001: /*
002: * This file or a portion of this file is licensed under the terms of
003: * the Globus Toolkit Public License, found in file GTPL, or at
004: * http://www.globus.org/toolkit/download/license.html. This notice must
005: * appear in redistributions of this file, with or without modification.
006: *
007: * Redistributions of this Software, with or without modification, must
008: * reproduce the GTPL in: (1) the Software, or (2) the Documentation or
009: * some other similar material which is provided with the Software (if
010: * any).
011: *
012: * Copyright 1999-2004 University of Chicago and The University of
013: * Southern California. All rights reserved.
014: */
015: package org.griphyn.common.catalog.replica;
016:
017: import org.griphyn.cPlanner.common.LogManager;
018:
019: import org.griphyn.common.catalog.ReplicaCatalog;
020: import org.griphyn.common.catalog.ReplicaCatalogEntry;
021:
022: import org.griphyn.common.util.VDSProperties;
023:
024: import java.util.List;
025: import java.util.LinkedList;
026: import java.util.Iterator;
027: import java.util.Map;
028: import java.util.HashMap;
029: import java.util.Set;
030: import java.util.HashSet;
031: import java.util.Collection;
032: import java.util.Properties;
033:
034: /**
035: * A multiple replica catalog implementation that allows users to query
036: * different multiple catalogs at the same time.<p>
037: *
038: * To use it set
039: * <pre>
040: * pegasus.catalog.replica MRC
041: * </pre>
042: *
043: * Each associated replica catalog can be configured via properties as follows.
044: * <p>
045: * The user associates a variable name referred to as [value]
046: * for each of the catalogs, where [value] is any legal identifier
047: * (concretely [A-Za-z][_A-Za-z0-9]*)
048: *
049: * For each associated replica catalogs the user specifies the following
050: * properties.
051: * <pre>
052: * pegasus.catalog.replica.mrc.[value] to specify the type of replica catalog.
053: * pegasus.catalog.replica.mrc.[value].key to specify a property name key for a
054: * particular catalog
055: * </pre>
056: *
057: * <p>
058: * For example, if a user wants to query two lrc's at the same time he/she
059: * can specify as follows
060: *
061: * <pre>
062: * pegasus.catalog.replica.mrc.lrc1 LRC
063: * pegasus.catalog.replica.mrc.lrc2.url rls://sukhna
064: *
065: * pegasus.catalog.replica.mrc.lrc2 LRC
066: * pegasus.catalog.replica.mrc.lrc2.url rls://smarty
067: *
068: * </pre>
069: *
070: * <p>
071: * In the above example, lrc1, lrc2 are any valid identifier names and url is
072: * the property key that needed to be specified.
073: *
074: * @author Karan Vahi
075: * @version $Revision: 429 $
076: */
077: public class MRC implements ReplicaCatalog {
078:
079: /**
080: * The prefix for the property subset for connecting to the individual
081: * catalogs.
082: */
083: public static final String PROPERTY_PREFIX = "mrc";
084:
085: /**
086: * The property key that designates the type of replica catalog to connect
087: * to.
088: */
089: public static final String TYPE_KEY = "type";
090:
091: /**
092: * The list of replica catalogs that need to be queried for.
093: */
094: protected List mRCList;
095:
096: /**
097: * The handle to the logging manager.
098: */
099: protected LogManager mLogger;
100:
101: /**
102: * The default constructor.
103: */
104: public MRC() {
105: mRCList = new LinkedList();
106: mLogger = LogManager.getInstance();
107: }
108:
109: /**
110: * Removes everything from the catalogs.
111: * Use with care!!!
112: *
113: * @return the number of removed entries.
114: */
115: public int clear() {
116: int result = 0;
117: for (Iterator it = this .rcIterator(); it.hasNext();) {
118: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
119: result += catalog.clear();
120: }
121: return 0;
122: }
123:
124: /**
125: * Explicitely free resources before the garbage collection hits.
126: *
127: *
128: */
129: public void close() {
130: for (Iterator it = this .rcIterator(); it.hasNext();) {
131: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
132: catalog.close();
133: }
134: }
135:
136: /**
137: * Establishes a link between the implementation and the thing the
138: * implementation is build upon.
139: *
140: * @param props contains all necessary data to establish the link.
141: *
142: * @return true if connected now, or false to indicate a failure.
143: */
144: public boolean connect(Properties props) {
145:
146: //get the subset for the properties
147: Properties subset = VDSProperties.matchingSubset(props,
148: PROPERTY_PREFIX, false);
149: mLogger.log("MRC Properties are " + subset,
150: LogManager.DEBUG_MESSAGE_LEVEL);
151:
152: //container for properties for each of the different catalogs
153: Map propertiesMap = new HashMap();
154:
155: //put each of the keys in the correct bin
156: for (Iterator it = subset.entrySet().iterator(); it.hasNext();) {
157: Map.Entry entry = (Map.Entry) it.next();
158: String key = (String) entry.getKey();
159: String value = (String) entry.getValue();
160: String name = getName(key); //bin stores the user defined name specified
161: //now determine the key
162: key = getKey(key, name);
163:
164: //store the key, value in the correct properties object
165: Properties p;
166: if (propertiesMap.containsKey(name)) {
167: p = (Properties) propertiesMap.get(name);
168: } else {
169: p = new Properties();
170: propertiesMap.put(name, p);
171: }
172: p.setProperty(key, value);
173: }
174:
175: //now that we have all the properties sorted accd to individual catalogs
176: //try connecting to them one by one
177: boolean result = true;
178: for (Iterator it = propertiesMap.entrySet().iterator(); it
179: .hasNext();) {
180: Map.Entry entry = (Map.Entry) it.next();
181: result &= connect((String) entry.getKey(),
182: (Properties) entry.getValue());
183:
184: //if unable to connect to any single
185: //break out and exit
186: if (!result) {
187: break;
188: }
189: }
190:
191: //if the result is false, then disconnect from
192: //already connected replica catalogs
193: if (!result) {
194: close();
195: }
196:
197: return result;
198: }
199:
200: /**
201: * Connects to an individual replica catalog. Also adds the handle to the
202: * connected replica catalog in the internal list.
203: *
204: * @param name the name given by the user in the properties file.
205: * @param properties the properties to use for connecting.
206: * @return boolean
207: */
208: protected boolean connect(String name, Properties properties) {
209:
210: //get the type first
211: String type = properties.getProperty(this .TYPE_KEY);
212: if (type == null) {
213: StringBuffer message = new StringBuffer();
214: message.append(
215: "No type associated with replica catalog of name ")
216: .append(name);
217: message.append("Set the property ").append(
218: ReplicaCatalog.c_prefix).append(".").append(name);
219: mLogger.log(message.toString(),
220: LogManager.DEBUG_MESSAGE_LEVEL);
221: return false;
222: }
223:
224: //try and connect
225: ReplicaCatalog catalog = null;
226: try {
227: catalog = ReplicaFactory.loadInstance(type, properties);
228: } catch (Exception e) {
229: //log the connection error
230: mLogger.log("Unable to connect to replica catalog of name "
231: + name, e, LogManager.ERROR_MESSAGE_LEVEL);
232: return false;
233: }
234:
235: mRCList.add(catalog);
236: return true;
237: }
238:
239: /**
240: * Returns an iterator to iterate through the list of ReplicaCatalogs that
241: * MRC is associated with.
242: *
243: * @return Iterator
244: */
245: protected Iterator rcIterator() {
246: return this .mRCList.iterator();
247: }
248:
249: /**
250: * Returns the name from the key. The name is first component of the key before
251: * the first dot (.).
252: *
253: * @param key String
254: * @return String
255: */
256: protected String getName(String key) {
257:
258: return (key.indexOf('.') == -1) ?
259: //if there is no instance of . then the key is the name
260: key
261: :
262: //else get the substring to first dot
263: key.substring(0, key.indexOf('.'));
264: }
265:
266: /**
267: * Returns the key with the prefix stripped off. In the case, where the key
268: * is the prefix, STYLE_KEY is returned. If the key does not start with the
269: * prefix, then null is returned.
270: *
271: * @param key the key
272: * @param prefix String
273: *
274: * @return key stripped off of the prefix
275: *
276: * @see #TYPE_KEY
277: */
278: protected String getKey(String key, String prefix) {
279: //sanity check
280: if (!key.startsWith(prefix))
281: return null;
282:
283: //if the key and prefix are same length
284: if (key.length() == prefix.length()) {
285: return this .TYPE_KEY;
286: }
287:
288: //if prefix does not end in a dot add a dot
289: if (prefix.charAt(prefix.length() - 1) != '.') {
290: prefix = prefix + '.';
291: }
292:
293: //for a valid subsetting operation there should be . at prefix.length() - 1
294: //allows us to distinguish between lrc1.url and lrc1a.url for prefix
295: //lrc1
296: return (key.charAt(prefix.length() - 1) != '.') ? null : key
297: .substring(prefix.length());
298: }
299:
300: /**
301: * Deletes all PFN entries for a given LFN from the replica catalog where
302: * the PFN attribute is found, and matches exactly the object value.
303: *
304: * @param lfn is the logical filename to look for.
305: * @param name is the PFN attribute name to look for.
306: * @param value is an exact match of the attribute value to match.
307: * @return the number of removed entries.
308: */
309: public int delete(String lfn, String name, Object value) {
310: int result = 0;
311: for (Iterator it = this .rcIterator(); it.hasNext();) {
312: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
313: result += catalog.delete(lfn, name, value);
314: }
315: return result;
316: }
317:
318: /**
319: * Deletes a very specific mapping from the replica catalog.
320: *
321: * @param lfn is the logical filename in the tuple.
322: * @param tuple is a description of the PFN and its attributes.
323: * @return the number of removed entries, either 0 or 1.
324: */
325: public int delete(String lfn, ReplicaCatalogEntry tuple) {
326: int result = 0;
327: for (Iterator it = this .rcIterator(); it.hasNext();) {
328: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
329: result += catalog.delete(lfn, tuple);
330: }
331: return result;
332:
333: }
334:
335: /**
336: * Deletes multiple mappings into the replica catalog.
337: *
338: * @param x is a map from logical filename string to list of replica
339: * catalog entries.
340: * @param matchAttributes whether mapping should be deleted only if all
341: * attributes match.
342: * @return the number of deletions.
343: */
344: public int delete(Map x, boolean matchAttributes) {
345: int result = 0;
346: for (Iterator it = this .rcIterator(); it.hasNext();) {
347: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
348: result += catalog.delete(x, matchAttributes);
349: }
350: return result;
351:
352: }
353:
354: /**
355: * Deletes a specific mapping from the replica catalog.
356: *
357: * @param lfn is the logical filename in the tuple.
358: * @param pfn is the physical filename in the tuple.
359: * @return the number of removed entries.
360: */
361: public int delete(String lfn, String pfn) {
362: int result = 0;
363: for (Iterator it = this .rcIterator(); it.hasNext();) {
364: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
365: result += catalog.delete(lfn, pfn);
366: }
367: return result;
368:
369: }
370:
371: /**
372: * Deletes all PFN entries for a given LFN from the replica catalog where
373: * the resource handle is found.
374: *
375: * @param lfn is the logical filename to look for.
376: * @param handle is the resource handle
377: *
378: * @return the number of entries removed.
379: */
380: public int deleteByResource(String lfn, String handle) {
381: int result = 0;
382: for (Iterator it = this .rcIterator(); it.hasNext();) {
383: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
384: result += catalog.delete(lfn, handle);
385: }
386: return result;
387:
388: }
389:
390: /**
391: * Inserts a new mapping into the replica catalog.
392: *
393: * @param lfn is the logical filename under which to book the entry.
394: * @param pfn is the physical filename associated with it.
395: * @param handle is a resource handle where the PFN resides.
396: *
397: * @return number of insertions, should always be 1. On failure, throw
398: * an exception, don't use zero.
399: *
400: * @throws UnsupportedOperationException
401: */
402: public int insert(String lfn, String pfn, String handle) {
403: throw new UnsupportedOperationException(
404: "Method insert( String, String, String ) not supported in MRC");
405: }
406:
407: /**
408: * Inserts a new mapping into the replica catalog.
409: *
410: * @param lfn is the logical filename under which to book the entry.
411: * @param tuple is the physical filename and associated PFN attributes.
412: *
413: * @return number of insertions, should always be 1. On failure, throw exception
414: * @throws UnsupportedOperationException
415: */
416: public int insert(String lfn, ReplicaCatalogEntry tuple) {
417: throw new UnsupportedOperationException(
418: "Method insert( String, ReplicaCatalogEntry ) not supported in MRC");
419: }
420:
421: /**
422: * Inserts multiple mappings into the replica catalog.
423: *
424: * @param x is a map from logical filename string to list of replica
425: * catalog entries.
426: *
427: * @return the number of insertions.
428: * @throws UnsupportedOperationException
429: */
430: public int insert(Map x) {
431: throw new UnsupportedOperationException(
432: "Method insert( Map ) not supported in MRC");
433: }
434:
435: /**
436: * Predicate to check, if the connection with the catalog's
437: * implementation is still active. Returns true only if the connections to
438: * all the associated replica catalogs is closed.
439: *
440: * @return true, if the implementation is disassociated, false otherwise.
441: */
442: public boolean isClosed() {
443: boolean result = true;
444: for (Iterator it = this .rcIterator(); it.hasNext();) {
445: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
446: result &= catalog.isClosed();
447: }
448: return result;
449:
450: }
451:
452: /**
453: * Lists a subset of all logical filenames in the catalog.
454: *
455: * @param constraint is a constraint for the logical filename only. It
456: * is a string that has some meaning to the implementing system. This
457: * can be a SQL wildcard for queries, or a regular expression for
458: * Java-based memory collections.
459: *
460: * @return A set of logical filenames that match. The set may be empty
461: *
462: */
463: public Set list(String constraint) {
464: Set result = new HashSet();
465: for (Iterator it = this .rcIterator(); it.hasNext();) {
466: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
467: result.addAll(catalog.list(constraint));
468: }
469: return result;
470:
471: }
472:
473: /**
474: * Lists all logical filenames in the catalog.
475: *
476: * @return A set of all logical filenames known to the catalog.
477: */
478: public Set list() {
479: Set result = new HashSet();
480: for (Iterator it = this .rcIterator(); it.hasNext();) {
481: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
482: result.addAll(catalog.list());
483: }
484: return result;
485:
486: }
487:
488: /**
489: * Retrieves the entry for a given filename and resource handle from the
490: * replica catalog.
491: *
492: * @param lfn is the logical filename to obtain information for.
493: * @param handle is the resource handle to obtain entries for.
494: * @return the (first) matching physical filename, or <code>null</code>
495: * if no match was found.
496: */
497: public String lookup(String lfn, String handle) {
498: String result = null;
499: for (Iterator it = this .rcIterator(); it.hasNext();) {
500: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
501:
502: if ((result = catalog.lookup(lfn, handle)) != null) {
503: return result;
504: }
505: }
506: return result;
507:
508: }
509:
510: /**
511: * Retrieves all entries for a given LFN from the replica catalog.
512: *
513: * @param lfn is the logical filename to obtain information for.
514: *
515: * @return a collection of replica catalog entries
516: *
517: */
518: public Collection lookup(String lfn) {
519: Collection result = new LinkedList();
520: for (Iterator it = this .rcIterator(); it.hasNext();) {
521: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
522: Collection l = catalog.lookup(lfn);
523: if (l != null) {
524: result.addAll(l);
525: }
526: }
527: return result;
528:
529: }
530:
531: /**
532: * Retrieves multiple entries for a given logical filename, up to the
533: * complete catalog.
534: *
535: * @param lfns is a set of logical filename strings to look up.
536: * @param handle is the resource handle, restricting the LFNs.
537: *
538: * @return a map indexed by the LFN. Each value is a collection of
539: * replica catalog entries (all attributes).
540: */
541: public Map lookup(Set lfns, String handle) {
542: Map result = new HashMap();
543:
544: for (Iterator it = this .rcIterator(); it.hasNext();) {
545: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
546: Map m = catalog.lookup(lfns, handle);
547:
548: //merge all the entries in the map into the result
549: for (Iterator mit = m.entrySet().iterator(); mit.hasNext();) {
550: Map.Entry entry = (Map.Entry) mit.next();
551: //merge the entries into the main result
552: String lfn = (String) entry.getKey(); //the lfn
553: if (result.containsKey(lfn)) {
554: //right now no merging of RCE being done on basis
555: //on them having same pfns. duplicate might occur.
556: ((Set) result.get(lfn)).addAll((Set) entry
557: .getValue());
558: } else {
559: result.put(lfn, entry.getValue());
560: }
561: }
562: }
563: return result;
564: }
565:
566: /**
567: * Retrieves multiple entries for a given logical filename, up to the
568: * complete catalog.
569: *
570: * @param constraints is mapping of keys 'lfn', 'pfn', or any attribute
571: * name, e.g. the resource handle 'site', to a string that has some
572: * meaning to the implementing system. This can be a SQL wildcard for
573: * queries, or a regular expression for Java-based memory collections.
574: * Unknown keys are ignored. Using an empty map requests the complete
575: * catalog.
576: *
577: * @return a map indexed by the LFN. Each value is a collection of
578: * replica catalog entries.
579: */
580: public Map lookup(Map constraints) {
581:
582: Map result = new HashMap();
583:
584: for (Iterator it = this .rcIterator(); it.hasNext();) {
585: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
586: Map m = catalog.lookup(constraints);
587:
588: //merge all the entries in the map into the result
589: for (Iterator mit = m.entrySet().iterator(); mit.hasNext();) {
590: Map.Entry entry = (Map.Entry) mit.next();
591: //merge the entries into the main result
592: String lfn = (String) entry.getKey(); //the lfn
593: if (result.containsKey(lfn)) {
594: //right now no merging of RCE being done on basis
595: //on them having same pfns. duplicate might occur.
596: ((Set) result.get(lfn)).addAll((Set) entry
597: .getValue());
598: } else {
599: result.put(lfn, entry.getValue());
600: }
601: }
602: }
603: return result;
604:
605: }
606:
607: /**
608: * Retrieves multiple entries for a given logical filename, up to the
609: * complete catalog.
610: *
611: * @param lfns is a set of logical filename strings to look up.
612: *
613: * @return a map indexed by the LFN. Each value is a collection of
614: * replica catalog entries for the LFN.
615: */
616: public Map lookup(Set lfns) {
617:
618: Map result = new HashMap();
619:
620: for (Iterator it = this .rcIterator(); it.hasNext();) {
621: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
622: Map m = catalog.lookup(lfns);
623:
624: //merge all the entries in the map into the result
625: for (Iterator mit = m.entrySet().iterator(); mit.hasNext();) {
626: Map.Entry entry = (Map.Entry) mit.next();
627: //merge the entries into the main result
628: String lfn = (String) entry.getKey(); //the lfn
629: if (result.containsKey(lfn)) {
630: //right now no merging of RCE being done on basis
631: //on them having same pfns. duplicate might occur.
632: ((Collection) result.get(lfn))
633: .addAll((Collection) entry.getValue());
634: } else {
635: result.put(lfn, entry.getValue());
636: }
637: }
638: }
639: return result;
640:
641: }
642:
643: /**
644: * Retrieves all entries for a given LFN from the replica catalog.
645: *
646: * @param lfn is the logical filename to obtain information for.
647: * @return a set of PFN strings
648: */
649: public Set lookupNoAttributes(String lfn) {
650: Set result = new HashSet();
651: for (Iterator it = this .rcIterator(); it.hasNext();) {
652: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
653: result.addAll(catalog.lookupNoAttributes(lfn));
654: }
655: return result;
656: }
657:
658: /**
659: * Retrieves multiple entries for a given logical filename, up to the
660: * complete catalog.
661: *
662: * @param lfns is a set of logical filename strings to look up.
663: * @return a map indexed by the LFN. Each value is a set of PFN strings.
664: */
665: public Map lookupNoAttributes(Set lfns) {
666: Map result = new HashMap();
667: for (Iterator it = lfns.iterator(); it.hasNext();) {
668: String lfn = (String) it.next();
669: result.put(lfn, this .lookupNoAttributes(lfn));
670: }
671: return result;
672:
673: }
674:
675: /**
676: * Retrieves multiple entries for a given logical filename, up to the
677: * complete catalog.
678: *
679: * @param lfns is a set of logical filename strings to look up.
680: * @param handle is the resource handle, restricting the LFNs.
681: * @return a map indexed by the LFN. Each value is a set of physical
682: * filenames.
683: */
684: public Map lookupNoAttributes(Set lfns, String handle) {
685: Map result = new HashMap();
686:
687: for (Iterator it = this .rcIterator(); it.hasNext();) {
688: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
689: Map m = this .lookupNoAttributes(lfns, handle);
690:
691: //merge the map into the result
692: for (Iterator mit = m.entrySet().iterator(); mit.hasNext();) {
693: Map.Entry entry = (Map.Entry) mit.next();
694: //merge the entries into the main result
695: String key = (String) entry.getKey(); //the lfn
696: if (result.containsKey(key)) {
697: //merge the results
698: ((Set) result.get(key)).addAll((Set) entry
699: .getValue());
700: } else {
701: result.put(key, entry.getValue());
702: }
703: }
704:
705: }
706: return result;
707:
708: }
709:
710: /**
711: * Removes all mappings for an LFN from the replica catalog.
712: *
713: * @param lfn is the logical filename to remove all mappings for.
714: * @return the number of removed entries.
715: */
716: public int remove(String lfn) {
717: int result = 0;
718: for (Iterator it = this .rcIterator(); it.hasNext();) {
719: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
720: result += catalog.remove(lfn);
721: }
722: return result;
723:
724: }
725:
726: /**
727: * Removes all mappings for a set of LFNs.
728: *
729: * @param lfns is a set of logical filename to remove all mappings for.
730: *
731: * @return the number of removed entries.
732: */
733: public int remove(Set lfns) {
734: int result = 0;
735: for (Iterator it = this .rcIterator(); it.hasNext();) {
736: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
737: result += catalog.remove(lfns);
738: }
739: return result;
740:
741: }
742:
743: /**
744: * Removes all entries from the replica catalog where the PFN attribute
745: * is found, and matches exactly the object value.
746: *
747: * @param name is the PFN attribute name to look for.
748: * @param value is an exact match of the attribute value to match.
749: *
750: * @return the number of removed entries.
751: */
752: public int removeByAttribute(String name, Object value) {
753: int result = 0;
754: for (Iterator it = this .rcIterator(); it.hasNext();) {
755: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
756: result += catalog.removeByAttribute(name, value);
757: }
758: return result;
759:
760: }
761:
762: /**
763: * Removes all entries associated with a particular resource handle.
764: *
765: * @param handle is the site handle to remove all entries for.
766: *
767: * @return the number of removed entries.
768: */
769: public int removeByAttribute(String handle) {
770: int result = 0;
771: for (Iterator it = this .rcIterator(); it.hasNext();) {
772: ReplicaCatalog catalog = (ReplicaCatalog) it.next();
773: result += catalog.removeByAttribute(handle);
774: }
775: return result;
776:
777: }
778: }
|