0001: /*
0002: * This file or a portion of this file is licensed under the terms of
0003: * the Globus Toolkit Public License, found in file GTPL, or at
0004: * http://www.globus.org/toolkit/download/license.html. This notice must
0005: * appear in redistributions of this file, with or without modification.
0006: *
0007: * Redistributions of this Software, with or without modification, must
0008: * reproduce the GTPL in: (1) the Software, or (2) the Documentation or
0009: * some other similar material which is provided with the Software (if
0010: * any).
0011: *
0012: * Copyright 1999-2004 University of Chicago and The University of
0013: * Southern California. All rights reserved.
0014: */
0015: package org.griphyn.common.catalog.replica;
0016:
0017: import org.griphyn.common.catalog.ReplicaCatalog;
0018: import org.griphyn.common.catalog.ReplicaCatalogEntry;
0019: import org.griphyn.common.catalog.CatalogException;
0020:
0021: import org.griphyn.cPlanner.common.LogManager;
0022:
0023: import org.globus.replica.rls.RLSClient;
0024: import org.globus.replica.rls.RLSException;
0025: import org.globus.replica.rls.RLSAttribute;
0026: import org.globus.replica.rls.RLSAttributeObject;
0027: import org.globus.replica.rls.RLSLRCInfo;
0028: import org.globus.replica.rls.RLSString2Bulk;
0029: import org.globus.replica.rls.RLSString2;
0030:
0031: import java.util.Collection;
0032: import java.util.Collections;
0033: import java.util.Comparator;
0034: import java.util.Map;
0035: import java.util.HashMap;
0036: import java.util.Set;
0037: import java.util.HashSet;
0038: import java.util.Properties;
0039: import java.util.List;
0040: import java.util.ArrayList;
0041: import java.util.LinkedHashSet;
0042: import java.util.Iterator;
0043:
0044: /**
0045: * This class implements the VDS replica catalog interface on top of RLI API.
0046: * A thing to take care of is that all delete and remove operations are
0047: * propoagated to all the Local Replica Catalogs (LRCs) reporting to the RLI.
0048: * Hence,
0049: * you should be careful while deleting LFNs, as deletions can cascade to
0050: * multiple LRCs. If you want to delete or remove an LFN from a particular LRC,
0051: * use the LRC implementation to connect to that LRC and call the corresponding
0052: * delete functions on that.
0053: * There is no transaction support in the implementation. The implementation
0054: * is best effort. Inconsistencies can occur if one of the LRCs goes offline,
0055: * or an operation fails for whatsoever reason.
0056: *
0057: * @author Karan Vahi
0058: * @author Gaurang Mehta
0059: *
0060: * @version $Revision: 159 $
0061: */
0062: public class RLI implements ReplicaCatalog {
0063:
0064: /**
0065: * The number of entries searched in each bulk query to RLS.
0066: */
0067: public static final int RLS_BULK_QUERY_SIZE = 1000;
0068:
0069: /**
0070: * The default timeout in seconds to be used while querying the RLI.
0071: */
0072: public static final String DEFAULT_RLI_TIMEOUT = "30";
0073:
0074: /**
0075: * The key that is used to get hold of the timeout value from the properties
0076: * object.
0077: */
0078: public static final String RLS_TIMEOUT_KEY = "rls.timeout";
0079:
0080: /**
0081: * The key that is used to get hold of the timeout value from the properties
0082: * object.
0083: */
0084: public static final String RLI_TIMEOUT_KEY = "rli.timeout";
0085:
0086: /**
0087: * The key that is used to designate the LRC whose results are to be
0088: * ignored.
0089: */
0090: public static final String LRC_IGNORE_KEY = "lrc.ignore";
0091:
0092: /**
0093: * The key that is used to designate the LRC whose results are to be
0094: * restricted.
0095: */
0096: public static final String LRC_RESTRICT_KEY = "lrc.restrict";
0097:
0098: /**
0099: * The attribute in RLS that maps to a site handle.
0100: */
0101: public static final String SITE_ATTRIBUTE = "pool";
0102:
0103: /**
0104: * The undefined pool attribute value. The pool attribute is assigned this
0105: * value if the pfn queried does not have a pool associated with it.
0106: */
0107: public static final String UNDEFINED_SITE = "UNDEFINED_POOL";
0108:
0109: /**
0110: * The key that is used to get hold of the url from the properties object.
0111: */
0112: public static final String URL_KEY = "url";
0113:
0114: /**
0115: * The key that if set, specifies the proxy to be picked up while connecting
0116: * to the RLS.
0117: */
0118: public static final String PROXY_KEY = "proxy";
0119:
0120: /**
0121: * The error message for not connected to RLI.
0122: */
0123: public static final String RLI_NOT_CONNECTED_MSG = "Not connected to RLI ";
0124:
0125: /**
0126: * The error message for not connected to LRC.
0127: */
0128: public static final String LRC_NOT_CONNECTED_MSG = "Unable to connect to LRC ";
0129:
0130: /**
0131: * The LRC query state indicating that LRC needs to queried fully. The LRC
0132: * returns all PFNs irrespective of whether they have a site attribute or
0133: * not.
0134: */
0135: public static final int LRC_QUERY_NORMAL = 0;
0136:
0137: /**
0138: * The LRC query state indicating that LRC has to be restricted query.
0139: * LRC should return only PFNs with site attributes tagged.
0140: */
0141: public static final int LRC_QUERY_RESTRICT = 1;
0142:
0143: /**
0144: * The LRC query state indicating that LRC has to be ignored.
0145: */
0146: public static final int LRC_QUERY_IGNORE = 2;
0147:
0148: /**
0149: * The handle to the client that allows access to both the RLI and the LRC
0150: * running at the url specified while connecting.
0151: */
0152: private RLSClient mRLS;
0153:
0154: /**
0155: * The handle to the client that allows access to the LRC running at the
0156: * url specified while connecting.
0157: */
0158: private RLSClient.RLI mRLI;
0159:
0160: /**
0161: * The url to the RLI to which this instance implementation talks to.
0162: */
0163: private String mRLIURL;
0164:
0165: /**
0166: * A String array contains the LRC URLs that have to be ignored for querying.
0167: */
0168: private String[] mLRCIgnoreList;
0169:
0170: /**
0171: * A String array contains the LRC URLs that have to be restricted for querying.
0172: * Only those entries are returned that have a site attribute associated
0173: * with them.
0174: */
0175: private String[] mLRCRestrictList;
0176:
0177: /**
0178: * The handle to the logging object. Should be log4j soon.
0179: */
0180: private LogManager mLogger;
0181:
0182: /**
0183: * The string holding the message that is logged in the logger.
0184: */
0185: private String mLogMsg;
0186:
0187: /**
0188: * The properties object containing all the properties, that are required
0189: * to connect to a RLS.
0190: */
0191: private Properties mConnectProps;
0192:
0193: /**
0194: * The batch size while querying the RLI in the bulk mode.
0195: */
0196: private int mBatchSize;
0197:
0198: /**
0199: * The timeout in seconds to be applied while querying the RLI.
0200: */
0201: private int mTimeout;
0202:
0203: /**
0204: * The default constructor, that creates an object which is not linked with
0205: * any RLS. Use the connect method to connect to the RLS.
0206: *
0207: * @see #connect(Properties).
0208: */
0209: public RLI() {
0210: mRLS = null;
0211: mLogger = LogManager.getInstance();
0212: mConnectProps = new Properties();
0213: mBatchSize = this .RLS_BULK_QUERY_SIZE;
0214: mTimeout = Integer.parseInt(DEFAULT_RLI_TIMEOUT);
0215:
0216: }
0217:
0218: /**
0219: * Establishes a connection to the RLI, picking up the proxy from the default
0220: * location usually /tmp/ directory.
0221: *
0222: * @param url the url to lrc to connect to.
0223: *
0224: * @return true if connected now, or false to indicate a failure.
0225: */
0226: public boolean connect(String url) {
0227: return connect(url, null);
0228: }
0229:
0230: /**
0231: * Establishes a connection to the RLI.
0232: *
0233: * @param props contains all necessary data to establish the link.
0234: *
0235: * @return true if connected now, or false to indicate a failure.
0236: */
0237: public boolean connect(Properties props) {
0238: boolean con = false;
0239: Object obj = props.remove(URL_KEY);
0240: mRLIURL = (obj == null) ? null : (String) obj;
0241:
0242: if (mRLIURL == null) {
0243: //nothing to connect to.
0244: mLogger.log("The RLI url is not specified",
0245: LogManager.ERROR_MESSAGE_LEVEL);
0246: return con;
0247: }
0248:
0249: //try to see if a proxy cert has been specified or not
0250: String proxy = props.getProperty(PROXY_KEY);
0251: mConnectProps = props;//??
0252:
0253: mLRCIgnoreList = this .getRLSLRCIgnoreURLs(props);
0254: mLRCRestrictList = this .getRLSLRCRestrictURLs(props);
0255:
0256: //determine timeout
0257: mTimeout = getTimeout(props);
0258:
0259: //set the batch size for queries
0260: setBatchSize(props);
0261:
0262: return connect(mRLIURL, proxy);
0263: }
0264:
0265: /**
0266: * Establishes a connection to the RLI.
0267: *
0268: * @param url the url to lrc to connect to.
0269: * @param proxy the path to the proxy file to be picked up. null denotes
0270: * default location.
0271: *
0272: * @return true if connected now, or false to indicate a failure.
0273: */
0274: public boolean connect(String url, String proxy) {
0275: mRLIURL = url;
0276: //push it into the internal properties object
0277: mConnectProps.setProperty(URL_KEY, url);
0278: if (proxy != null) {
0279: mConnectProps.setProperty(PROXY_KEY, proxy);
0280: }
0281: try {
0282: mRLS = (proxy == null) ? new RLSClient(url) : //proxy is picked up from default loc /tmp
0283: new RLSClient(url, proxy);
0284:
0285: //set RLI timeout
0286: mRLS.SetTimeout(mTimeout);
0287:
0288: //connect is only successful if we have
0289: //successfully connected to the LRC
0290: mRLI = mRLS.getRLI();
0291:
0292: } catch (RLSException e) {
0293: mLogger.log("RLS Exception", e,
0294: LogManager.ERROR_MESSAGE_LEVEL);
0295: return false;
0296: }
0297: return true;
0298: }
0299:
0300: /**
0301: * Gets a handle to the RLI that is associated with the RLS running at
0302: * url.
0303: *
0304: * @return <code>RLSClient.RLI</code> that points to the RLI that is
0305: * running , or null in case connect method not being called.
0306: * @see #mRLIURL
0307: */
0308: public RLSClient.RLI getRLI() {
0309: return (this .isClosed()) ? null : mRLS.getRLI();
0310: }
0311:
0312: /**
0313: * Gets a handle to the LRC that is associated with the RLS running at
0314: * url.
0315: *
0316: * @return <code>RLSClient.LRC</code> that points to the RLI that is
0317: * running , or null in case connect method not being called.
0318: * @see #mRLIURL
0319: */
0320: public RLSClient.LRC getLRC() {
0321: return (this .isClosed()) ? null : mRLS.getLRC();
0322: }
0323:
0324: /**
0325: * Retrieves the entry for a given filename and resource handle from
0326: * the RLS.
0327: *
0328: * @param lfn is the logical filename to obtain information for.
0329: * @param handle is the resource handle to obtain entries for.
0330: *
0331: * @return the (first) matching physical filename, or
0332: * <code>null</code> if no match was found.
0333: */
0334: public String lookup(String lfn, String handle) {
0335: //sanity check
0336: if (this .isClosed()) {
0337: //probably an exception should be thrown here!!
0338: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0339: + this .mRLIURL);
0340: }
0341: String pfn = null;
0342: ArrayList lrcList = null;
0343: try {
0344: lrcList = mRLI.getLRC(lfn);
0345: for (Iterator it = lrcList.iterator(); it.hasNext();) {
0346: //connect to an lrc
0347: String lrcURL = ((RLSString2) it.next()).s2;
0348: //push the lrcURL to the properties object
0349: mConnectProps.setProperty(this .URL_KEY, lrcURL);
0350: LRC lrc = new LRC();
0351: if (!lrc.connect(mConnectProps)) {
0352: //log an error/warning message
0353: mLogger.log("Unable to connect to LRC " + lrcURL,
0354: LogManager.ERROR_MESSAGE_LEVEL);
0355: continue;
0356: }
0357:
0358: //query the lrc
0359: try {
0360: pfn = lrc.lookup(lfn, handle);
0361: if (pfn != null)
0362: return pfn;
0363: } catch (Exception ex) {
0364: mLogger.log("lookup(String,String)", ex,
0365: LogManager.ERROR_MESSAGE_LEVEL);
0366: } finally {
0367: //disconnect
0368: lrc.close();
0369: }
0370: }
0371: } catch (RLSException ex) {
0372: mLogger.log("lookup(String,String)", ex,
0373: LogManager.ERROR_MESSAGE_LEVEL);
0374: }
0375:
0376: return null;
0377: }
0378:
0379: /**
0380: * Retrieves all entries for a given LFN from the replica catalog.
0381: * Each entry in the result set is a tuple of a PFN and all its
0382: * attributes.
0383: *
0384: * @param lfn is the logical filename to obtain information for.
0385: *
0386: * @return a collection of replica catalog entries, or null in case of
0387: * unable to connect to RLS.
0388: *
0389: * @see ReplicaCatalogEntry
0390: */
0391: public Collection lookup(String lfn) {
0392: //sanity check
0393: if (this .isClosed()) {
0394: //probably an exception should be thrown here!!
0395: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0396: + this .mRLIURL);
0397: }
0398:
0399: Collection result = null;
0400: ArrayList lrcList = null;
0401: try {
0402: lrcList = mRLI.getLRC(lfn);
0403: result = new ArrayList(lrcList.size());
0404:
0405: for (Iterator it = lrcList.iterator(); it.hasNext();) {
0406: //connect to an lrc
0407: String lrcURL = ((RLSString2) it.next()).s2;
0408:
0409: //push the lrcURL to the properties object
0410: mConnectProps.setProperty(this .URL_KEY, lrcURL);
0411: LRC lrc = new LRC();
0412: if (!lrc.connect(mConnectProps)) {
0413: //log an error/warning message
0414: mLogger.log("Unable to connect to LRC " + lrcURL,
0415: LogManager.ERROR_MESSAGE_LEVEL);
0416: continue;
0417: }
0418:
0419: //query the lrc
0420: try {
0421: Collection l = lrc.lookup(lfn);
0422: if (l != null)
0423: result.addAll(l);
0424: } catch (Exception ex) {
0425: mLogger.log("lookup(String)", ex,
0426: LogManager.ERROR_MESSAGE_LEVEL);
0427: } finally {
0428: //disconnect
0429: lrc.close();
0430: }
0431: }
0432: } catch (RLSException ex) {
0433: if (ex.GetRC() == RLSClient.RLS_LFN_NEXIST
0434: || ex.GetRC() == RLSClient.RLS_MAPPING_NEXIST) {
0435: mLogger.log("lookup(String) :Mapping for lfn " + lfn
0436: + " does not exist in RLI",
0437: LogManager.ERROR_MESSAGE_LEVEL);
0438: } else {
0439: mLogger.log("lookup(String)", ex,
0440: LogManager.ERROR_MESSAGE_LEVEL);
0441: }
0442: result = new ArrayList(1);
0443: }
0444:
0445: return result;
0446:
0447: }
0448:
0449: /**
0450: * Retrieves all entries for a given LFN from the replica catalog.
0451: * Each entry in the result set is just a PFN string. Duplicates
0452: * are reduced through the set paradigm.
0453: *
0454: * @param lfn is the logical filename to obtain information for.
0455: * @return a set of PFN strings, or null in case of unable to connect
0456: * to RLS.
0457: *
0458: */
0459: public Set lookupNoAttributes(String lfn) {
0460: //sanity check
0461: if (this .isClosed()) {
0462: //probably an exception should be thrown here!!
0463: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0464: + this .mRLIURL);
0465: }
0466:
0467: Set result = null;
0468: ArrayList lrcList = null;
0469: try {
0470: lrcList = mRLI.getLRC(lfn);
0471: result = new HashSet(lrcList.size());
0472:
0473: for (Iterator it = lrcList.iterator(); it.hasNext();) {
0474: //connect to an lrc
0475: String lrcURL = ((RLSString2) it.next()).s2;
0476:
0477: //push the lrcURL to the properties object
0478: mConnectProps.setProperty(this .URL_KEY, lrcURL);
0479: LRC lrc = new LRC();
0480: if (!lrc.connect(mConnectProps)) {
0481: //log an error/warning message
0482: mLogger.log("Unable to connect to LRC " + lrcURL,
0483: LogManager.ERROR_MESSAGE_LEVEL);
0484: continue;
0485: }
0486:
0487: //query the lrc
0488: try {
0489: Collection l = lrc.lookupNoAttributes(lfn);
0490: if (l != null)
0491: result.addAll(l);
0492:
0493: } catch (Exception ex) {
0494: mLogger.log("lookupNoAttributes(String,String)",
0495: ex, LogManager.ERROR_MESSAGE_LEVEL);
0496: } finally {
0497: //disconnect
0498: lrc.close();
0499: }
0500: }
0501: } catch (RLSException ex) {
0502: if (ex.GetRC() == RLSClient.RLS_LFN_NEXIST
0503: || ex.GetRC() == RLSClient.RLS_MAPPING_NEXIST) {
0504: mLogger.log("lookupNoAttributes(String,String): "
0505: + "Mapping for lfn " + lfn
0506: + " does not exist in RLI",
0507: LogManager.ERROR_MESSAGE_LEVEL);
0508: } else {
0509: mLogger.log("lookupNoAttributes(String,String)", ex,
0510: LogManager.ERROR_MESSAGE_LEVEL);
0511: }
0512: result = new HashSet(1);
0513: }
0514:
0515: return result;
0516:
0517: }
0518:
0519: /**
0520: * Retrieves multiple entries for a given logical filename, up to the
0521: * complete LRC. It uses the bulk query api to the LRC to query for stuff.
0522: * Bulk query has been in RLS since version 2.0.8. Internally, the bulk
0523: * queries are done is sizes specified by variable mBatchSize.
0524: *
0525: * @param lfns is a set of logical filename strings to look up.
0526: *
0527: * @return a map indexed by the LFN. Each value is a collection
0528: * of replica catalog entries for the LFN.
0529: *
0530: * @see ReplicaCatalogEntry
0531: * @see #getBatchSize()
0532: */
0533: public Map lookup(Set lfns) {
0534: //Map indexed by lrc url and each value a collection
0535: //of lfns that the RLI says are present in it.
0536: Map lrc2lfn = this .getLRC2LFNS(lfns);
0537:
0538: if (lrc2lfn == null) {
0539: //probably RLI is not connected!!
0540: return null;
0541: }
0542:
0543: // now query the LRCs with the LFNs that they are responsible for
0544: // and aggregate stuff.
0545: String key = null;
0546: Map result = new HashMap(lfns.size());
0547: String message;
0548: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
0549: Map.Entry entry = (Map.Entry) it.next();
0550: key = (String) entry.getKey();
0551: message = "Querying LRC " + key;
0552: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0553:
0554: //push the lrcURL to the properties object
0555: mConnectProps.setProperty(this .URL_KEY, key);
0556: LRC lrc = new LRC();
0557: if (!lrc.connect(mConnectProps)) {
0558: //log an error/warning message
0559: mLogger.log("Unable to connect to LRC " + key,
0560: LogManager.ERROR_MESSAGE_LEVEL);
0561: continue;
0562: }
0563:
0564: //query the lrc
0565: try {
0566: Map m = lrc.lookup((Set) entry.getValue());
0567:
0568: //figure out if we need to restrict our queries or not.
0569: //restrict means only include results if they have a site
0570: //handle associated
0571: boolean restrict = (this .determineQueryType(key) == this .LRC_QUERY_RESTRICT);
0572:
0573: for (Iterator mit = m.entrySet().iterator(); mit
0574: .hasNext();) {
0575: entry = (Map.Entry) mit.next();
0576: List pfns = ((List) entry.getValue());
0577: if (restrict) {
0578: //traverse through all the PFN's and check for resource handle
0579: for (Iterator pfnIterator = pfns.iterator(); pfnIterator
0580: .hasNext();) {
0581: ReplicaCatalogEntry pfn = (ReplicaCatalogEntry) pfnIterator
0582: .next();
0583: if (pfn.getResourceHandle() == null) {
0584: //do not include in the results if the entry does not have
0585: //a pool attribute associated with it.
0586: mLogger.log("Ignoring entry "
0587: + entry.getValue()
0588: + " from LRC " + key,
0589: LogManager.DEBUG_MESSAGE_LEVEL);
0590: pfnIterator.remove();
0591: }
0592: }
0593:
0594: }
0595:
0596: //if pfns are empty which could be due to
0597: //restriction case taking away all pfns
0598: //do not merge in result
0599: if (pfns.isEmpty()) {
0600: continue;
0601: }
0602:
0603: //merge the entries into the main result
0604: key = (String) entry.getKey(); //the lfn
0605: if (result.containsKey(key)) {
0606: //right now no merging of RCE being done on basis
0607: //on them having same pfns. duplicate might occur.
0608: ((List) result.get(key)).addAll(pfns);
0609: } else {
0610: result.put(key, pfns);
0611: }
0612: }
0613: } catch (Exception ex) {
0614: mLogger.log("lookup(Set)", ex,
0615: LogManager.ERROR_MESSAGE_LEVEL);
0616: } finally {
0617: //disconnect
0618: lrc.close();
0619: }
0620:
0621: mLogger.logCompletion(message,
0622: LogManager.DEBUG_MESSAGE_LEVEL);
0623: }
0624: return result;
0625: }
0626:
0627: /**
0628: * Retrieves all entries for a given LFN from the replica catalog.
0629: * Each entry in the result set is just a PFN string. Duplicates
0630: * are reduced through the set paradigm.
0631: *
0632: * @param lfns is a set of logical filename strings to look up.
0633: * @return a map indexed by the LFN. Each value is a collection
0634: * of replica catalog entries for the LFN.
0635: */
0636: public Map lookupNoAttributes(Set lfns) {
0637: //Map indexed by lrc url and each value a collection
0638: //of lfns that the RLI says are present in it.
0639: Map lrc2lfn = this .getLRC2LFNS(lfns);
0640: if (lrc2lfn == null) {
0641: //probably RLI is not connected!!
0642: return null;
0643: }
0644:
0645: // now query the LRCs with the LFNs that they are responsible for
0646: // and aggregate stuff.
0647: String key = null;
0648: String message;
0649: Map result = new HashMap(lfns.size());
0650: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
0651: Map.Entry entry = (Map.Entry) it.next();
0652: key = (String) entry.getKey();
0653: message = "Querying LRC " + key;
0654: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0655:
0656: //push the lrcURL to the properties object
0657: mConnectProps.setProperty(this .URL_KEY, key);
0658: LRC lrc = new LRC();
0659: if (!lrc.connect(mConnectProps)) {
0660: //log an error/warning message
0661: mLogger.log("Unable to connect to LRC " + key,
0662: LogManager.ERROR_MESSAGE_LEVEL);
0663: continue;
0664: }
0665:
0666: //query the lrc
0667: try {
0668: Map m = lrc.lookupNoAttributes((Set) entry.getValue());
0669: for (Iterator mit = m.entrySet().iterator(); mit
0670: .hasNext();) {
0671: entry = (Map.Entry) mit.next();
0672: //merge the entries into the main result
0673: key = (String) entry.getKey(); //the lfn
0674: if (result.containsKey(key)) {
0675: //right now no merging of RCE being done on basis
0676: //on them having same pfns. duplicate might occur.
0677: ((Set) result.get(key)).addAll((Set) entry
0678: .getValue());
0679: } else {
0680: result.put(key, entry.getValue());
0681: }
0682: }
0683: } catch (Exception ex) {
0684: mLogger.log("lookup(Set)", ex,
0685: LogManager.ERROR_MESSAGE_LEVEL);
0686: } finally {
0687: //disconnect
0688: lrc.close();
0689: }
0690:
0691: mLogger.logCompletion(message,
0692: LogManager.DEBUG_MESSAGE_LEVEL);
0693: }
0694: return result;
0695:
0696: }
0697:
0698: /**
0699: * Retrieves multiple entries for a given logical filenames, up to the
0700: * complete catalog. Retrieving full catalogs should be harmful, but
0701: * may be helpful in online display or portal.<p>
0702: *
0703: * @param lfns is a set of logical filename strings to look up.
0704: * @param handle is the resource handle, restricting the LFNs.
0705: *
0706: * @return a map indexed by the LFN. Each value is a collection
0707: * of replica catalog entries (all attributes).
0708: *
0709: * @see ReplicaCatalogEntry
0710: */
0711: public Map lookup(Set lfns, String handle) {
0712: //Map indexed by lrc url and each value a collection
0713: //of lfns that the RLI says are present in it.
0714: Map lrc2lfn = this .getLRC2LFNS(lfns);
0715: if (lrc2lfn == null) {
0716: //probably RLI is not connected!!
0717: return null;
0718: }
0719:
0720: // now query the LRCs with the LFNs they are responsible for
0721: // and aggregate stuff.
0722: String key = null, message = null;
0723: Map result = new HashMap(lfns.size());
0724: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
0725: Map.Entry entry = (Map.Entry) it.next();
0726: key = (String) entry.getKey();
0727: message = "Querying LRC " + key;
0728: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0729:
0730: //push the lrcURL to the properties object
0731: mConnectProps.setProperty(this .URL_KEY, key);
0732: LRC lrc = new LRC();
0733: if (!lrc.connect(mConnectProps)) {
0734: //log an error/warning message
0735: mLogger.log("Unable to connect to LRC " + key,
0736: LogManager.ERROR_MESSAGE_LEVEL);
0737: continue;
0738: }
0739:
0740: //query the lrc
0741: try {
0742: Map m = lrc.lookup((Set) entry.getValue(), handle);
0743: for (Iterator mit = m.entrySet().iterator(); mit
0744: .hasNext();) {
0745: entry = (Map.Entry) mit.next();
0746: //merge the entries into the main result
0747: key = (String) entry.getKey(); //the lfn
0748: if (result.containsKey(key)) {
0749: //right now no merging of RCE being done on basis
0750: //on them having same pfns. duplicate might occur.
0751: ((Set) result.get(key)).addAll((Set) entry
0752: .getValue());
0753: } else {
0754: result.put(key, entry.getValue());
0755: }
0756: }
0757: } catch (Exception ex) {
0758: mLogger.log("lookup(Set,String)", ex,
0759: LogManager.ERROR_MESSAGE_LEVEL);
0760: } finally {
0761: //disconnect
0762: lrc.close();
0763: }
0764:
0765: mLogger.logCompletion(message,
0766: LogManager.DEBUG_MESSAGE_LEVEL);
0767: }
0768: return result;
0769:
0770: }
0771:
0772: /**
0773: * Retrieves multiple entries for a given logical filename, up to the
0774: * complete catalog. Retrieving full catalogs should be harmful, but
0775: * may be helpful in online display or portal.<p>
0776: *
0777: * The <code>noAttributes</code> flag is missing on purpose, because
0778: * due to the resource handle, attribute lookups are already required.
0779: *
0780: * @param lfns is a set of logical filename strings to look up.
0781: * @param handle is the resource handle, restricting the LFNs.
0782: *
0783: * @return a map indexed by the LFN. Each value is a set of
0784: * physical filenames.
0785: */
0786: public Map lookupNoAttributes(Set lfns, String handle) {
0787: //Map indexed by lrc url and each value a collection
0788: //of lfns that the RLI says are present in it.
0789: Map lrc2lfn = this .getLRC2LFNS(lfns);
0790: if (lrc2lfn == null) {
0791: //probably RLI is not connected!!
0792: return null;
0793: }
0794:
0795: // now query the LRCs with the LFNs that they are responsible for
0796: // and aggregate stuff.
0797: String key = null, message = null;
0798: Map result = new HashMap(lfns.size());
0799: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
0800: Map.Entry entry = (Map.Entry) it.next();
0801: key = (String) entry.getKey();
0802: message = "Querying LRC " + key;
0803: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0804:
0805: //push the lrcURL to the properties object
0806: mConnectProps.setProperty(this .URL_KEY, key);
0807: LRC lrc = new LRC();
0808: if (!lrc.connect(mConnectProps)) {
0809: //log an error/warning message
0810: mLogger.log("Unable to connect to LRC " + key,
0811: LogManager.ERROR_MESSAGE_LEVEL);
0812: continue;
0813: }
0814:
0815: //query the lrc
0816: try {
0817: Map m = lrc.lookupNoAttributes((Set) entry.getValue(),
0818: handle);
0819: for (Iterator mit = m.entrySet().iterator(); mit
0820: .hasNext();) {
0821: entry = (Map.Entry) mit.next();
0822: //merge the entries into the main result
0823: key = (String) entry.getKey(); //the lfn
0824: if (result.containsKey(key)) {
0825: //right now no merging of RCE being done on basis
0826: //on them having same pfns. duplicate might occur.
0827: ((Set) result.get(key)).addAll((Set) entry
0828: .getValue());
0829: } else {
0830: result.put(key, entry.getValue());
0831: }
0832: }
0833: } catch (Exception ex) {
0834: mLogger.log("lookup(Set,String):", ex,
0835: LogManager.ERROR_MESSAGE_LEVEL);
0836: } finally {
0837: //disconnect
0838: lrc.close();
0839: }
0840:
0841: mLogger.logCompletion(message,
0842: LogManager.DEBUG_MESSAGE_LEVEL);
0843: }
0844: return result;
0845:
0846: }
0847:
0848: /**
0849: * Retrieves multiple entries for a given logical filename, up to the
0850: * complete catalog. Retrieving full catalogs should be harmful, but
0851: * may be helpful in online display or portal.<p>
0852: *
0853: *
0854: * At present it DOES NOT SUPPORT ATTRIBUTE MATCHING.
0855: *
0856: * @param constraints is mapping of keys 'lfn', 'pfn' to a string that
0857: * has some meaning to the implementing system. This can be a SQL
0858: * wildcard for queries, or a regular expression for Java-based memory
0859: * collections. Unknown keys are ignored. Using an empty map requests
0860: * the complete catalog.
0861: *
0862: * @return a map indexed by the LFN. Each value is a collection
0863: * of replica catalog entries.
0864: *
0865: * @see ReplicaCatalogEntry
0866: */
0867: public Map lookup(Map constraints) {
0868: //sanity check
0869: if (this .isClosed()) {
0870: //probably an exception should be thrown here!!
0871: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0872: + this .mRLIURL);
0873: }
0874: Map result = new HashMap();
0875: String url = null, message = null;
0876: //we need to get hold of all the LRC
0877: //that report to the RLI and call the
0878: //list() method on each of them
0879: for (Iterator it = this .getReportingLRC().iterator(); it
0880: .hasNext();) {
0881: url = (String) it.next();
0882: message = "Querying LRC " + url;
0883: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0884:
0885: //push the lrcURL to the properties object
0886: mConnectProps.setProperty(this .URL_KEY, url);
0887: LRC lrc = new LRC();
0888: if (!lrc.connect(mConnectProps)) {
0889: //log an error/warning message
0890: mLogger.log("Unable to connect to LRC " + url,
0891: LogManager.ERROR_MESSAGE_LEVEL);
0892: continue;
0893: }
0894: try {
0895: Map m = lrc.lookup(constraints);
0896: for (Iterator mit = m.entrySet().iterator(); mit
0897: .hasNext();) {
0898: Map.Entry entry = (Map.Entry) mit.next();
0899: //merge the entries into the main result
0900: String key = (String) entry.getKey(); //the lfn
0901: if (result.containsKey(key)) {
0902: //right now no merging of RCE being done on basis
0903: //on them having same pfns. duplicate might occur.
0904: ((List) result.get(key)).addAll((List) entry
0905: .getValue());
0906: } else {
0907: result.put(key, entry.getValue());
0908: }
0909: }
0910:
0911: } catch (Exception e) {
0912: mLogger.log("list(String)", e,
0913: LogManager.ERROR_MESSAGE_LEVEL);
0914: } finally {
0915: lrc.close();
0916: }
0917: }
0918:
0919: return result;
0920:
0921: }
0922:
0923: /**
0924: * Lists all logical filenames in the catalog.
0925: *
0926: * @return A set of all logical filenames known to the catalog.
0927: */
0928: public Set list() {
0929: //sanity check
0930: if (this .isClosed()) {
0931: //probably an exception should be thrown here!!
0932: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0933: + this .mRLIURL);
0934: }
0935: Set result = new HashSet();
0936: String url = null, message = null;
0937: //we need to get hold of all the LRC
0938: //that report to the RLI and call the
0939: //list() method on each of them
0940: for (Iterator it = this .getReportingLRC().iterator(); it
0941: .hasNext();) {
0942: url = (String) it.next();
0943: message = "Querying LRC " + url;
0944: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0945:
0946: //push the lrcURL to the properties object
0947: mConnectProps.setProperty(this .URL_KEY, url);
0948: LRC lrc = new LRC();
0949: if (!lrc.connect(mConnectProps)) {
0950: //log an error/warning message
0951: mLogger.log("Unable to connect to LRC " + url,
0952: LogManager.ERROR_MESSAGE_LEVEL);
0953: continue;
0954: }
0955: try {
0956: result.addAll(lrc.list());
0957: } catch (Exception e) {
0958: mLogger
0959: .log("list()", e,
0960: LogManager.ERROR_MESSAGE_LEVEL);
0961: } finally {
0962: lrc.close();
0963: }
0964: mLogger.logCompletion(message,
0965: LogManager.DEBUG_MESSAGE_LEVEL);
0966: }
0967:
0968: return result;
0969: }
0970:
0971: /**
0972: * Lists a subset of all logical filenames in the catalog.
0973: *
0974: * @param constraint is a constraint for the logical filename only. It
0975: * is a string that has some meaning to the implementing system. This
0976: * can be a SQL wildcard for queries, or a regular expression for
0977: * Java-based memory collections.
0978: *
0979: * @return A set of logical filenames that match. The set may be empty
0980: */
0981: public Set list(String constraint) {
0982: //sanity check
0983: if (this .isClosed()) {
0984: //probably an exception should be thrown here!!
0985: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
0986: + this .mRLIURL);
0987: }
0988: Set result = new HashSet();
0989: String url = null, message = null;
0990: //we need to get hold of all the LRC
0991: //that report to the RLI and call the
0992: //list() method on each of them
0993: for (Iterator it = this .getReportingLRC().iterator(); it
0994: .hasNext();) {
0995: url = (String) it.next();
0996: message = "Querying LRC " + url;
0997: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
0998:
0999: //push the lrcURL to the properties object
1000: mConnectProps.setProperty(this .URL_KEY, url);
1001: LRC lrc = new LRC();
1002: if (!lrc.connect(mConnectProps)) {
1003: //log an error/warning message
1004: mLogger.log("Unable to connect to LRC " + url,
1005: LogManager.ERROR_MESSAGE_LEVEL);
1006: continue;
1007: }
1008: try {
1009: result.addAll(lrc.list(constraint));
1010: } catch (Exception e) {
1011: mLogger.log("list(String)", e,
1012: LogManager.ERROR_MESSAGE_LEVEL);
1013: } finally {
1014: lrc.close();
1015: }
1016: mLogger.logCompletion(message,
1017: LogManager.DEBUG_MESSAGE_LEVEL);
1018: }
1019:
1020: return result;
1021: }
1022:
1023: /**
1024: * Inserts a new mapping into the LRC running at the URL, where the RLI
1025: * is running.
1026: *
1027: * @param lfn is the logical filename under which to book the entry.
1028: * @param tuple is the physical filename and associated PFN attributes.
1029: *
1030: * @return number of insertions, should always be 1. On failure,
1031: * throws a RuntimeException.
1032: */
1033: public int insert(String lfn, ReplicaCatalogEntry tuple) {
1034: //get hold of the LRC if that is running
1035: LRC lrc = new LRC();
1036: int result = 1;
1037: if (!lrc.connect(mConnectProps)) {
1038: //log an error/warning message
1039: throw new RuntimeException(LRC_NOT_CONNECTED_MSG
1040: + mConnectProps.getProperty(URL_KEY));
1041: }
1042: result = lrc.insert(lfn, tuple);
1043: //better to keep a handle to the running LRC
1044: //as a member variable, and close it in
1045: //RLI.close()
1046: lrc.close();
1047: return result;
1048:
1049: }
1050:
1051: /**
1052: * Inserts a new mapping into the LRC running at the URL, where the RLI
1053: * is running.
1054: * This is a convenience function exposing the resource handle. Internally,
1055: * the <code>ReplicaCatalogEntry</code> element will be contructed, and passed to
1056: * the appropriate insert function.
1057: *
1058: * @param lfn is the logical filename under which to book the entry.
1059: * @param pfn is the physical filename associated with it.
1060: * @param handle is a resource handle where the PFN resides.
1061: *
1062: * @return number of insertions, should always be 1. On failure,
1063: * throws a RuntimeException.
1064: *
1065: * @see #insert( String, ReplicaCatalogEntry )
1066: * @see ReplicaCatalogEntry
1067: */
1068: public int insert(String lfn, String pfn, String handle) {
1069: //get hold of the LRC if that is running
1070: LRC lrc = new LRC();
1071: int result = 1;
1072: if (!lrc.connect(mConnectProps)) {
1073: //log an error/warning message
1074: throw new RuntimeException(LRC_NOT_CONNECTED_MSG
1075: + mConnectProps.getProperty(URL_KEY));
1076: }
1077: result = lrc.insert(lfn, pfn, handle);
1078: //better to keep a handle to the running LRC
1079: //as a member variable, and close it in
1080: //RLI.close()
1081: lrc.close();
1082: return result;
1083:
1084: }
1085:
1086: /**
1087: * Inserts multiple mappings into the replica catalog. The input is a
1088: * map indexed by the LFN. The value for each LFN key is a collection
1089: * of replica catalog entries.
1090: *
1091: * @param x is a map from logical filename string to list of replica
1092: * catalog entries.
1093: *
1094: * @return the number of insertions.
1095: * @see ReplicaCatalogEntry
1096: */
1097: public int insert(Map x) {
1098: //get hold of the LRC if that is running
1099: LRC lrc = new LRC();
1100: int result = 1;
1101: if (!lrc.connect(mConnectProps)) {
1102: //log an error/warning message
1103: throw new RuntimeException(LRC_NOT_CONNECTED_MSG
1104: + mConnectProps.getProperty(URL_KEY));
1105: }
1106: result = lrc.insert(x);
1107: //better to keep a handle to the running LRC
1108: //as a member variable, and close it in
1109: //RLI.close()
1110: lrc.close();
1111: return result;
1112:
1113: }
1114:
1115: /**
1116: * Deletes a specific mapping from the replica catalog. We don't care
1117: * about the resource handle. More than one entry could theoretically
1118: * be removed. Upon removal of an entry, all attributes associated
1119: * with the PFN also evaporate (cascading deletion).
1120: *
1121: * It can result in a deletion of more than one entry, and from more
1122: * than one local replica catalog that might be reporting to the RLI.
1123: *
1124: * @param lfn is the logical filename in the tuple.
1125: * @param pfn is the physical filename in the tuple.
1126: *
1127: * @return the number of removed entries.
1128: */
1129: public int delete(String lfn, String pfn) {
1130: ReplicaCatalogEntry rce = new ReplicaCatalogEntry(pfn);
1131: return delete(lfn, rce);
1132: }
1133:
1134: /**
1135: * Deletes a very specific mapping from the replica catalog. The LFN
1136: * must be matches, the PFN, and all PFN attributes specified in the
1137: * replica catalog entry. More than one entry could theoretically be
1138: * removed. Upon removal of an entry, all attributes associated with
1139: * the PFN also evaporate (cascading deletion).
1140: * It can result in a deletion of more than one entry, and from more
1141: * than one local replica catalog that might be reporting to the RLI.
1142: *
1143: * @param lfn is the logical filename in the tuple.
1144: * @param tuple is a description of the PFN and its attributes.
1145: *
1146: * @return the number of removed entries, either 0 or 1.
1147: */
1148: public int delete(String lfn, ReplicaCatalogEntry tuple) {
1149: //Map indexed by lrc url and each value a collection
1150: //of lfns that the RLI says are present in it.
1151: Set lfns = new HashSet(1);
1152: lfns.add(lfn);
1153: Map lrc2lfn = this .getLRC2LFNS(lfns);
1154: int result = 0;
1155:
1156: if (lrc2lfn == null) {
1157: //probably RLI is not connected!!
1158: return 0;
1159: }
1160:
1161: // call the delete function on the individual
1162: // LRCs where the mapping resides
1163: String key = null, message = null;
1164: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
1165: Map.Entry entry = (Map.Entry) it.next();
1166: key = (String) entry.getKey();
1167: message = "Querying LRC " + key;
1168: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1169:
1170: //push the lrcURL to the properties object
1171: mConnectProps.setProperty(this .URL_KEY, key);
1172: LRC lrc = new LRC();
1173: if (!lrc.connect(mConnectProps)) {
1174: //log an error/warning message
1175: mLogger.log("Unable to connect to LRC " + key,
1176: LogManager.ERROR_MESSAGE_LEVEL);
1177: continue;
1178: }
1179:
1180: //delete from the LRC
1181: try {
1182: result += lrc.delete(lfn, tuple);
1183: } catch (Exception ex) {
1184: mLogger.log("delete(String, ReplicaCatalogEntry)", ex,
1185: LogManager.ERROR_MESSAGE_LEVEL);
1186: } finally {
1187: //disconnect
1188: lrc.close();
1189: }
1190: mLogger.logCompletion(message,
1191: LogManager.DEBUG_MESSAGE_LEVEL);
1192: }
1193:
1194: return result;
1195:
1196: }
1197:
1198: /**
1199: * Deletes all PFN entries for a given LFN from the replica catalog
1200: * where the PFN attribute is found, and matches exactly the object
1201: * value. This method may be useful to remove all replica entries that
1202: * have a certain MD5 sum associated with them. It may also be harmful
1203: * overkill.
1204: * It can result in a deletion of more than one entry, and from more
1205: * than one local replica catalog that might be reporting to the RLI.
1206: *
1207: * @param lfn is the logical filename to look for.
1208: * @param name is the PFN attribute name to look for.
1209: * @param value is an exact match of the attribute value to match.
1210: *
1211: * @return the number of removed entries.
1212: */
1213: public int delete(String lfn, String name, Object value) {
1214: //Map indexed by lrc url and each value a collection
1215: //of lfns that the RLI says are present in it.
1216: Set lfns = new HashSet(1);
1217: lfns.add(lfn);
1218: Map lrc2lfn = this .getLRC2LFNS(lfns);
1219: int result = 0;
1220:
1221: if (lrc2lfn == null) {
1222: //probably RLI is not connected!!
1223: return 0;
1224: }
1225:
1226: // call the delete function on the individual
1227: // LRCs where the mapping resides
1228: String key = null, message = null;
1229: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
1230: Map.Entry entry = (Map.Entry) it.next();
1231: key = (String) entry.getKey();
1232: message = "Deleting from LRC " + key;
1233: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1234:
1235: //push the lrcURL to the properties object
1236: mConnectProps.setProperty(this .URL_KEY, key);
1237: LRC lrc = new LRC();
1238: if (!lrc.connect(mConnectProps)) {
1239: //log an error/warning message
1240: mLogger.log("Unable to connect to LRC " + key,
1241: LogManager.ERROR_MESSAGE_LEVEL);
1242: continue;
1243: }
1244:
1245: //delete from the LRC
1246: try {
1247: result += lrc.delete(lfn, name, value);
1248: } catch (Exception ex) {
1249: mLogger.log("delete(String, String, Object)", ex,
1250: LogManager.ERROR_MESSAGE_LEVEL);
1251: } finally {
1252: //disconnect
1253: lrc.close();
1254: }
1255: mLogger.logCompletion(message,
1256: LogManager.DEBUG_MESSAGE_LEVEL);
1257: }
1258:
1259: return result;
1260:
1261: }
1262:
1263: /**
1264: * Deletes all PFN entries for a given LFN from the replica catalog
1265: * where the resource handle is found. Karan requested this
1266: * convenience method, which can be coded like
1267: * <pre>
1268: * delete( lfn, SITE_ATTRIBUTE, handle )
1269: * </pre>
1270: *
1271: * It can result in a deletion of more than one entry, and from more
1272: * than one local replica catalog that might be reporting to the RLI.
1273: *
1274: * @param lfn is the logical filename to look for.
1275: * @param handle is the resource handle
1276: *
1277: * @return the number of entries removed.
1278: */
1279: public int deleteByResource(String lfn, String handle) {
1280: return delete(lfn, SITE_ATTRIBUTE, handle);
1281: }
1282:
1283: /**
1284: * Deletes multiple mappings into the replica catalog. The input is a
1285: * map indexed by the LFN. The value for each LFN key is a collection
1286: * of replica catalog entries. On setting matchAttributes to false, all entries
1287: * having matching lfn pfn mapping to an entry in the Map are deleted.
1288: * However, upon removal of an entry, all attributes associated with the pfn
1289: * also evaporate (cascaded deletion).
1290: * The deletes are done in batches.
1291: *
1292: * @param x is a map from logical filename string to list of
1293: * replica catalog entries.
1294: * @param matchAttributes whether mapping should be deleted only if all
1295: * attributes match.
1296: *
1297: * @return the number of deletions.
1298: * @see ReplicaCatalogEntry
1299: */
1300: public int delete(Map x, boolean matchAttributes) {
1301: //Map indexed by lrc url and each value a collection
1302: //of lfns that the RLI says are present in it.
1303: Set lfns = new HashSet(x.size());
1304: for (Iterator it = x.keySet().iterator(); it.hasNext();) {
1305: lfns.add((String) it.next());
1306: }
1307: Map lrc2lfn = this .getLRC2LFNS(lfns);
1308: int result = 0;
1309:
1310: if (lrc2lfn == null) {
1311: //probably RLI is not connected!!
1312: return 0;
1313: }
1314:
1315: //compose an exception that might need to be thrown
1316: CatalogException exception = new ReplicaCatalogException();
1317:
1318: // call the delete function on the individual
1319: // LRCs where the mapping resides
1320: String key = null, message = null;
1321: String lfn;
1322: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
1323: Map.Entry entry = (Map.Entry) it.next();
1324: key = (String) entry.getKey();
1325: lfns = (Set) entry.getValue();
1326: message = "Deleting from LRC " + key;
1327: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1328:
1329: //push the lrcURL to the properties object
1330: mConnectProps.setProperty(this .URL_KEY, key);
1331: LRC lrc = new LRC();
1332: if (!lrc.connect(mConnectProps)) {
1333: //log an error/warning message
1334: mLogger.log("Unable to connect to LRC " + key,
1335: LogManager.ERROR_MESSAGE_LEVEL);
1336: continue;
1337: }
1338:
1339: //compose the map to delete for a particular LRC
1340: Map lmap = new HashMap(lfns.size());
1341: for (Iterator lfnIt = lfns.iterator(); lfnIt.hasNext();) {
1342: lfn = (String) lfnIt.next();
1343: lmap.put(lfn, x.get(lfn));
1344: }
1345:
1346: //delete from the LRC
1347: try {
1348: result += lrc.delete(x, matchAttributes);
1349: } catch (ReplicaCatalogException e) {
1350: exception.setNextException(e);
1351: } catch (Exception ex) {
1352: mLogger.log("delete(Map,boolean)", ex,
1353: LogManager.ERROR_MESSAGE_LEVEL);
1354: } finally {
1355: //disconnect
1356: lrc.close();
1357: }
1358: mLogger.logCompletion(message,
1359: LogManager.DEBUG_MESSAGE_LEVEL);
1360: }
1361:
1362: //throw an exception only if a nested exception
1363: if ((exception = exception.getNextException()) != null)
1364: throw exception;
1365:
1366: return result;
1367:
1368: }
1369:
1370: /**
1371: * Removes all mappings for an LFN from the replica catalog.
1372: * It can result in a deletion of more than one entry, and from more
1373: * than one local replica catalog that might be reporting to the RLI.
1374: *
1375: * @param lfn is the logical filename to remove all mappings for.
1376: *
1377: * @return the number of removed entries.
1378: */
1379: public int remove(String lfn) {
1380: //Map indexed by lrc url and each value a collection
1381: //of lfns that the RLI says are present in it.
1382: Set lfns = new HashSet(1);
1383: lfns.add(lfn);
1384: Map lrc2lfn = this .getLRC2LFNS(lfns);
1385: int result = 0;
1386:
1387: if (lrc2lfn == null) {
1388: //probably RLI is not connected!!
1389: return 0;
1390: }
1391:
1392: // call the delete function on the individual
1393: // LRCs where the mapping resides
1394: String key = null, message = null;
1395: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
1396: Map.Entry entry = (Map.Entry) it.next();
1397: key = (String) entry.getKey();
1398: message = "Deleting from LRC " + key;
1399: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1400:
1401: //push the lrcURL to the properties object
1402: mConnectProps.setProperty(this .URL_KEY, key);
1403: LRC lrc = new LRC();
1404: if (!lrc.connect(mConnectProps)) {
1405: //log an error/warning message
1406: mLogger.log("Unable to connect to LRC " + key,
1407: LogManager.ERROR_MESSAGE_LEVEL);
1408: continue;
1409: }
1410:
1411: //delete from the LRC
1412: try {
1413: result += lrc.remove(lfn);
1414: } catch (Exception ex) {
1415: mLogger.log("remove(String):", ex,
1416: LogManager.ERROR_MESSAGE_LEVEL);
1417: } finally {
1418: //disconnect
1419: lrc.close();
1420: }
1421: mLogger.logCompletion(message,
1422: LogManager.DEBUG_MESSAGE_LEVEL);
1423: }
1424:
1425: return result;
1426:
1427: }
1428:
1429: /**
1430: * Removes all mappings for a set of LFNs.
1431: * It can result in a deletion of more than one entry, and from more
1432: * than one local replica catalog that might be reporting to the RLI.
1433: *
1434: * @param lfns is a set of logical filename to remove all mappings for.
1435: *
1436: * @return the number of removed entries.
1437: */
1438: public int remove(Set lfns) {
1439: //Map indexed by lrc url and each value a collection
1440: //of lfns that the RLI says are present in it.
1441: Map lrc2lfn = this .getLRC2LFNS(lfns);
1442: int result = 0;
1443: Set s = null;
1444:
1445: if (lrc2lfn == null) {
1446: //probably RLI is not connected!!
1447: return 0;
1448: }
1449:
1450: // call the delete function on the individual
1451: // LRCs where the mapping resides
1452: String key = null, message = null;
1453: for (Iterator it = lrc2lfn.entrySet().iterator(); it.hasNext();) {
1454: Map.Entry entry = (Map.Entry) it.next();
1455: key = (String) entry.getKey();
1456: message = "Deleting from LRC " + key;
1457: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1458:
1459: //push the lrcURL to the properties object
1460: mConnectProps.setProperty(this .URL_KEY, key);
1461: LRC lrc = new LRC();
1462: if (!lrc.connect(mConnectProps)) {
1463: //log an error/warning message
1464: mLogger.log("Unable to connect to LRC " + key,
1465: LogManager.ERROR_MESSAGE_LEVEL);
1466: continue;
1467: }
1468:
1469: //delete from the LRC
1470: try {
1471: s = (Set) entry.getValue();
1472: mLogger.log("\tDeleting the following lfns " + s,
1473: LogManager.DEBUG_MESSAGE_LEVEL);
1474: result += lrc.remove((Set) entry.getValue());
1475: } catch (Exception ex) {
1476: mLogger.log("remove(Set)", ex,
1477: LogManager.ERROR_MESSAGE_LEVEL);
1478: } finally {
1479: //disconnect
1480: lrc.close();
1481: }
1482: mLogger.logCompletion(message,
1483: LogManager.DEBUG_MESSAGE_LEVEL);
1484: }
1485:
1486: return result;
1487:
1488: }
1489:
1490: /**
1491: * Removes all entries from the replica catalog where the PFN attribute
1492: * is found, and matches exactly the object value.
1493: * It can result in a deletion of more than one entry, and from more
1494: * than one local replica catalog that might be reporting to the RLI.
1495: *
1496: * @param name is the PFN attribute name to look for.
1497: * @param value is an exact match of the attribute value to match.
1498: *
1499: * @return the number of removed entries.
1500: */
1501: public int removeByAttribute(String name, Object value) {
1502: //sanity check
1503: if (this .isClosed()) {
1504: //probably an exception should be thrown here!!
1505: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
1506: + this .mRLIURL);
1507: }
1508: int result = 0;
1509: String url = null;
1510: //we need to get hold of all the LRC
1511: //that report to the RLI and call the
1512: //list() method on each of them
1513: for (Iterator it = this .getReportingLRC().iterator(); it
1514: .hasNext();) {
1515: url = (String) it.next();
1516:
1517: mLogger.log("Removing from LRC " + url,
1518: LogManager.DEBUG_MESSAGE_LEVEL);
1519:
1520: //push the lrcURL to the properties object
1521: mConnectProps.setProperty(this .URL_KEY, url);
1522: LRC lrc = new LRC();
1523: if (!lrc.connect(mConnectProps)) {
1524: //log an error/warning message
1525: mLogger.log("Unable to connect to LRC " + url,
1526: LogManager.ERROR_MESSAGE_LEVEL);
1527: continue;
1528: }
1529: try {
1530: result += lrc.removeByAttribute(name, value);
1531: } catch (Exception e) {
1532: mLogger.log("removeByAttribute(String,Object)", e,
1533: LogManager.ERROR_MESSAGE_LEVEL);
1534: } finally {
1535: lrc.close();
1536: }
1537: mLogger.logCompletion("Removing from LRC " + url,
1538: LogManager.DEBUG_MESSAGE_LEVEL);
1539: }
1540:
1541: return result;
1542:
1543: }
1544:
1545: /**
1546: * Removes all entries associated with a particular resource handle.
1547: * This is useful, if a site goes offline. It is a convenience method,
1548: * which calls the generic <code>removeByAttribute</code> method.
1549: * It can result in a deletion of more than one entry, and from more
1550: * than one local replica catalog that might be reporting to the RLI.
1551: *
1552: * @param handle is the site handle to remove all entries for.
1553: *
1554: * @return the number of removed entries.
1555: * @see #removeByAttribute( String, Object )
1556: */
1557: public int removeByAttribute(String handle) {
1558: return removeByAttribute(SITE_ATTRIBUTE, handle);
1559: }
1560:
1561: /**
1562: * Removes everything from all the LRCs that report to this RLI.
1563: * Use with caution!
1564: *
1565: * @return the number of removed entries.
1566: */
1567: public int clear() {
1568: //sanity check
1569: if (this .isClosed()) {
1570: //probably an exception should be thrown here!!
1571: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
1572: + this .mRLIURL);
1573: }
1574: int result = 0;
1575: String url = null, message = null;
1576: //we need to get hold of all the LRC
1577: //that report to the RLI and call the
1578: //list() method on each of them
1579: for (Iterator it = this .getReportingLRC().iterator(); it
1580: .hasNext();) {
1581: url = (String) it.next();
1582: message = "Querying LRC " + url;
1583: mLogger.log(message, LogManager.DEBUG_MESSAGE_LEVEL);
1584:
1585: //push the lrcURL to the properties object
1586: mConnectProps.setProperty(this .URL_KEY, url);
1587: LRC lrc = new LRC();
1588: if (!lrc.connect(mConnectProps)) {
1589: //log an error/warning message
1590: mLogger.log("Unable to connect to LRC " + url,
1591: LogManager.ERROR_MESSAGE_LEVEL);
1592: continue;
1593: }
1594: try {
1595: result += lrc.clear();
1596: } catch (Exception e) {
1597: mLogger.log("list(String)", e,
1598: LogManager.ERROR_MESSAGE_LEVEL);
1599: } finally {
1600: lrc.close();
1601: }
1602: mLogger.logCompletion(message,
1603: LogManager.DEBUG_MESSAGE_LEVEL);
1604: }
1605:
1606: return result;
1607: }
1608:
1609: /**
1610: * Explicitely free resources before the garbage collection hits.
1611: */
1612: public void close() {
1613: try {
1614: if (mRLS != null)
1615: mRLS.Close();
1616: } catch (RLSException e) {
1617: //ignore
1618: } finally {
1619: mRLS = null;
1620: }
1621: }
1622:
1623: /**
1624: * Returns whether the connection to the RLS with which this instance is
1625: * associated is closed or not.
1626: *
1627: * @return true, if the implementation is disassociated, false otherwise.
1628: * @see #close()
1629: */
1630: public boolean isClosed() {
1631: return (mRLS == null);
1632: }
1633:
1634: /**
1635: * It returns the timeout value in seconds after which to timeout in case of
1636: * no activity from the RLI.
1637: *
1638: * Referred to by the "rli.timeout" property.
1639: *
1640: * @param properties the properties passed in the connect method.
1641: *
1642: * @return the timeout value if specified else,
1643: * the value specified by "rls.timeout" property, else
1644: * DEFAULT_RLI_TIMEOUT.
1645: *
1646: * @see #DEFAULT_RLI_TIMEOUT
1647: */
1648: public int getTimeout(Properties properties) {
1649: String prop = properties.getProperty(this .RLI_TIMEOUT_KEY);
1650:
1651: //if prop is null get rls timeout,
1652: prop = (prop == null) ? properties
1653: .getProperty(this .RLS_TIMEOUT_KEY) : prop;
1654:
1655: int val = 0;
1656: try {
1657: val = Integer.parseInt(prop);
1658: } catch (Exception e) {
1659: val = Integer.parseInt(DEFAULT_RLI_TIMEOUT);
1660: }
1661: return val;
1662:
1663: }
1664:
1665: /**
1666: * Sets the number of lfns in each batch while querying the lrc in the
1667: * bulk mode.
1668: *
1669: * @param properties the properties passed while connecting.
1670: *
1671: */
1672: protected void setBatchSize(Properties properties) {
1673: String s = properties.getProperty(this .BATCH_KEY);
1674: int size = this .RLS_BULK_QUERY_SIZE;
1675: try {
1676: size = Integer.parseInt(s);
1677: } catch (Exception e) {
1678: }
1679: mBatchSize = size;
1680: }
1681:
1682: /**
1683: * Returns the number of lfns in each batch while querying the lrc in the
1684: * bulk mode.
1685: *
1686: * @return the batch size.
1687: */
1688: protected int getBatchSize() {
1689: return mBatchSize;
1690: }
1691:
1692: /**
1693: * Returns a map indexed by lrc urls. Each value is a set of
1694: * String objects referring to the logical filenames whose mappings reside
1695: * at a particular lrc amongst the set of logical filenames passed.
1696: *
1697: * @param lfns the set of lfns queried to the RLI.
1698: *
1699: * @return Map indexed by lrc urls. Each value is a set of lfn strings.
1700: * null in case the connection to RLI is closed or error.
1701: */
1702: private Map getLRC2LFNS(Set lfns) {
1703: int batch = lfns.size() > mBatchSize ? mBatchSize : lfns.size();
1704: //sanity check
1705: if (this .isClosed()) {
1706: //probably an exception should be thrown here!!
1707: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
1708: + this .mRLIURL);
1709: }
1710:
1711: Map lrc2lfn = new HashMap();//indexed by lrc url and each value a collection
1712: //of lfns that the RLI says are present init.
1713: //get a handle to the rli
1714:
1715: //we need to query the RLI in batches
1716: for (Iterator it = lfns.iterator(); it.hasNext();) {
1717: ArrayList l = new ArrayList(batch);
1718: for (int j = 0; (j < batch) && (it.hasNext()); j++) {
1719: l.add(it.next());
1720: }
1721:
1722: //query the RLI for one batch
1723: List res = null;
1724: try {
1725: res = mRLI.getLRCBulk(l);
1726: } catch (RLSException ex) {
1727: mLogger.log("getLRC2LFNS(Set)", ex,
1728: LogManager.ERROR_MESSAGE_LEVEL);
1729: //or throw a runtime exception
1730: return null;
1731: }
1732: //iterate through the results and put them in the map
1733: String lrc = null;
1734: String lfn = null;
1735: for (Iterator lit = res.iterator(); lit.hasNext();) {
1736: RLSString2Bulk s2b = (RLSString2Bulk) lit.next();
1737: lfn = s2b.s1;//s1 is the lfn
1738: lrc = s2b.s2;//s2 denotes the lrc which contains the mapping
1739:
1740: //rc is the exit status returned by the RLI
1741: if (s2b.rc == RLSClient.RLS_SUCCESS) {
1742: //we are really only concerned with success
1743: //and do not care about other exit codes
1744: Object val = null;
1745: Set s = null;
1746: s = ((val = lrc2lfn.get(lrc)) == null) ? new LinkedHashSet()
1747: : (LinkedHashSet) val;
1748: s.add(lfn);
1749: if (val == null)
1750: lrc2lfn.put(lrc, s);
1751:
1752: }
1753: }
1754: }
1755:
1756: //match LRC's just once against ingore and restrict lists
1757: for (Iterator it = lrc2lfn.keySet().iterator(); it.hasNext();) {
1758: String lrc = (String) it.next();
1759: int state = this .determineQueryType(lrc);
1760:
1761: //do the query on the basis of the state
1762: if (state == LRC_QUERY_IGNORE) {
1763: mLogger.log("Skipping LRC " + lrc,
1764: LogManager.DEBUG_MESSAGE_LEVEL);
1765: it.remove();
1766: }
1767: }
1768:
1769: return lrc2lfn;
1770: }
1771:
1772: /**
1773: * Returns a tri state indicating what type of query needs to be done to
1774: * a particular LRC.
1775: *
1776: * @param url the LRC url.
1777: *
1778: * @return tristate
1779: */
1780: private int determineQueryType(String url) {
1781: int type = this .LRC_QUERY_NORMAL;
1782:
1783: if (mLRCRestrictList != null) {
1784: for (int j = 0; j < mLRCRestrictList.length; j++) {
1785: if (url.indexOf(mLRCRestrictList[j]) != -1) {
1786: type = this .LRC_QUERY_RESTRICT;
1787: break;
1788: }
1789: }
1790: }
1791: if (mLRCIgnoreList != null) {
1792: for (int j = 0; j < mLRCIgnoreList.length; j++) {
1793: if (url.indexOf(mLRCIgnoreList[j]) != -1) {
1794: type = this .LRC_QUERY_IGNORE;
1795: break;
1796: }
1797: }
1798: }
1799:
1800: return type;
1801: }
1802:
1803: /**
1804: * Returns the rls LRC urls to ignore for querying (requested by LIGO).
1805: *
1806: * Referred to by the "pegasus.catalog.replica.lrc.ignore" property.
1807: *
1808: * @param properties the properties passed in the connect method.
1809: *
1810: * @return String[] if a comma separated list supplied as the property value,
1811: * else null
1812: */
1813: protected String[] getRLSLRCIgnoreURLs(Properties properties) {
1814: String urls = properties.getProperty(this .LRC_IGNORE_KEY, null);
1815: if (urls != null) {
1816: String[] urllist = urls.split(",");
1817: return urllist;
1818: } else {
1819: return null;
1820: }
1821: }
1822:
1823: /**
1824: * Returns the rls LRC urls to restrict for querying (requested by LIGO).
1825: *
1826: * Referred to by the "pegasus.catalog.replica.lrc.restrict" property.
1827: *
1828: * @param properties the properties passed in the connect method.
1829: *
1830: * @return String[] if a comma separated list supplied as the property value,
1831: * else null
1832: */
1833: protected String[] getRLSLRCRestrictURLs(Properties properties) {
1834: String urls = properties.getProperty(this .LRC_RESTRICT_KEY,
1835: null);
1836: if (urls != null) {
1837: String[] urllist = urls.split(",");
1838: return urllist;
1839: } else {
1840: return null;
1841: }
1842: }
1843:
1844: /**
1845: * Retrieves the URLs of all the LRCs that report to the RLI.
1846: *
1847: * @return a Set containing the URLs to all the LRCs that report to the
1848: * RLI.
1849: */
1850: private Set getReportingLRC() {
1851: //sanity check
1852: if (this .isClosed()) {
1853: //probably an exception should be thrown here!!
1854: throw new RuntimeException(RLI_NOT_CONNECTED_MSG
1855: + this .mRLIURL);
1856: }
1857: Set result = new HashSet();
1858: Collection c = null;
1859:
1860: try {
1861: c = mRLI.lrcList();
1862: } catch (RLSException e) {
1863: mLogger.log("getReportingLRC(Set)", e,
1864: LogManager.ERROR_MESSAGE_LEVEL);
1865: }
1866:
1867: for (Iterator it = c.iterator(); it.hasNext();) {
1868: RLSLRCInfo lrc = (RLSLRCInfo) it.next();
1869: result.add(lrc.url);
1870: }
1871:
1872: return result;
1873: }
1874:
1875: /**
1876: * Populates the mapping table by querying the LRC in the mLRCList. At
1877: * present it searches for all the files in the original DAG. At this point
1878: * it should be all the files in the Reduced Dag but not doing so in order to
1879: * conserve memory.
1880: *
1881: * @param allInCache indicates whether all input file entries were found in
1882: * cache or not.
1883: *
1884: * @return List
1885: */
1886: /*
1887: private List populateMapTable( boolean allInCache ) {
1888: String lrcURL = null;
1889: List list = null;
1890: RLSQuery client = null;
1891: ReplicaLocation rl = null;
1892: List pfnList = null;
1893:
1894: mTable = new HashMap( mSearchFiles.size() );
1895:
1896: int size = mLRCMap.size();
1897: mLogger.log("Number of LRCs that will be queried: "+size,
1898: LogManager.DEBUG_MESSAGE_LEVEL);
1899: for ( Iterator iter = mLRCMap.keySet().iterator(); iter.hasNext(); ) {
1900: lrcURL = ( String ) iter.next();
1901: int state = this.determineQueryType(lrcURL);
1902:
1903: //do the query on the basis of the state
1904: if ( state == LRC_QUERY_IGNORE ) {
1905: mLogger.log( "Skipping LRC " + lrcURL,
1906: LogManager.DEBUG_MESSAGE_LEVEL);
1907: }
1908: else{
1909: mLogger.log( "Querying LRC " + lrcURL,
1910: LogManager.DEBUG_MESSAGE_LEVEL);
1911: list = ( ArrayList ) mLRCMap.get( lrcURL );
1912: try {
1913: client = new RLSQuery( lrcURL );
1914: boolean restrict = (state == LRC_QUERY_RESTRICT);
1915: client.bulkQueryLRC( list, RLSQuery.RLS_BULK_QUERY_SIZE,
1916: mTable,restrict);
1917: client.close();
1918: } catch ( Exception e ) {
1919: mLogMsg =
1920: "RLSEngine.java: While getting connection to LRC " +
1921: lrcURL + " " + e;
1922: mLogger.log( mLogMsg, LogManager.ERROR_MESSAGE_LEVEL );
1923: size--;
1924:
1925: //do a hard fail only if the RLS exitmode is set to error
1926: //or we could not query to all the LRCs
1927: // and we could not find all the entries in the cache
1928: mLogger.log("RLS exit mode is " + mProps.getRLSExitMode(),
1929: LogManager.DEBUG_MESSAGE_LEVEL);
1930: boolean exitOnError = mProps.getRLSExitMode().equalsIgnoreCase( "error" );
1931: if ( exitOnError || ( size == 0 && !allInCache )) {
1932: mLogMsg = ( exitOnError ) ?
1933: "Unable to access LRC " + lrcURL :
1934: "Unable to query any LRC and not all input files are in cache!";
1935: throw new RuntimeException( mLogMsg );
1936: }
1937: }
1938: mLogger.logCompletion("Querying LRC " + lrcURL,
1939: LogManager.DEBUG_MESSAGE_LEVEL);
1940:
1941: }
1942: }
1943: return new ArrayList( mTable.keySet() );
1944:
1945: }
1946:
1947: */
1948: /**
1949: * The main program, for some unit testing.
1950: *
1951: * @param args String[]
1952: */
1953: public static void main(String[] args) {
1954: RLI rli = new RLI();
1955: String lfn = "test";
1956: Set s = new HashSet();
1957: s.add(lfn);
1958: s.add("testX");
1959: s.add("vahi.f.a");
1960: System.out.println("Connecting " + rli.connect("rls://sukhna"));
1961: boolean insert = false;
1962: LogManager.getInstance().setLevel(
1963: LogManager.DEBUG_MESSAGE_LEVEL);
1964:
1965: if (insert) {
1966: ReplicaCatalogEntry rce = new ReplicaCatalogEntry(
1967: "gsiftp://sukhna.isi.edu/tmp/test");
1968: rce.addAttribute("name", "karan");
1969: LRC sukhna = new LRC();
1970: sukhna.connect("rls://sukhna");
1971: sukhna.insert("test", rce);
1972: sukhna.insert("test", "gsiftp://sukhna.isi.edu/tmp/test1",
1973: "isi");
1974: sukhna.insert("vahi.f.a", "file:///tmp/vahi.f.a", "isi");
1975: sukhna.insert("testX", "gsiftp://sukhna.isi.edu/tmp/testX",
1976: "isi");
1977: sukhna.insert("testX",
1978: "gsiftp://sukhna.isi.edu/tmp/testXvahi", "isi");
1979: sukhna.close();
1980:
1981: LRC smarty = new LRC();
1982: ReplicaCatalogEntry rce1 = new ReplicaCatalogEntry(
1983: "gsiftp://smarty.isi.edu/tmp/test");
1984: rce1.addAttribute("name", "gaurang");
1985:
1986: smarty.connect("rlsn://smarty");
1987: smarty.insert("test", rce1);
1988: smarty.insert("test", "gsiftp://smarty.isi.edu/tmp/test1",
1989: "isi");
1990: smarty.insert("vahi.f.a", "file:///tmp-smarty/vahi.f.a",
1991: "isi");
1992: smarty.insert("testX", "gsiftp://smarty.isi.edu/tmp/testX",
1993: "isi");
1994: smarty.close();
1995: }
1996:
1997: System.out.println("\n Searching for lfn " + lfn);
1998: System.out.println(rli.lookup(lfn));
1999:
2000: System.out
2001: .println("\n Searching for lfn w/o attributes " + lfn);
2002: System.out.println(rli.lookupNoAttributes(lfn));
2003:
2004: System.out.println("\nSearching for a set of lfn " + s);
2005: System.out.println(rli.lookup(s));
2006:
2007: System.out
2008: .println("\nSearching for a set of lfn with handle matching"
2009: + s);
2010: System.out.println(rli.lookup(s, "isi"));
2011:
2012: System.out
2013: .println("\nSearching for a set of lfn with handle matching "
2014: + " returning only pfns" + s);
2015: System.out.println(rli.lookupNoAttributes(s, "isi"));
2016:
2017: System.out.println("\nListing all the lfns tracked in RLI");
2018: System.out.println(rli.list("*").size());
2019:
2020: //System.out.println("\n Removing entry for lfn " + lfn);
2021: //System.out.println(rli.remove(lfn));
2022:
2023: //System.out.println("\n Removing entry for lfns " + s);
2024: //System.out.println(rli.remove(s));
2025:
2026: //System.out.println("\n Removing entry for lfn by handle matching " + lfn);
2027: //System.out.println(rli.deleteByResource(lfn,"isi"));
2028:
2029: //System.out.println("\nSearching for a set of lfn " + s);
2030: //System.out.println(rli.lookup(s));
2031: Map m = new HashMap();
2032: m.put("lfn", "test*");
2033: m.put("pfn", "*vahi*");
2034: System.out.println("\nDoing a constraint lookup "
2035: + rli.lookup(m));
2036: rli.close();
2037: }
2038:
2039: }//end of main class
|