001: /**
002: * This file or a portion of this file is licensed under the terms of
003: * the Globus Toolkit Public License, found at $PEGASUS_HOME/GTPL or
004: * http://www.globus.org/toolkit/download/license.html.
005: * This notice must appear in redistributions of this file
006: * with or without modification.
007: *
008: * Redistributions of this Software, with or without modification, must reproduce
009: * the GTPL in:
010: * (1) the Software, or
011: * (2) the Documentation or
012: * some other similar material which is provided with the Software (if any).
013: *
014: * Copyright 1999-2004
015: * University of Chicago and The University of Southern California.
016: * All rights reserved.
017: */package org.griphyn.cPlanner.poolinfo;
018:
019: import org.griphyn.cPlanner.classes.GridFTPServer;
020: import org.griphyn.cPlanner.classes.LRC;
021: import org.griphyn.cPlanner.classes.PoolConfig;
022: import org.griphyn.cPlanner.classes.Profile;
023: import org.griphyn.cPlanner.classes.SiteInfo;
024: import org.griphyn.cPlanner.classes.SubInfo;
025: import org.griphyn.cPlanner.classes.WorkDir;
026:
027: import org.griphyn.cPlanner.common.LogManager;
028: import org.griphyn.cPlanner.common.PegRandom;
029: import org.griphyn.cPlanner.common.PegasusProperties;
030: import org.griphyn.cPlanner.common.UserOptions;
031: import org.griphyn.cPlanner.common.Utility;
032:
033: import org.griphyn.cPlanner.engine.Engine;
034:
035: import org.griphyn.cPlanner.namespace.Namespace;
036: import org.griphyn.cPlanner.namespace.VDS;
037:
038: import org.griphyn.common.classes.SysInfo;
039:
040: import java.io.File;
041:
042: import java.util.ArrayList;
043: import java.util.HashMap;
044: import java.util.Iterator;
045: import java.util.List;
046: import java.util.Map;
047: import java.util.Set;
048:
049: /**
050: * This is an abstract class which defines the interface for the information
051: * providers like sites.xml, sites.catalog.
052: *
053: * @author Karan Vahi
054: * @author Gaurang Mehta
055: * @version $Revision: 279 $
056: */
057:
058: public abstract class PoolInfoProvider {
059:
060: /**
061: * The name of the environment variable PEGASUS_HOME.
062: */
063: public static final String PEGASUS_HOME = "PEGASUS_HOME";
064:
065: /**
066: * The name of the environment variable VDS_HOME.
067: */
068: public static final String VDS_HOME = "VDS_HOME";
069:
070: /**
071: * The LogManager object which is used to log all the messages. It's values
072: * are set in the CPlanner class.
073: */
074: protected LogManager mLogger;
075:
076: /**
077: * The String holding the log message.
078: */
079: protected String mLogMsg;
080:
081: /**
082: * The path to the pool information provider.
083: */
084: protected String mPoolProvider;
085:
086: /**
087: * The object holding all the properties pertaining to Pegasus.
088: */
089: protected PegasusProperties mProps;
090:
091: /**
092: * The working directory relative to the mount point of the execution pool.
093: * It is populated from the pegasus.dir.exec property from the properties file.
094: * If not specified then it work_dir is supposed to be the exec mount point
095: * of the execution pool.
096: */
097: protected String mWorkDir;
098:
099: /**
100: * This contains the storage directory relative to the se mount point of the
101: * pool. It is populated from the pegasus.dir.storage property from the properties
102: * file. If not specified then the storage directory is the se mount point
103: * from the pool.config file.
104: */
105: protected String mStorageDir;
106:
107: /**
108: * Handle to the Singleton instance containing the options passed to the
109: * planner at run time.
110: */
111: protected UserOptions mUserOpts;
112:
113: /**
114: * A boolean indicating whether to have a deep directory structure for
115: * the storage directory or not.
116: */
117: protected boolean mDeepStorageStructure;
118:
119: /**
120: *
121: * The method which returns a Singleton instance of the derived InfoProvider
122: * class. It must be overriden, by implementing classes.
123: *
124: * @param poolProvider the url to site catalog source. Can be a URL.
125: *
126: * @return PoolInfoProvider
127: */
128: public static PoolInfoProvider singletonInstance(String poolProvider) {
129: String msg = "The method singletonInstance(String) not implemented "
130: + "\nby the pool provider implementing class";
131:
132: //throw an exception
133: //it seems the derived class did not define the method
134: throw new java.lang.UnsupportedOperationException(msg);
135:
136: }
137:
138: /**
139: * The method that returns a Non Singleton instance of the dervived
140: * InfoProvider class. This method if invoked should also ensure that all
141: * other internal Pegasus objectslike PegasusProperties are invoked
142: * in a non singleton manner. It must be overriden, by implementing
143: * classes.
144: *
145: * @param poolProvider the path to the file containing the pool information.
146: * @param propFileName the name of the properties file that needs to be
147: * picked up from PEGASUS_HOME/etc directory.If it is null,
148: * then the default file should be picked up.
149: * @return PoolInfoProvider
150: */
151: public static PoolInfoProvider nonSingletonInstance(
152: String poolProvider, String propFileName) {
153:
154: String msg = "The method nonSingletonInstance(String,String) not implemented "
155: + "\nby the pool provider implementing class";
156:
157: //throw an exception
158: //it seems the derived class did not define the method
159: throw new java.lang.UnsupportedOperationException(msg);
160:
161: }
162:
163: /**
164: * It loads the objects that the pool providers need in a singleton manner,
165: * wherever possible. If the class in not implemented in Singleton manner,
166: * the objects would be loaded normally.
167: */
168: protected void loadSingletonObjects() {
169: mLogger = LogManager.getInstance();
170: mLogMsg = new String();
171: mPoolProvider = new String();
172: mProps = PegasusProperties.getInstance();
173: mUserOpts = UserOptions.getInstance();
174: mWorkDir = mProps.getExecDirectory();
175: mStorageDir = mProps.getStorageDirectory();
176: mDeepStorageStructure = mProps
177: .useDeepStorageDirectoryStructure();
178: }
179:
180: /**
181: * It loads the objects using their non singleton implementations.
182: *
183: *
184: * @param propFileName the name of the properties file that needs to be
185: * picked up from PEGASUS_HOME/etc directory.If it is null,
186: * then the default properties file should be picked up.
187: *
188: */
189: public void loadNonSingletonObjects(String propFileName) {
190: //these should be invoked in non singleton
191: //manner but is not.
192: mLogger = LogManager.getInstance();
193:
194: mLogMsg = new String();
195: mPoolProvider = new String();
196: mProps = PegasusProperties.getInstance((propFileName == null) ?
197: //load the default properties file
198: org.griphyn.common.util.VDSProperties.PROPERTY_FILENAME
199: :
200: //load the file with this name from $PEGASUS_HOME/etc directory
201: propFileName);
202:
203: //these should be invoked in non singleton
204: //manner but is not.
205: mUserOpts = UserOptions.getInstance();
206:
207: mWorkDir = mProps.getExecDirectory();
208: mStorageDir = mProps.getStorageDirectory();
209: mDeepStorageStructure = mProps
210: .useDeepStorageDirectoryStructure();
211: }
212:
213: /**
214: * Returns the System information for a bunch of sites.
215: *
216: * @param siteids List The siteid whose system information is required
217: *
218: * @return Map The key is the siteid and the value is a SysInfo object
219: *
220: * @see org.griphyn.common.classes.SysInfo
221: */
222: public abstract Map getSysinfos(List siteids);
223:
224: /**
225: * Returns the System information for a single site.
226: *
227: * @param siteID String The site whose system information is requested
228: *
229: * @return SysInfo The system information as a SysInfo object
230: *
231: * @see org.griphyn.common.classes.SysInfo
232: */
233: public abstract SysInfo getSysinfo(String siteID);
234:
235: /**
236: * Gets the pool information from the pool.config file on the basis
237: * of the name of the pool, and the universe.
238: *
239: * @param siteID the name of the site
240: * @param universe the execution universe for the job
241: *
242: * @return the corresponding pool object for the entry if found
243: * else null
244: */
245: public abstract SiteInfo getPoolEntry(String siteID, String universe);
246:
247: /**
248: * It returns the profile information associated with a particular pool. If
249: * the pool provider has no such information it should return null.
250: * The name of the object may purport that it is specific to GVDS format, but
251: * in fact it a tuple consisting of namespace, key and value that can be used
252: * by other Pool providers too.
253: *
254: * @param siteID the name of the site, whose profile information you want.
255: *
256: * @return List of <code>Profile</code> objects
257: * null if the information about the site is not with the pool provider.
258: *
259: * @see org.griphyn.cPlanner.classes.Profile
260: */
261: public abstract List getPoolProfile(String siteID);
262:
263: /**
264: * It returns all the jobmanagers corresponding to a specified site.
265: *
266: * @param siteID the name of the site at which the jobmanager runs.
267: *
268: * @return list of <code>JobManager</code>, each referring to
269: * one jobmanager contact string. An empty list if no jobmanagers
270: * found.
271: */
272: public abstract List getJobmanagers(String siteID);
273:
274: /**
275: * It returns all the jobmanagers corresponding to a specified pool and
276: * universe.
277: *
278: * @param siteID the name of the site at which the jobmanager runs.
279: * @param universe the gvds universe with which it is associated.
280: *
281: * @return list of <code>JobManager</code>, each referring to
282: * one jobmanager contact string. An empty list if no jobmanagers
283: * found.
284: */
285: public abstract List getJobmanagers(String siteID, String universe);
286:
287: /**
288: * It returns all the gridftp servers corresponding to a specified pool.
289: *
290: * @param siteID the name of the site at which the jobmanager runs.
291: *
292: * @return List of <code>GridFTPServer</code>, each referring to one
293: * GridFtp Server.
294: */
295: public abstract List getGridFTPServers(String siteID);
296:
297: /**
298: * It returns all the pools available in the site catalog
299: *
300: * @return List of names of the pools available as String
301: */
302: public abstract List getPools();
303:
304: /**
305: * This is a soft state remove, that removes a jobmanager from a particular
306: * pool entry. The cause of this removal could be the inability to
307: * authenticate against it at runtime. The successful removal lead Pegasus
308: * not to schedule job on that particular jobmanager.
309: *
310: * @param siteID the name of the site at which the jobmanager runs.
311: * @param universe the gvds universe with which it is associated.
312: * @param jobManagerContact the contact string to the jobmanager.
313: *
314: * @return true if was able to remove the jobmanager from the cache
315: * false if unable to remove, or the matching entry is not found
316: * or if the implementing class does not maintain a soft state.
317: */
318: public abstract boolean removeJobManager(String siteID,
319: String universe, String jobManagerContact);
320:
321: /**
322: * This is a soft state remove, that removes a gridftp server from a particular
323: * pool entry. The cause of this removal could be the inability to
324: * authenticate against it at runtime. The successful removal lead Pegasus
325: * not to schedule any transfers on that particular gridftp server.
326: *
327: * @param siteID the name of the site at which the gridftp runs.
328: * @param urlPrefix the url prefix containing the protocol,hostname and port.
329: *
330: * @return true if was able to remove the gridftp from the cache
331: * false if unable to remove, or the matching entry is not found
332: * or if the implementing class does not maintain a soft state.
333: * or the information about site is not in the site catalog.
334: */
335: public abstract boolean removeGridFtp(String siteID,
336: String urlPrefix);
337:
338: /**
339: * Returns a textual description of the pool mode being used.
340: * @return String
341: */
342: public abstract String getPoolMode();
343:
344: /**
345: * Return a random lrc url from the list of lrc url's.
346: * @param lrcs Arraylist of <code>LRC</code> objects.
347: *
348: * @return String Returns one of lrc url's
349: * @see org.griphyn.cPlanner.classes.LRC
350: */
351: public String selectLRC(ArrayList lrcs) {
352: String lrcurl = null;
353: if (lrcs.size() == 1) {
354: lrcurl = ((LRC) (lrcs.get(0))).getURL();
355: } else {
356: lrcurl = ((LRC) (lrcs
357: .get((int) Math.random() * lrcs.size()))).getURL();
358: }
359: return lrcurl;
360: }
361:
362: /**
363: * Returns the path to the execution mount point (The Workdir).
364: *
365: * @param workdir the <code>WorkDir</code> object containing the workdir
366: * information.
367: *
368: * @return String The exec-mount point (aka workdir)
369: *
370: * @throws Exception
371: */
372: public String selectWorkdir(WorkDir workdir) throws Exception {
373: return workdir.getInfo(WorkDir.WORKDIR);
374: }
375:
376: /**
377: * Return a random gridftp url from the list of gridftp url's.
378: *
379: * @param ftp Takes an ArrayList of <code>GridFTPServer</code> Objects.
380: *
381: * @return String Returns a single gridftp url from among many
382: *
383: * @see org.griphyn.cPlanner.classes.GridFTPServer
384: */
385: public GridFTPServer selectGridFtp(ArrayList ftp) {
386: int sel = PegRandom.getInteger(ftp.size() - 1);
387: return (GridFTPServer) (ftp.get(sel));
388: }
389:
390: /**
391: * Returns the value of VDS_HOME for a site.
392: *
393: * @param siteID the name of the site.
394: * @return value if set else null.
395: */
396: public String getVDS_HOME(String siteID) {
397: return this .getEnvironmentVariable(siteID, VDS_HOME);
398: }
399:
400: /**
401: * Returns the value of PEGASUS_HOME for a site.
402: *
403: * @param siteID the name of the site.
404: * @return value if set else null.
405: */
406: public String getPegasusHome(String siteID) {
407: return this .getEnvironmentVariable(siteID, PEGASUS_HOME);
408: }
409:
410: /**
411: * Returns an environment variable for a particular site set in the
412: * Site Catalog.
413: *
414: * @param siteID the name of the site.
415: * @param envVariable the environment variable whose value is required.
416: *
417: * @return value of the environment variable if found, else null
418: */
419: public String getEnvironmentVariable(String siteID,
420: String envVariable) {
421: String result = null;
422:
423: //get all environment variables
424: List envs = this .getPoolProfile(siteID, Profile.ENV);
425: if (envs == null) {
426: return result;
427: }
428:
429: //traverse through all the environment variables
430: for (Iterator it = envs.iterator(); it.hasNext();) {
431: Profile p = (Profile) it.next();
432: if (p.getProfileKey().equals(envVariable)) {
433: result = p.getProfileValue();
434: break;
435: }
436: }
437:
438: return result;
439: }
440:
441: /**
442: * It returns profile information associated with a particular namespace and
443: * pool.
444: *
445: * @param siteID the name of the site, whose profile information you want.
446: * @param namespace the namespace correspoinding to which the profile
447: * information of a particular site is desired.
448: *
449: * @return List of <code>Profile</code> objects
450: * NULL when the information about the site is not there or no
451: * profile information associated with the site.
452: *
453: * @see org.griphyn.cPlanner.classes.Profile
454: */
455: public List getPoolProfile(String siteID, String namespace) {
456: logMessage("List getPoolProfile(String siteID, String namespace");
457: logMessage("\tList getPoolProfile(" + siteID + "," + namespace
458: + ")");
459: List profileList = null;
460: ArrayList namespList = null;
461: //sanity checks
462: if (siteID == null || namespace == null
463: || namespace.length() < 2) {
464: return null;
465: }
466:
467: //check if the namespace asked for
468: //is a valid namespace or not
469: if (!Namespace.isNamespaceValid(namespace)) {
470: mLogger.log("Namespace " + namespace
471: + " not suppored. Ignoring",
472: LogManager.WARNING_MESSAGE_LEVEL);
473: return null;
474: }
475:
476: //get information about all the profiles
477: profileList = this .getPoolProfile(siteID);
478:
479: if (profileList == null) {
480: return profileList;
481: }
482:
483: //iterate through the list and add to the namespace list
484: Iterator it = profileList.iterator();
485: namespList = new ArrayList(3);
486: Profile poolPf = null;
487: while (it.hasNext()) {
488: poolPf = (Profile) it.next();
489: if (poolPf.getProfileNamespace()
490: .equalsIgnoreCase(namespace)) {
491: namespList.add(poolPf);
492: }
493: }
494:
495: if (namespList.isEmpty()) {
496: namespList = null;
497:
498: }
499: return namespList;
500: }
501:
502: /**
503: * This determines the working directory on remote execution pool on the
504: * basis of whether an absolute path is specified in the pegasus.dir.exec directory
505: * or a relative path.
506: *
507: * @param executionPool the pool where a job has to be executed.
508: *
509: * @return the path to the pool work dir.
510: * @throws RuntimeException in case of site not found in the site catalog.
511: */
512: public String getExecPoolWorkDir(String executionPool) {
513: return this .getExecPoolWorkDir(executionPool, null, -1);
514: }
515:
516: /**
517: * This determines the working directory on remote execution pool for a
518: * particular job. The job should have it's execution pool set.
519: *
520: * @param job <code>SubInfo</code> object for the job.
521: *
522: * @return the path to the pool work dir.
523: * @throws RuntimeException in case of site not found in the site catalog.
524: */
525: public String getExecPoolWorkDir(SubInfo job) {
526: return this .getExecPoolWorkDir(job.executionPool, job.vdsNS
527: .getStringValue(VDS.REMOTE_INITIALDIR_KEY),
528: job.jobClass);
529: }
530:
531: /**
532: * This determines the working directory on remote execution pool on the
533: * basis of whether an absolute path is specified in the pegasus.dir.exec
534: * directory or a relative path.
535: *
536: * @param siteID the name of the site where a job has to be executed.
537: * @param path the relative path that needs to be appended to the
538: * workdir from the execution pool.
539: *
540: * @return the path to the pool work dir.
541: * @throws RuntimeException in case of site not found in the site catalog.
542: */
543: public String getExecPoolWorkDir(String siteID, String path) {
544: logMessage("String getExecPoolWorkDir(String siteID, String path)");
545: logMessage("\t String getExecPoolWorkDir(" + siteID + ","
546: + path + ")");
547: return this .getExecPoolWorkDir(siteID, path, -1);
548: }
549:
550: /**
551: * This determines the working directory on remote execution pool on the
552: * basis of whether an absolute path is specified in the pegasus.dir.exec directory
553: * or a relative path. If the job class happens to be a create directory job
554: * it does not append the name of the random directory since the job is
555: * trying to create that random directory.
556: *
557: * @param siteID the name of the site where the job has to be executed.
558: * @param path the relative path that needs to be appended to the
559: * workdir from the execution pool.
560: * @param jobClass the class of the job.
561: *
562: * @return the path to the pool work dir.
563: * @throws RuntimeException in case of site not found in the site catalog.
564: */
565: public String getExecPoolWorkDir(String siteID, String path,
566: int jobClass) {
567: SiteInfo execPool = this .getPoolEntry(siteID, "vanilla");
568: if (execPool == null) {
569: throw new RuntimeException("Entry for " + siteID
570: + " does not exist in the Site Catalog");
571: }
572: String execPoolDir = mWorkDir;
573:
574: if (jobClass == SubInfo.CREATE_DIR_JOB) {
575: //the create dir jobs always run in the
576: //workdir specified in the site catalog
577: return execPool.getExecMountPoint();
578: }
579:
580: if (mWorkDir.length() == 0 || mWorkDir.charAt(0) != '/') {
581: //means you have to append the
582: //value specfied by pegasus.dir.exec
583: File f = new File(execPool.getExecMountPoint(), mWorkDir);
584: execPoolDir = f.getAbsolutePath();
585: }
586:
587: //get the random directory name
588: String randDir = mUserOpts.getRandomDirName();
589:
590: if (randDir != null) {
591: //append the random dir name to the
592: //work dir constructed till now
593: File f = new File(execPoolDir, randDir);
594: execPoolDir = f.getAbsolutePath();
595: }
596:
597: //path takes precedence over random dir
598: if (path != null) {
599: //well i can do nesting conditional return but wont
600: return (path.length() == 0 || path.charAt(0) != '/') ?
601: //append the path
602: new File(execPoolDir, path).getAbsolutePath()
603: : //else absolute path specified
604: path;
605: }
606:
607: return execPoolDir;
608: }
609:
610: /**
611: * Returns the url prefix of a gridftp server on the pool.
612: * gsiftp://dataserver.phys.uwm.edu/~/griphyn_test/ligodemo_output/
613: * gives a URL prefix of gsiftp://dataserver.phys.uwm.edu
614: *
615: * @param poolName the name of the pool.
616: *
617: * @return String corresponding to the url prefix if the pool is found.
618: * null if pool entry is not found.
619: */
620: public String getURLPrefix(String poolName) {
621: SiteInfo pool = getPoolEntry(poolName, "vanilla");
622: String urlPrefix = pool.getURLPrefix(true);
623:
624: if (urlPrefix == null || urlPrefix.trim().length() == 0) {
625: throw new RuntimeException(
626: " URL prefix not specified for site " + poolName);
627: }
628:
629: return urlPrefix;
630:
631: }
632:
633: /**
634: * Return the storage mount point for a particular pool.
635: *
636: * @param site SiteInfo object of the site for which you want the
637: * storage-mount-point.
638: *
639: * @return String corresponding to the mount point if the pool is found.
640: * null if pool entry is not found.
641: */
642: public String getSeMountPoint(SiteInfo site) {
643: logMessage("String getSeMountPoint(SiteInfo site)");
644: String mount_point = mStorageDir;
645: GridFTPServer server = null;
646: if (mStorageDir.length() == 0 || mStorageDir.charAt(0) != '/') {
647: server = site.selectGridFTP(false);
648: mount_point = server.getInfo(GridFTPServer.STORAGE_DIR);
649:
650: //removing the trailing slash if there
651: int length = mount_point.length();
652: if (length > 1 && mount_point.charAt(length - 1) == '/') {
653: mount_point = mount_point.substring(0, length - 1);
654: }
655:
656: //append the Storage Dir
657: File f = new File(mount_point, mStorageDir);
658: mount_point = f.getAbsolutePath();
659:
660: }
661:
662: //check if we need to replicate the submit directory
663: //structure on the storage directory
664: if (mDeepStorageStructure) {
665: String leaf = (this .mUserOpts.getOptions()
666: .partOfDeferredRun()) ?
667: //if a deferred run then pick up the relative random directory
668: this .mUserOpts.getOptions().getRandomDir()
669: :
670: //for a normal run add the relative submit directory
671: this .mUserOpts.getOptions()
672: .getRelativeSubmitDirectory();
673: File f = new File(mount_point, leaf);
674: mount_point = f.getAbsolutePath();
675: }
676:
677: return mount_point;
678:
679: }
680:
681: /**
682: * Gets the pool object to be used for the transfer universe. If we
683: * do not get that then defaults back to globus universe for the same pool.
684: *
685: * @param poolName the name of the pool
686: * @return Pool
687: */
688: public SiteInfo getTXPoolEntry(String poolName) {
689: SiteInfo p = this .getPoolEntry(poolName,
690: Engine.TRANSFER_UNIVERSE);
691: return p;
692:
693: }
694:
695: /**
696: * Logs the message to a logging stream. Currently does not log to any stream.
697: *
698: * @param msg the message to be logged.
699: */
700: protected void logMessage(String msg) {
701: //mLogger.logMessage("[Shishir] Site Catalog : " + msg);
702: }
703: }
|