001: /*
002: * This file or a portion of this file is licensed under the terms of
003: * the Globus Toolkit Public License, found in file GTPL, or at
004: * http://www.globus.org/toolkit/download/license.html. This notice must
005: * appear in redistributions of this file, with or without modification.
006: *
007: * Redistributions of this Software, with or without modification, must
008: * reproduce the GTPL in: (1) the Software, or (2) the Documentation or
009: * some other similar material which is provided with the Software (if
010: * any).
011: *
012: * Copyright 1999-2004 University of Chicago and The University of
013: * Southern California. All rights reserved.
014: */
015:
016: package org.griphyn.cPlanner.visualize.spaceusage;
017:
018: import org.griphyn.cPlanner.common.LogManager;
019:
020: import org.griphyn.common.util.Currently;
021:
022: import org.griphyn.cPlanner.visualize.Callback;
023:
024: import org.griphyn.vdl.invocation.StatInfo;
025:
026: import java.util.List;
027: import java.util.Iterator;
028: import java.util.StringTokenizer;
029:
030: /**
031: * Implements callback interface to calculate space usage.
032: *
033: * @author not attributable
034: * @version 1.0
035: */
036:
037: public class SpaceUsageCallback implements Callback {
038:
039: /**
040: * The marker for the PREJOB. The stdout corresponding to the
041: * PREJOB is enclosed within this marker.
042: */
043: public static final String PREJOB_MARKER = "@@@PREJOB@@@";
044:
045: /**
046: * The marker for the MAINJOB. The stdout corresponding to the
047: * MAINJOB is enclosed within this marker.
048: */
049: public static final String MAINJOB_MARKER = "@@@MAINJOB@@@";
050:
051: /**
052: * The marker for the POSTJOB. The stdout corresponding to the
053: * POSTJOB is enclosed within this marker.
054: */
055: public static final String POSTJOB_MARKER = "@@@POSTJOB@@@";
056:
057: /**
058: * The logical site where the job was run.
059: */
060: protected String mSite;
061:
062: /**
063: * The SpaceUsage object created during the callback construction.
064: */
065: protected SpaceUsage mSpaceStore;
066:
067: /**
068: * The main job whose record is being parsed.
069: */
070: protected String mMainJob;
071:
072: /**
073: * The handle to the logger.
074: */
075: protected LogManager mLogger;
076:
077: /**
078: * Boolean indicating whether to use stat data or not for computing directory
079: * sizes.
080: */
081: protected boolean mUseStatInfo;
082:
083: /**
084: * Stores in bytes the size of all the output files for a job.
085: */
086: protected long mJobOutSize;
087:
088: /**
089: * Stores in bytes the size of all the input files for a job.
090: */
091: protected long mJobInSize;
092:
093: /**
094: * Stores all the space readings for the current invocation record.
095: */
096: protected JobSpace mJobSpace;
097:
098: /**
099: * The default constructor.
100: */
101: public SpaceUsageCallback() {
102: mSpaceStore = new SpaceUsage();
103: mLogger = LogManager.getInstance();
104: mUseStatInfo = false;
105: mJobOutSize = 0;
106: mJobInSize = 0;
107: }
108:
109: /**
110: * Initializes the callback.
111: *
112: * @param directory the directory where all the files reside.
113: * @param useStatInfo boolean indicating whether to use stat info or not.
114: */
115: public void initialize(String directory, boolean useStatInfo) {
116: mUseStatInfo = useStatInfo;
117: }
118:
119: /**
120: * Callback for the starting of an invocation record.
121: *
122: * @param job the job/file being parsed.
123: * @param resource the site id where the job was executed.
124: */
125: public void cbInvocationStart(String job, String resource) {
126: mMainJob = job;
127: mSite = resource;
128:
129: mJobOutSize = 0;
130: mJobSpace = new JobSpace(job);
131: }
132:
133: public void cbStdIN(List jobs, String data) {
134:
135: }
136:
137: public void cbStdOut(List jobs, String data) {
138: this .parseData(data);
139: }
140:
141: public void cbStdERR(List jobs, String data) {
142:
143: }
144:
145: /**
146: * Callback function for when stat information for an input file is
147: * encountered. Empty for time being.
148: *
149: * @param filename the name of the file.
150: * @param info the <code>StatInfo</code> about the file.
151: *
152: */
153: public void cbInputFile(String filename, StatInfo info) {
154: if (mUseStatInfo) {
155: //sanity check
156: if (info == null) {
157: //log a warning
158: mLogger.log("No stat info for input file " + filename,
159: LogManager.WARNING_MESSAGE_LEVEL);
160: return;
161: }
162:
163: //increment the size to the already stored size.
164: mJobInSize += info.getSize();
165: mLogger.log("\tInput file is " + filename + " of size "
166: + info.getSize(), LogManager.DEBUG_MESSAGE_LEVEL);
167: }
168:
169: }
170:
171: /**
172: * Callback function for when stat information for an output file is
173: * encountered. The size of the file is computed and stored.
174: *
175: * @param filename the name of the file.
176: * @param info the <code>StatInfo</code> about the file.
177: *
178: */
179: public void cbOutputFile(String filename, StatInfo info) {
180: if (mUseStatInfo) {
181: //sanity check
182: if (info == null) {
183: //log a warning
184: mLogger.log("No stat info for output file " + filename,
185: LogManager.WARNING_MESSAGE_LEVEL);
186: return;
187: }
188:
189: //increment the size to the already stored size.
190: mJobOutSize += info.getSize();
191: mLogger.log("\tOutput file is " + filename + " of size "
192: + info.getSize(), LogManager.DEBUG_MESSAGE_LEVEL);
193: }
194: }
195:
196: /**
197: * Callback signalling that an invocation record has been parsed.
198: * Stores the total compute size, somewhere in the space structure
199: * for the jobs.
200: *
201: *
202: */
203: public void cbInvocationEnd() {
204: //if we are using statinfo
205: if (mUseStatInfo) {
206: //we get just the post job record, and put the mJobOut size in
207: //there
208: Space s = mJobSpace
209: .getSpaceReading(JobSpace.GRIDSTART_POSTJOB_EVENT_TYPE);
210:
211: if (s == null) {
212: mLogger.log("No space reading for job " + mMainJob,
213: LogManager.WARNING_MESSAGE_LEVEL);
214: return;
215: }
216:
217: if (cleanupJob(mMainJob)) {
218: float size = (float) mJobInSize;
219: size = size / 1024;
220: //for cleanup jobs we take input size
221: s.setCleanupFlag(true);
222: s.setSize(size, 'K');
223: mLogger.log("For job " + mMainJob
224: + " total input file size in K " + size,
225: LogManager.DEBUG_MESSAGE_LEVEL);
226: mSpaceStore.addRecord(mSite, s);
227: //we add a duplicate space reading that gets populated
228: //later with the reading of directory after the cleanup job
229: Space s1 = (Space) s.clone();
230: s1.setCleanupFlag(false);
231: s1.setSize("0");
232: mSpaceStore.addRecord(mSite, s1);
233: } else {
234: //for all other jobs we take output sizes
235: float size = (float) mJobOutSize;
236: size = size / 1024;
237: s.setSize(size, 'K');
238: mSpaceStore.addRecord(mSite, s);
239: mLogger.log("For job " + mMainJob
240: + " total output file size in K " + size,
241: LogManager.DEBUG_MESSAGE_LEVEL);
242: }
243:
244: } else {
245: //we put all the valid records into the space store
246: for (Iterator it = mJobSpace.spaceReadingsIterator(); it
247: .hasNext();) {
248: Object obj = it.next();
249: if (obj == null) {
250: //go to next reading
251: continue;
252: }
253: mSpaceStore.addRecord(mSite, (Space) obj);
254: }
255: }
256:
257: //reset per site data
258: mSite = "";
259: mMainJob = "";
260: mJobOutSize = 0;
261: mJobInSize = 0;
262: mJobSpace = null;
263: }
264:
265: /**
266: * Returns the SpaceUsage store built.
267: *
268: * @return SpaceUsage
269: */
270: public Object getConstructedObject() {
271: return mSpaceStore;
272: }
273:
274: /**
275: * Parses the data in the data section.
276: *
277: * @param data String
278: */
279: private void parseData(String data) {
280: //sanity check
281: if (data == null) {
282: return;
283: }
284:
285: //System.out.println( "DATA is " + data );
286:
287: //parse through
288: String token;
289: String header = null;
290: StringBuffer content = new StringBuffer();
291:
292: boolean start = false;
293: for (StringTokenizer st = new StringTokenizer(data); st
294: .hasMoreTokens();) {
295: token = st.nextToken();
296: if (validHeader(token)) {
297:
298: //if start is true we have one job data
299: if (start) {
300:
301: //token needs to match the previous header
302: if (token.equals(header)) {
303:
304: mLogger.log(
305: "Content before parsing " + content,
306: LogManager.DEBUG_MESSAGE_LEVEL);
307:
308: Space s = parseContent(header, content
309: .toString());
310:
311: //if the content was set for the MAINJOB
312: //set the marker to be true
313: if (token.equals(this .MAINJOB_MARKER)) {
314: s.setCleanupFlag(true);
315: }
316:
317: // mSpaceStore.addRecord( mSite, s ) ;
318: mJobSpace.addSpaceReading(s, this
319: .getEventTypeForHeader(header));
320:
321: mLogger.log("Content after parsing is " + s,
322: LogManager.DEBUG_MESSAGE_LEVEL);
323: start = !start;
324: content = new StringBuffer();
325: } else {
326: /* error */
327: throw new RuntimeException(
328: "Incorrect placement of markers in stdout ("
329: + header + " , " + token + " )");
330:
331: }
332:
333: continue;
334:
335: }
336:
337: //token is a valid header
338: header = token;
339:
340: //header is matched.
341: start = !start;
342: } else if (start) {
343: //we have already matched a header.
344: content.append(token).append(" ");
345: }
346: }
347: }
348:
349: /**
350: * Callback signalling that we are done with the parsing of the files.
351: */
352: public void done() {
353: mSpaceStore.sort();
354:
355: //we have all the records.
356: //need to do some mischief in case of using statinfo
357: if (mUseStatInfo) {
358: //go thru space usage for each site.
359: for (Iterator it = mSpaceStore.siteIterator(); it.hasNext();) {
360: String site = (String) it.next();
361:
362: float dir_size = 0;
363:
364: //go through space usage for a particular site
365: for (Iterator sizeIT = mSpaceStore.getSizes(site)
366: .iterator(); sizeIT.hasNext();) {
367: Space s = (Space) sizeIT.next();
368:
369: if (s.getCleanupFlag()) {
370: //subtract directory size
371: dir_size -= s.getSize('K');
372:
373: } else {
374: dir_size += s.getSize('K');
375:
376: //set the size back
377: s.setSize(dir_size, 'K');
378: }
379: mLogger.log("Directory size after job "
380: + s.getAssociatedJob() + " in K is "
381: + dir_size, LogManager.DEBUG_MESSAGE_LEVEL);
382:
383: }
384:
385: }
386:
387: }
388: }
389:
390: /**
391: * Parses the data in the data section.
392: *
393: * @param data String
394: */
395: /*
396: private void parseData ( String data ) {
397: int length = ( data == null ) ? 0 : data.length();
398:
399: System.out.println( "Data is " + data );
400:
401: String header = PREJOB_MARKER;
402: StringBuffer content = new StringBuffer();
403: boolean start = false; boolean end = true;
404: for ( int i = 0; i < length; i++){
405: char c = data.charAt( i );
406:
407: if ( c == '@' ){
408: //see if look ahead matches
409: if ( i + header.length() < length &&
410: ( data.substring( i, i + header.length()).equals( header ) )
411: ){
412:
413: //if start is true we have one job data
414: if ( start ) {
415:
416: //we are capturing date for post jobs only
417: if ( header.equalsIgnoreCase( POSTJOB_MARKER ) ){
418: System.out.println("Content before parsing " +
419: content);
420: Space s = parseContent(content.toString());
421: mSpaceStore.addRecord(mSite, s);
422: System.out.println("CONTENT IS " + s);
423: }
424: content = new StringBuffer();
425: start = false; end = true;
426:
427: //skip to the character after the header
428: i = i + header.length() - 1;
429: header = POSTJOB_MARKER;
430: continue;
431: }
432:
433: //header is matched.
434: start = !start;
435: end = !end;
436:
437: //skip to the character after the header
438: i = i + header.length() - 1;
439: }
440: else if ( start ) { content.append( c ); }
441: }
442: else if ( start ){
443: //add to content
444: content.append( c );
445: }
446: }
447: }
448: */
449:
450: /**
451: * Returns a boolean indicating whether the token passed matches
452: * a header or not. In the specific case of using statinfo, for calculating
453: * directory sizes, we only mainjob and postjob markers are valid.
454: *
455: * @param token the token to be matched.
456: * @param state the current header being processed
457: *
458: * @return boolean
459: */
460: protected boolean validHeader(String token) {
461: return (this .mUseStatInfo) ?
462: //only two headers are valid.
463: (token.equals(this .MAINJOB_MARKER) || token
464: .equals(this .POSTJOB_MARKER))
465: :
466: //all three headers are valid.
467: (token.equals(this .MAINJOB_MARKER)
468: || token.equals(this .PREJOB_MARKER) || token
469: .equals(this .POSTJOB_MARKER));
470:
471: }
472:
473: /**
474: * Returns boolean indicating whether the job is a cleanup job or not.
475: * Does it on the basis of the name of the job.
476: *
477: * @param name the name of the job.
478: *
479: * @return boolean
480: */
481: public boolean cleanupJob(String name) {
482: return name.startsWith("cln_");
483: }
484:
485: /**
486: * Parses the content and stores it in a Space object.
487: *
488: * @param header the header from which the content was collected.
489: * @param content the Content.
490: *
491: * @return Space
492: */
493: protected Space parseContent(String header, String content) {
494: String date = null;
495: String size = null;
496: Space s;
497: for (StringTokenizer st = new StringTokenizer(content); st
498: .hasMoreTokens();) {
499: if (date == null) {
500: date = st.nextToken();
501: } else {
502: size = st.nextToken();
503: break;
504: }
505: }
506: s = new Space(Currently.parse(date), size);
507: s.setAssociatedJob(mMainJob);
508: return s;
509: }
510:
511: /**
512: * Returns the event type matching a header.
513: *
514: * @param header
515: *
516: * @return the corresponding event type
517: */
518: protected int getEventTypeForHeader(String marker) {
519: int event = -1;
520: if (marker.equals(this.PREJOB_MARKER)) {
521: event = JobSpace.GRIDSTART_PREJOB_EVENT_TYPE;
522: }
523: if (marker.equals(this.MAINJOB_MARKER)) {
524: event = JobSpace.GRIDSTART_MAINJOB_EVENT_TYPE;
525: }
526: if (marker.equals(this.POSTJOB_MARKER)) {
527: event = JobSpace.GRIDSTART_POSTJOB_EVENT_TYPE;
528: }
529: return event;
530: }
531:
532: }
|