001: /*
002: * This file or a portion of this file is licensed under the terms of
003: * the Globus Toolkit Public License, found in file ../GTPL, or at
004: * http://www.globus.org/toolkit/download/license.html. This notice must
005: * appear in redistributions of this file, with or without modification.
006: *
007: * Redistributions of this Software, with or without modification, must
008: * reproduce the GTPL in: (1) the Software, or (2) the Documentation or
009: * some other similar material which is provided with the Software (if
010: * any).
011: *
012: * Copyright 1999-2004 University of Chicago and The University of
013: * Southern California. All rights reserved.
014: */
015: package org.griphyn.vdl.euryale;
016:
017: import org.griphyn.common.util.Separator;
018: import org.griphyn.common.util.Currently;
019: import java.io.*;
020: import java.util.*;
021: import java.text.*;
022:
023: /**
024: * This file factory generates a stream of submit files in a dynamically
025: * determinable directory structure. By default, a 2-level subdirectory
026: * structure is assumed, which should be able to accomodate about 500k
027: * files.
028: *
029: * <pre>
030: * mult=16, offset=30, fpd=254: nr=15 => l=1
031: * mult=16, offset=30, fpd=254: nr=4047 => l=2
032: * mult=16, offset=30, fpd=254: nr=1028222 => l=3
033: * </pre>
034: *
035: * With the given multiplicator, offset and files per directory, nr is
036: * smallest number of jobs at which a level change to l occurs.
037: *
038: * @author Kavitha Ranganathan
039: * @author Jens-S. Vöckler
040: * @author Yong Zhao
041: * @version $Revision: 289 $
042: *
043: * @see DAX2DAG
044: */
045: public class HashedFileFactory extends FlatFileFactory {
046: /**
047: * Determines dynamically the number of directory levels required
048: * to accomodate a certain number of files.
049: *
050: * <pre>
051: * levels = |log ( tf * m + offset )|
052: * fpd
053: * </pre>
054: *
055: * @param totalFiles is the total number of files estimated to generate
056: * @param multiplicator is a corrective factor to account for files
057: * that are created by the run-time system on the fly. For Euryale and
058: * Pegasus it is safe to assume a multiplicator of at least 8.
059: * @param filesPerDirectory is the optimal maximum number of directory
060: * entries in any directory. The value of 254 for Linux ext2, and thus
061: * ext3, is a safe bet.
062: * @param offset is the number of (expected) files in the top level.
063: * @return the number of directory levels necessary to accomodate the
064: * given number of files.
065: */
066: public static int calculateLevels(int totalFiles,
067: int multiplicator, int filesPerDirectory, int offset) {
068: // total files to accomodate, including ones cropping up later
069: long total = totalFiles * multiplicator + offset;
070:
071: // "count" the levels
072: // return (int) Math.floor( Math.log(total) / Math.log(filesPerDirectory) );
073: int levels = 0;
074: while (total > filesPerDirectory) {
075: ++levels;
076: total /= filesPerDirectory;
077: }
078: return levels;
079: }
080:
081: /**
082: * Counts the number of times the structured virtual constructor was
083: * called.
084: * @see #getCount()
085: */
086: protected int m_count;
087:
088: /**
089: * Contains the total number of directory levels. Defaults to a
090: * reasonable level for hashing.
091: */
092: protected int m_levels = 2;
093:
094: /**
095: * Number of entries per level. The number 254 is optimized for the
096: * Linux VFS ext2, and consequently ext3, which works fastest, if the
097: * number of entries per directory, including dot and dotdot, don't
098: * exceed 256.
099: */
100: protected int m_filesPerDirectory = 254;
101:
102: /**
103: * Multiplicative factor to estimate the number of result leaf
104: * filenames for each virtual constructor invocation. We assume that
105: * Euryale produces ~12 files per submit file. It is better to err
106: * on the larger side than makeing the multiplicator too small.
107: */
108: protected int m_multiplicator = 16;
109:
110: /**
111: * Offset of files expected to reside at the top level directory.
112: * This is counted in addition to the directories being created.
113: */
114: protected int m_offset = 30;
115:
116: /**
117: * Helping structure to avoid repeated memory requests. Stores the
118: * directory number for each level.
119: * @see #createFile( String )
120: */
121: protected int mh_level[];
122:
123: /**
124: * Helping structure to avoid repeated memory requests. Stores the
125: * digits necessary to create one level's directory name.
126: * @see #format( int )
127: */
128: protected StringBuffer mh_buffer;
129:
130: /**
131: * Helping structure to avoid repeated memory requests. Stores the
132: * number of digits for hexadecimal formatting.
133: * @see #createFile( String )
134: */
135: protected int mh_digits;
136:
137: /**
138: * Resets the helper structures after changing layout parameters. You
139: * will also need to call this function after you invoked the virtual
140: * constructors, but want to change parameter pertaining to the
141: * directory structure. The structured file count will also be reset!
142: */
143: public void reset() {
144: super .reset();
145: m_count = 0;
146: mh_level = new int[m_levels];
147: mh_digits = (int) Math.ceil(Math.log(m_filesPerDirectory)
148: / Math.log(16));
149: mh_buffer = new StringBuffer(mh_digits);
150: }
151:
152: /**
153: * Constructor: Creates the base directory and employs sanity checks.
154: * @param baseDirectory is the place where the other dirs are created,
155: * and where the DAG file resides.
156: * @throws IOException if the location is not a writable directory,
157: * or cannot be created as such.
158: */
159: public HashedFileFactory(File baseDirectory) throws IOException {
160: super (baseDirectory);
161: reset();
162: }
163:
164: /**
165: * Constructor: Creates the directory and employs sanity checks.
166: * @param baseDirectory is the place where the other dirs are created,
167: * and where the DAG file resides.
168: * @throws IOException if the location is not a writable directory,
169: * or cannot be created as such.
170: */
171: public HashedFileFactory(String baseDirectory) throws IOException {
172: super (baseDirectory);
173: reset();
174: }
175:
176: /**
177: * Constructor: Creates the base directory and employs sanity checks.
178: * @param baseDirectory is the place where the other dirs are created,
179: * and where the DAG file resides.
180: * @param totalFiles is the number of files to support, and the number
181: * of times, the virtual constructor is expected to be called.
182: * @throws IOException if the location is not a writable directory,
183: * or cannot be created as such.
184: */
185: public HashedFileFactory(File baseDirectory, int totalFiles)
186: throws IOException {
187: super (baseDirectory);
188: m_levels = calculateLevels(totalFiles, m_multiplicator,
189: m_filesPerDirectory, m_offset);
190: reset();
191: }
192:
193: /**
194: * Constructor: Creates the directory and employs sanity checks.
195: * @param baseDirectory is the place where the other dirs are created,
196: * and where the DAG file resides.
197: * @param totalFiles is the number of files to support, and the number
198: * of times, the virtual constructor is expected to be called.
199: * @throws IOException if the location is not a writable directory,
200: * or cannot be created as such.
201: */
202: public HashedFileFactory(String baseDirectory, int totalFiles)
203: throws IOException {
204: super (baseDirectory);
205: m_levels = calculateLevels(totalFiles, m_multiplicator,
206: m_filesPerDirectory, m_offset);
207: reset();
208: }
209:
210: /**
211: * Converts the given integer into hexadecimal notation, using
212: * the given number of digits, prefixing with zeros as necessary.
213: *
214: * @param number is the number to format.
215: * @return a string of appropriate length, filled with leading zeros,
216: * representing the number hexadecimally.
217: */
218: public String format(int number) {
219: mh_buffer.delete(0, mh_digits);
220: mh_buffer.append(Integer.toHexString(number).toUpperCase());
221: while (mh_buffer.length() < mh_digits)
222: mh_buffer.insert(0, '0');
223: return mh_buffer.toString();
224: }
225:
226: /**
227: * Creates the next file with the given basename. This is the factory
228: * standard virtual constructor. Once invoked, the directory structure
229: * can not be changed any more.
230: *
231: * @param basename is the filename to create. Don't specify dirs here.
232: * @return a File structure which points to the new file. Nothing is
233: * created through this method, and creation may still fail.
234: * @see #getCount()
235: */
236: public File createFile(String basename) throws IOException {
237: // calculate the directory which this goes into
238: ////int estimate = m_count++ * m_multiplicator;
239: int estimate = (m_count++ * m_multiplicator) + m_offset;
240: for (int i = m_levels - 1; i >= 0; --i) {
241: estimate /= m_filesPerDirectory;
242: mh_level[i] = estimate % m_filesPerDirectory;
243: }
244: if (estimate > m_filesPerDirectory)
245: throw new RuntimeException(
246: "ERROR! Wrap-around of generator.");
247:
248: //create the base directory if required
249: File d = createDirectory();
250:
251: // return position in new (or old) directory
252: return new File(d, basename);
253: }
254:
255: /**
256: * Creates a directory for the hashed file directory structure on the
257: * submit host.
258: *
259: *
260: * @return the File structure to the created directory
261: *
262: * @throws IOException the exception.
263: */
264: protected File createDirectory() throws IOException {
265: // create directory, as necessary
266: File d = getBaseDirectory();
267: for (int i = 0; i < m_levels; ++i) {
268: d = new File(d, format(mh_level[i]));
269: if (d.exists()) {
270: if (!d.isDirectory()) {
271: throw new IOException(d.getPath()
272: + " is not a directory");
273: }
274: } else {
275: if (!d.mkdir()) {
276: throw new IOException("unable to create directory "
277: + d.getPath());
278: }
279: }
280: }
281: return d;
282: }
283:
284: /**
285: * Returns the number of times the regular virtual constructor for
286: * structured entries was called.
287: * @return the count for createFile invocations.
288: * @see #createFile( String )
289: */
290: public int getCount() {
291: return m_count;
292: }
293:
294: /**
295: * Accessor: Obtains the total number of directory levels.
296: * @return the total number of directory levels chosen.
297: */
298: public int getLevels() {
299: return m_levels;
300: }
301:
302: /**
303: * Accessor: Sets the number of directory levels. Note that this
304: * modificator can only be called before the virtual constructor
305: * is called the first time.
306: *
307: * @param levels is the number of directory levels to use
308: * @throws VTorInUseException if the virtual constructor is already in use.
309: * @throws IllegalArgumentException if the argument is less than zero.
310: * @see #getLevels()
311: */
312: public void setLevels(int levels) {
313: if (m_count != 0)
314: throw new VTorInUseException();
315: if (levels < 0)
316: throw new IllegalArgumentException();
317: m_levels = levels;
318: reset();
319: }
320:
321: /**
322: * Accessor: Sets the number of directory levels. Note that this
323: * modificator can only be called before the virtual constructor
324: * is called the first time. It takes as argument the total number
325: * of expected files instead of the level.
326: *
327: * @param totalFiles is the total number of files to accomodate.
328: * @throws VTorInUseException if the virtual constructor is already in use.
329: * @throws IllegalArgumentException if the argument is less than zero.
330: * @see #getLevels()
331: */
332: public void setLevelsFromTotals(int totalFiles) {
333: if (m_count != 0)
334: throw new VTorInUseException();
335: if (totalFiles < 0)
336: throw new IllegalArgumentException();
337: m_levels = calculateLevels(totalFiles, m_multiplicator,
338: m_filesPerDirectory, m_offset);
339: reset();
340: }
341:
342: /**
343: * Accessor: Obtains the number of entries per directory.
344: * @return the chosen number of entries per directory excluding the
345: * dot and dotdot files.
346: */
347: public int getFilesPerDirectory() {
348: return m_filesPerDirectory;
349: }
350:
351: /**
352: * Accessor: Sets the optimal maximum number of files per directory
353: * excluding dot and dotdot. For a Linux ext2 and thus ext3 system,
354: * the optimal maximum number is 254.
355: *
356: * @param entries is the number of optimal maximum entries per dir.
357: * @throws VTorInUseException if the virtual constructor is already in use.
358: * @throws IllegalArgumentException if the argument is less than one.
359: * @see #getFilesPerDirectory()
360: */
361: public void setFilesPerDirectory(int entries) {
362: if (m_count != 0)
363: throw new VTorInUseException();
364: if (entries <= 0)
365: throw new IllegalArgumentException();
366: m_filesPerDirectory = entries;
367: reset();
368: }
369:
370: /**
371: * Accessor: Obtains the multiplicative factor for an estimation
372: * of total files from calls to the virtual constructor.
373: * @return the multiplicator.
374: * @see #setMultiplicator(int)
375: */
376: public int getMultiplicator() {
377: return m_multiplicator;
378: }
379:
380: /**
381: * Accessor: Sets the multiplicative factor to account for files which
382: * may be created without calling the virtual constructor.
383: *
384: * @param multiplicator is the new multiplicator.
385: * @throws VTorInUseException if the virtual constructor is already in use.
386: * @throws IllegalArgumentException if the argument is less than one.
387: * @see #getMultiplicator()
388: */
389: public void setMultiplicator(int multiplicator) {
390: if (m_count != 0)
391: throw new VTorInUseException();
392: if (multiplicator < 1)
393: throw new IllegalArgumentException();
394: m_multiplicator = multiplicator;
395: reset();
396: }
397:
398: /**
399: * Accessor: Obtains the offset for an estimation of total files from
400: * calls to the virtual constructor.
401: * @return the offset
402: * @see #setOffset(int)
403: */
404: public int getOffset() {
405: return m_offset;
406: }
407:
408: /**
409: * Accessor: Sets the offset for files which may be created without
410: * calling the virtual constructor.
411: *
412: * @param offset is the new offset
413: * @throws VTorInUseException if the virtual constructor is already in use.
414: * @throws IllegalArgumentException if the argument is less than zero.
415: * @see #getOffset()
416: */
417: public void setOffset(int offset) {
418: if (m_count != 0)
419: throw new VTorInUseException();
420: if (offset < 0)
421: throw new IllegalArgumentException();
422: m_offset = offset;
423: reset();
424: }
425:
426: /**
427: * test function
428: */
429: public static void main(String arg[]) throws Exception {
430: if (arg.length == 0) {
431: // no arguments, spit out at which point levels change
432: HashedFileFactory def = new HashedFileFactory("/tmp");
433: int level = 0;
434: for (int i = 1; i < 4; ++i) {
435: def.reset();
436: int nr = 1;
437: for (int n = i; n > 0; n--)
438: nr *= def.getFilesPerDirectory();
439: nr -= def.getOffset();
440: nr /= def.getMultiplicator();
441: for (int j = -2; j < Integer.MAX_VALUE; ++j) {
442: int n = nr + j;
443: def.reset();
444: def.setLevelsFromTotals(n);
445: if (level < def.getLevels()) {
446: ++level;
447: System.out.println("mult="
448: + def.getMultiplicator() + ", offset="
449: + def.getOffset() + ", fpd="
450: + def.getFilesPerDirectory() + ": nr="
451: + n + " => l=" + level);
452: break;
453: }
454: }
455: }
456:
457: } else {
458: // arguments, assume numeric strings
459: for (int i = 0; i < arg.length; ++i) {
460: int nr = Integer.parseInt(arg[i]);
461: HashedFileFactory hff = new HashedFileFactory("/tmp");
462: hff.setLevelsFromTotals(nr);
463: System.out.println();
464: System.out.println("filesPerDirectory = "
465: + hff.getFilesPerDirectory());
466: System.out.println("multiplicator = "
467: + hff.getMultiplicator());
468: System.out.println("offset = " + hff.getOffset());
469: System.out.println("totalFiles = " + nr);
470: System.out.println("levels = " + hff.getLevels());
471:
472: File f = hff.createFile("ID000001");
473: System.out.println("example = \"" + f.getAbsolutePath()
474: + "\"");
475: }
476: System.out.println();
477: }
478: }
479: }
|