001: // kelondroDyn.java
002: // -----------------------
003: // part of The Kelondro Database
004: // (C) by Michael Peter Christen; mc@anomic.de
005: // first published on http://www.anomic.de
006: // Frankfurt, Germany, 2004
007: // last major change: 09.02.2004
008: //
009: // This program is free software; you can redistribute it and/or modify
010: // it under the terms of the GNU General Public License as published by
011: // the Free Software Foundation; either version 2 of the License, or
012: // (at your option) any later version.
013: //
014: // This program is distributed in the hope that it will be useful,
015: // but WITHOUT ANY WARRANTY; without even the implied warranty of
016: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017: // GNU General Public License for more details.
018: //
019: // You should have received a copy of the GNU General Public License
020: // along with this program; if not, write to the Free Software
021: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022: //
023: // Using this software in any meaning (reading, learning, copying, compiling,
024: // running) means that you agree that the Author(s) is (are) not responsible
025: // for cost, loss of data or any harm that may be caused directly or indirectly
026: // by usage of this softare or this documentation. The usage of this software
027: // is on your own risk. The installation and usage (starting/running) of this
028: // software may allow other people or application to access your computer and
029: // any attached devices and is highly dependent on the configuration of the
030: // software which must be done by the user of the software; the author(s) is
031: // (are) also not responsible for proper configuration and usage of the
032: // software, even if provoked by documentation provided together with
033: // the software.
034: //
035: // Any changes to this file according to the GPL as documented in the file
036: // gpl.txt aside this file in the shipment you received can be done to the
037: // lines that follows this copyright notice here, but changes must not be
038: // done inside the copyright notive above. A re-distribution must contain
039: // the intact and unchanged copyright notice.
040: // Contributions and changes to the program code must be marked as such.
041:
042: /*
043: This class extends the kelondroTree and adds dynamic data handling
044: A dynamic content is created, by using several tree nodes and
045: combining them over a set of associated keys.
046: Example: a byte[] of length 1000 shall be stored in a kelondroTree
047: with node size 256. The key for the entry is 'entry'.
048: Then kelondroDyn stores the first part of four into the entry
049: 'entry00', the second into 'entry01', and so on.
050:
051: */
052:
053: package de.anomic.kelondro;
054:
055: import java.io.File;
056: import java.io.FileInputStream;
057: import java.io.FileOutputStream;
058: import java.io.IOException;
059: import java.util.Iterator;
060:
061: public class kelondroDyn {
062:
063: private static final int counterlen = 8;
064: private static final int EcoFSBufferSize = 20;
065:
066: protected int keylen;
067: private int reclen;
068: //private int segmentCount;
069: private char fillChar;
070: private kelondroIndex index;
071: private kelondroObjectBuffer buffer;
072: private kelondroRow rowdef;
073:
074: public kelondroDyn(File file, boolean useNodeCache,
075: boolean useObjectCache, long preloadTime, int key,
076: int nodesize, char fillChar, kelondroByteOrder objectOrder,
077: boolean usetree, boolean writebuffer, boolean resetOnFail) {
078: // creates or opens a dynamic tree
079: rowdef = new kelondroRow("byte[] key-" + (key + counterlen)
080: + ", byte[] node-" + nodesize, objectOrder, 0);
081: kelondroIndex fbi;
082: if (usetree) {
083: try {
084: fbi = new kelondroTree(file, useNodeCache, preloadTime,
085: rowdef, 1, 8);
086: } catch (IOException e) {
087: e.printStackTrace();
088: if (resetOnFail) {
089: file.delete();
090: try {
091: fbi = new kelondroTree(file, useNodeCache, -1,
092: rowdef, 1, 8);
093: } catch (IOException e1) {
094: e1.printStackTrace();
095: throw new kelondroException(e.getMessage());
096: }
097: } else {
098: throw new kelondroException(e.getMessage());
099: }
100: }
101:
102: } else {
103: if (file.exists()) {
104: if (file.isDirectory()) {
105: fbi = new kelondroFlexTable(file.getParentFile(),
106: file.getName(), 10000, rowdef, 0,
107: resetOnFail);
108: } else {
109: fbi = new kelondroEcoTable(file, rowdef,
110: kelondroEcoTable.tailCacheUsageAuto,
111: EcoFSBufferSize, 0);
112: }
113: } else {
114: fbi = new kelondroEcoTable(file, rowdef,
115: kelondroEcoTable.tailCacheUsageAuto,
116: EcoFSBufferSize, 0);
117: }
118: }
119: this .index = ((useObjectCache) && (!(fbi instanceof kelondroEcoTable))) ? (kelondroIndex) new kelondroCache(
120: fbi)
121: : fbi;
122: this .keylen = key;
123: this .reclen = nodesize;
124: this .fillChar = fillChar;
125: //this.segmentCount = 0;
126: //if (!(tree.fileExisted)) writeSegmentCount();
127: buffer = new kelondroObjectBuffer(file.toString());
128: }
129:
130: public static final void delete(File file, boolean usetree) {
131: if (usetree) {
132: file.delete();
133: } else {
134: if (file.isDirectory()) {
135: kelondroFlexTable.delete(file.getParentFile(), file
136: .getName());
137: } else {
138: file.delete();
139: }
140: }
141: }
142:
143: public void reset() throws IOException {
144: String name = this .index.filename();
145: this .index.reset();
146: this .buffer = new kelondroObjectBuffer(name);
147: }
148:
149: public kelondroRow row() {
150: return this .rowdef;
151: }
152:
153: public synchronized int sizeDyn() {
154: //this.segmentCount = 0;
155: //Iterator i = keys(true); while (i.hasNext()) segmentCount++;
156: //return segmentCount;
157: return index.size();
158: }
159:
160: private static String counter(int c) {
161: String s = Integer.toHexString(c);
162: while (s.length() < counterlen)
163: s = "0" + s;
164: return s;
165: }
166:
167: private byte[] dynKey(String key, int record) {
168: if (key.length() > keylen)
169: throw new RuntimeException("key len (" + key.length()
170: + ") out of limit (" + keylen + "): '" + key + "'");
171: while (key.length() < keylen)
172: key = key + fillChar;
173: key = key + counter(record);
174: return key.getBytes();
175: }
176:
177: protected String origKey(byte[] rawKey) {
178: int n = keylen - 1;
179: if (n >= rawKey.length)
180: n = rawKey.length - 1;
181: while ((n > 0) && (rawKey[n] == (byte) fillChar))
182: n--;
183: return new String(rawKey, 0, n + 1);
184: }
185:
186: public class dynKeyIterator implements
187: kelondroCloneableIterator<String> {
188: // the iterator iterates all keys
189: kelondroCloneableIterator<kelondroRow.Entry> ri;
190: String nextKey;
191:
192: public dynKeyIterator(
193: kelondroCloneableIterator<kelondroRow.Entry> iter) {
194: ri = iter;
195: nextKey = n();
196: }
197:
198: public dynKeyIterator clone(Object modifier) {
199: return new dynKeyIterator(ri.clone(modifier));
200: }
201:
202: public boolean hasNext() {
203: return nextKey != null;
204: }
205:
206: public String next() {
207: String result = nextKey;
208: nextKey = n();
209: return origKey(result.getBytes());
210: }
211:
212: public void remove() {
213: throw new UnsupportedOperationException(
214: "no remove in RawKeyIterator");
215: }
216:
217: private String n() {
218: byte[] g;
219: String k;
220: String v;
221: int c;
222: kelondroRow.Entry nt;
223: while (ri.hasNext()) {
224: nt = (kelondroRow.Entry) ri.next();
225: if (nt == null)
226: throw new kelondroException(
227: "no more elements available");
228: g = nt.getColBytes(0);
229: if (g == null)
230: return null;
231: k = new String(g, 0, keylen);
232: v = new String(g, keylen, counterlen);
233: try {
234: c = Integer.parseInt(v, 16);
235: } catch (NumberFormatException e) {
236: c = -1;
237: }
238: if (c == 0)
239: return k;
240: }
241: return null;
242: }
243: }
244:
245: public synchronized kelondroCloneableIterator<String> dynKeys(
246: boolean up, boolean rotating) throws IOException {
247: // iterates only the keys of the Nodes
248: // enumerated objects are of type String
249: dynKeyIterator i = new dynKeyIterator(index.rows(up, null));
250: if (rotating)
251: return new kelondroRotateIterator<String>(i, null, index
252: .size());
253: else
254: return i;
255: }
256:
257: public synchronized dynKeyIterator dynKeys(boolean up,
258: byte[] firstKey) throws IOException {
259: return new dynKeyIterator(index.rows(up, firstKey));
260: }
261:
262: private byte[] getValueCached(byte[] key) throws IOException {
263:
264: // read from buffer
265: byte[] buffered = (byte[]) buffer.get(key);
266: if (buffered != null)
267: return buffered;
268:
269: // read from db
270: kelondroRow.Entry result = index.get(key);
271: if (result == null)
272: return null;
273:
274: // return result
275: return result.getColBytes(1);
276: }
277:
278: private synchronized void setValueCached(byte[] key, byte[] value)
279: throws IOException {
280: // update storage
281: synchronized (this ) {
282: index.put(rowdef.newEntry(new byte[][] { key, value }));
283: buffer.put(key, value);
284: }
285: }
286:
287: public synchronized int getDyn(String key, int pos)
288: throws IOException {
289: int reccnt = pos / reclen;
290: // read within a single record
291: byte[] buf = getValueCached(dynKey(key, reccnt));
292: if (buf == null)
293: return -1;
294: int recpos = pos % reclen;
295: if (buf.length <= recpos)
296: return -1;
297: return buf[recpos] & 0xFF;
298: }
299:
300: public synchronized byte[] getDyn(String key, int pos, int len)
301: throws IOException {
302: int recpos = pos % reclen;
303: int reccnt = pos / reclen;
304: byte[] segment1;
305: // read first within a single record
306: if ((recpos == 0) && (reclen == len)) {
307: segment1 = getValueCached(dynKey(key, reccnt));
308: if (segment1 == null)
309: return null;
310: } else {
311: byte[] buf = getValueCached(dynKey(key, reccnt));
312: if (buf == null)
313: return null;
314: if (buf.length < reclen) {
315: byte[] buff = new byte[reclen];
316: System.arraycopy(buf, 0, buff, 0, buf.length);
317: buf = buff;
318: buff = null;
319: }
320: // System.out.println("read:
321: // buf.length="+buf.length+",recpos="+recpos+",len="+len);
322: if (recpos + len <= reclen) {
323: segment1 = new byte[len];
324: System.arraycopy(buf, recpos, segment1, 0, len);
325: } else {
326: segment1 = new byte[reclen - recpos];
327: System.arraycopy(buf, recpos, segment1, 0, reclen
328: - recpos);
329: }
330: }
331: // if this is all, return
332: if (recpos + len <= reclen)
333: return segment1;
334: // read from several records
335: // we combine recursively all participating records
336: // we have two segments: the one in the starting record, and the remaining
337: // segment 1 in record <reccnt> : start = recpos, length = reclen - recpos
338: // segment 2 in record <reccnt>+1: start = 0, length = len - reclen + recpos
339: // recursively step further
340: byte[] segment2 = getDyn(key, pos + segment1.length, len
341: - segment1.length);
342: if (segment2 == null)
343: return segment1;
344: // now combine the two segments into the result
345: byte[] result = new byte[segment1.length + segment2.length];
346: System.arraycopy(segment1, 0, result, 0, segment1.length);
347: System.arraycopy(segment2, 0, result, segment1.length,
348: segment2.length);
349: return result;
350: }
351:
352: public synchronized void putDyn(String key, int pos, byte[] b,
353: int off, int len) throws IOException {
354: int recpos = pos % reclen;
355: int reccnt = pos / reclen;
356: byte[] buf;
357: // first write current record
358: if ((recpos == 0) && (reclen == len)) {
359: if (off == 0) {
360: setValueCached(dynKey(key, reccnt), b);
361: } else {
362: buf = new byte[len];
363: System.arraycopy(b, off, buf, 0, len);
364: setValueCached(dynKey(key, reccnt), b);
365: }
366: } else {
367: buf = getValueCached(dynKey(key, reccnt));
368: if (buf == null) {
369: buf = new byte[reclen];
370: } else if (buf.length < reclen) {
371: byte[] buff = new byte[reclen];
372: System.arraycopy(buf, 0, buff, 0, buf.length);
373: buf = buff;
374: buff = null;
375: }
376: // System.out.println("write:
377: // b.length="+b.length+",off="+off+",len="+(reclen-recpos));
378: if (len < (reclen - recpos))
379: System.arraycopy(b, off, buf, recpos, len);
380: else
381: System.arraycopy(b, off, buf, recpos, reclen - recpos);
382: setValueCached(dynKey(key, reccnt), buf);
383: }
384: // if more records are necessary, write to them also recursively
385: if (recpos + len > reclen) {
386: putDyn(key, pos + reclen - recpos, b,
387: off + reclen - recpos, len - reclen + recpos);
388: }
389: }
390:
391: public synchronized void remove(String key) throws IOException {
392: // remove value in cache and tree
393: if (key == null)
394: return;
395: int recpos = 0;
396: byte[] k;
397: while (index.get(k = dynKey(key, recpos)) != null) {
398: index.remove(k, false);
399: buffer.remove(k);
400: recpos++;
401: }
402: //segmentCount--; writeSegmentCount();
403: }
404:
405: public synchronized boolean existsDyn(String key)
406: throws IOException {
407: return (key != null)
408: && (getValueCached(dynKey(key, 0)) != null);
409: }
410:
411: public synchronized kelondroRA getRA(String filekey) {
412: // this returns always a RARecord, even if no existed bevore
413: //return new kelondroBufferedRA(new RARecord(filekey), 512, 0);
414: return new RARecord(filekey);
415: }
416:
417: public class RARecord extends kelondroAbstractRA implements
418: kelondroRA {
419:
420: int seekpos = 0;
421:
422: String filekey;
423:
424: public RARecord(String filekey) {
425: this .filekey = filekey;
426: }
427:
428: public long length() throws IOException {
429: return Long.MAX_VALUE;
430: }
431:
432: public long available() throws IOException {
433: return Long.MAX_VALUE;
434: }
435:
436: public int read() throws IOException {
437: return getDyn(filekey, seekpos++);
438: }
439:
440: public void write(int i) throws IOException {
441: byte[] b = new byte[1];
442: b[0] = (byte) i;
443: putDyn(filekey, seekpos++, b, 0, 1);
444: }
445:
446: public int read(byte[] b, int off, int len) throws IOException {
447: int l = Math.min(b.length - off, len);
448: byte[] buf = getDyn(filekey, seekpos, l);
449: if (buf == null)
450: return -1;
451: l = Math.min(buf.length, l);
452: System.arraycopy(buf, 0, b, off, l);
453: seekpos += l;
454: return l;
455: }
456:
457: public void write(byte[] b, int off, int len)
458: throws IOException {
459: putDyn(filekey, seekpos, b, off, len);
460: seekpos += len;
461: }
462:
463: public void seek(long pos) throws IOException {
464: seekpos = (int) pos;
465: }
466:
467: public void close() throws IOException {
468: // no need to do anything here
469: }
470:
471: }
472:
473: public synchronized void writeFile(String key, File f)
474: throws IOException {
475: // reads a file from the FS and writes it into the database
476: kelondroRA kra = null;
477: FileInputStream fis = null;
478: try {
479: kra = getRA(key);
480: byte[] buffer = new byte[1024];
481: byte[] result = new byte[(int) f.length()];
482: fis = new FileInputStream(f);
483: int i;
484: int pos = 0;
485: while ((i = fis.read(buffer)) > 0) {
486: System.arraycopy(buffer, 0, result, pos, i);
487: pos += i;
488: }
489: fis.close();
490: kra.writeArray(result);
491: } finally {
492: if (fis != null)
493: try {
494: fis.close();
495: } catch (Exception e) {
496: }
497: if (kra != null)
498: try {
499: kra.close();
500: } catch (Exception e) {
501: }
502: }
503: }
504:
505: public synchronized void readFile(String key, File f)
506: throws IOException {
507: // reads a file from the DB and writes it to the FS
508: kelondroRA kra = null;
509: FileOutputStream fos = null;
510: try {
511: kra = getRA(key);
512: byte[] result = kra.readArray();
513: fos = new FileOutputStream(f);
514: fos.write(result);
515: } finally {
516: if (fos != null)
517: try {
518: fos.close();
519: } catch (Exception e) {
520: }
521: if (kra != null)
522: try {
523: kra.close();
524: } catch (Exception e) {
525: }
526: }
527: }
528:
529: public synchronized void close() {
530: index.close();
531: }
532:
533: public static void main(String[] args) {
534: // test app for DB functions
535: // reads/writes files to a database table
536: // arguments:
537: // {-f2db/-db2f} <db-name> <key> <filename>
538:
539: if (args.length == 1) {
540: // open a db and list keys
541: try {
542: kelondroDyn kd = new kelondroDyn(new File(args[0]),
543: true, true, 0, 4, 100, '_',
544: kelondroNaturalOrder.naturalOrder, false,
545: false, true);
546: System.out.println(kd.sizeDyn() + " elements in DB");
547: Iterator<String> i = kd.dynKeys(true, false);
548: while (i.hasNext())
549: System.out.println(i.next());
550: kd.close();
551: } catch (IOException e) {
552: e.printStackTrace();
553: }
554: }
555: if (args.length == 4) {
556: boolean writeFile = (args[0].equals("-db2f"));
557: File db = new File(args[1]);
558: String key = args[2];
559: File f = new File(args[3]);
560: kelondroDyn kd;
561: try {
562: kd = new kelondroDyn(db, true, true, 0, 80, 200, '_',
563: kelondroNaturalOrder.naturalOrder, false,
564: false, true);
565: if (writeFile)
566: kd.readFile(key, f);
567: else
568: kd.writeFile(key, f);
569: } catch (IOException e) {
570: System.out.println("ERROR: " + e.toString());
571: }
572: }
573: }
574:
575: public static int countElementsDyn(kelondroDyn t) {
576: int count = 0;
577: try {
578: Iterator<String> iter = t.dynKeys(true, false);
579: while (iter.hasNext()) {
580: count++;
581: if (iter.next() == null)
582: System.out.println("ERROR! null element found");
583: }
584: return count;
585: } catch (IOException e) {
586: return -1;
587: }
588: }
589: }
|