001: /* Copyright (c) 2001-2005, The HSQL Development Group
002: * All rights reserved.
003: *
004: * Redistribution and use in source and binary forms, with or without
005: * modification, are permitted provided that the following conditions are met:
006: *
007: * Redistributions of source code must retain the above copyright notice, this
008: * list of conditions and the following disclaimer.
009: *
010: * Redistributions in binary form must reproduce the above copyright notice,
011: * this list of conditions and the following disclaimer in the documentation
012: * and/or other materials provided with the distribution.
013: *
014: * Neither the name of the HSQL Development Group nor the names of its
015: * contributors may be used to endorse or promote products derived from this
016: * software without specific prior written permission.
017: *
018: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
019: * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
020: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
021: * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
022: * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
023: * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
024: * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025: * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
026: * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028: * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029: */
030:
031: package org.hsqldb.persist;
032:
033: import java.io.File;
034: import java.io.IOException;
035:
036: import org.hsqldb.Database;
037: import org.hsqldb.HsqlException;
038: import org.hsqldb.Trace;
039: import org.hsqldb.lib.FileAccess;
040: import org.hsqldb.lib.FileUtil;
041: import org.hsqldb.lib.SimpleLog;
042: import org.hsqldb.lib.StopWatch;
043: import org.hsqldb.lib.Storage;
044: import org.hsqldb.lib.ZipUnzipFile;
045: import org.hsqldb.rowio.RowInputBinary;
046: import org.hsqldb.rowio.RowInputInterface;
047: import org.hsqldb.rowio.RowOutputBinary;
048: import org.hsqldb.rowio.RowOutputInterface;
049: import org.hsqldb.store.BitMap;
050:
051: /**
052: * Acts as a manager for CACHED table persistence.<p>
053: *
054: * This contains the top level functionality. Provides file management services
055: * and access.<p>
056: *
057: * Rewritten for 1.8.0 together with Cache.
058: *
059: * @author fredt@users
060: * @version 1.8.0
061: * @since 1.7.2
062: */
063: public class DataFileCache {
064:
065: protected FileAccess fa;
066:
067: // flags
068: public static final int FLAG_ISSAVED = 2;
069: public static final int FLAG_ROWINFO = 3;
070:
071: // file format fields
072: static final int LONG_EMPTY_SIZE = 4; // empty space size
073: static final int LONG_FREE_POS_POS = 12; // where iFreePos is saved
074: static final int LONG_EMPTY_INDEX_POS = 20; // empty space index
075: static final int FLAGS_POS = 28;
076: static final int INITIAL_FREE_POS = 32;
077:
078: //
079: DataFileBlockManager freeBlocks;
080: private static final int initIOBufferSize = 256;
081:
082: //
083: protected String fileName;
084: protected String backupFileName;
085: protected Database database;
086:
087: // this flag is used externally to determine if a backup is required
088: protected boolean fileModified;
089: protected int cacheFileScale;
090:
091: // post openning constant fields
092: protected boolean cacheReadonly;
093:
094: // cache operation mode
095: protected boolean storeOnInsert;
096:
097: //
098: protected int cachedRowPadding = 8;
099: protected boolean hasRowInfo = false;
100:
101: // reusable input / output streams
102: protected RowInputInterface rowIn;
103: protected RowOutputInterface rowOut;
104:
105: //
106: public long maxDataFileSize;
107:
108: //
109: protected Storage dataFile;
110: protected long fileFreePosition;
111: protected int maxCacheSize; // number of Rows
112: protected long maxCacheBytes; // number of bytes
113: protected int maxFreeBlocks;
114: protected Cache cache;
115:
116: public DataFileCache(Database db, String baseFileName)
117: throws HsqlException {
118:
119: initParams(db, baseFileName);
120:
121: cache = new Cache(this );
122: }
123:
124: /**
125: * initial external parameters are set here.
126: */
127: protected void initParams(Database database, String baseFileName)
128: throws HsqlException {
129:
130: HsqlDatabaseProperties props = database.getProperties();
131:
132: fileName = baseFileName + ".data";
133: backupFileName = baseFileName + ".backup";
134: this .database = database;
135: fa = database.getFileAccess();
136:
137: int cacheScale = props.getIntegerProperty(
138: HsqlDatabaseProperties.hsqldb_cache_scale, 14, 8, 18);
139: int cacheSizeScale = props.getIntegerProperty(
140: HsqlDatabaseProperties.hsqldb_cache_size_scale, 10, 6,
141: 20);
142: int cacheFreeCountScale = props.getIntegerProperty(
143: HsqlDatabaseProperties.hsqldb_cache_free_count_scale,
144: 9, 6, 12);
145:
146: cacheFileScale = database.getProperties().getIntegerProperty(
147: HsqlDatabaseProperties.hsqldb_cache_file_scale, 1);
148:
149: if (cacheFileScale != 1) {
150: cacheFileScale = 8;
151: }
152:
153: cacheReadonly = database.isFilesReadOnly();
154:
155: int lookupTableLength = 1 << cacheScale;
156: int avgRowBytes = 1 << cacheSizeScale;
157:
158: maxCacheSize = lookupTableLength * 3;
159: maxCacheBytes = maxCacheSize * avgRowBytes;
160: maxDataFileSize = cacheFileScale == 1 ? Integer.MAX_VALUE
161: : (long) Integer.MAX_VALUE * 4;
162: maxFreeBlocks = 1 << cacheFreeCountScale;
163: dataFile = null;
164: }
165:
166: /**
167: * Opens the *.data file for this cache, setting the variables that
168: * allow access to the particular database version of the *.data file.
169: */
170: public void open(boolean readonly) throws HsqlException {
171:
172: fileFreePosition = 0;
173:
174: database.logger.appLog
175: .logContext(SimpleLog.LOG_NORMAL, "start");
176:
177: try {
178: boolean preexists = database.isFilesInJar();
179: long freesize = 0;
180:
181: if (!preexists && fa.isStreamElement(fileName)) {
182: if (database.isStoredFileAccess()) {
183: preexists = true;
184: } else {
185:
186: // discard "empty" databases
187: File f = new File(fileName);
188:
189: preexists = f.length() > INITIAL_FREE_POS;
190: }
191: }
192:
193: if (preexists) {
194: String version = database.getProperties().getProperty(
195: HsqlDatabaseProperties.hsqldb_cache_version);
196: boolean v17 = HsqlDatabaseProperties.VERSION_STRING_1_7_0
197: .equals(version);
198:
199: // for later versions
200: boolean v18 = HsqlDatabaseProperties.VERSION_STRING_1_8_0
201: .equals(version);
202:
203: if (!v17) {
204: throw Trace
205: .error(Trace.WRONG_DATABASE_FILE_VERSION);
206: }
207: }
208:
209: boolean isNio = database.getProperties().isPropertyTrue(
210: HsqlDatabaseProperties.hsqldb_nio_data_file);
211: int fileType = isNio ? ScaledRAFile.DATA_FILE_NIO
212: : ScaledRAFile.DATA_FILE_RAF;
213:
214: if (database.isFilesInJar()) {
215: fileType = ScaledRAFile.DATA_FILE_JAR;
216: }
217:
218: // oj@openofice.org - change to file access api
219: String cname = database.getURLProperties().getProperty(
220: "storage_class_name");
221: String skey = database.getURLProperties().getProperty(
222: "storage_key");
223:
224: dataFile = ScaledRAFile.newScaledRAFile(database, fileName,
225: readonly, fileType, cname, skey);
226:
227: if (preexists) {
228: dataFile.seek(FLAGS_POS);
229:
230: int flags = dataFile.readInt();
231:
232: hasRowInfo = BitMap.isSet(flags, FLAG_ROWINFO);
233:
234: dataFile.seek(LONG_EMPTY_SIZE);
235:
236: freesize = dataFile.readLong();
237:
238: dataFile.seek(LONG_FREE_POS_POS);
239:
240: fileFreePosition = dataFile.readLong();
241:
242: if (fileFreePosition < INITIAL_FREE_POS) {
243: fileFreePosition = INITIAL_FREE_POS;
244: }
245: } else {
246: fileFreePosition = INITIAL_FREE_POS;
247:
248: dataFile.seek(LONG_FREE_POS_POS);
249: dataFile.writeLong(INITIAL_FREE_POS);
250:
251: // set unsaved flag;
252: dataFile.seek(FLAGS_POS);
253: dataFile.writeInt(0);
254: }
255:
256: initBuffers();
257:
258: fileModified = false;
259: freeBlocks = new DataFileBlockManager(maxFreeBlocks,
260: cacheFileScale, freesize);
261:
262: database.logger.appLog.logContext(SimpleLog.LOG_NORMAL,
263: "end");
264: } catch (Throwable e) {
265: database.logger.appLog.logContext(e, "failed");
266: close(false);
267:
268: throw Trace.error(Trace.FILE_IO_ERROR,
269: Trace.DataFileCache_open, new Object[] { e,
270: fileName });
271: }
272: }
273:
274: /**
275: * Parameter write indicates either an orderly close, or a fast close
276: * without backup.
277: *
278: * When false, just closes the file.
279: *
280: * When true, writes out all cached rows that have been modified and the
281: * free position pointer for the *.data file and then closes the file.
282: */
283: public void close(boolean write) throws HsqlException {
284:
285: SimpleLog appLog = database.logger.appLog;
286:
287: try {
288: if (cacheReadonly) {
289: if (dataFile != null) {
290: dataFile.close();
291: }
292:
293: return;
294: }
295:
296: StopWatch sw = new StopWatch();
297:
298: appLog.sendLine(SimpleLog.LOG_NORMAL,
299: "DataFileCache.close(" + write + ") : start");
300:
301: if (write) {
302: cache.saveAll();
303: Trace.printSystemOut("saveAll: " + sw.elapsedTime());
304: appLog.sendLine(SimpleLog.LOG_NORMAL,
305: "DataFileCache.close() : save data");
306:
307: if (fileModified || freeBlocks.isModified()) {
308:
309: // set empty
310: dataFile.seek(LONG_EMPTY_SIZE);
311: dataFile.writeLong(freeBlocks.getLostBlocksSize());
312:
313: // set end
314: dataFile.seek(LONG_FREE_POS_POS);
315: dataFile.writeLong(fileFreePosition);
316:
317: // set saved flag;
318: dataFile.seek(FLAGS_POS);
319:
320: int flag = BitMap.set(0, FLAG_ISSAVED);
321:
322: if (hasRowInfo) {
323: flag = BitMap.set(flag, FLAG_ROWINFO);
324: }
325:
326: dataFile.writeInt(flag);
327: appLog.sendLine(SimpleLog.LOG_NORMAL,
328: "DataFileCache.close() : flags");
329:
330: //
331: if (dataFile.length() != fileFreePosition) {
332: dataFile.seek(fileFreePosition);
333: }
334:
335: appLog.sendLine(SimpleLog.LOG_NORMAL,
336: "DataFileCache.close() : seek end");
337: Trace.printSystemOut("pos and flags: "
338: + sw.elapsedTime());
339: }
340: }
341:
342: if (dataFile != null) {
343: dataFile.close();
344: appLog.sendLine(SimpleLog.LOG_NORMAL,
345: "DataFileCache.close() : close");
346:
347: dataFile = null;
348:
349: Trace.printSystemOut("close: " + sw.elapsedTime());
350: }
351:
352: boolean empty = fileFreePosition == INITIAL_FREE_POS;
353:
354: if (empty) {
355: fa.removeElement(fileName);
356: fa.removeElement(backupFileName);
357: }
358: } catch (Throwable e) {
359: appLog.logContext(e, null);
360:
361: throw Trace.error(Trace.FILE_IO_ERROR,
362: Trace.DataFileCache_close, new Object[] { e,
363: fileName });
364: }
365: }
366:
367: protected void initBuffers() {
368:
369: if (rowOut == null
370: || ((RowOutputBinary) rowOut).getBuffer().length > initIOBufferSize) {
371: rowOut = new RowOutputBinary(256);
372: }
373:
374: if (rowIn == null
375: || ((RowInputBinary) rowIn).getBuffer().length > initIOBufferSize) {
376: rowIn = new RowInputBinary(new byte[256]);
377: }
378: }
379:
380: /**
381: * Writes out all the rows to a new file without fragmentation.
382: */
383: public void defrag() throws HsqlException {
384:
385: if (cacheReadonly) {
386: return;
387: }
388:
389: if (fileFreePosition == INITIAL_FREE_POS) {
390: return;
391: }
392:
393: database.logger.appLog
394: .logContext(SimpleLog.LOG_NORMAL, "start");
395:
396: try {
397: boolean wasNio = dataFile.wasNio();
398:
399: cache.saveAll();
400:
401: DataFileDefrag dfd = new DataFileDefrag(database, this ,
402: fileName);
403:
404: dfd.process();
405: close(false);
406: deleteFile(wasNio);
407: renameDataFile();
408: backupFile();
409: database.getProperties().setProperty(
410: HsqlDatabaseProperties.hsqldb_cache_version,
411: HsqlDatabaseProperties.THIS_CACHE_VERSION);
412: database.getProperties().save();
413: cache.clear();
414:
415: cache = new Cache(this );
416:
417: open(cacheReadonly);
418: dfd.updateTableIndexRoots();
419: dfd.updateTransactionRowIDs();
420: } catch (Throwable e) {
421: database.logger.appLog.logContext(e, null);
422:
423: throw new HsqlException(e, Trace
424: .getMessage(Trace.GENERAL_IO_ERROR),
425: Trace.GENERAL_IO_ERROR);
426: }
427:
428: database.logger.appLog.logContext(SimpleLog.LOG_NORMAL, "end");
429: }
430:
431: /**
432: * Used when a row is deleted as a result of some DML or DDL command.
433: * Removes the row from the cache data structures.
434: * Adds the file space for the row to the list of free positions.
435: */
436: public synchronized void remove(int i, PersistentStore store)
437: throws IOException {
438:
439: CachedObject r = release(i);
440: int size = r == null ? getStorageSize(i) : r.getStorageSize();
441:
442: freeBlocks.add(i, size);
443: }
444:
445: public synchronized void removePersistence(int i,
446: PersistentStore store) throws IOException {
447: }
448:
449: /**
450: * Allocates file space for the row. <p>
451: *
452: * Free space is requested from the block manager if it exists.
453: * Otherwise the file is grown to accommodate it.
454: */
455: private int setFilePos(CachedObject r) throws IOException {
456:
457: int rowSize = r.getStorageSize();
458: int i = freeBlocks == null ? -1 : freeBlocks.get(rowSize);
459:
460: if (i == -1) {
461: i = (int) (fileFreePosition / cacheFileScale);
462:
463: long newFreePosition = fileFreePosition + rowSize;
464:
465: if (newFreePosition > maxDataFileSize) {
466: throw new IOException(Trace
467: .getMessage(Trace.DATA_FILE_IS_FULL));
468: }
469:
470: fileFreePosition = newFreePosition;
471: }
472:
473: r.setPos(i);
474:
475: return i;
476: }
477:
478: public synchronized void add(CachedObject object)
479: throws IOException {
480:
481: int size = object.getRealSize(rowOut);
482:
483: size = ((size + cachedRowPadding - 1) / cachedRowPadding)
484: * cachedRowPadding;
485:
486: object.setStorageSize(size);
487:
488: int i = setFilePos(object);
489:
490: cache.put(i, object);
491:
492: // was previously used for text tables
493: if (storeOnInsert) {
494: saveRow(object);
495: }
496: }
497:
498: /**
499: * For a CacheObject that had been previously released from the cache.
500: * A new version is introduced, using the preallocated space for the object.
501: */
502: public synchronized void restore(CachedObject object)
503: throws IOException {
504:
505: int i = object.getPos();
506:
507: cache.put(i, object);
508:
509: // was previously used for text tables
510: if (storeOnInsert) {
511: saveRow(object);
512: }
513: }
514:
515: public synchronized int getStorageSize(int i) throws IOException {
516:
517: CachedObject value = cache.get(i);
518:
519: if (value != null) {
520: return value.getStorageSize();
521: }
522:
523: return readSize(i);
524: }
525:
526: public synchronized CachedObject get(int i, PersistentStore store,
527: boolean keep) throws HsqlException {
528:
529: if (i < 0) {
530: return null;
531: }
532:
533: try {
534: CachedObject object = cache.get(i);
535:
536: if (object == null) {
537: RowInputInterface rowInput = readObject(i);
538:
539: if (rowInput == null) {
540: return null;
541: }
542:
543: object = store.get(rowInput);
544:
545: // for text tables with empty rows at the beginning,
546: // pos may move forward in readObject
547: i = object.getPos();
548:
549: cache.put(i, object);
550: }
551:
552: if (keep) {
553: object.keepInMemory(true);
554: }
555:
556: return object;
557: } catch (IOException e) {
558: database.logger.appLog.logContext(e, fileName
559: + " get pos: " + i);
560:
561: throw Trace.error(Trace.DATA_FILE_ERROR,
562: Trace.DataFileCache_makeRow, new Object[] { e,
563: fileName });
564: }
565: }
566:
567: synchronized RowInputInterface getRaw(int i) throws IOException {
568: return readObject(i);
569: }
570:
571: protected synchronized int readSize(int pos) throws IOException {
572:
573: dataFile.seek((long) pos * cacheFileScale);
574:
575: return dataFile.readInt();
576: }
577:
578: protected synchronized RowInputInterface readObject(int pos)
579: throws IOException {
580:
581: dataFile.seek((long) pos * cacheFileScale);
582:
583: int size = dataFile.readInt();
584:
585: rowIn.resetRow(pos, size);
586: dataFile.read(rowIn.getBuffer(), 4, size - 4);
587:
588: return rowIn;
589: }
590:
591: public synchronized CachedObject release(int i) {
592: return cache.release(i);
593: }
594:
595: protected synchronized void saveRows(CachedObject[] rows,
596: int offset, int count) throws IOException {
597:
598: try {
599: for (int i = offset; i < offset + count; i++) {
600: CachedObject r = rows[i];
601:
602: saveRow(r);
603:
604: rows[i] = null;
605: }
606: } catch (IOException e) {
607: database.logger.appLog.logContext(e, null);
608:
609: throw e;
610: } catch (Throwable e) {
611: database.logger.appLog.logContext(e, null);
612:
613: throw new IOException(e.toString());
614: } finally {
615: initBuffers();
616: }
617: }
618:
619: /**
620: * Writes out the specified Row. Will write only the Nodes or both Nodes
621: * and table row data depending on what is not already persisted to disk.
622: */
623: public synchronized void saveRow(CachedObject row)
624: throws IOException {
625:
626: setFileModified();
627: rowOut.reset();
628: row.write(rowOut);
629: dataFile.seek((long) row.getPos() * cacheFileScale);
630: dataFile.write(rowOut.getOutputStream().getBuffer(), 0, rowOut
631: .getOutputStream().size());
632: }
633:
634: /**
635: * Saves the *.data file as compressed *.backup.
636: *
637: * @throws HsqlException
638: */
639: void backupFile() throws IOException {
640:
641: try {
642: if (fa.isStreamElement(fileName)) {
643: ZipUnzipFile.compressFile(fileName, backupFileName
644: + ".new", database.getFileAccess());
645: }
646: } catch (IOException e) {
647: database.logger.appLog.logContext(e, null);
648:
649: throw e;
650: }
651: }
652:
653: void renameBackupFile() {
654:
655: if (fa.isStreamElement(backupFileName + ".new")) {
656: fa.removeElement(backupFileName);
657: fa.renameElement(backupFileName + ".new", backupFileName);
658: }
659: }
660:
661: /**
662: * Renames the *.data.new file.
663: *
664: * @throws HsqlException
665: */
666: void renameDataFile() {
667:
668: if (fa.isStreamElement(fileName + ".new")) {
669: fa.removeElement(fileName);
670: fa.renameElement(fileName + ".new", fileName);
671: }
672: }
673:
674: void deleteFile(boolean wasNio) {
675:
676: // first attemp to delete
677: fa.removeElement(fileName);
678:
679: if (fa.isStreamElement(fileName)) {
680: if (wasNio) {
681: System.gc();
682: fa.removeElement(fileName);
683: }
684:
685: if (fa.isStreamElement(fileName)) {
686: fa.renameElement(fileName, fileName + ".old");
687:
688: File oldfile = new File(fileName + ".old");
689:
690: FileUtil.deleteOnExit(oldfile);
691: }
692: }
693: }
694:
695: void deleteBackup() {
696: fa.removeElement(backupFileName);
697: }
698:
699: /**
700: * This method deletes a data file or resets its free position.
701: * this is used only for nio files - not OOo files
702: */
703: static void deleteOrResetFreePos(Database database, String filename) {
704:
705: ScaledRAFile raFile = null;
706:
707: database.getFileAccess().removeElement(filename);
708:
709: if (database.isStoredFileAccess()) {
710: return;
711: }
712:
713: if (!database.getFileAccess().isStreamElement(filename)) {
714: return;
715: }
716:
717: try {
718: raFile = new ScaledRAFile(database, filename, false);
719:
720: raFile.seek(LONG_FREE_POS_POS);
721: raFile.writeLong(INITIAL_FREE_POS);
722: } catch (IOException e) {
723: database.logger.appLog.logContext(e, null);
724: } finally {
725: if (raFile != null) {
726: try {
727: raFile.close();
728: } catch (IOException e) {
729: database.logger.appLog.logContext(e, null);
730: }
731: }
732: }
733: }
734:
735: public int capacity() {
736: return maxCacheSize;
737: }
738:
739: public long bytesCapacity() {
740: return maxCacheBytes;
741: }
742:
743: public long getTotalCachedBlockSize() {
744: return cache.getTotalCachedBlockSize();
745: }
746:
747: public int getFreeBlockCount() {
748: return freeBlocks.size();
749: }
750:
751: public int getTotalFreeBlockSize() {
752: return 0;
753: }
754:
755: public long getFileFreePos() {
756: return fileFreePosition;
757: }
758:
759: public int getCachedObjectCount() {
760: return cache.size();
761: }
762:
763: public String getFileName() {
764: return fileName;
765: }
766:
767: public boolean hasRowInfo() {
768: return hasRowInfo;
769: }
770:
771: public boolean isFileModified() {
772: return fileModified;
773: }
774:
775: protected synchronized void setFileModified() throws IOException {
776:
777: if (!fileModified) {
778:
779: // unset saved flag;
780: dataFile.seek(FLAGS_POS);
781:
782: int flag = BitMap.set(0, FLAG_ISSAVED);
783:
784: if (hasRowInfo) {
785: flag = BitMap.set(flag, FLAG_ROWINFO);
786: }
787:
788: dataFile.writeInt(flag);
789:
790: fileModified = true;
791: }
792: }
793: }
|