001: /*
002: * Copyright 2004-2008 H2 Group. Licensed under the H2 License, Version 1.0
003: * (http://h2database.com/html/license.html).
004: * Initial Developer: H2 Group
005: */
006: package org.h2.value;
007:
008: import java.io.BufferedInputStream;
009: import java.io.ByteArrayInputStream;
010: import java.io.File;
011: import java.io.IOException;
012: import java.io.InputStream;
013: import java.io.Reader;
014: import java.sql.PreparedStatement;
015: import java.sql.SQLException;
016:
017: import org.h2.constant.SysProperties;
018: import org.h2.engine.Constants;
019: import org.h2.message.Message;
020: import org.h2.store.DataHandler;
021: import org.h2.store.FileStore;
022: import org.h2.store.FileStoreInputStream;
023: import org.h2.store.FileStoreOutputStream;
024: import org.h2.store.fs.FileSystem;
025: import org.h2.util.ByteUtils;
026: import org.h2.util.FileUtils;
027: import org.h2.util.IOUtils;
028: import org.h2.util.MathUtils;
029: import org.h2.util.SmallLRUCache;
030: import org.h2.util.StringUtils;
031:
032: /**
033: * Implementation of the BLOB and CLOB data types.
034: */
035: public class ValueLob extends Value {
036: // TODO lob: concatenate function for blob and clob
037: // (to create a large blob from pieces)
038: // and a getpart function (to get it in pieces) and make sure a file is created!
039:
040: public static final int TABLE_ID_SESSION = -1;
041:
042: private final int type;
043: private long precision;
044: private DataHandler handler;
045: private int tableId;
046: private int objectId;
047: private String fileName;
048: private boolean linked;
049: private byte[] small;
050: private int hash;
051: private boolean compression;
052: private FileStore tempFile;
053:
054: /**
055: * This counter is used to calculate the next directory to store lobs.
056: * It is better than using a random number because less directories are created.
057: */
058: private static int dirCounter;
059:
060: private ValueLob(int type, DataHandler handler, String fileName,
061: int tableId, int objectId, boolean linked, long precision,
062: boolean compression) {
063: this .type = type;
064: this .handler = handler;
065: this .fileName = fileName;
066: this .tableId = tableId;
067: this .objectId = objectId;
068: this .linked = linked;
069: this .precision = precision;
070: this .compression = compression;
071: }
072:
073: private static ValueLob copy(ValueLob lob) {
074: ValueLob copy = new ValueLob(lob.type, lob.handler,
075: lob.fileName, lob.tableId, lob.objectId, lob.linked,
076: lob.precision, lob.compression);
077: copy.small = lob.small;
078: copy.hash = lob.hash;
079: return copy;
080: }
081:
082: private ValueLob(int type, byte[] small) throws SQLException {
083: this .type = type;
084: this .small = small;
085: if (small != null) {
086: if (type == Value.BLOB) {
087: this .precision = small.length;
088: } else {
089: this .precision = getString().length();
090: }
091: }
092: }
093:
094: public static ValueLob createSmallLob(int type, byte[] small)
095: throws SQLException {
096: return new ValueLob(type, small);
097: }
098:
099: private static String getFileName(DataHandler handler, int tableId,
100: int objectId) {
101: if (SysProperties.CHECK && tableId == 0 && objectId == 0) {
102: throw Message.getInternalError("0 LOB");
103: }
104: if (handler.getLobFilesInDirectories()) {
105: String table = tableId < 0 ? ".temp" : ".t" + tableId;
106: return getFileNamePrefix(handler.getDatabasePath(),
107: objectId)
108: + table + Constants.SUFFIX_LOB_FILE;
109: } else {
110: return handler.getDatabasePath() + "." + tableId + "."
111: + objectId + Constants.SUFFIX_LOB_FILE;
112: }
113: }
114:
115: /**
116: * Create a LOB value with the given parameters.
117: *
118: * @param type the data type
119: * @param handler the file handler
120: * @param tableId the table object id
121: * @param objectId the object id
122: * @param precision the precision (length in elements)
123: * @param compression if compression is used
124: * @return the value object
125: */
126: public static ValueLob open(int type, DataHandler handler,
127: int tableId, int objectId, long precision,
128: boolean compression) {
129: String fileName = getFileName(handler, tableId, objectId);
130: return new ValueLob(type, handler, fileName, tableId, objectId,
131: true, precision, compression);
132: }
133:
134: public static ValueLob createClob(Reader in, long length,
135: DataHandler handler) throws SQLException {
136: try {
137: boolean compress = handler
138: .getLobCompressionAlgorithm(Value.CLOB) != null;
139: long remaining = Long.MAX_VALUE;
140: if (length >= 0 && length < remaining) {
141: remaining = length;
142: }
143: int len = getBufferSize(handler, compress, remaining);
144: char[] buff = new char[len];
145: len = IOUtils.readFully(in, buff, len);
146: len = len < 0 ? 0 : len;
147: if (len <= handler.getMaxLengthInplaceLob()) {
148: byte[] small = StringUtils.utf8Encode(new String(buff,
149: 0, len));
150: return ValueLob.createSmallLob(Value.CLOB, small);
151: }
152: ValueLob lob = new ValueLob(Value.CLOB, null);
153: lob.createFromReader(buff, len, in, remaining, handler);
154: return lob;
155: } catch (IOException e) {
156: throw Message.convertIOException(e, null);
157: }
158: }
159:
160: private static int getBufferSize(DataHandler handler,
161: boolean compress, long remaining) {
162: if (remaining < 0 || remaining > Integer.MAX_VALUE) {
163: remaining = Integer.MAX_VALUE;
164: }
165: long inplace = handler.getMaxLengthInplaceLob();
166: if (inplace >= Integer.MAX_VALUE) {
167: inplace = remaining;
168: }
169: long m = compress ? Constants.IO_BUFFER_SIZE_COMPRESS
170: : Constants.IO_BUFFER_SIZE;
171: if (m < remaining && m <= inplace) {
172: m = Math.min(remaining, inplace + 1);
173: // the buffer size must be bigger than the inplace lob, otherwise we can't
174: // know if it must be stored in-place or not
175: m = MathUtils.roundUpLong(m, Constants.IO_BUFFER_SIZE);
176: }
177: m = Math.min(remaining, m);
178: m = MathUtils.convertLongToInt(m);
179: if (m < 0) {
180: m = Integer.MAX_VALUE;
181: }
182: return (int) m;
183: }
184:
185: private void createFromReader(char[] buff, int len, Reader in,
186: long remaining, DataHandler handler) throws SQLException {
187: try {
188: FileStoreOutputStream out = initLarge(handler);
189: boolean compress = handler
190: .getLobCompressionAlgorithm(Value.CLOB) != null;
191: try {
192: while (true) {
193: precision += len;
194: byte[] b = StringUtils.utf8Encode(new String(buff,
195: 0, len));
196: out.write(b, 0, b.length);
197: remaining -= len;
198: if (remaining <= 0) {
199: break;
200: }
201: len = getBufferSize(handler, compress, remaining);
202: len = IOUtils.readFully(in, buff, len);
203: if (len <= 0) {
204: break;
205: }
206: }
207: } finally {
208: out.close();
209: }
210: } catch (IOException e) {
211: throw Message.convertIOException(e, null);
212: }
213: }
214:
215: private static String getFileNamePrefix(String path, int objectId) {
216: String name;
217: int f = objectId % SysProperties.LOB_FILES_PER_DIRECTORY;
218: if (f > 0) {
219: name = File.separator + objectId;
220: } else {
221: name = "";
222: }
223: objectId /= SysProperties.LOB_FILES_PER_DIRECTORY;
224: while (objectId > 0) {
225: f = objectId % SysProperties.LOB_FILES_PER_DIRECTORY;
226: name = File.separator + f + Constants.SUFFIX_LOBS_DIRECTORY
227: + name;
228: objectId /= SysProperties.LOB_FILES_PER_DIRECTORY;
229: }
230: name = path + Constants.SUFFIX_LOBS_DIRECTORY + name;
231: return name;
232: }
233:
234: private int getNewObjectId(DataHandler handler) throws SQLException {
235: String path = handler.getDatabasePath();
236: int objectId = 0;
237: while (true) {
238: String dir = getFileNamePrefix(path, objectId);
239: String[] list = getFileList(handler, dir);
240: int fileCount = 0;
241: boolean[] used = new boolean[SysProperties.LOB_FILES_PER_DIRECTORY];
242: for (int i = 0; i < list.length; i++) {
243: String name = list[i];
244: if (name.endsWith(Constants.SUFFIX_DB_FILE)) {
245: name = name.substring(name
246: .lastIndexOf(File.separatorChar) + 1);
247: String n = name.substring(0, name.indexOf('.'));
248: int id;
249: try {
250: id = Integer.parseInt(n);
251: } catch (NumberFormatException e) {
252: id = -1;
253: }
254: if (id > 0) {
255: fileCount++;
256: used[id % SysProperties.LOB_FILES_PER_DIRECTORY] = true;
257: }
258: }
259: }
260: int fileId = -1;
261: if (fileCount < SysProperties.LOB_FILES_PER_DIRECTORY) {
262: for (int i = 1; i < SysProperties.LOB_FILES_PER_DIRECTORY; i++) {
263: if (!used[i]) {
264: fileId = i;
265: break;
266: }
267: }
268: }
269: if (fileId > 0) {
270: objectId += fileId;
271: invalidateFileList(handler, dir);
272: break;
273: } else {
274: if (objectId > Integer.MAX_VALUE
275: / SysProperties.LOB_FILES_PER_DIRECTORY) {
276: // this directory path is full: start from zero
277: // (this can happen only theoretically,
278: // for example if the random number generator is broken)
279: objectId = 0;
280: } else {
281: // calculate the directory
282: // start with 1 (otherwise we don't know the number of directories)
283: // it doesn't really matter what directory is used, it might as well be random
284: // (but that would generate more directories):
285: // int dirId = RandomUtils.nextInt(
286: // SysProperties.LOB_FILES_PER_DIRECTORY - 1) + 1;
287: int dirId = (dirCounter++ / (SysProperties.LOB_FILES_PER_DIRECTORY - 1)) + 1;
288: objectId = objectId
289: * SysProperties.LOB_FILES_PER_DIRECTORY;
290: objectId += dirId
291: * SysProperties.LOB_FILES_PER_DIRECTORY;
292: }
293: }
294: }
295: return objectId;
296: }
297:
298: private void invalidateFileList(DataHandler handler, String dir) {
299: SmallLRUCache cache = handler.getLobFileListCache();
300: if (cache != null) {
301: synchronized (cache) {
302: cache.remove(dir);
303: }
304: }
305: }
306:
307: private String[] getFileList(DataHandler handler, String dir)
308: throws SQLException {
309: SmallLRUCache cache = handler.getLobFileListCache();
310: String[] list;
311: if (cache == null) {
312: list = FileUtils.listFiles(dir);
313: } else {
314: synchronized (cache) {
315: list = (String[]) cache.get(dir);
316: if (list == null) {
317: list = FileUtils.listFiles(dir);
318: cache.put(dir, list);
319: }
320: }
321: }
322: return list;
323: }
324:
325: public static ValueLob createBlob(InputStream in, long length,
326: DataHandler handler) throws SQLException {
327: try {
328: long remaining = Long.MAX_VALUE;
329: boolean compress = handler
330: .getLobCompressionAlgorithm(Value.BLOB) != null;
331: if (length >= 0 && length < remaining) {
332: remaining = length;
333: }
334: int len = getBufferSize(handler, compress, remaining);
335: byte[] buff = new byte[len];
336: len = IOUtils.readFully(in, buff, 0, len);
337: if (len <= handler.getMaxLengthInplaceLob()) {
338: byte[] small = new byte[len];
339: System.arraycopy(buff, 0, small, 0, len);
340: return ValueLob.createSmallLob(Value.BLOB, small);
341: }
342: ValueLob lob = new ValueLob(Value.BLOB, null);
343: lob.createFromStream(buff, len, in, remaining, handler);
344: return lob;
345: } catch (IOException e) {
346: throw Message.convertIOException(e, null);
347: }
348: }
349:
350: private FileStoreOutputStream initLarge(DataHandler handler)
351: throws IOException, SQLException {
352: this .handler = handler;
353: this .tableId = 0;
354: this .linked = false;
355: this .precision = 0;
356: this .small = null;
357: this .hash = 0;
358: String compressionAlgorithm = handler
359: .getLobCompressionAlgorithm(type);
360: this .compression = compressionAlgorithm != null;
361: synchronized (handler) {
362: if (handler.getLobFilesInDirectories()) {
363: objectId = getNewObjectId(handler);
364: fileName = getFileNamePrefix(handler.getDatabasePath(),
365: objectId)
366: + ".temp.db";
367: } else {
368: objectId = handler.allocateObjectId(false, true);
369: fileName = handler.createTempFile();
370: }
371: tempFile = handler.openFile(fileName, "rw", false);
372: tempFile.autoDelete();
373: }
374: FileStoreOutputStream out = new FileStoreOutputStream(tempFile,
375: handler, compressionAlgorithm);
376: return out;
377: }
378:
379: private void createFromStream(byte[] buff, int len, InputStream in,
380: long remaining, DataHandler handler) throws SQLException {
381: try {
382: FileStoreOutputStream out = initLarge(handler);
383: boolean compress = handler
384: .getLobCompressionAlgorithm(Value.BLOB) != null;
385: try {
386: while (true) {
387: precision += len;
388: out.write(buff, 0, len);
389: remaining -= len;
390: if (remaining <= 0) {
391: break;
392: }
393: len = getBufferSize(handler, compress, remaining);
394: len = IOUtils.readFully(in, buff, 0, len);
395: if (len <= 0) {
396: break;
397: }
398: }
399: } finally {
400: out.close();
401: }
402: } catch (IOException e) {
403: throw Message.convertIOException(e, null);
404: }
405: }
406:
407: public Value convertTo(int t) throws SQLException {
408: if (t == type) {
409: return this ;
410: } else if (t == Value.CLOB) {
411: ValueLob copy = ValueLob.createClob(getReader(), -1,
412: handler);
413: return copy;
414: } else if (t == Value.BLOB) {
415: ValueLob copy = ValueLob.createBlob(getInputStream(), -1,
416: handler);
417: return copy;
418: }
419: return super .convertTo(t);
420: }
421:
422: public boolean isLinked() {
423: return linked;
424: }
425:
426: public String getFileName() {
427: return fileName;
428: }
429:
430: public void close() throws SQLException {
431: if (fileName != null) {
432: if (tempFile != null) {
433: tempFile.stopAutoDelete();
434: }
435: deleteFile(handler, fileName);
436: }
437: }
438:
439: public void unlink() throws SQLException {
440: if (linked && fileName != null) {
441: String temp;
442: // synchronize on the database, to avoid concurrent temp file
443: // creation / deletion / backup
444: synchronized (handler) {
445: if (handler.getLobFilesInDirectories()) {
446: temp = getFileName(handler, -1, objectId);
447: } else {
448: // just to get a filename - an empty file will be created
449: temp = handler.createTempFile();
450: }
451: deleteFile(handler, temp);
452: renameFile(handler, fileName, temp);
453: tempFile = FileStore.open(handler, temp, "rw", null);
454: tempFile.autoDelete();
455: tempFile.closeSilently();
456: fileName = temp;
457: linked = false;
458: }
459: }
460: }
461:
462: public Value link(DataHandler handler, int tabId)
463: throws SQLException {
464: if (fileName == null) {
465: this .tableId = tabId;
466: return this ;
467: }
468: if (linked) {
469: ValueLob copy = ValueLob.copy(this );
470: if (handler.getLobFilesInDirectories()) {
471: copy.objectId = getNewObjectId(handler);
472: } else {
473: copy.objectId = handler.allocateObjectId(false, true);
474: }
475: copy.tableId = tabId;
476: String live = getFileName(handler, copy.tableId,
477: copy.objectId);
478: copyFile(handler, fileName, live);
479: copy.fileName = live;
480: copy.linked = true;
481: return copy;
482: }
483: if (!linked) {
484: this .tableId = tabId;
485: String live = getFileName(handler, tableId, objectId);
486: tempFile.stopAutoDelete();
487: tempFile = null;
488: renameFile(handler, fileName, live);
489: fileName = live;
490: linked = true;
491: }
492: return this ;
493: }
494:
495: public int getTableId() {
496: return tableId;
497: }
498:
499: public int getObjectId() {
500: return objectId;
501: }
502:
503: public int getType() {
504: return type;
505: }
506:
507: public long getPrecision() {
508: return precision;
509: }
510:
511: public String getString() {
512: int len = precision > Integer.MAX_VALUE || precision == 0 ? Integer.MAX_VALUE
513: : (int) precision;
514: try {
515: if (type == Value.CLOB) {
516: if (small != null) {
517: return StringUtils.utf8Decode(small);
518: }
519: return IOUtils.readStringAndClose(getReader(), len);
520: } else {
521: byte[] buff;
522: if (small != null) {
523: buff = small;
524: } else {
525: buff = IOUtils.readBytesAndClose(getInputStream(),
526: len);
527: }
528: return ByteUtils.convertBytesToString(buff);
529: }
530: } catch (IOException e) {
531: throw Message.convertToInternal(Message.convertIOException(
532: e, fileName));
533: }
534: }
535:
536: public byte[] getBytes() throws SQLException {
537: byte[] data = getBytesNoCopy();
538: return ByteUtils.cloneByteArray(data);
539: }
540:
541: public byte[] getBytesNoCopy() throws SQLException {
542: if (small != null) {
543: return small;
544: }
545: try {
546: return IOUtils.readBytesAndClose(getInputStream(),
547: Integer.MAX_VALUE);
548: } catch (IOException e) {
549: throw Message.convertIOException(e, fileName);
550: }
551: }
552:
553: public int hashCode() {
554: if (hash == 0) {
555: if (precision > 4096) {
556: // TODO: should calculate the hash code when saving, and store
557: // it in the data file
558: return (int) (precision ^ (precision >> 32));
559: }
560: try {
561: hash = ByteUtils.getByteArrayHash(getBytes());
562: } catch (SQLException e) {
563: throw Message.convertToInternal(e);
564: }
565: }
566: return hash;
567: }
568:
569: protected int compareSecure(Value v, CompareMode mode)
570: throws SQLException {
571: if (type == Value.CLOB) {
572: int c = getString().compareTo(v.getString());
573: return c == 0 ? 0 : (c < 0 ? -1 : 1);
574: } else {
575: byte[] v2 = v.getBytesNoCopy();
576: return ByteUtils.compareNotNull(getBytes(), v2);
577: }
578: }
579:
580: public Object getObject() {
581: if (type == Value.CLOB) {
582: return getReader();
583: } else {
584: return getInputStream();
585: }
586: }
587:
588: public Reader getReader() {
589: try {
590: return IOUtils.getReader(getInputStream());
591: } catch (SQLException e) {
592: throw Message.convertToInternal(e);
593: }
594: }
595:
596: public InputStream getInputStream() {
597: try {
598: if (fileName == null) {
599: return new ByteArrayInputStream(small);
600: }
601: FileStore store = handler.openFile(fileName, "r", true);
602: boolean alwaysClose = SysProperties.lobCloseBetweenReads;
603: return new BufferedInputStream(new FileStoreInputStream(
604: store, handler, compression, alwaysClose),
605: Constants.IO_BUFFER_SIZE);
606: } catch (SQLException e) {
607: throw Message.convertToInternal(e);
608: }
609: }
610:
611: public void set(PreparedStatement prep, int parameterIndex)
612: throws SQLException {
613: long p = getPrecision();
614: // TODO test if setBinaryStream with -1 works for other databases a well
615: if (p > Integer.MAX_VALUE || p <= 0) {
616: p = -1;
617: }
618: if (type == Value.BLOB) {
619: prep.setBinaryStream(parameterIndex, getInputStream(),
620: (int) p);
621: } else {
622: prep.setCharacterStream(parameterIndex, getReader(),
623: (int) p);
624: }
625: }
626:
627: public String getSQL() {
628: try {
629: String s;
630: if (type == Value.CLOB) {
631: s = getString();
632: return StringUtils.quoteStringSQL(s);
633: } else {
634: byte[] buff = getBytes();
635: s = ByteUtils.convertBytesToString(buff);
636: return "X'" + s + "'";
637: }
638: } catch (SQLException e) {
639: throw Message.convertToInternal(e);
640: }
641: }
642:
643: public String toString() {
644: if (small == null) {
645: return getClass().getName() + " file: " + fileName
646: + " type: " + type + " precision: " + precision;
647: } else {
648: return getSQL();
649: }
650: }
651:
652: public byte[] getSmall() {
653: return small;
654: }
655:
656: public int getDisplaySize() {
657: return MathUtils.convertLongToInt(getPrecision());
658: }
659:
660: public boolean equals(Object other) {
661: try {
662: return other instanceof ValueLob
663: && compareSecure((Value) other, null) == 0;
664: } catch (SQLException e) {
665: throw Message.convertToInternal(e);
666: }
667: }
668:
669: public void convertToFileIfRequired(DataHandler handler)
670: throws SQLException {
671: if (Constants.AUTO_CONVERT_LOB_TO_FILES && small != null
672: && small.length > handler.getMaxLengthInplaceLob()) {
673: boolean compress = handler.getLobCompressionAlgorithm(type) != null;
674: int len = getBufferSize(handler, compress, Long.MAX_VALUE);
675: int tabId = tableId;
676: if (type == Value.BLOB) {
677: createFromStream(new byte[len], 0, getInputStream(),
678: Long.MAX_VALUE, handler);
679: } else {
680: createFromReader(new char[len], 0, getReader(),
681: Long.MAX_VALUE, handler);
682: }
683: Value v2 = link(handler, tabId);
684: if (SysProperties.CHECK && v2 != this ) {
685: throw Message.getInternalError();
686: }
687: }
688: }
689:
690: public static void removeAllForTable(DataHandler handler,
691: int tableId) throws SQLException {
692: if (handler.getLobFilesInDirectories()) {
693: String dir = getFileNamePrefix(handler.getDatabasePath(), 0);
694: removeAllForTable(handler, dir, tableId);
695: } else {
696: String prefix = handler.getDatabasePath();
697: String dir = FileUtils.getParent(prefix);
698: String[] list = FileUtils.listFiles(dir);
699: for (int i = 0; i < list.length; i++) {
700: String name = list[i];
701: if (name.startsWith(prefix + "." + tableId + ".")
702: && name.endsWith(".lob.db")) {
703: deleteFile(handler, name);
704: }
705: }
706: }
707: }
708:
709: private static void removeAllForTable(DataHandler handler,
710: String dir, int tableId) throws SQLException {
711: String[] list = FileUtils.listFiles(dir);
712: for (int i = 0; i < list.length; i++) {
713: if (FileUtils.isDirectory(list[i])) {
714: removeAllForTable(handler, list[i], tableId);
715: } else {
716: String name = list[i];
717: if (name.endsWith(".t" + tableId + ".lob.db")) {
718: deleteFile(handler, name);
719: }
720: }
721: }
722: }
723:
724: public boolean useCompression() {
725: return compression;
726: }
727:
728: public boolean isFileBased() {
729: return fileName != null;
730: }
731:
732: private static synchronized void deleteFile(DataHandler handler,
733: String fileName) throws SQLException {
734: // synchronize on the database, to avoid concurrent temp file creation /
735: // deletion / backup
736: synchronized (handler.getLobSyncObject()) {
737: FileUtils.delete(fileName);
738: }
739: }
740:
741: private static synchronized void renameFile(DataHandler handler,
742: String oldName, String newName) throws SQLException {
743: synchronized (handler.getLobSyncObject()) {
744: FileUtils.rename(oldName, newName);
745: }
746: }
747:
748: private void copyFile(DataHandler handler, String fileName,
749: String live) throws SQLException {
750: synchronized (handler.getLobSyncObject()) {
751: FileSystem.getInstance(fileName).copy(fileName, live);
752: }
753: }
754:
755: public void setFileName(String fileName) {
756: this .fileName = fileName;
757: }
758:
759: public int getMemory() {
760: if (small != null) {
761: return small.length + 32;
762: }
763: return 128;
764: }
765:
766: }
|