001: /*
002: * $Id: BaseFlatfileTable.java,v 1.26 2005/11/24 20:47:32 ahimanikya Exp $
003: * =======================================================================
004: * Copyright (c) 2002-2005 Axion Development Team. All rights reserved.
005: *
006: * Redistribution and use in source and binary forms, with or without
007: * modification, are permitted provided that the following conditions
008: * are met:
009: *
010: * 1. Redistributions of source code must retain the above
011: * copyright notice, this list of conditions and the following
012: * disclaimer.
013: *
014: * 2. Redistributions in binary form must reproduce the above copyright
015: * notice, this list of conditions and the following disclaimer in
016: * the documentation and/or other materials provided with the
017: * distribution.
018: *
019: * 3. The names "Tigris", "Axion", nor the names of its contributors may
020: * not be used to endorse or promote products derived from this
021: * software without specific prior written permission.
022: *
023: * 4. Products derived from this software may not be called "Axion", nor
024: * may "Tigris" or "Axion" appear in their names without specific prior
025: * written permission.
026: *
027: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
028: * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
029: * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
030: * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
031: * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
032: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
033: * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
034: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
035: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
036: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
037: * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
038: * =======================================================================
039: */
040:
041: package org.axiondb.engine.tables;
042:
043: import java.io.File;
044: import java.io.IOException;
045: import java.util.Arrays;
046: import java.util.HashSet;
047: import java.util.Properties;
048: import java.util.Set;
049:
050: import org.apache.commons.collections.primitives.ArrayIntList;
051: import org.axiondb.AxionException;
052: import org.axiondb.Column;
053: import org.axiondb.DataType;
054: import org.axiondb.Database;
055: import org.axiondb.ExternalTable;
056: import org.axiondb.Row;
057: import org.axiondb.TableFactory;
058: import org.axiondb.TableOrganizationContext;
059: import org.axiondb.io.BufferedDataInputStream;
060: import org.axiondb.io.BufferedDataOutputStream;
061: import org.axiondb.io.FileUtil;
062: import org.axiondb.types.CharacterType;
063: import org.axiondb.types.LOBType;
064: import org.axiondb.types.ObjectType;
065: import org.axiondb.types.StringType;
066:
067: /**
068: * Base Flatfile Table<br>
069: *
070: * TODO: Support for decimal and thousand separator, trailing/leading minus sign
071: * TODO: Support for multiple record delimiter
072: *
073: * @author Ahimanikya Satapathy
074: * @author Jonathan Giron
075: */
076: public abstract class BaseFlatfileTable extends BaseDiskTable implements
077: ExternalTable {
078:
079: protected static final int EOF = -1;
080: protected static final char FILLER = ' ';
081:
082: public static final String PROP_FILENAME = "FILENAME";
083: protected static final String PROP_ISFIRSTLINEHEADER = "ISFIRSTLINEHEADER";
084: protected static final String PROP_RECORDDELIMITER = "RECORDDELIMITER";
085: protected static final String PROP_ROWSTOSKIP = "ROWSTOSKIP";
086: protected static final String PROP_MAXFAULTS = "MAXFAULTS";
087:
088: private static final Set PROPERTY_KEYS = new HashSet(4);
089:
090: static {
091: PROPERTY_KEYS.add(PROP_FILENAME);
092: PROPERTY_KEYS.add(PROP_CREATE_IF_NOT_EXIST);
093: PROPERTY_KEYS.add(PROP_ISFIRSTLINEHEADER);
094: PROPERTY_KEYS.add(PROP_RECORDDELIMITER);
095: PROPERTY_KEYS.add(PROP_ROWSTOSKIP);
096: PROPERTY_KEYS.add(PROP_MAXFAULTS);
097: PROPERTY_KEYS.add(ExternalTable.PROP_LOADTYPE);
098: }
099:
100: public BaseFlatfileTable(String name, Database db,
101: TableFactory factory) throws AxionException {
102: super (name, db, factory);
103: }
104:
105: public void addColumn(Column col, boolean metaUpdateNeeded)
106: throws AxionException {
107: if (col.getDataType() instanceof LOBType
108: || col.getDataType() instanceof ObjectType) {
109: throw new UnsupportedOperationException(
110: "Lob or Object Type not supported for this Table Type");
111: }
112: super .addColumn(col, metaUpdateNeeded);
113: }
114:
115: /**
116: * Loads external data using the given properties table - should be called only once
117: * by the table factory.
118: *
119: * @param table Table to be set
120: * @param props Properties for Table
121: * @exception AxionException thrown while setting Properties
122: */
123: public boolean loadExternalTable(Properties props)
124: throws AxionException {
125: try {
126: context.readOrSetDefaultProperties(props);
127: context.updateProperties();
128:
129: createOrLoadDataFile();
130:
131: initializeTable();
132:
133: writeMetaFile();
134: return true;
135: } catch (Exception e) {
136: // If we fail to initialize the flatfile table then drop the table.
137: // Table creation is not atomic in case of flat file.
138: try {
139: drop();
140: } catch (Throwable ignore) {
141: // this exception is secondary to the one we want to throw...
142: }
143: throw new AxionException(
144: "Failed to create table using supplied properties. ",
145: e);
146: }
147: }
148:
149: public void remount() throws AxionException {
150: remount(getRootDir(), false);
151: }
152:
153: public static String addEscapeSequence(String srcString) {
154: StringBuffer buf = new StringBuffer(srcString.length() + 2);
155:
156: char[] srcArray = srcString.toCharArray();
157: for (int i = 0; i < srcArray.length; i++) {
158: char c = srcArray[i];
159: switch (c) {
160: case '\t':
161: buf.append("\\t");
162: break;
163:
164: case '\r':
165: buf.append("\\r");
166: break;
167:
168: case '\b':
169: buf.append("\\b");
170: break;
171:
172: case '\n':
173: buf.append("\\n");
174: break;
175:
176: case '\f':
177: buf.append("\\f");
178: break;
179:
180: default:
181: buf.append(c);
182: break;
183: }
184: }
185:
186: return buf.toString();
187: }
188:
189: protected void createOrLoadDataFile() throws AxionException {
190: createOrLoadDataFile(_isCreateDataFileIfNotExist);
191: }
192:
193: protected void createOrLoadDataFile(boolean createNewDataFile)
194: throws AxionException {
195: if (!createNewDataFile && !getDataFile().exists()) {
196: throw new AxionException("Data file \"" + getDataFile()
197: + "\" for table " + getName() + " not found.");
198: }
199:
200: try {
201: if (getDataFile().createNewFile()) {
202: writeHeader(getOutputStream());
203: } else {
204: getOutputStream();
205: }
206: getInputStream();
207: } catch (IOException ioex) {
208: throw new AxionException(
209: "Unable to create data file \"" + getDataFile()
210: + "\" for table " + getName() + ".", ioex);
211: }
212: }
213:
214: public static String fixEscapeSequence(String srcString) {
215: // no escape sequences, return string as is
216: if (srcString == null || srcString.length() < 2)
217: return srcString;
218:
219: char[] srcArray = srcString.toCharArray();
220: char[] tgtArray = (char[]) srcArray.clone();
221: Arrays.fill(tgtArray, ' ');
222: int j = 0;
223: // e.g. curString = "\\r\\n" = {'\\', 'r', '\\', 'n'}
224: for (int i = 0; i < srcArray.length; i++) {
225: char c = srcArray[i];
226: if ((i + 1) == srcArray.length) {
227: // this is last char, so put it as is
228: tgtArray[j++] = c;
229: break;
230: }
231:
232: if (c == '\\') {
233: switch (srcArray[i + 1]) {
234: case 't':
235: c = '\t';
236: break;
237: case 'r':
238: c = '\r';
239: break;
240: case 'b':
241: c = '\b';
242: break;
243: case 'n':
244: c = '\n';
245: break;
246: case 'f':
247: c = '\f';
248: break;
249: }
250: i++;
251: }
252: tgtArray[j++] = c;
253: }
254:
255: return new String(tgtArray, 0, j);
256: }
257:
258: protected File getDataFile() {
259: if (_dataFile != null) {
260: return _dataFile;
261: }
262:
263: if ((_fileName.indexOf("\\") > -1)
264: || (_fileName.indexOf("/") > -1)
265: || _fileName.indexOf(File.separator) > -1) {
266: // if filename already contains path info, then use as-is
267: _dataFile = new File(_fileName);
268: } else {
269: // the reason why we are using the db dir as data file dir
270: // one may have flat files exported from some other system
271: // and want to create
272: _dataFile = new File(_dbdir, _fileName);
273: }
274: return _dataFile;
275: }
276:
277: protected synchronized void renameTableFiles(String oldName,
278: String name) {
279: super .renameTableFiles(oldName, name);
280: FileUtil.renameFile(_dbdir, oldName, name, "."
281: + getDefaultDataFileExtension());
282: _fileName = name + "." + getDefaultDataFileExtension();
283: context.setProperty(PROP_FILENAME, _fileName);
284: }
285:
286: protected File getLobDir() {
287: throw new UnsupportedOperationException(
288: "Lobs Datatype is not supported");
289: }
290:
291: protected long ignoreRowsToSkip() throws AxionException {
292: if (_rowsToSkip > 0) {
293: int offset = 0;
294: int i = _rowsToSkip;
295:
296: while (i-- > 0) {
297: offset += nextLineLength(offset);
298: }
299:
300: return offset;
301: } else if (_isFirstLineHeader) {
302: // Handle old versions of flatfiles where RowsToSkip is undefined and first
303: // line header is true.
304: return nextLineLength(0);
305: }
306: return 0;
307: }
308:
309: protected void initializeTable() throws AxionException {
310: try {
311: int faultCount = 0;
312: _rowCount = 0;
313:
314: final long fileLength = FileUtil.getLength(getDataFile());
315:
316: long fileOffset = ignoreRowsToSkip();
317: while (true) {
318: // If at EOF or current offset + length of line separator string is larger
319: // than length of file (less EOF character), then terminate.
320: // XXX: In case _lineSep holds multiple record delimiter we may have issue here ?
321: if (-1 == fileOffset
322: || (fileOffset + _lineSep.length() >= fileLength - 1)) {
323: break;
324: }
325:
326: getPidxList().add(fileOffset);
327: int idToAssign = getPidxList().size() - 1;
328:
329: try {
330: _rowCount++;
331: getRowByOffset(idToAssign, fileOffset);
332: } catch (AxionException e) {
333: // ignore this line and read next line
334: // set row as invalid so that we will not attempt to read
335: // next time
336: getPidxList().set(idToAssign, INVALID_OFFSET);
337: _freeIds.add(idToAssign);
338: _rowCount--;
339:
340: if (++faultCount > _maxFaults) {
341: // TODO: Write bad rows to a file with .bad extension
342: String msg = "Fault tolerance threshold ("
343: + _maxFaults + ") exceeded for table "
344: + getName() + ". ";
345: throw new AxionException(msg + e.getMessage(),
346: e);
347: }
348: }
349:
350: fileOffset = getInputStream().getPos();
351: }
352: getPidxList().flush();
353: } catch (Exception e) {
354: throw new AxionException(e);
355: }
356: }
357:
358: abstract protected boolean isEndOfRecord(int recLength,
359: int nextChar, BufferedDataInputStream data)
360: throws IOException;
361:
362: protected boolean isEOF(int nextChar) {
363: return nextChar == EOF;
364: }
365:
366: protected boolean isNullString(String str) {
367: return (str == null || str.trim().length() == 0);
368: }
369:
370: protected String getDefaultDataFileExtension() {
371: return "txt";
372: }
373:
374: protected void reloadFilesAfterTruncate() throws AxionException {
375: // zero out freeIds file
376: _freeIds = new ArrayIntList();
377: writeFridFile();
378:
379: initFiles(getDataFile(), true);
380:
381: // Create zero-record data file (with header if required).
382: createOrLoadDataFile(true);
383:
384: initializeTable();
385: initializeRowCount();
386: }
387:
388: protected Row trySettingColumn(int idToAssign, Row row, int i,
389: String colValue) throws AxionException {
390: DataType columnDataType = getColumn(i).getDataType();
391:
392: colValue = evaluateForNull(colValue, columnDataType);
393: if (colValue == null) {
394: row.set(i, null);
395: } else {
396: Object val = columnDataType.convert(colValue);
397: row.set(i, val);
398: }
399:
400: return row;
401: }
402:
403: protected String evaluateForNull(String colValue, DataType datatype) {
404: if (datatype instanceof CharacterType) {
405: int colWidth = datatype.getPrecision();
406: return (null == colValue || colWidth <= 0 || (colValue
407: .length() == colWidth && colValue.trim().length() == 0)) ? null
408: : colValue;
409: } else if (colValue == null
410: || (!(datatype instanceof StringType) && colValue
411: .length() == 0)) {
412: return null;
413: }
414:
415: return colValue;
416: }
417:
418: protected abstract void writeHeader(BufferedDataOutputStream data2)
419: throws AxionException;
420:
421: protected int nextLineLength(long fileOffset) throws AxionException {
422: if (fileOffset < 0) {
423: return EOF;
424: }
425:
426: try {
427: int nextChar;
428: int recLength = 0;
429: BufferedDataInputStream data = getInputStream();
430:
431: data.seek(fileOffset);
432: while (true) {
433: nextChar = data.read();
434: recLength++;
435:
436: if (isEOF(nextChar)) {
437: if (recLength > 1) {
438: //-- EOF reached but has some data
439: return recLength;
440: }
441: return EOF;
442: }
443:
444: if (isEndOfRecord(recLength, nextChar, data)) {
445: //recLength += (_lineSep.length() - 1);
446: recLength = (int) (data.getPos() - fileOffset);
447: break;
448: }
449: }
450: return recLength;
451:
452: } catch (Exception e) {
453: throw new AxionException("Unable to parse data file: ", e);
454: }
455: }
456:
457: protected abstract class BaseFlatfileTableOrganizationContext
458: extends BaseTableOrganizationContext {
459: public Set getPropertyKeys() {
460: Set keys = new HashSet(PROPERTY_KEYS.size());
461: keys.addAll(PROPERTY_KEYS);
462: return keys;
463: }
464:
465: public void readOrSetDefaultProperties(Properties props)
466: throws AxionException {
467: assertValidPropertyKeys(props);
468:
469: String rawLineSep = props.getProperty(PROP_RECORDDELIMITER);
470: if (rawLineSep != null) {
471: _lineSep = fixEscapeSequence(rawLineSep);
472: } else {
473: _lineSep = "";
474: }
475:
476: String firstLineHeader = props.getProperty(
477: PROP_ISFIRSTLINEHEADER, "false");
478: _isFirstLineHeader = Boolean.valueOf(firstLineHeader)
479: .booleanValue();
480:
481: _fileName = props.getProperty(PROP_FILENAME);
482: if (isNullString(_fileName)) {
483: _fileName = getName() + "."
484: + getDefaultDataFileExtension();
485: }
486:
487: if ((props.getProperty(PROP_CREATE_IF_NOT_EXIST) != null)
488: && ("FALSE".equalsIgnoreCase(props
489: .getProperty(PROP_CREATE_IF_NOT_EXIST)))) {
490: _isCreateDataFileIfNotExist = false;
491: }
492:
493: String skipRowsStr = props
494: .getProperty(PROP_ROWSTOSKIP, "0");
495: try {
496: _rowsToSkip = Integer.parseInt(skipRowsStr);
497: } catch (NumberFormatException e) {
498: _rowsToSkip = 0;
499: }
500:
501: String maxFaultStr = props.getProperty(PROP_MAXFAULTS, Long
502: .toString(Long.MAX_VALUE));
503: try {
504: _maxFaults = Long.parseLong(maxFaultStr);
505: } catch (NumberFormatException e) {
506: _maxFaults = Long.MAX_VALUE;
507: }
508:
509: // Negative values are meaningless...in this case use Long.MAX_VALUE to
510: // represent unlimited fault threshold.
511: if (_maxFaults < 0L) {
512: _maxFaults = Long.MAX_VALUE;
513: }
514: }
515:
516: public void updateProperties() {
517: super .updateProperties();
518:
519: setProperty(PROP_RECORDDELIMITER,
520: addEscapeSequence(_lineSep));
521: setProperty(PROP_ISFIRSTLINEHEADER, Boolean
522: .toString(_isFirstLineHeader));
523: setProperty(PROP_FILENAME, _fileName);
524: setProperty(PROP_ROWSTOSKIP, Integer.toString(_rowsToSkip));
525: setProperty(PROP_MAXFAULTS, Long.toString(_maxFaults));
526: setProperty(PROP_CREATE_IF_NOT_EXIST, Boolean
527: .toString(_isCreateDataFileIfNotExist));
528: }
529:
530: public Set getRequiredPropertyKeys() {
531: return getBaseRequiredPropertyKeys();
532: }
533: }
534:
535: protected TableOrganizationContext context;
536: protected String _fileName;
537: protected boolean _isFirstLineHeader = false;
538: protected int _rowsToSkip = 0;
539: protected long _maxFaults = Long.MAX_VALUE;
540: private boolean _isCreateDataFileIfNotExist = true;
541:
542: protected String _lineSep;
543: }
|