001: /*
002: * $Id: FixedWidthFlatfileTable.java,v 1.16 2005/12/20 18:32:28 ahimanikya Exp $
003: * =======================================================================
004: * Copyright (c) 2002-2005 Axion Development Team. All rights reserved.
005: *
006: * Redistribution and use in source and binary forms, with or without
007: * modification, are permitted provided that the following conditions
008: * are met:
009: *
010: * 1. Redistributions of source code must retain the above
011: * copyright notice, this list of conditions and the following
012: * disclaimer.
013: *
014: * 2. Redistributions in binary form must reproduce the above copyright
015: * notice, this list of conditions and the following disclaimer in
016: * the documentation and/or other materials provided with the
017: * distribution.
018: *
019: * 3. The names "Tigris", "Axion", nor the names of its contributors may
020: * not be used to endorse or promote products derived from this
021: * software without specific prior written permission.
022: *
023: * 4. Products derived from this software may not be called "Axion", nor
024: * may "Tigris" or "Axion" appear in their names without specific prior
025: * written permission.
026: *
027: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
028: * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
029: * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
030: * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
031: * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
032: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
033: * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
034: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
035: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
036: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
037: * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
038: * =======================================================================
039: */
040:
041: package org.axiondb.engine.tables;
042:
043: import java.io.IOException;
044: import java.io.ObjectInputStream;
045: import java.io.ObjectOutputStream;
046: import java.util.Arrays;
047: import java.util.HashSet;
048: import java.util.Properties;
049: import java.util.Set;
050:
051: import org.axiondb.AxionException;
052: import org.axiondb.Column;
053: import org.axiondb.DataType;
054: import org.axiondb.Database;
055: import org.axiondb.ExternalTable;
056: import org.axiondb.Row;
057: import org.axiondb.engine.rows.SimpleRow;
058: import org.axiondb.io.BufferedDataInputStream;
059: import org.axiondb.io.BufferedDataOutputStream;
060:
061: /**
062: * A disk-resident Fixed Width Flatfile {@link org.axiondb.Table}.
063: *
064: * @version $Revision: 1.16 $ $Date: 2005/12/20 18:32:28 $
065: * @author Ahimanikya Satapathy
066: * @author Jonathan Giron
067: */
068: public final class FixedWidthFlatfileTable extends BaseFlatfileTable {
069:
070: // TODO: Add Record Trailer byte size
071:
072: public static final String PROP_HEADERBYTESOFFSET = "HEADERBYTESOFFSET";
073:
074: private static final Set PROPERTY_KEYS = new HashSet(2);
075:
076: static {
077: PROPERTY_KEYS.add(PROP_HEADERBYTESOFFSET);
078: }
079:
080: public FixedWidthFlatfileTable(String name, Database db)
081: throws AxionException {
082: super (name, db, new FixedWidthFlatfileTableLoader());
083: setType(ExternalTable.FW_TABLE_TYPE);
084: }
085:
086: public void addColumn(Column col, boolean metaUpdateNeeded)
087: throws AxionException {
088: super .addColumn(col, metaUpdateNeeded);
089: updateRecordLength();
090: }
091:
092: protected Row getRowByOffset(int idToAssign, long ptr)
093: throws AxionException {
094: BufferedDataInputStream data = getInputStream();
095: int colCount = getColumnCount();
096: Row row = new SimpleRow(idToAssign, colCount);
097:
098: try {
099: synchronized (data) {
100: int colLength = 0;
101: int linePtr = 0;
102: char[] charArray = readLine(data, ptr);
103:
104: for (int i = 0; i < colCount; i++) {
105: colLength = getColumnSize(i);
106:
107: String columnValue = new String(charArray, linePtr,
108: colLength).trim();
109: if (columnValue.length() == 0) {
110: columnValue = null;
111: }
112:
113: row = trySettingColumn(idToAssign, row, i,
114: columnValue);
115: linePtr += colLength;
116: }
117: }
118: return row;
119: } catch (Exception e) {
120: if (e instanceof AxionException) {
121: throw (AxionException) e;
122: }
123: throw new AxionException(e);
124: }
125: }
126:
127: protected int nextLineLength(final long fileOffset)
128: throws AxionException {
129: try {
130: int nextChar;
131: int recLength = 0;
132: BufferedDataInputStream data = getInputStream();
133:
134: data.seek(fileOffset);
135: while (true) {
136: nextChar = data.read();
137: recLength++;
138:
139: if (isEOF(nextChar)) {
140: if (recLength > 1) {
141: //-- EOF reached but has some data
142: return recLength;
143: }
144: return EOF;
145: }
146:
147: if (isEndOfRecord(recLength, nextChar, data)) {
148: recLength = (recLength == 1 ? recLength
149: : _recordLength);
150: break;
151: }
152: }
153: return recLength;
154:
155: } catch (Exception e) {
156: throw new AxionException("Unable to parse data file: ", e);
157: }
158: }
159:
160: protected long ignoreRowsToSkip() throws AxionException {
161: long offset = super .ignoreRowsToSkip();
162: if (offset > 0) {
163: return offset;
164: } else if (_headerBytesOffset > 0) {
165: return _headerBytesOffset;
166: }
167: return 0;
168: }
169:
170: protected void initializeTable() throws AxionException {
171: _lineCharArray = new char[_recordLength];
172: super .initializeTable();
173: }
174:
175: protected boolean isEndOfRecord(int recLength, int nextChar,
176: BufferedDataInputStream data) throws IOException {
177: if (isEOF(nextChar)) {
178: return true;
179: }
180:
181: if (recLength >= _recordLength) {
182: if (!("".equals(_lineSep))
183: && _lineSep.charAt(0) != nextChar) {
184: throw new IOException(
185: "Corrupted data, record delimeter not found at specified record length.");
186: } else {
187: return true;
188: }
189: }
190:
191: if (!("".equals(_lineSep)) && _lineSep.charAt(0) == nextChar) {
192: char[] charBuf = _lineSep.toCharArray();
193: // Look ahead to see whether the following chars match EOL.
194: long lastDataFileOffset = data.getPos();
195: for (int i = 1, I = _lineSep.length(); i < I; i++) {
196: if (charBuf[i] != (char) data.read()) {
197: data.seek(lastDataFileOffset);
198: return false;
199: }
200: }
201: return true;
202: }
203: return false;
204: }
205:
206: public boolean loadExternalTable(Properties props)
207: throws AxionException {
208: context = new FixedwidthTableOrganizationContext();
209: return super .loadExternalTable(props);
210: }
211:
212: public Properties getTableProperties() {
213: return context.getTableProperties();
214: }
215:
216: protected void parseTableProperties(ObjectInputStream in)
217: throws AxionException {
218: try {
219: _lineSep = in.readUTF();
220: _isFirstLineHeader = Boolean.valueOf(in.readUTF())
221: .booleanValue();
222: in.readUTF(); // _eol will be computed , keep this for older version metadata
223: _headerBytesOffset = in.readInt();
224: _recordLength = in.readInt();
225: _fileName = in.readUTF();
226:
227: context = new FixedwidthTableOrganizationContext();
228: context.updateProperties();
229: context.readOrSetDefaultProperties(context
230: .getTableProperties());
231: createOrLoadDataFile();
232: _lineCharArray = new char[_recordLength];
233: } catch (IOException e) {
234: throw new AxionException(
235: "Unable to parse meta file for table " + getName(),
236: e);
237: }
238: }
239:
240: protected synchronized void renameTableFiles(String oldName,
241: String name) {
242: super .renameTableFiles(oldName, name);
243: updateRecordLength();
244: }
245:
246: private void updateRecordLength() {
247: if (context != null) {
248: _recordLength = 0;
249: for (int i = 0, I = getColumnCount(); i < I; i++) {
250: _recordLength += getColumnSize(i);
251: }
252: _lineCharArray = new char[_recordLength];
253: }
254: }
255:
256: protected void writeHeader(BufferedDataOutputStream dataFile)
257: throws AxionException {
258: if (_isFirstLineHeader) {
259: try {
260: for (int i = 0, I = getColumnCount(); i < I; i++) {
261: dataFile.write(writeColumn(i, getColumn(i)
262: .getName()));
263: }
264: dataFile.write(_lineSep.getBytes());
265: } catch (IOException ioex) {
266: throw new AxionException(
267: "Unable to write header for table: "
268: + getName(), ioex);
269: }
270: }
271: }
272:
273: protected void writeRow(BufferedDataOutputStream out, Row row)
274: throws AxionException {
275: Object colValue = null;
276: DataType type = null;
277:
278: try {
279: for (int i = 0, I = getColumnCount(); i < I; i++) {
280: colValue = row.get(i);
281: type = getColumn(i).getDataType();
282: out.write(writeColumn(i, type.toString(colValue)));
283: }
284:
285: // write new line
286: out.write(_lineSep.getBytes());
287: } catch (IOException e) {
288: throw new AxionException("Error writing row: " + row, e);
289: }
290: }
291:
292: protected void writeTableProperties(ObjectOutputStream out)
293: throws AxionException {
294: try {
295: if (_lineSep != null && _fileName != null) {
296: out.writeUTF(_lineSep);
297: out.writeUTF(Boolean.toString(_isFirstLineHeader));
298: out.writeUTF(_lineSep);
299: out.writeInt(_headerBytesOffset);
300: out.writeInt(_recordLength);
301: out.writeUTF(_fileName);
302: }
303: } catch (IOException e) {
304: throw new AxionException(
305: "Unable to write meta file for table " + getName(),
306: e);
307: }
308: }
309:
310: private int getColumnSize(int index) {
311: return getColumn(index).getDataType().getColumnDisplaySize();
312: }
313:
314: private char[] readLine(BufferedDataInputStream data,
315: long fileOffset) throws AxionException {
316: Arrays.fill(_lineCharArray, FILLER);
317: int recLength = 0;
318: try {
319: int nextChar;
320: data.seek(fileOffset);
321:
322: while (true) {
323: nextChar = data.read();
324: if (isEndOfRecord(recLength, nextChar, data)) {
325: if (recLength == 0) {
326: throw new AxionException(
327: "Empty line detected - invalid.");
328: } else if (_lineSep.length() == 0
329: && !isEOF(nextChar)) {
330: // If this is a packed data file, move pointer back by one - since
331: // there isn't a record delimiter, our call to data.read() has set
332: // the file pointer to the first char of the next record.
333: data.seek(data.getPos() - 1);
334: }
335: break;
336: }
337: _lineCharArray[recLength++] = ((char) nextChar);
338: }
339:
340: return _lineCharArray;
341:
342: } catch (IOException ioex) {
343: throw new AxionException("Unable to parse data file: ",
344: ioex);
345: }
346: }
347:
348: private byte[] writeColumn(int colIndex, String value) {
349: if (value == null) {
350: value = " ";
351: }
352:
353: byte byteData[] = new byte[getColumnSize(colIndex)];
354: Arrays.fill(byteData, (byte) FILLER);
355: byte colValue[] = value.getBytes();
356:
357: // truncate if required
358: int len = colValue.length <= byteData.length ? colValue.length
359: : byteData.length;
360: System.arraycopy(colValue, 0, byteData, 0, len);
361: return byteData;
362: }
363:
364: private class FixedwidthTableOrganizationContext extends
365: BaseFlatfileTableOrganizationContext {
366: public Set getPropertyKeys() {
367: Set baseKeys = super .getPropertyKeys();
368: Set keys = new HashSet(baseKeys.size()
369: + PROPERTY_KEYS.size());
370: keys.addAll(baseKeys);
371: keys.addAll(PROPERTY_KEYS);
372:
373: return keys;
374: }
375:
376: public void readOrSetDefaultProperties(Properties props)
377: throws AxionException {
378: super .readOrSetDefaultProperties(props);
379:
380: // compute record length; if not set yet
381: for (int i = 0, I = getColumnCount(); i < I; i++) {
382: _recordLength += getColumnSize(i);
383: }
384: _recordLength += _lineSep.length();
385:
386: try {
387: _headerBytesOffset = Integer.parseInt(props
388: .getProperty(PROP_HEADERBYTESOFFSET));
389: } catch (NumberFormatException e) {
390: // optional argument, ignore if not set
391: }
392:
393: // IsFirstLineHeader will override HeaderBytesOffset.
394: _isFirstLineHeader = Boolean.valueOf(
395: props.getProperty(PROP_ISFIRSTLINEHEADER
396: .toUpperCase(), "false")).booleanValue();
397: }
398:
399: public void updateProperties() {
400: super .updateProperties();
401: _props.setProperty(PROP_LOADTYPE,
402: ExternalTableFactory.TYPE_FIXEDWIDTH);
403: _props.setProperty(PROP_HEADERBYTESOFFSET, Integer
404: .toString(_headerBytesOffset));
405: }
406: }
407:
408: private int _headerBytesOffset = 0;
409: private char[] _lineCharArray;
410: private int _recordLength;
411:
412: }
|