001: /*
002: * $Id: TestDelimitedFlatfileTable.java,v 1.20 2005/12/20 19:08:58 ahimanikya Exp $
003: * =======================================================================
004: * Copyright (c) 2002 Axion Development Team. All rights reserved.
005: *
006: * Redistribution and use in source and binary forms, with or without
007: * modification, are permitted provided that the following conditions
008: * are met:
009: *
010: * 1. Redistributions of source code must retain the above
011: * copyright notice, this list of conditions and the following
012: * disclaimer.
013: *
014: * 2. Redistributions in binary form must reproduce the above copyright
015: * notice, this list of conditions and the following disclaimer in
016: * the documentation and/or other materials provided with the
017: * distribution.
018: *
019: * 3. The names "Tigris", "Axion", nor the names of its contributors may
020: * not be used to endorse or promote products derived from this
021: * software without specific prior written permission.
022: *
023: * 4. Products derived from this software may not be called "Axion", nor
024: * may "Tigris" or "Axion" appear in their names without specific prior
025: * written permission.
026: *
027: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
028: * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
029: * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
030: * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
031: * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
032: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
033: * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
034: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
035: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
036: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
037: * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
038: * =======================================================================
039: */
040:
041: package org.axiondb.engine.tables;
042:
043: import java.io.File;
044: import java.io.FileWriter;
045: import java.util.ArrayList;
046: import java.util.List;
047: import java.util.Properties;
048:
049: import junit.framework.Test;
050: import junit.framework.TestSuite;
051:
052: import org.axiondb.AxionException;
053: import org.axiondb.Column;
054: import org.axiondb.ColumnIdentifier;
055: import org.axiondb.DataType;
056: import org.axiondb.Database;
057: import org.axiondb.ExternalTable;
058: import org.axiondb.ExternalTableLoader;
059: import org.axiondb.Row;
060: import org.axiondb.RowIterator;
061: import org.axiondb.Table;
062: import org.axiondb.TableIdentifier;
063: import org.axiondb.engine.DiskDatabase;
064: import org.axiondb.engine.commands.AxionQueryContext;
065: import org.axiondb.engine.commands.CreateTableCommand;
066: import org.axiondb.engine.commands.CreateViewCommand;
067: import org.axiondb.engine.commands.SubSelectCommand;
068: import org.axiondb.types.CharacterVaryingType;
069: import org.axiondb.types.IntegerType;
070: import org.axiondb.types.LOBType;
071: import org.axiondb.types.ObjectType;
072:
073: /**
074: * @version $Revision: 1.20 $ $Date: 2005/12/20 19:08:58 $
075: * @author Ahimanikya Satapathy
076: */
077: public class TestDelimitedFlatfileTable extends AbstractTableTest {
078:
079: //------------------------------------------------------------ Conventional
080:
081: public TestDelimitedFlatfileTable(String testName) {
082: super (testName);
083: }
084:
085: public static Test suite() {
086: TestSuite suite = new TestSuite(
087: TestDelimitedFlatfileTable.class);
088: return suite;
089: }
090:
091: //--------------------------------------------------------------- Lifecycle
092:
093: protected DiskDatabase _db = null;
094: protected String tableName = null;
095: protected String dataFileName = null;
096:
097: protected Table createTable(String name) throws Exception {
098: tableName = name;
099: ExternalTableLoader loader = new DelimitedFlatfileTableLoader();
100: ExternalTable t = (ExternalTable) loader.createTable(_db, name);
101: t.loadExternalTable(setProperties(name));
102: return t;
103: }
104:
105: protected Database getDatabase() throws Exception {
106: return _db;
107: }
108:
109: protected File getDataFile() throws Exception {
110: return new File(getDbdir(), dataFileName);
111: }
112:
113: protected Properties setProperties(String name) {
114: Properties props = new Properties();
115:
116: props.setProperty(ExternalTable.PROP_LOADTYPE, "delimited");
117: props.setProperty(DelimitedFlatfileTable.PROP_FIELDDELIMITER,
118: ",");
119: String eol = System.getProperty("line.separator");
120: props.setProperty(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
121: props.setProperty(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER,
122: "false");
123:
124: dataFileName = name + ".csv";
125: props
126: .setProperty(BaseFlatfileTable.PROP_FILENAME,
127: dataFileName);
128:
129: return props;
130: }
131:
132: protected String getTableName() {
133: return tableName != null ? tableName : "FFCSV";
134: }
135:
136: public void setUp() throws Exception {
137: getDbdir().mkdirs();
138: _db = new DiskDatabase(getDbdir());
139: super .setUp();
140: }
141:
142: public void tearDown() throws Exception {
143: super .tearDown();
144: _db.shutdown();
145: File data = new File(getDbdir(), tableName + ".csv");
146: data.delete();
147: }
148:
149: //------------------------------------------------------------------- Tests
150:
151: public void testObjectTable() throws Exception {
152: // TODO: Make this test pass, define a interface MarshallableObject for method
153: // toString and toObject, or or MarshallableXMLObject toXMLString , toObject , If
154: // the Object implement this then we can use them in flat file.
155: }
156:
157: public void testInvalidPropertyKey() throws Exception {
158: try {
159: Properties badProps = new Properties();
160: badProps.put(ExternalTable.PROP_LOADTYPE, "delimited");
161: badProps.put("UNKNOWN_PROPERTY", Boolean.TRUE);
162: ExternalTableFactory factory = new ExternalTableFactory();
163: factory.createTable(_db, "BadTable", badProps,
164: buildColumns());
165: fail("Expected AxionException due to unrecognized property name 'UNKNOWN_PROPERTY'");
166: } catch (AxionException expected) {
167: // Expected AxionException due to unrecognized property name.
168: }
169: }
170:
171: public void testDiskInsert() throws Exception {
172: testAddRow();
173: table.shutdown();
174: File data = new File(getDbdir(), dataFileName);
175: assertTrue("Should have data file", data.exists());
176: assertTrue("Should have some data in data file",
177: data.length() >= 12);
178: }
179:
180: public void testInvalidDataType() throws Exception {
181: try {
182: table.addColumn(new Column("LOBCOL", new LOBType()));
183: fail("Expected Exception");
184: } catch (UnsupportedOperationException e) {
185: // expected
186: }
187:
188: try {
189: table.addColumn(new Column("OBJCOL", new ObjectType()));
190: fail("Expected Exception");
191: } catch (UnsupportedOperationException e) {
192: // expected
193: }
194: }
195:
196: public void testRemount() throws Exception {
197: testAddRow();
198: ((ExternalTable) table).remount();
199:
200: RowIterator iter = table.getRowIterator(true);
201: assertNotNull(iter);
202: assertTrue(iter.hasNext());
203: assertNotNull(iter.next());
204: assertTrue(iter.hasNext());
205: assertNotNull(iter.next());
206: assertTrue(!iter.hasNext());
207: }
208:
209: public void testDiskDrop() throws Exception {
210: testAddRow();
211: File tabledir = new File(getDbdir(), getTableName());
212: File meta = new File(tabledir, getTableName() + ".META");
213: assertTrue("Table directory should exist", tabledir.exists());
214: assertTrue("Meta file should exist", meta.exists());
215: table.drop();
216: assertTrue("Meta file should not exist", !meta.exists());
217: assertTrue("Table directory should not exist", !tabledir
218: .exists());
219: }
220:
221: public void testFileReadCustomRecordDelimiter() throws Exception {
222: File data = new File(getDbdir(), "FFTEST.csv");
223: FileWriter out = new FileWriter(data);
224:
225: String eol = "AA##BB";
226:
227: out.write("\"ID\", \"NAME\"" + eol); // Header
228: out.write("\"1\",\"aaaa\"" + eol); // 1
229: out.write("\"we\"," + eol); // bad 1
230: out.write("\"2\",\"bbbb\"" + eol); // 2
231: out.write("\"3\",\"cccc\"" + eol); // 3
232: out.write("\"4\",\"dddd\"" + eol); // 4
233: out.write("\"xx\",\"xx\"" + eol); // bad 2
234: out.write("\"5\",\"sfdf\"" + eol); // 5
235: out.write("\"6\",\"eeee\"" + eol); // 6
236: out.write("\"2004-10-10\",\"hhhh\"" + eol); // bad 3
237: out.write("\"7.0\", \"AAc\"" + eol); // 7
238: out.close();
239:
240: ExternalTableFactory factory = new ExternalTableFactory();
241: Properties prop = setProperties("FFTEST");
242: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
243: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
244: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, "AA##BB");
245:
246: Table table2 = factory.createTable(_db, "FFTEST", prop,
247: buildColumns());
248: RowIterator itr = table2.getRowIterator(false);
249:
250: int rowCount = 0;
251: while (itr.hasNext()) {
252: itr.next();
253: rowCount++;
254: }
255:
256: assertEquals("Valid row count should have correct value", 7,
257: rowCount);
258: table2.drop();
259: data.delete();
260: }
261:
262: public void testGetLobDir() {
263: try {
264: ((DelimitedFlatfileTable) table).getLobDir();
265: fail("Expected Exception");
266: } catch (UnsupportedOperationException e) {
267: // expected
268: }
269: }
270:
271: public void testFileReadQuotedMissingCloseQuote() throws Exception {
272: final String ffName = "QuotedTest";
273:
274: File data = new File(getDbdir(), ffName + ".csv");
275: FileWriter out = new FileWriter(data);
276:
277: String eol = System.getProperty("line.separator");
278: out.write("\"ID\", \"NAME\"" + eol); // Header
279: out.write("\"1\",\"dfdf" + eol); // 5 - missing close quote
280: out.close();
281:
282: Table table2 = null;
283: try {
284: ExternalTableFactory factory = new ExternalTableFactory();
285: Properties prop = setProperties("FFTEST");
286: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
287: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
288: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
289: prop.setProperty(BaseFlatfileTable.PROP_FILENAME, data
290: .getCanonicalPath());
291:
292: table2 = factory.createTable(_db, ffName, prop,
293: buildColumns());
294: RowIterator itr = table2.getRowIterator(false);
295:
296: int rowCount = 0;
297: while (itr.hasNext()) {
298: itr.next();
299: rowCount++;
300: Row row = itr.current();
301: assertEquals(new Integer(1), row.get(0));
302: assertEquals("dfdf", row.get(1));
303: }
304:
305: assertEquals("Valid row count should have correct value",
306: 1, rowCount);
307: } finally {
308: if (table2 != null) {
309: table2.drop();
310: }
311: data.delete();
312: }
313: }
314:
315: public void testFileReadQuoted() throws Exception {
316: final String ffName = "QuotedTest";
317:
318: File data = new File(getDbdir(), ffName + ".csv");
319: FileWriter out = new FileWriter(data);
320:
321: String eol = System.getProperty("line.separator");
322: out.write("\"ID\", \"NAME\"" + eol); // Header
323: out.write("\"1\", \"aa\"" + eol); // 1
324: out.write("\"2.00\", \"bbb\"" + eol); // 2
325: out.write("\"3.00\" , \"ccc\"" + eol); // 3
326: out.write("\"4.00\", \"\"" + eol); // 4
327: out.write("" + eol); // skip
328: out.write("\"we\"," + eol); // bad 1
329: out.write("\"7.0f\", \"ccc\"" + eol); // bad 3 ?
330: out.write("\"xx\",\"xx\"" + eol); // bad 4
331: out.write("\"2004-10-10\",\"hhhh\"" + eol); // bad 5
332: out.write("\"7\",\"dfdf" + eol); // 5 - missing end quote
333: out.write("\"\" , \"\"" + eol); // 6
334:
335: String name = "";
336: for (int i = 0; i < 20; i++) {
337: name += "cccdd";
338: }
339: out.write("3.00," + name + ""); // 7 - no quotes
340: out.close();
341:
342: Table table2 = null;
343: try {
344: ExternalTableFactory factory = new ExternalTableFactory();
345: Properties prop = setProperties("FFTEST");
346: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
347: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
348: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
349: prop.setProperty(BaseFlatfileTable.PROP_FILENAME, data
350: .getCanonicalPath());
351:
352: table2 = factory.createTable(_db, ffName, prop,
353: buildColumns());
354: RowIterator itr = table2.getRowIterator(false);
355:
356: int rowCount = 0;
357: while (itr.hasNext()) {
358: itr.next();
359: rowCount++;
360: }
361:
362: assertEquals("Valid row count should have correct value",
363: 7, rowCount);
364: } finally {
365: if (table2 != null) {
366: table2.drop();
367: }
368: data.delete();
369: }
370: }
371:
372: /**
373: * Tests whether parsing logic in org.axiondb.engine.tables.DelimitedFlatfileTable.CharTokenizer correctly handles the scenario where the
374: * parsed record fits exactly within the size of _lineCharBuffer, and the last field is
375: * numeric (non-char).
376: *
377: * @throws Exception if an unexpected error occurs
378: */
379: public void testCharTokenizerWithNumericLastColumnAtLineCharBufferCapacity()
380: throws Exception {
381: final String tableName1 = "CHAR_BUFFER_TEST";
382: Properties prop = setProperties(tableName1);
383:
384: File data = new File(getDbdir(), (String) prop
385: .get(BaseFlatfileTable.PROP_FILENAME));
386:
387: Table table1 = null;
388: try {
389: FileWriter out = new FileWriter(data);
390:
391: String eol = "\r\n";
392: out.write("NAME,QTY" + eol); // Header
393: out.write("\"aaa\",1" + eol); // 1
394: out.write("\"bbb\",2" + eol); // 2
395: out.write("\"ccc\",3" + eol); // 3
396:
397: StringBuffer buf = new StringBuffer(
398: DelimitedFlatfileTable.LINE_CHAR_ARRAY_SIZE);
399:
400: // Subtracting 5 from the default array size adjusts for the number of chars outside
401: // the quoted string, plus the newline that is appended to _lineCharArray. We want to
402: // fill the default array to capacity, and not trigger a resizing.
403: for (int i = 0; i < DelimitedFlatfileTable.LINE_CHAR_ARRAY_SIZE - 5; i++) {
404: buf.append(i % 10);
405: }
406: out.write("\"" + buf.toString() + "\",4" + eol);
407: out.close();
408:
409: ExternalTableFactory factory = new ExternalTableFactory();
410: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
411: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
412: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
413:
414: List list = new ArrayList(2);
415: list.add(new Column("NAME", new CharacterVaryingType(100)));
416: list.add(new Column("QTY", new IntegerType()));
417:
418: table1 = factory.createTable(_db, tableName1, prop, list);
419: assertEquals("Total lines should return correct value", 4,
420: table1.getRowCount());
421:
422: _db.addTable(table1);
423:
424: CreateViewCommand cmd = new CreateViewCommand();
425: final String viewName = tableName1 + "View";
426: cmd.setObjectName(viewName);
427: cmd.setIfNotExists(true);
428: cmd.setSubQuery("select * from " + tableName1);
429: cmd.execute(_db);
430: TableView view = (TableView) _db.getTable(viewName);
431:
432: // If CharTokenizer doesn't handle the fourth row correctly, we expect the row
433: // count to be less than four.
434: assertEquals("Valid row count should have correct value",
435: 4, view.getRowCount());
436: } finally {
437: if (table1 != null) {
438: table1.drop();
439: }
440: data.delete();
441: }
442: }
443:
444: /**
445: * Tests whether parsing logic in org.axiondb.engine.tables.DelimitedFlatfileTable.CharTokenizer correctly handles the scenario where the
446: * records are delimited with CR/LF but the declared record delimiter is LF only, and the
447: * last field is numeric (non-char).
448: *
449: * @throws Exception if an unexpected error occurs
450: */
451: // public void testCharTokenizerWithNumericLastColumnAndCRLFAsActualEOF() throws Exception {
452: // final String tableName1 = "CharBufferTest";
453: // Properties prop = setProperties(tableName1);
454: //
455: // File data = new File(getDbdir(), (String) prop.get(BaseFlatfileTable.PROP_FILENAME));
456: //
457: // Table table1 = null;
458: // try {
459: // FileWriter out = new FileWriter(data);
460: //
461: // String eol = "\r\n";
462: // out.write("NAME,QTY" + eol); // Header
463: // out.write("\"aaa\",1" + eol); // 1
464: // out.write("\"bbb\",2" + eol); // 2
465: // out.write("\"ccc\",3" + eol); // 3
466: // out.write("\"ddd\",4" + eol); // 4
467: // out.close();
468: //
469: // ExternalTableFactory factory = new ExternalTableFactory();
470: // prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
471: // prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
472: //
473: // // Stated EOF is LF but actual EOF is CRLF
474: // prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, "\n");
475: //
476: // List list = new ArrayList(2);
477: // list.add(new Column("NAME", new CharacterVaryingType(10)));
478: // list.add(new Column("QTY", new IntegerType()));
479: //
480: // table1 = factory.createTable(_db, tableName1, prop, list);
481: // assertEquals("Total lines should return correct value", 4, table1.getRowCount());
482: //
483: // _db.addTable(table1);
484: //
485: // CreateViewCommand cmd = new CreateViewCommand();
486: // final String viewName = tableName1 + "View";
487: // cmd.setObjectName(viewName);
488: // cmd.setIfNotExists(true);
489: // cmd.setSubQuery("select * from " + tableName1);
490: // cmd.execute(_db);
491: // TableView view = (TableView)_db.getTable(viewName);
492: //
493: // // If CharTokenizer doesn't handle the fourth row correctly, we expect the row
494: // // count to be less than four.
495: // assertEquals("Valid row count should have correct value", 4, view.getRowCount());
496: // } finally {
497: // if (table1 != null) {
498: // table1.drop();
499: // }
500: // data.delete();
501: // }
502: // }
503: public void testCharTokenizerWithQuotedCharLastColumnAtLineCharBufferCapacity()
504: throws Exception {
505: final String tableName1 = "CharBufferTest2";
506: Properties prop = setProperties(tableName1);
507:
508: File data = new File(getDbdir(), (String) prop
509: .get(BaseFlatfileTable.PROP_FILENAME));
510:
511: Table table1 = null;
512: try {
513: FileWriter out = new FileWriter(data);
514:
515: String eol = "\r\n";
516: out.write("QTY,NAME" + eol); // Header
517:
518: StringBuffer buf = new StringBuffer(
519: DelimitedFlatfileTable.LINE_CHAR_ARRAY_SIZE);
520:
521: // Subtracting 5 from the default array size adjusts for the number of chars outside
522: // the quoted string, plus the newline that is appended to _lineCharArray. We want to
523: // fill the default array to capacity, and not trigger a resizing.
524: for (int i = 0; i < DelimitedFlatfileTable.LINE_CHAR_ARRAY_SIZE - 5; i++) {
525: buf.append(i % 10);
526: }
527: out.write("1,\"" + buf.toString() + "\"" + eol);
528: out.close();
529:
530: ExternalTableFactory factory = new ExternalTableFactory();
531: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
532: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
533: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
534:
535: List list = new ArrayList(2);
536: DataType stringType = new CharacterVaryingType(100);
537: list.add(new Column("QTY", new IntegerType()));
538: list.add(new Column("NAME", stringType));
539:
540: table1 = factory.createTable(_db, tableName1, prop, list);
541: assertEquals("Total lines should return correct value", 1,
542: table1.getRowCount());
543:
544: _db.addTable(table1);
545:
546: // Ensure that NAME column does not contain any extraneous characters.
547: Row aRow = table1.getRow(0);
548: assertEquals(
549: "String column contains extraneous characters", buf
550: .toString(), (String) stringType
551: .convert(aRow.get(1)));
552: } finally {
553: if (table1 != null) {
554: table1.drop();
555: }
556: data.delete();
557: }
558: }
559:
560: public void testFileReadNoValidRow() throws Exception {
561: File data = new File(getDbdir(), "FFTest.csv");
562: FileWriter out = new FileWriter(data);
563:
564: String eol = System.getProperty("line.separator");
565:
566: out.write("\"ID\", \"NAME\"" + eol); // Header
567: out.write("\"aa\", \"aa\"" + eol); // 1
568: out.write("\"bbb\", \"bbb\"" + eol); // 2
569: out.write("\"ccc\" , \"ccc\"" + eol); // 3
570: out.close();
571:
572: ExternalTableFactory factory = new ExternalTableFactory();
573: Properties prop = setProperties("FFTEST");
574: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
575: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
576:
577: Table table2 = factory.createTable(_db, "FFTEST", prop,
578: buildColumns());
579: RowIterator itr = table2.getRowIterator(false);
580:
581: int rowCount = 0;
582: while (itr.hasNext()) {
583: itr.next();
584: rowCount++;
585: }
586: assertEquals("Valid row count should have correct value", 0,
587: rowCount);
588: table2.drop();
589:
590: table2 = factory.createTable(_db, "FFTEST", prop,
591: buildColumns());
592: itr = table2.getRowIterator(false);
593:
594: _db.addTable(table2);
595:
596: CreateViewCommand cmd = new CreateViewCommand();
597: cmd.setObjectName("FFTESTVIEW");
598: cmd.setIfNotExists(true);
599: cmd.setSubQuery("select * from FFTest");
600: cmd.execute(_db);
601: TableView view = (TableView) _db.getTable("FFTESTVIEW");
602: assertEquals("Valid row count should have correct value", 0,
603: view.getRowCount());
604:
605: table2.drop();
606: data.delete();
607: }
608:
609: public void testFileRead() throws Exception {
610: File data = new File(getDbdir(), "FFTEST.csv");
611: FileWriter out = new FileWriter(data);
612:
613: String eol = System.getProperty("line.separator");
614:
615: out.write("ID, NAME" + eol); // Header
616: out.write("1, aa" + eol); // 1
617: out.write("2.00, bbb" + eol); // 2
618: out.write("3.00, ccc" + eol); // 3
619: out.write("4.00, ddd" + eol); // 4
620: out.write("" + eol); // skip
621: out.write("we," + eol); // bad 1
622: out.write("7,dfdf" + eol); // 5
623: out.write("7.0f, ccc" + eol); // bad 3 ?
624: out.write("xx,xx" + eol); // bad 4
625: out.write("5, test" + eol); // 6
626: out.write("2004-10-10,hhhh" + eol); // bad 5
627: out.write("" + eol); // skip
628:
629: String name = "";
630: for (int i = 0; i < 20; i++) {
631: name += "cccdd";
632: }
633: out.write("3.00," + name + eol); // 7
634: out.write("" + eol); // skip
635: out.close();
636:
637: try {
638: ExternalTableFactory factory = new ExternalTableFactory();
639: Properties prop = setProperties("FFTEST");
640: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
641: prop.put(BaseFlatfileTable.PROP_ROWSTOSKIP, "1");
642:
643: Table table2 = factory.createTable(_db, "FFTEST", prop,
644: buildColumns());
645: RowIterator itr = table2.getRowIterator(false);
646:
647: int rowCount = 0;
648: while (itr.hasNext()) {
649: itr.next();
650: rowCount++;
651: }
652:
653: assertEquals("Valid row count should have correct value",
654: 7, rowCount);
655: table2.drop();
656:
657: prop.put(BaseFlatfileTable.PROP_MAXFAULTS, "2");
658:
659: try {
660: table2 = factory.createTable(_db, "FFTEST", prop,
661: buildColumns());
662: itr = table2.getRowIterator(false);
663: while (itr.hasNext()) {
664: itr.next();
665: }
666: fail("Expected Exception");
667: } catch (Exception e) {
668: // expected
669: }
670: table2.drop();
671:
672: // bad property value, shd use default value
673: prop.put(BaseFlatfileTable.PROP_MAXFAULTS, "-10");
674: prop.put(BaseFlatfileTable.PROP_ROWSTOSKIP, "-10");
675: table2 = factory.createTable(_db, "FFTEST", prop,
676: buildColumns());
677: itr = table2.getRowIterator(false);
678:
679: rowCount = 0;
680: while (itr.hasNext()) {
681: itr.next();
682: rowCount++;
683: }
684:
685: assertEquals("Valid row count should have correct value",
686: 7, rowCount);
687: table2.drop();
688:
689: // bad property value, shd use default value
690: prop.put(BaseFlatfileTable.PROP_MAXFAULTS, "ABC");
691: prop.put(BaseFlatfileTable.PROP_ROWSTOSKIP, "ABC");
692: table2 = factory.createTable(_db, "FFTEST", prop,
693: buildColumns());
694: itr = table2.getRowIterator(false);
695:
696: rowCount = 0;
697: while (itr.hasNext()) {
698: itr.next();
699: rowCount++;
700: }
701:
702: assertEquals("Valid row count should have correct value",
703: 7, rowCount);
704: table2.drop();
705:
706: prop.put(BaseFlatfileTable.PROP_MAXFAULTS, "6");
707: prop.put(BaseFlatfileTable.PROP_ROWSTOSKIP, "0");
708: table2 = factory.createTable(_db, "FFTEST", prop,
709: buildColumns());
710: itr = table2.getRowIterator(false);
711:
712: rowCount = 0;
713: while (itr.hasNext()) {
714: itr.next();
715: rowCount++;
716: }
717:
718: assertEquals("Valid row count should have correct value",
719: 7, rowCount);
720:
721: _db.addTable(table2);
722: CreateTableCommand cmd = new CreateTableCommand();
723: cmd.setObjectName("FFTEST2");
724: cmd.setType("external");
725: prop.put(BaseFlatfileTable.PROP_FILENAME, "FFTest2.csv");
726: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
727: cmd.setProperties(prop);
728: AxionQueryContext ctx = new AxionQueryContext();
729: ctx.addSelect(new ColumnIdentifier("*"));
730: ctx.addFrom(new TableIdentifier("FFTEST"));
731: SubSelectCommand subSelect = new SubSelectCommand(ctx);
732: cmd.setSubQuery(subSelect);
733: cmd.execute(_db);
734:
735: Table table3 = _db.getTable("FFTEST2");
736: itr = table3.getRowIterator(false);
737:
738: assertEquals("Valid row count should have correct value",
739: 7, table3.getRowCount());
740:
741: rowCount = 0;
742: while (itr.hasNext()) {
743: itr.next();
744: rowCount++;
745: }
746:
747: assertEquals("Valid row count should have correct value",
748: 7, rowCount);
749: } finally {
750: try {
751: _db.dropTable("FFTEST");
752: } catch (AxionException ignore) {
753: // ignore
754: }
755:
756: try {
757: _db.dropTable("FFTEST2");
758: } catch (AxionException ignore) {
759: // ignore
760: }
761:
762: if (data != null) {
763: data.delete();
764: }
765: }
766: }
767:
768: public void testRestartDB() throws Exception {
769: testAddRow();
770: table.shutdown();
771: table = null;
772: _db.shutdown();
773:
774: _db = new DiskDatabase(getDbdir());
775: assertTrue(_db.hasTable(getTableName()));
776: table = _db.getTable(getTableName());
777: testGetName();
778: RowIterator iter = table.getRowIterator(true);
779: assertNotNull(iter);
780: assertTrue(iter.hasNext());
781: assertNotNull(iter.next());
782: assertTrue(iter.hasNext());
783: assertNotNull(iter.next());
784: assertTrue(!iter.hasNext());
785: }
786:
787: public void testCharTokenizer() throws Exception {
788: final String eol = "" + Character.MAX_VALUE;
789: final String[] tokens = new String[] { "1",
790: "missing close quote" };
791:
792: String testStr = tokens[0] + ",\"" + tokens[1] + eol;
793:
794: ExternalTableFactory factory = new ExternalTableFactory();
795: Properties prop = setProperties("FFTEST");
796: prop.put(BaseFlatfileTable.PROP_ISFIRSTLINEHEADER, "true");
797: prop.put(BaseFlatfileTable.PROP_ROWSTOSKIP, "1");
798: prop.put(BaseFlatfileTable.PROP_RECORDDELIMITER, eol);
799: prop.put(DelimitedFlatfileTable.PROP_QUALIFIER, "\"");
800:
801: DelimitedFlatfileTable table2 = (DelimitedFlatfileTable) factory
802: .createTable(_db, "FFTEST", prop, buildColumns());
803: DelimitedFlatfileTable.CharTokenizer tokenizer = (table2).new CharTokenizer(
804: testStr.toCharArray(), ",");
805:
806: try {
807: int i = 0;
808: while (tokenizer.hasMoreTokens()) {
809: String currentToken = tokenizer.nextToken();
810: assertEquals(tokens[i++], currentToken);
811: }
812:
813: assertEquals("Did not parse expected number of tokens", 2,
814: i);
815: } finally {
816: table2.drop();
817: }
818: }
819:
820: private List buildColumns() {
821: List list = new ArrayList(2);
822: list.add(new Column("ID", new IntegerType()));
823: list.add(new Column("NAME", new CharacterVaryingType(100)));
824: return list;
825: }
826: }
|