001: package org.apache.lucene.index;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import java.io.IOException;
021: import java.io.File;
022:
023: import org.apache.lucene.util.LuceneTestCase;
024: import junit.framework.TestSuite;
025: import junit.textui.TestRunner;
026: import org.apache.lucene.store.IndexOutput;
027: import org.apache.lucene.store.Directory;
028: import org.apache.lucene.store.IndexInput;
029: import org.apache.lucene.store.FSDirectory;
030: import org.apache.lucene.store._TestHelper;
031: import org.apache.lucene.util._TestUtil;
032:
033: /**
034: *
035: * @version $Id: TestCompoundFile.java 602165 2007-12-07 17:42:33Z mikemccand $
036: */
037: public class TestCompoundFile extends LuceneTestCase {
038: /** Main for running test case by itself. */
039: public static void main(String args[]) {
040: TestRunner.run(new TestSuite(TestCompoundFile.class));
041: // TestRunner.run (new TestCompoundFile("testSingleFile"));
042: // TestRunner.run (new TestCompoundFile("testTwoFiles"));
043: // TestRunner.run (new TestCompoundFile("testRandomFiles"));
044: // TestRunner.run (new TestCompoundFile("testClonedStreamsClosing"));
045: // TestRunner.run (new TestCompoundFile("testReadAfterClose"));
046: // TestRunner.run (new TestCompoundFile("testRandomAccess"));
047: // TestRunner.run (new TestCompoundFile("testRandomAccessClones"));
048: // TestRunner.run (new TestCompoundFile("testFileNotFound"));
049: // TestRunner.run (new TestCompoundFile("testReadPastEOF"));
050:
051: // TestRunner.run (new TestCompoundFile("testIWCreate"));
052:
053: }
054:
055: private Directory dir;
056:
057: public void setUp() throws Exception {
058: super .setUp();
059: File file = new File(System.getProperty("tempDir"), "testIndex");
060: _TestUtil.rmDir(file);
061: dir = FSDirectory.getDirectory(file);
062: }
063:
064: /** Creates a file of the specified size with random data. */
065: private void createRandomFile(Directory dir, String name, int size)
066: throws IOException {
067: IndexOutput os = dir.createOutput(name);
068: for (int i = 0; i < size; i++) {
069: byte b = (byte) (Math.random() * 256);
070: os.writeByte(b);
071: }
072: os.close();
073: }
074:
075: /** Creates a file of the specified size with sequential data. The first
076: * byte is written as the start byte provided. All subsequent bytes are
077: * computed as start + offset where offset is the number of the byte.
078: */
079: private void createSequenceFile(Directory dir, String name,
080: byte start, int size) throws IOException {
081: IndexOutput os = dir.createOutput(name);
082: for (int i = 0; i < size; i++) {
083: os.writeByte(start);
084: start++;
085: }
086: os.close();
087: }
088:
089: private void assertSameStreams(String msg, IndexInput expected,
090: IndexInput test) throws IOException {
091: assertNotNull(msg + " null expected", expected);
092: assertNotNull(msg + " null test", test);
093: assertEquals(msg + " length", expected.length(), test.length());
094: assertEquals(msg + " position", expected.getFilePointer(), test
095: .getFilePointer());
096:
097: byte expectedBuffer[] = new byte[512];
098: byte testBuffer[] = new byte[expectedBuffer.length];
099:
100: long remainder = expected.length() - expected.getFilePointer();
101: while (remainder > 0) {
102: int readLen = (int) Math.min(remainder,
103: expectedBuffer.length);
104: expected.readBytes(expectedBuffer, 0, readLen);
105: test.readBytes(testBuffer, 0, readLen);
106: assertEqualArrays(msg + ", remainder " + remainder,
107: expectedBuffer, testBuffer, 0, readLen);
108: remainder -= readLen;
109: }
110: }
111:
112: private void assertSameStreams(String msg, IndexInput expected,
113: IndexInput actual, long seekTo) throws IOException {
114: if (seekTo >= 0 && seekTo < expected.length()) {
115: expected.seek(seekTo);
116: actual.seek(seekTo);
117: assertSameStreams(msg + ", seek(mid)", expected, actual);
118: }
119: }
120:
121: private void assertSameSeekBehavior(String msg,
122: IndexInput expected, IndexInput actual) throws IOException {
123: // seek to 0
124: long point = 0;
125: assertSameStreams(msg + ", seek(0)", expected, actual, point);
126:
127: // seek to middle
128: point = expected.length() / 2l;
129: assertSameStreams(msg + ", seek(mid)", expected, actual, point);
130:
131: // seek to end - 2
132: point = expected.length() - 2;
133: assertSameStreams(msg + ", seek(end-2)", expected, actual,
134: point);
135:
136: // seek to end - 1
137: point = expected.length() - 1;
138: assertSameStreams(msg + ", seek(end-1)", expected, actual,
139: point);
140:
141: // seek to the end
142: point = expected.length();
143: assertSameStreams(msg + ", seek(end)", expected, actual, point);
144:
145: // seek past end
146: point = expected.length() + 1;
147: assertSameStreams(msg + ", seek(end+1)", expected, actual,
148: point);
149: }
150:
151: private void assertEqualArrays(String msg, byte[] expected,
152: byte[] test, int start, int len) {
153: assertNotNull(msg + " null expected", expected);
154: assertNotNull(msg + " null test", test);
155:
156: for (int i = start; i < len; i++) {
157: assertEquals(msg + " " + i, expected[i], test[i]);
158: }
159: }
160:
161: // ===========================================================
162: // Tests of the basic CompoundFile functionality
163: // ===========================================================
164:
165: /** This test creates compound file based on a single file.
166: * Files of different sizes are tested: 0, 1, 10, 100 bytes.
167: */
168: public void testSingleFile() throws IOException {
169: int data[] = new int[] { 0, 1, 10, 100 };
170: for (int i = 0; i < data.length; i++) {
171: String name = "t" + data[i];
172: createSequenceFile(dir, name, (byte) 0, data[i]);
173: CompoundFileWriter csw = new CompoundFileWriter(dir, name
174: + ".cfs");
175: csw.addFile(name);
176: csw.close();
177:
178: CompoundFileReader csr = new CompoundFileReader(dir, name
179: + ".cfs");
180: IndexInput expected = dir.openInput(name);
181: IndexInput actual = csr.openInput(name);
182: assertSameStreams(name, expected, actual);
183: assertSameSeekBehavior(name, expected, actual);
184: expected.close();
185: actual.close();
186: csr.close();
187: }
188: }
189:
190: /** This test creates compound file based on two files.
191: *
192: */
193: public void testTwoFiles() throws IOException {
194: createSequenceFile(dir, "d1", (byte) 0, 15);
195: createSequenceFile(dir, "d2", (byte) 0, 114);
196:
197: CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
198: csw.addFile("d1");
199: csw.addFile("d2");
200: csw.close();
201:
202: CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
203: IndexInput expected = dir.openInput("d1");
204: IndexInput actual = csr.openInput("d1");
205: assertSameStreams("d1", expected, actual);
206: assertSameSeekBehavior("d1", expected, actual);
207: expected.close();
208: actual.close();
209:
210: expected = dir.openInput("d2");
211: actual = csr.openInput("d2");
212: assertSameStreams("d2", expected, actual);
213: assertSameSeekBehavior("d2", expected, actual);
214: expected.close();
215: actual.close();
216: csr.close();
217: }
218:
219: /** This test creates a compound file based on a large number of files of
220: * various length. The file content is generated randomly. The sizes range
221: * from 0 to 1Mb. Some of the sizes are selected to test the buffering
222: * logic in the file reading code. For this the chunk variable is set to
223: * the length of the buffer used internally by the compound file logic.
224: */
225: public void testRandomFiles() throws IOException {
226: // Setup the test segment
227: String segment = "test";
228: int chunk = 1024; // internal buffer size used by the stream
229: createRandomFile(dir, segment + ".zero", 0);
230: createRandomFile(dir, segment + ".one", 1);
231: createRandomFile(dir, segment + ".ten", 10);
232: createRandomFile(dir, segment + ".hundred", 100);
233: createRandomFile(dir, segment + ".big1", chunk);
234: createRandomFile(dir, segment + ".big2", chunk - 1);
235: createRandomFile(dir, segment + ".big3", chunk + 1);
236: createRandomFile(dir, segment + ".big4", 3 * chunk);
237: createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
238: createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
239: createRandomFile(dir, segment + ".big7", 1000 * chunk);
240:
241: // Setup extraneous files
242: createRandomFile(dir, "onetwothree", 100);
243: createRandomFile(dir, segment + ".notIn", 50);
244: createRandomFile(dir, segment + ".notIn2", 51);
245:
246: // Now test
247: CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
248: final String data[] = new String[] { ".zero", ".one", ".ten",
249: ".hundred", ".big1", ".big2", ".big3", ".big4",
250: ".big5", ".big6", ".big7" };
251: for (int i = 0; i < data.length; i++) {
252: csw.addFile(segment + data[i]);
253: }
254: csw.close();
255:
256: CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
257: for (int i = 0; i < data.length; i++) {
258: IndexInput check = dir.openInput(segment + data[i]);
259: IndexInput test = csr.openInput(segment + data[i]);
260: assertSameStreams(data[i], check, test);
261: assertSameSeekBehavior(data[i], check, test);
262: test.close();
263: check.close();
264: }
265: csr.close();
266: }
267:
268: /** Setup a larger compound file with a number of components, each of
269: * which is a sequential file (so that we can easily tell that we are
270: * reading in the right byte). The methods sets up 20 files - f0 to f19,
271: * the size of each file is 1000 bytes.
272: */
273: private void setUp_2() throws IOException {
274: CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
275: for (int i = 0; i < 20; i++) {
276: createSequenceFile(dir, "f" + i, (byte) 0, 2000);
277: cw.addFile("f" + i);
278: }
279: cw.close();
280: }
281:
282: public void testReadAfterClose() throws IOException {
283: demo_FSIndexInputBug((FSDirectory) dir, "test");
284: }
285:
286: private void demo_FSIndexInputBug(FSDirectory fsdir, String file)
287: throws IOException {
288: // Setup the test file - we need more than 1024 bytes
289: IndexOutput os = fsdir.createOutput(file);
290: for (int i = 0; i < 2000; i++) {
291: os.writeByte((byte) i);
292: }
293: os.close();
294:
295: IndexInput in = fsdir.openInput(file);
296:
297: // This read primes the buffer in IndexInput
298: byte b = in.readByte();
299:
300: // Close the file
301: in.close();
302:
303: // ERROR: this call should fail, but succeeds because the buffer
304: // is still filled
305: b = in.readByte();
306:
307: // ERROR: this call should fail, but succeeds for some reason as well
308: in.seek(1099);
309:
310: try {
311: // OK: this call correctly fails. We are now past the 1024 internal
312: // buffer, so an actual IO is attempted, which fails
313: b = in.readByte();
314: fail("expected readByte() to throw exception");
315: } catch (IOException e) {
316: // expected exception
317: }
318: }
319:
320: static boolean isCSIndexInput(IndexInput is) {
321: return is instanceof CompoundFileReader.CSIndexInput;
322: }
323:
324: static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
325: if (isCSIndexInput(is)) {
326: CompoundFileReader.CSIndexInput cis = (CompoundFileReader.CSIndexInput) is;
327:
328: return _TestHelper.isFSIndexInputOpen(cis.base);
329: } else {
330: return false;
331: }
332: }
333:
334: public void testClonedStreamsClosing() throws IOException {
335: setUp_2();
336: CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
337:
338: // basic clone
339: IndexInput expected = dir.openInput("f11");
340:
341: // this test only works for FSIndexInput
342: if (_TestHelper.isFSIndexInput(expected)) {
343:
344: assertTrue(_TestHelper.isFSIndexInputOpen(expected));
345:
346: IndexInput one = cr.openInput("f11");
347: assertTrue(isCSIndexInputOpen(one));
348:
349: IndexInput two = (IndexInput) one.clone();
350: assertTrue(isCSIndexInputOpen(two));
351:
352: assertSameStreams("basic clone one", expected, one);
353: expected.seek(0);
354: assertSameStreams("basic clone two", expected, two);
355:
356: // Now close the first stream
357: one.close();
358: assertTrue("Only close when cr is closed",
359: isCSIndexInputOpen(one));
360:
361: // The following should really fail since we couldn't expect to
362: // access a file once close has been called on it (regardless of
363: // buffering and/or clone magic)
364: expected.seek(0);
365: two.seek(0);
366: assertSameStreams("basic clone two/2", expected, two);
367:
368: // Now close the compound reader
369: cr.close();
370: assertFalse("Now closed one", isCSIndexInputOpen(one));
371: assertFalse("Now closed two", isCSIndexInputOpen(two));
372:
373: // The following may also fail since the compound stream is closed
374: expected.seek(0);
375: two.seek(0);
376: //assertSameStreams("basic clone two/3", expected, two);
377:
378: // Now close the second clone
379: two.close();
380: expected.seek(0);
381: two.seek(0);
382: //assertSameStreams("basic clone two/4", expected, two);
383: }
384:
385: expected.close();
386: }
387:
388: /** This test opens two files from a compound stream and verifies that
389: * their file positions are independent of each other.
390: */
391: public void testRandomAccess() throws IOException {
392: setUp_2();
393: CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
394:
395: // Open two files
396: IndexInput e1 = dir.openInput("f11");
397: IndexInput e2 = dir.openInput("f3");
398:
399: IndexInput a1 = cr.openInput("f11");
400: IndexInput a2 = dir.openInput("f3");
401:
402: // Seek the first pair
403: e1.seek(100);
404: a1.seek(100);
405: assertEquals(100, e1.getFilePointer());
406: assertEquals(100, a1.getFilePointer());
407: byte be1 = e1.readByte();
408: byte ba1 = a1.readByte();
409: assertEquals(be1, ba1);
410:
411: // Now seek the second pair
412: e2.seek(1027);
413: a2.seek(1027);
414: assertEquals(1027, e2.getFilePointer());
415: assertEquals(1027, a2.getFilePointer());
416: byte be2 = e2.readByte();
417: byte ba2 = a2.readByte();
418: assertEquals(be2, ba2);
419:
420: // Now make sure the first one didn't move
421: assertEquals(101, e1.getFilePointer());
422: assertEquals(101, a1.getFilePointer());
423: be1 = e1.readByte();
424: ba1 = a1.readByte();
425: assertEquals(be1, ba1);
426:
427: // Now more the first one again, past the buffer length
428: e1.seek(1910);
429: a1.seek(1910);
430: assertEquals(1910, e1.getFilePointer());
431: assertEquals(1910, a1.getFilePointer());
432: be1 = e1.readByte();
433: ba1 = a1.readByte();
434: assertEquals(be1, ba1);
435:
436: // Now make sure the second set didn't move
437: assertEquals(1028, e2.getFilePointer());
438: assertEquals(1028, a2.getFilePointer());
439: be2 = e2.readByte();
440: ba2 = a2.readByte();
441: assertEquals(be2, ba2);
442:
443: // Move the second set back, again cross the buffer size
444: e2.seek(17);
445: a2.seek(17);
446: assertEquals(17, e2.getFilePointer());
447: assertEquals(17, a2.getFilePointer());
448: be2 = e2.readByte();
449: ba2 = a2.readByte();
450: assertEquals(be2, ba2);
451:
452: // Finally, make sure the first set didn't move
453: // Now make sure the first one didn't move
454: assertEquals(1911, e1.getFilePointer());
455: assertEquals(1911, a1.getFilePointer());
456: be1 = e1.readByte();
457: ba1 = a1.readByte();
458: assertEquals(be1, ba1);
459:
460: e1.close();
461: e2.close();
462: a1.close();
463: a2.close();
464: cr.close();
465: }
466:
467: /** This test opens two files from a compound stream and verifies that
468: * their file positions are independent of each other.
469: */
470: public void testRandomAccessClones() throws IOException {
471: setUp_2();
472: CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
473:
474: // Open two files
475: IndexInput e1 = cr.openInput("f11");
476: IndexInput e2 = cr.openInput("f3");
477:
478: IndexInput a1 = (IndexInput) e1.clone();
479: IndexInput a2 = (IndexInput) e2.clone();
480:
481: // Seek the first pair
482: e1.seek(100);
483: a1.seek(100);
484: assertEquals(100, e1.getFilePointer());
485: assertEquals(100, a1.getFilePointer());
486: byte be1 = e1.readByte();
487: byte ba1 = a1.readByte();
488: assertEquals(be1, ba1);
489:
490: // Now seek the second pair
491: e2.seek(1027);
492: a2.seek(1027);
493: assertEquals(1027, e2.getFilePointer());
494: assertEquals(1027, a2.getFilePointer());
495: byte be2 = e2.readByte();
496: byte ba2 = a2.readByte();
497: assertEquals(be2, ba2);
498:
499: // Now make sure the first one didn't move
500: assertEquals(101, e1.getFilePointer());
501: assertEquals(101, a1.getFilePointer());
502: be1 = e1.readByte();
503: ba1 = a1.readByte();
504: assertEquals(be1, ba1);
505:
506: // Now more the first one again, past the buffer length
507: e1.seek(1910);
508: a1.seek(1910);
509: assertEquals(1910, e1.getFilePointer());
510: assertEquals(1910, a1.getFilePointer());
511: be1 = e1.readByte();
512: ba1 = a1.readByte();
513: assertEquals(be1, ba1);
514:
515: // Now make sure the second set didn't move
516: assertEquals(1028, e2.getFilePointer());
517: assertEquals(1028, a2.getFilePointer());
518: be2 = e2.readByte();
519: ba2 = a2.readByte();
520: assertEquals(be2, ba2);
521:
522: // Move the second set back, again cross the buffer size
523: e2.seek(17);
524: a2.seek(17);
525: assertEquals(17, e2.getFilePointer());
526: assertEquals(17, a2.getFilePointer());
527: be2 = e2.readByte();
528: ba2 = a2.readByte();
529: assertEquals(be2, ba2);
530:
531: // Finally, make sure the first set didn't move
532: // Now make sure the first one didn't move
533: assertEquals(1911, e1.getFilePointer());
534: assertEquals(1911, a1.getFilePointer());
535: be1 = e1.readByte();
536: ba1 = a1.readByte();
537: assertEquals(be1, ba1);
538:
539: e1.close();
540: e2.close();
541: a1.close();
542: a2.close();
543: cr.close();
544: }
545:
546: public void testFileNotFound() throws IOException {
547: setUp_2();
548: CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
549:
550: // Open two files
551: try {
552: IndexInput e1 = cr.openInput("bogus");
553: fail("File not found");
554:
555: } catch (IOException e) {
556: /* success */
557: //System.out.println("SUCCESS: File Not Found: " + e);
558: }
559:
560: cr.close();
561: }
562:
563: public void testReadPastEOF() throws IOException {
564: setUp_2();
565: CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
566: IndexInput is = cr.openInput("f2");
567: is.seek(is.length() - 10);
568: byte b[] = new byte[100];
569: is.readBytes(b, 0, 10);
570:
571: try {
572: byte test = is.readByte();
573: fail("Single byte read past end of file");
574: } catch (IOException e) {
575: /* success */
576: //System.out.println("SUCCESS: single byte read past end of file: " + e);
577: }
578:
579: is.seek(is.length() - 10);
580: try {
581: is.readBytes(b, 0, 50);
582: fail("Block read past end of file");
583: } catch (IOException e) {
584: /* success */
585: //System.out.println("SUCCESS: block read past end of file: " + e);
586: }
587:
588: is.close();
589: cr.close();
590: }
591:
592: /** This test that writes larger than the size of the buffer output
593: * will correctly increment the file pointer.
594: */
595: public void testLargeWrites() throws IOException {
596: IndexOutput os = dir.createOutput("testBufferStart.txt");
597:
598: byte[] largeBuf = new byte[2048];
599: for (int i = 0; i < largeBuf.length; i++) {
600: largeBuf[i] = (byte) (Math.random() * 256);
601: }
602:
603: long currentPos = os.getFilePointer();
604: os.writeBytes(largeBuf, largeBuf.length);
605:
606: try {
607: assertEquals(currentPos + largeBuf.length, os
608: .getFilePointer());
609: } finally {
610: os.close();
611: }
612:
613: }
614: }
|