0001: package org.apache.lucene.index;
0002:
0003: /**
0004: * Licensed to the Apache Software Foundation (ASF) under one or more
0005: * contributor license agreements. See the NOTICE file distributed with
0006: * this work for additional information regarding copyright ownership.
0007: * The ASF licenses this file to You under the Apache License, Version 2.0
0008: * (the "License"); you may not use this file except in compliance with
0009: * the License. You may obtain a copy of the License at
0010: *
0011: * http://www.apache.org/licenses/LICENSE-2.0
0012: *
0013: * Unless required by applicable law or agreed to in writing, software
0014: * distributed under the License is distributed on an "AS IS" BASIS,
0015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016: * See the License for the specific language governing permissions and
0017: * limitations under the License.
0018: */
0019:
0020: import org.apache.lucene.util.LuceneTestCase;
0021: import junit.framework.TestSuite;
0022: import junit.textui.TestRunner;
0023: import org.apache.lucene.analysis.WhitespaceAnalyzer;
0024: import org.apache.lucene.analysis.standard.StandardAnalyzer;
0025: import org.apache.lucene.document.Document;
0026: import org.apache.lucene.document.Field;
0027: import org.apache.lucene.index.IndexReader.FieldOption;
0028: import org.apache.lucene.search.Hits;
0029: import org.apache.lucene.search.IndexSearcher;
0030: import org.apache.lucene.search.TermQuery;
0031: import org.apache.lucene.store.*;
0032: import org.apache.lucene.util._TestUtil;
0033:
0034: import java.io.File;
0035: import java.io.FileNotFoundException;
0036: import java.io.IOException;
0037: import java.util.*;
0038:
0039: public class TestIndexReader extends LuceneTestCase {
0040: /** Main for running test case by itself. */
0041: public static void main(String args[]) {
0042: TestRunner.run(new TestSuite(TestIndexReader.class));
0043: // TestRunner.run (new TestIndexReader("testBasicDelete"));
0044: // TestRunner.run (new TestIndexReader("testDeleteReaderWriterConflict"));
0045: // TestRunner.run (new TestIndexReader("testDeleteReaderReaderConflict"));
0046: // TestRunner.run (new TestIndexReader("testFilesOpenClose"));
0047: }
0048:
0049: public TestIndexReader(String name) {
0050: super (name);
0051: }
0052:
0053: public void testIsCurrent() throws Exception {
0054: RAMDirectory d = new MockRAMDirectory();
0055: IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(),
0056: true);
0057: addDocumentWithFields(writer);
0058: writer.close();
0059: // set up reader:
0060: IndexReader reader = IndexReader.open(d);
0061: assertTrue(reader.isCurrent());
0062: // modify index by adding another document:
0063: writer = new IndexWriter(d, new StandardAnalyzer(), false);
0064: addDocumentWithFields(writer);
0065: writer.close();
0066: assertFalse(reader.isCurrent());
0067: // re-create index:
0068: writer = new IndexWriter(d, new StandardAnalyzer(), true);
0069: addDocumentWithFields(writer);
0070: writer.close();
0071: assertFalse(reader.isCurrent());
0072: reader.close();
0073: d.close();
0074: }
0075:
0076: /**
0077: * Tests the IndexReader.getFieldNames implementation
0078: * @throws Exception on error
0079: */
0080: public void testGetFieldNames() throws Exception {
0081: RAMDirectory d = new MockRAMDirectory();
0082: // set up writer
0083: IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(),
0084: true);
0085: addDocumentWithFields(writer);
0086: writer.close();
0087: // set up reader
0088: IndexReader reader = IndexReader.open(d);
0089: Collection fieldNames = reader
0090: .getFieldNames(IndexReader.FieldOption.ALL);
0091: assertTrue(fieldNames.contains("keyword"));
0092: assertTrue(fieldNames.contains("text"));
0093: assertTrue(fieldNames.contains("unindexed"));
0094: assertTrue(fieldNames.contains("unstored"));
0095: reader.close();
0096: // add more documents
0097: writer = new IndexWriter(d, new StandardAnalyzer(), false);
0098: // want to get some more segments here
0099: for (int i = 0; i < 5 * writer.getMergeFactor(); i++) {
0100: addDocumentWithFields(writer);
0101: }
0102: // new fields are in some different segments (we hope)
0103: for (int i = 0; i < 5 * writer.getMergeFactor(); i++) {
0104: addDocumentWithDifferentFields(writer);
0105: }
0106: // new termvector fields
0107: for (int i = 0; i < 5 * writer.getMergeFactor(); i++) {
0108: addDocumentWithTermVectorFields(writer);
0109: }
0110:
0111: writer.close();
0112: // verify fields again
0113: reader = IndexReader.open(d);
0114: fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
0115: assertEquals(13, fieldNames.size()); // the following fields
0116: assertTrue(fieldNames.contains("keyword"));
0117: assertTrue(fieldNames.contains("text"));
0118: assertTrue(fieldNames.contains("unindexed"));
0119: assertTrue(fieldNames.contains("unstored"));
0120: assertTrue(fieldNames.contains("keyword2"));
0121: assertTrue(fieldNames.contains("text2"));
0122: assertTrue(fieldNames.contains("unindexed2"));
0123: assertTrue(fieldNames.contains("unstored2"));
0124: assertTrue(fieldNames.contains("tvnot"));
0125: assertTrue(fieldNames.contains("termvector"));
0126: assertTrue(fieldNames.contains("tvposition"));
0127: assertTrue(fieldNames.contains("tvoffset"));
0128: assertTrue(fieldNames.contains("tvpositionoffset"));
0129:
0130: // verify that only indexed fields were returned
0131: fieldNames = reader
0132: .getFieldNames(IndexReader.FieldOption.INDEXED);
0133: assertEquals(11, fieldNames.size()); // 6 original + the 5 termvector fields
0134: assertTrue(fieldNames.contains("keyword"));
0135: assertTrue(fieldNames.contains("text"));
0136: assertTrue(fieldNames.contains("unstored"));
0137: assertTrue(fieldNames.contains("keyword2"));
0138: assertTrue(fieldNames.contains("text2"));
0139: assertTrue(fieldNames.contains("unstored2"));
0140: assertTrue(fieldNames.contains("tvnot"));
0141: assertTrue(fieldNames.contains("termvector"));
0142: assertTrue(fieldNames.contains("tvposition"));
0143: assertTrue(fieldNames.contains("tvoffset"));
0144: assertTrue(fieldNames.contains("tvpositionoffset"));
0145:
0146: // verify that only unindexed fields were returned
0147: fieldNames = reader
0148: .getFieldNames(IndexReader.FieldOption.UNINDEXED);
0149: assertEquals(2, fieldNames.size()); // the following fields
0150: assertTrue(fieldNames.contains("unindexed"));
0151: assertTrue(fieldNames.contains("unindexed2"));
0152:
0153: // verify index term vector fields
0154: fieldNames = reader
0155: .getFieldNames(IndexReader.FieldOption.TERMVECTOR);
0156: assertEquals(1, fieldNames.size()); // 1 field has term vector only
0157: assertTrue(fieldNames.contains("termvector"));
0158:
0159: fieldNames = reader
0160: .getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION);
0161: assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
0162: assertTrue(fieldNames.contains("tvposition"));
0163:
0164: fieldNames = reader
0165: .getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET);
0166: assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
0167: assertTrue(fieldNames.contains("tvoffset"));
0168:
0169: fieldNames = reader
0170: .getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET);
0171: assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
0172: assertTrue(fieldNames.contains("tvpositionoffset"));
0173: reader.close();
0174: d.close();
0175: }
0176:
0177: public void testTermVectors() throws Exception {
0178: RAMDirectory d = new MockRAMDirectory();
0179: // set up writer
0180: IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(),
0181: true);
0182: // want to get some more segments here
0183: // new termvector fields
0184: for (int i = 0; i < 5 * writer.getMergeFactor(); i++) {
0185: Document doc = new Document();
0186: doc.add(new Field("tvnot", "one two two three three three",
0187: Field.Store.YES, Field.Index.TOKENIZED,
0188: Field.TermVector.NO));
0189: doc.add(new Field("termvector",
0190: "one two two three three three", Field.Store.YES,
0191: Field.Index.TOKENIZED, Field.TermVector.YES));
0192: doc.add(new Field("tvoffset",
0193: "one two two three three three", Field.Store.YES,
0194: Field.Index.TOKENIZED,
0195: Field.TermVector.WITH_OFFSETS));
0196: doc.add(new Field("tvposition",
0197: "one two two three three three", Field.Store.YES,
0198: Field.Index.TOKENIZED,
0199: Field.TermVector.WITH_POSITIONS));
0200: doc.add(new Field("tvpositionoffset",
0201: "one two two three three three", Field.Store.YES,
0202: Field.Index.TOKENIZED,
0203: Field.TermVector.WITH_POSITIONS_OFFSETS));
0204:
0205: writer.addDocument(doc);
0206: }
0207: writer.close();
0208: IndexReader reader = IndexReader.open(d);
0209: FieldSortedTermVectorMapper mapper = new FieldSortedTermVectorMapper(
0210: new TermVectorEntryFreqSortedComparator());
0211: reader.getTermFreqVector(0, mapper);
0212: Map map = mapper.getFieldToTerms();
0213: assertTrue("map is null and it shouldn't be", map != null);
0214: assertTrue("map Size: " + map.size() + " is not: " + 4, map
0215: .size() == 4);
0216: Set set = (Set) map.get("termvector");
0217: for (Iterator iterator = set.iterator(); iterator.hasNext();) {
0218: TermVectorEntry entry = (TermVectorEntry) iterator.next();
0219: assertTrue("entry is null and it shouldn't be",
0220: entry != null);
0221: System.out.println("Entry: " + entry);
0222: }
0223:
0224: }
0225:
0226: private void assertTermDocsCount(String msg, IndexReader reader,
0227: Term term, int expected) throws IOException {
0228: TermDocs tdocs = null;
0229:
0230: try {
0231: tdocs = reader.termDocs(term);
0232: assertNotNull(msg + ", null TermDocs", tdocs);
0233: int count = 0;
0234: while (tdocs.next()) {
0235: count++;
0236: }
0237: assertEquals(msg + ", count mismatch", expected, count);
0238:
0239: } finally {
0240: if (tdocs != null)
0241: tdocs.close();
0242: }
0243:
0244: }
0245:
0246: public void testBasicDelete() throws IOException {
0247: Directory dir = new MockRAMDirectory();
0248:
0249: IndexWriter writer = null;
0250: IndexReader reader = null;
0251: Term searchTerm = new Term("content", "aaa");
0252:
0253: // add 100 documents with term : aaa
0254: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0255: for (int i = 0; i < 100; i++) {
0256: addDoc(writer, searchTerm.text());
0257: }
0258: writer.close();
0259:
0260: // OPEN READER AT THIS POINT - this should fix the view of the
0261: // index at the point of having 100 "aaa" documents and 0 "bbb"
0262: reader = IndexReader.open(dir);
0263: assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
0264: assertTermDocsCount("first reader", reader, searchTerm, 100);
0265: reader.close();
0266:
0267: // DELETE DOCUMENTS CONTAINING TERM: aaa
0268: int deleted = 0;
0269: reader = IndexReader.open(dir);
0270: deleted = reader.deleteDocuments(searchTerm);
0271: assertEquals("deleted count", 100, deleted);
0272: assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
0273: assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
0274:
0275: // open a 2nd reader to make sure first reader can
0276: // commit its changes (.del) while second reader
0277: // is open:
0278: IndexReader reader2 = IndexReader.open(dir);
0279: reader.close();
0280:
0281: // CREATE A NEW READER and re-test
0282: reader = IndexReader.open(dir);
0283: assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
0284: assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
0285: reader.close();
0286: reader2.close();
0287: dir.close();
0288: }
0289:
0290: // Make sure attempts to make changes after reader is
0291: // closed throws IOException:
0292: public void testChangesAfterClose() throws IOException {
0293: Directory dir = new RAMDirectory();
0294:
0295: IndexWriter writer = null;
0296: IndexReader reader = null;
0297: Term searchTerm = new Term("content", "aaa");
0298:
0299: // add 11 documents with term : aaa
0300: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0301: for (int i = 0; i < 11; i++) {
0302: addDoc(writer, searchTerm.text());
0303: }
0304: writer.close();
0305:
0306: reader = IndexReader.open(dir);
0307:
0308: // Close reader:
0309: reader.close();
0310:
0311: // Then, try to make changes:
0312: try {
0313: reader.deleteDocument(4);
0314: fail("deleteDocument after close failed to throw IOException");
0315: } catch (AlreadyClosedException e) {
0316: // expected
0317: }
0318:
0319: try {
0320: reader.setNorm(5, "aaa", 2.0f);
0321: fail("setNorm after close failed to throw IOException");
0322: } catch (AlreadyClosedException e) {
0323: // expected
0324: }
0325:
0326: try {
0327: reader.undeleteAll();
0328: fail("undeleteAll after close failed to throw IOException");
0329: } catch (AlreadyClosedException e) {
0330: // expected
0331: }
0332: }
0333:
0334: // Make sure we get lock obtain failed exception with 2 writers:
0335: public void testLockObtainFailed() throws IOException {
0336: Directory dir = new RAMDirectory();
0337:
0338: IndexWriter writer = null;
0339: IndexReader reader = null;
0340: Term searchTerm = new Term("content", "aaa");
0341:
0342: // add 11 documents with term : aaa
0343: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0344: for (int i = 0; i < 11; i++) {
0345: addDoc(writer, searchTerm.text());
0346: }
0347:
0348: // Create reader:
0349: reader = IndexReader.open(dir);
0350:
0351: // Try to make changes
0352: try {
0353: reader.deleteDocument(4);
0354: fail("deleteDocument should have hit LockObtainFailedException");
0355: } catch (LockObtainFailedException e) {
0356: // expected
0357: }
0358:
0359: try {
0360: reader.setNorm(5, "aaa", 2.0f);
0361: fail("setNorm should have hit LockObtainFailedException");
0362: } catch (LockObtainFailedException e) {
0363: // expected
0364: }
0365:
0366: try {
0367: reader.undeleteAll();
0368: fail("undeleteAll should have hit LockObtainFailedException");
0369: } catch (LockObtainFailedException e) {
0370: // expected
0371: }
0372: writer.close();
0373: reader.close();
0374: }
0375:
0376: // Make sure you can set norms & commit even if a reader
0377: // is open against the index:
0378: public void testWritingNorms() throws IOException {
0379: String tempDir = System.getProperty("tempDir");
0380: if (tempDir == null)
0381: throw new IOException("tempDir undefined, cannot run test");
0382:
0383: File indexDir = new File(tempDir, "lucenetestnormwriter");
0384: Directory dir = FSDirectory.getDirectory(indexDir);
0385: IndexWriter writer;
0386: IndexReader reader;
0387: Term searchTerm = new Term("content", "aaa");
0388:
0389: // add 1 documents with term : aaa
0390: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0391: addDoc(writer, searchTerm.text());
0392: writer.close();
0393:
0394: // now open reader & set norm for doc 0
0395: reader = IndexReader.open(dir);
0396: reader.setNorm(0, "content", (float) 2.0);
0397:
0398: // we should be holding the write lock now:
0399: assertTrue("locked", IndexReader.isLocked(dir));
0400:
0401: reader.commit();
0402:
0403: // we should not be holding the write lock now:
0404: assertTrue("not locked", !IndexReader.isLocked(dir));
0405:
0406: // open a 2nd reader:
0407: IndexReader reader2 = IndexReader.open(dir);
0408:
0409: // set norm again for doc 0
0410: reader.setNorm(0, "content", (float) 3.0);
0411: assertTrue("locked", IndexReader.isLocked(dir));
0412:
0413: reader.close();
0414:
0415: // we should not be holding the write lock now:
0416: assertTrue("not locked", !IndexReader.isLocked(dir));
0417:
0418: reader2.close();
0419: dir.close();
0420:
0421: rmDir(indexDir);
0422: }
0423:
0424: // Make sure you can set norms & commit, and there are
0425: // no extra norms files left:
0426: public void testWritingNormsNoReader() throws IOException {
0427: Directory dir = new MockRAMDirectory();
0428: IndexWriter writer = null;
0429: IndexReader reader = null;
0430: Term searchTerm = new Term("content", "aaa");
0431:
0432: // add 1 documents with term : aaa
0433: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0434: writer.setUseCompoundFile(false);
0435: addDoc(writer, searchTerm.text());
0436: writer.close();
0437:
0438: // now open reader & set norm for doc 0 (writes to
0439: // _0_1.s0)
0440: reader = IndexReader.open(dir);
0441: reader.setNorm(0, "content", (float) 2.0);
0442: reader.close();
0443:
0444: // now open reader again & set norm for doc 0 (writes to _0_2.s0)
0445: reader = IndexReader.open(dir);
0446: reader.setNorm(0, "content", (float) 2.0);
0447: reader.close();
0448: assertFalse(
0449: "failed to remove first generation norms file on writing second generation",
0450: dir.fileExists("_0_1.s0"));
0451:
0452: dir.close();
0453: }
0454:
0455: public void testDeleteReaderWriterConflictUnoptimized()
0456: throws IOException {
0457: deleteReaderWriterConflict(false);
0458: }
0459:
0460: public void testOpenEmptyDirectory() throws IOException {
0461: String dirName = "test.empty";
0462: File fileDirName = new File(dirName);
0463: if (!fileDirName.exists()) {
0464: fileDirName.mkdir();
0465: }
0466: try {
0467: IndexReader reader = IndexReader.open(fileDirName);
0468: fail("opening IndexReader on empty directory failed to produce FileNotFoundException");
0469: } catch (FileNotFoundException e) {
0470: // GOOD
0471: }
0472: rmDir(fileDirName);
0473: }
0474:
0475: public void testDeleteReaderWriterConflictOptimized()
0476: throws IOException {
0477: deleteReaderWriterConflict(true);
0478: }
0479:
0480: private void deleteReaderWriterConflict(boolean optimize)
0481: throws IOException {
0482: //Directory dir = new RAMDirectory();
0483: Directory dir = getDirectory();
0484:
0485: Term searchTerm = new Term("content", "aaa");
0486: Term searchTerm2 = new Term("content", "bbb");
0487:
0488: // add 100 documents with term : aaa
0489: IndexWriter writer = new IndexWriter(dir,
0490: new WhitespaceAnalyzer(), true);
0491: for (int i = 0; i < 100; i++) {
0492: addDoc(writer, searchTerm.text());
0493: }
0494: writer.close();
0495:
0496: // OPEN READER AT THIS POINT - this should fix the view of the
0497: // index at the point of having 100 "aaa" documents and 0 "bbb"
0498: IndexReader reader = IndexReader.open(dir);
0499: assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
0500: assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
0501: assertTermDocsCount("first reader", reader, searchTerm, 100);
0502: assertTermDocsCount("first reader", reader, searchTerm2, 0);
0503:
0504: // add 100 documents with term : bbb
0505: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
0506: for (int i = 0; i < 100; i++) {
0507: addDoc(writer, searchTerm2.text());
0508: }
0509:
0510: // REQUEST OPTIMIZATION
0511: // This causes a new segment to become current for all subsequent
0512: // searchers. Because of this, deletions made via a previously open
0513: // reader, which would be applied to that reader's segment, are lost
0514: // for subsequent searchers/readers
0515: if (optimize)
0516: writer.optimize();
0517: writer.close();
0518:
0519: // The reader should not see the new data
0520: assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
0521: assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
0522: assertTermDocsCount("first reader", reader, searchTerm, 100);
0523: assertTermDocsCount("first reader", reader, searchTerm2, 0);
0524:
0525: // DELETE DOCUMENTS CONTAINING TERM: aaa
0526: // NOTE: the reader was created when only "aaa" documents were in
0527: int deleted = 0;
0528: try {
0529: deleted = reader.deleteDocuments(searchTerm);
0530: fail("Delete allowed on an index reader with stale segment information");
0531: } catch (StaleReaderException e) {
0532: /* success */
0533: }
0534:
0535: // Re-open index reader and try again. This time it should see
0536: // the new data.
0537: reader.close();
0538: reader = IndexReader.open(dir);
0539: assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
0540: assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
0541: assertTermDocsCount("first reader", reader, searchTerm, 100);
0542: assertTermDocsCount("first reader", reader, searchTerm2, 100);
0543:
0544: deleted = reader.deleteDocuments(searchTerm);
0545: assertEquals("deleted count", 100, deleted);
0546: assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
0547: assertEquals("deleted docFreq", 100, reader
0548: .docFreq(searchTerm2));
0549: assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
0550: assertTermDocsCount("deleted termDocs", reader, searchTerm2,
0551: 100);
0552: reader.close();
0553:
0554: // CREATE A NEW READER and re-test
0555: reader = IndexReader.open(dir);
0556: assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
0557: assertEquals("deleted docFreq", 100, reader
0558: .docFreq(searchTerm2));
0559: assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
0560: assertTermDocsCount("deleted termDocs", reader, searchTerm2,
0561: 100);
0562: reader.close();
0563: }
0564:
0565: private Directory getDirectory() throws IOException {
0566: return FSDirectory.getDirectory(new File(System
0567: .getProperty("tempDir"), "testIndex"));
0568: }
0569:
0570: public void testFilesOpenClose() throws IOException {
0571: // Create initial data set
0572: File dirFile = new File(System.getProperty("tempDir"),
0573: "testIndex");
0574: Directory dir = getDirectory();
0575: IndexWriter writer = new IndexWriter(dir,
0576: new WhitespaceAnalyzer(), true);
0577: addDoc(writer, "test");
0578: writer.close();
0579: dir.close();
0580:
0581: // Try to erase the data - this ensures that the writer closed all files
0582: _TestUtil.rmDir(dirFile);
0583: dir = getDirectory();
0584:
0585: // Now create the data set again, just as before
0586: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0587: addDoc(writer, "test");
0588: writer.close();
0589: dir.close();
0590:
0591: // Now open existing directory and test that reader closes all files
0592: dir = getDirectory();
0593: IndexReader reader1 = IndexReader.open(dir);
0594: reader1.close();
0595: dir.close();
0596:
0597: // The following will fail if reader did not close
0598: // all files
0599: _TestUtil.rmDir(dirFile);
0600: }
0601:
0602: public void testLastModified() throws IOException {
0603: assertFalse(IndexReader.indexExists("there_is_no_such_index"));
0604: final File fileDir = new File(System.getProperty("tempDir"),
0605: "testIndex");
0606: for (int i = 0; i < 2; i++) {
0607: try {
0608: final Directory dir;
0609: if (0 == i)
0610: dir = new MockRAMDirectory();
0611: else
0612: dir = getDirectory();
0613: assertFalse(IndexReader.indexExists(dir));
0614: IndexWriter writer = new IndexWriter(dir,
0615: new WhitespaceAnalyzer(), true);
0616: addDocumentWithFields(writer);
0617: assertTrue(IndexReader.isLocked(dir)); // writer open, so dir is locked
0618: writer.close();
0619: assertTrue(IndexReader.indexExists(dir));
0620: IndexReader reader = IndexReader.open(dir);
0621: assertFalse(IndexReader.isLocked(dir)); // reader only, no lock
0622: long version = IndexReader.lastModified(dir);
0623: if (i == 1) {
0624: long version2 = IndexReader.lastModified(fileDir);
0625: assertEquals(version, version2);
0626: }
0627: reader.close();
0628: // modify index and check version has been
0629: // incremented:
0630: while (true) {
0631: try {
0632: Thread.sleep(1000);
0633: break;
0634: } catch (InterruptedException ie) {
0635: Thread.currentThread().interrupt();
0636: }
0637: }
0638:
0639: writer = new IndexWriter(dir, new WhitespaceAnalyzer(),
0640: true);
0641: addDocumentWithFields(writer);
0642: writer.close();
0643: reader = IndexReader.open(dir);
0644: assertTrue("old lastModified is " + version
0645: + "; new lastModified is "
0646: + IndexReader.lastModified(dir),
0647: version <= IndexReader.lastModified(dir));
0648: reader.close();
0649: dir.close();
0650: } finally {
0651: if (i == 1)
0652: _TestUtil.rmDir(fileDir);
0653: }
0654: }
0655: }
0656:
0657: public void testVersion() throws IOException {
0658: assertFalse(IndexReader.indexExists("there_is_no_such_index"));
0659: Directory dir = new MockRAMDirectory();
0660: assertFalse(IndexReader.indexExists(dir));
0661: IndexWriter writer = new IndexWriter(dir,
0662: new WhitespaceAnalyzer(), true);
0663: addDocumentWithFields(writer);
0664: assertTrue(IndexReader.isLocked(dir)); // writer open, so dir is locked
0665: writer.close();
0666: assertTrue(IndexReader.indexExists(dir));
0667: IndexReader reader = IndexReader.open(dir);
0668: assertFalse(IndexReader.isLocked(dir)); // reader only, no lock
0669: long version = IndexReader.getCurrentVersion(dir);
0670: reader.close();
0671: // modify index and check version has been
0672: // incremented:
0673: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
0674: addDocumentWithFields(writer);
0675: writer.close();
0676: reader = IndexReader.open(dir);
0677: assertTrue("old version is " + version + "; new version is "
0678: + IndexReader.getCurrentVersion(dir),
0679: version < IndexReader.getCurrentVersion(dir));
0680: reader.close();
0681: dir.close();
0682: }
0683:
0684: public void testLock() throws IOException {
0685: Directory dir = new MockRAMDirectory();
0686: IndexWriter writer = new IndexWriter(dir,
0687: new WhitespaceAnalyzer(), true);
0688: addDocumentWithFields(writer);
0689: writer.close();
0690: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
0691: IndexReader reader = IndexReader.open(dir);
0692: try {
0693: reader.deleteDocument(0);
0694: fail("expected lock");
0695: } catch (IOException e) {
0696: // expected exception
0697: }
0698: IndexReader.unlock(dir); // this should not be done in the real world!
0699: reader.deleteDocument(0);
0700: reader.close();
0701: writer.close();
0702: dir.close();
0703: }
0704:
0705: public void testUndeleteAll() throws IOException {
0706: Directory dir = new MockRAMDirectory();
0707: IndexWriter writer = new IndexWriter(dir,
0708: new WhitespaceAnalyzer(), true);
0709: addDocumentWithFields(writer);
0710: addDocumentWithFields(writer);
0711: writer.close();
0712: IndexReader reader = IndexReader.open(dir);
0713: reader.deleteDocument(0);
0714: reader.deleteDocument(1);
0715: reader.undeleteAll();
0716: reader.close();
0717: reader = IndexReader.open(dir);
0718: assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
0719: reader.close();
0720: dir.close();
0721: }
0722:
0723: public void testUndeleteAllAfterClose() throws IOException {
0724: Directory dir = new MockRAMDirectory();
0725: IndexWriter writer = new IndexWriter(dir,
0726: new WhitespaceAnalyzer(), true);
0727: addDocumentWithFields(writer);
0728: addDocumentWithFields(writer);
0729: writer.close();
0730: IndexReader reader = IndexReader.open(dir);
0731: reader.deleteDocument(0);
0732: reader.deleteDocument(1);
0733: reader.close();
0734: reader = IndexReader.open(dir);
0735: reader.undeleteAll();
0736: assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
0737: reader.close();
0738: dir.close();
0739: }
0740:
0741: public void testUndeleteAllAfterCloseThenReopen()
0742: throws IOException {
0743: Directory dir = new MockRAMDirectory();
0744: IndexWriter writer = new IndexWriter(dir,
0745: new WhitespaceAnalyzer(), true);
0746: addDocumentWithFields(writer);
0747: addDocumentWithFields(writer);
0748: writer.close();
0749: IndexReader reader = IndexReader.open(dir);
0750: reader.deleteDocument(0);
0751: reader.deleteDocument(1);
0752: reader.close();
0753: reader = IndexReader.open(dir);
0754: reader.undeleteAll();
0755: reader.close();
0756: reader = IndexReader.open(dir);
0757: assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll()
0758: reader.close();
0759: dir.close();
0760: }
0761:
0762: public void testDeleteReaderReaderConflictUnoptimized()
0763: throws IOException {
0764: deleteReaderReaderConflict(false);
0765: }
0766:
0767: public void testDeleteReaderReaderConflictOptimized()
0768: throws IOException {
0769: deleteReaderReaderConflict(true);
0770: }
0771:
0772: /**
0773: * Make sure if reader tries to commit but hits disk
0774: * full that reader remains consistent and usable.
0775: */
0776: public void testDiskFull() throws IOException {
0777:
0778: boolean debug = false;
0779: Term searchTerm = new Term("content", "aaa");
0780: int START_COUNT = 157;
0781: int END_COUNT = 144;
0782:
0783: // First build up a starting index:
0784: RAMDirectory startDir = new MockRAMDirectory();
0785: IndexWriter writer = new IndexWriter(startDir,
0786: new WhitespaceAnalyzer(), true);
0787: for (int i = 0; i < 157; i++) {
0788: Document d = new Document();
0789: d.add(new Field("id", Integer.toString(i), Field.Store.YES,
0790: Field.Index.UN_TOKENIZED));
0791: d.add(new Field("content", "aaa " + i, Field.Store.NO,
0792: Field.Index.TOKENIZED));
0793: writer.addDocument(d);
0794: }
0795: writer.close();
0796:
0797: long diskUsage = startDir.sizeInBytes();
0798: long diskFree = diskUsage + 100;
0799:
0800: IOException err = null;
0801:
0802: boolean done = false;
0803:
0804: // Iterate w/ ever increasing free disk space:
0805: while (!done) {
0806: MockRAMDirectory dir = new MockRAMDirectory(startDir);
0807: IndexReader reader = IndexReader.open(dir);
0808:
0809: // For each disk size, first try to commit against
0810: // dir that will hit random IOExceptions & disk
0811: // full; after, give it infinite disk space & turn
0812: // off random IOExceptions & retry w/ same reader:
0813: boolean success = false;
0814:
0815: for (int x = 0; x < 2; x++) {
0816:
0817: double rate = 0.05;
0818: double diskRatio = ((double) diskFree) / diskUsage;
0819: long this DiskFree;
0820: String testName;
0821:
0822: if (0 == x) {
0823: this DiskFree = diskFree;
0824: if (diskRatio >= 2.0) {
0825: rate /= 2;
0826: }
0827: if (diskRatio >= 4.0) {
0828: rate /= 2;
0829: }
0830: if (diskRatio >= 6.0) {
0831: rate = 0.0;
0832: }
0833: if (debug) {
0834: System.out.println("\ncycle: " + diskFree
0835: + " bytes");
0836: }
0837: testName = "disk full during reader.close() @ "
0838: + this DiskFree + " bytes";
0839: } else {
0840: this DiskFree = 0;
0841: rate = 0.0;
0842: if (debug) {
0843: System.out
0844: .println("\ncycle: same writer: unlimited disk space");
0845: }
0846: testName = "reader re-use after disk full";
0847: }
0848:
0849: dir.setMaxSizeInBytes(this DiskFree);
0850: dir.setRandomIOExceptionRate(rate, diskFree);
0851:
0852: try {
0853: if (0 == x) {
0854: int docId = 12;
0855: for (int i = 0; i < 13; i++) {
0856: reader.deleteDocument(docId);
0857: reader.setNorm(docId, "contents",
0858: (float) 2.0);
0859: docId += 12;
0860: }
0861: }
0862: reader.close();
0863: success = true;
0864: if (0 == x) {
0865: done = true;
0866: }
0867: } catch (IOException e) {
0868: if (debug) {
0869: System.out.println(" hit IOException: " + e);
0870: }
0871: err = e;
0872: if (1 == x) {
0873: e.printStackTrace();
0874: fail(testName
0875: + " hit IOException after disk space was freed up");
0876: }
0877: }
0878:
0879: // Whether we succeeded or failed, check that all
0880: // un-referenced files were in fact deleted (ie,
0881: // we did not create garbage). Just create a
0882: // new IndexFileDeleter, have it delete
0883: // unreferenced files, then verify that in fact
0884: // no files were deleted:
0885: String[] startFiles = dir.list();
0886: SegmentInfos infos = new SegmentInfos();
0887: infos.read(dir);
0888: IndexFileDeleter d = new IndexFileDeleter(dir,
0889: new KeepOnlyLastCommitDeletionPolicy(), infos,
0890: null, null);
0891: String[] endFiles = dir.list();
0892:
0893: Arrays.sort(startFiles);
0894: Arrays.sort(endFiles);
0895:
0896: //for(int i=0;i<startFiles.length;i++) {
0897: // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
0898: //}
0899:
0900: if (!Arrays.equals(startFiles, endFiles)) {
0901: String successStr;
0902: if (success) {
0903: successStr = "success";
0904: } else {
0905: successStr = "IOException";
0906: err.printStackTrace();
0907: }
0908: fail("reader.close() failed to delete unreferenced files after "
0909: + successStr
0910: + " ("
0911: + diskFree
0912: + " bytes): before delete:\n "
0913: + arrayToString(startFiles)
0914: + "\n after delete:\n "
0915: + arrayToString(endFiles));
0916: }
0917:
0918: // Finally, verify index is not corrupt, and, if
0919: // we succeeded, we see all docs changed, and if
0920: // we failed, we see either all docs or no docs
0921: // changed (transactional semantics):
0922: IndexReader newReader = null;
0923: try {
0924: newReader = IndexReader.open(dir);
0925: } catch (IOException e) {
0926: e.printStackTrace();
0927: fail(testName
0928: + ":exception when creating IndexReader after disk full during close: "
0929: + e);
0930: }
0931: /*
0932: int result = newReader.docFreq(searchTerm);
0933: if (success) {
0934: if (result != END_COUNT) {
0935: fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT);
0936: }
0937: } else {
0938: // On hitting exception we still may have added
0939: // all docs:
0940: if (result != START_COUNT && result != END_COUNT) {
0941: err.printStackTrace();
0942: fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT);
0943: }
0944: }
0945: */
0946:
0947: IndexSearcher searcher = new IndexSearcher(newReader);
0948: Hits hits = null;
0949: try {
0950: hits = searcher.search(new TermQuery(searchTerm));
0951: } catch (IOException e) {
0952: e.printStackTrace();
0953: fail(testName + ": exception when searching: " + e);
0954: }
0955: int result2 = hits.length();
0956: if (success) {
0957: if (result2 != END_COUNT) {
0958: fail(testName
0959: + ": method did not throw exception but hits.length for search on term 'aaa' is "
0960: + result2 + " instead of expected "
0961: + END_COUNT);
0962: }
0963: } else {
0964: // On hitting exception we still may have added
0965: // all docs:
0966: if (result2 != START_COUNT && result2 != END_COUNT) {
0967: err.printStackTrace();
0968: fail(testName
0969: + ": method did throw exception but hits.length for search on term 'aaa' is "
0970: + result2 + " instead of expected "
0971: + START_COUNT);
0972: }
0973: }
0974:
0975: searcher.close();
0976: newReader.close();
0977:
0978: if (result2 == END_COUNT) {
0979: break;
0980: }
0981: }
0982:
0983: dir.close();
0984:
0985: // Try again with 10 more bytes of free space:
0986: diskFree += 10;
0987: }
0988:
0989: startDir.close();
0990: }
0991:
0992: public void testDocsOutOfOrderJIRA140() throws IOException {
0993: Directory dir = new MockRAMDirectory();
0994: IndexWriter writer = new IndexWriter(dir,
0995: new WhitespaceAnalyzer(), true);
0996: for (int i = 0; i < 11; i++) {
0997: addDoc(writer, "aaa");
0998: }
0999: writer.close();
1000: IndexReader reader = IndexReader.open(dir);
1001:
1002: // Try to delete an invalid docId, yet, within range
1003: // of the final bits of the BitVector:
1004:
1005: boolean gotException = false;
1006: try {
1007: reader.deleteDocument(11);
1008: } catch (ArrayIndexOutOfBoundsException e) {
1009: gotException = true;
1010: }
1011: reader.close();
1012:
1013: writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
1014:
1015: // We must add more docs to get a new segment written
1016: for (int i = 0; i < 11; i++) {
1017: addDoc(writer, "aaa");
1018: }
1019:
1020: // Without the fix for LUCENE-140 this call will
1021: // [incorrectly] hit a "docs out of order"
1022: // IllegalStateException because above out-of-bounds
1023: // deleteDocument corrupted the index:
1024: writer.optimize();
1025:
1026: if (!gotException) {
1027: fail("delete of out-of-bounds doc number failed to hit exception");
1028: }
1029: dir.close();
1030: }
1031:
1032: public void testExceptionReleaseWriteLockJIRA768()
1033: throws IOException {
1034:
1035: Directory dir = new MockRAMDirectory();
1036: IndexWriter writer = new IndexWriter(dir,
1037: new WhitespaceAnalyzer(), true);
1038: addDoc(writer, "aaa");
1039: writer.close();
1040:
1041: IndexReader reader = IndexReader.open(dir);
1042: try {
1043: reader.deleteDocument(1);
1044: fail("did not hit exception when deleting an invalid doc number");
1045: } catch (ArrayIndexOutOfBoundsException e) {
1046: // expected
1047: }
1048: reader.close();
1049: if (IndexReader.isLocked(dir)) {
1050: fail("write lock is still held after close");
1051: }
1052:
1053: reader = IndexReader.open(dir);
1054: try {
1055: reader.setNorm(1, "content", (float) 2.0);
1056: fail("did not hit exception when calling setNorm on an invalid doc number");
1057: } catch (ArrayIndexOutOfBoundsException e) {
1058: // expected
1059: }
1060: reader.close();
1061: if (IndexReader.isLocked(dir)) {
1062: fail("write lock is still held after close");
1063: }
1064: dir.close();
1065: }
1066:
1067: private String arrayToString(String[] l) {
1068: String s = "";
1069: for (int i = 0; i < l.length; i++) {
1070: if (i > 0) {
1071: s += "\n ";
1072: }
1073: s += l[i];
1074: }
1075: return s;
1076: }
1077:
1078: public void testOpenReaderAfterDelete() throws IOException {
1079: File dirFile = new File(System.getProperty("tempDir"),
1080: "deletetest");
1081: Directory dir = FSDirectory.getDirectory(dirFile);
1082: try {
1083: IndexReader reader = IndexReader.open(dir);
1084: fail("expected FileNotFoundException");
1085: } catch (FileNotFoundException e) {
1086: // expected
1087: }
1088:
1089: dirFile.delete();
1090:
1091: // Make sure we still get a CorruptIndexException (not NPE):
1092: try {
1093: IndexReader reader = IndexReader.open(dir);
1094: fail("expected FileNotFoundException");
1095: } catch (FileNotFoundException e) {
1096: // expected
1097: }
1098: }
1099:
1100: private void deleteReaderReaderConflict(boolean optimize)
1101: throws IOException {
1102: Directory dir = getDirectory();
1103:
1104: Term searchTerm1 = new Term("content", "aaa");
1105: Term searchTerm2 = new Term("content", "bbb");
1106: Term searchTerm3 = new Term("content", "ccc");
1107:
1108: // add 100 documents with term : aaa
1109: // add 100 documents with term : bbb
1110: // add 100 documents with term : ccc
1111: IndexWriter writer = new IndexWriter(dir,
1112: new WhitespaceAnalyzer(), true);
1113: for (int i = 0; i < 100; i++) {
1114: addDoc(writer, searchTerm1.text());
1115: addDoc(writer, searchTerm2.text());
1116: addDoc(writer, searchTerm3.text());
1117: }
1118: if (optimize)
1119: writer.optimize();
1120: writer.close();
1121:
1122: // OPEN TWO READERS
1123: // Both readers get segment info as exists at this time
1124: IndexReader reader1 = IndexReader.open(dir);
1125: assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
1126: assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
1127: assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
1128: assertTermDocsCount("first opened", reader1, searchTerm1, 100);
1129: assertTermDocsCount("first opened", reader1, searchTerm2, 100);
1130: assertTermDocsCount("first opened", reader1, searchTerm3, 100);
1131:
1132: IndexReader reader2 = IndexReader.open(dir);
1133: assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
1134: assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
1135: assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
1136: assertTermDocsCount("first opened", reader2, searchTerm1, 100);
1137: assertTermDocsCount("first opened", reader2, searchTerm2, 100);
1138: assertTermDocsCount("first opened", reader2, searchTerm3, 100);
1139:
1140: // DELETE DOCS FROM READER 2 and CLOSE IT
1141: // delete documents containing term: aaa
1142: // when the reader is closed, the segment info is updated and
1143: // the first reader is now stale
1144: reader2.deleteDocuments(searchTerm1);
1145: assertEquals("after delete 1", 100, reader2
1146: .docFreq(searchTerm1));
1147: assertEquals("after delete 1", 100, reader2
1148: .docFreq(searchTerm2));
1149: assertEquals("after delete 1", 100, reader2
1150: .docFreq(searchTerm3));
1151: assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
1152: assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
1153: assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
1154: reader2.close();
1155:
1156: // Make sure reader 1 is unchanged since it was open earlier
1157: assertEquals("after delete 1", 100, reader1
1158: .docFreq(searchTerm1));
1159: assertEquals("after delete 1", 100, reader1
1160: .docFreq(searchTerm2));
1161: assertEquals("after delete 1", 100, reader1
1162: .docFreq(searchTerm3));
1163: assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
1164: assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
1165: assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
1166:
1167: // ATTEMPT TO DELETE FROM STALE READER
1168: // delete documents containing term: bbb
1169: try {
1170: reader1.deleteDocuments(searchTerm2);
1171: fail("Delete allowed from a stale index reader");
1172: } catch (IOException e) {
1173: /* success */
1174: }
1175:
1176: // RECREATE READER AND TRY AGAIN
1177: reader1.close();
1178: reader1 = IndexReader.open(dir);
1179: assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
1180: assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
1181: assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
1182: assertTermDocsCount("reopened", reader1, searchTerm1, 0);
1183: assertTermDocsCount("reopened", reader1, searchTerm2, 100);
1184: assertTermDocsCount("reopened", reader1, searchTerm3, 100);
1185:
1186: reader1.deleteDocuments(searchTerm2);
1187: assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
1188: assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
1189: assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
1190: assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
1191: assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
1192: assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
1193: reader1.close();
1194:
1195: // Open another reader to confirm that everything is deleted
1196: reader2 = IndexReader.open(dir);
1197: assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
1198: assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
1199: assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
1200: assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
1201: assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
1202: assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
1203: reader2.close();
1204:
1205: dir.close();
1206: }
1207:
1208: private void addDocumentWithFields(IndexWriter writer)
1209: throws IOException {
1210: Document doc = new Document();
1211: doc.add(new Field("keyword", "test1", Field.Store.YES,
1212: Field.Index.UN_TOKENIZED));
1213: doc.add(new Field("text", "test1", Field.Store.YES,
1214: Field.Index.TOKENIZED));
1215: doc.add(new Field("unindexed", "test1", Field.Store.YES,
1216: Field.Index.NO));
1217: doc.add(new Field("unstored", "test1", Field.Store.NO,
1218: Field.Index.TOKENIZED));
1219: writer.addDocument(doc);
1220: }
1221:
1222: private void addDocumentWithDifferentFields(IndexWriter writer)
1223: throws IOException {
1224: Document doc = new Document();
1225: doc.add(new Field("keyword2", "test1", Field.Store.YES,
1226: Field.Index.UN_TOKENIZED));
1227: doc.add(new Field("text2", "test1", Field.Store.YES,
1228: Field.Index.TOKENIZED));
1229: doc.add(new Field("unindexed2", "test1", Field.Store.YES,
1230: Field.Index.NO));
1231: doc.add(new Field("unstored2", "test1", Field.Store.NO,
1232: Field.Index.TOKENIZED));
1233: writer.addDocument(doc);
1234: }
1235:
1236: private void addDocumentWithTermVectorFields(IndexWriter writer)
1237: throws IOException {
1238: Document doc = new Document();
1239: doc.add(new Field("tvnot", "tvnot", Field.Store.YES,
1240: Field.Index.TOKENIZED, Field.TermVector.NO));
1241: doc.add(new Field("termvector", "termvector", Field.Store.YES,
1242: Field.Index.TOKENIZED, Field.TermVector.YES));
1243: doc.add(new Field("tvoffset", "tvoffset", Field.Store.YES,
1244: Field.Index.TOKENIZED, Field.TermVector.WITH_OFFSETS));
1245: doc
1246: .add(new Field("tvposition", "tvposition",
1247: Field.Store.YES, Field.Index.TOKENIZED,
1248: Field.TermVector.WITH_POSITIONS));
1249: doc.add(new Field("tvpositionoffset", "tvpositionoffset",
1250: Field.Store.YES, Field.Index.TOKENIZED,
1251: Field.TermVector.WITH_POSITIONS_OFFSETS));
1252:
1253: writer.addDocument(doc);
1254: }
1255:
1256: private void addDoc(IndexWriter writer, String value)
1257: throws IOException {
1258: Document doc = new Document();
1259: doc.add(new Field("content", value, Field.Store.NO,
1260: Field.Index.TOKENIZED));
1261: writer.addDocument(doc);
1262: }
1263:
1264: private void rmDir(File dir) {
1265: File[] files = dir.listFiles();
1266: for (int i = 0; i < files.length; i++) {
1267: files[i].delete();
1268: }
1269: dir.delete();
1270: }
1271:
1272: public static void assertIndexEquals(IndexReader index1,
1273: IndexReader index2) throws IOException {
1274: assertEquals("IndexReaders have different values for numDocs.",
1275: index1.numDocs(), index2.numDocs());
1276: assertEquals("IndexReaders have different values for maxDoc.",
1277: index1.maxDoc(), index2.maxDoc());
1278: assertEquals("Only one IndexReader has deletions.", index1
1279: .hasDeletions(), index2.hasDeletions());
1280: assertEquals("Only one index is optimized.", index1
1281: .isOptimized(), index2.isOptimized());
1282:
1283: // check field names
1284: Collection fields1 = index1.getFieldNames(FieldOption.ALL);
1285: Collection fields2 = index1.getFieldNames(FieldOption.ALL);
1286: assertEquals("IndexReaders have different numbers of fields.",
1287: fields1.size(), fields2.size());
1288: Iterator it1 = fields1.iterator();
1289: Iterator it2 = fields1.iterator();
1290: while (it1.hasNext()) {
1291: assertEquals("Different field names.", (String) it1.next(),
1292: (String) it2.next());
1293: }
1294:
1295: // check norms
1296: it1 = fields1.iterator();
1297: while (it1.hasNext()) {
1298: String curField = (String) it1.next();
1299: byte[] norms1 = index1.norms(curField);
1300: byte[] norms2 = index2.norms(curField);
1301: assertEquals(norms1.length, norms2.length);
1302: for (int i = 0; i < norms1.length; i++) {
1303: assertEquals("Norm different for doc " + i
1304: + " and field '" + curField + "'.", norms1[i],
1305: norms2[i]);
1306: }
1307: }
1308:
1309: // check deletions
1310: for (int i = 0; i < index1.maxDoc(); i++) {
1311: assertEquals("Doc " + i + " only deleted in one index.",
1312: index1.isDeleted(i), index2.isDeleted(i));
1313: }
1314:
1315: // check stored fields
1316: for (int i = 0; i < index1.maxDoc(); i++) {
1317: if (!index1.isDeleted(i)) {
1318: Document doc1 = index1.document(i);
1319: Document doc2 = index2.document(i);
1320: fields1 = doc1.getFields();
1321: fields2 = doc2.getFields();
1322: assertEquals("Different numbers of fields for doc " + i
1323: + ".", fields1.size(), fields2.size());
1324: it1 = fields1.iterator();
1325: it2 = fields2.iterator();
1326: while (it1.hasNext()) {
1327: Field curField1 = (Field) it1.next();
1328: Field curField2 = (Field) it2.next();
1329: assertEquals("Different fields names for doc " + i
1330: + ".", curField1.name(), curField2.name());
1331: assertEquals("Different field values for doc " + i
1332: + ".", curField1.stringValue(), curField2
1333: .stringValue());
1334: }
1335: }
1336: }
1337:
1338: // check dictionary and posting lists
1339: TermEnum enum1 = index1.terms();
1340: TermEnum enum2 = index2.terms();
1341: TermPositions tp1 = index1.termPositions();
1342: TermPositions tp2 = index2.termPositions();
1343: while (enum1.next()) {
1344: assertTrue(enum2.next());
1345: assertEquals("Different term in dictionary.", enum1.term(),
1346: enum2.term());
1347: tp1.seek(enum1.term());
1348: tp2.seek(enum1.term());
1349: while (tp1.next()) {
1350: assertTrue(tp2.next());
1351: assertEquals("Different doc id in postinglist of term "
1352: + enum1.term() + ".", tp1.doc(), tp2.doc());
1353: assertEquals(
1354: "Different term frequence in postinglist of term "
1355: + enum1.term() + ".", tp1.freq(), tp2
1356: .freq());
1357: for (int i = 0; i < tp1.freq(); i++) {
1358: assertEquals(
1359: "Different positions in postinglist of term "
1360: + enum1.term() + ".", tp1
1361: .nextPosition(), tp2.nextPosition());
1362: }
1363: }
1364: }
1365: }
1366:
1367: }
|