001: package org.apache.lucene.index;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import org.apache.lucene.util.LuceneTestCase;
021: import java.util.Vector;
022: import java.util.Arrays;
023: import java.io.ByteArrayOutputStream;
024: import java.io.ObjectOutputStream;
025: import java.io.IOException;
026: import java.io.File;
027:
028: import org.apache.lucene.analysis.WhitespaceAnalyzer;
029: import org.apache.lucene.search.IndexSearcher;
030: import org.apache.lucene.search.TermQuery;
031: import org.apache.lucene.search.Hits;
032: import org.apache.lucene.store.Directory;
033: import org.apache.lucene.store.FSDirectory;
034: import org.apache.lucene.document.Document;
035: import org.apache.lucene.document.Field;
036: import java.io.*;
037: import java.util.*;
038: import java.util.zip.*;
039:
040: /*
041: Verify we can read the pre-2.1 file format, do searches
042: against it, and add documents to it.
043: */
044:
045: public class TestBackwardsCompatibility extends LuceneTestCase {
046:
047: // Uncomment these cases & run in a pre-lockless checkout
048: // to create indices:
049:
050: /*
051: public void testCreatePreLocklessCFS() throws IOException {
052: createIndex("src/test/org/apache/lucene/index/index.prelockless.cfs", true);
053: }
054:
055: public void testCreatePreLocklessNoCFS() throws IOException {
056: createIndex("src/test/org/apache/lucene/index/index.prelockless.nocfs", false);
057: }
058: */
059:
060: /* Unzips dirName + ".zip" --> dirName, removing dirName
061: first */
062: public void unzip(String zipName, String destDirName)
063: throws IOException {
064:
065: Enumeration entries;
066: ZipFile zipFile;
067: zipFile = new ZipFile(zipName + ".zip");
068:
069: entries = zipFile.entries();
070:
071: String dirName = fullDir(destDirName);
072:
073: File fileDir = new File(dirName);
074: rmDir(destDirName);
075:
076: fileDir.mkdir();
077:
078: while (entries.hasMoreElements()) {
079: ZipEntry entry = (ZipEntry) entries.nextElement();
080:
081: InputStream in = zipFile.getInputStream(entry);
082: OutputStream out = new BufferedOutputStream(
083: new FileOutputStream(new File(fileDir, entry
084: .getName())));
085:
086: byte[] buffer = new byte[8192];
087: int len;
088: while ((len = in.read(buffer)) >= 0) {
089: out.write(buffer, 0, len);
090: }
091:
092: in.close();
093: out.close();
094: }
095:
096: zipFile.close();
097: }
098:
099: public void testCreateCFS() throws IOException {
100: String dirName = "testindex.cfs";
101: createIndex(dirName, true);
102: rmDir(dirName);
103: }
104:
105: public void testCreateNoCFS() throws IOException {
106: String dirName = "testindex.nocfs";
107: createIndex(dirName, true);
108: rmDir(dirName);
109: }
110:
111: final String[] oldNames = { "prelockless.cfs", "prelockless.nocfs",
112: "presharedstores.cfs", "presharedstores.nocfs" };
113:
114: public void testSearchOldIndex() throws IOException {
115: for (int i = 0; i < oldNames.length; i++) {
116: String dirName = "src/test/org/apache/lucene/index/index."
117: + oldNames[i];
118: unzip(dirName, oldNames[i]);
119: searchIndex(oldNames[i]);
120: rmDir(oldNames[i]);
121: }
122: }
123:
124: public void testIndexOldIndexNoAdds() throws IOException {
125: for (int i = 0; i < oldNames.length; i++) {
126: String dirName = "src/test/org/apache/lucene/index/index."
127: + oldNames[i];
128: unzip(dirName, oldNames[i]);
129: changeIndexNoAdds(oldNames[i], true);
130: rmDir(oldNames[i]);
131:
132: unzip(dirName, oldNames[i]);
133: changeIndexNoAdds(oldNames[i], false);
134: rmDir(oldNames[i]);
135: }
136: }
137:
138: public void testIndexOldIndex() throws IOException {
139: for (int i = 0; i < oldNames.length; i++) {
140: String dirName = "src/test/org/apache/lucene/index/index."
141: + oldNames[i];
142: unzip(dirName, oldNames[i]);
143: changeIndexWithAdds(oldNames[i], true);
144: rmDir(oldNames[i]);
145:
146: unzip(dirName, oldNames[i]);
147: changeIndexWithAdds(oldNames[i], false);
148: rmDir(oldNames[i]);
149: }
150: }
151:
152: public void searchIndex(String dirName) throws IOException {
153: //QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer());
154: //Query query = parser.parse("handle:1");
155:
156: dirName = fullDir(dirName);
157:
158: Directory dir = FSDirectory.getDirectory(dirName);
159: IndexSearcher searcher = new IndexSearcher(dir);
160:
161: Hits hits = searcher.search(new TermQuery(new Term("content",
162: "aaa")));
163: assertEquals(34, hits.length());
164: Document d = hits.doc(0);
165:
166: // First document should be #21 since it's norm was increased:
167: assertEquals("didn't get the right document first", "21", d
168: .get("id"));
169:
170: searcher.close();
171: dir.close();
172: }
173:
174: /* Open pre-lockless index, add docs, do a delete &
175: * setNorm, and search */
176: public void changeIndexWithAdds(String dirName, boolean autoCommit)
177: throws IOException {
178:
179: dirName = fullDir(dirName);
180:
181: Directory dir = FSDirectory.getDirectory(dirName);
182:
183: // open writer
184: IndexWriter writer = new IndexWriter(dir, autoCommit,
185: new WhitespaceAnalyzer(), false);
186:
187: // add 10 docs
188: for (int i = 0; i < 10; i++) {
189: addDoc(writer, 35 + i);
190: }
191:
192: // make sure writer sees right total -- writer seems not to know about deletes in .del?
193: assertEquals("wrong doc count", 45, writer.docCount());
194: writer.close();
195:
196: // make sure searching sees right # hits
197: IndexSearcher searcher = new IndexSearcher(dir);
198: Hits hits = searcher.search(new TermQuery(new Term("content",
199: "aaa")));
200: assertEquals("wrong number of hits", 44, hits.length());
201: Document d = hits.doc(0);
202: assertEquals("wrong first document", "21", d.get("id"));
203: searcher.close();
204:
205: // make sure we can do delete & setNorm against this
206: // pre-lockless segment:
207: IndexReader reader = IndexReader.open(dir);
208: Term searchTerm = new Term("id", "6");
209: int delCount = reader.deleteDocuments(searchTerm);
210: assertEquals("wrong delete count", 1, delCount);
211: reader.setNorm(22, "content", (float) 2.0);
212: reader.close();
213:
214: // make sure they "took":
215: searcher = new IndexSearcher(dir);
216: hits = searcher
217: .search(new TermQuery(new Term("content", "aaa")));
218: assertEquals("wrong number of hits", 43, hits.length());
219: d = hits.doc(0);
220: assertEquals("wrong first document", "22", d.get("id"));
221: searcher.close();
222:
223: // optimize
224: writer = new IndexWriter(dir, autoCommit,
225: new WhitespaceAnalyzer(), false);
226: writer.optimize();
227: writer.close();
228:
229: searcher = new IndexSearcher(dir);
230: hits = searcher
231: .search(new TermQuery(new Term("content", "aaa")));
232: assertEquals("wrong number of hits", 43, hits.length());
233: d = hits.doc(0);
234: assertEquals("wrong first document", "22", d.get("id"));
235: searcher.close();
236:
237: dir.close();
238: }
239:
240: /* Open pre-lockless index, add docs, do a delete &
241: * setNorm, and search */
242: public void changeIndexNoAdds(String dirName, boolean autoCommit)
243: throws IOException {
244:
245: dirName = fullDir(dirName);
246:
247: Directory dir = FSDirectory.getDirectory(dirName);
248:
249: // make sure searching sees right # hits
250: IndexSearcher searcher = new IndexSearcher(dir);
251: Hits hits = searcher.search(new TermQuery(new Term("content",
252: "aaa")));
253: assertEquals("wrong number of hits", 34, hits.length());
254: Document d = hits.doc(0);
255: assertEquals("wrong first document", "21", d.get("id"));
256: searcher.close();
257:
258: // make sure we can do a delete & setNorm against this
259: // pre-lockless segment:
260: IndexReader reader = IndexReader.open(dir);
261: Term searchTerm = new Term("id", "6");
262: int delCount = reader.deleteDocuments(searchTerm);
263: assertEquals("wrong delete count", 1, delCount);
264: reader.setNorm(22, "content", (float) 2.0);
265: reader.close();
266:
267: // make sure they "took":
268: searcher = new IndexSearcher(dir);
269: hits = searcher
270: .search(new TermQuery(new Term("content", "aaa")));
271: assertEquals("wrong number of hits", 33, hits.length());
272: d = hits.doc(0);
273: assertEquals("wrong first document", "22", d.get("id"));
274: searcher.close();
275:
276: // optimize
277: IndexWriter writer = new IndexWriter(dir, autoCommit,
278: new WhitespaceAnalyzer(), false);
279: writer.optimize();
280: writer.close();
281:
282: searcher = new IndexSearcher(dir);
283: hits = searcher
284: .search(new TermQuery(new Term("content", "aaa")));
285: assertEquals("wrong number of hits", 33, hits.length());
286: d = hits.doc(0);
287: assertEquals("wrong first document", "22", d.get("id"));
288: searcher.close();
289:
290: dir.close();
291: }
292:
293: public void createIndex(String dirName, boolean doCFS)
294: throws IOException {
295:
296: rmDir(dirName);
297:
298: dirName = fullDir(dirName);
299:
300: Directory dir = FSDirectory.getDirectory(dirName);
301: IndexWriter writer = new IndexWriter(dir,
302: new WhitespaceAnalyzer(), true);
303: writer.setUseCompoundFile(doCFS);
304:
305: for (int i = 0; i < 35; i++) {
306: addDoc(writer, i);
307: }
308: assertEquals("wrong doc count", 35, writer.docCount());
309: writer.close();
310:
311: // Delete one doc so we get a .del file:
312: IndexReader reader = IndexReader.open(dir);
313: Term searchTerm = new Term("id", "7");
314: int delCount = reader.deleteDocuments(searchTerm);
315: assertEquals("didn't delete the right number of documents", 1,
316: delCount);
317:
318: // Set one norm so we get a .s0 file:
319: reader.setNorm(21, "content", (float) 1.5);
320: reader.close();
321: }
322:
323: /* Verifies that the expected file names were produced */
324:
325: public void testExactFileNames() throws IOException {
326:
327: for (int pass = 0; pass < 2; pass++) {
328:
329: String outputDir = "lucene.backwardscompat0.index";
330: rmDir(outputDir);
331:
332: try {
333: Directory dir = FSDirectory
334: .getDirectory(fullDir(outputDir));
335:
336: boolean autoCommit = 0 == pass;
337:
338: IndexWriter writer = new IndexWriter(dir, autoCommit,
339: new WhitespaceAnalyzer(), true);
340: writer.setRAMBufferSizeMB(16.0);
341: //IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
342: for (int i = 0; i < 35; i++) {
343: addDoc(writer, i);
344: }
345: assertEquals("wrong doc count", 35, writer.docCount());
346: writer.close();
347:
348: // Delete one doc so we get a .del file:
349: IndexReader reader = IndexReader.open(dir);
350: Term searchTerm = new Term("id", "7");
351: int delCount = reader.deleteDocuments(searchTerm);
352: assertEquals(
353: "didn't delete the right number of documents",
354: 1, delCount);
355:
356: // Set one norm so we get a .s0 file:
357: reader.setNorm(21, "content", (float) 1.5);
358: reader.close();
359:
360: // The numbering of fields can vary depending on which
361: // JRE is in use. On some JREs we see content bound to
362: // field 0; on others, field 1. So, here we have to
363: // figure out which field number corresponds to
364: // "content", and then set our expected file names below
365: // accordingly:
366: CompoundFileReader cfsReader = new CompoundFileReader(
367: dir, "_0.cfs");
368: FieldInfos fieldInfos = new FieldInfos(cfsReader,
369: "_0.fnm");
370: int contentFieldIndex = -1;
371: for (int i = 0; i < fieldInfos.size(); i++) {
372: FieldInfo fi = fieldInfos.fieldInfo(i);
373: if (fi.name.equals("content")) {
374: contentFieldIndex = i;
375: break;
376: }
377: }
378: cfsReader.close();
379: assertTrue(
380: "could not locate the 'content' field number in the _2.cfs segment",
381: contentFieldIndex != -1);
382:
383: // Now verify file names:
384: String[] expected;
385: expected = new String[] { "_0.cfs", "_0_1.del",
386: "_0_1.s" + contentFieldIndex, "segments_4",
387: "segments.gen" };
388:
389: if (!autoCommit)
390: expected[3] = "segments_3";
391:
392: String[] actual = dir.list();
393: Arrays.sort(expected);
394: Arrays.sort(actual);
395: if (!Arrays.equals(expected, actual)) {
396: fail("incorrect filenames in index: expected:\n "
397: + asString(expected)
398: + "\n actual:\n "
399: + asString(actual));
400: }
401: dir.close();
402: } finally {
403: rmDir(outputDir);
404: }
405: }
406: }
407:
408: private String asString(String[] l) {
409: String s = "";
410: for (int i = 0; i < l.length; i++) {
411: if (i > 0) {
412: s += "\n ";
413: }
414: s += l[i];
415: }
416: return s;
417: }
418:
419: private void addDoc(IndexWriter writer, int id) throws IOException {
420: Document doc = new Document();
421: doc.add(new Field("content", "aaa", Field.Store.NO,
422: Field.Index.TOKENIZED));
423: doc.add(new Field("id", Integer.toString(id), Field.Store.YES,
424: Field.Index.UN_TOKENIZED));
425: writer.addDocument(doc);
426: }
427:
428: private void rmDir(String dir) throws IOException {
429: File fileDir = new File(fullDir(dir));
430: if (fileDir.exists()) {
431: File[] files = fileDir.listFiles();
432: if (files != null) {
433: for (int i = 0; i < files.length; i++) {
434: files[i].delete();
435: }
436: }
437: fileDir.delete();
438: }
439: }
440:
441: public static String fullDir(String dirName) throws IOException {
442: return new File(System.getProperty("tempDir"), dirName)
443: .getCanonicalPath();
444: }
445: }
|