001: package org.apache.lucene.index;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import org.apache.lucene.util.LuceneTestCase;
021: import org.apache.lucene.analysis.WhitespaceAnalyzer;
022: import org.apache.lucene.document.*;
023: import org.apache.lucene.search.Similarity;
024: import org.apache.lucene.store.FSDirectory;
025: import org.apache.lucene.store.RAMDirectory;
026: import org.apache.lucene.store.AlreadyClosedException;
027: import org.apache.lucene.util._TestUtil;
028:
029: import java.io.File;
030: import java.io.IOException;
031: import java.util.*;
032:
033: public class TestFieldsReader extends LuceneTestCase {
034: private RAMDirectory dir = new RAMDirectory();
035: private Document testDoc = new Document();
036: private FieldInfos fieldInfos = null;
037:
038: private final static String TEST_SEGMENT_NAME = "_0";
039:
040: public TestFieldsReader(String s) {
041: super (s);
042: }
043:
044: protected void setUp() throws Exception {
045: super .setUp();
046: fieldInfos = new FieldInfos();
047: DocHelper.setupDoc(testDoc);
048: fieldInfos.add(testDoc);
049: IndexWriter writer = new IndexWriter(dir,
050: new WhitespaceAnalyzer(), true);
051: writer.setUseCompoundFile(false);
052: writer.addDocument(testDoc);
053: writer.close();
054: }
055:
056: public void test() throws IOException {
057: assertTrue(dir != null);
058: assertTrue(fieldInfos != null);
059: FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME,
060: fieldInfos);
061: assertTrue(reader != null);
062: assertTrue(reader.size() == 1);
063: Document doc = reader.doc(0, null);
064: assertTrue(doc != null);
065: assertTrue(doc.getField(DocHelper.TEXT_FIELD_1_KEY) != null);
066:
067: Fieldable field = doc.getField(DocHelper.TEXT_FIELD_2_KEY);
068: assertTrue(field != null);
069: assertTrue(field.isTermVectorStored() == true);
070:
071: assertTrue(field.isStoreOffsetWithTermVector() == true);
072: assertTrue(field.isStorePositionWithTermVector() == true);
073: assertTrue(field.getOmitNorms() == false);
074:
075: field = doc.getField(DocHelper.TEXT_FIELD_3_KEY);
076: assertTrue(field != null);
077: assertTrue(field.isTermVectorStored() == false);
078: assertTrue(field.isStoreOffsetWithTermVector() == false);
079: assertTrue(field.isStorePositionWithTermVector() == false);
080: assertTrue(field.getOmitNorms() == true);
081:
082: reader.close();
083: }
084:
085: public void testLazyFields() throws Exception {
086: assertTrue(dir != null);
087: assertTrue(fieldInfos != null);
088: FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME,
089: fieldInfos);
090: assertTrue(reader != null);
091: assertTrue(reader.size() == 1);
092: Set loadFieldNames = new HashSet();
093: loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
094: loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
095: Set lazyFieldNames = new HashSet();
096: //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
097: lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
098: lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
099: lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
100: lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
101: lazyFieldNames.add(DocHelper.COMPRESSED_TEXT_FIELD_2_KEY);
102: SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(
103: loadFieldNames, lazyFieldNames);
104: Document doc = reader.doc(0, fieldSelector);
105: assertTrue("doc is null and it shouldn't be", doc != null);
106: Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
107: assertTrue("field is null and it shouldn't be", field != null);
108: assertTrue("field is not lazy and it should be", field.isLazy());
109: String value = field.stringValue();
110: assertTrue("value is null and it shouldn't be", value != null);
111: assertTrue(value + " is not equal to "
112: + DocHelper.LAZY_FIELD_TEXT, value
113: .equals(DocHelper.LAZY_FIELD_TEXT) == true);
114: field = doc.getFieldable(DocHelper.COMPRESSED_TEXT_FIELD_2_KEY);
115: assertTrue("field is null and it shouldn't be", field != null);
116: assertTrue("field is not lazy and it should be", field.isLazy());
117: value = field.stringValue();
118: assertTrue("value is null and it shouldn't be", value != null);
119: assertTrue(value + " is not equal to "
120: + DocHelper.FIELD_2_COMPRESSED_TEXT, value
121: .equals(DocHelper.FIELD_2_COMPRESSED_TEXT) == true);
122: field = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
123: assertTrue("field is null and it shouldn't be", field != null);
124: assertTrue("Field is lazy and it should not be",
125: field.isLazy() == false);
126: field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
127: assertTrue("field is null and it shouldn't be", field != null);
128: assertTrue("Field is lazy and it should not be",
129: field.isLazy() == false);
130: assertTrue(field.stringValue() + " is not equal to "
131: + DocHelper.FIELD_UTF1_TEXT, field.stringValue()
132: .equals(DocHelper.FIELD_UTF1_TEXT) == true);
133:
134: field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
135: assertTrue("field is null and it shouldn't be", field != null);
136: assertTrue("Field is lazy and it should not be",
137: field.isLazy() == true);
138: assertTrue(field.stringValue() + " is not equal to "
139: + DocHelper.FIELD_UTF2_TEXT, field.stringValue()
140: .equals(DocHelper.FIELD_UTF2_TEXT) == true);
141:
142: field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
143: assertTrue("field is null and it shouldn't be", field != null);
144: byte[] bytes = field.binaryValue();
145: assertTrue("bytes is null and it shouldn't be", bytes != null);
146: assertTrue(
147: "",
148: DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
149: for (int i = 0; i < bytes.length; i++) {
150: assertTrue("byte[" + i + "] is mismatched",
151: bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
152:
153: }
154: }
155:
156: public void testLazyFieldsAfterClose() throws Exception {
157: assertTrue(dir != null);
158: assertTrue(fieldInfos != null);
159: FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME,
160: fieldInfos);
161: assertTrue(reader != null);
162: assertTrue(reader.size() == 1);
163: Set loadFieldNames = new HashSet();
164: loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
165: loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
166: Set lazyFieldNames = new HashSet();
167: lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
168: lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
169: lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
170: lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
171: lazyFieldNames.add(DocHelper.COMPRESSED_TEXT_FIELD_2_KEY);
172: SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(
173: loadFieldNames, lazyFieldNames);
174: Document doc = reader.doc(0, fieldSelector);
175: assertTrue("doc is null and it shouldn't be", doc != null);
176: Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
177: assertTrue("field is null and it shouldn't be", field != null);
178: assertTrue("field is not lazy and it should be", field.isLazy());
179: reader.close();
180: try {
181: String value = field.stringValue();
182: fail("did not hit AlreadyClosedException as expected");
183: } catch (AlreadyClosedException e) {
184: // expected
185: }
186: }
187:
188: public void testLoadFirst() throws Exception {
189: assertTrue(dir != null);
190: assertTrue(fieldInfos != null);
191: FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME,
192: fieldInfos);
193: assertTrue(reader != null);
194: assertTrue(reader.size() == 1);
195: LoadFirstFieldSelector fieldSelector = new LoadFirstFieldSelector();
196: Document doc = reader.doc(0, fieldSelector);
197: assertTrue("doc is null and it shouldn't be", doc != null);
198: int count = 0;
199: List l = doc.getFields();
200: for (Iterator iter = l.iterator(); iter.hasNext();) {
201: Field field = (Field) iter.next();
202: assertTrue("field is null and it shouldn't be",
203: field != null);
204: String sv = field.stringValue();
205: assertTrue("sv is null and it shouldn't be", sv != null);
206: count++;
207: }
208: assertTrue(count + " does not equal: " + 1, count == 1);
209: }
210:
211: /**
212: * Not really a test per se, but we should have some way of assessing whether this is worthwhile.
213: * <p/>
214: * Must test using a File based directory
215: *
216: * @throws Exception
217: */
218: public void testLazyPerformance() throws Exception {
219: String tmpIODir = System.getProperty("tempDir");
220: String userName = System.getProperty("user.name");
221: String path = tmpIODir + File.separator + "lazyDir" + userName;
222: File file = new File(path);
223: _TestUtil.rmDir(file);
224: FSDirectory tmpDir = FSDirectory.getDirectory(file);
225: assertTrue(tmpDir != null);
226:
227: IndexWriter writer = new IndexWriter(tmpDir,
228: new WhitespaceAnalyzer(), true);
229: writer.setUseCompoundFile(false);
230: writer.addDocument(testDoc);
231: writer.close();
232:
233: assertTrue(fieldInfos != null);
234: FieldsReader reader;
235: long lazyTime = 0;
236: long regularTime = 0;
237: int length = 50;
238: Set lazyFieldNames = new HashSet();
239: lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
240: SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(
241: Collections.EMPTY_SET, lazyFieldNames);
242:
243: for (int i = 0; i < length; i++) {
244: reader = new FieldsReader(tmpDir, TEST_SEGMENT_NAME,
245: fieldInfos);
246: assertTrue(reader != null);
247: assertTrue(reader.size() == 1);
248:
249: Document doc;
250: doc = reader.doc(0, null);//Load all of them
251: assertTrue("doc is null and it shouldn't be", doc != null);
252: Fieldable field = doc
253: .getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);
254: assertTrue("field is lazy", field.isLazy() == false);
255: String value;
256: long start;
257: long finish;
258: start = System.currentTimeMillis();
259: //On my machine this was always 0ms.
260: value = field.stringValue();
261: finish = System.currentTimeMillis();
262: assertTrue("value is null and it shouldn't be",
263: value != null);
264: assertTrue("field is null and it shouldn't be",
265: field != null);
266: regularTime += (finish - start);
267: reader.close();
268: reader = null;
269: doc = null;
270: //Hmmm, are we still in cache???
271: System.gc();
272: reader = new FieldsReader(tmpDir, TEST_SEGMENT_NAME,
273: fieldInfos);
274: doc = reader.doc(0, fieldSelector);
275: field = doc.getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);
276: assertTrue("field is not lazy", field.isLazy() == true);
277: start = System.currentTimeMillis();
278: //On my machine this took around 50 - 70ms
279: value = field.stringValue();
280: finish = System.currentTimeMillis();
281: assertTrue("value is null and it shouldn't be",
282: value != null);
283: lazyTime += (finish - start);
284: reader.close();
285:
286: }
287: System.out
288: .println("Average Non-lazy time (should be very close to zero): "
289: + regularTime
290: / length
291: + " ms for "
292: + length
293: + " reads");
294: System.out
295: .println("Average Lazy Time (should be greater than zero): "
296: + lazyTime
297: / length
298: + " ms for "
299: + length
300: + " reads");
301: }
302:
303: public void testLoadSize() throws IOException {
304: FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME,
305: fieldInfos);
306: Document doc;
307:
308: doc = reader.doc(0, new FieldSelector() {
309: public FieldSelectorResult accept(String fieldName) {
310: if (fieldName.equals(DocHelper.TEXT_FIELD_1_KEY)
311: || fieldName
312: .equals(DocHelper.COMPRESSED_TEXT_FIELD_2_KEY)
313: || fieldName
314: .equals(DocHelper.LAZY_FIELD_BINARY_KEY))
315: return FieldSelectorResult.SIZE;
316: else if (fieldName.equals(DocHelper.TEXT_FIELD_3_KEY))
317: return FieldSelectorResult.LOAD;
318: else
319: return FieldSelectorResult.NO_LOAD;
320: }
321: });
322: Fieldable f1 = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
323: Fieldable f3 = doc.getFieldable(DocHelper.TEXT_FIELD_3_KEY);
324: Fieldable fb = doc
325: .getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
326: assertTrue(f1.isBinary());
327: assertTrue(!f3.isBinary());
328: assertTrue(fb.isBinary());
329: assertSizeEquals(2 * DocHelper.FIELD_1_TEXT.length(), f1
330: .binaryValue());
331: assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());
332: assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb
333: .binaryValue());
334:
335: reader.close();
336: }
337:
338: private void assertSizeEquals(int size, byte[] sizebytes) {
339: assertEquals((byte) (size >>> 24), sizebytes[0]);
340: assertEquals((byte) (size >>> 16), sizebytes[1]);
341: assertEquals((byte) (size >>> 8), sizebytes[2]);
342: assertEquals((byte) size, sizebytes[3]);
343: }
344:
345: }
|