001: package org.apache.lucene.index;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import java.io.IOException;
021: import java.io.Reader;
022:
023: import org.apache.lucene.util.LuceneTestCase;
024:
025: import org.apache.lucene.analysis.Analyzer;
026: import org.apache.lucene.analysis.LowerCaseTokenizer;
027: import org.apache.lucene.analysis.Token;
028: import org.apache.lucene.analysis.TokenFilter;
029: import org.apache.lucene.analysis.TokenStream;
030: import org.apache.lucene.document.Document;
031: import org.apache.lucene.document.Field;
032: import org.apache.lucene.document.Field.Index;
033: import org.apache.lucene.document.Field.Store;
034: import org.apache.lucene.store.IndexInput;
035: import org.apache.lucene.store.RAMDirectory;
036:
037: /**
038: * This testcase tests whether multi-level skipping is being used
039: * to reduce I/O while skipping through posting lists.
040: *
041: * Skipping in general is already covered by several other
042: * testcases.
043: *
044: */
045: public class TestMultiLevelSkipList extends LuceneTestCase {
046: public void testSimpleSkip() throws IOException {
047: RAMDirectory dir = new RAMDirectory();
048: IndexWriter writer = new IndexWriter(dir,
049: new PayloadAnalyzer(), true);
050: Term term = new Term("test", "a");
051: for (int i = 0; i < 5000; i++) {
052: Document d1 = new Document();
053: d1.add(new Field(term.field(), term.text(), Store.NO,
054: Index.TOKENIZED));
055: writer.addDocument(d1);
056: }
057: writer.flush();
058: writer.optimize();
059: writer.close();
060:
061: IndexReader reader = IndexReader.open(dir);
062: SegmentTermPositions tp = (SegmentTermPositions) reader
063: .termPositions();
064: tp.freqStream = new CountingStream(tp.freqStream);
065:
066: for (int i = 0; i < 2; i++) {
067: counter = 0;
068: tp.seek(term);
069:
070: checkSkipTo(tp, 14, 185); // no skips
071: checkSkipTo(tp, 17, 190); // one skip on level 0
072: checkSkipTo(tp, 287, 200); // one skip on level 1, two on level 0
073:
074: // this test would fail if we had only one skip level,
075: // because than more bytes would be read from the freqStream
076: checkSkipTo(tp, 4800, 250);// one skip on level 2
077: }
078: }
079:
080: public void checkSkipTo(TermPositions tp, int target, int maxCounter)
081: throws IOException {
082: tp.skipTo(target);
083: if (maxCounter < counter) {
084: fail("Too many bytes read: " + counter);
085: }
086:
087: assertEquals("Wrong document " + tp.doc()
088: + " after skipTo target " + target, target, tp.doc());
089: assertEquals("Frequency is not 1: " + tp.freq(), 1, tp.freq());
090: tp.nextPosition();
091: byte[] b = new byte[1];
092: tp.getPayload(b, 0);
093: assertEquals("Wrong payload for the target " + target + ": "
094: + b[0], (byte) target, b[0]);
095: }
096:
097: private static class PayloadAnalyzer extends Analyzer {
098: public TokenStream tokenStream(String fieldName, Reader reader) {
099: return new PayloadFilter(new LowerCaseTokenizer(reader));
100: }
101:
102: }
103:
104: private static class PayloadFilter extends TokenFilter {
105: static int count = 0;
106:
107: protected PayloadFilter(TokenStream input) {
108: super (input);
109: }
110:
111: public Token next() throws IOException {
112: Token t = input.next();
113: if (t != null) {
114: t
115: .setPayload(new Payload(
116: new byte[] { (byte) count++ }));
117: }
118: return t;
119: }
120:
121: }
122:
123: private int counter = 0;
124:
125: // Simply extends IndexInput in a way that we are able to count the number
126: // of bytes read
127: class CountingStream extends IndexInput {
128: private IndexInput input;
129:
130: CountingStream(IndexInput input) {
131: this .input = input;
132: }
133:
134: public byte readByte() throws IOException {
135: TestMultiLevelSkipList.this .counter++;
136: return this .input.readByte();
137: }
138:
139: public void readBytes(byte[] b, int offset, int len)
140: throws IOException {
141: TestMultiLevelSkipList.this .counter += len;
142: this .input.readBytes(b, offset, len);
143: }
144:
145: public void close() throws IOException {
146: this .input.close();
147: }
148:
149: public long getFilePointer() {
150: return this .input.getFilePointer();
151: }
152:
153: public void seek(long pos) throws IOException {
154: this .input.seek(pos);
155: }
156:
157: public long length() {
158: return this .input.length();
159: }
160:
161: public Object clone() {
162: return new CountingStream((IndexInput) this.input.clone());
163: }
164:
165: }
166: }
|