001: package org.apache.lucene.benchmark.byTask.feeds;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import org.apache.lucene.benchmark.byTask.utils.Config;
021: import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask;
022:
023: import org.apache.lucene.document.Document;
024: import org.apache.lucene.document.Field;
025:
026: import java.io.BufferedReader;
027: import java.io.IOException;
028: import java.io.FileInputStream;
029: import java.io.InputStreamReader;
030:
031: /**
032: * A DocMaker reading one line at a time as a Document from
033: * a single file. This saves IO cost (over DirDocMaker) of
034: * recursing through a directory and opening a new file for
035: * every document. It also re-uses its Document and Field
036: * instance to improve indexing speed.
037: *
038: * Config properties:
039: * docs.file=<path to the file%gt;
040: */
041: public class LineDocMaker extends BasicDocMaker {
042:
043: FileInputStream fileIS;
044: BufferedReader fileIn;
045: ThreadLocal docState = new ThreadLocal();
046: private String fileName;
047:
048: private static int READER_BUFFER_BYTES = 64 * 1024;
049:
050: class DocState {
051: Document doc;
052: Field bodyField;
053: Field titleField;
054: Field dateField;
055: Field idField;
056:
057: public DocState() {
058:
059: bodyField = new Field(BasicDocMaker.BODY_FIELD, "",
060: storeVal, Field.Index.TOKENIZED, termVecVal);
061: titleField = new Field(BasicDocMaker.TITLE_FIELD, "",
062: storeVal, Field.Index.TOKENIZED, termVecVal);
063: dateField = new Field(BasicDocMaker.DATE_FIELD, "",
064: storeVal, Field.Index.TOKENIZED, termVecVal);
065: idField = new Field(BasicDocMaker.ID_FIELD, "",
066: Field.Store.YES, Field.Index.NO_NORMS);
067:
068: doc = new Document();
069: doc.add(bodyField);
070: doc.add(titleField);
071: doc.add(dateField);
072: doc.add(idField);
073: }
074:
075: final static String SEP = WriteLineDocTask.SEP;
076:
077: public Document setFields(String line) {
078: // title <TAB> date <TAB> body <NEWLINE>
079: int spot = line.indexOf(SEP);
080: if (spot != -1) {
081: titleField.setValue(line.substring(0, spot));
082: int spot2 = line.indexOf(SEP, 1 + spot);
083: if (spot2 != -1) {
084: dateField.setValue(line.substring(1 + spot, spot2));
085: bodyField.setValue(line.substring(1 + spot2, line
086: .length()));
087: } else {
088: dateField.setValue("");
089: bodyField.setValue("");
090: }
091: } else
092: titleField.setValue("");
093: return doc;
094: }
095: }
096:
097: protected DocData getNextDocData() throws Exception {
098: throw new RuntimeException("not implemented");
099: }
100:
101: private DocState getDocState() {
102: DocState ds = (DocState) docState.get();
103: if (ds == null) {
104: ds = new DocState();
105: docState.set(ds);
106: }
107: return ds;
108: }
109:
110: public Document makeDocument() throws Exception {
111:
112: String line;
113: synchronized (this ) {
114: while (true) {
115: line = fileIn.readLine();
116: if (line == null) {
117: // Reset the file
118: openFile();
119: if (!forever)
120: throw new NoMoreDataException();
121: } else {
122: break;
123: }
124: }
125: }
126:
127: return getDocState().setFields(line);
128: }
129:
130: public Document makeDocument(int size) throws Exception {
131: throw new RuntimeException(
132: "cannot change document size with LineDocMaker; please use DirDocMaker instead");
133: }
134:
135: public synchronized void resetInputs() {
136: super .resetInputs();
137: fileName = config.get("docs.file", null);
138: if (fileName == null)
139: throw new RuntimeException("docs.file must be set");
140: openFile();
141: }
142:
143: synchronized void openFile() {
144: try {
145: if (fileIn != null)
146: fileIn.close();
147: fileIS = new FileInputStream(fileName);
148: fileIn = new BufferedReader(new InputStreamReader(fileIS,
149: "UTF-8"), READER_BUFFER_BYTES);
150: } catch (IOException e) {
151: throw new RuntimeException(e);
152: }
153: }
154:
155: public int numUniqueTexts() {
156: return -1;
157: }
158: }
|