001: package org.apache.lucene.benchmark.quality;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import java.io.BufferedReader;
021: import java.io.File;
022: import java.io.FileReader;
023: import java.io.PrintWriter;
024:
025: import org.apache.lucene.benchmark.byTask.TestPerfTasksLogic;
026: import org.apache.lucene.benchmark.byTask.feeds.ReutersDocMaker;
027: import org.apache.lucene.benchmark.quality.Judge;
028: import org.apache.lucene.benchmark.quality.QualityQuery;
029: import org.apache.lucene.benchmark.quality.QualityQueryParser;
030: import org.apache.lucene.benchmark.quality.QualityBenchmark;
031: import org.apache.lucene.benchmark.quality.trec.TrecJudge;
032: import org.apache.lucene.benchmark.quality.trec.TrecTopicsReader;
033: import org.apache.lucene.benchmark.quality.utils.SimpleQQParser;
034: import org.apache.lucene.benchmark.quality.utils.SubmissionReport;
035: import org.apache.lucene.search.IndexSearcher;
036: import org.apache.lucene.store.FSDirectory;
037:
038: import junit.framework.TestCase;
039:
040: /**
041: * Test that quality run does its job.
042: */
043: public class TestQualityRun extends TestCase {
044:
045: private static boolean DEBUG = Boolean.getBoolean("tests.verbose");
046:
047: /**
048: * @param arg0
049: */
050: public TestQualityRun(String name) {
051: super (name);
052: }
053:
054: public void testTrecQuality() throws Exception {
055: // first create the complete reuters index
056: createReutersIndex();
057:
058: File workDir = new File(System.getProperty(
059: "benchmark.work.dir", "work"));
060: assertTrue("Bad workDir: " + workDir, workDir.exists()
061: && workDir.isDirectory());
062:
063: int maxResults = 1000;
064: String docNameField = "docid";
065:
066: PrintWriter logger = DEBUG ? new PrintWriter(System.out, true)
067: : null;
068:
069: // <tests src dir> for topics/qrels files - src/test/org/apache/lucene/benchmark/quality
070: File srcTestDir = new File(new File(new File(new File(new File(
071: new File(new File(workDir.getAbsoluteFile()
072: .getParentFile(), "src"), "test"), "org"),
073: "apache"), "lucene"), "benchmark"), "quality");
074:
075: // prepare topics
076: File topicsFile = new File(srcTestDir, "trecTopics.txt");
077: assertTrue("Bad topicsFile: " + topicsFile, topicsFile.exists()
078: && topicsFile.isFile());
079: TrecTopicsReader qReader = new TrecTopicsReader();
080: QualityQuery qqs[] = qReader.readQueries(new BufferedReader(
081: new FileReader(topicsFile)));
082:
083: // prepare judge
084: File qrelsFile = new File(srcTestDir, "trecQRels.txt");
085: assertTrue("Bad qrelsFile: " + qrelsFile, qrelsFile.exists()
086: && qrelsFile.isFile());
087: Judge judge = new TrecJudge(new BufferedReader(new FileReader(
088: qrelsFile)));
089:
090: // validate topics & judgments match each other
091: judge.validateData(qqs, logger);
092:
093: IndexSearcher searcher = new IndexSearcher(FSDirectory
094: .getDirectory(new File(workDir, "index")));
095:
096: QualityQueryParser qqParser = new SimpleQQParser("title",
097: "body");
098: QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser,
099: searcher, docNameField);
100:
101: SubmissionReport submitLog = DEBUG ? new SubmissionReport(
102: logger, "TestRun") : null;
103: qrun.setMaxResults(maxResults);
104: QualityStats stats[] = qrun.execute(judge, submitLog, logger);
105:
106: // --------- verify by the way judgments were altered for this test:
107: // for some queries, depending on m = qnum % 8
108: // m==0: avg_precision and recall are hurt, by marking fake docs as relevant
109: // m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
110: // m==2: all precision, precision_at_n and recall are hurt.
111: // m>=3: these queries remain perfect
112: for (int i = 0; i < stats.length; i++) {
113: QualityStats s = stats[i];
114: switch (i % 8) {
115:
116: case 0:
117: assertTrue("avg-p should be hurt: " + s.getAvp(),
118: 1.0 > s.getAvp());
119: assertTrue("recall should be hurt: " + s.getRecall(),
120: 1.0 > s.getRecall());
121: for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
122: assertEquals("p_at_" + j + " should be perfect: "
123: + s.getPrecisionAt(j), 1.0, s
124: .getPrecisionAt(j), 1E-9);
125: }
126: break;
127:
128: case 1:
129: assertTrue("avg-p should be hurt", 1.0 > s.getAvp());
130: assertEquals("recall should be perfect: "
131: + s.getRecall(), 1.0, s.getRecall(), 1E-9);
132: for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
133: assertTrue("p_at_" + j + " should be hurt: "
134: + s.getPrecisionAt(j), 1.0 > s
135: .getPrecisionAt(j));
136: }
137: break;
138:
139: case 2:
140: assertTrue("avg-p should be hurt: " + s.getAvp(),
141: 1.0 > s.getAvp());
142: assertTrue("recall should be hurt: " + s.getRecall(),
143: 1.0 > s.getRecall());
144: for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
145: assertTrue("p_at_" + j + " should be hurt: "
146: + s.getPrecisionAt(j), 1.0 > s
147: .getPrecisionAt(j));
148: }
149: break;
150:
151: default: {
152: assertEquals("avg-p should be perfect: " + s.getAvp(),
153: 1.0, s.getAvp(), 1E-9);
154: assertEquals("recall should be perfect: "
155: + s.getRecall(), 1.0, s.getRecall(), 1E-9);
156: for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
157: assertEquals("p_at_" + j + " should be perfect: "
158: + s.getPrecisionAt(j), 1.0, s
159: .getPrecisionAt(j), 1E-9);
160: }
161: }
162:
163: }
164: }
165:
166: QualityStats avg = QualityStats.average(stats);
167: if (logger != null) {
168: avg.log("Average statistis:", 1, logger, " ");
169: }
170:
171: assertTrue("mean avg-p should be hurt: " + avg.getAvp(),
172: 1.0 > avg.getAvp());
173: assertTrue("avg recall should be hurt: " + avg.getRecall(),
174: 1.0 > avg.getRecall());
175: for (int j = 1; j <= QualityStats.MAX_POINTS; j++) {
176: assertTrue("avg p_at_" + j + " should be hurt: "
177: + avg.getPrecisionAt(j), 1.0 > avg
178: .getPrecisionAt(j));
179: }
180:
181: }
182:
183: // use benchmark logic to create the full Reuters index
184: private void createReutersIndex() throws Exception {
185: // 1. alg definition
186: String algLines[] = { "# ----- properties ",
187: "doc.maker=" + ReutersDocMaker.class.getName(),
188: "doc.add.log.step=2500", "doc.term.vector=false",
189: "doc.maker.forever=false", "directory=FSDirectory",
190: "doc.stored=true", "doc.tokenized=true",
191: "# ----- alg ", "ResetSystemErase", "CreateIndex",
192: "{ AddDoc } : *", "CloseIndex", };
193:
194: // 2. execute the algorithm (required in every "logic" test)
195: TestPerfTasksLogic.execBenchmark(algLines);
196: }
197: }
|