001: package org.apache.lucene.benchmark.utils;
002:
003: /**
004: * Copyright 2005 The Apache Software Foundation
005: *
006: * Licensed under the Apache License, Version 2.0 (the "License");
007: * you may not use this file except in compliance with the License.
008: * You may obtain a copy of the License at
009: *
010: * http://www.apache.org/licenses/LICENSE-2.0
011: *
012: * Unless required by applicable law or agreed to in writing, software
013: * distributed under the License is distributed on an "AS IS" BASIS,
014: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015: * See the License for the specific language governing permissions and
016: * limitations under the License.
017: */
018:
019: import java.io.BufferedReader;
020: import java.io.File;
021: import java.io.FileFilter;
022: import java.io.FileReader;
023: import java.io.FileWriter;
024: import java.io.IOException;
025: import java.util.regex.Matcher;
026: import java.util.regex.Pattern;
027:
028: /**
029: * Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
030: */
031: public class ExtractReuters {
032: private File reutersDir;
033: private File outputDir;
034: private static final String LINE_SEPARATOR = System
035: .getProperty("line.separator");
036:
037: public ExtractReuters(File reutersDir, File outputDir) {
038: this .reutersDir = reutersDir;
039: this .outputDir = outputDir;
040: System.out.println("Deleting all files in " + outputDir);
041: File[] files = outputDir.listFiles();
042: for (int i = 0; i < files.length; i++) {
043: files[i].delete();
044: }
045:
046: }
047:
048: public void extract() {
049: File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
050: public boolean accept(File file) {
051: return file.getName().endsWith(".sgm");
052: }
053: });
054: if (sgmFiles != null && sgmFiles.length > 0) {
055: for (int i = 0; i < sgmFiles.length; i++) {
056: File sgmFile = sgmFiles[i];
057: extractFile(sgmFile);
058: }
059: } else {
060: System.err.println("No .sgm files in " + reutersDir);
061: }
062: }
063:
064: Pattern EXTRACTION_PATTERN = Pattern
065: .compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
066:
067: private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };
068:
069: private static String[] META_CHARS_SERIALIZATIONS = { "&",
070: "<", ">", """, "'" };
071:
072: /**
073: * Override if you wish to change what is extracted
074: *
075: * @param sgmFile
076: */
077: protected void extractFile(File sgmFile) {
078: try {
079: BufferedReader reader = new BufferedReader(new FileReader(
080: sgmFile));
081:
082: StringBuffer buffer = new StringBuffer(1024);
083: StringBuffer outBuffer = new StringBuffer(1024);
084:
085: String line = null;
086: int index = -1;
087: int docNumber = 0;
088: while ((line = reader.readLine()) != null) {
089: //when we see a closing reuters tag, flush the file
090:
091: if ((index = line.indexOf("</REUTERS")) == -1) {
092: //Replace the SGM escape sequences
093:
094: buffer.append(line).append(' ');//accumulate the strings for now, then apply regular expression to get the pieces,
095: } else {
096: //Extract the relevant pieces and write to a file in the output dir
097: Matcher matcher = EXTRACTION_PATTERN
098: .matcher(buffer);
099: while (matcher.find()) {
100: for (int i = 1; i <= matcher.groupCount(); i++) {
101: if (matcher.group(i) != null) {
102: outBuffer.append(matcher.group(i));
103: }
104: }
105: outBuffer.append(LINE_SEPARATOR).append(
106: LINE_SEPARATOR);
107: }
108: String out = outBuffer.toString();
109: for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
110: out = out.replaceAll(
111: META_CHARS_SERIALIZATIONS[i],
112: META_CHARS[i]);
113: }
114: File outFile = new File(outputDir, sgmFile
115: .getName()
116: + "-" + (docNumber++) + ".txt");
117: //System.out.println("Writing " + outFile);
118: FileWriter writer = new FileWriter(outFile);
119: writer.write(out);
120: writer.close();
121: outBuffer.setLength(0);
122: buffer.setLength(0);
123: }
124: }
125: reader.close();
126: }
127:
128: catch (IOException e)
129:
130: {
131: throw new RuntimeException(e);
132: }
133: }
134:
135: public static void main(String[] args) {
136: if (args.length != 2) {
137: printUsage();
138: }
139: File reutersDir = new File(args[0]);
140:
141: if (reutersDir.exists()) {
142: File outputDir = new File(args[1]);
143: outputDir.mkdirs();
144: ExtractReuters extractor = new ExtractReuters(reutersDir,
145: outputDir);
146: extractor.extract();
147: } else {
148: printUsage();
149: }
150: }
151:
152: private static void printUsage() {
153: System.err
154: .println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
155: }
156: }
|