001: //xlsParser.java
002: //------------------------
003: //part of YaCy
004: //(C) by Michael Peter Christen; mc@anomic.de
005: //first published on http://www.anomic.de
006: //Frankfurt, Germany, 2005
007:
008: //this file is contributed by Tim Riemann
009: //last major change: 12.09.2006
010:
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015:
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020:
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024:
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036:
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.plasma.parser.xls;
045:
046: import java.io.InputStream;
047: import java.util.Hashtable;
048:
049: import org.apache.poi.hssf.eventusermodel.HSSFEventFactory;
050: import org.apache.poi.hssf.eventusermodel.HSSFListener;
051: import org.apache.poi.hssf.eventusermodel.HSSFRequest;
052: import org.apache.poi.hssf.record.LabelSSTRecord;
053: import org.apache.poi.hssf.record.NumberRecord;
054: import org.apache.poi.hssf.record.Record;
055: import org.apache.poi.hssf.record.SSTRecord;
056: import org.apache.poi.poifs.filesystem.POIFSFileSystem;
057:
058: import de.anomic.plasma.plasmaParserDocument;
059: import de.anomic.plasma.parser.AbstractParser;
060: import de.anomic.plasma.parser.Parser;
061: import de.anomic.plasma.parser.ParserException;
062: import de.anomic.yacy.yacyURL;
063:
064: public class xlsParser extends AbstractParser implements Parser,
065: HSSFListener {
066:
067: //StringBuffer for parsed text
068: private StringBuffer sbFoundStrings = null;
069:
070: //sstrecord needed for event parsing
071: private SSTRecord sstrec;
072:
073: /**
074: * a list of mime types that are supported by this parser class
075: * @see #getSupportedMimeTypes()
076: */
077: public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
078: static {
079: SUPPORTED_MIME_TYPES.put("application/msexcel", "xls");
080: SUPPORTED_MIME_TYPES.put("application/excel", "xls");
081: SUPPORTED_MIME_TYPES.put("application/vnd.ms-excel", "xls");
082: SUPPORTED_MIME_TYPES.put("application/x-excel", "xls");
083: SUPPORTED_MIME_TYPES.put("application/x-msexcel", "xls");
084: }
085:
086: /**
087: * a list of library names that are needed by this parser
088: * @see Parser#getLibxDependences()
089: */
090: private static final String[] LIBX_DEPENDENCIES = new String[] {
091: "poi-3.0-alpha2-20060616.jar",
092: "poi-scratchpad-3.0-alpha2-20060616.jar", };
093:
094: public xlsParser() {
095: super (LIBX_DEPENDENCIES);
096: this .parserName = "Microsoft Excel Parser";
097: this .parserVersionNr = "0.1";
098: }
099:
100: /*
101: * parses the source documents and returns a plasmaParserDocument containing
102: * all extracted information about the parsed document
103: */
104: public plasmaParserDocument parse(yacyURL location,
105: String mimeType, String charset, InputStream source)
106: throws ParserException, InterruptedException {
107: try {
108: //generate new StringBuffer for parsing
109: sbFoundStrings = new StringBuffer();
110:
111: //create a new org.apache.poi.poifs.filesystem.Filesystem
112: POIFSFileSystem poifs = new POIFSFileSystem(source);
113: //get the Workbook (excel part) stream in a InputStream
114: InputStream din = poifs
115: .createDocumentInputStream("Workbook");
116: //construct out HSSFRequest object
117: HSSFRequest req = new HSSFRequest();
118: //lazy listen for ALL records with the listener shown above
119: req.addListenerForAllRecords(this );
120: //create our event factory
121: HSSFEventFactory factory = new HSSFEventFactory();
122: //process our events based on the document input stream
123: factory.processEvents(req, din);
124: //close our document input stream (don't want to leak these!)
125: din.close();
126:
127: //now the parsed strings are in the StringBuffer, now convert them to a String
128: String contents = sbFoundStrings.toString();
129:
130: /*
131: * create the plasmaParserDocument for the database
132: * and set shortText and bodyText properly
133: */
134: plasmaParserDocument theDoc = new plasmaParserDocument(
135: location, mimeType, "UTF-8", null, ((contents
136: .length() > 80) ? contents.substring(0, 80)
137: : contents.trim()).replaceAll("\r\n", " ")
138: .replaceAll("\n", " ")
139: .replaceAll("\r", " ")
140: .replaceAll("\t", " "), "", // TODO: AUTHOR
141: null, null, contents.getBytes("UTF-8"), null, null);
142: return theDoc;
143: } catch (Exception e) {
144: if (e instanceof InterruptedException)
145: throw (InterruptedException) e;
146:
147: /*
148: * an unexpected error occurred, log it and throw a ParserException
149: */
150: String errorMsg = "Unable to parse the xls document '"
151: + location + "':" + e.getMessage();
152: this .theLogger.logSevere(errorMsg);
153: throw new ParserException(errorMsg, location);
154: } finally {
155: sbFoundStrings = null;
156: }
157: }
158:
159: public Hashtable<String, String> getSupportedMimeTypes() {
160: return SUPPORTED_MIME_TYPES;
161: }
162:
163: public void reset() {
164: //nothing to do
165: super .reset();
166: }
167:
168: public void processRecord(Record record) {
169: switch (record.getSid()) {
170: case NumberRecord.sid: {
171: NumberRecord numrec = (NumberRecord) record;
172: sbFoundStrings.append(numrec.getValue());
173: break;
174: }
175: //unique string records
176: case SSTRecord.sid: {
177: sstrec = (SSTRecord) record;
178: for (int k = 0; k < sstrec.getNumUniqueStrings(); k++) {
179: sbFoundStrings.append(sstrec.getString(k));
180:
181: //add line seperator
182: sbFoundStrings.append("\n");
183: }
184: break;
185: }
186:
187: case LabelSSTRecord.sid: {
188: LabelSSTRecord lsrec = (LabelSSTRecord) record;
189: sbFoundStrings
190: .append(sstrec.getString(lsrec.getSSTIndex()));
191: break;
192: }
193: }
194:
195: //add line seperator
196: sbFoundStrings.append("\n");
197: }
198: }
|