001: //pdfParser.java
002: //------------------------
003: //part of YaCy
004: //(C) by Michael Peter Christen; mc@anomic.de
005: //first published on http://www.anomic.de
006: //Frankfurt, Germany, 2005
007: //
008: //this file is contributed by Martin Thelian
009: //last major change: 24.04.2005
010: //
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015: //
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020: //
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: //
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036: //
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.plasma.parser.pdf;
045:
046: import java.io.File;
047: import java.io.FileOutputStream;
048: import java.io.InputStream;
049: import java.io.OutputStreamWriter;
050: import java.io.Writer;
051: import java.util.Hashtable;
052:
053: import org.pdfbox.pdfparser.PDFParser;
054: import org.pdfbox.pdmodel.PDDocument;
055: import org.pdfbox.pdmodel.PDDocumentInformation;
056: import org.pdfbox.pdmodel.encryption.StandardDecryptionMaterial;
057: import org.pdfbox.util.PDFTextStripper;
058:
059: import de.anomic.plasma.plasmaCrawlEURL;
060: import de.anomic.plasma.plasmaParserDocument;
061: import de.anomic.plasma.parser.AbstractParser;
062: import de.anomic.plasma.parser.Parser;
063: import de.anomic.plasma.parser.ParserException;
064: import de.anomic.server.serverCharBuffer;
065: import de.anomic.yacy.yacyURL;
066:
067: public class pdfParser extends AbstractParser implements Parser {
068:
069: /**
070: * a list of mime types that are supported by this parser class
071: * @see #getSupportedMimeTypes()
072: */
073: public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
074: static {
075: SUPPORTED_MIME_TYPES.put("application/pdf", "pdf");
076: }
077:
078: /**
079: * a list of library names that are needed by this parser
080: * @see Parser#getLibxDependences()
081: */
082: private static final String[] LIBX_DEPENDENCIES = new String[] {
083: "PDFBox-0.7.3.jar", "FontBox-0.1.0-dev.jar",
084: "bcprov-jdk14-132.jar" };
085:
086: public pdfParser() {
087: super (LIBX_DEPENDENCIES);
088: this .parserName = "Acrobat Portable Document Parser";
089: }
090:
091: public Hashtable<String, String> getSupportedMimeTypes() {
092: return SUPPORTED_MIME_TYPES;
093: }
094:
095: public plasmaParserDocument parse(yacyURL location,
096: String mimeType, String charset, InputStream source)
097: throws ParserException, InterruptedException {
098:
099: PDDocument theDocument = null;
100: Writer writer = null;
101: File writerFile = null;
102: try {
103: // reducing thread priority
104: Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
105:
106: // deactivating the logging for jMimeMagic
107: // Logger theLogger = Logger.getLogger("org.pdfbox");
108: // theLogger.setLevel(Level.INFO);
109:
110: String docTitle = null, docSubject = null, docAuthor = null, docKeywordStr = null;
111:
112: // check for interruption
113: checkInterruption();
114:
115: // creating a pdf parser
116: PDFParser parser = new PDFParser(source);
117: parser.parse();
118:
119: // check for interruption
120: checkInterruption();
121:
122: // creating a text stripper
123: PDFTextStripper stripper = new PDFTextStripper();
124: theDocument = parser.getPDDocument();
125:
126: if (theDocument.isEncrypted()) {
127: theDocument
128: .openProtection(new StandardDecryptionMaterial(
129: ""));
130: if (!theDocument.getCurrentAccessPermission()
131: .canExtractContent())
132: throw new ParserException("Document is encrypted",
133: location,
134: plasmaCrawlEURL.DENIED_DOCUMENT_ENCRYPTED);
135: }
136:
137: // extracting some metadata
138: PDDocumentInformation theDocInfo = theDocument
139: .getDocumentInformation();
140: if (theDocInfo != null) {
141: docTitle = theDocInfo.getTitle();
142: docSubject = theDocInfo.getSubject();
143: docAuthor = theDocInfo.getAuthor();
144: docKeywordStr = theDocInfo.getKeywords();
145: }
146:
147: // creating a writer for output
148: if ((this .contentLength == -1)
149: || (this .contentLength > Parser.MAX_KEEP_IN_MEMORY_SIZE)) {
150: writerFile = File.createTempFile("pdfParser", ".tmp");
151: writer = new OutputStreamWriter(new FileOutputStream(
152: writerFile), "UTF-8");
153: } else {
154: writer = new serverCharBuffer();
155: }
156:
157: stripper.writeText(theDocument, writer);
158: theDocument.close();
159: theDocument = null;
160: writer.close();
161:
162: String[] docKeywords = null;
163: if (docKeywordStr != null)
164: docKeywords = docKeywordStr.split(" |,");
165:
166: plasmaParserDocument theDoc = null;
167:
168: if (writer instanceof serverCharBuffer) {
169: byte[] contentBytes = ((serverCharBuffer) writer)
170: .toString().getBytes("UTF-8");
171: theDoc = new plasmaParserDocument(location, mimeType,
172: "UTF-8", docKeywords,
173: (docTitle == null) ? docSubject : docTitle,
174: docAuthor, null, null, contentBytes, null, null);
175: } else {
176: theDoc = new plasmaParserDocument(location, mimeType,
177: "UTF-8", docKeywords,
178: (docTitle == null) ? docSubject : docTitle,
179: docAuthor, null, null, writerFile, null, null);
180: }
181:
182: return theDoc;
183: } catch (Exception e) {
184: if (e instanceof InterruptedException)
185: throw (InterruptedException) e;
186: if (e instanceof ParserException)
187: throw (ParserException) e;
188:
189: // close the writer
190: if (writer != null)
191: try {
192: writer.close();
193: } catch (Exception ex) {/* ignore this */
194: }
195:
196: // delete the file
197: if (writerFile != null)
198: try {
199: writerFile.delete();
200: } catch (Exception ex) {/* ignore this */
201: }
202:
203: throw new ParserException(
204: "Unexpected error while parsing pdf file. "
205: + e.getMessage(), location);
206: } finally {
207: if (theDocument != null)
208: try {
209: theDocument.close();
210: } catch (Exception e) {/* ignore this */
211: }
212: if (writer != null)
213: try {
214: writer.close();
215: } catch (Exception e) {/* ignore this */
216: }
217: Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
218: }
219: }
220:
221: public void reset() {
222: // Nothing todo here at the moment
223: super.reset();
224: }
225:
226: }
|