001: //docParser.java
002: //------------------------
003: //part of YaCy
004: //(C) by Michael Peter Christen; mc@anomic.de
005: //first published on http://www.anomic.de
006: //Frankfurt, Germany, 2005
007: //
008: //this file is contributed by Marc Nause
009: //last major change: 01.11.2006
010: //
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015: //
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020: //
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: //
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036: //
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.plasma.parser.swf;
045:
046: import java.io.InputStream;
047: import java.util.HashMap;
048: import java.util.Hashtable;
049:
050: import pt.tumba.parser.swf.SWF2HTML;
051: import de.anomic.plasma.plasmaParserDocument;
052: import de.anomic.plasma.parser.AbstractParser;
053: import de.anomic.plasma.parser.Parser;
054: import de.anomic.plasma.parser.ParserException;
055: import de.anomic.yacy.yacyURL;
056:
057: public class swfParser extends AbstractParser implements Parser {
058:
059: /**
060: * a list of mime types that are supported by this parser class
061: * @see #getSupportedMimeTypes()
062: */
063: public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
064: static {
065: SUPPORTED_MIME_TYPES
066: .put("application/x-shockwave-flash", "swf");
067: SUPPORTED_MIME_TYPES.put(
068: "application/x-shockwave-flash2-preview", "swf");
069: }
070:
071: /**
072: * a list of library names that are needed by this parser
073: * @see Parser#getLibxDependences()
074: */
075: private static final String[] LIBX_DEPENDENCIES = new String[] { "webcat-0.1-swf.jar" };
076:
077: public swfParser() {
078: super (LIBX_DEPENDENCIES);
079: this .parserName = "Adobe Flash Parser";
080: this .parserVersionNr = "0.1";
081: }
082:
083: /**
084: * returns a hashtable containing the mimetypes that are supported by this class
085: */
086: public Hashtable<String, String> getSupportedMimeTypes() {
087: return SUPPORTED_MIME_TYPES;
088: }
089:
090: /*
091: * parses the source documents and returns a plasmaParserDocument containing
092: * all extracted information about the parsed document
093: */
094: public plasmaParserDocument parse(yacyURL location,
095: String mimeType, String charset, InputStream source)
096: throws ParserException, InterruptedException {
097:
098: try {
099: SWF2HTML swf2html = new SWF2HTML();
100: String contents = swf2html.convertSWFToHTML(source);
101: String url = null;
102: String urlnr = null;
103: String linebreak = System.getProperty("line.separator");
104: String[] sections = null;
105: String abstrct = null;
106: //TreeSet images = null;
107: HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
108: int urls = 0;
109: int urlStart = -1;
110: int urlEnd = 0;
111: int p0 = 0;
112:
113: //getting rid of HTML-Tags
114: p0 = contents.indexOf("<html><body>");
115: contents = contents.substring(p0 + 12);
116: p0 = contents.indexOf("</body></html>");
117: contents = contents.substring(0, p0);
118:
119: //extracting urls
120: while ((urlStart = contents.indexOf("http://", urlEnd)) >= 0) {
121: urlEnd = contents.indexOf(linebreak, urlStart);
122: url = contents.substring(urlStart, urlEnd);
123: urlnr = (new Integer(++urls)).toString();
124: anchors.put(new yacyURL(url, null), urlnr);
125: contents = contents.substring(0, urlStart)
126: + contents.substring(urlEnd);
127: }
128:
129: // As the result of parsing this function must return a plasmaParserDocument object
130: plasmaParserDocument theDoc = new plasmaParserDocument(
131: location, // url of the source document
132: mimeType, // the documents mime type
133: "UTF-8", // charset of the document text
134: null, //keywords
135: ((contents.length() > 80) ? contents.substring(0,
136: 80) : contents.trim()).replaceAll("\r\n",
137: " ").replaceAll("\n", " ").replaceAll("\r",
138: " ").replaceAll("\t", " "), // title
139: "", // TODO: AUTHOR
140: sections, // an array of section headlines
141: abstrct, // an abstract
142: contents.getBytes("UTF-8"), // the parsed document text
143: anchors, // a map of extracted anchors
144: null); // a treeset of image URLs
145: return theDoc;
146: } catch (Exception e) {
147: if (e instanceof InterruptedException)
148: throw (InterruptedException) e;
149:
150: // if an unexpected error occures just log the error and raise a new ParserException
151: String errorMsg = "Unable to parse the swf document '"
152: + location + "':" + e.getMessage();
153: this .theLogger.logSevere(errorMsg);
154: throw new ParserException(errorMsg, location);
155: }
156: }
157:
158: public void reset() {
159: // this code is executed if the parser class is returned into the parser pool
160: super.reset();
161:
162: }
163: }
|