001: //rssParser.java
002: //------------------------
003: //part of YaCy
004: //(C) by Michael Peter Christen; mc@anomic.de
005: //first published on http://www.anomic.de
006: //Frankfurt, Germany, 2005
007: //
008: //this file is contributed by Martin Thelian
009: //last major change: 16.05.2005
010: //
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015: //
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020: //
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: //
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036: //
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.plasma.parser.rss;
045:
046: import java.io.ByteArrayInputStream;
047: import java.io.InputStream;
048: import java.io.Writer;
049: import java.util.HashMap;
050: import java.util.Hashtable;
051: import java.util.LinkedList;
052: import java.util.Map;
053: import java.util.TreeSet;
054:
055: import de.anomic.htmlFilter.htmlFilterAbstractScraper;
056: import de.anomic.htmlFilter.htmlFilterContentScraper;
057: import de.anomic.htmlFilter.htmlFilterImageEntry;
058: import de.anomic.htmlFilter.htmlFilterWriter;
059: import de.anomic.plasma.plasmaParserDocument;
060: import de.anomic.plasma.parser.AbstractParser;
061: import de.anomic.plasma.parser.Parser;
062: import de.anomic.plasma.parser.ParserException;
063: import de.anomic.server.serverByteBuffer;
064: import de.anomic.server.serverCharBuffer;
065: import de.anomic.server.serverFileUtils;
066: import de.anomic.xml.rssReader;
067: import de.anomic.xml.rssReader.Item;
068: import de.anomic.yacy.yacyURL;
069:
070: public class rssParser extends AbstractParser implements Parser {
071:
072: /**
073: * a list of mime types that are supported by this parser class
074: * @see #getSupportedMimeTypes()
075: */
076: public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
077: static {
078: SUPPORTED_MIME_TYPES.put("text/rss", "xml,rss,rdf");
079: SUPPORTED_MIME_TYPES.put("application/rdf+xml", "xml,rss,rdf");
080: SUPPORTED_MIME_TYPES.put("application/rss+xml", "xml,rss,rdf");
081: SUPPORTED_MIME_TYPES.put("application/atom+xml", "xml,atom");
082: }
083:
084: /**
085: * a list of library names that are needed by this parser
086: * @see Parser#getLibxDependences()
087: */
088: private static final String[] LIBX_DEPENDENCIES = new String[] {};
089:
090: public rssParser() {
091: super (LIBX_DEPENDENCIES);
092: this .parserName = "Rich Site Summary/Atom Feed Parser";
093: }
094:
095: public plasmaParserDocument parse(yacyURL location,
096: String mimeType, String charset, InputStream source)
097: throws ParserException, InterruptedException {
098:
099: try {
100: LinkedList<String> feedSections = new LinkedList<String>();
101: HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
102: TreeSet<htmlFilterImageEntry> images = new TreeSet<htmlFilterImageEntry>();
103: serverByteBuffer text = new serverByteBuffer();
104: serverCharBuffer authors = new serverCharBuffer();
105:
106: rssReader reader = new rssReader(source);
107:
108: // getting the rss feed title and description
109: String feedTitle = reader.getChannel().getTitle();
110:
111: // getting feed creator
112: String feedCreator = reader.getChannel().getAuthor();
113: if (feedCreator != null && feedCreator.length() > 0)
114: authors.append(",").append(feedCreator);
115:
116: // getting the feed description
117: String feedDescription = reader.getChannel()
118: .getDescription();
119:
120: if (reader.getImage() != null) {
121: images.add(new htmlFilterImageEntry(new yacyURL(reader
122: .getImage(), null), feedTitle, -1, -1));
123: }
124:
125: // loop through the feed items
126: for (int i = 0; i < reader.items(); i++) {
127: // check for interruption
128: checkInterruption();
129:
130: // getting the next item
131: Item item = reader.getItem(i);
132:
133: String itemTitle = item.getTitle();
134: yacyURL itemURL = new yacyURL(item.getLink(), null);
135: String itemDescr = item.getDescription();
136: String itemCreator = item.getCreator();
137: if (itemCreator != null && itemCreator.length() > 0)
138: authors.append(",").append(itemCreator);
139:
140: feedSections.add(itemTitle);
141: anchors.put(itemURL, itemTitle);
142:
143: if ((text.length() != 0)
144: && (text.byteAt(text.length() - 1) != 32))
145: text.append((byte) 32);
146: text.append(
147: new serverCharBuffer(htmlFilterAbstractScraper
148: .stripAll(new serverCharBuffer(
149: itemDescr.toCharArray())))
150: .trim().toString()).append(' ');
151:
152: String itemContent = item.getDescription();
153: if ((itemContent != null) && (itemContent.length() > 0)) {
154:
155: htmlFilterContentScraper scraper = new htmlFilterContentScraper(
156: itemURL);
157: Writer writer = new htmlFilterWriter(null, null,
158: scraper, null, false);
159: serverFileUtils.copy(new ByteArrayInputStream(
160: itemContent.getBytes("UTF-8")), writer,
161: "UTF-8");
162:
163: String itemHeadline = scraper.getTitle();
164: if ((itemHeadline != null)
165: && (itemHeadline.length() > 0)) {
166: feedSections.add(itemHeadline);
167: }
168:
169: Map<yacyURL, String> itemLinks = scraper
170: .getAnchors();
171: if ((itemLinks != null) && (itemLinks.size() > 0)) {
172: anchors.putAll(itemLinks);
173: }
174:
175: TreeSet<htmlFilterImageEntry> itemImages = scraper
176: .getImages();
177: if ((itemImages != null) && (itemImages.size() > 0)) {
178: images.addAll(itemImages);
179: }
180:
181: byte[] extractedText = scraper.getText();
182: if ((extractedText != null)
183: && (extractedText.length > 0)) {
184: if ((text.length() != 0)
185: && (text.byteAt(text.length() - 1) != 32))
186: text.append((byte) 32);
187: text.append(scraper.getText());
188: }
189:
190: }
191: }
192:
193: plasmaParserDocument theDoc = new plasmaParserDocument(
194: location, mimeType, "UTF-8", null, feedTitle,
195: (authors.length() > 0) ? authors.toString(1,
196: authors.length()) : "",
197: (String[]) feedSections
198: .toArray(new String[feedSections.size()]),
199: feedDescription, text.getBytes(), anchors, images);
200:
201: return theDoc;
202:
203: } catch (Exception e) {
204: if (e instanceof InterruptedException)
205: throw (InterruptedException) e;
206: if (e instanceof ParserException)
207: throw (ParserException) e;
208:
209: throw new ParserException(
210: "Unexpected error while parsing rss file."
211: + e.getMessage(), location);
212: }
213: }
214:
215: public Hashtable<String, String> getSupportedMimeTypes() {
216: return SUPPORTED_MIME_TYPES;
217: }
218:
219: public void reset() {
220: // Nothing todo here at the moment
221: super.reset();
222: }
223:
224: }
|