001: //vcfParser.java
002: //------------------------
003: //part of YaCy
004: //(C) by Michael Peter Christen; mc@anomic.de
005: //first published on http://www.anomic.de
006: //Frankfurt, Germany, 2005
007: //
008: //this file is contributed by Martin Thelian
009: //last major change: 20.11.2005
010: //
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015: //
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020: //
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: //
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036: //
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.plasma.parser.vcf;
045:
046: import java.io.BufferedReader;
047: import java.io.ByteArrayInputStream;
048: import java.io.InputStream;
049: import java.io.InputStreamReader;
050: import java.net.MalformedURLException;
051: import java.util.HashMap;
052: import java.util.Hashtable;
053: import java.util.Iterator;
054: import java.util.LinkedList;
055:
056: import de.anomic.http.httpc;
057: import de.anomic.kelondro.kelondroBase64Order;
058: import de.anomic.plasma.plasmaParserDocument;
059: import de.anomic.plasma.parser.AbstractParser;
060: import de.anomic.plasma.parser.Parser;
061: import de.anomic.plasma.parser.ParserException;
062: import de.anomic.yacy.yacyURL;
063:
064: /**
065: * Vcard specification: http://www.imc.org/pdi/vcard-21.txt
066: * @author theli
067: *
068: */
069: public class vcfParser extends AbstractParser implements Parser {
070:
071: /**
072: * a list of mime types that are supported by this parser class
073: * @see #getSupportedMimeTypes()
074: *
075: * TODO: support of x-mozilla-cpt and x-mozilla-html tags
076: */
077: public static final Hashtable<String, String> SUPPORTED_MIME_TYPES = new Hashtable<String, String>();
078: static {
079: SUPPORTED_MIME_TYPES.put("text/x-vcard", "vcf");
080: SUPPORTED_MIME_TYPES.put("application/vcard", "vcf");
081: }
082:
083: /**
084: * a list of library names that are needed by this parser
085: * @see Parser#getLibxDependences()
086: */
087: private static final String[] LIBX_DEPENDENCIES = new String[] {};
088:
089: public vcfParser() {
090: super (LIBX_DEPENDENCIES);
091: this .parserName = "vCard Parser";
092: }
093:
094: public Hashtable<String, String> getSupportedMimeTypes() {
095: return SUPPORTED_MIME_TYPES;
096: }
097:
098: public plasmaParserDocument parse(yacyURL location,
099: String mimeType, String charset, InputStream source)
100: throws ParserException, InterruptedException {
101:
102: try {
103: StringBuffer parsedTitle = new StringBuffer();
104: StringBuffer parsedDataText = new StringBuffer();
105: HashMap<String, String> parsedData = new HashMap<String, String>();
106: HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
107: LinkedList<String> parsedNames = new LinkedList<String>();
108:
109: boolean useLastLine = false;
110: int lineNr = 0;
111: String line = null;
112: BufferedReader inputReader = (charset != null) ? new BufferedReader(
113: new InputStreamReader(source, charset))
114: : new BufferedReader(new InputStreamReader(source));
115: while (true) {
116: // check for interruption
117: checkInterruption();
118:
119: // getting the next line
120: if (!useLastLine) {
121: line = inputReader.readLine();
122: } else {
123: useLastLine = false;
124: }
125:
126: if (line == null)
127: break;
128: else if (line.length() == 0)
129: continue;
130:
131: lineNr++;
132: int pos = line.indexOf(":");
133: if (pos != -1) {
134: String key = line.substring(0, pos).trim()
135: .toUpperCase();
136: String value = line.substring(pos + 1).trim();
137:
138: String encoding = null;
139: String[] keyParts = key.split(";");
140: if (keyParts.length > 1) {
141: for (int i = 0; i < keyParts.length; i++) {
142: if (keyParts[i].toUpperCase().startsWith(
143: "ENCODING")) {
144: encoding = keyParts[i]
145: .substring("ENCODING".length() + 1);
146: } else if (keyParts[i].toUpperCase()
147: .startsWith("QUOTED-PRINTABLE")) {
148: encoding = "QUOTED-PRINTABLE";
149: } else if (keyParts[i].toUpperCase()
150: .startsWith("BASE64")) {
151: encoding = "BASE64";
152: }
153:
154: }
155: if (encoding != null) {
156: try {
157: if (encoding
158: .equalsIgnoreCase("QUOTED-PRINTABLE")) {
159: // if the value has multiple lines ...
160: if (line.endsWith("=")) {
161: do {
162: value = value.substring(0,
163: value.length() - 1);
164: line = inputReader
165: .readLine();
166: if (line == null)
167: break;
168: value += line;
169: } while (line.endsWith("="));
170: }
171: value = decodeQuotedPrintable(value);
172: } else if (encoding
173: .equalsIgnoreCase("base64")) {
174: do {
175: line = inputReader.readLine();
176: if (line == null)
177: break;
178: if (line.indexOf(":") != -1) {
179: // we have detected an illegal block end of the base64 data
180: useLastLine = true;
181: }
182: if (!useLastLine)
183: value += line.trim();
184: else
185: break;
186: } while (line.length() != 0);
187: value = kelondroBase64Order.standardCoder
188: .decodeString(value,
189: "de.anomic.plasma.parser.vcf.vcfParser.parse(...)");
190: }
191: } catch (Exception ey) {
192: // Encoding error: This could occure e.g. if the base64 doesn't
193: // end with an empty newline
194: //
195: // We can simply ignore it.
196: }
197: }
198: }
199:
200: if (key.equalsIgnoreCase("END")) {
201: String name = null, title = null;
202:
203: // using the name of the current version as section headline
204: if (parsedData.containsKey("FN")) {
205: parsedNames.add(name = (String) parsedData
206: .get("FN"));
207: } else if (parsedData.containsKey("N")) {
208: parsedNames.add(name = (String) parsedData
209: .get("N"));
210: } else {
211: parsedNames.add(name = "unknown name");
212: }
213:
214: // getting the vcard title
215: if (parsedData.containsKey("TITLE")) {
216: parsedNames.add(title = (String) parsedData
217: .get("TITLE"));
218: }
219:
220: if (parsedTitle.length() > 0)
221: parsedTitle.append(", ");
222: parsedTitle.append((title == null) ? name
223: : name + " - " + title);
224:
225: // looping through the properties and add there values to
226: // the text representation of the vCard
227: Iterator<String> iter = parsedData.values()
228: .iterator();
229: while (iter.hasNext()) {
230: value = (String) iter.next();
231: parsedDataText.append(value).append("\r\n");
232: }
233: parsedDataText.append("\r\n");
234: parsedData.clear();
235: } else if (key.toUpperCase().startsWith("URL")) {
236: try {
237: yacyURL newURL = new yacyURL(value, null);
238: anchors.put(newURL, newURL.toString());
239: //parsedData.put(key,value);
240: } catch (MalformedURLException ex) {/* ignore this */
241: }
242: } else if (!key.equalsIgnoreCase("BEGIN")
243: && !key.equalsIgnoreCase("END")
244: && !key.equalsIgnoreCase("VERSION")
245: && !key.toUpperCase().startsWith("LOGO")
246: && !key.toUpperCase().startsWith("PHOTO")
247: && !key.toUpperCase().startsWith("SOUND")
248: && !key.toUpperCase().startsWith("KEY")
249: && !key.toUpperCase().startsWith("X-")) {
250: // value = value.replaceAll(";","\t");
251: if ((value.length() > 0))
252: parsedData.put(key, value);
253: }
254:
255: } else {
256: this .theLogger.logFinest("Invalid data in vcf file"
257: + "\n\tURL: " + location + "\n\tLine: "
258: + line + "\n\tLine-Nr: " + lineNr);
259: }
260: }
261:
262: String[] sections = (String[]) parsedNames
263: .toArray(new String[parsedNames.size()]);
264: byte[] text = parsedDataText.toString().getBytes();
265: plasmaParserDocument theDoc = new plasmaParserDocument(
266: location, // url of the source document
267: mimeType, // the documents mime type
268: null, null, // a list of extracted keywords
269: parsedTitle.toString(), // a long document title
270: "", // TODO: AUTHOR
271: sections, // an array of section headlines
272: "vCard", // an abstract
273: text, // the parsed document text
274: anchors, // a map of extracted anchors
275: null); // a treeset of image URLs
276: return theDoc;
277: } catch (Exception e) {
278: if (e instanceof InterruptedException)
279: throw (InterruptedException) e;
280: if (e instanceof ParserException)
281: throw (ParserException) e;
282:
283: throw new ParserException(
284: "Unexpected error while parsing vcf resource. "
285: + e.getMessage(), location);
286: }
287: }
288:
289: public void reset() {
290: // Nothing todo here at the moment
291: super .reset();
292: }
293:
294: public static final String decodeQuotedPrintable(String s) {
295: if (s == null)
296: return null;
297: byte[] b = s.getBytes();
298: StringBuffer sb = new StringBuffer();
299: for (int i = 0; i < b.length; i++) {
300: int c = b[i];
301: if (c == '=') {
302: try {
303: int u = Character.digit((char) b[++i], 16);
304: int l = Character.digit((char) b[++i], 16);
305: if (u == -1 || l == -1)
306: throw new RuntimeException(
307: "bad quoted-printable encoding");
308: sb.append((char) ((u << 4) + l));
309: } catch (ArrayIndexOutOfBoundsException e) {
310: throw new RuntimeException(
311: "bad quoted-printable encoding");
312: }
313: } else {
314: sb.append((char) c);
315: }
316: }
317: return sb.toString();
318: }
319:
320: public static void main(String[] args) {
321: try {
322: yacyURL contentUrl = new yacyURL(args[0], null);
323:
324: vcfParser testParser = new vcfParser();
325: byte[] content = httpc.singleGET(contentUrl, contentUrl
326: .getHost(), 10000, null, null, null, null);
327: ByteArrayInputStream input = new ByteArrayInputStream(
328: content);
329: testParser
330: .parse(contentUrl, "text/x-vcard", "UTF-8", input);
331: } catch (Exception e) {
332: e.printStackTrace();
333: }
334: }
335: }
|