01: package net.matuschek.examples;
02:
03: import java.io.FileWriter;
04: import java.net.URL;
05: import net.matuschek.http.URLLogger;
06: import net.matuschek.spider.WebRobot;
07:
08: /*********************************************
09: Copyright (c) 2001 by Daniel Matuschek
10: *********************************************/
11:
12: /**
13: * This example program downloads a web page. It does not
14: * store the documents but only logs the visited URLs.
15: *
16: * @author Daniel Matuschek
17: * @version $Revision: 1.2 $
18: */
19: public class LogURL {
20:
21: public static void main(String[] args) throws Exception {
22: System.out.println("URLs will be logged to urls.txt\n\n");
23:
24: WebRobot robby = new WebRobot();
25: robby.setStartURL(new URL("http://www.matuschek.net"));
26: robby.setMaxDepth(1);
27: robby.setSleepTime(0);
28:
29: FileWriter logfile = new FileWriter("urls.txt");
30: URLLogger log = new URLLogger(logfile);
31: robby.setDocManager(log);
32:
33: robby.run();
34: logfile.close();
35: }
36: }
|