001: // URLFetcherStack.java
002: // -------------------------------------
003: // part of YACY
004: //
005: // (C) 2007 by Franz Brausze
006: //
007: // last change: $LastChangedDate: $ by $LastChangedBy: $
008: //
009: // This program is free software; you can redistribute it and/or modify
010: // it under the terms of the GNU General Public License as published by
011: // the Free Software Foundation; either version 2 of the License, or
012: // (at your option) any later version.
013: //
014: // This program is distributed in the hope that it will be useful,
015: // but WITHOUT ANY WARRANTY; without even the implied warranty of
016: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017: // GNU General Public License for more details.
018: //
019: // You should have received a copy of the GNU General Public License
020: // along with this program; if not, write to the Free Software
021: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022: //
023: // Using this software in any meaning (reading, learning, copying, compiling,
024: // running) means that you agree that the Author(s) is (are) not responsible
025: // for cost, loss of data or any harm that may be caused directly or indirectly
026: // by usage of this softare or this documentation. The usage of this software
027: // is on your own risk. The installation and usage (starting/running) of this
028: // software may allow other people or application to access your computer and
029: // any attached devices and is highly dependent on the configuration of the
030: // software which must be done by the user of the software; the author(s) is
031: // (are) also not responsible for proper configuration and usage of the
032: // software, even if provoked by documentation provided together with
033: // the software.
034: //
035: // Any changes to this file according to the GPL as documented in the file
036: // gpl.txt aside this file in the shipment you received can be done to the
037: // lines that follows this copyright notice here, but changes must not be
038: // done inside the copyright notive above. A re-distribution must contain
039: // the intact and unchanged copyright notice.
040: // Contributions and changes to the program code must be marked as such.
041:
042: package de.anomic.data;
043:
044: import java.io.File;
045: import java.io.IOException;
046: import java.net.MalformedURLException;
047: import java.util.ArrayList;
048: import java.util.Iterator;
049:
050: import de.anomic.kelondro.kelondroBase64Order;
051: import de.anomic.kelondro.kelondroException;
052: import de.anomic.kelondro.kelondroRow;
053: import de.anomic.kelondro.kelondroStack;
054: import de.anomic.kelondro.kelondroRow.EntryIndex;
055: import de.anomic.server.logging.serverLog;
056: import de.anomic.yacy.yacyURL;
057:
058: public class URLFetcherStack {
059:
060: public static final String DBFILE = "urlRemote2.stack";
061:
062: private static final kelondroRow rowdef = new kelondroRow(
063: "String urlstring-256", kelondroBase64Order.enhancedCoder,
064: 0);
065: private final kelondroStack db;
066: private final serverLog log;
067:
068: private int popped = 0;
069: private int pushed = 0;
070:
071: public URLFetcherStack(File path) throws IOException {
072: this .db = new kelondroStack(new File(path, DBFILE), rowdef);
073: this .log = new serverLog("URLFETCHERSTACK");
074: }
075:
076: public int getPopped() {
077: return this .popped;
078: }
079:
080: public int getPushed() {
081: return this .pushed;
082: }
083:
084: public void clearStat() {
085: this .popped = 0;
086: this .pushed = 0;
087: }
088:
089: public void finalize() throws Throwable {
090: this .db.close();
091: }
092:
093: public boolean push(yacyURL url) {
094: try {
095: this .db.push(this .db.row().newEntry(
096: new byte[][] { url.toNormalform(true, true)
097: .getBytes() }));
098: this .pushed++;
099: return true;
100: } catch (IOException e) {
101: this .log.logSevere("error storing entry", e);
102: return false;
103: }
104: }
105:
106: public yacyURL pop() {
107: try {
108: kelondroRow.Entry r = this .db.pop();
109: if (r == null)
110: return null;
111: final String url = r.getColString(0, null);
112: try {
113: this .popped++;
114: return new yacyURL(url, null);
115: } catch (MalformedURLException e) {
116: this .log.logSevere("found invalid URL-entry: " + url);
117: return null;
118: }
119: } catch (IOException e) {
120: this .log.logSevere("error retrieving entry", e);
121: return null;
122: }
123: }
124:
125: public String[] top(int count) {
126: try {
127: final ArrayList<String> ar = new ArrayList<String>();
128: Iterator<EntryIndex> it = db.contentRows(500);
129: kelondroRow.EntryIndex ei;
130: for (int i = 0; i < count && it.hasNext(); i++) {
131: ei = it.next();
132: if (ei == null)
133: continue;
134: ar.add(ei.getColString(0, null));
135: }
136: return ar.toArray(new String[ar.size()]);
137: } catch (kelondroException e) {
138: this .log.logSevere("error retrieving entry", e);
139: return null;
140: }
141: }
142:
143: public int size() {
144: return this.db.size();
145: }
146: }
|