001: // kelondroSplitTable.java
002: // (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
003: // first published 12.10.2006 on http://www.anomic.de
004: //
005: // This is a part of YaCy, a peer-to-peer based web search engine
006: //
007: // $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
008: // $LastChangedRevision: 1986 $
009: // $LastChangedBy: orbiter $
010: //
011: // LICENSE
012: //
013: // This program is free software; you can redistribute it and/or modify
014: // it under the terms of the GNU General Public License as published by
015: // the Free Software Foundation; either version 2 of the License, or
016: // (at your option) any later version.
017: //
018: // This program is distributed in the hope that it will be useful,
019: // but WITHOUT ANY WARRANTY; without even the implied warranty of
020: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
021: // GNU General Public License for more details.
022: //
023: // You should have received a copy of the GNU General Public License
024: // along with this program; if not, write to the Free Software
025: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
026:
027: package de.anomic.kelondro;
028:
029: import java.io.File;
030: import java.io.IOException;
031: import java.util.ArrayList;
032: import java.util.Calendar;
033: import java.util.Date;
034: import java.util.HashMap;
035: import java.util.HashSet;
036: import java.util.Iterator;
037: import java.util.List;
038: import java.util.Map;
039:
040: import de.anomic.server.serverMemory;
041:
042: public class kelondroSplitTable implements kelondroIndex {
043:
044: // this is a set of kelondro tables
045: // the set is divided into tables with different entry date
046: // the table type can be either kelondroFlex or kelondroEco
047:
048: private static final long minimumRAM4Eco = 80 * 1024 * 1024;
049: private static final int EcoFSBufferSize = 20;
050:
051: private HashMap<String, kelondroIndex> tables; // a map from a date string to a kelondroIndex object
052: private kelondroRow rowdef;
053: private File path;
054: private String tablename;
055: private kelondroOrder<kelondroRow.Entry> entryOrder;
056:
057: public kelondroSplitTable(File path, String tablename,
058: long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
059: this .path = path;
060: this .tablename = tablename;
061: this .rowdef = rowdef;
062: this .entryOrder = new kelondroRow.EntryComparator(
063: rowdef.objectOrder);
064: init(preloadTime, resetOnFail);
065: }
066:
067: public void init(long preloadTime, boolean resetOnFail) {
068:
069: // initialized tables map
070: this .tables = new HashMap<String, kelondroIndex>();
071: if (!(path.exists()))
072: path.mkdirs();
073: String[] tablefile = path.list();
074: String date;
075:
076: // first pass: find tables
077: HashMap<String, Long> t = new HashMap<String, Long>();
078: long ram, sum = 0;
079: File f;
080: for (int i = 0; i < tablefile.length; i++) {
081: if ((tablefile[i].startsWith(tablename))
082: && (tablefile[i].charAt(tablename.length()) == '.')
083: && (tablefile[i].length() == tablename.length() + 7)) {
084: f = new File(path, tablefile[i]);
085: if (f.isDirectory()) {
086: ram = kelondroFlexTable.staticRAMIndexNeed(path,
087: tablefile[i], rowdef);
088: } else {
089: ram = kelondroEcoTable
090: .staticRAMIndexNeed(f, rowdef);
091: }
092: if (ram > 0) {
093: t.put(tablefile[i], new Long(ram));
094: sum += ram;
095: }
096: }
097: }
098:
099: // second pass: open tables
100: Iterator<Map.Entry<String, Long>> i;
101: Map.Entry<String, Long> entry;
102: String maxf;
103: long maxram;
104: kelondroIndex table;
105: while (t.size() > 0) {
106: // find maximum table
107: maxram = 0;
108: maxf = null;
109: i = t.entrySet().iterator();
110: while (i.hasNext()) {
111: entry = i.next();
112: ram = entry.getValue().longValue();
113: if (ram > maxram) {
114: maxf = entry.getKey();
115: maxram = ram;
116: }
117: }
118:
119: // open next biggest table
120: t.remove(maxf);
121: date = maxf.substring(tablename.length() + 1);
122: f = new File(path, maxf);
123: if (f.isDirectory()) {
124: // this is a kelonodroFlex table
125: table = new kelondroCache(new kelondroFlexTable(path,
126: maxf, preloadTime, rowdef, 0, resetOnFail));
127: } else {
128: table = new kelondroEcoTable(f, rowdef,
129: kelondroEcoTable.tailCacheDenyUsage,
130: EcoFSBufferSize, 0);
131: }
132: tables.put(date, table);
133: }
134: }
135:
136: public void reset() throws IOException {
137: this .close();
138: String[] l = path.list();
139: for (int i = 0; i < l.length; i++) {
140: if (l[i].startsWith(tablename)) {
141: File f = new File(path, l[i]);
142: if (f.isDirectory())
143: kelondroFlexTable.delete(path, l[i]);
144: else
145: f.delete();
146: }
147: }
148: init(-1, true);
149: }
150:
151: public String filename() {
152: return new File(path, tablename).toString();
153: }
154:
155: private static final Calendar this Calendar = Calendar.getInstance();
156:
157: public static final String dateSuffix(Date date) {
158: int month, year;
159: StringBuffer suffix = new StringBuffer(6);
160: synchronized (this Calendar) {
161: this Calendar.setTime(date);
162: month = this Calendar.get(Calendar.MONTH) + 1;
163: year = this Calendar.get(Calendar.YEAR);
164: }
165: if ((year < 1970) && (year >= 70))
166: suffix.append("19").append(Integer.toString(year));
167: else if (year < 1970)
168: suffix.append("20").append(Integer.toString(year));
169: else if (year > 3000)
170: return null;
171: else
172: suffix.append(Integer.toString(year));
173: if (month < 10)
174: suffix.append("0").append(Integer.toString(month));
175: else
176: suffix.append(Integer.toString(month));
177: return new String(suffix);
178: }
179:
180: public int size() {
181: Iterator<kelondroIndex> i = tables.values().iterator();
182: int s = 0;
183: while (i.hasNext())
184: s += i.next().size();
185: return s;
186: }
187:
188: public synchronized kelondroProfile profile() {
189: kelondroProfile[] profiles = new kelondroProfile[tables.size()];
190: Iterator<kelondroIndex> i = tables.values().iterator();
191: int c = 0;
192: while (i.hasNext())
193: profiles[c++] = ((kelondroIndex) i.next()).profile();
194: return kelondroProfile.consolidate(profiles);
195: }
196:
197: public int writeBufferSize() {
198: Iterator<kelondroIndex> i = tables.values().iterator();
199: int s = 0;
200: kelondroIndex ki;
201: while (i.hasNext()) {
202: ki = ((kelondroIndex) i.next());
203: if (ki instanceof kelondroCache)
204: s += ((kelondroCache) ki).writeBufferSize();
205: }
206: return s;
207: }
208:
209: public kelondroRow row() {
210: return this .rowdef;
211: }
212:
213: public boolean has(byte[] key) throws IOException {
214: Iterator<kelondroIndex> i = tables.values().iterator();
215: kelondroIndex table;
216: while (i.hasNext()) {
217: table = (kelondroIndex) i.next();
218: if (table.has(key))
219: return true;
220: }
221: return false;
222: }
223:
224: public synchronized kelondroRow.Entry get(byte[] key)
225: throws IOException {
226: Object[] keeper = keeperOf(key);
227: if (keeper == null)
228: return null;
229: return (kelondroRow.Entry) keeper[1];
230: }
231:
232: public synchronized void putMultiple(List<kelondroRow.Entry> rows)
233: throws IOException {
234: throw new UnsupportedOperationException("not yet implemented");
235: }
236:
237: public synchronized kelondroRow.Entry put(kelondroRow.Entry row)
238: throws IOException {
239: return put(row, null); // entry for current date
240: }
241:
242: public synchronized kelondroRow.Entry put(kelondroRow.Entry row,
243: Date entryDate) throws IOException {
244: assert row.objectsize() <= this .rowdef.objectsize;
245: Object[] keeper = keeperOf(row.getColBytes(0));
246: if (keeper != null)
247: return ((kelondroIndex) keeper[0]).put(row);
248: if ((entryDate == null) || (entryDate.after(new Date())))
249: entryDate = new Date(); // fix date
250: String suffix = dateSuffix(entryDate);
251: if (suffix == null)
252: return null;
253: kelondroIndex table = (kelondroIndex) tables.get(suffix);
254: if (table == null) {
255: // open table
256: File f = new File(path, tablename + "." + suffix);
257: if (f.exists()) {
258: if (f.isDirectory()) {
259: // open a flex table
260: table = new kelondroFlexTable(path, tablename + "."
261: + suffix, -1, rowdef, 0, true);
262: } else {
263: // open a eco table
264: table = new kelondroEcoTable(f, rowdef,
265: kelondroEcoTable.tailCacheDenyUsage,
266: EcoFSBufferSize, 0);
267: }
268: } else {
269: // make new table
270: if (serverMemory.request(minimumRAM4Eco, true)) {
271: // enough memory for a ecoTable
272: table = new kelondroEcoTable(f, rowdef,
273: kelondroEcoTable.tailCacheDenyUsage,
274: EcoFSBufferSize, 0);
275: } else {
276: // use the flex table
277: table = new kelondroFlexTable(path, tablename + "."
278: + suffix, -1, rowdef, 0, true);
279: }
280: }
281: tables.put(suffix, table);
282: }
283: table.put(row);
284: return null;
285: }
286:
287: public synchronized Object[] keeperOf(byte[] key)
288: throws IOException {
289: Iterator<kelondroIndex> i = tables.values().iterator();
290: kelondroIndex table;
291: kelondroRow.Entry entry;
292: while (i.hasNext()) {
293: table = (kelondroIndex) i.next();
294: entry = table.get(key);
295: if (entry != null)
296: return new Object[] { table, entry };
297: }
298: return null;
299: }
300:
301: public synchronized void addUnique(kelondroRow.Entry row)
302: throws IOException {
303: addUnique(row, null);
304: }
305:
306: public synchronized void addUnique(kelondroRow.Entry row,
307: Date entryDate) throws IOException {
308: assert row.objectsize() <= this .rowdef.objectsize;
309: if ((entryDate == null) || (entryDate.after(new Date())))
310: entryDate = new Date(); // fix date
311: String suffix = dateSuffix(entryDate);
312: if (suffix == null)
313: return;
314: kelondroIndex table = (kelondroIndex) tables.get(suffix);
315: if (table == null) {
316: // make new table
317: if (serverMemory.request(minimumRAM4Eco, true)) {
318: // enough memory for a ecoTable
319: table = new kelondroEcoTable(new File(path, tablename
320: + "." + suffix), rowdef,
321: kelondroEcoTable.tailCacheDenyUsage,
322: EcoFSBufferSize, 0);
323: } else {
324: // use the flex table
325: table = new kelondroFlexTable(path, tablename + "."
326: + suffix, -1, rowdef, 0, true);
327: }
328: tables.put(suffix, table);
329: }
330: table.addUnique(row);
331: }
332:
333: public synchronized void addUniqueMultiple(
334: List<kelondroRow.Entry> rows) throws IOException {
335: Iterator<kelondroRow.Entry> i = rows.iterator();
336: while (i.hasNext())
337: addUnique(i.next());
338: }
339:
340: public synchronized void addUniqueMultiple(
341: List<kelondroRow.Entry> rows, Date entryDate)
342: throws IOException {
343: Iterator<kelondroRow.Entry> i = rows.iterator();
344: while (i.hasNext())
345: addUnique(i.next(), entryDate);
346: }
347:
348: public ArrayList<kelondroRowSet> removeDoubles() throws IOException {
349: Iterator<kelondroIndex> i = tables.values().iterator();
350: ArrayList<kelondroRowSet> report = new ArrayList<kelondroRowSet>();
351: while (i.hasNext()) {
352: report.addAll(i.next().removeDoubles());
353: }
354: return report;
355: }
356:
357: public synchronized kelondroRow.Entry remove(byte[] key,
358: boolean keepOrder) throws IOException {
359: Iterator<kelondroIndex> i = tables.values().iterator();
360: kelondroIndex table;
361: kelondroRow.Entry entry;
362: while (i.hasNext()) {
363: table = i.next();
364: entry = table.remove(key, keepOrder);
365: if (entry != null)
366: return entry;
367: }
368: return null;
369: }
370:
371: public synchronized kelondroRow.Entry removeOne()
372: throws IOException {
373: Iterator<kelondroIndex> i = tables.values().iterator();
374: kelondroIndex table, maxtable = null;
375: int maxcount = -1;
376: while (i.hasNext()) {
377: table = (kelondroIndex) i.next();
378: if (table.size() > maxcount) {
379: maxtable = table;
380: maxcount = table.size();
381: }
382: }
383: if (maxtable == null) {
384: return null;
385: } else {
386: return maxtable.removeOne();
387: }
388: }
389:
390: public synchronized kelondroCloneableIterator<byte[]> keys(
391: boolean up, byte[] firstKey) throws IOException {
392: HashSet<kelondroCloneableIterator<byte[]>> set = new HashSet<kelondroCloneableIterator<byte[]>>();
393: Iterator<kelondroIndex> i = tables.values().iterator();
394: while (i.hasNext()) {
395: set.add(i.next().keys(up, firstKey));
396: }
397: return kelondroMergeIterator.cascade(set, rowdef.objectOrder,
398: kelondroMergeIterator.simpleMerge, up);
399: }
400:
401: public synchronized kelondroCloneableIterator<kelondroRow.Entry> rows(
402: boolean up, byte[] firstKey) throws IOException {
403: HashSet<kelondroCloneableIterator<kelondroRow.Entry>> set = new HashSet<kelondroCloneableIterator<kelondroRow.Entry>>();
404: Iterator<kelondroIndex> i = tables.values().iterator();
405: while (i.hasNext()) {
406: set.add(i.next().rows(up, firstKey));
407: }
408: return kelondroMergeIterator.cascade(set, entryOrder,
409: kelondroMergeIterator.simpleMerge, up);
410: }
411:
412: public final int cacheObjectChunkSize() {
413: // dummy method
414: return -1;
415: }
416:
417: public long[] cacheObjectStatus() {
418: // dummy method
419: return null;
420: }
421:
422: public final int cacheNodeChunkSize() {
423: // returns the size that the node cache uses for a single entry
424: return -1;
425: }
426:
427: public final int[] cacheNodeStatus() {
428: // a collection of different node cache status values
429: return new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
430: }
431:
432: public synchronized void close() {
433: if (tables == null)
434: return;
435: Iterator<kelondroIndex> i = tables.values().iterator();
436: while (i.hasNext()) {
437: i.next().close();
438: }
439: tables = null;
440: }
441:
442: public static void main(String[] args) {
443: System.out.println(dateSuffix(new Date()));
444: }
445:
446: }
|