001: /* ARCWriterPoolTest
002: *
003: * $Id: ARCWriterPoolTest.java 5029 2007-03-29 23:53:50Z gojomo $
004: *
005: * Created on Jan 22, 2004
006: *
007: * Copyright (C) 2004 Internet Archive.
008: *
009: * This file is part of the Heritrix web crawler (crawler.archive.org).
010: *
011: * Heritrix is free software; you can redistribute it and/or modify
012: * it under the terms of the GNU Lesser Public License as published by
013: * the Free Software Foundation; either version 2.1 of the License, or
014: * any later version.
015: *
016: * Heritrix is distributed in the hope that it will be useful,
017: * but WITHOUT ANY WARRANTY; without even the implied warranty of
018: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: * GNU Lesser Public License for more details.
020: *
021: * You should have received a copy of the GNU Lesser Public License
022: * along with Heritrix; if not, write to the Free Software
023: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: */
025: package org.archive.io.arc;
026:
027: import java.io.ByteArrayOutputStream;
028: import java.io.File;
029: import java.util.Arrays;
030: import java.util.Date;
031: import java.util.List;
032: import java.util.NoSuchElementException;
033:
034: import org.archive.io.WriterPoolMember;
035: import org.archive.io.WriterPool;
036: import org.archive.io.WriterPoolSettings;
037: import org.archive.util.TmpDirTestCase;
038:
039: /**
040: * Test ARCWriterPool
041: */
042: public class ARCWriterPoolTest extends TmpDirTestCase {
043: private static final String PREFIX = "TEST";
044:
045: public void testARCWriterPool() throws Exception {
046: final int MAX_ACTIVE = 3;
047: final int MAX_WAIT_MILLISECONDS = 100;
048: cleanUpOldFiles(PREFIX);
049: WriterPool pool = new ARCWriterPool(getSettings(true),
050: MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
051: WriterPoolMember[] writers = new WriterPoolMember[MAX_ACTIVE];
052: final String CONTENT = "Any old content";
053: ByteArrayOutputStream baos = new ByteArrayOutputStream();
054: baos.write(CONTENT.getBytes());
055: for (int i = 0; i < MAX_ACTIVE; i++) {
056: writers[i] = pool.borrowFile();
057: assertEquals("Number active", i + 1, pool.getNumActive());
058: ((ARCWriter) writers[i]).write("http://one.two.three",
059: "no-type", "0.0.0.0", 1234567890, CONTENT.length(),
060: baos);
061: }
062:
063: // Pool is maxed out. Try and get a new ARCWriter. We'll block for
064: // MAX_WAIT_MILLISECONDS. Should get exception.
065: long start = (new Date()).getTime();
066: boolean isException = false;
067: try {
068: pool.borrowFile();
069: } catch (NoSuchElementException e) {
070: isException = true;
071: long end = (new Date()).getTime();
072: // This test can fail on a loaded machine if the wait period is
073: // only MAX_WAIT_MILLISECONDS. Up the time to wait.
074: final int WAIT = MAX_WAIT_MILLISECONDS * 100;
075: if ((end - start) > (WAIT)) {
076: fail("More than " + MAX_WAIT_MILLISECONDS
077: + " elapsed: " + WAIT);
078: }
079: }
080: assertTrue("Did not get NoSuchElementException", isException);
081:
082: for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
083: pool.returnFile(writers[i]);
084: assertEquals("Number active", i, pool.getNumActive());
085: assertEquals("Number idle", MAX_ACTIVE
086: - pool.getNumActive(), pool.getNumIdle());
087: }
088: pool.close();
089: }
090:
091: public void testInvalidate() throws Exception {
092: final int MAX_ACTIVE = 3;
093: final int MAX_WAIT_MILLISECONDS = 100;
094: cleanUpOldFiles(PREFIX);
095: WriterPool pool = new ARCWriterPool(getSettings(true),
096: MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
097: WriterPoolMember[] writers = new WriterPoolMember[MAX_ACTIVE];
098: final String CONTENT = "Any old content";
099: ByteArrayOutputStream baos = new ByteArrayOutputStream();
100: baos.write(CONTENT.getBytes());
101: for (int i = 0; i < MAX_ACTIVE; i++) {
102: writers[i] = pool.borrowFile();
103: assertEquals("Number active", i + 1, pool.getNumActive());
104: ((ARCWriter) writers[i]).write("http://one.two.three",
105: "no-type", "0.0.0.0", 1234567890, CONTENT.length(),
106: baos);
107: }
108:
109: WriterPoolMember writer2Invalidate = writers[pool
110: .getNumActive() - 1];
111: writers[pool.getNumActive() - 1] = null;
112: pool.invalidateFile(writer2Invalidate);
113: for (int i = 0; i < (MAX_ACTIVE - 1); i++) {
114: if (writers[i] == null) {
115: continue;
116: }
117: pool.returnFile(writers[i]);
118: }
119:
120: for (int i = 0; i < MAX_ACTIVE; i++) {
121: writers[i] = pool.borrowFile();
122: assertEquals("Number active", i + 1, pool.getNumActive());
123: ((ARCWriter) writers[i]).write("http://one.two.three",
124: "no-type", "0.0.0.0", 1234567890, CONTENT.length(),
125: baos);
126: }
127: for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
128: pool.returnFile(writers[i]);
129: assertEquals("Number active", i, pool.getNumActive());
130: assertEquals("Number idle", MAX_ACTIVE
131: - pool.getNumActive(), pool.getNumIdle());
132: }
133: pool.close();
134: }
135:
136: private WriterPoolSettings getSettings(final boolean isCompressed) {
137: return new WriterPoolSettings() {
138: public long getMaxSize() {
139: return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE;
140: }
141:
142: public String getPrefix() {
143: return PREFIX;
144: }
145:
146: public String getSuffix() {
147: return "";
148: }
149:
150: public List<File> getOutputDirs() {
151: File[] files = { getTmpDir() };
152: return Arrays.asList(files);
153: }
154:
155: public boolean isCompressed() {
156: return isCompressed;
157: }
158:
159: public List getMetadata() {
160: return null;
161: }
162: };
163: }
164: }
|