001: /* WriterPool
002: *
003: * $Id: WriterPool.java 4535 2006-08-25 00:14:29Z stack-sf $
004: *
005: * Created July 19th, 2006.
006: *
007: * Copyright (C) 2006 Internet Archive.
008: *
009: * This file is part of the Heritrix web crawler (crawler.archive.org).
010: *
011: * Heritrix is free software; you can redistribute it and/or modify
012: * it under the terms of the GNU Lesser Public License as published by
013: * the Free Software Foundation; either version 2.1 of the License, or
014: * any later version.
015: *
016: * Heritrix is distributed in the hope that it will be useful,
017: * but WITHOUT ANY WARRANTY; without even the implied warranty of
018: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: * GNU Lesser Public License for more details.
020: *
021: * You should have received a copy of the GNU Lesser Public License
022: * along with Heritrix; if not, write to the Free Software
023: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: */
025: package org.archive.io;
026:
027: import java.io.File;
028: import java.io.IOException;
029: import java.util.NoSuchElementException;
030: import java.util.concurrent.atomic.AtomicInteger;
031: import java.util.logging.Level;
032: import java.util.logging.Logger;
033:
034: import org.apache.commons.pool.BasePoolableObjectFactory;
035: import org.apache.commons.pool.impl.FairGenericObjectPool;
036: import org.apache.commons.pool.impl.GenericObjectPool;
037:
038: /**
039: * Pool of Writers.
040: *
041: * Abstract. Override and pass in the Constructor a factory that creates
042: * {@link WriterPoolMember} implementations.
043: *
044: * @author stack
045: */
046: public abstract class WriterPool {
047: final Logger logger = Logger.getLogger(this .getClass().getName());
048:
049: /**
050: * Used to generate unique filename sequences.
051: */
052: final private AtomicInteger serialNo;
053:
054: /**
055: * Don't enforce a maximum number of idle instances in pool.
056: * To do so means GenericObjectPool will close files prematurely.
057: */
058: protected static final int NO_MAX_IDLE = -1;
059:
060: /**
061: * Retry getting a file on fail the below arbitrary amount of times.
062: * This facility is not configurable. If we fail this many times
063: * getting a file, something is seriously wrong.
064: */
065: private final int arbitraryRetryMax = 10;
066:
067: /**
068: * Default maximum active number of files in the pool.
069: */
070: public static final int DEFAULT_MAX_ACTIVE = 5;
071:
072: /**
073: * Maximum time to wait on a free file..
074: */
075: public static final int DEFAULT_MAXIMUM_WAIT = 1000 * 60 * 5;
076:
077: /**
078: * Pool instance.
079: */
080: private GenericObjectPool pool = null;
081:
082: /**
083: * File settings.
084: * Keep in data structure rather than as individual values.
085: */
086: private final WriterPoolSettings settings;
087:
088: /**
089: * Shutdown default constructor.
090: */
091: private WriterPool() {
092: this (null, null, null, -1, -1);
093: }
094:
095: /**
096: * Constructor
097: * @param serial Used to generate unique filename sequences
098: * @param factory Factory that knows how to make a {@link WriterPoolMember}.
099: * @param settings Settings for this pool.
100: * @param poolMaximumActive
101: * @param poolMaximumWait
102: */
103: public WriterPool(final AtomicInteger serial,
104: final BasePoolableObjectFactory factory,
105: final WriterPoolSettings settings,
106: final int poolMaximumActive, final int poolMaximumWait) {
107: logger.info("Initial configuration:" + " prefix="
108: + settings.getPrefix() + ", suffix="
109: + settings.getSuffix() + ", compress="
110: + settings.isCompressed() + ", maxSize="
111: + settings.getMaxSize() + ", maxActive="
112: + poolMaximumActive + ", maxWait=" + poolMaximumWait);
113: this .settings = settings;
114: this .pool = new FairGenericObjectPool(factory,
115: poolMaximumActive,
116: GenericObjectPool.WHEN_EXHAUSTED_BLOCK,
117: poolMaximumWait, NO_MAX_IDLE);
118: this .serialNo = serial;
119: }
120:
121: /**
122: * Check out a {@link WriterPoolMember}.
123: *
124: * This method must be answered by a call to
125: * {@link #returnFile(WriterPoolMember)} else pool starts leaking.
126: *
127: * @return Writer checked out of a pool of files.
128: * @throws IOException Problem getting Writer from pool (Converted
129: * from Exception to IOException so this pool can live as a good citizen
130: * down in depths of ARCSocketFactory).
131: * @throws NoSuchElementException If we time out waiting on a pool member.
132: */
133: public WriterPoolMember borrowFile() throws IOException {
134: WriterPoolMember f = null;
135: for (int i = 0; f == null; i++) {
136: long waitStart = System.currentTimeMillis();
137: try {
138: f = (WriterPoolMember) this .pool.borrowObject();
139: if (logger.getLevel() == Level.FINE) {
140: logger.fine("Borrowed " + f + " (Pool State: "
141: + getPoolState(waitStart) + ").");
142: }
143: } catch (NoSuchElementException e) {
144: // Let this exception out. Unit test at least depends on it.
145: // Log current state of the pool.
146: logger.warning(e.getMessage() + ": Retry #" + i
147: + " of " + " max of " + arbitraryRetryMax
148: + ": NSEE Pool State: "
149: + getPoolState(waitStart));
150: if (i >= arbitraryRetryMax) {
151: logger.log(Level.SEVERE,
152: "maximum retries exceeded; rethrowing", e);
153: throw e;
154: }
155: } catch (Exception e) {
156: // Convert.
157: logger.severe(e.getMessage() + ": E Pool State: "
158: + getPoolState(waitStart));
159: throw new IOException(
160: "Failed getting writer from pool: "
161: + e.getMessage());
162: }
163: }
164: return f;
165: }
166:
167: /**
168: * @param writer Writer to return to the pool.
169: * @throws IOException Problem returning File to pool.
170: */
171: public void returnFile(WriterPoolMember writer) throws IOException {
172: try {
173: if (logger.getLevel() == Level.FINE) {
174: logger.fine("Returned " + writer);
175: }
176: this .pool.returnObject(writer);
177: } catch (Exception e) {
178: throw new IOException("Failed restoring writer to pool: "
179: + e.getMessage());
180: }
181: }
182:
183: public void invalidateFile(WriterPoolMember f) throws IOException {
184: try {
185: this .pool.invalidateObject(f);
186: } catch (Exception e) {
187: // Convert exception.
188: throw new IOException(e.getMessage());
189: }
190: // It'll have been closed. Rename with an '.invalid' suffix so it
191: // gets attention.
192: File file = f.getFile();
193: file.renameTo(new File(file.getAbsoluteFile()
194: + WriterPoolMember.INVALID_SUFFIX));
195: }
196:
197: /**
198: * @return Number of {@link WriterPoolMember}s checked out of pool.
199: * @throws java.lang.UnsupportedOperationException
200: */
201: public int getNumActive() throws UnsupportedOperationException {
202: return this .pool.getNumActive();
203: }
204:
205: /**
206: * @return Number of {@link WriterPoolMember} instances still in the pool.
207: * @throws java.lang.UnsupportedOperationException
208: */
209: public int getNumIdle() throws UnsupportedOperationException {
210: return this .pool.getNumIdle();
211: }
212:
213: /**
214: * Close all {@link WriterPoolMember}s in pool.
215: */
216: public void close() {
217: this .pool.clear();
218: }
219:
220: /**
221: * @return Returns settings.
222: */
223: public WriterPoolSettings getSettings() {
224: return this .settings;
225: }
226:
227: /**
228: * @return State of the pool string
229: */
230: protected String getPoolState() {
231: return getPoolState(-1);
232: }
233:
234: /**
235: * @param startTime If we are passed a start time, we'll add difference
236: * between it and now to end of string. Pass -1 if don't want this
237: * added to end of state string.
238: * @return State of the pool string
239: */
240: protected String getPoolState(long startTime) {
241: StringBuffer buffer = new StringBuffer("Active ");
242: buffer.append(getNumActive());
243: buffer.append(" of max ");
244: buffer.append(this .pool.getMaxActive());
245: buffer.append(", idle ");
246: buffer.append(this .pool.getNumIdle());
247: if (startTime != -1) {
248: buffer.append(", time ");
249: buffer.append(System.currentTimeMillis() - startTime);
250: buffer.append("ms of max ");
251: buffer.append(this .pool.getMaxWait());
252: buffer.append("ms");
253: }
254: return buffer.toString();
255: }
256:
257: /**
258: * Returns the atomic integer used to generate serial numbers
259: * for files.
260: *
261: * @return the serial number generator
262: */
263: public AtomicInteger getSerialNo() {
264: return serialNo;
265: }
266: }
|