01: package org.apache.lucene.analysis;
02:
03: import java.io.IOException;
04: import java.util.ArrayList;
05: import java.util.Iterator;
06: import java.util.List;
07:
08: /**
09: * A SinkTokenizer can be used to cache Tokens for use in an Analyzer
10: *
11: * @see TeeTokenFilter
12: *
13: **/
14: public class SinkTokenizer extends Tokenizer {
15: protected List/*<Token>*/lst = new ArrayList/*<Token>*/();
16: protected Iterator/*<Token>*/iter;
17:
18: public SinkTokenizer(List/*<Token>*/input) {
19: this .lst = input;
20: if (this .lst == null)
21: this .lst = new ArrayList/*<Token>*/();
22: }
23:
24: public SinkTokenizer() {
25: this .lst = new ArrayList();
26: }
27:
28: public SinkTokenizer(int initCap) {
29: this .lst = new ArrayList(initCap);
30: }
31:
32: /**
33: * Get the tokens in the internal List.
34: * <p/>
35: * WARNING: Adding tokens to this list requires the {@link #reset()} method to be called in order for them
36: * to be made available. Also, this Tokenizer does nothing to protect against {@link java.util.ConcurrentModificationException}s
37: * in the case of adds happening while {@link #next(org.apache.lucene.analysis.Token)} is being called.
38: *
39: * @return A List of {@link org.apache.lucene.analysis.Token}s
40: */
41: public List/*<Token>*/getTokens() {
42: return lst;
43: }
44:
45: /**
46: * Returns the next token out of the list of cached tokens
47: * @return The next {@link org.apache.lucene.analysis.Token} in the Sink.
48: * @throws IOException
49: */
50: public Token next() throws IOException {
51: if (iter == null)
52: iter = lst.iterator();
53: return iter.hasNext() ? (Token) iter.next() : null;
54: }
55:
56: /**
57: * Override this method to cache only certain tokens, or new tokens based
58: * on the old tokens.
59: *
60: * @param t The {@link org.apache.lucene.analysis.Token} to add to the sink
61: */
62: public void add(Token t) {
63: if (t == null)
64: return;
65: lst.add((Token) t.clone());
66: }
67:
68: public void close() throws IOException {
69: //nothing to close
70: input = null;
71: lst = null;
72: }
73:
74: /**
75: * Reset the internal data structures to the start at the front of the list of tokens. Should be called
76: * if tokens were added to the list after an invocation of {@link #next(Token)}
77: * @throws IOException
78: */
79: public void reset() throws IOException {
80: iter = lst.iterator();
81: }
82: }
|