01: package org.apache.lucene.analysis;
02:
03: /**
04: * Licensed to the Apache Software Foundation (ASF) under one or more
05: * contributor license agreements. See the NOTICE file distributed with
06: * this work for additional information regarding copyright ownership.
07: * The ASF licenses this file to You under the Apache License, Version 2.0
08: * (the "License"); you may not use this file except in compliance with
09: * the License. You may obtain a copy of the License at
10: *
11: * http://www.apache.org/licenses/LICENSE-2.0
12: *
13: * Unless required by applicable law or agreed to in writing, software
14: * distributed under the License is distributed on an "AS IS" BASIS,
15: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16: * See the License for the specific language governing permissions and
17: * limitations under the License.
18: */
19:
20: import java.io.Reader;
21: import java.io.IOException;
22:
23: /** A Tokenizer is a TokenStream whose input is a Reader.
24: <p>
25: This is an abstract class.
26: <p>
27: NOTE: subclasses must override at least one of {@link
28: #next()} or {@link #next(Token)}.
29: <p>
30: NOTE: subclasses overriding {@link #next(Token)} must
31: call {@link Token#clear()}.
32: */
33:
34: public abstract class Tokenizer extends TokenStream {
35: /** The text source for this Tokenizer. */
36: protected Reader input;
37:
38: /** Construct a tokenizer with null input. */
39: protected Tokenizer() {
40: }
41:
42: /** Construct a token stream processing the given input. */
43: protected Tokenizer(Reader input) {
44: this .input = input;
45: }
46:
47: /** By default, closes the input Reader. */
48: public void close() throws IOException {
49: input.close();
50: }
51:
52: /** Expert: Reset the tokenizer to a new reader. Typically, an
53: * analyzer (in its reusableTokenStream method) will use
54: * this to re-use a previously created tokenizer. */
55: public void reset(Reader input) throws IOException {
56: this.input = input;
57: }
58: }
|