01: /**
02: * Licensed to the Apache Software Foundation (ASF) under one or more
03: * contributor license agreements. See the NOTICE file distributed with
04: * this work for additional information regarding copyright ownership.
05: * The ASF licenses this file to You under the Apache License, Version 2.0
06: * (the "License"); you may not use this file except in compliance with
07: * the License. You may obtain a copy of the License at
08: *
09: * http://www.apache.org/licenses/LICENSE-2.0
10: *
11: * Unless required by applicable law or agreed to in writing, software
12: * distributed under the License is distributed on an "AS IS" BASIS,
13: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14: * See the License for the specific language governing permissions and
15: * limitations under the License.
16: */package org.apache.solr.analysis;
17:
18: import junit.framework.TestCase;
19: import org.apache.lucene.analysis.Token;
20: import org.apache.lucene.analysis.TokenStream;
21: import org.apache.lucene.analysis.WhitespaceTokenizer;
22:
23: import java.io.IOException;
24: import java.io.StringReader;
25:
26: /**
27: * Test that BufferedTokenStream behaves as advertised in subclasses.
28: */
29: public class TestBufferedTokenStream extends BaseTokenTestCase {
30:
31: /** Example of a class implementing the rule "A" "B" => "Q" "B" */
32: public static class AB_Q_Stream extends BufferedTokenStream {
33: public AB_Q_Stream(TokenStream input) {
34: super (input);
35: }
36:
37: protected Token process(Token t) throws IOException {
38: if ("A".equals(t.termText())) {
39: Token t2 = read();
40: if (t2 != null && "B".equals(t2.termText()))
41: t.setTermText("Q");
42: if (t2 != null)
43: pushBack(t2);
44: }
45: return t;
46: }
47: }
48:
49: /** Example of a class implementing "A" "B" => "A" "A" "B" */
50: public static class AB_AAB_Stream extends BufferedTokenStream {
51: public AB_AAB_Stream(TokenStream input) {
52: super (input);
53: }
54:
55: protected Token process(Token t) throws IOException {
56: if ("A".equals(t.termText())
57: && "B".equals(peek(1).termText()))
58: write(t);
59: return t;
60: }
61: }
62:
63: public void testABQ() throws Exception {
64: final String input = "How now A B brown A cow B like A B thing?";
65: final String expected = "How now Q B brown A cow B like Q B thing?";
66: TokenStream ts = new AB_Q_Stream(new WhitespaceTokenizer(
67: new StringReader(input)));
68: final String actual = tsToString(ts);
69: //System.out.println(actual);
70: assertEquals(expected, actual);
71: }
72:
73: public void testABAAB() throws Exception {
74: final String input = "How now A B brown A cow B like A B thing?";
75: final String expected = "How now A A B brown A cow B like A A B thing?";
76: TokenStream ts = new AB_AAB_Stream(new WhitespaceTokenizer(
77: new StringReader(input)));
78: final String actual = tsToString(ts);
79: //System.out.println(actual);
80: assertEquals(expected, actual);
81: }
82: }
|