01: package org.apache.lucene.analysis.ru;
02:
03: /**
04: * Licensed to the Apache Software Foundation (ASF) under one or more
05: * contributor license agreements. See the NOTICE file distributed with
06: * this work for additional information regarding copyright ownership.
07: * The ASF licenses this file to You under the Apache License, Version 2.0
08: * (the "License"); you may not use this file except in compliance with
09: * the License. You may obtain a copy of the License at
10: *
11: * http://www.apache.org/licenses/LICENSE-2.0
12: *
13: * Unless required by applicable law or agreed to in writing, software
14: * distributed under the License is distributed on an "AS IS" BASIS,
15: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16: * See the License for the specific language governing permissions and
17: * limitations under the License.
18: */
19:
20: import java.io.Reader;
21: import org.apache.lucene.analysis.CharTokenizer;
22:
23: /**
24: * A RussianLetterTokenizer is a tokenizer that extends LetterTokenizer by additionally looking up letters
25: * in a given "russian charset". The problem with LeterTokenizer is that it uses Character.isLetter() method,
26: * which doesn't know how to detect letters in encodings like CP1252 and KOI8
27: * (well-known problems with 0xD7 and 0xF7 chars)
28: *
29: *
30: * @version $Id: RussianLetterTokenizer.java 564236 2007-08-09 15:21:19Z gsingers $
31: */
32:
33: public class RussianLetterTokenizer extends CharTokenizer {
34: /** Construct a new LetterTokenizer. */
35: private char[] charset;
36:
37: public RussianLetterTokenizer(Reader in, char[] charset) {
38: super (in);
39: this .charset = charset;
40: }
41:
42: /**
43: * Collects only characters which satisfy
44: * {@link Character#isLetter(char)}.
45: */
46: protected boolean isTokenChar(char c) {
47: if (Character.isLetter(c))
48: return true;
49: for (int i = 0; i < charset.length; i++) {
50: if (c == charset[i])
51: return true;
52: }
53: return false;
54: }
55: }
|