001: /*
002: * GrammarTokenizer.java
003: *
004: * THIS FILE HAS BEEN GENERATED AUTOMATICALLY. DO NOT EDIT!
005: *
006: * This work is free software; you can redistribute it and/or modify
007: * it under the terms of the GNU General Public License as published
008: * by the Free Software Foundation; either version 2 of the License,
009: * or (at your option) any later version.
010: *
011: * This work is distributed in the hope that it will be useful, but
012: * WITHOUT ANY WARRANTY; without even the implied warranty of
013: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014: * General Public License for more details.
015: *
016: * You should have received a copy of the GNU General Public License
017: * along with this program; if not, write to the Free Software
018: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
019: * USA
020: *
021: * As a special exception, the copyright holders of this library give
022: * you permission to link this library with independent modules to
023: * produce an executable, regardless of the license terms of these
024: * independent modules, and to copy and distribute the resulting
025: * executable under terms of your choice, provided that you also meet,
026: * for each linked independent module, the terms and conditions of the
027: * license of that module. An independent module is a module which is
028: * not derived from or based on this library. If you modify this
029: * library, you may extend this exception to your version of the
030: * library, but you are not obligated to do so. If you do not wish to
031: * do so, delete this exception statement from your version.
032: *
033: * Copyright (c) 2003 Per Cederberg. All rights reserved.
034: */
035:
036: package net.percederberg.grammatica;
037:
038: import java.io.Reader;
039:
040: import net.percederberg.grammatica.parser.ParserCreationException;
041: import net.percederberg.grammatica.parser.TokenPattern;
042: import net.percederberg.grammatica.parser.Tokenizer;
043:
044: /**
045: * A character stream tokenizer.
046: *
047: * @author Per Cederberg, <per at percederberg dot net>
048: * @version 1.2
049: */
050: class GrammarTokenizer extends Tokenizer {
051:
052: /**
053: * Creates a new tokenizer for the specified input stream.
054: *
055: * @param input the input stream to read
056: *
057: * @throws ParserCreationException if the tokenizer couldn't be
058: * initialized correctly
059: */
060: public GrammarTokenizer(Reader input)
061: throws ParserCreationException {
062:
063: super (input);
064: createPatterns();
065: }
066:
067: /**
068: * Initializes the tokenizer by creating all the token patterns.
069: *
070: * @throws ParserCreationException if the tokenizer couldn't be
071: * initialized correctly
072: */
073: private void createPatterns() throws ParserCreationException {
074: TokenPattern pattern;
075:
076: pattern = new TokenPattern(GrammarConstants.HEADER, "HEADER",
077: TokenPattern.STRING_TYPE, "%header%");
078: addPattern(pattern);
079:
080: pattern = new TokenPattern(GrammarConstants.TOKENS, "TOKENS",
081: TokenPattern.STRING_TYPE, "%tokens%");
082: addPattern(pattern);
083:
084: pattern = new TokenPattern(GrammarConstants.PRODUCTIONS,
085: "PRODUCTIONS", TokenPattern.STRING_TYPE,
086: "%productions%");
087: addPattern(pattern);
088:
089: pattern = new TokenPattern(GrammarConstants.IGNORE, "IGNORE",
090: TokenPattern.REGEXP_TYPE, "%ignore([^%]*)%");
091: addPattern(pattern);
092:
093: pattern = new TokenPattern(GrammarConstants.ERROR, "ERROR",
094: TokenPattern.REGEXP_TYPE, "%error([^%]*)%");
095: addPattern(pattern);
096:
097: pattern = new TokenPattern(
098: GrammarConstants.UNTERMINATED_DIRECTIVE,
099: "UNTERMINATED_DIRECTIVE", TokenPattern.REGEXP_TYPE,
100: "%[^%\\n\\r]*");
101: pattern.setError("unterminated directive");
102: addPattern(pattern);
103:
104: pattern = new TokenPattern(GrammarConstants.EQUALS, "EQUALS",
105: TokenPattern.STRING_TYPE, "=");
106: addPattern(pattern);
107:
108: pattern = new TokenPattern(GrammarConstants.LEFT_PAREN,
109: "LEFT_PAREN", TokenPattern.STRING_TYPE, "(");
110: addPattern(pattern);
111:
112: pattern = new TokenPattern(GrammarConstants.RIGHT_PAREN,
113: "RIGHT_PAREN", TokenPattern.STRING_TYPE, ")");
114: addPattern(pattern);
115:
116: pattern = new TokenPattern(GrammarConstants.LEFT_BRACE,
117: "LEFT_BRACE", TokenPattern.STRING_TYPE, "{");
118: addPattern(pattern);
119:
120: pattern = new TokenPattern(GrammarConstants.RIGHT_BRACE,
121: "RIGHT_BRACE", TokenPattern.STRING_TYPE, "}");
122: addPattern(pattern);
123:
124: pattern = new TokenPattern(GrammarConstants.LEFT_BRACKET,
125: "LEFT_BRACKET", TokenPattern.STRING_TYPE, "[");
126: addPattern(pattern);
127:
128: pattern = new TokenPattern(GrammarConstants.RIGHT_BRACKET,
129: "RIGHT_BRACKET", TokenPattern.STRING_TYPE, "]");
130: addPattern(pattern);
131:
132: pattern = new TokenPattern(GrammarConstants.QUESTION_MARK,
133: "QUESTION_MARK", TokenPattern.STRING_TYPE, "?");
134: addPattern(pattern);
135:
136: pattern = new TokenPattern(GrammarConstants.PLUS_SIGN,
137: "PLUS_SIGN", TokenPattern.STRING_TYPE, "+");
138: addPattern(pattern);
139:
140: pattern = new TokenPattern(GrammarConstants.ASTERISK,
141: "ASTERISK", TokenPattern.STRING_TYPE, "*");
142: addPattern(pattern);
143:
144: pattern = new TokenPattern(GrammarConstants.VERTICAL_BAR,
145: "VERTICAL_BAR", TokenPattern.STRING_TYPE, "|");
146: addPattern(pattern);
147:
148: pattern = new TokenPattern(GrammarConstants.SEMICOLON,
149: "SEMICOLON", TokenPattern.STRING_TYPE, ";");
150: addPattern(pattern);
151:
152: pattern = new TokenPattern(GrammarConstants.IDENTIFIER,
153: "IDENTIFIER", TokenPattern.REGEXP_TYPE,
154: "[A-Za-z][A-Za-z0-9_]*");
155: addPattern(pattern);
156:
157: pattern = new TokenPattern(GrammarConstants.QUOTED_STRING,
158: "QUOTED_STRING", TokenPattern.REGEXP_TYPE,
159: "(\"[^\"]*\")|('[^']*')");
160: addPattern(pattern);
161:
162: pattern = new TokenPattern(GrammarConstants.REGEXP, "REGEXP",
163: TokenPattern.REGEXP_TYPE,
164: "<<([^\\\\>]|(\\\\.)|(>[^>]))*>>");
165: addPattern(pattern);
166:
167: pattern = new TokenPattern(
168: GrammarConstants.SINGLE_LINE_COMMENT,
169: "SINGLE_LINE_COMMENT", TokenPattern.REGEXP_TYPE, "//.*");
170: pattern.setIgnore();
171: addPattern(pattern);
172:
173: pattern = new TokenPattern(GrammarConstants.MULTI_LINE_COMMENT,
174: "MULTI_LINE_COMMENT", TokenPattern.REGEXP_TYPE,
175: "/\\*([^*]|\\*[^/])*\\*/");
176: pattern.setIgnore();
177: addPattern(pattern);
178:
179: pattern = new TokenPattern(GrammarConstants.WHITESPACE,
180: "WHITESPACE", TokenPattern.REGEXP_TYPE, "[ \\t\\n\\r]+");
181: pattern.setIgnore();
182: addPattern(pattern);
183: }
184: }
|