001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.lib.lexer.lang;
043:
044: import org.netbeans.lib.lexer.lang.TestTokenId;
045: import java.util.HashMap;
046: import java.util.Map;
047: import org.netbeans.api.lexer.Token;
048: import org.netbeans.spi.lexer.Lexer;
049: import org.netbeans.spi.lexer.LexerInput;
050: import org.netbeans.spi.lexer.LexerRestartInfo;
051: import org.netbeans.spi.lexer.TokenFactory;
052:
053: /**
054: * Simple implementation a lexer.
055: *
056: * @author mmetelka
057: */
058: final class TestLexer implements Lexer<TestTokenId> {
059:
060: // Copy of LexerInput.EOF
061: private static final int EOF = LexerInput.EOF;
062:
063: private static final Map<String, TestTokenId> keywords = new HashMap<String, TestTokenId>();
064: static {
065: keywords
066: .put(TestTokenId.PUBLIC.fixedText(), TestTokenId.PUBLIC);
067: keywords.put(TestTokenId.PRIVATE.fixedText(),
068: TestTokenId.PRIVATE);
069: keywords
070: .put(TestTokenId.STATIC.fixedText(), TestTokenId.STATIC);
071: }
072:
073: private final LexerInput input;
074:
075: private final TokenFactory<TestTokenId> tokenFactory;
076:
077: private final int version;
078:
079: TestLexer(LexerRestartInfo<TestTokenId> info) {
080: this .input = info.input();
081: this .tokenFactory = info.tokenFactory();
082: assert (info.state() == null); // never set to non-null value in state()
083:
084: Integer ver = (Integer) info.getAttributeValue("version");
085: this .version = (ver != null) ? ver.intValue() : 2;
086: }
087:
088: public Object state() {
089: return null; // always in default state after token recognition
090: }
091:
092: public Token<TestTokenId> nextToken() {
093: while (true) {
094: int c = input.read();
095: switch (c) {
096: case '"': // string literal
097: while (true) {
098: switch (input.read()) {
099: case '"': // NOI18N
100: return token(TestTokenId.STRING_LITERAL);
101: case '\\':
102: input.read(); // skip the next character
103: break;
104: case '\r':
105: input.consumeNewline();
106: case '\n':
107: case EOF:
108: return token(TestTokenId.STRING_LITERAL_INCOMPLETE);
109: }
110: }
111:
112: case '+':
113: switch (input.read()) {
114: case '-':
115: if (input.read() == '+')
116: return token(TestTokenId.PLUS_MINUS_PLUS);
117: input.backup(2);
118: return token(TestTokenId.PLUS);
119:
120: }
121: input.backup(1);
122: return token(TestTokenId.PLUS);
123:
124: case '-':
125: return token(TestTokenId.MINUS);
126:
127: case '*':
128: return token(TestTokenId.STAR);
129:
130: case '/':
131: switch (input.read()) {
132: case '/': // in single-line comment
133: while (true)
134: switch (input.read()) {
135: case '\r':
136: input.consumeNewline();
137: case '\n':
138: case EOF:
139: return token(TestTokenId.LINE_COMMENT);
140: }
141: case '*': // in multi-line or javadoc comment
142: c = input.read();
143: if (c == '*') { // either javadoc comment or empty multi-line comment /**/
144: c = input.read();
145: if (c == '/')
146: return token(TestTokenId.BLOCK_COMMENT);
147: while (true) { // in javadoc comment
148: while (c == '*') {
149: c = input.read();
150: if (c == '/')
151: return token(TestTokenId.JAVADOC_COMMENT);
152: else if (c == EOF)
153: return token(TestTokenId.JAVADOC_COMMENT_INCOMPLETE);
154: }
155: if (c == EOF)
156: return token(TestTokenId.JAVADOC_COMMENT_INCOMPLETE);
157: c = input.read();
158: }
159:
160: } else { // in multi-line comment (and not after '*')
161: while (true) {
162: c = input.read();
163: while (c == '*') {
164: c = input.read();
165: if (c == '/')
166: return token(TestTokenId.BLOCK_COMMENT);
167: else if (c == EOF)
168: return token(TestTokenId.BLOCK_COMMENT_INCOMPLETE);
169: }
170: if (c == EOF)
171: return token(TestTokenId.BLOCK_COMMENT_INCOMPLETE);
172: }
173: }
174: }
175: input.backup(1);
176: return token(TestTokenId.DIV);
177:
178: // All Character.isWhitespace(c) below 0x80 follow
179: // ['\t' - '\r'] and [0x1c - ' ']
180: case '\t':
181: case '\n':
182: case 0x0b:
183: case '\f':
184: case '\r':
185: case 0x1c:
186: case 0x1d:
187: case 0x1e:
188: case 0x1f:
189: return finishWhitespace();
190: case ' ':
191: c = input.read();
192: if (c == EOF || !Character.isWhitespace(c)) { // Return single space as flyweight token
193: input.backup(1);
194: return tokenFactory.getFlyweightToken(
195: TestTokenId.WHITESPACE, " ");
196: }
197: return finishWhitespace();
198:
199: case EOF: // no more chars on the input
200: return null; // the only legal situation when null can be returned
201:
202: default:
203: if (Character.isJavaIdentifierStart((char) c))
204: return finishIdentifier();
205: if (c >= 0x80 && Character.isWhitespace((char) c))
206: return finishWhitespace();
207:
208: // Invalid char
209: return token(TestTokenId.ERROR);
210: }
211: }
212: }
213:
214: private Token<TestTokenId> finishWhitespace() {
215: while (true) {
216: int c = input.read();
217: if (c == EOF || !Character.isWhitespace(c)) {
218: input.backup(1);
219: return tokenFactory.createToken(TestTokenId.WHITESPACE);
220: }
221: }
222: }
223:
224: private Token<TestTokenId> finishIdentifier() {
225: while (true) {
226: int c = input.read();
227: if (c == EOF || !Character.isJavaIdentifierPart(c)) {
228: input.backup(1);
229: // Check whether the identifier is not a keyword
230: TestTokenId id = keywords.get(input.readText());
231: // Test: Only recognize STATIC keyword in version >= 2
232: if (id == TestTokenId.STATIC && version < 2) {
233: id = null; // force IDENTIFIER
234: }
235: return (id == null) ? tokenFactory
236: .createToken(TestTokenId.IDENTIFIER)
237: : token(id);
238: }
239: }
240: }
241:
242: private Token<TestTokenId> token(TestTokenId id) {
243: String fixedText = id.fixedText();
244: return (fixedText != null) ? tokenFactory.getFlyweightToken(id,
245: fixedText) : tokenFactory.createToken(id);
246: }
247:
248: public void release() {
249: }
250:
251: }
|