001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.spi.lexer;
043:
044: import java.util.Set;
045: import org.netbeans.api.lexer.PartType;
046: import org.netbeans.api.lexer.Token;
047: import org.netbeans.api.lexer.TokenId;
048: import org.netbeans.lib.editor.util.CharSequenceUtilities;
049: import org.netbeans.lib.lexer.LanguageOperation;
050: import org.netbeans.lib.lexer.LexerInputOperation;
051: import org.netbeans.lib.lexer.TokenIdImpl;
052: import org.netbeans.lib.lexer.token.CustomTextToken;
053: import org.netbeans.lib.lexer.token.DefaultToken;
054: import org.netbeans.lib.lexer.token.ComplexToken;
055: import org.netbeans.lib.lexer.token.ComplexToken;
056: import org.netbeans.lib.lexer.token.PropertyToken;
057: import org.netbeans.lib.lexer.token.TextToken;
058:
059: /**
060: * Lexer should delegate all the token instances creation to this class.
061: * <br/>
062: * It's not allowed to create empty tokens.
063: *
064: * @author Miloslav Metelka
065: * @version 1.00
066: */
067:
068: public final class TokenFactory<T extends TokenId> {
069:
070: /** Flag for additional correctness checks (may degrade performance). */
071: private static final boolean testing = Boolean
072: .getBoolean("netbeans.debug.lexer.test");
073:
074: /**
075: * Token instance that should be returned by the lexer
076: * if there is an active filtering of certain token ids
077: * and the just recognized token-id should be skipped.
078: */
079: public static final Token SKIP_TOKEN = new TextToken<TokenId>(
080: new TokenIdImpl(
081: "skip-token-id; special id of TokenFactory.SKIP_TOKEN; "
082: + // NOI18N
083: " It should never be part of token sequence",
084: 0, null), // NOI18N
085: "" // empty skip token text NOI18N
086: );
087:
088: private final LexerInputOperation<T> operation;
089:
090: TokenFactory(LexerInputOperation<T> operation) {
091: this .operation = operation;
092: }
093:
094: /**
095: * Create token with token length corresponding
096: * to the number of characters read from the lexer input.
097: *
098: * @see #createToken(TokenId, int)
099: */
100: public Token<T> createToken(T id) {
101: return createToken(id, operation.readIndex());
102: }
103:
104: /**
105: * Create regular token instance with an explicit length.
106: *
107: * @param id non-null token id recognized by the lexer.
108: * @param length >=0 length of the token to be created. The length must not
109: * exceed the number of characters read from the lexer input.
110: * @return non-null regular token instance.
111: * <br/>
112: * {@link #SKIP_TOKEN} will be returned
113: * if tokens for the given token id should be skipped
114: * because of token id filter.
115: */
116: public Token<T> createToken(T id, int length) {
117: if (isSkipToken(id)) {
118: operation.tokenRecognized(length, true);
119: return skipToken();
120: } else { // Do not skip the token
121: if (operation.tokenRecognized(length, false)) { // Create preprocessed token
122: // return new PreprocessedTextToken<T>(id, operation.tokenLength());
123: return new DefaultToken<T>(id, operation.tokenLength());
124: } else {
125: return new DefaultToken<T>(id, operation.tokenLength());
126: }
127: }
128: }
129:
130: /**
131: * Create regular token instance with an explicit length and part type.
132: *
133: * @param id non-null token id recognized by the lexer.
134: * @param length >=0 length of the token to be created. The length must not
135: * exceed the number of characters read from the lexer input.
136: * @param partType whether this token is complete token or a part of a complete token.
137: * @return non-null regular token instance.
138: * <br/>
139: * {@link #SKIP_TOKEN} will be returned
140: * if tokens for the given token id should be skipped
141: * because of token id filter.
142: */
143: public Token<T> createToken(T id, int length, PartType partType) {
144: checkPartTypeNonNull(partType);
145: if (partType == PartType.COMPLETE)
146: return createToken(id, length);
147:
148: if (isSkipToken(id)) {
149: operation.tokenRecognized(length, true);
150: return skipToken();
151: } else { // Do not skip the token
152: if (operation.tokenRecognized(length, false)) { // Create preprocessed token
153: // return new ComplexToken<T>(id, operation.tokenLength(), null, partType, null);
154: return new PropertyToken<T>(id,
155: operation.tokenLength(), null, partType);
156: } else {
157: return new PropertyToken<T>(id,
158: operation.tokenLength(), null, partType);
159: }
160: }
161: }
162:
163: /**
164: * Get flyweight token for the given arguments.
165: * <br/>
166: * <b>Note:</b> The returned token will not be flyweight under certain
167: * conditions - see return value description.
168: *
169: * @param id non-null token id.
170: * @param text non-null text that the flyweight token should carry.
171: * @return non-null flyweight token instance.
172: * <br/>
173: * For performance reasons there is a limit for number of successive
174: * flyweight tokens. If this limit would be exceeded a single non-flyweight
175: * token gets created instead of flyweight one.
176: * <br/>
177: * {@link #SKIP_TOKEN} will be returned
178: * if tokens for the given token id should be skipped
179: * because of token id filter.
180: */
181: public Token<T> getFlyweightToken(T id, String text) {
182: assert (text.length() <= operation.readIndex());
183: // Compare each recognized char with the corresponding char in text
184: if (testing) {
185: for (int i = 0; i < text.length(); i++) {
186: if (text.charAt(i) != operation.readExisting(i)) {
187: throw new IllegalArgumentException(
188: "Flyweight text in "
189: + // NOI18N
190: "TokenFactory.getFlyweightToken("
191: + id
192: + ", \""
193: + // NOI18N
194: CharSequenceUtilities
195: .debugText(text)
196: + "\") "
197: + // NOI18N
198: "differs from recognized text: '"
199: + // NOI18N
200: CharSequenceUtilities
201: .debugChar(operation
202: .readExisting(i))
203: + "' != '"
204: + CharSequenceUtilities
205: .debugChar(text.charAt(i)) + // NOI18N
206: "' at index=" + i // NOI18N
207: );
208: }
209: }
210: }
211:
212: // Check whether token with given id should be created
213: if (isSkipToken(id)) {
214: operation.tokenRecognized(text.length(), true);
215: return skipToken();
216: } else { // Do not skip the token
217: if (operation.tokenRecognized(text.length(), false)) { // Create preprocessed token
218: // return new PreprocessedTextToken<T>(id, operation.tokenLength());
219: return new DefaultToken<T>(id, operation.tokenLength());
220: } else if (operation.isFlyTokenAllowed()) {
221: LanguageOperation<T> langOp = operation
222: .languageOperation();
223: return langOp.getFlyweightToken(id, text);
224: } else { // return non-flyweight token
225: return new DefaultToken<T>(id, operation.tokenLength());
226: }
227: }
228: }
229:
230: /**
231: * Create token with properties.
232: *
233: * @param id non-null token id.
234: * @param length >=0 length of the token to be created. The length must not
235: * exceed the number of characters read from the lexer input.
236: * @param propertyProvider non-null token property provider.
237: * @param partType whether this token is complete or just a part of complete token.
238: * See {@link TokenPropertyProvider} for examples how this parameter may be used.
239: * @return non-null property token instance.
240: * <br/>
241: * {@link #SKIP_TOKEN} will be returned
242: * if tokens for the given token id should be skipped
243: * because of token id filter.
244: */
245: public Token<T> createPropertyToken(T id, int length,
246: TokenPropertyProvider propertyProvider, PartType partType) {
247: checkPartTypeNonNull(partType);
248: if (isSkipToken(id)) {
249: operation.tokenRecognized(length, true);
250: return skipToken();
251: } else { // Do not skip the token
252: if (operation.tokenRecognized(length, false)) { // Create preprocessed token
253: // return new ComplexToken<T>(id, operation.tokenLength(),
254: // propertyProvider, null, partType);
255: return new PropertyToken<T>(id,
256: operation.tokenLength(), propertyProvider,
257: partType);
258: } else {
259: return new PropertyToken<T>(id,
260: operation.tokenLength(), propertyProvider,
261: partType);
262: }
263: }
264: }
265:
266: /**
267: * Create token with a custom text that possibly differs from the text
268: * represented by the token in the input text.
269: */
270: public Token<T> createCustomTextToken(T id, CharSequence text,
271: int length, PartType partType) {
272: checkPartTypeNonNull(partType);
273: if (isSkipToken(id)) {
274: operation.tokenRecognized(length, true);
275: return skipToken();
276: } else { // Do not skip the token
277: if (operation.tokenRecognized(length, false)) { // Create preprocessed token
278: return new CustomTextToken<T>(id, operation
279: .tokenLength(), text, partType);
280: // return new ComplexToken<T>(id, operation.tokenLength(), null, text, partType);
281: } else {
282: return new CustomTextToken<T>(id, operation
283: .tokenLength(), text, partType);
284: }
285: }
286: }
287:
288: private boolean isSkipToken(T id) {
289: Set<? extends TokenId> skipTokenIds = operation.skipTokenIds();
290: return (skipTokenIds != null) && skipTokenIds.contains(id);
291: }
292:
293: @SuppressWarnings("unchecked")
294: // NOI18N
295: private Token<T> skipToken() {
296: return SKIP_TOKEN;
297: }
298:
299: private void checkPartTypeNonNull(PartType partType) {
300: if (partType == null)
301: throw new IllegalArgumentException(
302: "partType must be non-null");
303: }
304:
305: }
|