001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.lib.lexer.inc;
043:
044: import java.util.List;
045: import java.util.Set;
046: import org.netbeans.api.lexer.LanguagePath;
047: import org.netbeans.lib.lexer.LAState;
048: import org.netbeans.lib.lexer.LexerSpiPackageAccessor;
049: import org.netbeans.lib.lexer.TextLexerInputOperation;
050: import org.netbeans.lib.lexer.TokenList;
051: import org.netbeans.lib.editor.util.FlyOffsetGapList;
052: import org.netbeans.lib.lexer.EmbeddingContainer;
053: import org.netbeans.lib.lexer.LexerInputOperation;
054: import org.netbeans.lib.lexer.LexerUtilsConstants;
055: import org.netbeans.api.lexer.InputAttributes;
056: import org.netbeans.api.lexer.Language;
057: import org.netbeans.api.lexer.Token;
058: import org.netbeans.api.lexer.TokenId;
059: import org.netbeans.lib.lexer.TokenHierarchyOperation;
060: import org.netbeans.lib.lexer.token.AbstractToken;
061: import org.netbeans.lib.lexer.token.TextToken;
062: import org.netbeans.spi.lexer.MutableTextInput;
063:
064: /**
065: * Incremental token list maintains a list of tokens
066: * at the root language level.
067: * <br/>
068: * The physical storage contains a gap to speed up list modifications
069: * during typing in a document when tokens are typically added/removed
070: * at the same index in the list.
071: *
072: * <p>
073: * There is an intent to not degrade performance significantly
074: * with each extra language embedding level so the token list maintains direct
075: * link to the root level.
076: *
077: * @author Miloslav Metelka
078: * @version 1.00
079: */
080:
081: public final class IncTokenList<T extends TokenId> extends
082: FlyOffsetGapList<Object> implements MutableTokenList<T> {
083:
084: private final TokenHierarchyOperation<?, T> tokenHierarchyOperation;
085:
086: private LanguagePath languagePath;
087:
088: private CharSequence text;
089:
090: /**
091: * Lexer input operation used for lexing of the input.
092: */
093: private LexerInputOperation<T> lexerInputOperation;
094:
095: private int rootModCount;
096:
097: private LAState laState;
098:
099: public IncTokenList(
100: TokenHierarchyOperation<?, T> tokenHierarchyOperation) {
101: this .tokenHierarchyOperation = tokenHierarchyOperation;
102: }
103:
104: /**
105: * Activate this list internally if it's currently active (its languagePath() != null)
106: * or deactivate if LP == null.
107: */
108: public void reinit() {
109: if (languagePath != null) {
110: MutableTextInput input = tokenHierarchyOperation
111: .mutableTextInput();
112: this .text = LexerSpiPackageAccessor.get().text(input);
113: this .lexerInputOperation = new TextLexerInputOperation<T>(
114: this , text);
115: } else {
116: this .text = null;
117: releaseLexerInputOperation();
118: }
119: this .laState = LAState.empty();
120: }
121:
122: private void releaseLexerInputOperation() {
123: if (lexerInputOperation != null)
124: lexerInputOperation.release();
125: }
126:
127: public void refreshLexerInputOperation() {
128: releaseLexerInputOperation();
129: int lastTokenIndex = tokenCountCurrent() - 1;
130: lexerInputOperation = createLexerInputOperation(
131: lastTokenIndex + 1, existingTokensEndOffset(),
132: (lastTokenIndex >= 0) ? state(lastTokenIndex) : null);
133: }
134:
135: public LanguagePath languagePath() {
136: return languagePath;
137: }
138:
139: public void setLanguagePath(LanguagePath languagePath) {
140: this .languagePath = languagePath;
141: }
142:
143: public boolean updateLanguagePath() {
144: Language<?> language = LexerSpiPackageAccessor.get().language(
145: tokenHierarchyOperation.mutableTextInput());
146: if (language != null) {
147: setLanguagePath(LanguagePath.get(language));
148: return true;
149: }
150: return false;
151: }
152:
153: public synchronized int tokenCount() {
154: if (lexerInputOperation != null) { // still lexing
155: tokenOrEmbeddingContainerImpl(Integer.MAX_VALUE);
156: }
157: return size();
158: }
159:
160: public char childTokenCharAt(int rawOffset, int index) {
161: index += childTokenOffset(rawOffset);
162: return text.charAt(index);
163: }
164:
165: public int childTokenOffset(int rawOffset) {
166: return (rawOffset < offsetGapStart() ? rawOffset : rawOffset
167: - offsetGapLength());
168: }
169:
170: public int tokenOffset(int index) {
171: return elementOffset(index);
172: }
173:
174: public int existingTokensEndOffset() {
175: return elementOrEndOffset(tokenCountCurrent());
176: }
177:
178: /**
179: * Get modification count for which this token list was last updated
180: * (mainly its cached start offset).
181: */
182: public synchronized int modCount() {
183: return rootModCount;
184: }
185:
186: public void incrementModCount() {
187: rootModCount++;
188: }
189:
190: public synchronized Object tokenOrEmbeddingContainer(int index) {
191: return tokenOrEmbeddingContainerImpl(index);
192: }
193:
194: private Object tokenOrEmbeddingContainerImpl(int index) {
195: while (lexerInputOperation != null && index >= size()) {
196: Token token = lexerInputOperation.nextToken();
197: if (token != null) { // lexer returned valid token
198: updateElementOffsetAdd(token);
199: add(token);
200: laState = laState.add(lexerInputOperation.lookahead(),
201: lexerInputOperation.lexerState());
202: } else { // no more tokens from lexer
203: lexerInputOperation.release();
204: lexerInputOperation = null;
205: trimToSize();
206: laState.trimToSize();
207: }
208: }
209: return (index < size()) ? get(index) : null;
210: }
211:
212: public synchronized AbstractToken<T> replaceFlyToken(int index,
213: AbstractToken<T> flyToken, int offset) {
214: TextToken<T> nonFlyToken = ((TextToken<T>) flyToken)
215: .createCopy(this , offset2Raw(offset));
216: set(index, nonFlyToken);
217: return nonFlyToken;
218: }
219:
220: public synchronized void wrapToken(int index,
221: EmbeddingContainer embeddingContainer) {
222: set(index, embeddingContainer);
223: }
224:
225: public InputAttributes inputAttributes() {
226: return LexerSpiPackageAccessor.get().inputAttributes(
227: tokenHierarchyOperation.mutableTextInput());
228: }
229:
230: protected int elementRawOffset(Object elem) {
231: return LexerUtilsConstants.token(elem).rawOffset();
232: }
233:
234: protected void setElementRawOffset(Object elem, int rawOffset) {
235: LexerUtilsConstants.token(elem).setRawOffset(rawOffset);
236: }
237:
238: protected boolean isElementFlyweight(Object elem) {
239: // token wrapper always contains non-flyweight token
240: return (elem.getClass() != EmbeddingContainer.class)
241: && ((AbstractToken) elem).isFlyweight();
242: }
243:
244: protected int elementLength(Object elem) {
245: return LexerUtilsConstants.token(elem).length();
246: }
247:
248: private AbstractToken<T> existingToken(int index) {
249: // Must use synced tokenOrEmbeddingContainer() because of possible change
250: // of the underlying list impl when adding lazily requested tokens
251: return LexerUtilsConstants
252: .token(tokenOrEmbeddingContainer(index));
253: }
254:
255: public Object tokenOrEmbeddingContainerUnsync(int index) {
256: // Solely for token list updater or token hierarchy snapshots
257: // having single-threaded exclusive write access
258: return get(index);
259: }
260:
261: public int lookahead(int index) {
262: return laState.lookahead(index);
263: }
264:
265: public Object state(int index) {
266: return laState.state(index);
267: }
268:
269: public int tokenCountCurrent() {
270: return size();
271: }
272:
273: public TokenList<?> root() {
274: return this ;
275: }
276:
277: public TokenHierarchyOperation<?, ?> tokenHierarchyOperation() {
278: return tokenHierarchyOperation;
279: }
280:
281: public LexerInputOperation<T> createLexerInputOperation(
282: int tokenIndex, int relexOffset, Object relexState) {
283: // Used for mutable lists only so maintain LA and state
284: return new TextLexerInputOperation<T>(this , tokenIndex,
285: relexState, text, 0, relexOffset, text.length());
286: }
287:
288: public boolean isFullyLexed() {
289: return (lexerInputOperation == null);
290: }
291:
292: public void replaceTokens(TokenListChange<T> change,
293: int removeTokenCount, int diffLength) {
294: int index = change.index();
295: // Remove obsolete tokens (original offsets are retained)
296: Object[] removedTokensOrEmbeddingContainers = new Object[removeTokenCount];
297: copyElements(index, index + removeTokenCount,
298: removedTokensOrEmbeddingContainers, 0);
299: int offset = change.offset();
300: for (int i = 0; i < removeTokenCount; i++) {
301: Object tokenOrEmbeddingContainer = removedTokensOrEmbeddingContainers[i];
302: AbstractToken<?> token;
303: // It's necessary to update-status of all removed tokens' contained embeddings
304: // since otherwise (if they would not be up-to-date) they could not be updated later
305: // as they lose their parent token list which the update-status relies on.
306: if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {
307: EmbeddingContainer<?> ec = (EmbeddingContainer<?>) tokenOrEmbeddingContainer;
308: ec.updateStatusAndInvalidate();
309: token = ec.token();
310: } else { // Regular token
311: token = (AbstractToken<?>) tokenOrEmbeddingContainer;
312: }
313: if (!token.isFlyweight()) {
314: updateElementOffsetRemove(token);
315: token.setTokenList(null);
316: }
317: offset += token.length();
318: }
319: remove(index, removeTokenCount); // Retain original offsets
320: laState.remove(index, removeTokenCount); // Remove lookaheads and states
321: change.setRemovedTokens(removedTokensOrEmbeddingContainers);
322: change.setRemovedEndOffset(offset);
323:
324: // Move and fix the gap according to the performed modification.
325: if (offsetGapStart() != change.offset()) {
326: // Minimum of the index of the first removed index and original computed index
327: moveOffsetGap(change.offset(), Math.min(index, change
328: .offsetGapIndex()));
329: }
330: updateOffsetGapLength(-diffLength);
331:
332: // Add created tokens.
333: List<Object> addedTokensOrBranches = change
334: .addedTokensOrBranches();
335: if (addedTokensOrBranches != null) {
336: for (Object tokenOrBranch : addedTokensOrBranches) {
337: @SuppressWarnings("unchecked")
338: AbstractToken<T> token = (AbstractToken<T>) tokenOrBranch;
339: updateElementOffsetAdd(token);
340: }
341: addAll(index, addedTokensOrBranches);
342: laState = laState.addAll(index, change.laState());
343: change.syncAddedTokenCount();
344: // Check for bounds change only
345: if (removeTokenCount == 1
346: && addedTokensOrBranches.size() == 1) {
347: // Compare removed and added token ids and part types
348: AbstractToken<T> removedToken = LexerUtilsConstants
349: .token(removedTokensOrEmbeddingContainers[0]);
350: AbstractToken<T> addedToken = change.addedToken(0);
351: if (removedToken.id() == addedToken.id()
352: && removedToken.partType() == addedToken
353: .partType()) {
354: change.markBoundsChange();
355: }
356: }
357: }
358: }
359:
360: public boolean isContinuous() {
361: return true;
362: }
363:
364: public Set<T> skipTokenIds() {
365: return null;
366: }
367:
368: public int startOffset() {
369: return 0;
370: }
371:
372: public int endOffset() {
373: return text.length();
374: }
375:
376: public boolean isRemoved() {
377: return false; // Should never become removed
378: }
379:
380: @Override
381: public String toString() {
382: return LexerUtilsConstants.appendTokenList(null, this )
383: .toString();
384: }
385:
386: public CharSequence text() {
387: return text;
388: }
389:
390: public void setText(CharSequence text) {
391: this.text = text;
392: }
393:
394: }
|