001: /*******************************************************************************
002: * Copyright (c) 2000, 2005 IBM Corporation and others.
003: * All rights reserved. This program and the accompanying materials
004: * are made available under the terms of the Eclipse Public License v1.0
005: * which accompanies this distribution, and is available at
006: * http://www.eclipse.org/legal/epl-v10.html
007: *
008: * Contributors:
009: * IBM Corporation - initial API and implementation
010: *******************************************************************************/package org.eclipse.jdt.internal.ui.text;
011:
012: import org.eclipse.jface.text.IDocument;
013: import org.eclipse.jface.text.rules.ICharacterScanner;
014: import org.eclipse.jface.text.rules.IPartitionTokenScanner;
015: import org.eclipse.jface.text.rules.IToken;
016: import org.eclipse.jface.text.rules.Token;
017:
018: import org.eclipse.jdt.ui.text.IJavaPartitions;
019:
020: /**
021: * This scanner recognizes the JavaDoc comments, Java multi line comments, Java single line comments,
022: * Java strings and Java characters.
023: */
024: public class FastJavaPartitionScanner implements
025: IPartitionTokenScanner, IJavaPartitions {
026:
027: // states
028: private static final int JAVA = 0;
029: private static final int SINGLE_LINE_COMMENT = 1;
030: private static final int MULTI_LINE_COMMENT = 2;
031: private static final int JAVADOC = 3;
032: private static final int CHARACTER = 4;
033: private static final int STRING = 5;
034:
035: // beginning of prefixes and postfixes
036: private static final int NONE = 0;
037: private static final int BACKSLASH = 1; // postfix for STRING and CHARACTER
038: private static final int SLASH = 2; // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
039: private static final int SLASH_STAR = 3; // prefix for MULTI_LINE_COMMENT or JAVADOC
040: private static final int SLASH_STAR_STAR = 4; // prefix for MULTI_LINE_COMMENT or JAVADOC
041: private static final int STAR = 5; // postfix for MULTI_LINE_COMMENT or JAVADOC
042: private static final int CARRIAGE_RETURN = 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
043:
044: /** The scanner. */
045: private final BufferedDocumentScanner fScanner = new BufferedDocumentScanner(
046: 1000); // faster implementation
047:
048: /** The offset of the last returned token. */
049: private int fTokenOffset;
050: /** The length of the last returned token. */
051: private int fTokenLength;
052:
053: /** The state of the scanner. */
054: private int fState;
055: /** The last significant characters read. */
056: private int fLast;
057: /** The amount of characters already read on first call to nextToken(). */
058: private int fPrefixLength;
059:
060: // emulate JavaPartitionScanner
061: private boolean fEmulate = false;
062: private int fJavaOffset;
063: private int fJavaLength;
064:
065: private final IToken[] fTokens = new IToken[] { new Token(null),
066: new Token(JAVA_SINGLE_LINE_COMMENT),
067: new Token(JAVA_MULTI_LINE_COMMENT), new Token(JAVA_DOC),
068: new Token(JAVA_CHARACTER), new Token(JAVA_STRING) };
069:
070: public FastJavaPartitionScanner(boolean emulate) {
071: fEmulate = emulate;
072: }
073:
074: public FastJavaPartitionScanner() {
075: this (false);
076: }
077:
078: /*
079: * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
080: */
081: public IToken nextToken() {
082:
083: // emulate JavaPartitionScanner
084: if (fEmulate) {
085: if (fJavaOffset != -1
086: && fTokenOffset + fTokenLength != fJavaOffset
087: + fJavaLength) {
088: fTokenOffset += fTokenLength;
089: return fTokens[JAVA];
090: } else {
091: fJavaOffset = -1;
092: fJavaLength = 0;
093: }
094: }
095:
096: fTokenOffset += fTokenLength;
097: fTokenLength = fPrefixLength;
098:
099: while (true) {
100: final int ch = fScanner.read();
101:
102: // characters
103: switch (ch) {
104: case ICharacterScanner.EOF:
105: if (fTokenLength > 0) {
106: fLast = NONE; // ignore last
107: return preFix(fState, JAVA, NONE, 0);
108:
109: } else {
110: fLast = NONE;
111: fPrefixLength = 0;
112: return Token.EOF;
113: }
114:
115: case '\r':
116: // emulate JavaPartitionScanner
117: if (!fEmulate && fLast != CARRIAGE_RETURN) {
118: fLast = CARRIAGE_RETURN;
119: fTokenLength++;
120: continue;
121:
122: } else {
123:
124: switch (fState) {
125: case SINGLE_LINE_COMMENT:
126: case CHARACTER:
127: case STRING:
128: if (fTokenLength > 0) {
129: IToken token = fTokens[fState];
130:
131: // emulate JavaPartitionScanner
132: if (fEmulate) {
133: fTokenLength++;
134: fLast = NONE;
135: fPrefixLength = 0;
136: } else {
137: fLast = CARRIAGE_RETURN;
138: fPrefixLength = 1;
139: }
140:
141: fState = JAVA;
142: return token;
143:
144: } else {
145: consume();
146: continue;
147: }
148:
149: default:
150: consume();
151: continue;
152: }
153: }
154:
155: case '\n':
156: switch (fState) {
157: case SINGLE_LINE_COMMENT:
158: case CHARACTER:
159: case STRING:
160: // assert(fTokenLength > 0);
161: return postFix(fState);
162:
163: default:
164: consume();
165: continue;
166: }
167:
168: default:
169: if (!fEmulate && fLast == CARRIAGE_RETURN) {
170: switch (fState) {
171: case SINGLE_LINE_COMMENT:
172: case CHARACTER:
173: case STRING:
174:
175: int last;
176: int newState;
177: switch (ch) {
178: case '/':
179: last = SLASH;
180: newState = JAVA;
181: break;
182:
183: case '*':
184: last = STAR;
185: newState = JAVA;
186: break;
187:
188: case '\'':
189: last = NONE;
190: newState = CHARACTER;
191: break;
192:
193: case '"':
194: last = NONE;
195: newState = STRING;
196: break;
197:
198: case '\r':
199: last = CARRIAGE_RETURN;
200: newState = JAVA;
201: break;
202:
203: case '\\':
204: last = BACKSLASH;
205: newState = JAVA;
206: break;
207:
208: default:
209: last = NONE;
210: newState = JAVA;
211: break;
212: }
213:
214: fLast = NONE; // ignore fLast
215: return preFix(fState, newState, last, 1);
216:
217: default:
218: break;
219: }
220: }
221: }
222:
223: // states
224: switch (fState) {
225: case JAVA:
226: switch (ch) {
227: case '/':
228: if (fLast == SLASH) {
229: if (fTokenLength - getLastLength(fLast) > 0) {
230: return preFix(JAVA, SINGLE_LINE_COMMENT,
231: NONE, 2);
232: } else {
233: preFix(JAVA, SINGLE_LINE_COMMENT, NONE, 2);
234: fTokenOffset += fTokenLength;
235: fTokenLength = fPrefixLength;
236: break;
237: }
238:
239: } else {
240: fTokenLength++;
241: fLast = SLASH;
242: break;
243: }
244:
245: case '*':
246: if (fLast == SLASH) {
247: if (fTokenLength - getLastLength(fLast) > 0)
248: return preFix(JAVA, MULTI_LINE_COMMENT,
249: SLASH_STAR, 2);
250: else {
251: preFix(JAVA, MULTI_LINE_COMMENT,
252: SLASH_STAR, 2);
253: fTokenOffset += fTokenLength;
254: fTokenLength = fPrefixLength;
255: break;
256: }
257:
258: } else {
259: consume();
260: break;
261: }
262:
263: case '\'':
264: fLast = NONE; // ignore fLast
265: if (fTokenLength > 0)
266: return preFix(JAVA, CHARACTER, NONE, 1);
267: else {
268: preFix(JAVA, CHARACTER, NONE, 1);
269: fTokenOffset += fTokenLength;
270: fTokenLength = fPrefixLength;
271: break;
272: }
273:
274: case '"':
275: fLast = NONE; // ignore fLast
276: if (fTokenLength > 0)
277: return preFix(JAVA, STRING, NONE, 1);
278: else {
279: preFix(JAVA, STRING, NONE, 1);
280: fTokenOffset += fTokenLength;
281: fTokenLength = fPrefixLength;
282: break;
283: }
284:
285: default:
286: consume();
287: break;
288: }
289: break;
290:
291: case SINGLE_LINE_COMMENT:
292: consume();
293: break;
294:
295: case JAVADOC:
296: switch (ch) {
297: case '/':
298: switch (fLast) {
299: case SLASH_STAR_STAR:
300: return postFix(MULTI_LINE_COMMENT);
301:
302: case STAR:
303: return postFix(JAVADOC);
304:
305: default:
306: consume();
307: break;
308: }
309: break;
310:
311: case '*':
312: fTokenLength++;
313: fLast = STAR;
314: break;
315:
316: default:
317: consume();
318: break;
319: }
320: break;
321:
322: case MULTI_LINE_COMMENT:
323: switch (ch) {
324: case '*':
325: if (fLast == SLASH_STAR) {
326: fLast = SLASH_STAR_STAR;
327: fTokenLength++;
328: fState = JAVADOC;
329: } else {
330: fTokenLength++;
331: fLast = STAR;
332: }
333: break;
334:
335: case '/':
336: if (fLast == STAR) {
337: return postFix(MULTI_LINE_COMMENT);
338: } else {
339: consume();
340: break;
341: }
342:
343: default:
344: consume();
345: break;
346: }
347: break;
348:
349: case STRING:
350: switch (ch) {
351: case '\\':
352: fLast = (fLast == BACKSLASH) ? NONE : BACKSLASH;
353: fTokenLength++;
354: break;
355:
356: case '\"':
357: if (fLast != BACKSLASH) {
358: return postFix(STRING);
359:
360: } else {
361: consume();
362: break;
363: }
364:
365: default:
366: consume();
367: break;
368: }
369: break;
370:
371: case CHARACTER:
372: switch (ch) {
373: case '\\':
374: fLast = (fLast == BACKSLASH) ? NONE : BACKSLASH;
375: fTokenLength++;
376: break;
377:
378: case '\'':
379: if (fLast != BACKSLASH) {
380: return postFix(CHARACTER);
381:
382: } else {
383: consume();
384: break;
385: }
386:
387: default:
388: consume();
389: break;
390: }
391: break;
392: }
393: }
394: }
395:
396: private static final int getLastLength(int last) {
397: switch (last) {
398: default:
399: return -1;
400:
401: case NONE:
402: return 0;
403:
404: case CARRIAGE_RETURN:
405: case BACKSLASH:
406: case SLASH:
407: case STAR:
408: return 1;
409:
410: case SLASH_STAR:
411: return 2;
412:
413: case SLASH_STAR_STAR:
414: return 3;
415: }
416: }
417:
418: private final void consume() {
419: fTokenLength++;
420: fLast = NONE;
421: }
422:
423: private final IToken postFix(int state) {
424: fTokenLength++;
425: fLast = NONE;
426: fState = JAVA;
427: fPrefixLength = 0;
428: return fTokens[state];
429: }
430:
431: private final IToken preFix(int state, int newState, int last,
432: int prefixLength) {
433: // emulate JavaPartitionScanner
434: if (fEmulate && state == JAVA
435: && (fTokenLength - getLastLength(fLast) > 0)) {
436: fTokenLength -= getLastLength(fLast);
437: fJavaOffset = fTokenOffset;
438: fJavaLength = fTokenLength;
439: fTokenLength = 1;
440: fState = newState;
441: fPrefixLength = prefixLength;
442: fLast = last;
443: return fTokens[state];
444:
445: } else {
446: fTokenLength -= getLastLength(fLast);
447: fLast = last;
448: fPrefixLength = prefixLength;
449: IToken token = fTokens[state];
450: fState = newState;
451: return token;
452: }
453: }
454:
455: private static int getState(String contentType) {
456:
457: if (contentType == null)
458: return JAVA;
459:
460: else if (contentType.equals(JAVA_SINGLE_LINE_COMMENT))
461: return SINGLE_LINE_COMMENT;
462:
463: else if (contentType.equals(JAVA_MULTI_LINE_COMMENT))
464: return MULTI_LINE_COMMENT;
465:
466: else if (contentType.equals(JAVA_DOC))
467: return JAVADOC;
468:
469: else if (contentType.equals(JAVA_STRING))
470: return STRING;
471:
472: else if (contentType.equals(JAVA_CHARACTER))
473: return CHARACTER;
474:
475: else
476: return JAVA;
477: }
478:
479: /*
480: * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String, int)
481: */
482: public void setPartialRange(IDocument document, int offset,
483: int length, String contentType, int partitionOffset) {
484:
485: fScanner.setRange(document, offset, length);
486: fTokenOffset = partitionOffset;
487: fTokenLength = 0;
488: fPrefixLength = offset - partitionOffset;
489: fLast = NONE;
490:
491: if (offset == partitionOffset) {
492: // restart at beginning of partition
493: fState = JAVA;
494: } else {
495: fState = getState(contentType);
496: }
497:
498: // emulate JavaPartitionScanner
499: if (fEmulate) {
500: fJavaOffset = -1;
501: fJavaLength = 0;
502: }
503: }
504:
505: /*
506: * @see ITokenScanner#setRange(IDocument, int, int)
507: */
508: public void setRange(IDocument document, int offset, int length) {
509:
510: fScanner.setRange(document, offset, length);
511: fTokenOffset = offset;
512: fTokenLength = 0;
513: fPrefixLength = 0;
514: fLast = NONE;
515: fState = JAVA;
516:
517: // emulate JavaPartitionScanner
518: if (fEmulate) {
519: fJavaOffset = -1;
520: fJavaLength = 0;
521: }
522: }
523:
524: /*
525: * @see ITokenScanner#getTokenLength()
526: */
527: public int getTokenLength() {
528: return fTokenLength;
529: }
530:
531: /*
532: * @see ITokenScanner#getTokenOffset()
533: */
534: public int getTokenOffset() {
535: return fTokenOffset;
536: }
537:
538: }
|