001: /*******************************************************************************
002: * Copyright (c) 2000, 2007 IBM Corporation and others.
003: * All rights reserved. This program and the accompanying materials
004: * are made available under the terms of the Eclipse Public License v1.0
005: * which accompanies this distribution, and is available at
006: * http://www.eclipse.org/legal/epl-v10.html
007: *
008: * Contributors:
009: * IBM Corporation - initial API and implementation
010: *******************************************************************************/package org.eclipse.jdt.internal.codeassist.complete;
011:
012: /*
013: * Scanner aware of a cursor location so as to discard trailing portions of identifiers
014: * containing the cursor location.
015: *
016: * Cursor location denotes the position of the last character behind which completion
017: * got requested:
018: * -1 means completion at the very beginning of the source
019: * 0 means completion behind the first character
020: * n means completion behind the n-th character
021: */
022: import org.eclipse.jdt.core.compiler.*;
023: import org.eclipse.jdt.internal.compiler.parser.Scanner;
024: import org.eclipse.jdt.internal.compiler.parser.ScannerHelper;
025:
026: public class CompletionScanner extends Scanner {
027:
028: public char[] completionIdentifier;
029: public int cursorLocation;
030: public int endOfEmptyToken = -1;
031:
032: /* Source positions of the completedIdentifier
033: * if inside actual identifier, end goes to the actual identifier
034: * end, in other words, beyond cursor location
035: */
036: public int completedIdentifierStart = 0;
037: public int completedIdentifierEnd = -1;
038: public int unicodeCharSize;
039:
040: public static final char[] EmptyCompletionIdentifier = {};
041:
042: public CompletionScanner(long sourceLevel) {
043: super (false /*comment*/, false /*whitespace*/,
044: false /*nls*/, sourceLevel, null /*taskTags*/,
045: null/*taskPriorities*/, true/*taskCaseSensitive*/);
046: }
047:
048: /*
049: * Truncate the current identifier if it is containing the cursor location. Since completion is performed
050: * on an identifier prefix.
051: *
052: */
053: public char[] getCurrentIdentifierSource() {
054:
055: if (this .completionIdentifier == null) {
056: if (this .cursorLocation < this .startPosition
057: && this .currentPosition == this .startPosition) { // fake empty identifier got issued
058: // remember actual identifier positions
059: this .completedIdentifierStart = this .startPosition;
060: this .completedIdentifierEnd = this .completedIdentifierStart - 1;
061: return this .completionIdentifier = EmptyCompletionIdentifier;
062: }
063: if (this .cursorLocation + 1 >= this .startPosition
064: && this .cursorLocation < this .currentPosition) {
065: // remember actual identifier positions
066: this .completedIdentifierStart = this .startPosition;
067: this .completedIdentifierEnd = this .currentPosition - 1;
068: if (this .withoutUnicodePtr != 0) { // check unicode scenario
069: int length = this .cursorLocation + 1
070: - this .startPosition - this .unicodeCharSize;
071: System
072: .arraycopy(
073: this .withoutUnicodeBuffer,
074: 1,
075: this .completionIdentifier = new char[length],
076: 0, length);
077: } else {
078: // no char[] sharing around completionIdentifier, we want it to be unique so as to use identity checks
079: int length = this .cursorLocation + 1
080: - this .startPosition;
081: System
082: .arraycopy(
083: this .source,
084: this .startPosition,
085: (this .completionIdentifier = new char[length]),
086: 0, length);
087: }
088: return this .completionIdentifier;
089: }
090: }
091: return super .getCurrentIdentifierSource();
092: }
093:
094: public char[] getCurrentTokenSourceString() {
095: if (this .completionIdentifier == null) {
096: if (this .cursorLocation + 1 >= this .startPosition
097: && this .cursorLocation < this .currentPosition) {
098: // remember actual identifier positions
099: this .completedIdentifierStart = this .startPosition;
100: this .completedIdentifierEnd = this .currentPosition - 1;
101: if (this .withoutUnicodePtr != 0) { // check unicode scenario
102: int length = this .cursorLocation
103: - this .startPosition - this .unicodeCharSize;
104: System
105: .arraycopy(
106: this .withoutUnicodeBuffer,
107: 2,
108: this .completionIdentifier = new char[length],
109: 0, length);
110: } else {
111: // no char[] sharing around completionIdentifier, we want it to be unique so as to use identity checks
112: int length = this .cursorLocation
113: - this .startPosition;
114: System
115: .arraycopy(
116: this .source,
117: this .startPosition + 1,
118: (this .completionIdentifier = new char[length]),
119: 0, length);
120: }
121: return this .completionIdentifier;
122: }
123: }
124: return super .getCurrentTokenSourceString();
125: }
126:
127: public int getNextToken() throws InvalidInputException {
128:
129: this .wasAcr = false;
130: this .unicodeCharSize = 0;
131: if (this .diet) {
132: jumpOverMethodBody();
133: this .diet = false;
134: return this .currentPosition > this .eofPosition ? TokenNameEOF
135: : TokenNameRBRACE;
136: }
137: int whiteStart = 0;
138: try {
139: while (true) { //loop for jumping over comments
140: this .withoutUnicodePtr = 0;
141: //start with a new token (even comment written with unicode )
142:
143: // ---------Consume white space and handles start position---------
144: whiteStart = this .currentPosition;
145: boolean isWhiteSpace, hasWhiteSpaces = false;
146: int offset = 0;
147: do {
148: this .startPosition = this .currentPosition;
149: boolean checkIfUnicode = false;
150: try {
151: checkIfUnicode = ((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
152: && (this .source[this .currentPosition] == 'u');
153: } catch (IndexOutOfBoundsException e) {
154: if (this .tokenizeWhiteSpace
155: && (whiteStart != this .currentPosition - 1)) {
156: // reposition scanner in case we are interested by spaces as tokens
157: this .currentPosition--;
158: this .startPosition = whiteStart;
159: return TokenNameWHITESPACE;
160: }
161: if (this .currentPosition > this .eofPosition) {
162: /* might be completing at eof (e.g. behind a dot) */
163: if (this .completionIdentifier == null
164: && this .startPosition == this .cursorLocation + 1) {
165: this .currentPosition = this .startPosition; // for being detected as empty free identifier
166: return TokenNameIdentifier;
167: }
168: return TokenNameEOF;
169: }
170: }
171: if (checkIfUnicode) {
172: isWhiteSpace = jumpOverUnicodeWhiteSpace();
173: offset = 6;
174: } else {
175: offset = 1;
176: if ((this .currentCharacter == '\r')
177: || (this .currentCharacter == '\n')) {
178: //checkNonExternalizedString();
179: if (this .recordLineSeparator) {
180: pushLineSeparator();
181: }
182: }
183: isWhiteSpace = (this .currentCharacter == ' ')
184: || CharOperation
185: .isWhitespace(this .currentCharacter);
186: }
187: if (isWhiteSpace) {
188: hasWhiteSpaces = true;
189: }
190: /* completion requesting strictly inside blanks */
191: if ((whiteStart != this .currentPosition)
192: //&& (previousToken == TokenNameDOT)
193: && (this .completionIdentifier == null)
194: && (whiteStart <= this .cursorLocation + 1)
195: && (this .cursorLocation < this .startPosition)
196: && !ScannerHelper
197: .isJavaIdentifierStart(this .currentCharacter)) {
198: this .currentPosition = this .startPosition; // for next token read
199: return TokenNameIdentifier;
200: }
201: } while (isWhiteSpace);
202: if (this .tokenizeWhiteSpace && hasWhiteSpaces) {
203: // reposition scanner in case we are interested by spaces as tokens
204: this .currentPosition -= offset;
205: this .startPosition = whiteStart;
206: return TokenNameWHITESPACE;
207: }
208: //little trick to get out in the middle of a source computation
209: if (this .currentPosition > this .eofPosition) {
210: /* might be completing at eof (e.g. behind a dot) */
211: if (this .completionIdentifier == null
212: && this .startPosition == this .cursorLocation + 1) {
213: // compute end of empty identifier.
214: // if the empty identifier is at the start of a next token the end of
215: // empty identifier is the end of the next token (eg. "<empty token>next").
216: int temp = this .eofPosition;
217: this .eofPosition = this .source.length;
218: while (getNextCharAsJavaIdentifierPart()) {/*empty*/
219: }
220: this .eofPosition = temp;
221: this .endOfEmptyToken = this .currentPosition - 1;
222: this .currentPosition = this .startPosition; // for being detected as empty free identifier
223: return TokenNameIdentifier;
224: }
225: return TokenNameEOF;
226: }
227:
228: // ---------Identify the next token-------------
229:
230: switch (this .currentCharacter) {
231: case '@':
232: return TokenNameAT;
233: case '(':
234: return TokenNameLPAREN;
235: case ')':
236: return TokenNameRPAREN;
237: case '{':
238: return TokenNameLBRACE;
239: case '}':
240: return TokenNameRBRACE;
241: case '[':
242: return TokenNameLBRACKET;
243: case ']':
244: return TokenNameRBRACKET;
245: case ';':
246: return TokenNameSEMICOLON;
247: case ',':
248: return TokenNameCOMMA;
249: case '.':
250: if (this .startPosition <= this .cursorLocation
251: && this .cursorLocation < this .currentPosition) {
252: return TokenNameDOT; // completion inside .<|>12
253: }
254: if (getNextCharAsDigit()) {
255: return scanNumber(true);
256: }
257: int temp = this .currentPosition;
258: if (getNextChar('.')) {
259: if (getNextChar('.')) {
260: return TokenNameELLIPSIS;
261: } else {
262: this .currentPosition = temp;
263: return TokenNameDOT;
264: }
265: } else {
266: this .currentPosition = temp;
267: return TokenNameDOT;
268: }
269: case '+': {
270: int test;
271: if ((test = getNextChar('+', '=')) == 0)
272: return TokenNamePLUS_PLUS;
273: if (test > 0)
274: return TokenNamePLUS_EQUAL;
275: return TokenNamePLUS;
276: }
277: case '-': {
278: int test;
279: if ((test = getNextChar('-', '=')) == 0)
280: return TokenNameMINUS_MINUS;
281: if (test > 0)
282: return TokenNameMINUS_EQUAL;
283: return TokenNameMINUS;
284: }
285: case '~':
286: return TokenNameTWIDDLE;
287: case '!':
288: if (getNextChar('='))
289: return TokenNameNOT_EQUAL;
290: return TokenNameNOT;
291: case '*':
292: if (getNextChar('='))
293: return TokenNameMULTIPLY_EQUAL;
294: return TokenNameMULTIPLY;
295: case '%':
296: if (getNextChar('='))
297: return TokenNameREMAINDER_EQUAL;
298: return TokenNameREMAINDER;
299: case '<': {
300: int test;
301: if ((test = getNextChar('=', '<')) == 0)
302: return TokenNameLESS_EQUAL;
303: if (test > 0) {
304: if (getNextChar('='))
305: return TokenNameLEFT_SHIFT_EQUAL;
306: return TokenNameLEFT_SHIFT;
307: }
308: return TokenNameLESS;
309: }
310: case '>': {
311: int test;
312: if (this .returnOnlyGreater) {
313: return TokenNameGREATER;
314: }
315: if ((test = getNextChar('=', '>')) == 0)
316: return TokenNameGREATER_EQUAL;
317: if (test > 0) {
318: if ((test = getNextChar('=', '>')) == 0)
319: return TokenNameRIGHT_SHIFT_EQUAL;
320: if (test > 0) {
321: if (getNextChar('='))
322: return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
323: return TokenNameUNSIGNED_RIGHT_SHIFT;
324: }
325: return TokenNameRIGHT_SHIFT;
326: }
327: return TokenNameGREATER;
328: }
329: case '=':
330: if (getNextChar('='))
331: return TokenNameEQUAL_EQUAL;
332: return TokenNameEQUAL;
333: case '&': {
334: int test;
335: if ((test = getNextChar('&', '=')) == 0)
336: return TokenNameAND_AND;
337: if (test > 0)
338: return TokenNameAND_EQUAL;
339: return TokenNameAND;
340: }
341: case '|': {
342: int test;
343: if ((test = getNextChar('|', '=')) == 0)
344: return TokenNameOR_OR;
345: if (test > 0)
346: return TokenNameOR_EQUAL;
347: return TokenNameOR;
348: }
349: case '^':
350: if (getNextChar('='))
351: return TokenNameXOR_EQUAL;
352: return TokenNameXOR;
353: case '?':
354: return TokenNameQUESTION;
355: case ':':
356: return TokenNameCOLON;
357: case '\'': {
358: int test;
359: if ((test = getNextChar('\n', '\r')) == 0) {
360: throw new InvalidInputException(
361: INVALID_CHARACTER_CONSTANT);
362: }
363: if (test > 0) {
364: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
365: for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
366: if (this .currentPosition + lookAhead == this .eofPosition)
367: break;
368: if (this .source[this .currentPosition
369: + lookAhead] == '\n')
370: break;
371: if (this .source[this .currentPosition
372: + lookAhead] == '\'') {
373: this .currentPosition += lookAhead + 1;
374: break;
375: }
376: }
377: throw new InvalidInputException(
378: INVALID_CHARACTER_CONSTANT);
379: }
380: }
381: if (getNextChar('\'')) {
382: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
383: for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
384: if (this .currentPosition + lookAhead == this .eofPosition)
385: break;
386: if (this .source[this .currentPosition
387: + lookAhead] == '\n')
388: break;
389: if (this .source[this .currentPosition
390: + lookAhead] == '\'') {
391: this .currentPosition += lookAhead + 1;
392: break;
393: }
394: }
395: throw new InvalidInputException(
396: INVALID_CHARACTER_CONSTANT);
397: }
398: if (getNextChar('\\')) {
399: if (this .unicodeAsBackSlash) {
400: // consume next character
401: this .unicodeAsBackSlash = false;
402: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
403: && (this .source[this .currentPosition] == 'u')) {
404: getNextUnicodeChar();
405: } else {
406: if (this .withoutUnicodePtr != 0) {
407: unicodeStore();
408: }
409: }
410: } else {
411: this .currentCharacter = this .source[this .currentPosition++];
412: }
413: scanEscapeCharacter();
414: } else { // consume next character
415: this .unicodeAsBackSlash = false;
416: boolean checkIfUnicode = false;
417: try {
418: checkIfUnicode = ((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
419: && (this .source[this .currentPosition] == 'u');
420: } catch (IndexOutOfBoundsException e) {
421: this .currentPosition--;
422: throw new InvalidInputException(
423: INVALID_CHARACTER_CONSTANT);
424: }
425: if (checkIfUnicode) {
426: getNextUnicodeChar();
427: } else {
428: if (this .withoutUnicodePtr != 0) {
429: this .unicodeStore();
430: }
431: }
432: }
433: if (getNextChar('\''))
434: return TokenNameCharacterLiteral;
435: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
436: for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
437: if (this .currentPosition + lookAhead == this .eofPosition)
438: break;
439: if (this .source[this .currentPosition
440: + lookAhead] == '\n')
441: break;
442: if (this .source[this .currentPosition
443: + lookAhead] == '\'') {
444: this .currentPosition += lookAhead + 1;
445: break;
446: }
447: }
448: throw new InvalidInputException(
449: INVALID_CHARACTER_CONSTANT);
450: case '"':
451: try {
452: // consume next character
453: this .unicodeAsBackSlash = false;
454: boolean isUnicode = false;
455: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
456: && (this .source[this .currentPosition] == 'u')) {
457: getNextUnicodeChar();
458: isUnicode = true;
459: } else {
460: if (this .withoutUnicodePtr != 0) {
461: this .unicodeStore();
462: }
463: }
464:
465: while (this .currentCharacter != '"') {
466: /**** \r and \n are not valid in string literals ****/
467: if ((this .currentCharacter == '\n')
468: || (this .currentCharacter == '\r')) {
469: if (isUnicode) {
470: int start = this .currentPosition - 5;
471: while (this .source[start] != '\\') {
472: start--;
473: }
474: if (this .startPosition <= this .cursorLocation
475: && this .cursorLocation <= this .currentPosition - 1) {
476: this .currentPosition = start;
477: // complete inside a string literal
478: return TokenNameStringLiteral;
479: }
480: start = this .currentPosition;
481: for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
482: if (this .currentPosition >= this .eofPosition) {
483: this .currentPosition = start;
484: break;
485: }
486: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
487: && (this .source[this .currentPosition] == 'u')) {
488: isUnicode = true;
489: getNextUnicodeChar();
490: } else {
491: isUnicode = false;
492: }
493: if (!isUnicode
494: && this .currentCharacter == '\n') {
495: this .currentPosition--; // set current position on new line character
496: break;
497: }
498: if (this .currentCharacter == '\"') {
499: throw new InvalidInputException(
500: INVALID_CHAR_IN_STRING);
501: }
502: }
503: } else {
504: this .currentPosition--; // set current position on new line character
505: if (this .startPosition <= this .cursorLocation
506: && this .cursorLocation <= this .currentPosition - 1) {
507: // complete inside a string literal
508: return TokenNameStringLiteral;
509: }
510: }
511: throw new InvalidInputException(
512: INVALID_CHAR_IN_STRING);
513: }
514: if (this .currentCharacter == '\\') {
515: if (this .unicodeAsBackSlash) {
516: this .withoutUnicodePtr--;
517: // consume next character
518: this .unicodeAsBackSlash = false;
519: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
520: && (this .source[this .currentPosition] == 'u')) {
521: getNextUnicodeChar();
522: isUnicode = true;
523: this .withoutUnicodePtr--;
524: } else {
525: isUnicode = false;
526: }
527: } else {
528: if (this .withoutUnicodePtr == 0) {
529: unicodeInitializeBuffer(this .currentPosition
530: - this .startPosition);
531: }
532: this .withoutUnicodePtr--;
533: this .currentCharacter = this .source[this .currentPosition++];
534: }
535: // we need to compute the escape character in a separate buffer
536: scanEscapeCharacter();
537: if (this .withoutUnicodePtr != 0) {
538: unicodeStore();
539: }
540: }
541: // consume next character
542: this .unicodeAsBackSlash = false;
543: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
544: && (this .source[this .currentPosition] == 'u')) {
545: getNextUnicodeChar();
546: isUnicode = true;
547: } else {
548: isUnicode = false;
549: if (this .withoutUnicodePtr != 0) {
550: this .unicodeStore();
551: }
552: }
553:
554: }
555: } catch (IndexOutOfBoundsException e) {
556: this .currentPosition--;
557: if (this .startPosition <= this .cursorLocation
558: && this .cursorLocation < this .currentPosition) {
559: // complete inside a string literal
560: return TokenNameStringLiteral;
561: }
562: throw new InvalidInputException(
563: UNTERMINATED_STRING);
564: } catch (InvalidInputException e) {
565: if (e.getMessage().equals(INVALID_ESCAPE)) {
566: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
567: for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
568: if (this .currentPosition + lookAhead == this .eofPosition)
569: break;
570: if (this .source[this .currentPosition
571: + lookAhead] == '\n')
572: break;
573: if (this .source[this .currentPosition
574: + lookAhead] == '\"') {
575: this .currentPosition += lookAhead + 1;
576: break;
577: }
578: }
579:
580: }
581: throw e; // rethrow
582: }
583: return TokenNameStringLiteral;
584: case '/': {
585: int test;
586: if ((test = getNextChar('/', '*')) == 0) { //line comment
587: this .lastCommentLinePosition = this .currentPosition;
588: try { //get the next char
589: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
590: && (this .source[this .currentPosition] == 'u')) {
591: //-------------unicode traitement ------------
592: int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
593: this .currentPosition++;
594: while (this .source[this .currentPosition] == 'u') {
595: this .currentPosition++;
596: }
597: if ((c1 = ScannerHelper
598: .getNumericValue(this .source[this .currentPosition++])) > 15
599: || c1 < 0
600: || (c2 = ScannerHelper
601: .getNumericValue(this .source[this .currentPosition++])) > 15
602: || c2 < 0
603: || (c3 = ScannerHelper
604: .getNumericValue(this .source[this .currentPosition++])) > 15
605: || c3 < 0
606: || (c4 = ScannerHelper
607: .getNumericValue(this .source[this .currentPosition++])) > 15
608: || c4 < 0) {
609: throw new InvalidInputException(
610: INVALID_UNICODE_ESCAPE);
611: } else {
612: this .currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
613: }
614: }
615:
616: //handle the \\u case manually into comment
617:if (this .currentCharacter == '\\') {
618: if (this .source[this .currentPosition] == '\\')
619: this .currentPosition++;
620: } //jump over the \\
621: boolean isUnicode = false;
622: while (this .currentCharacter != '\r'
623: && this .currentCharacter != '\n') {
624: this .lastCommentLinePosition = this .currentPosition;
625: //get the next char
626: isUnicode = false;
627: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
628: && (this .source[this .currentPosition] == 'u')) {
629: isUnicode = true;
630: //-------------unicode traitement ------------
631: int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
632: this .currentPosition++;
633: while (this .source[this .currentPosition] == 'u') {
634: this .currentPosition++;
635: }
636: if ((c1 = ScannerHelper
637: .getNumericValue(this .source[this .currentPosition++])) > 15
638: || c1 < 0
639: || (c2 = ScannerHelper
640: .getNumericValue(this .source[this .currentPosition++])) > 15
641: || c2 < 0
642: || (c3 = ScannerHelper
643: .getNumericValue(this .source[this .currentPosition++])) > 15
644: || c3 < 0
645: || (c4 = ScannerHelper
646: .getNumericValue(this .source[this .currentPosition++])) > 15
647: || c4 < 0) {
648: throw new InvalidInputException(
649: INVALID_UNICODE_ESCAPE);
650: } else {
651: this .currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
652: }
653: }
654: //handle the \\u case manually into comment
655:if (this .currentCharacter == '\\') {
656: if (this .source[this .currentPosition] == '\\')
657: this .currentPosition++;
658: } //jump over the \\
659: }
660: /*
661: * We need to completely consume the line break
662: */
663: if (this .currentCharacter == '\r'
664: && this .eofPosition > this .currentPosition) {
665: if (this .source[this .currentPosition] == '\n') {
666: this .currentPosition++;
667: this .currentCharacter = '\n';
668: } else if ((this .source[this .currentPosition] == '\\')
669: && (this .source[this .currentPosition + 1] == 'u')) {
670: isUnicode = true;
671: char unicodeChar;
672: int index = this .currentPosition + 1;
673: index++;
674: while (this .source[index] == 'u') {
675: index++;
676: }
677: //-------------unicode traitement ------------
678: int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
679: if ((c1 = ScannerHelper
680: .getNumericValue(this .source[index++])) > 15
681: || c1 < 0
682: || (c2 = ScannerHelper
683: .getNumericValue(this .source[index++])) > 15
684: || c2 < 0
685: || (c3 = ScannerHelper
686: .getNumericValue(this .source[index++])) > 15
687: || c3 < 0
688: || (c4 = ScannerHelper
689: .getNumericValue(this .source[index++])) > 15
690: || c4 < 0) {
691: this .currentPosition = index;
692: throw new InvalidInputException(
693: INVALID_UNICODE_ESCAPE);
694: } else {
695: unicodeChar = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
696: }
697: if (unicodeChar == '\n') {
698: this .currentPosition = index;
699: this .currentCharacter = '\n';
700: }
701: }
702: }
703: recordComment(TokenNameCOMMENT_LINE);
704: if (this .startPosition <= this .cursorLocation
705: && this .cursorLocation < this .currentPosition - 1) {
706: throw new InvalidCursorLocation(
707: InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
708: }
709: if (this .taskTags != null)
710: checkTaskTag(this .startPosition,
711: this .currentPosition);
712: if ((this .currentCharacter == '\r')
713: || (this .currentCharacter == '\n')) {
714: //checkNonExternalizedString();
715: if (this .recordLineSeparator) {
716: if (isUnicode) {
717: pushUnicodeLineSeparator();
718: } else {
719: pushLineSeparator();
720: }
721: }
722: }
723: if (this .tokenizeComments) {
724: return TokenNameCOMMENT_LINE;
725: }
726: } catch (IndexOutOfBoundsException e) {
727: this .currentPosition--;
728: recordComment(TokenNameCOMMENT_LINE);
729: if (this .taskTags != null)
730: checkTaskTag(this .startPosition,
731: this .currentPosition);
732: if (this .tokenizeComments) {
733: return TokenNameCOMMENT_LINE;
734: } else {
735: this .currentPosition++;
736: }
737: }
738: break;
739: }
740: if (test > 0) { //traditional and javadoc comment
741: try { //get the next char
742: boolean isJavadoc = false, star = false;
743: boolean isUnicode = false;
744: // consume next character
745: this .unicodeAsBackSlash = false;
746: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
747: && (this .source[this .currentPosition] == 'u')) {
748: getNextUnicodeChar();
749: isUnicode = true;
750: } else {
751: isUnicode = false;
752: if (this .withoutUnicodePtr != 0) {
753: this .unicodeStore();
754: }
755: }
756:
757: if (this .currentCharacter == '*') {
758: isJavadoc = true;
759: star = true;
760: }
761: if ((this .currentCharacter == '\r')
762: || (this .currentCharacter == '\n')) {
763: //checkNonExternalizedString();
764: if (this .recordLineSeparator) {
765: if (!isUnicode) {
766: pushLineSeparator();
767: }
768: }
769: }
770: isUnicode = false;
771: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
772: && (this .source[this .currentPosition] == 'u')) {
773: //-------------unicode traitement ------------
774: getNextUnicodeChar();
775: isUnicode = true;
776: } else {
777: isUnicode = false;
778: }
779: //handle the \\u case manually into comment
780:if (this .currentCharacter == '\\') {
781: if (this .source[this .currentPosition] == '\\')
782: this .currentPosition++;
783: } //jump over the \\
784: // empty comment is not a javadoc /**/
785: if (this .currentCharacter == '/') {
786: isJavadoc = false;
787: }
788: //loop until end of comment */
789: while ((this .currentCharacter != '/')
790: || (!star)) {
791: if ((this .currentCharacter == '\r')
792: || (this .currentCharacter == '\n')) {
793: //checkNonExternalizedString();
794: if (this .recordLineSeparator) {
795: if (!isUnicode) {
796: pushLineSeparator();
797: }
798: }
799: }
800: star = this .currentCharacter == '*';
801: //get next char
802: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
803: && (this .source[this .currentPosition] == 'u')) {
804: //-------------unicode traitement ------------
805: getNextUnicodeChar();
806: isUnicode = true;
807: } else {
808: isUnicode = false;
809: }
810: //handle the \\u case manually into comment
811:if (this .currentCharacter == '\\') {
812: if (this .source[this .currentPosition] == '\\')
813: this .currentPosition++;
814: } //jump over the \\
815: }
816: int token = isJavadoc ? TokenNameCOMMENT_JAVADOC
817: : TokenNameCOMMENT_BLOCK;
818: recordComment(token);
819: if (!isJavadoc
820: && this .startPosition <= this .cursorLocation
821: && this .cursorLocation < this .currentPosition - 1) {
822: throw new InvalidCursorLocation(
823: InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
824: }
825: if (this .taskTags != null)
826: checkTaskTag(this .startPosition,
827: this .currentPosition);
828: if (this .tokenizeComments) {
829: /*
830: if (isJavadoc)
831: return TokenNameCOMMENT_JAVADOC;
832: return TokenNameCOMMENT_BLOCK;
833: */
834: return token;
835: }
836: } catch (IndexOutOfBoundsException e) {
837: this .currentPosition--;
838: throw new InvalidInputException(
839: UNTERMINATED_COMMENT);
840: }
841: break;
842: }
843: if (getNextChar('='))
844: return TokenNameDIVIDE_EQUAL;
845: return TokenNameDIVIDE;
846: }
847: case '\u001a':
848: if (atEnd())
849: return TokenNameEOF;
850: //the atEnd may not be <this.currentPosition == this.source.length> if source is only some part of a real (external) stream
851: throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
852:
853: default:
854: if (ScannerHelper
855: .isJavaIdentifierStart(this .currentCharacter))
856: return scanIdentifierOrKeyword();
857: if (ScannerHelper.isDigit(this .currentCharacter)) {
858: return scanNumber(false);
859: }
860: return TokenNameERROR;
861: }
862: }
863: } //-----------------end switch while try--------------------
864: catch (IndexOutOfBoundsException e) {
865: if (this .tokenizeWhiteSpace
866: && (whiteStart != this .currentPosition - 1)) {
867: // reposition scanner in case we are interested by spaces as tokens
868: this .currentPosition--;
869: this .startPosition = whiteStart;
870: return TokenNameWHITESPACE;
871: }
872: }
873: /* might be completing at very end of file (e.g. behind a dot) */
874: if (this .completionIdentifier == null
875: && this .startPosition == this .cursorLocation + 1) {
876: this .currentPosition = this .startPosition; // for being detected as empty free identifier
877: return TokenNameIdentifier;
878: }
879: return TokenNameEOF;
880: }
881:
882: public final void getNextUnicodeChar() throws InvalidInputException {
883: int temp = this .currentPosition; // the \ is already read
884: super .getNextUnicodeChar();
885: if (this .cursorLocation > temp) {
886: this .unicodeCharSize += (this .currentPosition - temp);
887: }
888: if (temp < this .cursorLocation
889: && this .cursorLocation < this .currentPosition - 1) {
890: throw new InvalidCursorLocation(
891: InvalidCursorLocation.NO_COMPLETION_INSIDE_UNICODE);
892: }
893: }
894:
895: public final void jumpOverBlock() {
896: this .jumpOverMethodBody();
897: }
898:
899: ///*
900: // * In case we actually read a keyword, but the cursor is located inside,
901: // * we pretend we read an identifier.
902: // */
903: public int scanIdentifierOrKeyword() {
904:
905: int id = super .scanIdentifierOrKeyword();
906:
907: if (this .startPosition <= this .cursorLocation + 1
908: && this .cursorLocation < this .currentPosition) {
909:
910: // extends the end of the completion token even if the end is after eofPosition
911: if (this .cursorLocation + 1 == this .eofPosition) {
912: int temp = this .eofPosition;
913: this .eofPosition = this .source.length;
914: while (getNextCharAsJavaIdentifierPart()) {/*empty*/
915: }
916: this .eofPosition = temp;
917: }
918: // convert completed keyword into an identifier
919: return TokenNameIdentifier;
920: }
921: return id;
922: }
923:
924: public int scanNumber(boolean dotPrefix)
925: throws InvalidInputException {
926:
927: int token = super .scanNumber(dotPrefix);
928:
929: // consider completion just before a number to be ok, will insert before it
930: if (this .startPosition <= this .cursorLocation
931: && this .cursorLocation < this .currentPosition) {
932: throw new InvalidCursorLocation(
933: InvalidCursorLocation.NO_COMPLETION_INSIDE_NUMBER);
934: }
935: return token;
936: }
937: }
|