0001: /*******************************************************************************
0002: * Copyright (c) 2000, 2007 IBM Corporation and others.
0003: * All rights reserved. This program and the accompanying materials
0004: * are made available under the terms of the Eclipse Public License v1.0
0005: * which accompanies this distribution, and is available at
0006: * http://www.eclipse.org/legal/epl-v10.html
0007: *
0008: * Contributors:
0009: * IBM Corporation - initial API and implementation
0010: *******************************************************************************/package org.eclipse.jdt.internal.compiler.parser;
0011:
0012: import org.eclipse.jdt.core.compiler.CharOperation;
0013: import org.eclipse.jdt.core.compiler.InvalidInputException;
0014: import org.eclipse.jdt.internal.compiler.CompilationResult;
0015: import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
0016: import org.eclipse.jdt.internal.compiler.util.Util;
0017:
0018: /**
0019: * IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in
0020: * org.eclipse.jdt.core.compiler public package where it is API.
0021: * The mirror implementation is using the backward compatible ITerminalSymbols constant
0022: * definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens
0023: * which constant values reflect the latest parser generation state.
0024: */
0025: public class Scanner implements TerminalTokens {
0026:
0027: //public int newIdentCount = 0;
0028:
0029: /* APIs ares
0030: - getNextToken() which return the current type of the token
0031: (this value is not memorized by the scanner)
0032: - getCurrentTokenSource() which provides with the token "REAL" source
0033: (aka all unicode have been transformed into a correct char)
0034: - sourceStart gives the position into the stream
0035: - currentPosition-1 gives the sourceEnd position into the stream
0036: */
0037: public long sourceLevel;
0038: public long complianceLevel;
0039:
0040: // 1.4 feature
0041: public boolean useAssertAsAnIndentifier = false;
0042: //flag indicating if processed source contains occurrences of keyword assert
0043: public boolean containsAssertKeyword = false;
0044:
0045: // 1.5 feature
0046: public boolean useEnumAsAnIndentifier = false;
0047:
0048: public boolean recordLineSeparator = false;
0049: public char currentCharacter;
0050: public int startPosition;
0051: public int currentPosition;
0052: public int initialPosition, eofPosition;
0053: // after this position eof are generated instead of real token from the source
0054:
0055: public boolean tokenizeComments = false;
0056: public boolean tokenizeWhiteSpace = false;
0057:
0058: //source should be viewed as a window (aka a part)
0059: //of a entire very large stream
0060: public char source[];
0061:
0062: //unicode support
0063: public char[] withoutUnicodeBuffer;
0064: public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
0065: public boolean unicodeAsBackSlash = false;
0066:
0067: public boolean scanningFloatLiteral = false;
0068:
0069: //support for /** comments
0070: public final static int COMMENT_ARRAYS_SIZE = 30;
0071: public int[] commentStops = new int[COMMENT_ARRAYS_SIZE];
0072: public int[] commentStarts = new int[COMMENT_ARRAYS_SIZE];
0073: public int[] commentTagStarts = new int[COMMENT_ARRAYS_SIZE];
0074: public int commentPtr = -1; // no comment test with commentPtr value -1
0075: protected int lastCommentLinePosition = -1;
0076:
0077: // task tag support
0078: public char[][] foundTaskTags = null;
0079: public char[][] foundTaskMessages;
0080: public char[][] foundTaskPriorities = null;
0081: public int[][] foundTaskPositions;
0082: public int foundTaskCount = 0;
0083: public char[][] taskTags = null;
0084: public char[][] taskPriorities = null;
0085: public boolean isTaskCaseSensitive = true;
0086:
0087: //diet parsing support - jump over some method body when requested
0088: public boolean diet = false;
0089:
0090: //support for the poor-line-debuggers ....
0091: //remember the position of the cr/lf
0092: public int[] lineEnds = new int[250];
0093: public int linePtr = -1;
0094: public boolean wasAcr = false;
0095:
0096: public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
0097:
0098: public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
0099: public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
0100: public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
0101: public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
0102: public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
0103: public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
0104: public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
0105: public static final String INVALID_LOW_SURROGATE = "Invalid_Low_Surrogate"; //$NON-NLS-1$
0106: public static final String INVALID_HIGH_SURROGATE = "Invalid_High_Surrogate"; //$NON-NLS-1$
0107:
0108: public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
0109: public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
0110: public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
0111: public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
0112: public static final String INVALID_DIGIT = "Invalid_Digit"; //$NON-NLS-1$
0113: private static final int[] EMPTY_LINE_ENDS = Util.EMPTY_INT_ARRAY;
0114:
0115: //----------------optimized identifier managment------------------
0116: static final char[] charArray_a = new char[] { 'a' },
0117: charArray_b = new char[] { 'b' },
0118: charArray_c = new char[] { 'c' },
0119: charArray_d = new char[] { 'd' },
0120: charArray_e = new char[] { 'e' },
0121: charArray_f = new char[] { 'f' },
0122: charArray_g = new char[] { 'g' },
0123: charArray_h = new char[] { 'h' },
0124: charArray_i = new char[] { 'i' },
0125: charArray_j = new char[] { 'j' },
0126: charArray_k = new char[] { 'k' },
0127: charArray_l = new char[] { 'l' },
0128: charArray_m = new char[] { 'm' },
0129: charArray_n = new char[] { 'n' },
0130: charArray_o = new char[] { 'o' },
0131: charArray_p = new char[] { 'p' },
0132: charArray_q = new char[] { 'q' },
0133: charArray_r = new char[] { 'r' },
0134: charArray_s = new char[] { 's' },
0135: charArray_t = new char[] { 't' },
0136: charArray_u = new char[] { 'u' },
0137: charArray_v = new char[] { 'v' },
0138: charArray_w = new char[] { 'w' },
0139: charArray_x = new char[] { 'x' },
0140: charArray_y = new char[] { 'y' },
0141: charArray_z = new char[] { 'z' };
0142:
0143: static final char[] initCharArray = new char[] { '\u0000',
0144: '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
0145: static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries
0146:
0147: public static final int OptimizedLength = 7;
0148: public/*static*/final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
0149: // support for detecting non-externalized string literals
0150: public static final char[] TAG_PREFIX = "//$NON-NLS-".toCharArray(); //$NON-NLS-1$
0151: public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length;
0152: public static final char TAG_POSTFIX = '$';
0153: public static final int TAG_POSTFIX_LENGTH = 1;
0154: private NLSTag[] nlsTags = null;
0155: protected int nlsTagsPtr;
0156: public boolean checkNonExternalizedStringLiterals;
0157:
0158: // generic support
0159: public boolean returnOnlyGreater = false;
0160:
0161: /*static*/{
0162: for (int i = 0; i < 6; i++) {
0163: for (int j = 0; j < TableSize; j++) {
0164: for (int k = 0; k < InternalTableSize; k++) {
0165: this .charArray_length[i][j][k] = initCharArray;
0166: }
0167: }
0168: }
0169: }
0170: /*static*/int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0,
0171: newEntry5 = 0, newEntry6 = 0;
0172: public boolean insideRecovery = false;
0173:
0174: public static final int RoundBracket = 0;
0175: public static final int SquareBracket = 1;
0176: public static final int CurlyBracket = 2;
0177: public static final int BracketKinds = 3;
0178:
0179: // extended unicode support
0180: public static final int LOW_SURROGATE_MIN_VALUE = 0xDC00;
0181: public static final int HIGH_SURROGATE_MIN_VALUE = 0xD800;
0182: public static final int HIGH_SURROGATE_MAX_VALUE = 0xDBFF;
0183: public static final int LOW_SURROGATE_MAX_VALUE = 0xDFFF;
0184:
0185: public Scanner() {
0186: this (false /*comment*/, false /*whitespace*/, false /*nls*/,
0187: ClassFileConstants.JDK1_3 /*sourceLevel*/,
0188: null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/);
0189: }
0190:
0191: public Scanner(boolean tokenizeComments,
0192: boolean tokenizeWhiteSpace,
0193: boolean checkNonExternalizedStringLiterals,
0194: long sourceLevel, long complianceLevel, char[][] taskTags,
0195: char[][] taskPriorities, boolean isTaskCaseSensitive) {
0196:
0197: this .eofPosition = Integer.MAX_VALUE;
0198: this .tokenizeComments = tokenizeComments;
0199: this .tokenizeWhiteSpace = tokenizeWhiteSpace;
0200: this .sourceLevel = sourceLevel;
0201: this .complianceLevel = complianceLevel;
0202: this .checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
0203: if (taskTags != null) {
0204: int length = taskTags.length;
0205: if (taskPriorities != null) {
0206: int[] initialIndexes = new int[length];
0207: for (int i = 0; i < length; i++) {
0208: initialIndexes[i] = i;
0209: }
0210: Util.reverseQuickSort(taskTags, 0, taskTags.length - 1,
0211: initialIndexes);
0212: char[][] temp = new char[length][];
0213: for (int i = 0; i < length; i++) {
0214: temp[i] = taskPriorities[initialIndexes[i]];
0215: }
0216: this .taskPriorities = temp;
0217: } else {
0218: Util.reverseQuickSort(taskTags, 0, taskTags.length - 1);
0219: }
0220: this .taskTags = taskTags;
0221: this .isTaskCaseSensitive = isTaskCaseSensitive;
0222: }
0223: }
0224:
0225: public Scanner(boolean tokenizeComments,
0226: boolean tokenizeWhiteSpace,
0227: boolean checkNonExternalizedStringLiterals,
0228: long sourceLevel, char[][] taskTags,
0229: char[][] taskPriorities, boolean isTaskCaseSensitive) {
0230:
0231: this (tokenizeComments, tokenizeWhiteSpace,
0232: checkNonExternalizedStringLiterals, sourceLevel,
0233: sourceLevel, taskTags, taskPriorities,
0234: isTaskCaseSensitive);
0235: }
0236:
0237: public final boolean atEnd() {
0238: // This code is not relevant if source is
0239: // Only a part of the real stream input
0240:
0241: return this .eofPosition <= this .currentPosition;
0242: }
0243:
0244: // chech presence of task: tags
0245: // TODO (frederic) see if we need to take unicode characters into account...
0246: public void checkTaskTag(int commentStart, int commentEnd)
0247: throws InvalidInputException {
0248: char[] src = this .source;
0249:
0250: // only look for newer task: tags
0251: if (this .foundTaskCount > 0
0252: && this .foundTaskPositions[this .foundTaskCount - 1][0] >= commentStart) {
0253: return;
0254: }
0255: int foundTaskIndex = this .foundTaskCount;
0256: char previous = src[commentStart + 1]; // should be '*' or '/'
0257: for (int i = commentStart + 2; i < commentEnd
0258: && i < this .eofPosition; i++) {
0259: char[] tag = null;
0260: char[] priority = null;
0261: // check for tag occurrence only if not ambiguous with javadoc tag
0262: if (previous != '@') {
0263: nextTag: for (int itag = 0; itag < this .taskTags.length; itag++) {
0264: tag = this .taskTags[itag];
0265: int tagLength = tag.length;
0266: if (tagLength == 0)
0267: continue nextTag;
0268:
0269: // ensure tag is not leaded with letter if tag starts with a letter
0270: if (ScannerHelper.isJavaIdentifierStart(tag[0])) {
0271: if (ScannerHelper
0272: .isJavaIdentifierPart(previous)) {
0273: continue nextTag;
0274: }
0275: }
0276:
0277: for (int t = 0; t < tagLength; t++) {
0278: char sc, tc;
0279: int x = i + t;
0280: if (x >= this .eofPosition || x >= commentEnd)
0281: continue nextTag;
0282: // case sensitive check
0283: if ((sc = src[i + t]) != (tc = tag[t])) {
0284: // case insensitive check
0285: if (this .isTaskCaseSensitive
0286: || (ScannerHelper.toLowerCase(sc) != ScannerHelper
0287: .toLowerCase(tc))) {
0288: continue nextTag;
0289: }
0290: }
0291: }
0292: // ensure tag is not followed with letter if tag finishes with a letter
0293: if (i + tagLength < commentEnd
0294: && ScannerHelper.isJavaIdentifierPart(src[i
0295: + tagLength - 1])) {
0296: if (ScannerHelper.isJavaIdentifierPart(src[i
0297: + tagLength]))
0298: continue nextTag;
0299: }
0300: if (this .foundTaskTags == null) {
0301: this .foundTaskTags = new char[5][];
0302: this .foundTaskMessages = new char[5][];
0303: this .foundTaskPriorities = new char[5][];
0304: this .foundTaskPositions = new int[5][];
0305: } else if (this .foundTaskCount == this .foundTaskTags.length) {
0306: System
0307: .arraycopy(
0308: this .foundTaskTags,
0309: 0,
0310: this .foundTaskTags = new char[this .foundTaskCount * 2][],
0311: 0, this .foundTaskCount);
0312: System
0313: .arraycopy(
0314: this .foundTaskMessages,
0315: 0,
0316: this .foundTaskMessages = new char[this .foundTaskCount * 2][],
0317: 0, this .foundTaskCount);
0318: System
0319: .arraycopy(
0320: this .foundTaskPriorities,
0321: 0,
0322: this .foundTaskPriorities = new char[this .foundTaskCount * 2][],
0323: 0, this .foundTaskCount);
0324: System
0325: .arraycopy(
0326: this .foundTaskPositions,
0327: 0,
0328: this .foundTaskPositions = new int[this .foundTaskCount * 2][],
0329: 0, this .foundTaskCount);
0330: }
0331:
0332: priority = this .taskPriorities != null
0333: && itag < this .taskPriorities.length ? this .taskPriorities[itag]
0334: : null;
0335:
0336: this .foundTaskTags[this .foundTaskCount] = tag;
0337: this .foundTaskPriorities[this .foundTaskCount] = priority;
0338: this .foundTaskPositions[this .foundTaskCount] = new int[] {
0339: i, i + tagLength - 1 };
0340: this .foundTaskMessages[this .foundTaskCount] = CharOperation.NO_CHAR;
0341: this .foundTaskCount++;
0342: i += tagLength - 1; // will be incremented when looping
0343: break nextTag;
0344: }
0345: }
0346: previous = src[i];
0347: }
0348: boolean containsEmptyTask = false;
0349: for (int i = foundTaskIndex; i < this .foundTaskCount; i++) {
0350: // retrieve message start and end positions
0351: int msgStart = this .foundTaskPositions[i][0]
0352: + this .foundTaskTags[i].length;
0353: int max_value = i + 1 < this .foundTaskCount ? this .foundTaskPositions[i + 1][0] - 1
0354: : commentEnd - 1;
0355: // at most beginning of next task
0356: if (max_value < msgStart) {
0357: max_value = msgStart; // would only occur if tag is before EOF.
0358: }
0359: int end = -1;
0360: char c;
0361: for (int j = msgStart; j < max_value; j++) {
0362: if ((c = src[j]) == '\n' || c == '\r') {
0363: end = j - 1;
0364: break;
0365: }
0366: }
0367: if (end == -1) {
0368: for (int j = max_value; j > msgStart; j--) {
0369: if ((c = src[j]) == '*') {
0370: end = j - 1;
0371: break;
0372: }
0373: }
0374: if (end == -1)
0375: end = max_value;
0376: }
0377: if (msgStart == end) {
0378: // if the description is empty, we might want to see if two tags are not sharing the same message
0379: // see https://bugs.eclipse.org/bugs/show_bug.cgi?id=110797
0380: containsEmptyTask = true;
0381: continue;
0382: }
0383: // trim the message
0384: while (CharOperation.isWhitespace(src[end])
0385: && msgStart <= end)
0386: end--;
0387: while (CharOperation.isWhitespace(src[msgStart])
0388: && msgStart <= end)
0389: msgStart++;
0390: // update the end position of the task
0391: this .foundTaskPositions[i][1] = end;
0392: // get the message source
0393: final int messageLength = end - msgStart + 1;
0394: char[] message = new char[messageLength];
0395: System.arraycopy(src, msgStart, message, 0, messageLength);
0396: this .foundTaskMessages[i] = message;
0397: }
0398: if (containsEmptyTask) {
0399: for (int i = foundTaskIndex, max = this .foundTaskCount; i < max; i++) {
0400: if (this .foundTaskMessages[i].length == 0) {
0401: loop: for (int j = i + 1; j < max; j++) {
0402: if (this .foundTaskMessages[j].length != 0) {
0403: this .foundTaskMessages[i] = this .foundTaskMessages[j];
0404: this .foundTaskPositions[i][1] = this .foundTaskPositions[j][1];
0405: break loop;
0406: }
0407: }
0408: }
0409: }
0410: }
0411: }
0412:
0413: public char[] getCurrentIdentifierSource() {
0414: //return the token REAL source (aka unicodes are precomputed)
0415:
0416: char[] result;
0417: if (this .withoutUnicodePtr != 0) {
0418: //0 is used as a fast test flag so the real first char is in position 1
0419: System.arraycopy(this .withoutUnicodeBuffer, 1,
0420: result = new char[this .withoutUnicodePtr], 0,
0421: this .withoutUnicodePtr);
0422: } else {
0423: int length = this .currentPosition - this .startPosition;
0424: if (length == this .eofPosition)
0425: return this .source;
0426: switch (length) { // see OptimizedLength
0427: case 1:
0428: return optimizedCurrentTokenSource1();
0429: case 2:
0430: return optimizedCurrentTokenSource2();
0431: case 3:
0432: return optimizedCurrentTokenSource3();
0433: case 4:
0434: return optimizedCurrentTokenSource4();
0435: case 5:
0436: return optimizedCurrentTokenSource5();
0437: case 6:
0438: return optimizedCurrentTokenSource6();
0439: }
0440: //no optimization
0441: System.arraycopy(this .source, this .startPosition,
0442: result = new char[length], 0, length);
0443: }
0444: //newIdentCount++;
0445: return result;
0446: }
0447:
0448: public int getCurrentTokenEndPosition() {
0449: return this .currentPosition - 1;
0450: }
0451:
0452: public char[] getCurrentTokenSource() {
0453: // Return the token REAL source (aka unicodes are precomputed)
0454:
0455: char[] result;
0456: if (this .withoutUnicodePtr != 0)
0457: // 0 is used as a fast test flag so the real first char is in position 1
0458: System.arraycopy(this .withoutUnicodeBuffer, 1,
0459: result = new char[this .withoutUnicodePtr], 0,
0460: this .withoutUnicodePtr);
0461: else {
0462: int length;
0463: System.arraycopy(this .source, this .startPosition,
0464: result = new char[length = this .currentPosition
0465: - this .startPosition], 0, length);
0466: }
0467: return result;
0468: }
0469:
0470: public final String getCurrentTokenString() {
0471: // Return current token as a string
0472:
0473: if (this .withoutUnicodePtr != 0) {
0474: // 0 is used as a fast test flag so the real first char is in position 1
0475: return new String(this .withoutUnicodeBuffer, 1,
0476: this .withoutUnicodePtr);
0477: }
0478: return new String(this .source, this .startPosition,
0479: this .currentPosition - this .startPosition);
0480: }
0481:
0482: public char[] getCurrentTokenSourceString() {
0483: //return the token REAL source (aka unicodes are precomputed).
0484: //REMOVE the two " that are at the beginning and the end.
0485:
0486: char[] result;
0487: if (this .withoutUnicodePtr != 0)
0488: //0 is used as a fast test flag so the real first char is in position 1
0489: System.arraycopy(this .withoutUnicodeBuffer, 2,
0490: //2 is 1 (real start) + 1 (to jump over the ")
0491: result = new char[this .withoutUnicodePtr - 2], 0,
0492: this .withoutUnicodePtr - 2);
0493: else {
0494: int length;
0495: System.arraycopy(this .source, this .startPosition + 1,
0496: result = new char[length = this .currentPosition
0497: - this .startPosition - 2], 0, length);
0498: }
0499: return result;
0500: }
0501:
0502: public final String getCurrentStringLiteral() {
0503: //return the token REAL source (aka unicodes are precomputed).
0504: //REMOVE the two " that are at the beginning and the end.
0505:
0506: if (this .withoutUnicodePtr != 0)
0507: //0 is used as a fast test flag so the real first char is in position 1
0508: //2 is 1 (real start) + 1 (to jump over the ")
0509: return new String(this .withoutUnicodeBuffer, 2,
0510: this .withoutUnicodePtr - 2);
0511: else {
0512: return new String(this .source, this .startPosition + 1,
0513: this .currentPosition - this .startPosition - 2);
0514: }
0515: }
0516:
0517: public final char[] getRawTokenSource() {
0518: int length = this .currentPosition - this .startPosition;
0519: char[] tokenSource = new char[length];
0520: System.arraycopy(this .source, this .startPosition, tokenSource,
0521: 0, length);
0522: return tokenSource;
0523: }
0524:
0525: public final char[] getRawTokenSourceEnd() {
0526: int length = this .eofPosition - this .currentPosition - 1;
0527: char[] sourceEnd = new char[length];
0528: System.arraycopy(this .source, this .currentPosition, sourceEnd,
0529: 0, length);
0530: return sourceEnd;
0531: }
0532:
0533: public int getCurrentTokenStartPosition() {
0534: return this .startPosition;
0535: }
0536:
0537: /*
0538: * Search the source position corresponding to the end of a given line number
0539: *
0540: * Line numbers are 1-based, and relative to the scanner initialPosition.
0541: * Character positions are 0-based.
0542: *
0543: * In case the given line number is inconsistent, answers -1.
0544: */
0545: public final int getLineEnd(int lineNumber) {
0546:
0547: if (this .lineEnds == null || this .linePtr == -1)
0548: return -1;
0549: if (lineNumber > this .lineEnds.length + 1)
0550: return -1;
0551: if (lineNumber <= 0)
0552: return -1;
0553: if (lineNumber == this .lineEnds.length + 1)
0554: return this .eofPosition;
0555: return this .lineEnds[lineNumber - 1]; // next line start one character behind the lineEnd of the previous line
0556: }
0557:
0558: public final int[] getLineEnds() {
0559: //return a bounded copy of this.lineEnds
0560: if (this .linePtr == -1) {
0561: return EMPTY_LINE_ENDS;
0562: }
0563: int[] copy;
0564: System.arraycopy(this .lineEnds, 0,
0565: copy = new int[this .linePtr + 1], 0, this .linePtr + 1);
0566: return copy;
0567: }
0568:
0569: /**
0570: * Search the source position corresponding to the beginning of a given line number
0571: *
0572: * Line numbers are 1-based, and relative to the scanner initialPosition.
0573: * Character positions are 0-based.
0574: *
0575: * e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0.
0576: *
0577: * In case the given line number is inconsistent, answers -1.
0578: *
0579: * @param lineNumber int
0580: * @return int
0581: */
0582: public final int getLineStart(int lineNumber) {
0583:
0584: if (this .lineEnds == null || this .linePtr == -1)
0585: return -1;
0586: if (lineNumber > this .lineEnds.length + 1)
0587: return -1;
0588: if (lineNumber <= 0)
0589: return -1;
0590:
0591: if (lineNumber == 1)
0592: return this .initialPosition;
0593: return this .lineEnds[lineNumber - 2] + 1; // next line start one character behind the lineEnd of the previous line
0594: }
0595:
0596: public final int getNextChar() {
0597: try {
0598: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0599: && (this .source[this .currentPosition] == 'u')) {
0600: getNextUnicodeChar();
0601: } else {
0602: this .unicodeAsBackSlash = false;
0603: if (this .withoutUnicodePtr != 0) {
0604: unicodeStore();
0605: }
0606: }
0607: return this .currentCharacter;
0608: } catch (IndexOutOfBoundsException e) {
0609: return -1;
0610: } catch (InvalidInputException e) {
0611: return -1;
0612: }
0613: }
0614:
0615: public final int getNextCharWithBoundChecks() {
0616: if (this .currentPosition >= this .eofPosition) {
0617: return -1;
0618: }
0619: this .currentCharacter = this .source[this .currentPosition++];
0620: if (this .currentPosition >= this .eofPosition) {
0621: this .unicodeAsBackSlash = false;
0622: if (this .withoutUnicodePtr != 0) {
0623: unicodeStore();
0624: }
0625: return this .currentCharacter;
0626: }
0627: if (this .currentCharacter == '\\'
0628: && this .source[this .currentPosition] == 'u') {
0629: try {
0630: getNextUnicodeChar();
0631: } catch (InvalidInputException e) {
0632: return -1;
0633: }
0634: } else {
0635: this .unicodeAsBackSlash = false;
0636: if (this .withoutUnicodePtr != 0) {
0637: unicodeStore();
0638: }
0639: }
0640: return this .currentCharacter;
0641: }
0642:
0643: public final boolean getNextChar(char testedChar) {
0644: //BOOLEAN
0645: //handle the case of unicode.
0646: //when a unicode appears then we must use a buffer that holds char internal values
0647: //At the end of this method currentCharacter holds the new visited char
0648: //and currentPosition points right next after it
0649: //Both previous lines are true if the currentCharacter is == to the testedChar
0650: //On false, no side effect has occured.
0651:
0652: //ALL getNextChar.... ARE OPTIMIZED COPIES
0653:
0654: if (this .currentPosition >= this .eofPosition) { // handle the obvious case upfront
0655: this .unicodeAsBackSlash = false;
0656: return false;
0657: }
0658:
0659: int temp = this .currentPosition;
0660: try {
0661: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0662: && (this .source[this .currentPosition] == 'u')) {
0663: getNextUnicodeChar();
0664: if (this .currentCharacter != testedChar) {
0665: this .currentPosition = temp;
0666: this .withoutUnicodePtr--;
0667: return false;
0668: }
0669: return true;
0670: } //-------------end unicode traitement--------------
0671: else {
0672: if (this .currentCharacter != testedChar) {
0673: this .currentPosition = temp;
0674: return false;
0675: }
0676: this .unicodeAsBackSlash = false;
0677: if (this .withoutUnicodePtr != 0)
0678: unicodeStore();
0679: return true;
0680: }
0681: } catch (IndexOutOfBoundsException e) {
0682: this .unicodeAsBackSlash = false;
0683: this .currentPosition = temp;
0684: return false;
0685: } catch (InvalidInputException e) {
0686: this .unicodeAsBackSlash = false;
0687: this .currentPosition = temp;
0688: return false;
0689: }
0690: }
0691:
0692: public final int getNextChar(char testedChar1, char testedChar2) {
0693: //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
0694: //test can be done with (x==0) for the first and (x>0) for the second
0695: //handle the case of unicode.
0696: //when a unicode appears then we must use a buffer that holds char internal values
0697: //At the end of this method currentCharacter holds the new visited char
0698: //and currentPosition points right next after it
0699: //Both previous lines are true if the currentCharacter is == to the testedChar1/2
0700: //On false, no side effect has occured.
0701:
0702: //ALL getNextChar.... ARE OPTIMIZED COPIES
0703: if (this .currentPosition >= this .eofPosition) // handle the obvious case upfront
0704: return -1;
0705:
0706: int temp = this .currentPosition;
0707: try {
0708: int result;
0709: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0710: && (this .source[this .currentPosition] == 'u')) {
0711: getNextUnicodeChar();
0712: if (this .currentCharacter == testedChar1) {
0713: result = 0;
0714: } else if (this .currentCharacter == testedChar2) {
0715: result = 1;
0716: } else {
0717: this .currentPosition = temp;
0718: this .withoutUnicodePtr--;
0719: result = -1;
0720: }
0721: return result;
0722: } else {
0723: if (this .currentCharacter == testedChar1) {
0724: result = 0;
0725: } else if (this .currentCharacter == testedChar2) {
0726: result = 1;
0727: } else {
0728: this .currentPosition = temp;
0729: return -1;
0730: }
0731:
0732: if (this .withoutUnicodePtr != 0)
0733: unicodeStore();
0734: return result;
0735: }
0736: } catch (IndexOutOfBoundsException e) {
0737: this .currentPosition = temp;
0738: return -1;
0739: } catch (InvalidInputException e) {
0740: this .currentPosition = temp;
0741: return -1;
0742: }
0743: }
0744:
0745: public final boolean getNextCharAsDigit()
0746: throws InvalidInputException {
0747: //BOOLEAN
0748: //handle the case of unicode.
0749: //when a unicode appears then we must use a buffer that holds char internal values
0750: //At the end of this method currentCharacter holds the new visited char
0751: //and currentPosition points right next after it
0752: //Both previous lines are true if the currentCharacter is a digit
0753: //On false, no side effect has occured.
0754:
0755: //ALL getNextChar.... ARE OPTIMIZED COPIES
0756: if (this .currentPosition >= this .eofPosition) // handle the obvious case upfront
0757: return false;
0758:
0759: int temp = this .currentPosition;
0760: try {
0761: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0762: && (this .source[this .currentPosition] == 'u')) {
0763: getNextUnicodeChar();
0764: if (!ScannerHelper.isDigit(this .currentCharacter)) {
0765: this .currentPosition = temp;
0766: this .withoutUnicodePtr--;
0767: return false;
0768: }
0769: return true;
0770: } else {
0771: if (!ScannerHelper.isDigit(this .currentCharacter)) {
0772: this .currentPosition = temp;
0773: return false;
0774: }
0775: if (this .withoutUnicodePtr != 0)
0776: unicodeStore();
0777: return true;
0778: }
0779: } catch (IndexOutOfBoundsException e) {
0780: this .currentPosition = temp;
0781: return false;
0782: } catch (InvalidInputException e) {
0783: this .currentPosition = temp;
0784: return false;
0785: }
0786: }
0787:
0788: public final boolean getNextCharAsDigit(int radix) {
0789: //BOOLEAN
0790: //handle the case of unicode.
0791: //when a unicode appears then we must use a buffer that holds char internal values
0792: //At the end of this method currentCharacter holds the new visited char
0793: //and currentPosition points right next after it
0794: //Both previous lines are true if the currentCharacter is a digit base on radix
0795: //On false, no side effect has occured.
0796:
0797: //ALL getNextChar.... ARE OPTIMIZED COPIES
0798: if (this .currentPosition >= this .eofPosition) // handle the obvious case upfront
0799: return false;
0800:
0801: int temp = this .currentPosition;
0802: try {
0803: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0804: && (this .source[this .currentPosition] == 'u')) {
0805: getNextUnicodeChar();
0806: if (ScannerHelper.digit(this .currentCharacter, radix) == -1) {
0807: this .currentPosition = temp;
0808: this .withoutUnicodePtr--;
0809: return false;
0810: }
0811: return true;
0812: } else {
0813: if (ScannerHelper.digit(this .currentCharacter, radix) == -1) {
0814: this .currentPosition = temp;
0815: return false;
0816: }
0817: if (this .withoutUnicodePtr != 0)
0818: unicodeStore();
0819: return true;
0820: }
0821: } catch (IndexOutOfBoundsException e) {
0822: this .currentPosition = temp;
0823: return false;
0824: } catch (InvalidInputException e) {
0825: this .currentPosition = temp;
0826: return false;
0827: }
0828: }
0829:
0830: public boolean getNextCharAsJavaIdentifierPartWithBoundCheck() {
0831: //BOOLEAN
0832: //handle the case of unicode.
0833: //when a unicode appears then we must use a buffer that holds char internal values
0834: //At the end of this method currentCharacter holds the new visited char
0835: //and currentPosition points right next after it
0836: //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
0837: //On false, no side effect has occured.
0838:
0839: //ALL getNextChar.... ARE OPTIMIZED COPIES
0840: int pos = this .currentPosition;
0841: if (pos >= this .eofPosition) // handle the obvious case upfront
0842: return false;
0843:
0844: int temp2 = this .withoutUnicodePtr;
0845: try {
0846: boolean unicode = false;
0847: this .currentCharacter = this .source[this .currentPosition++];
0848: if (this .currentPosition < this .eofPosition) {
0849: if (this .currentCharacter == '\\'
0850: && this .source[this .currentPosition] == 'u') {
0851: getNextUnicodeChar();
0852: unicode = true;
0853: }
0854: }
0855: char c = this .currentCharacter;
0856: boolean isJavaIdentifierPart = false;
0857: if (c >= HIGH_SURROGATE_MIN_VALUE
0858: && c <= HIGH_SURROGATE_MAX_VALUE) {
0859: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
0860: this .currentPosition = pos;
0861: this .withoutUnicodePtr = temp2;
0862: return false;
0863: }
0864: // Unicode 4 detection
0865: char low = (char) getNextCharWithBoundChecks();
0866: if (low < LOW_SURROGATE_MIN_VALUE
0867: || low > LOW_SURROGATE_MAX_VALUE) {
0868: // illegal low surrogate
0869: this .currentPosition = pos;
0870: this .withoutUnicodePtr = temp2;
0871: return false;
0872: }
0873: isJavaIdentifierPart = ScannerHelper
0874: .isJavaIdentifierPart(c, low);
0875: } else if (c >= LOW_SURROGATE_MIN_VALUE
0876: && c <= LOW_SURROGATE_MAX_VALUE) {
0877: this .currentPosition = pos;
0878: this .withoutUnicodePtr = temp2;
0879: return false;
0880: } else {
0881: isJavaIdentifierPart = ScannerHelper
0882: .isJavaIdentifierPart(c);
0883: }
0884: if (unicode) {
0885: if (!isJavaIdentifierPart) {
0886: this .currentPosition = pos;
0887: this .withoutUnicodePtr = temp2;
0888: return false;
0889: }
0890: return true;
0891: } else {
0892: if (!isJavaIdentifierPart) {
0893: this .currentPosition = pos;
0894: return false;
0895: }
0896:
0897: if (this .withoutUnicodePtr != 0)
0898: unicodeStore();
0899: return true;
0900: }
0901: } catch (InvalidInputException e) {
0902: this .currentPosition = pos;
0903: this .withoutUnicodePtr = temp2;
0904: return false;
0905: }
0906: }
0907:
0908: public boolean getNextCharAsJavaIdentifierPart() {
0909: //BOOLEAN
0910: //handle the case of unicode.
0911: //when a unicode appears then we must use a buffer that holds char internal values
0912: //At the end of this method currentCharacter holds the new visited char
0913: //and currentPosition points right next after it
0914: //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
0915: //On false, no side effect has occured.
0916:
0917: //ALL getNextChar.... ARE OPTIMIZED COPIES
0918: int pos;
0919: if ((pos = this .currentPosition) >= this .eofPosition) // handle the obvious case upfront
0920: return false;
0921:
0922: int temp2 = this .withoutUnicodePtr;
0923: try {
0924: boolean unicode = false;
0925: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
0926: && (this .source[this .currentPosition] == 'u')) {
0927: getNextUnicodeChar();
0928: unicode = true;
0929: }
0930: char c = this .currentCharacter;
0931: boolean isJavaIdentifierPart = false;
0932: if (c >= HIGH_SURROGATE_MIN_VALUE
0933: && c <= HIGH_SURROGATE_MAX_VALUE) {
0934: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
0935: this .currentPosition = pos;
0936: this .withoutUnicodePtr = temp2;
0937: return false;
0938: }
0939: // Unicode 4 detection
0940: char low = (char) getNextChar();
0941: if (low < LOW_SURROGATE_MIN_VALUE
0942: || low > LOW_SURROGATE_MAX_VALUE) {
0943: // illegal low surrogate
0944: this .currentPosition = pos;
0945: this .withoutUnicodePtr = temp2;
0946: return false;
0947: }
0948: isJavaIdentifierPart = ScannerHelper
0949: .isJavaIdentifierPart(c, low);
0950: } else if (c >= LOW_SURROGATE_MIN_VALUE
0951: && c <= LOW_SURROGATE_MAX_VALUE) {
0952: this .currentPosition = pos;
0953: this .withoutUnicodePtr = temp2;
0954: return false;
0955: } else {
0956: isJavaIdentifierPart = ScannerHelper
0957: .isJavaIdentifierPart(c);
0958: }
0959: if (unicode) {
0960: if (!isJavaIdentifierPart) {
0961: this .currentPosition = pos;
0962: this .withoutUnicodePtr = temp2;
0963: return false;
0964: }
0965: return true;
0966: } else {
0967: if (!isJavaIdentifierPart) {
0968: this .currentPosition = pos;
0969: return false;
0970: }
0971:
0972: if (this .withoutUnicodePtr != 0)
0973: unicodeStore();
0974: return true;
0975: }
0976: } catch (IndexOutOfBoundsException e) {
0977: this .currentPosition = pos;
0978: this .withoutUnicodePtr = temp2;
0979: return false;
0980: } catch (InvalidInputException e) {
0981: this .currentPosition = pos;
0982: this .withoutUnicodePtr = temp2;
0983: return false;
0984: }
0985: }
0986:
0987: /*
0988: * External API in JavaConventions.
0989: * This is used to optimize the case where the scanner is used to scan a single identifier.
0990: * In this case, the AIOOBE is slower to handle than a bound check
0991: */
0992: public int scanIdentifier() throws InvalidInputException {
0993: int whiteStart = 0;
0994: while (true) { //loop for jumping over comments
0995: this .withoutUnicodePtr = 0;
0996: //start with a new token (even comment written with unicode )
0997: // ---------Consume white space and handles startPosition---------
0998: whiteStart = this .currentPosition;
0999: boolean isWhiteSpace, hasWhiteSpaces = false;
1000: int offset;
1001: int unicodePtr;
1002: boolean checkIfUnicode = false;
1003: do {
1004: unicodePtr = this .withoutUnicodePtr;
1005: offset = this .currentPosition;
1006: this .startPosition = this .currentPosition;
1007: if (this .currentPosition < this .eofPosition) {
1008: this .currentCharacter = this .source[this .currentPosition++];
1009: checkIfUnicode = this .currentPosition < this .eofPosition
1010: && this .currentCharacter == '\\'
1011: && this .source[this .currentPosition] == 'u';
1012: } else if (this .tokenizeWhiteSpace
1013: && (whiteStart != this .currentPosition - 1)) {
1014: // reposition scanner in case we are interested by spaces as tokens
1015: this .currentPosition--;
1016: this .startPosition = whiteStart;
1017: return TokenNameWHITESPACE;
1018: } else {
1019: return TokenNameEOF;
1020: }
1021: if (checkIfUnicode) {
1022: isWhiteSpace = jumpOverUnicodeWhiteSpace();
1023: offset = this .currentPosition - offset;
1024: } else {
1025: offset = this .currentPosition - offset;
1026: // inline version of:
1027: //isWhiteSpace =
1028: // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter);
1029: switch (this .currentCharacter) {
1030: case 10: /* \ u000a: LINE FEED */
1031: case 12: /* \ u000c: FORM FEED */
1032: case 13: /* \ u000d: CARRIAGE RETURN */
1033: case 32: /* \ u0020: SPACE */
1034: case 9: /* \ u0009: HORIZONTAL TABULATION */
1035: isWhiteSpace = true;
1036: break;
1037: default:
1038: isWhiteSpace = false;
1039: }
1040: }
1041: if (isWhiteSpace) {
1042: hasWhiteSpaces = true;
1043: }
1044: } while (isWhiteSpace);
1045: if (hasWhiteSpaces) {
1046: if (this .tokenizeWhiteSpace) {
1047: // reposition scanner in case we are interested by spaces as tokens
1048: this .currentPosition -= offset;
1049: this .startPosition = whiteStart;
1050: if (checkIfUnicode) {
1051: this .withoutUnicodePtr = unicodePtr;
1052: }
1053: return TokenNameWHITESPACE;
1054: } else if (checkIfUnicode) {
1055: this .withoutUnicodePtr = 0;
1056: unicodeStore();
1057: } else {
1058: this .withoutUnicodePtr = 0;
1059: }
1060: }
1061: char c = this .currentCharacter;
1062: if (c < ScannerHelper.MAX_OBVIOUS) {
1063: if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
1064: return scanIdentifierOrKeywordWithBoundCheck();
1065: }
1066: return TokenNameERROR;
1067: }
1068: boolean isJavaIdStart;
1069: if (c >= HIGH_SURROGATE_MIN_VALUE
1070: && c <= HIGH_SURROGATE_MAX_VALUE) {
1071: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
1072: throw new InvalidInputException(
1073: INVALID_UNICODE_ESCAPE);
1074: }
1075: // Unicode 4 detection
1076: char low = (char) getNextCharWithBoundChecks();
1077: if (low < LOW_SURROGATE_MIN_VALUE
1078: || low > LOW_SURROGATE_MAX_VALUE) {
1079: // illegal low surrogate
1080: throw new InvalidInputException(
1081: INVALID_LOW_SURROGATE);
1082: }
1083: isJavaIdStart = ScannerHelper.isJavaIdentifierStart(c,
1084: low);
1085: } else if (c >= LOW_SURROGATE_MIN_VALUE
1086: && c <= LOW_SURROGATE_MAX_VALUE) {
1087: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
1088: throw new InvalidInputException(
1089: INVALID_UNICODE_ESCAPE);
1090: }
1091: throw new InvalidInputException(INVALID_HIGH_SURROGATE);
1092: } else {
1093: // optimized case already checked
1094: isJavaIdStart = Character.isJavaIdentifierStart(c);
1095: }
1096: if (isJavaIdStart)
1097: return scanIdentifierOrKeywordWithBoundCheck();
1098: return TokenNameERROR;
1099: }
1100: }
1101:
1102: public int getNextToken() throws InvalidInputException {
1103: this .wasAcr = false;
1104: if (this .diet) {
1105: jumpOverMethodBody();
1106: this .diet = false;
1107: return this .currentPosition > this .eofPosition ? TokenNameEOF
1108: : TokenNameRBRACE;
1109: }
1110: int whiteStart = 0;
1111: try {
1112: while (true) { //loop for jumping over comments
1113: this .withoutUnicodePtr = 0;
1114: //start with a new token (even comment written with unicode )
1115:
1116: // ---------Consume white space and handles startPosition---------
1117: whiteStart = this .currentPosition;
1118: boolean isWhiteSpace, hasWhiteSpaces = false;
1119: int offset;
1120: int unicodePtr;
1121: boolean checkIfUnicode = false;
1122: do {
1123: unicodePtr = this .withoutUnicodePtr;
1124: offset = this .currentPosition;
1125: this .startPosition = this .currentPosition;
1126: try {
1127: checkIfUnicode = ((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1128: && (this .source[this .currentPosition] == 'u');
1129: } catch (IndexOutOfBoundsException e) {
1130: if (this .tokenizeWhiteSpace
1131: && (whiteStart != this .currentPosition - 1)) {
1132: // reposition scanner in case we are interested by spaces as tokens
1133: this .currentPosition--;
1134: this .startPosition = whiteStart;
1135: return TokenNameWHITESPACE;
1136: }
1137: if (this .currentPosition > this .eofPosition)
1138: return TokenNameEOF;
1139: }
1140: if (this .currentPosition > this .eofPosition)
1141: return TokenNameEOF;
1142: if (checkIfUnicode) {
1143: isWhiteSpace = jumpOverUnicodeWhiteSpace();
1144: offset = this .currentPosition - offset;
1145: } else {
1146: offset = this .currentPosition - offset;
1147: if ((this .currentCharacter == '\r')
1148: || (this .currentCharacter == '\n')) {
1149: if (this .recordLineSeparator) {
1150: pushLineSeparator();
1151: }
1152: }
1153: // inline version of:
1154: //isWhiteSpace =
1155: // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter);
1156: switch (this .currentCharacter) {
1157: case 10: /* \ u000a: LINE FEED */
1158: case 12: /* \ u000c: FORM FEED */
1159: case 13: /* \ u000d: CARRIAGE RETURN */
1160: case 32: /* \ u0020: SPACE */
1161: case 9: /* \ u0009: HORIZONTAL TABULATION */
1162: isWhiteSpace = true;
1163: break;
1164: default:
1165: isWhiteSpace = false;
1166: }
1167: }
1168: if (isWhiteSpace) {
1169: hasWhiteSpaces = true;
1170: }
1171: } while (isWhiteSpace);
1172: if (hasWhiteSpaces) {
1173: if (this .tokenizeWhiteSpace) {
1174: // reposition scanner in case we are interested by spaces as tokens
1175: this .currentPosition -= offset;
1176: this .startPosition = whiteStart;
1177: if (checkIfUnicode) {
1178: this .withoutUnicodePtr = unicodePtr;
1179: }
1180: return TokenNameWHITESPACE;
1181: } else if (checkIfUnicode) {
1182: this .withoutUnicodePtr = 0;
1183: unicodeStore();
1184: } else {
1185: this .withoutUnicodePtr = 0;
1186: }
1187: }
1188: // ---------Identify the next token-------------
1189: switch (this .currentCharacter) {
1190: case '@':
1191: /* if (this.sourceLevel >= ClassFileConstants.JDK1_5) {
1192: return TokenNameAT;
1193: } else {
1194: return TokenNameERROR;
1195: }*/
1196: return TokenNameAT;
1197: case '(':
1198: return TokenNameLPAREN;
1199: case ')':
1200: return TokenNameRPAREN;
1201: case '{':
1202: return TokenNameLBRACE;
1203: case '}':
1204: return TokenNameRBRACE;
1205: case '[':
1206: return TokenNameLBRACKET;
1207: case ']':
1208: return TokenNameRBRACKET;
1209: case ';':
1210: return TokenNameSEMICOLON;
1211: case ',':
1212: return TokenNameCOMMA;
1213: case '.':
1214: if (getNextCharAsDigit()) {
1215: return scanNumber(true);
1216: }
1217: int temp = this .currentPosition;
1218: if (getNextChar('.')) {
1219: if (getNextChar('.')) {
1220: return TokenNameELLIPSIS;
1221: } else {
1222: this .currentPosition = temp;
1223: return TokenNameDOT;
1224: }
1225: } else {
1226: this .currentPosition = temp;
1227: return TokenNameDOT;
1228: }
1229: case '+': {
1230: int test;
1231: if ((test = getNextChar('+', '=')) == 0)
1232: return TokenNamePLUS_PLUS;
1233: if (test > 0)
1234: return TokenNamePLUS_EQUAL;
1235: return TokenNamePLUS;
1236: }
1237: case '-': {
1238: int test;
1239: if ((test = getNextChar('-', '=')) == 0)
1240: return TokenNameMINUS_MINUS;
1241: if (test > 0)
1242: return TokenNameMINUS_EQUAL;
1243: return TokenNameMINUS;
1244: }
1245: case '~':
1246: return TokenNameTWIDDLE;
1247: case '!':
1248: if (getNextChar('='))
1249: return TokenNameNOT_EQUAL;
1250: return TokenNameNOT;
1251: case '*':
1252: if (getNextChar('='))
1253: return TokenNameMULTIPLY_EQUAL;
1254: return TokenNameMULTIPLY;
1255: case '%':
1256: if (getNextChar('='))
1257: return TokenNameREMAINDER_EQUAL;
1258: return TokenNameREMAINDER;
1259: case '<': {
1260: int test;
1261: if ((test = getNextChar('=', '<')) == 0)
1262: return TokenNameLESS_EQUAL;
1263: if (test > 0) {
1264: if (getNextChar('='))
1265: return TokenNameLEFT_SHIFT_EQUAL;
1266: return TokenNameLEFT_SHIFT;
1267: }
1268: return TokenNameLESS;
1269: }
1270: case '>': {
1271: int test;
1272: if (this .returnOnlyGreater) {
1273: return TokenNameGREATER;
1274: }
1275: if ((test = getNextChar('=', '>')) == 0)
1276: return TokenNameGREATER_EQUAL;
1277: if (test > 0) {
1278: if ((test = getNextChar('=', '>')) == 0)
1279: return TokenNameRIGHT_SHIFT_EQUAL;
1280: if (test > 0) {
1281: if (getNextChar('='))
1282: return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
1283: return TokenNameUNSIGNED_RIGHT_SHIFT;
1284: }
1285: return TokenNameRIGHT_SHIFT;
1286: }
1287: return TokenNameGREATER;
1288: }
1289: case '=':
1290: if (getNextChar('='))
1291: return TokenNameEQUAL_EQUAL;
1292: return TokenNameEQUAL;
1293: case '&': {
1294: int test;
1295: if ((test = getNextChar('&', '=')) == 0)
1296: return TokenNameAND_AND;
1297: if (test > 0)
1298: return TokenNameAND_EQUAL;
1299: return TokenNameAND;
1300: }
1301: case '|': {
1302: int test;
1303: if ((test = getNextChar('|', '=')) == 0)
1304: return TokenNameOR_OR;
1305: if (test > 0)
1306: return TokenNameOR_EQUAL;
1307: return TokenNameOR;
1308: }
1309: case '^':
1310: if (getNextChar('='))
1311: return TokenNameXOR_EQUAL;
1312: return TokenNameXOR;
1313: case '?':
1314: return TokenNameQUESTION;
1315: case ':':
1316: return TokenNameCOLON;
1317: case '\'': {
1318: int test;
1319: if ((test = getNextChar('\n', '\r')) == 0) {
1320: throw new InvalidInputException(
1321: INVALID_CHARACTER_CONSTANT);
1322: }
1323: if (test > 0) {
1324: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1325: for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1326: if (this .currentPosition + lookAhead == this .eofPosition)
1327: break;
1328: if (this .source[this .currentPosition
1329: + lookAhead] == '\n')
1330: break;
1331: if (this .source[this .currentPosition
1332: + lookAhead] == '\'') {
1333: this .currentPosition += lookAhead + 1;
1334: break;
1335: }
1336: }
1337: throw new InvalidInputException(
1338: INVALID_CHARACTER_CONSTANT);
1339: }
1340: }
1341: if (getNextChar('\'')) {
1342: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1343: for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1344: if (this .currentPosition + lookAhead == this .eofPosition)
1345: break;
1346: if (this .source[this .currentPosition
1347: + lookAhead] == '\n')
1348: break;
1349: if (this .source[this .currentPosition
1350: + lookAhead] == '\'') {
1351: this .currentPosition += lookAhead + 1;
1352: break;
1353: }
1354: }
1355: throw new InvalidInputException(
1356: INVALID_CHARACTER_CONSTANT);
1357: }
1358: if (getNextChar('\\')) {
1359: if (this .unicodeAsBackSlash) {
1360: // consume next character
1361: this .unicodeAsBackSlash = false;
1362: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1363: && (this .source[this .currentPosition] == 'u')) {
1364: getNextUnicodeChar();
1365: } else {
1366: if (this .withoutUnicodePtr != 0) {
1367: unicodeStore();
1368: }
1369: }
1370: } else {
1371: this .currentCharacter = this .source[this .currentPosition++];
1372: }
1373: scanEscapeCharacter();
1374: } else { // consume next character
1375: this .unicodeAsBackSlash = false;
1376: checkIfUnicode = false;
1377: try {
1378: checkIfUnicode = ((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1379: && (this .source[this .currentPosition] == 'u');
1380: } catch (IndexOutOfBoundsException e) {
1381: this .currentPosition--;
1382: throw new InvalidInputException(
1383: INVALID_CHARACTER_CONSTANT);
1384: }
1385: if (checkIfUnicode) {
1386: getNextUnicodeChar();
1387: } else {
1388: if (this .withoutUnicodePtr != 0) {
1389: unicodeStore();
1390: }
1391: }
1392: }
1393: if (getNextChar('\''))
1394: return TokenNameCharacterLiteral;
1395: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1396: for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
1397: if (this .currentPosition + lookAhead == this .eofPosition)
1398: break;
1399: if (this .source[this .currentPosition
1400: + lookAhead] == '\n')
1401: break;
1402: if (this .source[this .currentPosition
1403: + lookAhead] == '\'') {
1404: this .currentPosition += lookAhead + 1;
1405: break;
1406: }
1407: }
1408: throw new InvalidInputException(
1409: INVALID_CHARACTER_CONSTANT);
1410: case '"':
1411: try {
1412: // consume next character
1413: this .unicodeAsBackSlash = false;
1414: boolean isUnicode = false;
1415: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1416: && (this .source[this .currentPosition] == 'u')) {
1417: getNextUnicodeChar();
1418: isUnicode = true;
1419: } else {
1420: if (this .withoutUnicodePtr != 0) {
1421: unicodeStore();
1422: }
1423: }
1424:
1425: while (this .currentCharacter != '"') {
1426: /**** \r and \n are not valid in string literals ****/
1427: if ((this .currentCharacter == '\n')
1428: || (this .currentCharacter == '\r')) {
1429: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1430: if (isUnicode) {
1431: int start = this .currentPosition;
1432: for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1433: if (this .currentPosition >= this .eofPosition) {
1434: this .currentPosition = start;
1435: break;
1436: }
1437: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1438: && (this .source[this .currentPosition] == 'u')) {
1439: isUnicode = true;
1440: getNextUnicodeChar();
1441: } else {
1442: isUnicode = false;
1443: }
1444: if (!isUnicode
1445: && this .currentCharacter == '\n') {
1446: this .currentPosition--; // set current position on new line character
1447: break;
1448: }
1449: if (this .currentCharacter == '\"') {
1450: throw new InvalidInputException(
1451: INVALID_CHAR_IN_STRING);
1452: }
1453: }
1454: } else {
1455: this .currentPosition--; // set current position on new line character
1456: }
1457: throw new InvalidInputException(
1458: INVALID_CHAR_IN_STRING);
1459: }
1460: if (this .currentCharacter == '\\') {
1461: if (this .unicodeAsBackSlash) {
1462: this .withoutUnicodePtr--;
1463: // consume next character
1464: this .unicodeAsBackSlash = false;
1465: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1466: && (this .source[this .currentPosition] == 'u')) {
1467: getNextUnicodeChar();
1468: isUnicode = true;
1469: this .withoutUnicodePtr--;
1470: } else {
1471: isUnicode = false;
1472: }
1473: } else {
1474: if (this .withoutUnicodePtr == 0) {
1475: unicodeInitializeBuffer(this .currentPosition
1476: - this .startPosition);
1477: }
1478: this .withoutUnicodePtr--;
1479: this .currentCharacter = this .source[this .currentPosition++];
1480: }
1481: // we need to compute the escape character in a separate buffer
1482: scanEscapeCharacter();
1483: if (this .withoutUnicodePtr != 0) {
1484: unicodeStore();
1485: }
1486: }
1487: // consume next character
1488: this .unicodeAsBackSlash = false;
1489: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1490: && (this .source[this .currentPosition] == 'u')) {
1491: getNextUnicodeChar();
1492: isUnicode = true;
1493: } else {
1494: isUnicode = false;
1495: if (this .withoutUnicodePtr != 0) {
1496: unicodeStore();
1497: }
1498: }
1499:
1500: }
1501: } catch (IndexOutOfBoundsException e) {
1502: this .currentPosition--;
1503: throw new InvalidInputException(
1504: UNTERMINATED_STRING);
1505: } catch (InvalidInputException e) {
1506: if (e.getMessage().equals(INVALID_ESCAPE)) {
1507: // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1508: for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1509: if (this .currentPosition + lookAhead == this .eofPosition)
1510: break;
1511: if (this .source[this .currentPosition
1512: + lookAhead] == '\n')
1513: break;
1514: if (this .source[this .currentPosition
1515: + lookAhead] == '\"') {
1516: this .currentPosition += lookAhead + 1;
1517: break;
1518: }
1519: }
1520:
1521: }
1522: throw e; // rethrow
1523: }
1524: return TokenNameStringLiteral;
1525: case '/': {
1526: int test;
1527: if ((test = getNextChar('/', '*')) == 0) { //line comment
1528: this .lastCommentLinePosition = this .currentPosition;
1529: try { //get the next char
1530: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1531: && (this .source[this .currentPosition] == 'u')) {
1532: getNextUnicodeChar();
1533: }
1534:
1535: //handle the \\u case manually into comment
1536:if (this .currentCharacter == '\\') {
1537: if (this .source[this .currentPosition] == '\\')
1538: this .currentPosition++;
1539: } //jump over the \\
1540: boolean isUnicode = false;
1541: while (this .currentCharacter != '\r'
1542: && this .currentCharacter != '\n') {
1543: this .lastCommentLinePosition = this .currentPosition;
1544: //get the next char
1545: isUnicode = false;
1546: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1547: && (this .source[this .currentPosition] == 'u')) {
1548: getNextUnicodeChar();
1549: isUnicode = true;
1550: }
1551: //handle the \\u case manually into comment
1552:if (this .currentCharacter == '\\') {
1553: if (this .source[this .currentPosition] == '\\')
1554: this .currentPosition++;
1555: } //jump over the \\
1556: }
1557: /*
1558: * We need to completely consume the line break
1559: */
1560: if (this .currentCharacter == '\r'
1561: && this .eofPosition > this .currentPosition) {
1562: if (this .source[this .currentPosition] == '\n') {
1563: this .currentPosition++;
1564: this .currentCharacter = '\n';
1565: } else if ((this .source[this .currentPosition] == '\\')
1566: && (this .source[this .currentPosition + 1] == 'u')) {
1567: getNextUnicodeChar();
1568: isUnicode = true;
1569: }
1570: }
1571: recordComment(TokenNameCOMMENT_LINE);
1572: if (this .taskTags != null)
1573: checkTaskTag(this .startPosition,
1574: this .currentPosition);
1575: if ((this .currentCharacter == '\r')
1576: || (this .currentCharacter == '\n')) {
1577: if (this .checkNonExternalizedStringLiterals) {
1578: parseTags();
1579: }
1580: if (this .recordLineSeparator) {
1581: if (isUnicode) {
1582: pushUnicodeLineSeparator();
1583: } else {
1584: pushLineSeparator();
1585: }
1586: }
1587: }
1588: if (this .tokenizeComments) {
1589: return TokenNameCOMMENT_LINE;
1590: }
1591: } catch (IndexOutOfBoundsException e) {
1592: this .currentPosition--;
1593: recordComment(TokenNameCOMMENT_LINE);
1594: if (this .taskTags != null)
1595: checkTaskTag(this .startPosition,
1596: this .currentPosition);
1597: if (this .checkNonExternalizedStringLiterals) {
1598: parseTags();
1599: }
1600: if (this .tokenizeComments) {
1601: return TokenNameCOMMENT_LINE;
1602: } else {
1603: this .currentPosition++;
1604: }
1605: }
1606: break;
1607: }
1608: if (test > 0) { //traditional and javadoc comment
1609: try { //get the next char
1610: boolean isJavadoc = false, star = false;
1611: boolean isUnicode = false;
1612: int previous;
1613: // consume next character
1614: this .unicodeAsBackSlash = false;
1615: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1616: && (this .source[this .currentPosition] == 'u')) {
1617: getNextUnicodeChar();
1618: isUnicode = true;
1619: } else {
1620: isUnicode = false;
1621: if (this .withoutUnicodePtr != 0) {
1622: unicodeStore();
1623: }
1624: }
1625:
1626: if (this .currentCharacter == '*') {
1627: isJavadoc = true;
1628: star = true;
1629: }
1630: if ((this .currentCharacter == '\r')
1631: || (this .currentCharacter == '\n')) {
1632: if (this .recordLineSeparator) {
1633: if (isUnicode) {
1634: pushUnicodeLineSeparator();
1635: } else {
1636: pushLineSeparator();
1637: }
1638: }
1639: }
1640: isUnicode = false;
1641: previous = this .currentPosition;
1642: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1643: && (this .source[this .currentPosition] == 'u')) {
1644: //-------------unicode traitement ------------
1645: getNextUnicodeChar();
1646: isUnicode = true;
1647: } else {
1648: isUnicode = false;
1649: }
1650: //handle the \\u case manually into comment
1651:if (this .currentCharacter == '\\') {
1652: if (this .source[this .currentPosition] == '\\')
1653: this .currentPosition++; //jump over the \\
1654: }
1655: // empty comment is not a javadoc /**/
1656: if (this .currentCharacter == '/') {
1657: isJavadoc = false;
1658: }
1659: //loop until end of comment */
1660: int firstTag = 0;
1661: while ((this .currentCharacter != '/')
1662: || (!star)) {
1663: if ((this .currentCharacter == '\r')
1664: || (this .currentCharacter == '\n')) {
1665: if (this .recordLineSeparator) {
1666: if (isUnicode) {
1667: pushUnicodeLineSeparator();
1668: } else {
1669: pushLineSeparator();
1670: }
1671: }
1672: }
1673: switch (this .currentCharacter) {
1674: case '*':
1675: star = true;
1676: break;
1677: case '@':
1678: if (firstTag == 0) {
1679: firstTag = previous;
1680: }
1681: // fall through default case to set star to false
1682: default:
1683: star = false;
1684: }
1685: //get next char
1686: previous = this .currentPosition;
1687: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1688: && (this .source[this .currentPosition] == 'u')) {
1689: //-------------unicode traitement ------------
1690: getNextUnicodeChar();
1691: isUnicode = true;
1692: } else {
1693: isUnicode = false;
1694: }
1695: //handle the \\u case manually into comment
1696:if (this .currentCharacter == '\\') {
1697: if (this .source[this .currentPosition] == '\\')
1698: this .currentPosition++;
1699: } //jump over the \\
1700: }
1701: int token = isJavadoc ? TokenNameCOMMENT_JAVADOC
1702: : TokenNameCOMMENT_BLOCK;
1703: recordComment(token);
1704: this .commentTagStarts[this .commentPtr] = firstTag;
1705: if (this .taskTags != null)
1706: checkTaskTag(this .startPosition,
1707: this .currentPosition);
1708: if (this .tokenizeComments) {
1709: /*
1710: if (isJavadoc)
1711: return TokenNameCOMMENT_JAVADOC;
1712: return TokenNameCOMMENT_BLOCK;
1713: */
1714: return token;
1715: }
1716: } catch (IndexOutOfBoundsException e) {
1717: this .currentPosition--;
1718: throw new InvalidInputException(
1719: UNTERMINATED_COMMENT);
1720: }
1721: break;
1722: }
1723: if (getNextChar('='))
1724: return TokenNameDIVIDE_EQUAL;
1725: return TokenNameDIVIDE;
1726: }
1727: case '\u001a':
1728: if (atEnd())
1729: return TokenNameEOF;
1730: //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
1731: throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1732: default:
1733: char c = this .currentCharacter;
1734: if (c < ScannerHelper.MAX_OBVIOUS) {
1735: if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
1736: return scanIdentifierOrKeyword();
1737: } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) {
1738: return scanNumber(false);
1739: } else {
1740: return TokenNameERROR;
1741: }
1742: }
1743: boolean isJavaIdStart;
1744: if (c >= HIGH_SURROGATE_MIN_VALUE
1745: && c <= HIGH_SURROGATE_MAX_VALUE) {
1746: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
1747: throw new InvalidInputException(
1748: INVALID_UNICODE_ESCAPE);
1749: }
1750: // Unicode 4 detection
1751: char low = (char) getNextChar();
1752: if (low < LOW_SURROGATE_MIN_VALUE
1753: || low > LOW_SURROGATE_MAX_VALUE) {
1754: // illegal low surrogate
1755: throw new InvalidInputException(
1756: INVALID_LOW_SURROGATE);
1757: }
1758: isJavaIdStart = ScannerHelper
1759: .isJavaIdentifierStart(c, low);
1760: } else if (c >= LOW_SURROGATE_MIN_VALUE
1761: && c <= LOW_SURROGATE_MAX_VALUE) {
1762: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
1763: throw new InvalidInputException(
1764: INVALID_UNICODE_ESCAPE);
1765: }
1766: throw new InvalidInputException(
1767: INVALID_HIGH_SURROGATE);
1768: } else {
1769: // optimized case already checked
1770: isJavaIdStart = Character
1771: .isJavaIdentifierStart(c);
1772: }
1773: if (isJavaIdStart)
1774: return scanIdentifierOrKeyword();
1775: if (ScannerHelper.isDigit(this .currentCharacter)) {
1776: return scanNumber(false);
1777: }
1778: return TokenNameERROR;
1779: }
1780: }
1781: } //-----------------end switch while try--------------------
1782: catch (IndexOutOfBoundsException e) {
1783: if (this .tokenizeWhiteSpace
1784: && (whiteStart != this .currentPosition - 1)) {
1785: // reposition scanner in case we are interested by spaces as tokens
1786: this .currentPosition--;
1787: this .startPosition = whiteStart;
1788: return TokenNameWHITESPACE;
1789: }
1790: }
1791: return TokenNameEOF;
1792: }
1793:
1794: public void getNextUnicodeChar() throws InvalidInputException {
1795: //VOID
1796: //handle the case of unicode.
1797: //when a unicode appears then we must use a buffer that holds char internal values
1798: //At the end of this method currentCharacter holds the new visited char
1799: //and currentPosition points right next after it
1800:
1801: //ALL getNextChar.... ARE OPTIMIZED COPIES
1802: int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
1803: this .currentPosition++;
1804: if (this .currentPosition < this .eofPosition) {
1805: while (this .source[this .currentPosition] == 'u') {
1806: this .currentPosition++;
1807: if (this .currentPosition >= this .eofPosition) {
1808: this .currentPosition--;
1809: throw new InvalidInputException(
1810: INVALID_UNICODE_ESCAPE);
1811: }
1812: unicodeSize++;
1813: }
1814: } else {
1815: this .currentPosition--;
1816: throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1817: }
1818:
1819: if ((this .currentPosition + 4) > this .eofPosition) {
1820: this .currentPosition += (this .eofPosition - this .currentPosition);
1821: throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1822: }
1823: if ((c1 = ScannerHelper
1824: .getNumericValue(this .source[this .currentPosition++])) > 15
1825: || c1 < 0
1826: || (c2 = ScannerHelper
1827: .getNumericValue(this .source[this .currentPosition++])) > 15
1828: || c2 < 0
1829: || (c3 = ScannerHelper
1830: .getNumericValue(this .source[this .currentPosition++])) > 15
1831: || c3 < 0
1832: || (c4 = ScannerHelper
1833: .getNumericValue(this .source[this .currentPosition++])) > 15
1834: || c4 < 0) {
1835: throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1836: }
1837: this .currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1838: //need the unicode buffer
1839: if (this .withoutUnicodePtr == 0) {
1840: //buffer all the entries that have been left aside....
1841: unicodeInitializeBuffer(this .currentPosition - unicodeSize
1842: - this .startPosition);
1843: }
1844: //fill the buffer with the char
1845: unicodeStore();
1846: this .unicodeAsBackSlash = this .currentCharacter == '\\';
1847: }
1848:
1849: public NLSTag[] getNLSTags() {
1850: final int length = this .nlsTagsPtr;
1851: if (length != 0) {
1852: NLSTag[] result = new NLSTag[length];
1853: System.arraycopy(this .nlsTags, 0, result, 0, length);
1854: this .nlsTagsPtr = 0;
1855: return result;
1856: }
1857: return null;
1858: }
1859:
1860: public char[] getSource() {
1861: return this .source;
1862: }
1863:
1864: public final void jumpOverMethodBody() {
1865:
1866: this .wasAcr = false;
1867: int found = 1;
1868: try {
1869: while (true) { //loop for jumping over comments
1870: this .withoutUnicodePtr = 0;
1871: // ---------Consume white space and handles startPosition---------
1872: boolean isWhiteSpace;
1873: do {
1874: this .startPosition = this .currentPosition;
1875: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1876: && (this .source[this .currentPosition] == 'u')) {
1877: isWhiteSpace = jumpOverUnicodeWhiteSpace();
1878: } else {
1879: if (this .recordLineSeparator
1880: && ((this .currentCharacter == '\r') || (this .currentCharacter == '\n'))) {
1881: pushLineSeparator();
1882: }
1883: isWhiteSpace = CharOperation
1884: .isWhitespace(this .currentCharacter);
1885: }
1886: } while (isWhiteSpace);
1887:
1888: // -------consume token until } is found---------
1889: NextToken: switch (this .currentCharacter) {
1890: case '{':
1891: found++;
1892: break NextToken;
1893: case '}':
1894: found--;
1895: if (found == 0)
1896: return;
1897: break NextToken;
1898: case '\'': {
1899: boolean test;
1900: test = getNextChar('\\');
1901: if (test) {
1902: try {
1903: if (this .unicodeAsBackSlash) {
1904: // consume next character
1905: this .unicodeAsBackSlash = false;
1906: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1907: && (this .source[this .currentPosition] == 'u')) {
1908: getNextUnicodeChar();
1909: } else {
1910: if (this .withoutUnicodePtr != 0) {
1911: unicodeStore();
1912: }
1913: }
1914: } else {
1915: this .currentCharacter = this .source[this .currentPosition++];
1916: }
1917: scanEscapeCharacter();
1918: } catch (InvalidInputException ex) {
1919: // ignore
1920: }
1921: } else {
1922: try { // consume next character
1923: this .unicodeAsBackSlash = false;
1924: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1925: && (this .source[this .currentPosition] == 'u')) {
1926: getNextUnicodeChar();
1927: } else {
1928: if (this .withoutUnicodePtr != 0) {
1929: unicodeStore();
1930: }
1931: }
1932: } catch (InvalidInputException ex) {
1933: // ignore
1934: }
1935: }
1936: getNextChar('\'');
1937: break NextToken;
1938: }
1939: case '"':
1940: try {
1941: try { // consume next character
1942: this .unicodeAsBackSlash = false;
1943: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1944: && (this .source[this .currentPosition] == 'u')) {
1945: getNextUnicodeChar();
1946: } else {
1947: if (this .withoutUnicodePtr != 0) {
1948: unicodeStore();
1949: }
1950: }
1951: } catch (InvalidInputException ex) {
1952: // ignore
1953: }
1954: while (this .currentCharacter != '"') {
1955: if (this .currentCharacter == '\r') {
1956: if (this .source[this .currentPosition] == '\n')
1957: this .currentPosition++;
1958: break NextToken; // the string cannot go further that the line
1959: }
1960: if (this .currentCharacter == '\n') {
1961: break; // the string cannot go further that the line
1962: }
1963: if (this .currentCharacter == '\\') {
1964: try {
1965: if (this .unicodeAsBackSlash) {
1966: // consume next character
1967: this .unicodeAsBackSlash = false;
1968: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1969: && (this .source[this .currentPosition] == 'u')) {
1970: getNextUnicodeChar();
1971: } else {
1972: if (this .withoutUnicodePtr != 0) {
1973: unicodeStore();
1974: }
1975: }
1976: } else {
1977: this .currentCharacter = this .source[this .currentPosition++];
1978: }
1979: scanEscapeCharacter();
1980: } catch (InvalidInputException ex) {
1981: // ignore
1982: }
1983: }
1984: try { // consume next character
1985: this .unicodeAsBackSlash = false;
1986: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
1987: && (this .source[this .currentPosition] == 'u')) {
1988: getNextUnicodeChar();
1989: } else {
1990: if (this .withoutUnicodePtr != 0) {
1991: unicodeStore();
1992: }
1993: }
1994: } catch (InvalidInputException ex) {
1995: // ignore
1996: }
1997: }
1998: } catch (IndexOutOfBoundsException e) {
1999: return;
2000: }
2001: break NextToken;
2002: case '/': {
2003: int test;
2004: if ((test = getNextChar('/', '*')) == 0) { //line comment
2005: try {
2006: this .lastCommentLinePosition = this .currentPosition;
2007: //get the next char
2008: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
2009: && (this .source[this .currentPosition] == 'u')) {
2010: getNextUnicodeChar();
2011: }
2012: //handle the \\u case manually into comment
2013:if (this .currentCharacter == '\\') {
2014: if (this .source[this .currentPosition] == '\\')
2015: this .currentPosition++;
2016: } //jump over the \\
2017: boolean isUnicode = false;
2018: while (this .currentCharacter != '\r'
2019: && this .currentCharacter != '\n') {
2020: this .lastCommentLinePosition = this .currentPosition;
2021: //get the next char
2022: isUnicode = false;
2023: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
2024: && (this .source[this .currentPosition] == 'u')) {
2025: isUnicode = true;
2026: getNextUnicodeChar();
2027: }
2028: //handle the \\u case manually into comment
2029:if (this .currentCharacter == '\\') {
2030: if (this .source[this .currentPosition] == '\\')
2031: this .currentPosition++;
2032: } //jump over the \\
2033: }
2034: /*
2035: * We need to completely consume the line break
2036: */
2037: if (this .currentCharacter == '\r'
2038: && this .eofPosition > this .currentPosition) {
2039: if (this .source[this .currentPosition] == '\n') {
2040: this .currentPosition++;
2041: this .currentCharacter = '\n';
2042: } else if ((this .source[this .currentPosition] == '\\')
2043: && (this .source[this .currentPosition + 1] == 'u')) {
2044: isUnicode = true;
2045: getNextUnicodeChar();
2046: }
2047: }
2048: recordComment(TokenNameCOMMENT_LINE);
2049: if (this .recordLineSeparator
2050: && ((this .currentCharacter == '\r') || (this .currentCharacter == '\n'))) {
2051: if (this .checkNonExternalizedStringLiterals) {
2052: parseTags();
2053: }
2054: if (this .recordLineSeparator) {
2055: if (isUnicode) {
2056: pushUnicodeLineSeparator();
2057: } else {
2058: pushLineSeparator();
2059: }
2060: }
2061: }
2062: } catch (IndexOutOfBoundsException e) {
2063: //an eof will then be generated
2064: this .currentPosition--;
2065: recordComment(TokenNameCOMMENT_LINE);
2066: if (this .checkNonExternalizedStringLiterals) {
2067: parseTags();
2068: }
2069: if (!this .tokenizeComments) {
2070: this .currentPosition++;
2071: }
2072: }
2073: break NextToken;
2074: }
2075: if (test > 0) { //traditional and javadoc comment
2076: boolean isJavadoc = false;
2077: try { //get the next char
2078: boolean star = false;
2079: int previous;
2080: boolean isUnicode = false;
2081: // consume next character
2082: this .unicodeAsBackSlash = false;
2083: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
2084: && (this .source[this .currentPosition] == 'u')) {
2085: getNextUnicodeChar();
2086: isUnicode = true;
2087: } else {
2088: isUnicode = false;
2089: if (this .withoutUnicodePtr != 0) {
2090: unicodeStore();
2091: }
2092: }
2093:
2094: if (this .currentCharacter == '*') {
2095: isJavadoc = true;
2096: star = true;
2097: }
2098: if ((this .currentCharacter == '\r')
2099: || (this .currentCharacter == '\n')) {
2100: if (this .recordLineSeparator) {
2101: if (isUnicode) {
2102: pushUnicodeLineSeparator();
2103: } else {
2104: pushLineSeparator();
2105: }
2106: }
2107: }
2108: isUnicode = false;
2109: previous = this .currentPosition;
2110: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
2111: && (this .source[this .currentPosition] == 'u')) {
2112: getNextUnicodeChar();
2113: isUnicode = true;
2114: } else {
2115: isUnicode = false;
2116: }
2117: //handle the \\u case manually into comment
2118:if (this .currentCharacter == '\\') {
2119: if (this .source[this .currentPosition] == '\\')
2120: this .currentPosition++; //jump over the \\
2121: }
2122: // empty comment is not a javadoc /**/
2123: if (this .currentCharacter == '/') {
2124: isJavadoc = false;
2125: }
2126: //loop until end of comment */
2127: int firstTag = 0;
2128: while ((this .currentCharacter != '/')
2129: || (!star)) {
2130: if ((this .currentCharacter == '\r')
2131: || (this .currentCharacter == '\n')) {
2132: if (this .recordLineSeparator) {
2133: if (isUnicode) {
2134: pushUnicodeLineSeparator();
2135: } else {
2136: pushLineSeparator();
2137: }
2138: }
2139: }
2140: switch (this .currentCharacter) {
2141: case '*':
2142: star = true;
2143: break;
2144: case '@':
2145: if (firstTag == 0) {
2146: firstTag = previous;
2147: }
2148: // fall through default case to set star to false
2149: default:
2150: star = false;
2151: }
2152: //get next char
2153: previous = this .currentPosition;
2154: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
2155: && (this .source[this .currentPosition] == 'u')) {
2156: getNextUnicodeChar();
2157: isUnicode = true;
2158: } else {
2159: isUnicode = false;
2160: }
2161: //handle the \\u case manually into comment
2162:if (this .currentCharacter == '\\') {
2163: if (this .source[this .currentPosition] == '\\')
2164: this .currentPosition++;
2165: } //jump over the \\
2166: }
2167: recordComment(isJavadoc ? TokenNameCOMMENT_JAVADOC
2168: : TokenNameCOMMENT_BLOCK);
2169: this .commentTagStarts[this .commentPtr] = firstTag;
2170: } catch (IndexOutOfBoundsException e) {
2171: return;
2172: }
2173: break NextToken;
2174: }
2175: break NextToken;
2176: }
2177:
2178: default:
2179: try {
2180: char c = this .currentCharacter;
2181: if (c < ScannerHelper.MAX_OBVIOUS) {
2182: if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
2183: scanIdentifierOrKeyword();
2184: break NextToken;
2185: } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) {
2186: scanNumber(false);
2187: break NextToken;
2188: } else {
2189: break NextToken;
2190: }
2191: }
2192: boolean isJavaIdStart;
2193: if (c >= HIGH_SURROGATE_MIN_VALUE
2194: && c <= HIGH_SURROGATE_MAX_VALUE) {
2195: if (this .complianceLevel < ClassFileConstants.JDK1_5) {
2196: throw new InvalidInputException(
2197: INVALID_UNICODE_ESCAPE);
2198: }
2199: // Unicode 4 detection
2200: char low = (char) getNextChar();
2201: if (low < LOW_SURROGATE_MIN_VALUE
2202: || low > LOW_SURROGATE_MAX_VALUE) {
2203: // illegal low surrogate
2204: break NextToken;
2205: }
2206: isJavaIdStart = ScannerHelper
2207: .isJavaIdentifierStart(c, low);
2208: } else if (c >= LOW_SURROGATE_MIN_VALUE
2209: && c <= LOW_SURROGATE_MAX_VALUE) {
2210: break NextToken;
2211: } else {
2212: // optimized case already checked
2213: isJavaIdStart = Character
2214: .isJavaIdentifierStart(c);
2215: }
2216: if (isJavaIdStart) {
2217: scanIdentifierOrKeyword();
2218: break NextToken;
2219: }
2220: // if (ScannerHelper.isDigit(this.currentCharacter)) {
2221: // scanNumber(false);
2222: // break NextToken;
2223: // }
2224: } catch (InvalidInputException ex) {
2225: // ignore
2226: }
2227: }
2228: }
2229: //-----------------end switch while try--------------------
2230: } catch (IndexOutOfBoundsException e) {
2231: // ignore
2232: } catch (InvalidInputException e) {
2233: // ignore
2234: }
2235: return;
2236: }
2237:
2238: public final boolean jumpOverUnicodeWhiteSpace()
2239: throws InvalidInputException {
2240: //BOOLEAN
2241: //handle the case of unicode. Jump over the next whiteSpace
2242: //making startPosition pointing on the next available char
2243: //On false, the currentCharacter is filled up with a potential
2244: //correct char
2245:
2246: this .wasAcr = false;
2247: getNextUnicodeChar();
2248: return CharOperation.isWhitespace(this .currentCharacter);
2249: }
2250:
2251: final char[] optimizedCurrentTokenSource1() {
2252: //return always the same char[] build only once
2253:
2254: //optimization at no speed cost of 99.5 % of the singleCharIdentifier
2255: char charOne = this .source[this .startPosition];
2256: switch (charOne) {
2257: case 'a':
2258: return charArray_a;
2259: case 'b':
2260: return charArray_b;
2261: case 'c':
2262: return charArray_c;
2263: case 'd':
2264: return charArray_d;
2265: case 'e':
2266: return charArray_e;
2267: case 'f':
2268: return charArray_f;
2269: case 'g':
2270: return charArray_g;
2271: case 'h':
2272: return charArray_h;
2273: case 'i':
2274: return charArray_i;
2275: case 'j':
2276: return charArray_j;
2277: case 'k':
2278: return charArray_k;
2279: case 'l':
2280: return charArray_l;
2281: case 'm':
2282: return charArray_m;
2283: case 'n':
2284: return charArray_n;
2285: case 'o':
2286: return charArray_o;
2287: case 'p':
2288: return charArray_p;
2289: case 'q':
2290: return charArray_q;
2291: case 'r':
2292: return charArray_r;
2293: case 's':
2294: return charArray_s;
2295: case 't':
2296: return charArray_t;
2297: case 'u':
2298: return charArray_u;
2299: case 'v':
2300: return charArray_v;
2301: case 'w':
2302: return charArray_w;
2303: case 'x':
2304: return charArray_x;
2305: case 'y':
2306: return charArray_y;
2307: case 'z':
2308: return charArray_z;
2309: default:
2310: return new char[] { charOne };
2311: }
2312: }
2313:
2314: final char[] optimizedCurrentTokenSource2() {
2315: //try to return the same char[] build only once
2316:
2317: char[] src = this .source;
2318: int start = this .startPosition;
2319: char c0, c1;
2320: int hash = (((c0 = src[start]) << 6) + (c1 = src[start + 1]))
2321: % TableSize;
2322: char[][] table = this .charArray_length[0][hash];
2323: int i = newEntry2;
2324: while (++i < InternalTableSize) {
2325: char[] charArray = table[i];
2326: if ((c0 == charArray[0]) && (c1 == charArray[1]))
2327: return charArray;
2328: }
2329: //---------other side---------
2330: i = -1;
2331: int max = newEntry2;
2332: while (++i <= max) {
2333: char[] charArray = table[i];
2334: if ((c0 == charArray[0]) && (c1 == charArray[1]))
2335: return charArray;
2336: }
2337: //--------add the entry-------
2338: if (++max >= InternalTableSize)
2339: max = 0;
2340: char[] r;
2341: System.arraycopy(src, start, r = new char[2], 0, 2);
2342: //newIdentCount++;
2343: return table[newEntry2 = max] = r; //(r = new char[] {c0, c1});
2344: }
2345:
2346: final char[] optimizedCurrentTokenSource3() {
2347: //try to return the same char[] build only once
2348:
2349: char[] src = this .source;
2350: int start = this .startPosition;
2351: char c0, c1 = src[start + 1], c2;
2352: int hash = (((c0 = src[start]) << 6) + (c2 = src[start + 2]))
2353: % TableSize;
2354: // int hash = ((c0 << 12) + (c1<< 6) + c2) % TableSize;
2355: char[][] table = this .charArray_length[1][hash];
2356: int i = newEntry3;
2357: while (++i < InternalTableSize) {
2358: char[] charArray = table[i];
2359: if ((c0 == charArray[0]) && (c1 == charArray[1])
2360: && (c2 == charArray[2]))
2361: return charArray;
2362: }
2363: //---------other side---------
2364: i = -1;
2365: int max = newEntry3;
2366: while (++i <= max) {
2367: char[] charArray = table[i];
2368: if ((c0 == charArray[0]) && (c1 == charArray[1])
2369: && (c2 == charArray[2]))
2370: return charArray;
2371: }
2372: //--------add the entry-------
2373: if (++max >= InternalTableSize)
2374: max = 0;
2375: char[] r;
2376: System.arraycopy(src, start, r = new char[3], 0, 3);
2377: //newIdentCount++;
2378: return table[newEntry3 = max] = r; //(r = new char[] {c0, c1, c2});
2379: }
2380:
2381: final char[] optimizedCurrentTokenSource4() {
2382: //try to return the same char[] build only once
2383:
2384: char[] src = this .source;
2385: int start = this .startPosition;
2386: char c0, c1 = src[start + 1], c2, c3 = src[start + 3];
2387: int hash = (((c0 = src[start]) << 6) + (c2 = src[start + 2]))
2388: % TableSize;
2389: // int hash = (int) (((((long) c0) << 18) + (c1 << 12) + (c2 << 6) + c3) % TableSize);
2390: char[][] table = this .charArray_length[2][hash];
2391: int i = newEntry4;
2392: while (++i < InternalTableSize) {
2393: char[] charArray = table[i];
2394: if ((c0 == charArray[0]) && (c1 == charArray[1])
2395: && (c2 == charArray[2]) && (c3 == charArray[3]))
2396: return charArray;
2397: }
2398: //---------other side---------
2399: i = -1;
2400: int max = newEntry4;
2401: while (++i <= max) {
2402: char[] charArray = table[i];
2403: if ((c0 == charArray[0]) && (c1 == charArray[1])
2404: && (c2 == charArray[2]) && (c3 == charArray[3]))
2405: return charArray;
2406: }
2407: //--------add the entry-------
2408: if (++max >= InternalTableSize)
2409: max = 0;
2410: char[] r;
2411: System.arraycopy(src, start, r = new char[4], 0, 4);
2412: //newIdentCount++;
2413: return table[newEntry4 = max] = r; //(r = new char[] {c0, c1, c2, c3});
2414: }
2415:
2416: final char[] optimizedCurrentTokenSource5() {
2417: //try to return the same char[] build only once
2418:
2419: char[] src = this .source;
2420: int start = this .startPosition;
2421: char c0, c1 = src[start + 1], c2, c3 = src[start + 3], c4;
2422: int hash = (((c0 = src[start]) << 12)
2423: + ((c2 = src[start + 2]) << 6) + (c4 = src[start + 4]))
2424: % TableSize;
2425: // int hash = (int) (((((long) c0) << 24) + (((long) c1) << 18) + (c2 << 12) + (c3 << 6) + c4) % TableSize);
2426: char[][] table = this .charArray_length[3][hash];
2427: int i = newEntry5;
2428: while (++i < InternalTableSize) {
2429: char[] charArray = table[i];
2430: if ((c0 == charArray[0]) && (c1 == charArray[1])
2431: && (c2 == charArray[2]) && (c3 == charArray[3])
2432: && (c4 == charArray[4]))
2433: return charArray;
2434: }
2435: //---------other side---------
2436: i = -1;
2437: int max = newEntry5;
2438: while (++i <= max) {
2439: char[] charArray = table[i];
2440: if ((c0 == charArray[0]) && (c1 == charArray[1])
2441: && (c2 == charArray[2]) && (c3 == charArray[3])
2442: && (c4 == charArray[4]))
2443: return charArray;
2444: }
2445: //--------add the entry-------
2446: if (++max >= InternalTableSize)
2447: max = 0;
2448: char[] r;
2449: System.arraycopy(src, start, r = new char[5], 0, 5);
2450: //newIdentCount++;
2451: return table[newEntry5 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4});
2452: }
2453:
2454: final char[] optimizedCurrentTokenSource6() {
2455: //try to return the same char[] build only once
2456:
2457: char[] src = this .source;
2458: int start = this .startPosition;
2459: char c0, c1 = src[start + 1], c2, c3 = src[start + 3], c4, c5 = src[start + 5];
2460: int hash = (((c0 = src[start]) << 12)
2461: + ((c2 = src[start + 2]) << 6) + (c4 = src[start + 4]))
2462: % TableSize;
2463: // int hash = (int)(((((long) c0) << 32) + (((long) c1) << 24) + (((long) c2) << 18) + (c3 << 12) + (c4 << 6) + c5) % TableSize);
2464: char[][] table = this .charArray_length[4][hash];
2465: int i = newEntry6;
2466: while (++i < InternalTableSize) {
2467: char[] charArray = table[i];
2468: if ((c0 == charArray[0]) && (c1 == charArray[1])
2469: && (c2 == charArray[2]) && (c3 == charArray[3])
2470: && (c4 == charArray[4]) && (c5 == charArray[5]))
2471: return charArray;
2472: }
2473: //---------other side---------
2474: i = -1;
2475: int max = newEntry6;
2476: while (++i <= max) {
2477: char[] charArray = table[i];
2478: if ((c0 == charArray[0]) && (c1 == charArray[1])
2479: && (c2 == charArray[2]) && (c3 == charArray[3])
2480: && (c4 == charArray[4]) && (c5 == charArray[5]))
2481: return charArray;
2482: }
2483: //--------add the entry-------
2484: if (++max >= InternalTableSize)
2485: max = 0;
2486: char[] r;
2487: System.arraycopy(src, start, r = new char[6], 0, 6);
2488: //newIdentCount++;
2489: return table[newEntry6 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4, c5});
2490: }
2491:
2492: private void parseTags() {
2493: int position = 0;
2494: final int currentStartPosition = this .startPosition;
2495: final int currentLinePtr = this .linePtr;
2496: if (currentLinePtr >= 0) {
2497: position = this .lineEnds[currentLinePtr] + 1;
2498: }
2499: while (ScannerHelper.isWhitespace(this .source[position])) {
2500: position++;
2501: }
2502: if (currentStartPosition == position) {
2503: // the whole line is commented out
2504: return;
2505: }
2506: char[] s = null;
2507: int sourceEnd = this .currentPosition;
2508: int sourceStart = currentStartPosition;
2509: int sourceDelta = 0;
2510: if (this .withoutUnicodePtr != 0) {
2511: // 0 is used as a fast test flag so the real first char is in position 1
2512: System.arraycopy(this .withoutUnicodeBuffer, 1,
2513: s = new char[this .withoutUnicodePtr], 0,
2514: this .withoutUnicodePtr);
2515: sourceEnd = this .withoutUnicodePtr;
2516: sourceStart = 1;
2517: sourceDelta = currentStartPosition;
2518: } else {
2519: s = this .source;
2520: }
2521: int pos = CharOperation.indexOf(TAG_PREFIX, s, true,
2522: sourceStart, sourceEnd);
2523: if (pos != -1) {
2524: if (this .nlsTags == null) {
2525: this .nlsTags = new NLSTag[10];
2526: this .nlsTagsPtr = 0;
2527: }
2528: while (pos != -1) {
2529: int start = pos + TAG_PREFIX_LENGTH;
2530: int end = CharOperation.indexOf(TAG_POSTFIX, s, start,
2531: sourceEnd);
2532: if (end != -1) {
2533: NLSTag currentTag = null;
2534: final int currentLine = currentLinePtr + 1;
2535: try {
2536: currentTag = new NLSTag(pos + sourceDelta, end
2537: + sourceDelta, currentLine, extractInt(
2538: s, start, end));
2539: } catch (NumberFormatException e) {
2540: currentTag = new NLSTag(pos + sourceDelta, end
2541: + sourceDelta, currentLine, -1);
2542: }
2543: if (this .nlsTagsPtr == this .nlsTags.length) {
2544: // resize
2545: System
2546: .arraycopy(
2547: this .nlsTags,
2548: 0,
2549: (this .nlsTags = new NLSTag[this .nlsTagsPtr + 10]),
2550: 0, this .nlsTagsPtr);
2551: }
2552: this .nlsTags[this .nlsTagsPtr++] = currentTag;
2553: } else {
2554: end = start;
2555: }
2556: pos = CharOperation.indexOf(TAG_PREFIX, s, true, end,
2557: sourceEnd);
2558: }
2559: }
2560: }
2561:
2562: private int extractInt(char[] array, int start, int end) {
2563: int value = 0;
2564: for (int i = start; i < end; i++) {
2565: final char currentChar = array[i];
2566: int digit = 0;
2567: switch (currentChar) {
2568: case '0':
2569: digit = 0;
2570: break;
2571: case '1':
2572: digit = 1;
2573: break;
2574: case '2':
2575: digit = 2;
2576: break;
2577: case '3':
2578: digit = 3;
2579: break;
2580: case '4':
2581: digit = 4;
2582: break;
2583: case '5':
2584: digit = 5;
2585: break;
2586: case '6':
2587: digit = 6;
2588: break;
2589: case '7':
2590: digit = 7;
2591: break;
2592: case '8':
2593: digit = 8;
2594: break;
2595: case '9':
2596: digit = 9;
2597: break;
2598: default:
2599: throw new NumberFormatException();
2600: }
2601: value *= 10;
2602: if (digit < 0)
2603: throw new NumberFormatException();
2604: value += digit;
2605: }
2606: return value;
2607: }
2608:
2609: public final void pushLineSeparator() {
2610: //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2611: final int INCREMENT = 250;
2612: //currentCharacter is at position currentPosition-1
2613: // cr 000D
2614: if (this .currentCharacter == '\r') {
2615: int separatorPos = this .currentPosition - 1;
2616: if ((this .linePtr >= 0)
2617: && (this .lineEnds[this .linePtr] >= separatorPos))
2618: return;
2619: int length = this .lineEnds.length;
2620: if (++this .linePtr >= length)
2621: System.arraycopy(this .lineEnds, 0,
2622: this .lineEnds = new int[length + INCREMENT], 0,
2623: length);
2624: this .lineEnds[this .linePtr] = separatorPos;
2625: // look-ahead for merged cr+lf
2626: try {
2627: if (this .source[this .currentPosition] == '\n') {
2628: //System.out.println("look-ahead LF-" + this.currentPosition);
2629: this .lineEnds[this .linePtr] = this .currentPosition;
2630: this .currentPosition++;
2631: this .wasAcr = false;
2632: } else {
2633: this .wasAcr = true;
2634: }
2635: } catch (IndexOutOfBoundsException e) {
2636: this .wasAcr = true;
2637: }
2638: } else {
2639: // lf 000A
2640: if (this .currentCharacter == '\n') { //must merge eventual cr followed by lf
2641: if (this .wasAcr
2642: && (this .lineEnds[this .linePtr] == (this .currentPosition - 2))) {
2643: //System.out.println("merge LF-" + (this.currentPosition - 1));
2644: this .lineEnds[this .linePtr] = this .currentPosition - 1;
2645: } else {
2646: int separatorPos = this .currentPosition - 1;
2647: if ((this .linePtr >= 0)
2648: && (this .lineEnds[this .linePtr] >= separatorPos))
2649: return;
2650: int length = this .lineEnds.length;
2651: if (++this .linePtr >= length)
2652: System.arraycopy(this .lineEnds, 0,
2653: this .lineEnds = new int[length
2654: + INCREMENT], 0, length);
2655: this .lineEnds[this .linePtr] = separatorPos;
2656: }
2657: this .wasAcr = false;
2658: }
2659: }
2660: }
2661:
2662: public final void pushUnicodeLineSeparator() {
2663: // cr 000D
2664: if (this .currentCharacter == '\r') {
2665: if (this .source[this .currentPosition] == '\n') {
2666: this .wasAcr = false;
2667: } else {
2668: this .wasAcr = true;
2669: }
2670: } else {
2671: // lf 000A
2672: if (this .currentCharacter == '\n') { //must merge eventual cr followed by lf
2673: this .wasAcr = false;
2674: }
2675: }
2676: }
2677:
2678: public void recordComment(int token) {
2679: // compute position
2680: int stopPosition = this .currentPosition;
2681: switch (token) {
2682: case TokenNameCOMMENT_LINE:
2683: stopPosition = -this .lastCommentLinePosition;
2684: break;
2685: case TokenNameCOMMENT_BLOCK:
2686: stopPosition = -this .currentPosition;
2687: break;
2688: }
2689:
2690: // a new comment is recorded
2691: int length = this .commentStops.length;
2692: if (++this .commentPtr >= length) {
2693: int newLength = length + COMMENT_ARRAYS_SIZE * 10;
2694: System.arraycopy(this .commentStops, 0,
2695: this .commentStops = new int[newLength], 0, length);
2696: System.arraycopy(this .commentStarts, 0,
2697: this .commentStarts = new int[newLength], 0, length);
2698: System.arraycopy(this .commentTagStarts, 0,
2699: this .commentTagStarts = new int[newLength], 0,
2700: length);
2701: }
2702: this .commentStops[this .commentPtr] = stopPosition;
2703: this .commentStarts[this .commentPtr] = this .startPosition;
2704: }
2705:
2706: /**
2707: * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position.
2708: * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>).
2709: *
2710: * @param begin the given start position
2711: * @param end the given end position
2712: */
2713: public void resetTo(int begin, int end) {
2714: //reset the scanner to a given position where it may rescan again
2715:
2716: this .diet = false;
2717: this .initialPosition = this .startPosition = this .currentPosition = begin;
2718: if (this .source != null && this .source.length < end) {
2719: this .eofPosition = this .source.length;
2720: } else {
2721: this .eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
2722: }
2723: this .commentPtr = -1; // reset comment stack
2724: this .foundTaskCount = 0;
2725: }
2726:
2727: public final void scanEscapeCharacter()
2728: throws InvalidInputException {
2729: // the string with "\\u" is a legal string of two chars \ and u
2730://thus we use a direct access to the source (for regular cases).
2731: switch (this .currentCharacter) {
2732: case 'b':
2733: this .currentCharacter = '\b';
2734: break;
2735: case 't':
2736: this .currentCharacter = '\t';
2737: break;
2738: case 'n':
2739: this .currentCharacter = '\n';
2740: break;
2741: case 'f':
2742: this .currentCharacter = '\f';
2743: break;
2744: case 'r':
2745: this .currentCharacter = '\r';
2746: break;
2747: case '\"':
2748: this .currentCharacter = '\"';
2749: break;
2750: case '\'':
2751: this .currentCharacter = '\'';
2752: break;
2753: case '\\':
2754: this .currentCharacter = '\\';
2755: break;
2756: default:
2757: // -----------octal escape--------------
2758: // OctalDigit
2759: // OctalDigit OctalDigit
2760: // ZeroToThree OctalDigit OctalDigit
2761:
2762: int number = ScannerHelper
2763: .getNumericValue(this .currentCharacter);
2764: if (number >= 0 && number <= 7) {
2765: boolean zeroToThreeNot = number > 3;
2766: if (ScannerHelper
2767: .isDigit(this .currentCharacter = this .source[this .currentPosition++])) {
2768: int digit = ScannerHelper
2769: .getNumericValue(this .currentCharacter);
2770: if (digit >= 0 && digit <= 7) {
2771: number = (number * 8) + digit;
2772: if (ScannerHelper
2773: .isDigit(this .currentCharacter = this .source[this .currentPosition++])) {
2774: if (zeroToThreeNot) {// has read \NotZeroToThree OctalDigit Digit --> ignore last character
2775: this .currentPosition--;
2776: } else {
2777: digit = ScannerHelper
2778: .getNumericValue(this .currentCharacter);
2779: if (digit >= 0 && digit <= 7) { // has read \ZeroToThree OctalDigit OctalDigit
2780: number = (number * 8) + digit;
2781: } else {// has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
2782: this .currentPosition--;
2783: }
2784: }
2785: } else { // has read \OctalDigit NonDigit--> ignore last character
2786: this .currentPosition--;
2787: }
2788: } else { // has read \OctalDigit NonOctalDigit--> ignore last character
2789: this .currentPosition--;
2790: }
2791: } else { // has read \OctalDigit --> ignore last character
2792: this .currentPosition--;
2793: }
2794: if (number > 255)
2795: throw new InvalidInputException(INVALID_ESCAPE);
2796: this .currentCharacter = (char) number;
2797: } else
2798: throw new InvalidInputException(INVALID_ESCAPE);
2799: }
2800: }
2801:
2802: public int scanIdentifierOrKeywordWithBoundCheck() {
2803: //test keywords
2804:
2805: //first dispatch on the first char.
2806: //then the length. If there are several
2807: //keywors with the same length AND the same first char, then do another
2808: //dispatch on the second char
2809: this .useAssertAsAnIndentifier = false;
2810: this .useEnumAsAnIndentifier = false;
2811:
2812: char[] src = this .source;
2813: identLoop: {
2814: int pos;
2815: int srcLength = this .eofPosition;
2816: while (true) {
2817: if ((pos = this .currentPosition) >= srcLength) // handle the obvious case upfront
2818: break identLoop;
2819: char c = src[pos];
2820: if (c < ScannerHelper.MAX_OBVIOUS) {
2821: if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_UPPER_LETTER
2822: | ScannerHelper.C_LOWER_LETTER
2823: | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) {
2824: if (this .withoutUnicodePtr != 0) {
2825: this .currentCharacter = c;
2826: unicodeStore();
2827: }
2828: this .currentPosition++;
2829: } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) {
2830: this .currentCharacter = c;
2831: break identLoop;
2832: } else {
2833: //System.out.println("slow<=128: "+ c);
2834: while (getNextCharAsJavaIdentifierPartWithBoundCheck()) {/*empty*/
2835: }
2836: break identLoop;
2837: }
2838: } else {
2839: //System.out.println("slow>>128: "+ c);
2840: while (getNextCharAsJavaIdentifierPartWithBoundCheck()) {/*empty*/
2841: }
2842: break identLoop;
2843: }
2844: }
2845: }
2846:
2847: int index, length;
2848: char[] data;
2849: if (this .withoutUnicodePtr == 0) {
2850: //quick test on length == 1 but not on length > 12 while most identifier
2851: //have a length which is <= 12...but there are lots of identifier with
2852: //only one char....
2853: if ((length = this .currentPosition - this .startPosition) == 1) {
2854: return TokenNameIdentifier;
2855: }
2856: data = this .source;
2857: index = this .startPosition;
2858: } else {
2859: if ((length = this .withoutUnicodePtr) == 1)
2860: return TokenNameIdentifier;
2861: data = this .withoutUnicodeBuffer;
2862: index = 1;
2863: }
2864:
2865: return internalScanIdentifierOrKeyword(index, length, data);
2866: }
2867:
2868: public int scanIdentifierOrKeyword() {
2869: //test keywords
2870:
2871: //first dispatch on the first char.
2872: //then the length. If there are several
2873: //keywors with the same length AND the same first char, then do another
2874: //dispatch on the second char
2875: this .useAssertAsAnIndentifier = false;
2876: this .useEnumAsAnIndentifier = false;
2877:
2878: char[] src = this .source;
2879: identLoop: {
2880: int pos;
2881: int srcLength = this .eofPosition;
2882: while (true) {
2883: if ((pos = this .currentPosition) >= srcLength) // handle the obvious case upfront
2884: break identLoop;
2885: char c = src[pos];
2886: if (c < ScannerHelper.MAX_OBVIOUS) {
2887: if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_UPPER_LETTER
2888: | ScannerHelper.C_LOWER_LETTER
2889: | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) {
2890: if (this .withoutUnicodePtr != 0) {
2891: this .currentCharacter = c;
2892: unicodeStore();
2893: }
2894: this .currentPosition++;
2895: } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) {
2896: this .currentCharacter = c;
2897: break identLoop;
2898: } else {
2899: //System.out.println("slow<=128: "+ c);
2900: while (getNextCharAsJavaIdentifierPart()) {/*empty*/
2901: }
2902: break identLoop;
2903: }
2904: } else {
2905: //System.out.println("slow>>128: "+ c);
2906: while (getNextCharAsJavaIdentifierPart()) {/*empty*/
2907: }
2908: break identLoop;
2909: }
2910: }
2911: }
2912:
2913: int index, length;
2914: char[] data;
2915: if (this .withoutUnicodePtr == 0) {
2916: //quick test on length == 1 but not on length > 12 while most identifier
2917: //have a length which is <= 12...but there are lots of identifier with
2918: //only one char....
2919: if ((length = this .currentPosition - this .startPosition) == 1) {
2920: return TokenNameIdentifier;
2921: }
2922: data = this .source;
2923: index = this .startPosition;
2924: } else {
2925: if ((length = this .withoutUnicodePtr) == 1)
2926: return TokenNameIdentifier;
2927: data = this .withoutUnicodeBuffer;
2928: index = 1;
2929: }
2930:
2931: return internalScanIdentifierOrKeyword(index, length, data);
2932: }
2933:
2934: private int internalScanIdentifierOrKeyword(int index, int length,
2935: char[] data) {
2936: switch (data[index]) {
2937: case 'a':
2938: switch (length) {
2939: case 8: //abstract
2940: if ((data[++index] == 'b') && (data[++index] == 's')
2941: && (data[++index] == 't')
2942: && (data[++index] == 'r')
2943: && (data[++index] == 'a')
2944: && (data[++index] == 'c')
2945: && (data[++index] == 't')) {
2946: return TokenNameabstract;
2947: } else {
2948: return TokenNameIdentifier;
2949: }
2950: case 6: // assert
2951: if ((data[++index] == 's') && (data[++index] == 's')
2952: && (data[++index] == 'e')
2953: && (data[++index] == 'r')
2954: && (data[++index] == 't')) {
2955: if (this .sourceLevel >= ClassFileConstants.JDK1_4) {
2956: this .containsAssertKeyword = true;
2957: return TokenNameassert;
2958: } else {
2959: this .useAssertAsAnIndentifier = true;
2960: return TokenNameIdentifier;
2961: }
2962: } else {
2963: return TokenNameIdentifier;
2964: }
2965: default:
2966: return TokenNameIdentifier;
2967: }
2968: case 'b': //boolean break byte
2969: switch (length) {
2970: case 4:
2971: if ((data[++index] == 'y') && (data[++index] == 't')
2972: && (data[++index] == 'e'))
2973: return TokenNamebyte;
2974: else
2975: return TokenNameIdentifier;
2976: case 5:
2977: if ((data[++index] == 'r') && (data[++index] == 'e')
2978: && (data[++index] == 'a')
2979: && (data[++index] == 'k'))
2980: return TokenNamebreak;
2981: else
2982: return TokenNameIdentifier;
2983: case 7:
2984: if ((data[++index] == 'o') && (data[++index] == 'o')
2985: && (data[++index] == 'l')
2986: && (data[++index] == 'e')
2987: && (data[++index] == 'a')
2988: && (data[++index] == 'n'))
2989: return TokenNameboolean;
2990: else
2991: return TokenNameIdentifier;
2992: default:
2993: return TokenNameIdentifier;
2994: }
2995:
2996: case 'c': //case char catch const class continue
2997: switch (length) {
2998: case 4:
2999: if (data[++index] == 'a')
3000: if ((data[++index] == 's')
3001: && (data[++index] == 'e'))
3002: return TokenNamecase;
3003: else
3004: return TokenNameIdentifier;
3005: else if ((data[index] == 'h') && (data[++index] == 'a')
3006: && (data[++index] == 'r'))
3007: return TokenNamechar;
3008: else
3009: return TokenNameIdentifier;
3010: case 5:
3011: if (data[++index] == 'a')
3012: if ((data[++index] == 't')
3013: && (data[++index] == 'c')
3014: && (data[++index] == 'h'))
3015: return TokenNamecatch;
3016: else
3017: return TokenNameIdentifier;
3018: else if (data[index] == 'l')
3019: if ((data[++index] == 'a')
3020: && (data[++index] == 's')
3021: && (data[++index] == 's'))
3022: return TokenNameclass;
3023: else
3024: return TokenNameIdentifier;
3025: else if ((data[index] == 'o') && (data[++index] == 'n')
3026: && (data[++index] == 's')
3027: && (data[++index] == 't'))
3028: return TokenNameconst; //const is not used in java ???????
3029: else
3030: return TokenNameIdentifier;
3031: case 8:
3032: if ((data[++index] == 'o') && (data[++index] == 'n')
3033: && (data[++index] == 't')
3034: && (data[++index] == 'i')
3035: && (data[++index] == 'n')
3036: && (data[++index] == 'u')
3037: && (data[++index] == 'e'))
3038: return TokenNamecontinue;
3039: else
3040: return TokenNameIdentifier;
3041: default:
3042: return TokenNameIdentifier;
3043: }
3044:
3045: case 'd': //default do double
3046: switch (length) {
3047: case 2:
3048: if ((data[++index] == 'o'))
3049: return TokenNamedo;
3050: else
3051: return TokenNameIdentifier;
3052: case 6:
3053: if ((data[++index] == 'o') && (data[++index] == 'u')
3054: && (data[++index] == 'b')
3055: && (data[++index] == 'l')
3056: && (data[++index] == 'e'))
3057: return TokenNamedouble;
3058: else
3059: return TokenNameIdentifier;
3060: case 7:
3061: if ((data[++index] == 'e') && (data[++index] == 'f')
3062: && (data[++index] == 'a')
3063: && (data[++index] == 'u')
3064: && (data[++index] == 'l')
3065: && (data[++index] == 't'))
3066: return TokenNamedefault;
3067: else
3068: return TokenNameIdentifier;
3069: default:
3070: return TokenNameIdentifier;
3071: }
3072: case 'e': //else extends
3073: switch (length) {
3074: case 4:
3075: if ((data[++index] == 'l') && (data[++index] == 's')
3076: && (data[++index] == 'e'))
3077: return TokenNameelse;
3078: else if ((data[index] == 'n') && (data[++index] == 'u')
3079: && (data[++index] == 'm')) {
3080: if (this .sourceLevel >= ClassFileConstants.JDK1_5) {
3081: return TokenNameenum;
3082: } else {
3083: this .useEnumAsAnIndentifier = true;
3084: return TokenNameIdentifier;
3085: }
3086: } else {
3087: return TokenNameIdentifier;
3088: }
3089: case 7:
3090: if ((data[++index] == 'x') && (data[++index] == 't')
3091: && (data[++index] == 'e')
3092: && (data[++index] == 'n')
3093: && (data[++index] == 'd')
3094: && (data[++index] == 's'))
3095: return TokenNameextends;
3096: else
3097: return TokenNameIdentifier;
3098: default:
3099: return TokenNameIdentifier;
3100: }
3101:
3102: case 'f': //final finally float for false
3103: switch (length) {
3104: case 3:
3105: if ((data[++index] == 'o') && (data[++index] == 'r'))
3106: return TokenNamefor;
3107: else
3108: return TokenNameIdentifier;
3109: case 5:
3110: if (data[++index] == 'i')
3111: if ((data[++index] == 'n')
3112: && (data[++index] == 'a')
3113: && (data[++index] == 'l')) {
3114: return TokenNamefinal;
3115: } else
3116: return TokenNameIdentifier;
3117: else if (data[index] == 'l')
3118: if ((data[++index] == 'o')
3119: && (data[++index] == 'a')
3120: && (data[++index] == 't'))
3121: return TokenNamefloat;
3122: else
3123: return TokenNameIdentifier;
3124: else if ((data[index] == 'a') && (data[++index] == 'l')
3125: && (data[++index] == 's')
3126: && (data[++index] == 'e'))
3127: return TokenNamefalse;
3128: else
3129: return TokenNameIdentifier;
3130: case 7:
3131: if ((data[++index] == 'i') && (data[++index] == 'n')
3132: && (data[++index] == 'a')
3133: && (data[++index] == 'l')
3134: && (data[++index] == 'l')
3135: && (data[++index] == 'y'))
3136: return TokenNamefinally;
3137: else
3138: return TokenNameIdentifier;
3139:
3140: default:
3141: return TokenNameIdentifier;
3142: }
3143: case 'g': //goto
3144: if (length == 4) {
3145: if ((data[++index] == 'o') && (data[++index] == 't')
3146: && (data[++index] == 'o')) {
3147: return TokenNamegoto;
3148: }
3149: } //no goto in java are allowed, so why java removes this keyword ???
3150: return TokenNameIdentifier;
3151:
3152: case 'i': //if implements import instanceof int interface
3153: switch (length) {
3154: case 2:
3155: if (data[++index] == 'f')
3156: return TokenNameif;
3157: else
3158: return TokenNameIdentifier;
3159: case 3:
3160: if ((data[++index] == 'n') && (data[++index] == 't'))
3161: return TokenNameint;
3162: else
3163: return TokenNameIdentifier;
3164: case 6:
3165: if ((data[++index] == 'm') && (data[++index] == 'p')
3166: && (data[++index] == 'o')
3167: && (data[++index] == 'r')
3168: && (data[++index] == 't'))
3169: return TokenNameimport;
3170: else
3171: return TokenNameIdentifier;
3172: case 9:
3173: if ((data[++index] == 'n') && (data[++index] == 't')
3174: && (data[++index] == 'e')
3175: && (data[++index] == 'r')
3176: && (data[++index] == 'f')
3177: && (data[++index] == 'a')
3178: && (data[++index] == 'c')
3179: && (data[++index] == 'e'))
3180: return TokenNameinterface;
3181: else
3182: return TokenNameIdentifier;
3183: case 10:
3184: if (data[++index] == 'm')
3185: if ((data[++index] == 'p')
3186: && (data[++index] == 'l')
3187: && (data[++index] == 'e')
3188: && (data[++index] == 'm')
3189: && (data[++index] == 'e')
3190: && (data[++index] == 'n')
3191: && (data[++index] == 't')
3192: && (data[++index] == 's'))
3193: return TokenNameimplements ;
3194: else
3195: return TokenNameIdentifier;
3196: else if ((data[index] == 'n') && (data[++index] == 's')
3197: && (data[++index] == 't')
3198: && (data[++index] == 'a')
3199: && (data[++index] == 'n')
3200: && (data[++index] == 'c')
3201: && (data[++index] == 'e')
3202: && (data[++index] == 'o')
3203: && (data[++index] == 'f'))
3204: return TokenNameinstanceof ;
3205: else
3206: return TokenNameIdentifier;
3207:
3208: default:
3209: return TokenNameIdentifier;
3210: }
3211:
3212: case 'l': //long
3213: if (length == 4) {
3214: if ((data[++index] == 'o') && (data[++index] == 'n')
3215: && (data[++index] == 'g')) {
3216: return TokenNamelong;
3217: }
3218: }
3219: return TokenNameIdentifier;
3220:
3221: case 'n': //native new null
3222: switch (length) {
3223: case 3:
3224: if ((data[++index] == 'e') && (data[++index] == 'w'))
3225: return TokenNamenew;
3226: else
3227: return TokenNameIdentifier;
3228: case 4:
3229: if ((data[++index] == 'u') && (data[++index] == 'l')
3230: && (data[++index] == 'l'))
3231: return TokenNamenull;
3232: else
3233: return TokenNameIdentifier;
3234: case 6:
3235: if ((data[++index] == 'a') && (data[++index] == 't')
3236: && (data[++index] == 'i')
3237: && (data[++index] == 'v')
3238: && (data[++index] == 'e')) {
3239: return TokenNamenative;
3240: } else
3241: return TokenNameIdentifier;
3242: default:
3243: return TokenNameIdentifier;
3244: }
3245:
3246: case 'p': //package private protected public
3247: switch (length) {
3248: case 6:
3249: if ((data[++index] == 'u') && (data[++index] == 'b')
3250: && (data[++index] == 'l')
3251: && (data[++index] == 'i')
3252: && (data[++index] == 'c')) {
3253: return TokenNamepublic;
3254: } else
3255: return TokenNameIdentifier;
3256: case 7:
3257: if (data[++index] == 'a')
3258: if ((data[++index] == 'c')
3259: && (data[++index] == 'k')
3260: && (data[++index] == 'a')
3261: && (data[++index] == 'g')
3262: && (data[++index] == 'e'))
3263: return TokenNamepackage;
3264: else
3265: return TokenNameIdentifier;
3266: else if ((data[index] == 'r') && (data[++index] == 'i')
3267: && (data[++index] == 'v')
3268: && (data[++index] == 'a')
3269: && (data[++index] == 't')
3270: && (data[++index] == 'e')) {
3271: return TokenNameprivate;
3272: } else
3273: return TokenNameIdentifier;
3274: case 9:
3275: if ((data[++index] == 'r') && (data[++index] == 'o')
3276: && (data[++index] == 't')
3277: && (data[++index] == 'e')
3278: && (data[++index] == 'c')
3279: && (data[++index] == 't')
3280: && (data[++index] == 'e')
3281: && (data[++index] == 'd')) {
3282: return TokenNameprotected;
3283: } else
3284: return TokenNameIdentifier;
3285:
3286: default:
3287: return TokenNameIdentifier;
3288: }
3289:
3290: case 'r': //return
3291: if (length == 6) {
3292: if ((data[++index] == 'e') && (data[++index] == 't')
3293: && (data[++index] == 'u')
3294: && (data[++index] == 'r')
3295: && (data[++index] == 'n')) {
3296: return TokenNamereturn;
3297: }
3298: }
3299: return TokenNameIdentifier;
3300:
3301: case 's': //short static super switch synchronized strictfp
3302: switch (length) {
3303: case 5:
3304: if (data[++index] == 'h')
3305: if ((data[++index] == 'o')
3306: && (data[++index] == 'r')
3307: && (data[++index] == 't'))
3308: return TokenNameshort;
3309: else
3310: return TokenNameIdentifier;
3311: else if ((data[index] == 'u') && (data[++index] == 'p')
3312: && (data[++index] == 'e')
3313: && (data[++index] == 'r'))
3314: return TokenNamesuper ;
3315: else
3316: return TokenNameIdentifier;
3317:
3318: case 6:
3319: if (data[++index] == 't')
3320: if ((data[++index] == 'a')
3321: && (data[++index] == 't')
3322: && (data[++index] == 'i')
3323: && (data[++index] == 'c')) {
3324: return TokenNamestatic;
3325: } else
3326: return TokenNameIdentifier;
3327: else if ((data[index] == 'w') && (data[++index] == 'i')
3328: && (data[++index] == 't')
3329: && (data[++index] == 'c')
3330: && (data[++index] == 'h'))
3331: return TokenNameswitch;
3332: else
3333: return TokenNameIdentifier;
3334: case 8:
3335: if ((data[++index] == 't') && (data[++index] == 'r')
3336: && (data[++index] == 'i')
3337: && (data[++index] == 'c')
3338: && (data[++index] == 't')
3339: && (data[++index] == 'f')
3340: && (data[++index] == 'p'))
3341: return TokenNamestrictfp;
3342: else
3343: return TokenNameIdentifier;
3344: case 12:
3345: if ((data[++index] == 'y') && (data[++index] == 'n')
3346: && (data[++index] == 'c')
3347: && (data[++index] == 'h')
3348: && (data[++index] == 'r')
3349: && (data[++index] == 'o')
3350: && (data[++index] == 'n')
3351: && (data[++index] == 'i')
3352: && (data[++index] == 'z')
3353: && (data[++index] == 'e')
3354: && (data[++index] == 'd')) {
3355: return TokenNamesynchronized;
3356: } else
3357: return TokenNameIdentifier;
3358: default:
3359: return TokenNameIdentifier;
3360: }
3361:
3362: case 't': //try throw throws transient this true
3363: switch (length) {
3364: case 3:
3365: if ((data[++index] == 'r') && (data[++index] == 'y'))
3366: return TokenNametry;
3367: else
3368: return TokenNameIdentifier;
3369: case 4:
3370: if (data[++index] == 'h')
3371: if ((data[++index] == 'i')
3372: && (data[++index] == 's'))
3373: return TokenNamethis ;
3374: else
3375: return TokenNameIdentifier;
3376: else if ((data[index] == 'r') && (data[++index] == 'u')
3377: && (data[++index] == 'e'))
3378: return TokenNametrue;
3379: else
3380: return TokenNameIdentifier;
3381: case 5:
3382: if ((data[++index] == 'h') && (data[++index] == 'r')
3383: && (data[++index] == 'o')
3384: && (data[++index] == 'w'))
3385: return TokenNamethrow;
3386: else
3387: return TokenNameIdentifier;
3388: case 6:
3389: if ((data[++index] == 'h') && (data[++index] == 'r')
3390: && (data[++index] == 'o')
3391: && (data[++index] == 'w')
3392: && (data[++index] == 's'))
3393: return TokenNamethrows;
3394: else
3395: return TokenNameIdentifier;
3396: case 9:
3397: if ((data[++index] == 'r') && (data[++index] == 'a')
3398: && (data[++index] == 'n')
3399: && (data[++index] == 's')
3400: && (data[++index] == 'i')
3401: && (data[++index] == 'e')
3402: && (data[++index] == 'n')
3403: && (data[++index] == 't')) {
3404: return TokenNametransient;
3405: } else
3406: return TokenNameIdentifier;
3407:
3408: default:
3409: return TokenNameIdentifier;
3410: }
3411:
3412: case 'v': //void volatile
3413: switch (length) {
3414: case 4:
3415: if ((data[++index] == 'o') && (data[++index] == 'i')
3416: && (data[++index] == 'd'))
3417: return TokenNamevoid;
3418: else
3419: return TokenNameIdentifier;
3420: case 8:
3421: if ((data[++index] == 'o') && (data[++index] == 'l')
3422: && (data[++index] == 'a')
3423: && (data[++index] == 't')
3424: && (data[++index] == 'i')
3425: && (data[++index] == 'l')
3426: && (data[++index] == 'e')) {
3427: return TokenNamevolatile;
3428: } else
3429: return TokenNameIdentifier;
3430:
3431: default:
3432: return TokenNameIdentifier;
3433: }
3434:
3435: case 'w': //while widefp
3436: switch (length) {
3437: case 5:
3438: if ((data[++index] == 'h') && (data[++index] == 'i')
3439: && (data[++index] == 'l')
3440: && (data[++index] == 'e'))
3441: return TokenNamewhile;
3442: else
3443: return TokenNameIdentifier;
3444: //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&& (data[++index]=='p'))
3445: //return TokenNamewidefp ;
3446: //else
3447: //return TokenNameIdentifier;
3448: default:
3449: return TokenNameIdentifier;
3450: }
3451:
3452: default:
3453: return TokenNameIdentifier;
3454: }
3455: }
3456:
3457: public int scanNumber(boolean dotPrefix)
3458: throws InvalidInputException {
3459:
3460: //when entering this method the currentCharacter is the first
3461: //digit of the number. It may be preceeded by a '.' when
3462: //dotPrefix is true
3463:
3464: boolean floating = dotPrefix;
3465: if ((!dotPrefix) && (this .currentCharacter == '0')) {
3466: if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
3467: int start = this .currentPosition;
3468: while (getNextCharAsDigit(16)) {/*empty*/
3469: }
3470: int end = this .currentPosition;
3471: if (getNextChar('l', 'L') >= 0) {
3472: if (end == start) {
3473: throw new InvalidInputException(INVALID_HEXA);
3474: }
3475: return TokenNameLongLiteral;
3476: } else if (getNextChar('.')) {
3477: if (this .sourceLevel < ClassFileConstants.JDK1_5) {
3478: if (end == start) {
3479: throw new InvalidInputException(
3480: INVALID_HEXA);
3481: }
3482: this .currentPosition = end;
3483: return TokenNameIntegerLiteral;
3484: }
3485: // hexadecimal floating point literal
3486: // read decimal part
3487: boolean hasNoDigitsBeforeDot = end == start;
3488: start = this .currentPosition;
3489: while (getNextCharAsDigit(16)) {/*empty*/
3490: }
3491: end = this .currentPosition;
3492: if (hasNoDigitsBeforeDot && end == start) {
3493: throw new InvalidInputException(INVALID_HEXA);
3494: }
3495:
3496: if (getNextChar('p', 'P') >= 0) { // consume next character
3497: this .unicodeAsBackSlash = false;
3498: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3499: && (this .source[this .currentPosition] == 'u')) {
3500: getNextUnicodeChar();
3501: } else {
3502: if (this .withoutUnicodePtr != 0) {
3503: unicodeStore();
3504: }
3505: }
3506:
3507: if ((this .currentCharacter == '-')
3508: || (this .currentCharacter == '+')) { // consume next character
3509: this .unicodeAsBackSlash = false;
3510: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3511: && (this .source[this .currentPosition] == 'u')) {
3512: getNextUnicodeChar();
3513: } else {
3514: if (this .withoutUnicodePtr != 0) {
3515: unicodeStore();
3516: }
3517: }
3518: }
3519: if (!ScannerHelper
3520: .isDigit(this .currentCharacter)) {
3521: throw new InvalidInputException(
3522: INVALID_HEXA);
3523: }
3524: while (getNextCharAsDigit()) {/*empty*/
3525: }
3526: if (getNextChar('f', 'F') >= 0) {
3527: return TokenNameFloatingPointLiteral;
3528: }
3529: if (getNextChar('d', 'D') >= 0) {
3530: return TokenNameDoubleLiteral;
3531: }
3532: if (getNextChar('l', 'L') >= 0) {
3533: throw new InvalidInputException(
3534: INVALID_HEXA);
3535: }
3536: return TokenNameDoubleLiteral;
3537: } else {
3538: throw new InvalidInputException(INVALID_HEXA);
3539: }
3540: } else if (getNextChar('p', 'P') >= 0) { // consume next character
3541: if (this .sourceLevel < ClassFileConstants.JDK1_5) {
3542: // if we are in source level < 1.5 we report an integer literal
3543: this .currentPosition = end;
3544: return TokenNameIntegerLiteral;
3545: }
3546: this .unicodeAsBackSlash = false;
3547: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3548: && (this .source[this .currentPosition] == 'u')) {
3549: getNextUnicodeChar();
3550: } else {
3551: if (this .withoutUnicodePtr != 0) {
3552: unicodeStore();
3553: }
3554: }
3555:
3556: if ((this .currentCharacter == '-')
3557: || (this .currentCharacter == '+')) { // consume next character
3558: this .unicodeAsBackSlash = false;
3559: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3560: && (this .source[this .currentPosition] == 'u')) {
3561: getNextUnicodeChar();
3562: } else {
3563: if (this .withoutUnicodePtr != 0) {
3564: unicodeStore();
3565: }
3566: }
3567: }
3568: if (!ScannerHelper.isDigit(this .currentCharacter))
3569: throw new InvalidInputException(INVALID_FLOAT);
3570: while (getNextCharAsDigit()) {/*empty*/
3571: }
3572: if (getNextChar('f', 'F') >= 0)
3573: return TokenNameFloatingPointLiteral;
3574: if (getNextChar('d', 'D') >= 0)
3575: return TokenNameDoubleLiteral;
3576: if (getNextChar('l', 'L') >= 0) {
3577: throw new InvalidInputException(INVALID_HEXA);
3578: }
3579: return TokenNameDoubleLiteral;
3580: } else {
3581: if (end == start)
3582: throw new InvalidInputException(INVALID_HEXA);
3583: return TokenNameIntegerLiteral;
3584: }
3585: }
3586:
3587: //there is x or X in the number
3588: //potential octal ! ... some one may write 000099.0 ! thus 00100 < 00078.0 is true !!!!! crazy language
3589: if (getNextCharAsDigit()) { //-------------potential octal-----------------
3590: while (getNextCharAsDigit()) {/*empty*/
3591: }
3592:
3593: if (getNextChar('l', 'L') >= 0) {
3594: return TokenNameLongLiteral;
3595: }
3596:
3597: if (getNextChar('f', 'F') >= 0) {
3598: return TokenNameFloatingPointLiteral;
3599: }
3600:
3601: if (getNextChar('d', 'D') >= 0) {
3602: return TokenNameDoubleLiteral;
3603: } else { //make the distinction between octal and float ....
3604: boolean isInteger = true;
3605: if (getNextChar('.')) {
3606: isInteger = false;
3607: while (getNextCharAsDigit()) {/*empty*/
3608: }
3609: }
3610: if (getNextChar('e', 'E') >= 0) { // consume next character
3611: isInteger = false;
3612: this .unicodeAsBackSlash = false;
3613: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3614: && (this .source[this .currentPosition] == 'u')) {
3615: getNextUnicodeChar();
3616: } else {
3617: if (this .withoutUnicodePtr != 0) {
3618: unicodeStore();
3619: }
3620: }
3621:
3622: if ((this .currentCharacter == '-')
3623: || (this .currentCharacter == '+')) { // consume next character
3624: this .unicodeAsBackSlash = false;
3625: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3626: && (this .source[this .currentPosition] == 'u')) {
3627: getNextUnicodeChar();
3628: } else {
3629: if (this .withoutUnicodePtr != 0) {
3630: unicodeStore();
3631: }
3632: }
3633: }
3634: if (!ScannerHelper
3635: .isDigit(this .currentCharacter))
3636: throw new InvalidInputException(
3637: INVALID_FLOAT);
3638: while (getNextCharAsDigit()) {/*empty*/
3639: }
3640: }
3641: if (getNextChar('f', 'F') >= 0)
3642: return TokenNameFloatingPointLiteral;
3643: if (getNextChar('d', 'D') >= 0 || !isInteger)
3644: return TokenNameDoubleLiteral;
3645: return TokenNameIntegerLiteral;
3646: }
3647: } else {
3648: /* carry on */
3649: }
3650: }
3651:
3652: while (getNextCharAsDigit()) {/*empty*/
3653: }
3654:
3655: if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3656: return TokenNameLongLiteral;
3657:
3658: if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
3659: while (getNextCharAsDigit()) {/*empty*/
3660: }
3661: floating = true;
3662: }
3663:
3664: //if floating is true both exponant and suffix may be optional
3665:
3666: if (getNextChar('e', 'E') >= 0) {
3667: floating = true;
3668: // consume next character
3669: this .unicodeAsBackSlash = false;
3670: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3671: && (this .source[this .currentPosition] == 'u')) {
3672: getNextUnicodeChar();
3673: } else {
3674: if (this .withoutUnicodePtr != 0) {
3675: unicodeStore();
3676: }
3677: }
3678:
3679: if ((this .currentCharacter == '-')
3680: || (this .currentCharacter == '+')) { // consume next character
3681: this .unicodeAsBackSlash = false;
3682: if (((this .currentCharacter = this .source[this .currentPosition++]) == '\\')
3683: && (this .source[this .currentPosition] == 'u')) {
3684: getNextUnicodeChar();
3685: } else {
3686: if (this .withoutUnicodePtr != 0) {
3687: unicodeStore();
3688: }
3689: }
3690: }
3691: if (!ScannerHelper.isDigit(this .currentCharacter))
3692: throw new InvalidInputException(INVALID_FLOAT);
3693: while (getNextCharAsDigit()) {/*empty*/
3694: }
3695: }
3696:
3697: if (getNextChar('d', 'D') >= 0)
3698: return TokenNameDoubleLiteral;
3699: if (getNextChar('f', 'F') >= 0)
3700: return TokenNameFloatingPointLiteral;
3701:
3702: //the long flag has been tested before
3703:
3704: return floating ? TokenNameDoubleLiteral
3705: : TokenNameIntegerLiteral;
3706: }
3707:
3708: /**
3709: * Search the line number corresponding to a specific position
3710: * @param position int
3711: * @return int
3712: */
3713: public final int getLineNumber(int position) {
3714: return Util.getLineNumber(position, this .lineEnds, 0,
3715: this .linePtr);
3716: }
3717:
3718: public final void setSource(char[] sourceString) {
3719: //the source-buffer is set to sourceString
3720:
3721: int sourceLength;
3722: if (sourceString == null) {
3723: this .source = CharOperation.NO_CHAR;
3724: sourceLength = 0;
3725: } else {
3726: this .source = sourceString;
3727: sourceLength = sourceString.length;
3728: }
3729: this .startPosition = -1;
3730: this .eofPosition = sourceLength;
3731: this .initialPosition = this .currentPosition = 0;
3732: this .containsAssertKeyword = false;
3733: this .linePtr = -1;
3734: }
3735:
3736: /*
3737: * Should be used if a parse (usually a diet parse) has already been performed on the unit,
3738: * so as to get the already computed line end positions.
3739: */
3740: public final void setSource(char[] contents,
3741: CompilationResult compilationResult) {
3742: if (contents == null) {
3743: char[] cuContents = compilationResult.compilationUnit
3744: .getContents();
3745: setSource(cuContents);
3746: } else {
3747: setSource(contents);
3748: }
3749: int[] lineSeparatorPositions = compilationResult.lineSeparatorPositions;
3750: if (lineSeparatorPositions != null) {
3751: this .lineEnds = lineSeparatorPositions;
3752: this .linePtr = lineSeparatorPositions.length - 1;
3753: }
3754: }
3755:
3756: /*
3757: * Should be used if a parse (usually a diet parse) has already been performed on the unit,
3758: * so as to get the already computed line end positions.
3759: */
3760: public final void setSource(CompilationResult compilationResult) {
3761: setSource(null, compilationResult);
3762: }
3763:
3764: public String toString() {
3765: if (this .startPosition == this .eofPosition)
3766: return "EOF\n\n" + new String(this .source); //$NON-NLS-1$
3767: if (this .currentPosition > this .eofPosition)
3768: return "behind the EOF\n\n" + new String(this .source); //$NON-NLS-1$
3769:
3770: char front[] = new char[this .startPosition];
3771: System.arraycopy(this .source, 0, front, 0, this .startPosition);
3772:
3773: int middleLength = (this .currentPosition - 1)
3774: - this .startPosition + 1;
3775: char middle[];
3776: if (middleLength > -1) {
3777: middle = new char[middleLength];
3778: System.arraycopy(this .source, this .startPosition, middle,
3779: 0, middleLength);
3780: } else {
3781: middle = CharOperation.NO_CHAR;
3782: }
3783:
3784: char end[] = new char[this .eofPosition
3785: - (this .currentPosition - 1)];
3786: System.arraycopy(this .source, (this .currentPosition - 1) + 1,
3787: end, 0, this .eofPosition - (this .currentPosition - 1)
3788: - 1);
3789:
3790: return new String(front)
3791: + "\n===============================\nStarts here -->" //$NON-NLS-1$
3792: + new String(middle)
3793: + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3794: + new String(end);
3795: }
3796:
3797: public String toStringAction(int act) {
3798: switch (act) {
3799: case TokenNameIdentifier:
3800: return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3801: case TokenNameabstract:
3802: return "abstract"; //$NON-NLS-1$
3803: case TokenNameboolean:
3804: return "boolean"; //$NON-NLS-1$
3805: case TokenNamebreak:
3806: return "break"; //$NON-NLS-1$
3807: case TokenNamebyte:
3808: return "byte"; //$NON-NLS-1$
3809: case TokenNamecase:
3810: return "case"; //$NON-NLS-1$
3811: case TokenNamecatch:
3812: return "catch"; //$NON-NLS-1$
3813: case TokenNamechar:
3814: return "char"; //$NON-NLS-1$
3815: case TokenNameclass:
3816: return "class"; //$NON-NLS-1$
3817: case TokenNamecontinue:
3818: return "continue"; //$NON-NLS-1$
3819: case TokenNamedefault:
3820: return "default"; //$NON-NLS-1$
3821: case TokenNamedo:
3822: return "do"; //$NON-NLS-1$
3823: case TokenNamedouble:
3824: return "double"; //$NON-NLS-1$
3825: case TokenNameelse:
3826: return "else"; //$NON-NLS-1$
3827: case TokenNameextends:
3828: return "extends"; //$NON-NLS-1$
3829: case TokenNamefalse:
3830: return "false"; //$NON-NLS-1$
3831: case TokenNamefinal:
3832: return "final"; //$NON-NLS-1$
3833: case TokenNamefinally:
3834: return "finally"; //$NON-NLS-1$
3835: case TokenNamefloat:
3836: return "float"; //$NON-NLS-1$
3837: case TokenNamefor:
3838: return "for"; //$NON-NLS-1$
3839: case TokenNameif:
3840: return "if"; //$NON-NLS-1$
3841: case TokenNameimplements :
3842: return "implements"; //$NON-NLS-1$
3843: case TokenNameimport:
3844: return "import"; //$NON-NLS-1$
3845: case TokenNameinstanceof :
3846: return "instanceof"; //$NON-NLS-1$
3847: case TokenNameint:
3848: return "int"; //$NON-NLS-1$
3849: case TokenNameinterface:
3850: return "interface"; //$NON-NLS-1$
3851: case TokenNamelong:
3852: return "long"; //$NON-NLS-1$
3853: case TokenNamenative:
3854: return "native"; //$NON-NLS-1$
3855: case TokenNamenew:
3856: return "new"; //$NON-NLS-1$
3857: case TokenNamenull:
3858: return "null"; //$NON-NLS-1$
3859: case TokenNamepackage:
3860: return "package"; //$NON-NLS-1$
3861: case TokenNameprivate:
3862: return "private"; //$NON-NLS-1$
3863: case TokenNameprotected:
3864: return "protected"; //$NON-NLS-1$
3865: case TokenNamepublic:
3866: return "public"; //$NON-NLS-1$
3867: case TokenNamereturn:
3868: return "return"; //$NON-NLS-1$
3869: case TokenNameshort:
3870: return "short"; //$NON-NLS-1$
3871: case TokenNamestatic:
3872: return "static"; //$NON-NLS-1$
3873: case TokenNamesuper :
3874: return "super"; //$NON-NLS-1$
3875: case TokenNameswitch:
3876: return "switch"; //$NON-NLS-1$
3877: case TokenNamesynchronized:
3878: return "synchronized"; //$NON-NLS-1$
3879: case TokenNamethis :
3880: return "this"; //$NON-NLS-1$
3881: case TokenNamethrow:
3882: return "throw"; //$NON-NLS-1$
3883: case TokenNamethrows:
3884: return "throws"; //$NON-NLS-1$
3885: case TokenNametransient:
3886: return "transient"; //$NON-NLS-1$
3887: case TokenNametrue:
3888: return "true"; //$NON-NLS-1$
3889: case TokenNametry:
3890: return "try"; //$NON-NLS-1$
3891: case TokenNamevoid:
3892: return "void"; //$NON-NLS-1$
3893: case TokenNamevolatile:
3894: return "volatile"; //$NON-NLS-1$
3895: case TokenNamewhile:
3896: return "while"; //$NON-NLS-1$
3897:
3898: case TokenNameIntegerLiteral:
3899: return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3900: case TokenNameLongLiteral:
3901: return "Long(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3902: case TokenNameFloatingPointLiteral:
3903: return "Float(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3904: case TokenNameDoubleLiteral:
3905: return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3906: case TokenNameCharacterLiteral:
3907: return "Char(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3908: case TokenNameStringLiteral:
3909: return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3910:
3911: case TokenNamePLUS_PLUS:
3912: return "++"; //$NON-NLS-1$
3913: case TokenNameMINUS_MINUS:
3914: return "--"; //$NON-NLS-1$
3915: case TokenNameEQUAL_EQUAL:
3916: return "=="; //$NON-NLS-1$
3917: case TokenNameLESS_EQUAL:
3918: return "<="; //$NON-NLS-1$
3919: case TokenNameGREATER_EQUAL:
3920: return ">="; //$NON-NLS-1$
3921: case TokenNameNOT_EQUAL:
3922: return "!="; //$NON-NLS-1$
3923: case TokenNameLEFT_SHIFT:
3924: return "<<"; //$NON-NLS-1$
3925: case TokenNameRIGHT_SHIFT:
3926: return ">>"; //$NON-NLS-1$
3927: case TokenNameUNSIGNED_RIGHT_SHIFT:
3928: return ">>>"; //$NON-NLS-1$
3929: case TokenNamePLUS_EQUAL:
3930: return "+="; //$NON-NLS-1$
3931: case TokenNameMINUS_EQUAL:
3932: return "-="; //$NON-NLS-1$
3933: case TokenNameMULTIPLY_EQUAL:
3934: return "*="; //$NON-NLS-1$
3935: case TokenNameDIVIDE_EQUAL:
3936: return "/="; //$NON-NLS-1$
3937: case TokenNameAND_EQUAL:
3938: return "&="; //$NON-NLS-1$
3939: case TokenNameOR_EQUAL:
3940: return "|="; //$NON-NLS-1$
3941: case TokenNameXOR_EQUAL:
3942: return "^="; //$NON-NLS-1$
3943: case TokenNameREMAINDER_EQUAL:
3944: return "%="; //$NON-NLS-1$
3945: case TokenNameLEFT_SHIFT_EQUAL:
3946: return "<<="; //$NON-NLS-1$
3947: case TokenNameRIGHT_SHIFT_EQUAL:
3948: return ">>="; //$NON-NLS-1$
3949: case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL:
3950: return ">>>="; //$NON-NLS-1$
3951: case TokenNameOR_OR:
3952: return "||"; //$NON-NLS-1$
3953: case TokenNameAND_AND:
3954: return "&&"; //$NON-NLS-1$
3955: case TokenNamePLUS:
3956: return "+"; //$NON-NLS-1$
3957: case TokenNameMINUS:
3958: return "-"; //$NON-NLS-1$
3959: case TokenNameNOT:
3960: return "!"; //$NON-NLS-1$
3961: case TokenNameREMAINDER:
3962: return "%"; //$NON-NLS-1$
3963: case TokenNameXOR:
3964: return "^"; //$NON-NLS-1$
3965: case TokenNameAND:
3966: return "&"; //$NON-NLS-1$
3967: case TokenNameMULTIPLY:
3968: return "*"; //$NON-NLS-1$
3969: case TokenNameOR:
3970: return "|"; //$NON-NLS-1$
3971: case TokenNameTWIDDLE:
3972: return "~"; //$NON-NLS-1$
3973: case TokenNameDIVIDE:
3974: return "/"; //$NON-NLS-1$
3975: case TokenNameGREATER:
3976: return ">"; //$NON-NLS-1$
3977: case TokenNameLESS:
3978: return "<"; //$NON-NLS-1$
3979: case TokenNameLPAREN:
3980: return "("; //$NON-NLS-1$
3981: case TokenNameRPAREN:
3982: return ")"; //$NON-NLS-1$
3983: case TokenNameLBRACE:
3984: return "{"; //$NON-NLS-1$
3985: case TokenNameRBRACE:
3986: return "}"; //$NON-NLS-1$
3987: case TokenNameLBRACKET:
3988: return "["; //$NON-NLS-1$
3989: case TokenNameRBRACKET:
3990: return "]"; //$NON-NLS-1$
3991: case TokenNameSEMICOLON:
3992: return ";"; //$NON-NLS-1$
3993: case TokenNameQUESTION:
3994: return "?"; //$NON-NLS-1$
3995: case TokenNameCOLON:
3996: return ":"; //$NON-NLS-1$
3997: case TokenNameCOMMA:
3998: return ","; //$NON-NLS-1$
3999: case TokenNameDOT:
4000: return "."; //$NON-NLS-1$
4001: case TokenNameEQUAL:
4002: return "="; //$NON-NLS-1$
4003: case TokenNameEOF:
4004: return "EOF"; //$NON-NLS-1$
4005: case TokenNameWHITESPACE:
4006: return "white_space(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4007: default:
4008: return "not-a-token"; //$NON-NLS-1$
4009: }
4010: }
4011:
4012: public void unicodeInitializeBuffer(int length) {
4013: this .withoutUnicodePtr = length;
4014: if (this .withoutUnicodeBuffer == null)
4015: this .withoutUnicodeBuffer = new char[length + (1 + 10)];
4016: int bLength = this .withoutUnicodeBuffer.length;
4017: if (1 + length >= bLength) {
4018: System.arraycopy(this .withoutUnicodeBuffer, 0,
4019: this .withoutUnicodeBuffer = new char[length
4020: + (1 + 10)], 0, bLength);
4021: }
4022: System.arraycopy(this .source, this .startPosition,
4023: this .withoutUnicodeBuffer, 1, length);
4024: }
4025:
4026: public void unicodeStore() {
4027: int pos = ++this .withoutUnicodePtr;
4028: if (this .withoutUnicodeBuffer == null)
4029: this .withoutUnicodeBuffer = new char[10];
4030: int length = this .withoutUnicodeBuffer.length;
4031: if (pos == length) {
4032: System.arraycopy(this .withoutUnicodeBuffer, 0,
4033: this .withoutUnicodeBuffer = new char[length * 2],
4034: 0, length);
4035: }
4036: this.withoutUnicodeBuffer[pos] = this.currentCharacter;
4037: }
4038: }
|