0001: /*
0002: * TestLargeSource.java: JUnit test for the StandardTokenizer
0003: *
0004: * Copyright (C) 2003 Heiko Blau
0005: *
0006: * This file belongs to the JTopas test suite.
0007: * The JTopas test suite is free software; you can redistribute it and/or modify it
0008: * under the terms of the GNU Lesser General Public License as published by the
0009: * Free Software Foundation; either version 2.1 of the License, or (at your option)
0010: * any later version.
0011: *
0012: * This software is distributed in the hope that it will be useful, but WITHOUT
0013: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
0014: * FITNESS FOR A PARTICULAR PURPOSE.
0015: * See the GNU Lesser General Public License for more details.
0016: *
0017: * You should have received a copy of the GNU Lesser General Public License along
0018: * with the JTopas test suite. If not, write to the
0019: *
0020: * Free Software Foundation, Inc.
0021: * 59 Temple Place, Suite 330,
0022: * Boston, MA 02111-1307
0023: * USA
0024: *
0025: * or check the Internet: http://www.fsf.org
0026: *
0027: * The JTopas test suite uses the test framework JUnit by Kent Beck and Erich Gamma.
0028: * You should have received a copy of their JUnit licence agreement along with
0029: * the JTopas test suite.
0030: *
0031: * We do NOT provide the JUnit archive junit.jar nessecary to compile and run
0032: * our tests, since we assume, that You either have it already or would like
0033: * to get the current release Yourself.
0034: * Please visit either:
0035: * http://sourceforge.net/projects/junit
0036: * or
0037: * http://junit.org
0038: * to obtain JUnit.
0039: *
0040: * Contact:
0041: * email: heiko@susebox.de
0042: */
0043:
0044: package de.susebox.jtopas;
0045:
0046: //-----------------------------------------------------------------------------
0047: // Imports
0048: //
0049: import java.util.Iterator;
0050: import java.text.MessageFormat;
0051:
0052: import java.io.File;
0053: import java.io.FileReader;
0054: import java.io.FileOutputStream;
0055: import java.io.FileInputStream;
0056: import java.io.OutputStreamWriter;
0057: import java.io.InputStreamReader;
0058: import java.io.IOException;
0059:
0060: import junit.framework.Test;
0061: import junit.framework.TestCase;
0062: import junit.framework.TestSuite;
0063: import junit.framework.Assert;
0064:
0065: import de.susebox.jtopas.Token;
0066: import de.susebox.jtopas.Tokenizer;
0067: import de.susebox.jtopas.StandardTokenizer;
0068: import de.susebox.jtopas.StandardTokenizerProperties;
0069: import de.susebox.jtopas.TokenizerProperty;
0070: import de.susebox.jtopas.TokenizerException;
0071:
0072: import de.susebox.TestUtilities;
0073:
0074: //-----------------------------------------------------------------------------
0075: // Class TestLargeSource
0076: //
0077:
0078: /**<p>
0079: * This test suite generates a huge file with a common mix of comments, special
0080: * sequences, keywords, separators etc. It is mainly designed for profiling
0081: * sessions to find the hot spots.
0082: *</p>
0083: *
0084: * @author Heiko Blau
0085: */
0086: public class TestLargeSource extends TestCase {
0087:
0088: //---------------------------------------------------------------------------
0089: // properties
0090: //
0091:
0092: //---------------------------------------------------------------------------
0093: // main method
0094: //
0095:
0096: /**
0097: * call this method to invoke the tests
0098: */
0099: public static void main(String[] args) {
0100: String[] tests = { TestLargeSource.class.getName() };
0101:
0102: TestUtilities.run(tests, args);
0103: }
0104:
0105: //---------------------------------------------------------------------------
0106: // suite method
0107: //
0108:
0109: /**
0110: * Implementation of the JUnit method <code>suite</code>. This one creates
0111: * a large source file java style.
0112: *
0113: * @return a test suite
0114: */
0115: public static Test suite() {
0116: // Can we run Java 1.4 test cases?
0117: boolean charSequenceAvailable;
0118:
0119: try {
0120: Class.forName("java.lang.CharSequence"); // Java 1.4 or higher?
0121: charSequenceAvailable = true;
0122: } catch (Throwable throwable) {
0123: charSequenceAvailable = false;
0124: }
0125:
0126: // Construct the test suite
0127: TestSuite suite = new TestSuite(TestLargeSource.class.getName());
0128: int[] flags = {
0129: Flags.F_RETURN_WHITESPACES | Flags.F_TOKEN_POS_ONLY
0130: | Flags.F_COUNT_LINES,
0131: Flags.F_RETURN_WHITESPACES | Flags.F_TOKEN_POS_ONLY,
0132: Flags.F_RETURN_BLOCK_COMMENTS
0133: | Flags.F_RETURN_LINE_COMMENTS
0134: | Flags.F_TOKEN_POS_ONLY,
0135: Flags.F_RETURN_BLOCK_COMMENTS
0136: | Flags.F_RETURN_LINE_COMMENTS,
0137: Flags.F_RETURN_WHITESPACES,
0138: Flags.F_RETURN_WHITESPACES | Flags.F_COUNT_LINES,
0139: Flags.F_RETURN_WHITESPACES | Flags.F_TOKEN_POS_ONLY
0140: | Flags.F_KEEP_DATA,
0141: Flags.F_RETURN_WHITESPACES | Flags.F_KEEP_DATA,
0142: Flags.F_RETURN_BLOCK_COMMENTS
0143: | Flags.F_RETURN_LINE_COMMENTS
0144: | Flags.F_KEEP_DATA };
0145:
0146: long[] types = {
0147: (1 << Token.PATTERN) + (1 << Token.KEYWORD)
0148: + (1 << Token.SPECIAL_SEQUENCE)
0149: + (1 << Token.BLOCK_COMMENT)
0150: + (1 << Token.LINE_COMMENT)
0151: + (1 << Token.STRING),
0152: (1 << Token.KEYWORD) + (1 << Token.SPECIAL_SEQUENCE)
0153: + (1 << Token.BLOCK_COMMENT)
0154: + (1 << Token.LINE_COMMENT)
0155: + (1 << Token.STRING),
0156: (1 << Token.SPECIAL_SEQUENCE)
0157: + (1 << Token.BLOCK_COMMENT)
0158: + (1 << Token.LINE_COMMENT)
0159: + (1 << Token.STRING) };
0160:
0161: for (int flagsIndex = 0; flagsIndex < flags.length; ++flagsIndex) {
0162: for (int typesIndex = 0; typesIndex < types.length; ++typesIndex) {
0163: suite.addTest(new TestLargeSource("parseFile",
0164: flags[flagsIndex], types[typesIndex]));
0165: if (charSequenceAvailable) {
0166: suite.addTest(new TestLargeSource(
0167: "parseCharSequence", flags[flagsIndex],
0168: types[typesIndex]));
0169: }
0170: }
0171: }
0172:
0173: return suite;
0174: }
0175:
0176: //---------------------------------------------------------------------------
0177: // Constructor
0178: //
0179:
0180: /**
0181: * Default constructor.
0182: */
0183: public TestLargeSource(String test, int flags, long typeMask) {
0184: super (test);
0185: _flags = flags;
0186: _typeMask = typeMask;
0187: }
0188:
0189: //---------------------------------------------------------------------------
0190: // Fixture setup and release
0191: //
0192:
0193: /**
0194: * Sets up the fixture, for example, open a network connection.
0195: * This method is called before a test is executed.
0196: */
0197: protected void setUp() throws Exception {
0198: // create a temporary file that is removed on exit
0199: OutputStreamWriter writer;
0200:
0201: _smallFile = File.createTempFile("jtopas", null);
0202: _smallFile.deleteOnExit();
0203: _largeFile = File.createTempFile("jtopas", null);
0204: _largeFile.deleteOnExit();
0205:
0206: writer = new OutputStreamWriter(
0207: new FileOutputStream(_smallFile));
0208: for (int count = 0; count < SMALL_LOOPS; ++count) {
0209: writer.write(CODE_PIECE);
0210: }
0211: writer.close();
0212:
0213: writer = new OutputStreamWriter(
0214: new FileOutputStream(_largeFile));
0215: for (int count = 0; count < SMALL_LOOPS * LARGE_SMALL_RATIO; ++count) {
0216: writer.write(CODE_PIECE);
0217: }
0218: writer.close();
0219:
0220: // initializing the tokenizer properties
0221: _properties = new StandardTokenizerProperties(_flags);
0222:
0223: for (int index = 0; index < _javaProperties.length; ++index) {
0224: if ((_typeMask & (1 << _javaProperties[index].getType())) != 0) {
0225: _properties.addProperty(_javaProperties[index]);
0226: }
0227: }
0228: }
0229:
0230: /**
0231: * Tears down the fixture, for example, close a network connection.
0232: * This method is called after a test is executed.
0233: */
0234: protected void tearDown() throws Exception {
0235: }
0236:
0237: //---------------------------------------------------------------------------
0238: // test cases
0239: //
0240:
0241: /**
0242: * Tokenizing a large source
0243: */
0244: public void parseFile() throws Throwable {
0245: long smallTime = tokenizeFile(MessageFormat.format(
0246: TEST_MESSAGE, new Object[] { "parseFile",
0247: flags2String(), types2String(),
0248: Long.toString(_smallFile.length()) }),
0249: _smallFile, true);
0250: long largeTime = tokenizeFile(MessageFormat.format(
0251: TEST_MESSAGE, new Object[] { "parseFile",
0252: flags2String(), types2String(),
0253: Long.toString(_largeFile.length()) }),
0254: _largeFile, true);
0255:
0256: compareTime(smallTime, largeTime);
0257: }
0258:
0259: /**
0260: * Tokenizing a large source through a {@link CharSequenceTokenizerSource}.
0261: */
0262: public void parseCharSequence() throws Throwable {
0263: long smallTime = tokenizeFile(MessageFormat.format(
0264: TEST_MESSAGE, new Object[] { "parseCharSequence",
0265: flags2String(), types2String(),
0266: Long.toString(_smallFile.length()) }),
0267: _smallFile, false);
0268: long largeTime = tokenizeFile(MessageFormat.format(
0269: TEST_MESSAGE, new Object[] { "parseCharSequence",
0270: flags2String(), types2String(),
0271: Long.toString(_largeFile.length()) }),
0272: _largeFile, false);
0273:
0274: compareTime(smallTime, largeTime);
0275: }
0276:
0277: //---------------------------------------------------------------------------
0278: // implementation
0279: //
0280:
0281: /**
0282: * Removing a {@link TokenizerProperty} class.
0283: */
0284: private void removeProperties(int propertyType) {
0285: Iterator iter = _properties.getProperties();
0286:
0287: while (iter.hasNext()) {
0288: TokenizerProperty prop = (TokenizerProperty) iter.next();
0289:
0290: if (prop.getType() == propertyType) {
0291: iter.remove();
0292: }
0293: }
0294: }
0295:
0296: /**
0297: * Creating and tokenizing the source.
0298: */
0299: private long tokenizeFile(String message, File file,
0300: boolean useReaderSource) throws Throwable {
0301: System.out.println(message);
0302:
0303: TokenizerSource source = null;
0304: StandardTokenizer tokenizer = new StandardTokenizer(_properties);
0305:
0306: try {
0307: if (useReaderSource) {
0308: source = new ReaderSource(file);
0309: } else {
0310: source = new CharArraySource(readFile(file));
0311: }
0312: tokenizer.setSource(source);
0313: return tokenize(tokenizer);
0314: } finally {
0315: if (useReaderSource && source != null) {
0316: ((ReaderSource) source).close();
0317: }
0318: tokenizer.close();
0319: }
0320: }
0321:
0322: /**
0323: * Inner method for tokenizing
0324: */
0325: private long tokenize(Tokenizer tokenizer) throws Throwable {
0326: double[] times = { -1, 0, -1 }; // min, avg, max
0327: int index = 0;
0328: int count = 0;
0329: boolean hasPattern = tokenizer.getTokenizerProperties()
0330: .getPatterns().hasNext();
0331: boolean hasKeywords = tokenizer.getTokenizerProperties()
0332: .getKeywords().hasNext();
0333: int[] expected = hasPattern ? EXPECTED_TOKEN
0334: : hasKeywords ? EXPECTED_TOKEN_WITHOUT_PATTERN
0335: : EXPECTED_TOKEN_WITHOUT_PATTERN_AND_KEYWORDS;
0336:
0337: long diffTime = 0;
0338: long start = System.currentTimeMillis();
0339: long localStart = start;
0340:
0341: // tokenizer loop
0342: while (tokenizer.hasMoreToken()) {
0343: Token token = tokenizer.nextToken();
0344: int type = token.getType();
0345:
0346: switch (type) {
0347: case Token.WHITESPACE:
0348: case Token.EOF:
0349: break;
0350: default:
0351: // use if instead of assertTrue alone for performance reasons (no string
0352: // concatenation nessecary)
0353: // System.out.println(tokenizer.currentImage());
0354: if (expected[index] != type) {
0355: assertTrue("Line/Column " + token.getStartLine()
0356: + "/" + token.getStartColumn()
0357: + ": Expected "
0358: + Token.getTypeName(expected[index])
0359: + ", got " + Token.getTypeName(type)
0360: + ".\n" + tokenizer.currentImage(), false);
0361: }
0362: if (++index >= expected.length) {
0363: long localEnd = System.currentTimeMillis();
0364:
0365: diffTime = localEnd - localStart;
0366: localStart = localEnd;
0367:
0368: // Minimum time
0369: if (times[0] < 0 || times[0] > diffTime) {
0370: times[0] = diffTime;
0371: }
0372:
0373: // Max. time
0374: if (times[2] < 0 || times[2] < diffTime) {
0375: times[2] = diffTime;
0376: }
0377:
0378: // Prepare average time
0379: times[1] += diffTime;
0380: count++;
0381:
0382: // System.out.println("Next " + index + " token processed after " + diffTime + " milliseconds.");
0383: index = 0;
0384: }
0385: }
0386: }
0387:
0388: // ready
0389: diffTime = System.currentTimeMillis() - start;
0390: times[1] = times[1] / (double) count;
0391: System.out.println(" Finished after " + diffTime
0392: + " milliseconds.");
0393: System.out.println(" Min/avg/max tokenize time for "
0394: + expected.length + " token: " + times[0] + "/"
0395: + times[1] + "/" + times[2] + " milliseconds.");
0396:
0397: return diffTime;
0398: }
0399:
0400: //---------------------------------------------------------------------------
0401: // utility methods
0402: //
0403:
0404: /**
0405: * Compare the time for the small to the time for the large file
0406: *
0407: * @param smallTime time to parse the small file
0408: * @param largeTime time to parse the large file
0409: */
0410: private void compareTime(double smallTime, double largeTime) {
0411: if (largeTime / (smallTime + 1) > LARGE_SMALL_RATIO) {
0412: System.out
0413: .println("Tokenizer too slow. Time for small / large file "
0414: + smallTime
0415: + "/"
0416: + largeTime
0417: + "ms. Exceeding limit ratio of "
0418: + LARGE_SMALL_RATIO + ".");
0419: }
0420: }
0421:
0422: /**
0423: * Read a file into a character buffer.
0424: *
0425: * @param file the file to read
0426: * @return the buffer with the read characters
0427: */
0428: private char[] readFile(File file) throws Throwable {
0429: char[] cbuf = new char[(int) file.length()];
0430: int chars = 0;
0431: FileReader reader = new FileReader(file);
0432:
0433: try {
0434: while (chars < cbuf.length) {
0435: int read = reader
0436: .read(cbuf, chars, cbuf.length - chars);
0437:
0438: if (read < 0) {
0439: throw new IOException("Unexpected EOF after "
0440: + chars + " characters. Expected "
0441: + cbuf.length + ".");
0442: }
0443: chars += read;
0444: }
0445: } finally {
0446: try {
0447: reader.close();
0448: } catch (IOException ex) {
0449: }
0450: }
0451: return cbuf;
0452: }
0453:
0454: /**
0455: * Returns a string representation for the used flags
0456: */
0457: private String flags2String() {
0458: StringBuffer buffer = new StringBuffer();
0459:
0460: if ((_flags & Flags.F_KEEP_DATA) != 0) {
0461: buffer.append("F_KEEP_DATA");
0462: }
0463:
0464: if ((_flags & Flags.F_RETURN_WHITESPACES) == Flags.F_RETURN_WHITESPACES) {
0465: if (buffer.length() > 0) {
0466: buffer.append(" + ");
0467: }
0468: buffer.append("F_RETURN_WHITESPACES");
0469: } else {
0470: if ((_flags & Flags.F_RETURN_BLOCK_COMMENTS) != 0) {
0471: if (buffer.length() > 0) {
0472: buffer.append(" + ");
0473: }
0474: buffer.append("F_RETURN_BLOCK_COMMENTS");
0475: }
0476: if ((_flags & Flags.F_RETURN_LINE_COMMENTS) != 0) {
0477: if (buffer.length() > 0) {
0478: buffer.append(" + ");
0479: }
0480: buffer.append("F_RETURN_LINE_COMMENTS");
0481: }
0482: if ((_flags & Flags.F_RETURN_SIMPLE_WHITESPACES) != 0) {
0483: if (buffer.length() > 0) {
0484: buffer.append(" + ");
0485: }
0486: buffer.append("F_RETURN_SIMPLE_WHITESPACES");
0487: }
0488: }
0489:
0490: if ((_flags & Flags.F_COUNT_LINES) != 0) {
0491: if (buffer.length() > 0) {
0492: buffer.append(" + ");
0493: }
0494: buffer.append("F_COUNT_LINES");
0495: }
0496:
0497: if ((_flags & Flags.F_TOKEN_POS_ONLY) != 0) {
0498: if (buffer.length() > 0) {
0499: buffer.append(" + ");
0500: }
0501: buffer.append("F_TOKEN_POS_ONLY");
0502: }
0503: return buffer.toString();
0504: }
0505:
0506: /**
0507: * Returns a string representation for the used flags
0508: */
0509: private String types2String() {
0510: StringBuffer buffer = new StringBuffer();
0511:
0512: if ((_typeMask & (1 << Token.PATTERN)) != 0) {
0513: buffer.append("PATTERN");
0514: }
0515: if ((_typeMask & (1 << Token.KEYWORD)) != 0) {
0516: if (buffer.length() > 0) {
0517: buffer.append(" + ");
0518: }
0519: buffer.append("KEYWORD");
0520: }
0521: return buffer.toString();
0522: }
0523:
0524: //---------------------------------------------------------------------------
0525: // class constants
0526: //
0527:
0528: // piece of code the temporary large file consists of
0529: private static final String CODE_PIECE = "/**\n"
0530: + " * A Java-like code example with lots of comments, strings, special\n"
0531: + " * sequences etc.\n"
0532: + " *<br>\n"
0533: + " * Even some HTML tags like in real javadoc comments are present :-)\n"
0534: + " * This piece of code is multiplied into a temporary file to get a really\n"
0535: + " * huge source file (nothing that should happen in real life).\n"
0536: + " */\n"
0537: + "\n"
0538: + "// package declaration\n"
0539: + "package ours.my.subpackage;\n"
0540: + "\n"
0541: + "// imports\n"
0542: + "import java.util.*;\n"
0543: + "import java.io.InputStream;\n"
0544: + "import java.io.InputStreamReader;\n"
0545: + "import java.io.OutputStream;\n"
0546: + "import java.io.OutputStreamWriter;\n"
0547: + "import java.net.URL;\n"
0548: + "import java.net.URI;\n"
0549: + "import javax.swing.*;\n"
0550: + "\n"
0551: + "// class declaration\n"
0552: + "\n"
0553: + "/**\n"
0554: + " * An example Java class probably not even syntactically ok.\n"
0555: + " *\n"
0556: + " * @see OtherClass\n"
0557: + " * @see java.io.File\n"
0558: + " * @author me\n"
0559: + " */\n"
0560: + "public class MyTestClass implements Serializable {\n"
0561: + "\n"
0562: + " /**\n"
0563: + " * The usual main method.\n"
0564: + " *\n"
0565: + " * @param args the command line options and arguments\n"
0566: + " */\n"
0567: + " public static void main(String[] args) {\n"
0568: + " // create the argument store\n"
0569: + " argStore = new ArrayList(32);\n"
0570: + "\n"
0571: + " // wich GUI should be used?\n"
0572: + " if (args != null && args.length > 0) {\n"
0573: + " if (args[0].equals(\"swingui\")) {\n"
0574: + " new junit.swingui.TestRunner().main(tests);\n"
0575: + " } else if (args[0].equals(\"awtui\")) {\n"
0576: + " new junit.awtui.TestRunner().main(tests);\n"
0577: + " } else {\n"
0578: + " new junit.textui.TestRunner().main(tests);\n"
0579: + " }\n"
0580: + " } else {\n"
0581: + " new junit.textui.TestRunner().main(tests);\n"
0582: + " }\n"
0583: + "\n"
0584: + " // get all the other command line arguments\n"
0585: + " double doubleValue = 0.0;\n"
0586: + " int intValue = 0;\n"
0587: + " String stringValue = null;\n"
0588: + "\n"
0589: + " for (int index = 1; args != null && index < args.length; ++index) {\n"
0590: + " if (args[index].charAt(0) == '-') {\n"
0591: + " // options\n"
0592: + " switch (args[index].charAt(1)) {\n"
0593: + " case 'd':\n"
0594: + " doubleValue = Double.valueOf(args[index].substring(2)).doubleValue();\n"
0595: + " break;\n"
0596: + " case 's':\n"
0597: + " stringValue = args[index].substring(2);\n"
0598: + " break;\n"
0599: + " case 'i':\n"
0600: + " intValue = Integer.valueOf(args[index].substring(2)).intValue();\n"
0601: + " break;\n"
0602: + " default:\n"
0603: + " stringValue = \"\";\n"
0604: + " doubleValue = 0.0;\n"
0605: + " intValue = 0;\n"
0606: + " }\n"
0607: + "\n"
0608: + " } else {\n"
0609: + " // normal arguments\n"
0610: + " if ( ! argStore.contains(args[index])) {\n"
0611: + " argStore.add(args[index]);\n"
0612: + " } else {\n"
0613: + " System.out.println(\"Duplicate element \\\"\" + args[index] + \"\\\".\");\n"
0614: + " /* perhaps better use Environment.getEnvironment(this).out().println() */\n"
0615: + " }\n" + " }\n" + " }\n" + " }\n"
0616: + "\n" + " /**\n" + " * The argument store.\n"
0617: + " */\n" + " private ArrayList argStore = null;\n"
0618: + "}\n" + "\n" + "\n";
0619:
0620: // expected token types (without whitespaces)
0621: private static final int EXPECTED_TOKEN[] = {
0622: Token.BLOCK_COMMENT,
0623: Token.LINE_COMMENT, // "// package declaration\n"
0624: Token.KEYWORD,
0625: Token.NORMAL,
0626: Token.SPECIAL_SEQUENCE,
0627: Token.NORMAL,
0628: Token.SPECIAL_SEQUENCE,
0629: Token.NORMAL,
0630: Token.SPECIAL_SEQUENCE, // "package ours.my.subpackage;\n"
0631: Token.LINE_COMMENT, // "// imports\n"
0632: Token.KEYWORD,
0633: Token.NORMAL,
0634: Token.SPECIAL_SEQUENCE,
0635: Token.NORMAL,
0636: Token.SPECIAL_SEQUENCE,
0637: Token.SPECIAL_SEQUENCE,
0638: Token.SPECIAL_SEQUENCE, // "import java.util.*;\n"
0639: Token.KEYWORD,
0640: Token.NORMAL,
0641: Token.SPECIAL_SEQUENCE,
0642: Token.NORMAL,
0643: Token.SPECIAL_SEQUENCE,
0644: Token.NORMAL,
0645: Token.SPECIAL_SEQUENCE, // "import java.io.InputStream;\n"
0646: Token.KEYWORD,
0647: Token.NORMAL,
0648: Token.SPECIAL_SEQUENCE,
0649: Token.NORMAL,
0650: Token.SPECIAL_SEQUENCE,
0651: Token.NORMAL,
0652: Token.SPECIAL_SEQUENCE, // "import java.io.InputStreamReader;\n"
0653: Token.KEYWORD,
0654: Token.NORMAL,
0655: Token.SPECIAL_SEQUENCE,
0656: Token.NORMAL,
0657: Token.SPECIAL_SEQUENCE,
0658: Token.NORMAL,
0659: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStream;\n"
0660: Token.KEYWORD,
0661: Token.NORMAL,
0662: Token.SPECIAL_SEQUENCE,
0663: Token.NORMAL,
0664: Token.SPECIAL_SEQUENCE,
0665: Token.NORMAL,
0666: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStreamWriter;\n"
0667: Token.KEYWORD,
0668: Token.NORMAL,
0669: Token.SPECIAL_SEQUENCE,
0670: Token.NORMAL,
0671: Token.SPECIAL_SEQUENCE,
0672: Token.NORMAL,
0673: Token.SPECIAL_SEQUENCE, // "import java.net.URL;\n"
0674: Token.KEYWORD,
0675: Token.NORMAL,
0676: Token.SPECIAL_SEQUENCE,
0677: Token.NORMAL,
0678: Token.SPECIAL_SEQUENCE,
0679: Token.NORMAL,
0680: Token.SPECIAL_SEQUENCE, // "import java.net.URI;\n"
0681: Token.KEYWORD,
0682: Token.NORMAL,
0683: Token.SPECIAL_SEQUENCE,
0684: Token.NORMAL,
0685: Token.SPECIAL_SEQUENCE,
0686: Token.SPECIAL_SEQUENCE,
0687: Token.SPECIAL_SEQUENCE, // "import javax.swing.*;\n"
0688: Token.LINE_COMMENT, // "// class declaration\n"
0689: Token.BLOCK_COMMENT,
0690: Token.KEYWORD,
0691: Token.KEYWORD,
0692: Token.NORMAL,
0693: Token.KEYWORD,
0694: Token.NORMAL,
0695: Token.SPECIAL_SEQUENCE, // "public class MyTestClass implementes Serializable {\n"
0696: Token.BLOCK_COMMENT,
0697: Token.KEYWORD,
0698: Token.KEYWORD,
0699: Token.KEYWORD,
0700: Token.NORMAL,
0701: Token.SPECIAL_SEQUENCE,
0702: Token.KEYWORD,
0703: Token.SPECIAL_SEQUENCE,
0704: Token.SPECIAL_SEQUENCE,
0705: Token.NORMAL,
0706: Token.SPECIAL_SEQUENCE,
0707: Token.SPECIAL_SEQUENCE, // " public static void main(String[] args) {\n"
0708: Token.LINE_COMMENT, // "// create the argument store\n"
0709: Token.NORMAL,
0710: Token.SPECIAL_SEQUENCE,
0711: Token.KEYWORD,
0712: Token.NORMAL,
0713: Token.SPECIAL_SEQUENCE,
0714: Token.PATTERN,
0715: Token.SPECIAL_SEQUENCE,
0716: Token.SPECIAL_SEQUENCE, // " argStore = new ArrayList(32);\n"
0717: Token.LINE_COMMENT, // " // wich GUI should be used?\n"
0718: Token.KEYWORD,
0719: Token.SPECIAL_SEQUENCE,
0720: Token.NORMAL,
0721: Token.SPECIAL_SEQUENCE,
0722: Token.KEYWORD,
0723: Token.SPECIAL_SEQUENCE,
0724: Token.NORMAL,
0725: Token.SPECIAL_SEQUENCE,
0726: Token.NORMAL,
0727: Token.SPECIAL_SEQUENCE,
0728: Token.PATTERN,
0729: Token.SPECIAL_SEQUENCE,
0730: Token.SPECIAL_SEQUENCE, // " if (args != null && args.length > 0) {\n"
0731: Token.KEYWORD,
0732: Token.SPECIAL_SEQUENCE,
0733: Token.NORMAL,
0734: Token.SPECIAL_SEQUENCE,
0735: Token.PATTERN,
0736: Token.SPECIAL_SEQUENCE,
0737: Token.SPECIAL_SEQUENCE,
0738: Token.NORMAL,
0739: Token.SPECIAL_SEQUENCE,
0740: Token.STRING,
0741: Token.SPECIAL_SEQUENCE,
0742: Token.SPECIAL_SEQUENCE,
0743: Token.SPECIAL_SEQUENCE, // " if (args[0].equals(\"swingui\")) {\n"
0744: Token.KEYWORD,
0745: Token.NORMAL,
0746: Token.SPECIAL_SEQUENCE,
0747: Token.NORMAL,
0748: Token.SPECIAL_SEQUENCE,
0749: Token.NORMAL,
0750: Token.SPECIAL_SEQUENCE,
0751: Token.SPECIAL_SEQUENCE,
0752: Token.SPECIAL_SEQUENCE,
0753: Token.NORMAL,
0754: Token.SPECIAL_SEQUENCE,
0755: Token.NORMAL,
0756: Token.SPECIAL_SEQUENCE,
0757: Token.SPECIAL_SEQUENCE, // " new junit.swingui.TestRunner().main(tests);\n"
0758: Token.SPECIAL_SEQUENCE,
0759: Token.KEYWORD,
0760: Token.KEYWORD,
0761: Token.SPECIAL_SEQUENCE,
0762: Token.NORMAL,
0763: Token.SPECIAL_SEQUENCE,
0764: Token.PATTERN,
0765: Token.SPECIAL_SEQUENCE,
0766: Token.SPECIAL_SEQUENCE,
0767: Token.NORMAL,
0768: Token.SPECIAL_SEQUENCE,
0769: Token.STRING,
0770: Token.SPECIAL_SEQUENCE,
0771: Token.SPECIAL_SEQUENCE,
0772: Token.SPECIAL_SEQUENCE, // " } else if (args[0].equals(\"awtui\")) {\n"
0773: Token.KEYWORD,
0774: Token.NORMAL,
0775: Token.SPECIAL_SEQUENCE,
0776: Token.NORMAL,
0777: Token.SPECIAL_SEQUENCE,
0778: Token.NORMAL,
0779: Token.SPECIAL_SEQUENCE,
0780: Token.SPECIAL_SEQUENCE,
0781: Token.SPECIAL_SEQUENCE,
0782: Token.NORMAL,
0783: Token.SPECIAL_SEQUENCE,
0784: Token.NORMAL,
0785: Token.SPECIAL_SEQUENCE,
0786: Token.SPECIAL_SEQUENCE, // " new junit.awtui.TestRunner().main(tests);\n"
0787: Token.SPECIAL_SEQUENCE,
0788: Token.KEYWORD,
0789: Token.SPECIAL_SEQUENCE, // " } else {\n"
0790: Token.KEYWORD,
0791: Token.NORMAL,
0792: Token.SPECIAL_SEQUENCE,
0793: Token.NORMAL,
0794: Token.SPECIAL_SEQUENCE,
0795: Token.NORMAL,
0796: Token.SPECIAL_SEQUENCE,
0797: Token.SPECIAL_SEQUENCE,
0798: Token.SPECIAL_SEQUENCE,
0799: Token.NORMAL,
0800: Token.SPECIAL_SEQUENCE,
0801: Token.NORMAL,
0802: Token.SPECIAL_SEQUENCE,
0803: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
0804: Token.SPECIAL_SEQUENCE, // " }\n"
0805: Token.SPECIAL_SEQUENCE,
0806: Token.KEYWORD,
0807: Token.SPECIAL_SEQUENCE, // " } else {\n"
0808: Token.KEYWORD,
0809: Token.NORMAL,
0810: Token.SPECIAL_SEQUENCE,
0811: Token.NORMAL,
0812: Token.SPECIAL_SEQUENCE,
0813: Token.NORMAL,
0814: Token.SPECIAL_SEQUENCE,
0815: Token.SPECIAL_SEQUENCE,
0816: Token.SPECIAL_SEQUENCE,
0817: Token.NORMAL,
0818: Token.SPECIAL_SEQUENCE,
0819: Token.NORMAL,
0820: Token.SPECIAL_SEQUENCE,
0821: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
0822: Token.SPECIAL_SEQUENCE, // " }\n"
0823: Token.LINE_COMMENT, // " // get all the other command line arguments\n"
0824: Token.KEYWORD,
0825: Token.NORMAL,
0826: Token.SPECIAL_SEQUENCE,
0827: Token.PATTERN,
0828: Token.SPECIAL_SEQUENCE, // " double doubleValue = 0.0;\n"
0829: Token.KEYWORD,
0830: Token.NORMAL,
0831: Token.SPECIAL_SEQUENCE,
0832: Token.PATTERN,
0833: Token.SPECIAL_SEQUENCE, // " int intValue = 0;\n"
0834: Token.KEYWORD,
0835: Token.NORMAL,
0836: Token.SPECIAL_SEQUENCE,
0837: Token.KEYWORD,
0838: Token.SPECIAL_SEQUENCE, // " String stringValue = null;\n"
0839: Token.KEYWORD,
0840: Token.SPECIAL_SEQUENCE,
0841: Token.KEYWORD,
0842: Token.NORMAL,
0843: Token.SPECIAL_SEQUENCE,
0844: Token.PATTERN,
0845: Token.SPECIAL_SEQUENCE,
0846: Token.NORMAL,
0847: Token.SPECIAL_SEQUENCE,
0848: Token.KEYWORD,
0849: Token.SPECIAL_SEQUENCE,
0850: Token.NORMAL,
0851: Token.SPECIAL_SEQUENCE,
0852: Token.NORMAL,
0853: Token.SPECIAL_SEQUENCE,
0854: Token.NORMAL,
0855: Token.SPECIAL_SEQUENCE,
0856: Token.SPECIAL_SEQUENCE,
0857: Token.NORMAL,
0858: Token.SPECIAL_SEQUENCE,
0859: Token.SPECIAL_SEQUENCE, //" for (int index = 1; args != null && index < args.length; ++index) {\n"
0860: Token.KEYWORD,
0861: Token.SPECIAL_SEQUENCE,
0862: Token.NORMAL,
0863: Token.SPECIAL_SEQUENCE,
0864: Token.NORMAL,
0865: Token.SPECIAL_SEQUENCE,
0866: Token.SPECIAL_SEQUENCE,
0867: Token.NORMAL,
0868: Token.SPECIAL_SEQUENCE,
0869: Token.PATTERN,
0870: Token.SPECIAL_SEQUENCE,
0871: Token.SPECIAL_SEQUENCE,
0872: Token.STRING,
0873: Token.SPECIAL_SEQUENCE,
0874: Token.SPECIAL_SEQUENCE, // " if (args[index].charAt(0) == '-') {\n"
0875: Token.LINE_COMMENT, // " // options\n"
0876: Token.KEYWORD,
0877: Token.SPECIAL_SEQUENCE,
0878: Token.NORMAL,
0879: Token.SPECIAL_SEQUENCE,
0880: Token.NORMAL,
0881: Token.SPECIAL_SEQUENCE,
0882: Token.SPECIAL_SEQUENCE,
0883: Token.NORMAL,
0884: Token.SPECIAL_SEQUENCE,
0885: Token.PATTERN,
0886: Token.SPECIAL_SEQUENCE,
0887: Token.SPECIAL_SEQUENCE,
0888: Token.SPECIAL_SEQUENCE, // " switch (args[index].charAt(1)) {\n"
0889: Token.KEYWORD,
0890: Token.STRING,
0891: Token.SEPARATOR, // " case 'd':\n"
0892: Token.NORMAL,
0893: Token.SPECIAL_SEQUENCE,
0894: Token.NORMAL,
0895: Token.SPECIAL_SEQUENCE,
0896: Token.NORMAL,
0897: Token.SPECIAL_SEQUENCE,
0898: Token.NORMAL,
0899: Token.SPECIAL_SEQUENCE,
0900: Token.NORMAL,
0901: Token.SPECIAL_SEQUENCE,
0902: Token.SPECIAL_SEQUENCE,
0903: Token.NORMAL,
0904: Token.SPECIAL_SEQUENCE,
0905: Token.PATTERN,
0906: Token.SPECIAL_SEQUENCE,
0907: Token.SPECIAL_SEQUENCE,
0908: Token.SPECIAL_SEQUENCE,
0909: Token.NORMAL,
0910: Token.SPECIAL_SEQUENCE,
0911: Token.SPECIAL_SEQUENCE,
0912: Token.SPECIAL_SEQUENCE, // " doubleValue = Double.valueOf(args[index].substring(2)).doubleValue();\n"
0913: Token.KEYWORD,
0914: Token.SPECIAL_SEQUENCE, // " break;\n"
0915: Token.KEYWORD,
0916: Token.STRING,
0917: Token.SEPARATOR, // " case 's':\n"
0918: Token.NORMAL,
0919: Token.SPECIAL_SEQUENCE,
0920: Token.NORMAL,
0921: Token.SPECIAL_SEQUENCE,
0922: Token.NORMAL,
0923: Token.SPECIAL_SEQUENCE,
0924: Token.SPECIAL_SEQUENCE,
0925: Token.NORMAL,
0926: Token.SPECIAL_SEQUENCE,
0927: Token.PATTERN,
0928: Token.SPECIAL_SEQUENCE,
0929: Token.SPECIAL_SEQUENCE, // " stringValue = args[index].substring(2);\n"
0930: Token.KEYWORD,
0931: Token.SPECIAL_SEQUENCE, // " break;\n"
0932: Token.KEYWORD,
0933: Token.STRING,
0934: Token.SEPARATOR, // " case 'i':\n"
0935: Token.NORMAL,
0936: Token.SPECIAL_SEQUENCE,
0937: Token.NORMAL,
0938: Token.SPECIAL_SEQUENCE,
0939: Token.NORMAL,
0940: Token.SPECIAL_SEQUENCE,
0941: Token.NORMAL,
0942: Token.SPECIAL_SEQUENCE,
0943: Token.NORMAL,
0944: Token.SPECIAL_SEQUENCE,
0945: Token.SPECIAL_SEQUENCE,
0946: Token.NORMAL,
0947: Token.SPECIAL_SEQUENCE,
0948: Token.PATTERN,
0949: Token.SPECIAL_SEQUENCE,
0950: Token.SPECIAL_SEQUENCE,
0951: Token.SPECIAL_SEQUENCE,
0952: Token.NORMAL,
0953: Token.SPECIAL_SEQUENCE,
0954: Token.SPECIAL_SEQUENCE,
0955: Token.SPECIAL_SEQUENCE, // " intValue = Integer.valueOf(args[index].substring(2)).intValue();\n"
0956: Token.KEYWORD,
0957: Token.SPECIAL_SEQUENCE, // " break;\n"
0958: Token.KEYWORD,
0959: Token.SEPARATOR, // " default:\n"
0960: Token.NORMAL,
0961: Token.SPECIAL_SEQUENCE,
0962: Token.STRING,
0963: Token.SPECIAL_SEQUENCE, // " stringValue = \"\";\n"
0964: Token.NORMAL,
0965: Token.SPECIAL_SEQUENCE,
0966: Token.PATTERN,
0967: Token.SPECIAL_SEQUENCE, // " doubleValue = 0.0;\n"
0968: Token.NORMAL,
0969: Token.SPECIAL_SEQUENCE,
0970: Token.PATTERN,
0971: Token.SPECIAL_SEQUENCE, // " intValue = 0;\n"
0972: Token.SPECIAL_SEQUENCE, // " }\n"
0973: Token.SPECIAL_SEQUENCE,
0974: Token.KEYWORD,
0975: Token.SPECIAL_SEQUENCE, // " } else {\n"
0976: Token.LINE_COMMENT, // " // normal arguments\n"
0977: Token.KEYWORD,
0978: Token.SPECIAL_SEQUENCE,
0979: Token.SPECIAL_SEQUENCE,
0980: Token.NORMAL,
0981: Token.SPECIAL_SEQUENCE,
0982: Token.NORMAL,
0983: Token.SPECIAL_SEQUENCE,
0984: Token.NORMAL,
0985: Token.SPECIAL_SEQUENCE,
0986: Token.NORMAL,
0987: Token.SPECIAL_SEQUENCE,
0988: Token.SPECIAL_SEQUENCE,
0989: Token.SPECIAL_SEQUENCE,
0990: Token.SPECIAL_SEQUENCE, // " if ( ! argStore.contains(args[index])) {\n"
0991: Token.NORMAL,
0992: Token.SPECIAL_SEQUENCE,
0993: Token.NORMAL,
0994: Token.SPECIAL_SEQUENCE,
0995: Token.NORMAL,
0996: Token.SPECIAL_SEQUENCE,
0997: Token.NORMAL,
0998: Token.SPECIAL_SEQUENCE,
0999: Token.SPECIAL_SEQUENCE,
1000: Token.SPECIAL_SEQUENCE, // " argStore.add(args[index]);\n"
1001: Token.SPECIAL_SEQUENCE,
1002: Token.KEYWORD,
1003: Token.SPECIAL_SEQUENCE, // " } else {\n"
1004: Token.NORMAL, Token.SPECIAL_SEQUENCE,
1005: Token.NORMAL,
1006: Token.SPECIAL_SEQUENCE,
1007: Token.NORMAL,
1008: Token.SPECIAL_SEQUENCE,
1009: Token.STRING,
1010: Token.SPECIAL_SEQUENCE,
1011: Token.NORMAL,
1012: Token.SPECIAL_SEQUENCE,
1013: Token.NORMAL,
1014: Token.SPECIAL_SEQUENCE,
1015: Token.SPECIAL_SEQUENCE,
1016: Token.STRING,
1017: Token.SPECIAL_SEQUENCE,
1018: Token.SPECIAL_SEQUENCE, // " System.out.println(\"Duplicate element \\\" + args[index] + \"\\\".\");\n"
1019: Token.BLOCK_COMMENT, // " /* perhaps better use Environment.getEnvironment(this).out().println() */\n"
1020: Token.SPECIAL_SEQUENCE, // " }\n"
1021: Token.SPECIAL_SEQUENCE, // " }\n"
1022: Token.SPECIAL_SEQUENCE, // " }\n"
1023: Token.SPECIAL_SEQUENCE, // " }\n"
1024: Token.BLOCK_COMMENT, Token.KEYWORD, Token.NORMAL,
1025: Token.NORMAL, Token.SPECIAL_SEQUENCE, Token.KEYWORD,
1026: Token.SPECIAL_SEQUENCE, // " private ArrayList argStore = null;\n"
1027: Token.SPECIAL_SEQUENCE // "}\n"
1028: };
1029:
1030: // expected token types (without whitespaces) when no pattern matching is performed
1031: private static final int EXPECTED_TOKEN_WITHOUT_PATTERN[] = {
1032: Token.BLOCK_COMMENT,
1033: Token.LINE_COMMENT, // "// package declaration\n"
1034: Token.KEYWORD,
1035: Token.NORMAL,
1036: Token.SPECIAL_SEQUENCE,
1037: Token.NORMAL,
1038: Token.SPECIAL_SEQUENCE,
1039: Token.NORMAL,
1040: Token.SPECIAL_SEQUENCE, // "package ours.my.subpackage;\n"
1041: Token.LINE_COMMENT, // "// imports\n"
1042: Token.KEYWORD,
1043: Token.NORMAL,
1044: Token.SPECIAL_SEQUENCE,
1045: Token.NORMAL,
1046: Token.SPECIAL_SEQUENCE,
1047: Token.SPECIAL_SEQUENCE,
1048: Token.SPECIAL_SEQUENCE, // "import java.util.*;\n"
1049: Token.KEYWORD,
1050: Token.NORMAL,
1051: Token.SPECIAL_SEQUENCE,
1052: Token.NORMAL,
1053: Token.SPECIAL_SEQUENCE,
1054: Token.NORMAL,
1055: Token.SPECIAL_SEQUENCE, // "import java.io.InputStream;\n"
1056: Token.KEYWORD,
1057: Token.NORMAL,
1058: Token.SPECIAL_SEQUENCE,
1059: Token.NORMAL,
1060: Token.SPECIAL_SEQUENCE,
1061: Token.NORMAL,
1062: Token.SPECIAL_SEQUENCE, // "import java.io.InputStreamReader;\n"
1063: Token.KEYWORD,
1064: Token.NORMAL,
1065: Token.SPECIAL_SEQUENCE,
1066: Token.NORMAL,
1067: Token.SPECIAL_SEQUENCE,
1068: Token.NORMAL,
1069: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStream;\n"
1070: Token.KEYWORD,
1071: Token.NORMAL,
1072: Token.SPECIAL_SEQUENCE,
1073: Token.NORMAL,
1074: Token.SPECIAL_SEQUENCE,
1075: Token.NORMAL,
1076: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStreamWriter;\n"
1077: Token.KEYWORD,
1078: Token.NORMAL,
1079: Token.SPECIAL_SEQUENCE,
1080: Token.NORMAL,
1081: Token.SPECIAL_SEQUENCE,
1082: Token.NORMAL,
1083: Token.SPECIAL_SEQUENCE, // "import java.net.URL;\n"
1084: Token.KEYWORD,
1085: Token.NORMAL,
1086: Token.SPECIAL_SEQUENCE,
1087: Token.NORMAL,
1088: Token.SPECIAL_SEQUENCE,
1089: Token.NORMAL,
1090: Token.SPECIAL_SEQUENCE, // "import java.net.URI;\n"
1091: Token.KEYWORD,
1092: Token.NORMAL,
1093: Token.SPECIAL_SEQUENCE,
1094: Token.NORMAL,
1095: Token.SPECIAL_SEQUENCE,
1096: Token.SPECIAL_SEQUENCE,
1097: Token.SPECIAL_SEQUENCE, // "import javax.swing.*;\n"
1098: Token.LINE_COMMENT, // "// class declaration\n"
1099: Token.BLOCK_COMMENT,
1100: Token.KEYWORD,
1101: Token.KEYWORD,
1102: Token.NORMAL,
1103: Token.KEYWORD,
1104: Token.NORMAL,
1105: Token.SPECIAL_SEQUENCE, // "public class MyTestClass implementes Serializable {\n"
1106: Token.BLOCK_COMMENT,
1107: Token.KEYWORD,
1108: Token.KEYWORD,
1109: Token.KEYWORD,
1110: Token.NORMAL,
1111: Token.SPECIAL_SEQUENCE,
1112: Token.KEYWORD,
1113: Token.SPECIAL_SEQUENCE,
1114: Token.SPECIAL_SEQUENCE,
1115: Token.NORMAL,
1116: Token.SPECIAL_SEQUENCE,
1117: Token.SPECIAL_SEQUENCE, // " public static void main(String[] args) {\n"
1118: Token.LINE_COMMENT, // "// create the argument store\n"
1119: Token.NORMAL,
1120: Token.SPECIAL_SEQUENCE,
1121: Token.KEYWORD,
1122: Token.NORMAL,
1123: Token.SPECIAL_SEQUENCE,
1124: Token.NORMAL,
1125: Token.SPECIAL_SEQUENCE,
1126: Token.SPECIAL_SEQUENCE, // " argStore = new ArrayList(32);\n"
1127: Token.LINE_COMMENT, // " // wich GUI should be used?\n"
1128: Token.KEYWORD,
1129: Token.SPECIAL_SEQUENCE,
1130: Token.NORMAL,
1131: Token.SPECIAL_SEQUENCE,
1132: Token.KEYWORD,
1133: Token.SPECIAL_SEQUENCE,
1134: Token.NORMAL,
1135: Token.SPECIAL_SEQUENCE,
1136: Token.NORMAL,
1137: Token.SPECIAL_SEQUENCE,
1138: Token.NORMAL,
1139: Token.SPECIAL_SEQUENCE,
1140: Token.SPECIAL_SEQUENCE, // " if (args != null && args.length > 0) {\n"
1141: Token.KEYWORD,
1142: Token.SPECIAL_SEQUENCE,
1143: Token.NORMAL,
1144: Token.SPECIAL_SEQUENCE,
1145: Token.NORMAL,
1146: Token.SPECIAL_SEQUENCE,
1147: Token.SPECIAL_SEQUENCE,
1148: Token.NORMAL,
1149: Token.SPECIAL_SEQUENCE,
1150: Token.STRING,
1151: Token.SPECIAL_SEQUENCE,
1152: Token.SPECIAL_SEQUENCE,
1153: Token.SPECIAL_SEQUENCE, // " if (args[0].equals(\"swingui\")) {\n"
1154: Token.KEYWORD,
1155: Token.NORMAL,
1156: Token.SPECIAL_SEQUENCE,
1157: Token.NORMAL,
1158: Token.SPECIAL_SEQUENCE,
1159: Token.NORMAL,
1160: Token.SPECIAL_SEQUENCE,
1161: Token.SPECIAL_SEQUENCE,
1162: Token.SPECIAL_SEQUENCE,
1163: Token.NORMAL,
1164: Token.SPECIAL_SEQUENCE,
1165: Token.NORMAL,
1166: Token.SPECIAL_SEQUENCE,
1167: Token.SPECIAL_SEQUENCE, // " new junit.swingui.TestRunner().main(tests);\n"
1168: Token.SPECIAL_SEQUENCE,
1169: Token.KEYWORD,
1170: Token.KEYWORD,
1171: Token.SPECIAL_SEQUENCE,
1172: Token.NORMAL,
1173: Token.SPECIAL_SEQUENCE,
1174: Token.NORMAL,
1175: Token.SPECIAL_SEQUENCE,
1176: Token.SPECIAL_SEQUENCE,
1177: Token.NORMAL,
1178: Token.SPECIAL_SEQUENCE,
1179: Token.STRING,
1180: Token.SPECIAL_SEQUENCE,
1181: Token.SPECIAL_SEQUENCE,
1182: Token.SPECIAL_SEQUENCE, // " } else if (args[0].equals(\"awtui\")) {\n"
1183: Token.KEYWORD,
1184: Token.NORMAL,
1185: Token.SPECIAL_SEQUENCE,
1186: Token.NORMAL,
1187: Token.SPECIAL_SEQUENCE,
1188: Token.NORMAL,
1189: Token.SPECIAL_SEQUENCE,
1190: Token.SPECIAL_SEQUENCE,
1191: Token.SPECIAL_SEQUENCE,
1192: Token.NORMAL,
1193: Token.SPECIAL_SEQUENCE,
1194: Token.NORMAL,
1195: Token.SPECIAL_SEQUENCE,
1196: Token.SPECIAL_SEQUENCE, // " new junit.awtui.TestRunner().main(tests);\n"
1197: Token.SPECIAL_SEQUENCE,
1198: Token.KEYWORD,
1199: Token.SPECIAL_SEQUENCE, // " } else {\n"
1200: Token.KEYWORD,
1201: Token.NORMAL,
1202: Token.SPECIAL_SEQUENCE,
1203: Token.NORMAL,
1204: Token.SPECIAL_SEQUENCE,
1205: Token.NORMAL,
1206: Token.SPECIAL_SEQUENCE,
1207: Token.SPECIAL_SEQUENCE,
1208: Token.SPECIAL_SEQUENCE,
1209: Token.NORMAL,
1210: Token.SPECIAL_SEQUENCE,
1211: Token.NORMAL,
1212: Token.SPECIAL_SEQUENCE,
1213: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
1214: Token.SPECIAL_SEQUENCE, // " }\n"
1215: Token.SPECIAL_SEQUENCE,
1216: Token.KEYWORD,
1217: Token.SPECIAL_SEQUENCE, // " } else {\n"
1218: Token.KEYWORD,
1219: Token.NORMAL,
1220: Token.SPECIAL_SEQUENCE,
1221: Token.NORMAL,
1222: Token.SPECIAL_SEQUENCE,
1223: Token.NORMAL,
1224: Token.SPECIAL_SEQUENCE,
1225: Token.SPECIAL_SEQUENCE,
1226: Token.SPECIAL_SEQUENCE,
1227: Token.NORMAL,
1228: Token.SPECIAL_SEQUENCE,
1229: Token.NORMAL,
1230: Token.SPECIAL_SEQUENCE,
1231: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
1232: Token.SPECIAL_SEQUENCE, // " }\n"
1233: Token.LINE_COMMENT, // " // get all the other command line arguments\n"
1234: Token.KEYWORD,
1235: Token.NORMAL,
1236: Token.SPECIAL_SEQUENCE,
1237: Token.NORMAL,
1238: Token.SPECIAL_SEQUENCE,
1239: Token.NORMAL,
1240: Token.SPECIAL_SEQUENCE, // " double doubleValue = 0.0;\n"
1241: Token.KEYWORD,
1242: Token.NORMAL,
1243: Token.SPECIAL_SEQUENCE,
1244: Token.NORMAL,
1245: Token.SPECIAL_SEQUENCE, // " int intValue = 0;\n"
1246: Token.KEYWORD,
1247: Token.NORMAL,
1248: Token.SPECIAL_SEQUENCE,
1249: Token.KEYWORD,
1250: Token.SPECIAL_SEQUENCE, // " String stringValue = null;\n"
1251: Token.KEYWORD,
1252: Token.SPECIAL_SEQUENCE,
1253: Token.KEYWORD,
1254: Token.NORMAL,
1255: Token.SPECIAL_SEQUENCE,
1256: Token.NORMAL,
1257: Token.SPECIAL_SEQUENCE,
1258: Token.NORMAL,
1259: Token.SPECIAL_SEQUENCE,
1260: Token.KEYWORD,
1261: Token.SPECIAL_SEQUENCE,
1262: Token.NORMAL,
1263: Token.SPECIAL_SEQUENCE,
1264: Token.NORMAL,
1265: Token.SPECIAL_SEQUENCE,
1266: Token.NORMAL,
1267: Token.SPECIAL_SEQUENCE,
1268: Token.SPECIAL_SEQUENCE,
1269: Token.NORMAL,
1270: Token.SPECIAL_SEQUENCE,
1271: Token.SPECIAL_SEQUENCE, //" for (int index = 1; args != null && index < args.length; ++index) {\n"
1272: Token.KEYWORD,
1273: Token.SPECIAL_SEQUENCE,
1274: Token.NORMAL,
1275: Token.SPECIAL_SEQUENCE,
1276: Token.NORMAL,
1277: Token.SPECIAL_SEQUENCE,
1278: Token.SPECIAL_SEQUENCE,
1279: Token.NORMAL,
1280: Token.SPECIAL_SEQUENCE,
1281: Token.NORMAL,
1282: Token.SPECIAL_SEQUENCE,
1283: Token.SPECIAL_SEQUENCE,
1284: Token.STRING,
1285: Token.SPECIAL_SEQUENCE,
1286: Token.SPECIAL_SEQUENCE, // " if (args[index].charAt(0) == '-') {\n"
1287: Token.LINE_COMMENT, // " // options\n"
1288: Token.KEYWORD,
1289: Token.SPECIAL_SEQUENCE,
1290: Token.NORMAL,
1291: Token.SPECIAL_SEQUENCE,
1292: Token.NORMAL,
1293: Token.SPECIAL_SEQUENCE,
1294: Token.SPECIAL_SEQUENCE,
1295: Token.NORMAL,
1296: Token.SPECIAL_SEQUENCE,
1297: Token.NORMAL,
1298: Token.SPECIAL_SEQUENCE,
1299: Token.SPECIAL_SEQUENCE,
1300: Token.SPECIAL_SEQUENCE, // " switch (args[index].charAt(1)) {\n"
1301: Token.KEYWORD,
1302: Token.STRING,
1303: Token.SEPARATOR, // " case 'd':\n"
1304: Token.NORMAL,
1305: Token.SPECIAL_SEQUENCE,
1306: Token.NORMAL,
1307: Token.SPECIAL_SEQUENCE,
1308: Token.NORMAL,
1309: Token.SPECIAL_SEQUENCE,
1310: Token.NORMAL,
1311: Token.SPECIAL_SEQUENCE,
1312: Token.NORMAL,
1313: Token.SPECIAL_SEQUENCE,
1314: Token.SPECIAL_SEQUENCE,
1315: Token.NORMAL,
1316: Token.SPECIAL_SEQUENCE,
1317: Token.NORMAL,
1318: Token.SPECIAL_SEQUENCE,
1319: Token.SPECIAL_SEQUENCE,
1320: Token.SPECIAL_SEQUENCE,
1321: Token.NORMAL,
1322: Token.SPECIAL_SEQUENCE,
1323: Token.SPECIAL_SEQUENCE,
1324: Token.SPECIAL_SEQUENCE, // " doubleValue = Double.valueOf(args[index].substring(2)).doubleValue();\n"
1325: Token.KEYWORD,
1326: Token.SPECIAL_SEQUENCE, // " break;\n"
1327: Token.KEYWORD,
1328: Token.STRING,
1329: Token.SEPARATOR, // " case 's':\n"
1330: Token.NORMAL,
1331: Token.SPECIAL_SEQUENCE,
1332: Token.NORMAL,
1333: Token.SPECIAL_SEQUENCE,
1334: Token.NORMAL,
1335: Token.SPECIAL_SEQUENCE,
1336: Token.SPECIAL_SEQUENCE,
1337: Token.NORMAL,
1338: Token.SPECIAL_SEQUENCE,
1339: Token.NORMAL,
1340: Token.SPECIAL_SEQUENCE,
1341: Token.SPECIAL_SEQUENCE, // " stringValue = args[index].substring(2);\n"
1342: Token.KEYWORD,
1343: Token.SPECIAL_SEQUENCE, // " break;\n"
1344: Token.KEYWORD,
1345: Token.STRING,
1346: Token.SEPARATOR, // " case 'i':\n"
1347: Token.NORMAL,
1348: Token.SPECIAL_SEQUENCE,
1349: Token.NORMAL,
1350: Token.SPECIAL_SEQUENCE,
1351: Token.NORMAL,
1352: Token.SPECIAL_SEQUENCE,
1353: Token.NORMAL,
1354: Token.SPECIAL_SEQUENCE,
1355: Token.NORMAL,
1356: Token.SPECIAL_SEQUENCE,
1357: Token.SPECIAL_SEQUENCE,
1358: Token.NORMAL,
1359: Token.SPECIAL_SEQUENCE,
1360: Token.NORMAL,
1361: Token.SPECIAL_SEQUENCE,
1362: Token.SPECIAL_SEQUENCE,
1363: Token.SPECIAL_SEQUENCE,
1364: Token.NORMAL,
1365: Token.SPECIAL_SEQUENCE,
1366: Token.SPECIAL_SEQUENCE,
1367: Token.SPECIAL_SEQUENCE, // " intValue = Integer.valueOf(args[index].substring(2)).intValue();\n"
1368: Token.KEYWORD,
1369: Token.SPECIAL_SEQUENCE, // " break;\n"
1370: Token.KEYWORD,
1371: Token.SEPARATOR, // " default:\n"
1372: Token.NORMAL,
1373: Token.SPECIAL_SEQUENCE,
1374: Token.STRING,
1375: Token.SPECIAL_SEQUENCE, // " stringValue = \"\";\n"
1376: Token.NORMAL,
1377: Token.SPECIAL_SEQUENCE,
1378: Token.NORMAL,
1379: Token.SPECIAL_SEQUENCE,
1380: Token.NORMAL,
1381: Token.SPECIAL_SEQUENCE, // " doubleValue = 0.0;\n"
1382: Token.NORMAL,
1383: Token.SPECIAL_SEQUENCE,
1384: Token.NORMAL,
1385: Token.SPECIAL_SEQUENCE, // " intValue = 0;\n"
1386: Token.SPECIAL_SEQUENCE, // " }\n"
1387: Token.SPECIAL_SEQUENCE,
1388: Token.KEYWORD,
1389: Token.SPECIAL_SEQUENCE, // " } else {\n"
1390: Token.LINE_COMMENT, // " // normal arguments\n"
1391: Token.KEYWORD,
1392: Token.SPECIAL_SEQUENCE,
1393: Token.SPECIAL_SEQUENCE,
1394: Token.NORMAL,
1395: Token.SPECIAL_SEQUENCE,
1396: Token.NORMAL,
1397: Token.SPECIAL_SEQUENCE,
1398: Token.NORMAL,
1399: Token.SPECIAL_SEQUENCE,
1400: Token.NORMAL,
1401: Token.SPECIAL_SEQUENCE,
1402: Token.SPECIAL_SEQUENCE,
1403: Token.SPECIAL_SEQUENCE,
1404: Token.SPECIAL_SEQUENCE, // " if ( ! argStore.contains(args[index])) {\n"
1405: Token.NORMAL,
1406: Token.SPECIAL_SEQUENCE,
1407: Token.NORMAL,
1408: Token.SPECIAL_SEQUENCE,
1409: Token.NORMAL,
1410: Token.SPECIAL_SEQUENCE,
1411: Token.NORMAL,
1412: Token.SPECIAL_SEQUENCE,
1413: Token.SPECIAL_SEQUENCE,
1414: Token.SPECIAL_SEQUENCE, // " argStore.add(args[index]);\n"
1415: Token.SPECIAL_SEQUENCE,
1416: Token.KEYWORD,
1417: Token.SPECIAL_SEQUENCE, // " } else {\n"
1418: Token.NORMAL, Token.SPECIAL_SEQUENCE,
1419: Token.NORMAL,
1420: Token.SPECIAL_SEQUENCE,
1421: Token.NORMAL,
1422: Token.SPECIAL_SEQUENCE,
1423: Token.STRING,
1424: Token.SPECIAL_SEQUENCE,
1425: Token.NORMAL,
1426: Token.SPECIAL_SEQUENCE,
1427: Token.NORMAL,
1428: Token.SPECIAL_SEQUENCE,
1429: Token.SPECIAL_SEQUENCE,
1430: Token.STRING,
1431: Token.SPECIAL_SEQUENCE,
1432: Token.SPECIAL_SEQUENCE, // " System.out.println(\"Duplicate element \\\" + args[index] + \"\\\".\");\n"
1433: Token.BLOCK_COMMENT, // " /* perhaps better use Environment.getEnvironment(this).out().println() */\n"
1434: Token.SPECIAL_SEQUENCE, // " }\n"
1435: Token.SPECIAL_SEQUENCE, // " }\n"
1436: Token.SPECIAL_SEQUENCE, // " }\n"
1437: Token.SPECIAL_SEQUENCE, // " }\n"
1438: Token.BLOCK_COMMENT, Token.KEYWORD, Token.NORMAL,
1439: Token.NORMAL, Token.SPECIAL_SEQUENCE, Token.KEYWORD,
1440: Token.SPECIAL_SEQUENCE, // " private ArrayList argStore = null;\n"
1441: Token.SPECIAL_SEQUENCE // "}\n"
1442: };
1443:
1444: // expected token types (without whitespaces) when no pattern matching is performed
1445: private static final int EXPECTED_TOKEN_WITHOUT_PATTERN_AND_KEYWORDS[] = {
1446: Token.BLOCK_COMMENT,
1447: Token.LINE_COMMENT, // "// package declaration\n"
1448: Token.NORMAL,
1449: Token.NORMAL,
1450: Token.SPECIAL_SEQUENCE,
1451: Token.NORMAL,
1452: Token.SPECIAL_SEQUENCE,
1453: Token.NORMAL,
1454: Token.SPECIAL_SEQUENCE, // "package ours.my.subpackage;\n"
1455: Token.LINE_COMMENT, // "// imports\n"
1456: Token.NORMAL,
1457: Token.NORMAL,
1458: Token.SPECIAL_SEQUENCE,
1459: Token.NORMAL,
1460: Token.SPECIAL_SEQUENCE,
1461: Token.SPECIAL_SEQUENCE,
1462: Token.SPECIAL_SEQUENCE, // "import java.util.*;\n"
1463: Token.NORMAL,
1464: Token.NORMAL,
1465: Token.SPECIAL_SEQUENCE,
1466: Token.NORMAL,
1467: Token.SPECIAL_SEQUENCE,
1468: Token.NORMAL,
1469: Token.SPECIAL_SEQUENCE, // "import java.io.InputStream;\n"
1470: Token.NORMAL,
1471: Token.NORMAL,
1472: Token.SPECIAL_SEQUENCE,
1473: Token.NORMAL,
1474: Token.SPECIAL_SEQUENCE,
1475: Token.NORMAL,
1476: Token.SPECIAL_SEQUENCE, // "import java.io.InputStreamReader;\n"
1477: Token.NORMAL,
1478: Token.NORMAL,
1479: Token.SPECIAL_SEQUENCE,
1480: Token.NORMAL,
1481: Token.SPECIAL_SEQUENCE,
1482: Token.NORMAL,
1483: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStream;\n"
1484: Token.NORMAL,
1485: Token.NORMAL,
1486: Token.SPECIAL_SEQUENCE,
1487: Token.NORMAL,
1488: Token.SPECIAL_SEQUENCE,
1489: Token.NORMAL,
1490: Token.SPECIAL_SEQUENCE, // "import java.io.OutputStreamWriter;\n"
1491: Token.NORMAL,
1492: Token.NORMAL,
1493: Token.SPECIAL_SEQUENCE,
1494: Token.NORMAL,
1495: Token.SPECIAL_SEQUENCE,
1496: Token.NORMAL,
1497: Token.SPECIAL_SEQUENCE, // "import java.net.URL;\n"
1498: Token.NORMAL,
1499: Token.NORMAL,
1500: Token.SPECIAL_SEQUENCE,
1501: Token.NORMAL,
1502: Token.SPECIAL_SEQUENCE,
1503: Token.NORMAL,
1504: Token.SPECIAL_SEQUENCE, // "import java.net.URI;\n"
1505: Token.NORMAL,
1506: Token.NORMAL,
1507: Token.SPECIAL_SEQUENCE,
1508: Token.NORMAL,
1509: Token.SPECIAL_SEQUENCE,
1510: Token.SPECIAL_SEQUENCE,
1511: Token.SPECIAL_SEQUENCE, // "import javax.swing.*;\n"
1512: Token.LINE_COMMENT, // "// class declaration\n"
1513: Token.BLOCK_COMMENT,
1514: Token.NORMAL,
1515: Token.NORMAL,
1516: Token.NORMAL,
1517: Token.NORMAL,
1518: Token.NORMAL,
1519: Token.SPECIAL_SEQUENCE, // "public class MyTestClass implementes Serializable {\n"
1520: Token.BLOCK_COMMENT,
1521: Token.NORMAL,
1522: Token.NORMAL,
1523: Token.NORMAL,
1524: Token.NORMAL,
1525: Token.SPECIAL_SEQUENCE,
1526: Token.NORMAL,
1527: Token.SPECIAL_SEQUENCE,
1528: Token.SPECIAL_SEQUENCE,
1529: Token.NORMAL,
1530: Token.SPECIAL_SEQUENCE,
1531: Token.SPECIAL_SEQUENCE, // " public static void main(String[] args) {\n"
1532: Token.LINE_COMMENT, // "// create the argument store\n"
1533: Token.NORMAL,
1534: Token.SPECIAL_SEQUENCE,
1535: Token.NORMAL,
1536: Token.NORMAL,
1537: Token.SPECIAL_SEQUENCE,
1538: Token.NORMAL,
1539: Token.SPECIAL_SEQUENCE,
1540: Token.SPECIAL_SEQUENCE, // " argStore = new ArrayList(32);\n"
1541: Token.LINE_COMMENT, // " // wich GUI should be used?\n"
1542: Token.NORMAL,
1543: Token.SPECIAL_SEQUENCE,
1544: Token.NORMAL,
1545: Token.SPECIAL_SEQUENCE,
1546: Token.NORMAL,
1547: Token.SPECIAL_SEQUENCE,
1548: Token.NORMAL,
1549: Token.SPECIAL_SEQUENCE,
1550: Token.NORMAL,
1551: Token.SPECIAL_SEQUENCE,
1552: Token.NORMAL,
1553: Token.SPECIAL_SEQUENCE,
1554: Token.SPECIAL_SEQUENCE, // " if (args != null && args.length > 0) {\n"
1555: Token.NORMAL,
1556: Token.SPECIAL_SEQUENCE,
1557: Token.NORMAL,
1558: Token.SPECIAL_SEQUENCE,
1559: Token.NORMAL,
1560: Token.SPECIAL_SEQUENCE,
1561: Token.SPECIAL_SEQUENCE,
1562: Token.NORMAL,
1563: Token.SPECIAL_SEQUENCE,
1564: Token.STRING,
1565: Token.SPECIAL_SEQUENCE,
1566: Token.SPECIAL_SEQUENCE,
1567: Token.SPECIAL_SEQUENCE, // " if (args[0].equals(\"swingui\")) {\n"
1568: Token.NORMAL,
1569: Token.NORMAL,
1570: Token.SPECIAL_SEQUENCE,
1571: Token.NORMAL,
1572: Token.SPECIAL_SEQUENCE,
1573: Token.NORMAL,
1574: Token.SPECIAL_SEQUENCE,
1575: Token.SPECIAL_SEQUENCE,
1576: Token.SPECIAL_SEQUENCE,
1577: Token.NORMAL,
1578: Token.SPECIAL_SEQUENCE,
1579: Token.NORMAL,
1580: Token.SPECIAL_SEQUENCE,
1581: Token.SPECIAL_SEQUENCE, // " new junit.swingui.TestRunner().main(tests);\n"
1582: Token.SPECIAL_SEQUENCE,
1583: Token.NORMAL,
1584: Token.NORMAL,
1585: Token.SPECIAL_SEQUENCE,
1586: Token.NORMAL,
1587: Token.SPECIAL_SEQUENCE,
1588: Token.NORMAL,
1589: Token.SPECIAL_SEQUENCE,
1590: Token.SPECIAL_SEQUENCE,
1591: Token.NORMAL,
1592: Token.SPECIAL_SEQUENCE,
1593: Token.STRING,
1594: Token.SPECIAL_SEQUENCE,
1595: Token.SPECIAL_SEQUENCE,
1596: Token.SPECIAL_SEQUENCE, // " } else if (args[0].equals(\"awtui\")) {\n"
1597: Token.NORMAL,
1598: Token.NORMAL,
1599: Token.SPECIAL_SEQUENCE,
1600: Token.NORMAL,
1601: Token.SPECIAL_SEQUENCE,
1602: Token.NORMAL,
1603: Token.SPECIAL_SEQUENCE,
1604: Token.SPECIAL_SEQUENCE,
1605: Token.SPECIAL_SEQUENCE,
1606: Token.NORMAL,
1607: Token.SPECIAL_SEQUENCE,
1608: Token.NORMAL,
1609: Token.SPECIAL_SEQUENCE,
1610: Token.SPECIAL_SEQUENCE, // " new junit.awtui.TestRunner().main(tests);\n"
1611: Token.SPECIAL_SEQUENCE,
1612: Token.NORMAL,
1613: Token.SPECIAL_SEQUENCE, // " } else {\n"
1614: Token.NORMAL,
1615: Token.NORMAL,
1616: Token.SPECIAL_SEQUENCE,
1617: Token.NORMAL,
1618: Token.SPECIAL_SEQUENCE,
1619: Token.NORMAL,
1620: Token.SPECIAL_SEQUENCE,
1621: Token.SPECIAL_SEQUENCE,
1622: Token.SPECIAL_SEQUENCE,
1623: Token.NORMAL,
1624: Token.SPECIAL_SEQUENCE,
1625: Token.NORMAL,
1626: Token.SPECIAL_SEQUENCE,
1627: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
1628: Token.SPECIAL_SEQUENCE, // " }\n"
1629: Token.SPECIAL_SEQUENCE,
1630: Token.NORMAL,
1631: Token.SPECIAL_SEQUENCE, // " } else {\n"
1632: Token.NORMAL,
1633: Token.NORMAL,
1634: Token.SPECIAL_SEQUENCE,
1635: Token.NORMAL,
1636: Token.SPECIAL_SEQUENCE,
1637: Token.NORMAL,
1638: Token.SPECIAL_SEQUENCE,
1639: Token.SPECIAL_SEQUENCE,
1640: Token.SPECIAL_SEQUENCE,
1641: Token.NORMAL,
1642: Token.SPECIAL_SEQUENCE,
1643: Token.NORMAL,
1644: Token.SPECIAL_SEQUENCE,
1645: Token.SPECIAL_SEQUENCE, // " new junit.textui.TestRunner().main(tests);\n"
1646: Token.SPECIAL_SEQUENCE, // " }\n"
1647: Token.LINE_COMMENT, // " // get all the other command line arguments\n"
1648: Token.NORMAL,
1649: Token.NORMAL,
1650: Token.SPECIAL_SEQUENCE,
1651: Token.NORMAL,
1652: Token.SPECIAL_SEQUENCE,
1653: Token.NORMAL,
1654: Token.SPECIAL_SEQUENCE, // " double doubleValue = 0.0;\n"
1655: Token.NORMAL,
1656: Token.NORMAL,
1657: Token.SPECIAL_SEQUENCE,
1658: Token.NORMAL,
1659: Token.SPECIAL_SEQUENCE, // " int intValue = 0;\n"
1660: Token.NORMAL,
1661: Token.NORMAL,
1662: Token.SPECIAL_SEQUENCE,
1663: Token.NORMAL,
1664: Token.SPECIAL_SEQUENCE, // " String stringValue = null;\n"
1665: Token.NORMAL,
1666: Token.SPECIAL_SEQUENCE,
1667: Token.NORMAL,
1668: Token.NORMAL,
1669: Token.SPECIAL_SEQUENCE,
1670: Token.NORMAL,
1671: Token.SPECIAL_SEQUENCE,
1672: Token.NORMAL,
1673: Token.SPECIAL_SEQUENCE,
1674: Token.NORMAL,
1675: Token.SPECIAL_SEQUENCE,
1676: Token.NORMAL,
1677: Token.SPECIAL_SEQUENCE,
1678: Token.NORMAL,
1679: Token.SPECIAL_SEQUENCE,
1680: Token.NORMAL,
1681: Token.SPECIAL_SEQUENCE,
1682: Token.SPECIAL_SEQUENCE,
1683: Token.NORMAL,
1684: Token.SPECIAL_SEQUENCE,
1685: Token.SPECIAL_SEQUENCE, //" for (int index = 1; args != null && index < args.length; ++index) {\n"
1686: Token.NORMAL,
1687: Token.SPECIAL_SEQUENCE,
1688: Token.NORMAL,
1689: Token.SPECIAL_SEQUENCE,
1690: Token.NORMAL,
1691: Token.SPECIAL_SEQUENCE,
1692: Token.SPECIAL_SEQUENCE,
1693: Token.NORMAL,
1694: Token.SPECIAL_SEQUENCE,
1695: Token.NORMAL,
1696: Token.SPECIAL_SEQUENCE,
1697: Token.SPECIAL_SEQUENCE,
1698: Token.STRING,
1699: Token.SPECIAL_SEQUENCE,
1700: Token.SPECIAL_SEQUENCE, // " if (args[index].charAt(0) == '-') {\n"
1701: Token.LINE_COMMENT, // " // options\n"
1702: Token.NORMAL,
1703: Token.SPECIAL_SEQUENCE,
1704: Token.NORMAL,
1705: Token.SPECIAL_SEQUENCE,
1706: Token.NORMAL,
1707: Token.SPECIAL_SEQUENCE,
1708: Token.SPECIAL_SEQUENCE,
1709: Token.NORMAL,
1710: Token.SPECIAL_SEQUENCE,
1711: Token.NORMAL,
1712: Token.SPECIAL_SEQUENCE,
1713: Token.SPECIAL_SEQUENCE,
1714: Token.SPECIAL_SEQUENCE, // " switch (args[index].charAt(1)) {\n"
1715: Token.NORMAL,
1716: Token.STRING,
1717: Token.SEPARATOR, // " case 'd':\n"
1718: Token.NORMAL,
1719: Token.SPECIAL_SEQUENCE,
1720: Token.NORMAL,
1721: Token.SPECIAL_SEQUENCE,
1722: Token.NORMAL,
1723: Token.SPECIAL_SEQUENCE,
1724: Token.NORMAL,
1725: Token.SPECIAL_SEQUENCE,
1726: Token.NORMAL,
1727: Token.SPECIAL_SEQUENCE,
1728: Token.SPECIAL_SEQUENCE,
1729: Token.NORMAL,
1730: Token.SPECIAL_SEQUENCE,
1731: Token.NORMAL,
1732: Token.SPECIAL_SEQUENCE,
1733: Token.SPECIAL_SEQUENCE,
1734: Token.SPECIAL_SEQUENCE,
1735: Token.NORMAL,
1736: Token.SPECIAL_SEQUENCE,
1737: Token.SPECIAL_SEQUENCE,
1738: Token.SPECIAL_SEQUENCE, // " doubleValue = Double.valueOf(args[index].substring(2)).doubleValue();\n"
1739: Token.NORMAL,
1740: Token.SPECIAL_SEQUENCE, // " break;\n"
1741: Token.NORMAL,
1742: Token.STRING,
1743: Token.SEPARATOR, // " case 's':\n"
1744: Token.NORMAL,
1745: Token.SPECIAL_SEQUENCE,
1746: Token.NORMAL,
1747: Token.SPECIAL_SEQUENCE,
1748: Token.NORMAL,
1749: Token.SPECIAL_SEQUENCE,
1750: Token.SPECIAL_SEQUENCE,
1751: Token.NORMAL,
1752: Token.SPECIAL_SEQUENCE,
1753: Token.NORMAL,
1754: Token.SPECIAL_SEQUENCE,
1755: Token.SPECIAL_SEQUENCE, // " stringValue = args[index].substring(2);\n"
1756: Token.NORMAL,
1757: Token.SPECIAL_SEQUENCE, // " break;\n"
1758: Token.NORMAL,
1759: Token.STRING,
1760: Token.SEPARATOR, // " case 'i':\n"
1761: Token.NORMAL,
1762: Token.SPECIAL_SEQUENCE,
1763: Token.NORMAL,
1764: Token.SPECIAL_SEQUENCE,
1765: Token.NORMAL,
1766: Token.SPECIAL_SEQUENCE,
1767: Token.NORMAL,
1768: Token.SPECIAL_SEQUENCE,
1769: Token.NORMAL,
1770: Token.SPECIAL_SEQUENCE,
1771: Token.SPECIAL_SEQUENCE,
1772: Token.NORMAL,
1773: Token.SPECIAL_SEQUENCE,
1774: Token.NORMAL,
1775: Token.SPECIAL_SEQUENCE,
1776: Token.SPECIAL_SEQUENCE,
1777: Token.SPECIAL_SEQUENCE,
1778: Token.NORMAL,
1779: Token.SPECIAL_SEQUENCE,
1780: Token.SPECIAL_SEQUENCE,
1781: Token.SPECIAL_SEQUENCE, // " intValue = Integer.valueOf(args[index].substring(2)).intValue();\n"
1782: Token.NORMAL,
1783: Token.SPECIAL_SEQUENCE, // " break;\n"
1784: Token.NORMAL,
1785: Token.SEPARATOR, // " default:\n"
1786: Token.NORMAL,
1787: Token.SPECIAL_SEQUENCE,
1788: Token.STRING,
1789: Token.SPECIAL_SEQUENCE, // " stringValue = \"\";\n"
1790: Token.NORMAL,
1791: Token.SPECIAL_SEQUENCE,
1792: Token.NORMAL,
1793: Token.SPECIAL_SEQUENCE,
1794: Token.NORMAL,
1795: Token.SPECIAL_SEQUENCE, // " doubleValue = 0.0;\n"
1796: Token.NORMAL,
1797: Token.SPECIAL_SEQUENCE,
1798: Token.NORMAL,
1799: Token.SPECIAL_SEQUENCE, // " intValue = 0;\n"
1800: Token.SPECIAL_SEQUENCE, // " }\n"
1801: Token.SPECIAL_SEQUENCE,
1802: Token.NORMAL,
1803: Token.SPECIAL_SEQUENCE, // " } else {\n"
1804: Token.LINE_COMMENT, // " // normal arguments\n"
1805: Token.NORMAL,
1806: Token.SPECIAL_SEQUENCE,
1807: Token.SPECIAL_SEQUENCE,
1808: Token.NORMAL,
1809: Token.SPECIAL_SEQUENCE,
1810: Token.NORMAL,
1811: Token.SPECIAL_SEQUENCE,
1812: Token.NORMAL,
1813: Token.SPECIAL_SEQUENCE,
1814: Token.NORMAL,
1815: Token.SPECIAL_SEQUENCE,
1816: Token.SPECIAL_SEQUENCE,
1817: Token.SPECIAL_SEQUENCE,
1818: Token.SPECIAL_SEQUENCE, // " if ( ! argStore.contains(args[index])) {\n"
1819: Token.NORMAL,
1820: Token.SPECIAL_SEQUENCE,
1821: Token.NORMAL,
1822: Token.SPECIAL_SEQUENCE,
1823: Token.NORMAL,
1824: Token.SPECIAL_SEQUENCE,
1825: Token.NORMAL,
1826: Token.SPECIAL_SEQUENCE,
1827: Token.SPECIAL_SEQUENCE,
1828: Token.SPECIAL_SEQUENCE, // " argStore.add(args[index]);\n"
1829: Token.SPECIAL_SEQUENCE,
1830: Token.NORMAL,
1831: Token.SPECIAL_SEQUENCE, // " } else {\n"
1832: Token.NORMAL, Token.SPECIAL_SEQUENCE,
1833: Token.NORMAL,
1834: Token.SPECIAL_SEQUENCE,
1835: Token.NORMAL,
1836: Token.SPECIAL_SEQUENCE,
1837: Token.STRING,
1838: Token.SPECIAL_SEQUENCE,
1839: Token.NORMAL,
1840: Token.SPECIAL_SEQUENCE,
1841: Token.NORMAL,
1842: Token.SPECIAL_SEQUENCE,
1843: Token.SPECIAL_SEQUENCE,
1844: Token.STRING,
1845: Token.SPECIAL_SEQUENCE,
1846: Token.SPECIAL_SEQUENCE, // " System.out.println(\"Duplicate element \\\" + args[index] + \"\\\".\");\n"
1847: Token.BLOCK_COMMENT, // " /* perhaps better use Environment.getEnvironment(this).out().println() */\n"
1848: Token.SPECIAL_SEQUENCE, // " }\n"
1849: Token.SPECIAL_SEQUENCE, // " }\n"
1850: Token.SPECIAL_SEQUENCE, // " }\n"
1851: Token.SPECIAL_SEQUENCE, // " }\n"
1852: Token.BLOCK_COMMENT, Token.NORMAL, Token.NORMAL,
1853: Token.NORMAL, Token.SPECIAL_SEQUENCE, Token.NORMAL,
1854: Token.SPECIAL_SEQUENCE, // " private ArrayList argStore = null;\n"
1855: Token.SPECIAL_SEQUENCE // "}\n"
1856: };
1857:
1858: // Tokenizer properties
1859: private static final Object JAVADOC_COMPANION = new Object();
1860: private static final Object BLOCK_COMMENT_COMPANION = new Object();
1861: private static final Object LINE_COMMENT_COMPANION = new Object();
1862: private static final Object STRING_COMPANION = new Object();
1863: private static final Object CHAR_COMPANION = new Object();
1864: private static final Object BRACE_CLOSE_COMPANION = new Object();
1865: private static final Object COLON_COMPANION = new Object();
1866: private static final Object SEMICOLON_COMPANION = new Object();
1867: private static final Object STAR_COMPANION = new Object();
1868:
1869: // Properties
1870: static final TokenizerProperty[] _javaProperties = {
1871: new TokenizerProperty(Token.BLOCK_COMMENT, new String[] {
1872: "/**", "*/" }, JAVADOC_COMPANION),
1873: new TokenizerProperty(Token.BLOCK_COMMENT, new String[] {
1874: "/*", "*/" }, BLOCK_COMMENT_COMPANION),
1875: new TokenizerProperty(Token.LINE_COMMENT,
1876: new String[] { "//" }, LINE_COMMENT_COMPANION),
1877: new TokenizerProperty(Token.STRING, new String[] { "\"",
1878: "\"", "\\" }, STRING_COMPANION),
1879: new TokenizerProperty(Token.STRING, new String[] { "'",
1880: "'", "\\" }, CHAR_COMPANION),
1881: new TokenizerProperty(Token.PATTERN,
1882: new String[] { "[+\\-]?[0-9]+\\.?[0-9]*" }),
1883: new TokenizerProperty(Token.KEYWORD,
1884: new String[] { "package" }),
1885: new TokenizerProperty(Token.KEYWORD,
1886: new String[] { "abstract" }),
1887: new TokenizerProperty(Token.KEYWORD,
1888: new String[] { "public" }),
1889: new TokenizerProperty(Token.KEYWORD,
1890: new String[] { "protected" }),
1891: new TokenizerProperty(Token.KEYWORD,
1892: new String[] { "private" }),
1893: new TokenizerProperty(Token.KEYWORD,
1894: new String[] { "class" }),
1895: new TokenizerProperty(Token.KEYWORD,
1896: new String[] { "final" }),
1897: new TokenizerProperty(Token.KEYWORD,
1898: new String[] { "static" }),
1899: new TokenizerProperty(Token.KEYWORD,
1900: new String[] { "interface" }),
1901: new TokenizerProperty(Token.KEYWORD,
1902: new String[] { "extends" }),
1903: new TokenizerProperty(Token.KEYWORD,
1904: new String[] { "implements" }),
1905: new TokenizerProperty(Token.KEYWORD,
1906: new String[] { "synchronized" }),
1907: new TokenizerProperty(Token.KEYWORD,
1908: new String[] { "null" }),
1909: new TokenizerProperty(Token.KEYWORD,
1910: new String[] { "this" }),
1911: new TokenizerProperty(Token.KEYWORD,
1912: new String[] { "super" }),
1913: new TokenizerProperty(Token.KEYWORD, new String[] { "new" }),
1914: new TokenizerProperty(Token.KEYWORD,
1915: new String[] { "void" }),
1916: new TokenizerProperty(Token.KEYWORD,
1917: new String[] { "byte" }),
1918: new TokenizerProperty(Token.KEYWORD,
1919: new String[] { "char" }),
1920: new TokenizerProperty(Token.KEYWORD,
1921: new String[] { "short" }),
1922: new TokenizerProperty(Token.KEYWORD, new String[] { "int" }),
1923: new TokenizerProperty(Token.KEYWORD,
1924: new String[] { "long" }),
1925: new TokenizerProperty(Token.KEYWORD,
1926: new String[] { "double" }),
1927: new TokenizerProperty(Token.KEYWORD,
1928: new String[] { "float" }),
1929: new TokenizerProperty(Token.KEYWORD,
1930: new String[] { "String" }),
1931: new TokenizerProperty(Token.KEYWORD,
1932: new String[] { "throws" }),
1933: new TokenizerProperty(Token.KEYWORD,
1934: new String[] { "static" }),
1935: new TokenizerProperty(Token.KEYWORD,
1936: new String[] { "import" }),
1937: new TokenizerProperty(Token.KEYWORD,
1938: new String[] { "package" }),
1939: new TokenizerProperty(Token.KEYWORD, new String[] { "if" }),
1940: new TokenizerProperty(Token.KEYWORD,
1941: new String[] { "else" }),
1942: new TokenizerProperty(Token.KEYWORD, new String[] { "for" }),
1943: new TokenizerProperty(Token.KEYWORD,
1944: new String[] { "while" }),
1945: new TokenizerProperty(Token.KEYWORD,
1946: new String[] { "switch" }),
1947: new TokenizerProperty(Token.KEYWORD,
1948: new String[] { "case" }),
1949: new TokenizerProperty(Token.KEYWORD,
1950: new String[] { "break" }),
1951: new TokenizerProperty(Token.KEYWORD,
1952: new String[] { "default" }),
1953: new TokenizerProperty(Token.KEYWORD,
1954: new String[] { "continue" }),
1955: new TokenizerProperty(Token.KEYWORD,
1956: new String[] { "goto" }),
1957: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1958: new String[] { "." }),
1959: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1960: new String[] { ";" }),
1961: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1962: new String[] { "," }),
1963: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1964: new String[] { "=" }),
1965: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1966: new String[] { "==" }),
1967: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1968: new String[] { "!=" }),
1969: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1970: new String[] { ">" }),
1971: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1972: new String[] { "<" }),
1973: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1974: new String[] { ">=" }),
1975: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1976: new String[] { "<=" }),
1977: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1978: new String[] { "+=" }),
1979: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1980: new String[] { "-=" }),
1981: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1982: new String[] { "*=" }),
1983: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1984: new String[] { "/=" }),
1985: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1986: new String[] { ">>=" }),
1987: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1988: new String[] { "<<=" }),
1989: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1990: new String[] { "+" }),
1991: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1992: new String[] { "-" }),
1993: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1994: new String[] { "*" }),
1995: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1996: new String[] { "/" }),
1997: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
1998: new String[] { "++" }),
1999: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2000: new String[] { "--" }),
2001: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2002: new String[] { ">>" }),
2003: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2004: new String[] { "<<" }),
2005: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2006: new String[] { ">>>" }),
2007: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2008: new String[] { "&" }),
2009: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2010: new String[] { "|" }),
2011: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2012: new String[] { "^" }),
2013: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2014: new String[] { "&&" }),
2015: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2016: new String[] { "||" }),
2017: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2018: new String[] { "!" }),
2019: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2020: new String[] { "{" }),
2021: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2022: new String[] { "}" }),
2023: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2024: new String[] { "(" }),
2025: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2026: new String[] { ")" }),
2027: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2028: new String[] { "[" }),
2029: new TokenizerProperty(Token.SPECIAL_SEQUENCE,
2030: new String[] { "]" }) };
2031:
2032: // ratio between small and large file
2033: private static final int SMALL_LOOPS = 10;
2034: private static final double LARGE_SMALL_RATIO = 100.0;
2035:
2036: // Test message format
2037: private static final String TEST_MESSAGE = System.getProperty(
2038: "line.separator", "\n")
2039: + "{0}, flags \"{1}\", types \"{2}\": {3} bytes.";
2040:
2041: //---------------------------------------------------------------------------
2042: // Members
2043: //
2044: private TokenizerProperties _properties = null;
2045: private int _flags = 0;
2046: private long _typeMask = 0;
2047:
2048: //---------------------------------------------------------------------------
2049: // class members
2050: //
2051: private static File _smallFile = null;
2052: private static File _largeFile = null;
2053: }
|