001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.lib.lexer.test.dump;
043:
044: import java.io.File;
045: import java.io.FileNotFoundException;
046: import java.io.FileReader;
047: import java.io.FileWriter;
048: import java.nio.CharBuffer;
049: import java.util.ArrayList;
050: import java.util.Arrays;
051: import java.util.List;
052: import org.netbeans.api.lexer.Language;
053: import org.netbeans.api.lexer.Token;
054: import org.netbeans.api.lexer.TokenHierarchy;
055: import org.netbeans.api.lexer.TokenHierarchy;
056: import org.netbeans.api.lexer.TokenId;
057: import org.netbeans.api.lexer.TokenSequence;
058: import org.netbeans.api.lexer.TokenUtilities;
059: import org.netbeans.junit.NbTestCase;
060: import org.netbeans.lib.editor.util.CharSequenceUtilities;
061: import org.netbeans.lib.lexer.batch.BatchTokenList;
062: import org.netbeans.lib.lexer.test.LexerTestUtilities;
063:
064: /**
065: * Check whether generated token dump corresponds to the one read from a file.
066: *
067: * @author mmetelka
068: */
069: public final class TokenDumpCheck {
070:
071: public static void checkTokenDump(NbTestCase test,
072: String relFilePath, Language<?> language) throws Exception {
073: // Request lookaheads and states maintaining
074: boolean origMaintainLAState = BatchTokenList
075: .isMaintainLAState();
076: BatchTokenList.setMaintainLAState(true);
077: try {
078: File wholeInputFile = new File(test.getDataDir(),
079: relFilePath);
080: if (!wholeInputFile.exists()) {
081: NbTestCase.fail("File " + wholeInputFile
082: + " not found.");
083: }
084: String wholeInput = readFile(test, wholeInputFile);
085: // Scan for EOF markers and create multiple char sequences
086: TokenHierarchy<?> hi = TokenHierarchy.create(wholeInput,
087: TokenDumpTokenId.language());
088: TokenSequence<TokenDumpTokenId> ts = hi
089: .tokenSequence(TokenDumpTokenId.language());
090: boolean afterEOF = true; // Ignore newlines when after eof
091: boolean newline = false;
092: int textStartIndex = 0;
093: List<String> inputs = new ArrayList<String>();
094: List<String> testNames = new ArrayList<String>();
095: // Estimate 4 subtests equally long
096: StringBuilder inputBuffer = new StringBuilder(wholeInput
097: .length() / 4);
098: String testName = "<Unnamed test>";
099: while (ts.moveNext()) {
100: Token<TokenDumpTokenId> token = ts.token();
101: switch (token.id()) {
102: case NEWLINE:
103: if (newline) { // Was empty line
104: inputBuffer.append('\n');
105: }
106: if (!afterEOF) {
107: newline = true;
108: }
109: break;
110:
111: case TEST_NAME:
112: testName = token.text().toString();
113: ts.moveNext(); // skip newline (might be false for EOF)
114: newline = false;
115: break;
116:
117: case EOF_VIRTUAL:
118: // All except newline-eof_mark-newline
119: ts.moveNext(); // skip newline (might be false for EOF)
120: newline = false;
121: inputs.add(inputBuffer.toString());
122: inputBuffer.setLength(0);
123: testNames.add(testName);
124: testName = "<Unnamed test>";
125: afterEOF = true;
126: break;
127:
128: case TEXT:
129: if (newline) {
130: inputBuffer.append('\n');
131: newline = false;
132: }
133: inputBuffer.append(token.text());
134: afterEOF = false;
135: break;
136:
137: default:
138: if (TokenDumpTokenId.isCharLiteral(token.id())) {
139: Character ch = (Character) token
140: .getProperty(TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY);
141: assert (ch != null);
142: inputBuffer.append(ch);
143: } else {
144: throw new IllegalStateException(
145: "Unknown token id=" + token.id());
146: }
147: ts.moveNext(); // skip newline (might be false for EOF)
148: newline = false;
149: afterEOF = false;
150:
151: }
152: }
153: inputs.add(inputBuffer.toString());
154: testNames.add(testName);
155:
156: // Check whether token dump file exists
157: // Try to remove "/build/" from the dump file name if it exists.
158: // Otherwise give a warning.
159: File tokenDescInputFile = new File(test.getDataDir(),
160: relFilePath + ".tokens.txt");
161: String tokenDescInputFilePath = tokenDescInputFile
162: .getAbsolutePath();
163: boolean replaced = false;
164: if (tokenDescInputFilePath.indexOf("/build/test/") != -1) {
165: tokenDescInputFilePath = tokenDescInputFilePath
166: .replace("/build/test/", "/test/");
167: replaced = true;
168: }
169: if (!replaced
170: && tokenDescInputFilePath
171: .indexOf("/test/work/sys/") != -1) {
172: tokenDescInputFilePath = tokenDescInputFilePath
173: .replace("/test/work/sys/", "/test/unit/");
174: replaced = true;
175: }
176: if (!replaced) {
177: System.err
178: .println("Warning: Attempt to use tokens dump file "
179: + "from sources instead of the generated test files failed.\n"
180: + "Patterns '/build/test/' or '/test/work/sys/' not found in "
181: + tokenDescInputFilePath);
182: }
183: tokenDescInputFile = new File(tokenDescInputFilePath);
184:
185: String tokenDescInput = null;
186: if (tokenDescInputFile.exists()) {
187: tokenDescInput = readFile(test, tokenDescInputFile);
188: }
189: TokenDescCompare tdc = new TokenDescCompare(tokenDescInput,
190: tokenDescInputFile);
191:
192: // Check individual token descriptions
193: StringBuilder tokenDesc = new StringBuilder(40);
194: for (int i = 0; i < inputs.size(); i++) {
195: String input = inputs.get(i);
196: testName = testNames.get(i);
197: tdc.setTestName(testName);
198: TokenHierarchy<?> langHi = TokenHierarchy.create(input,
199: language);
200: TokenSequence<?> langTS = langHi.tokenSequence();
201: tdc.compareLine(testName, -1);
202: while (langTS.moveNext()) {
203: // Debug the token
204: Token<?> token = langTS.token();
205: tokenDesc.append(token.id().name());
206: int spaceCount = 14 - token.id().name().length();
207: while (--spaceCount >= 0) {
208: tokenDesc.append(' ');
209: }
210: tokenDesc.append(" \"");
211: tokenDesc.append(TokenUtilities.debugText(token
212: .text()));
213: tokenDesc.append('"');
214: int lookahead = LexerTestUtilities
215: .lookahead(langTS);
216: if (lookahead > 0) {
217: tokenDesc.append(", la=");
218: tokenDesc.append(lookahead);
219: }
220: Object state = LexerTestUtilities.state(langTS);
221: if (state != null) {
222: tokenDesc.append(", st=");
223: tokenDesc.append(state);
224: }
225: tdc.compareLine(tokenDesc, langTS.index());
226: tokenDesc.setLength(0);
227: }
228: tdc.compareLine("----- EOF -----\n", -1);
229: }
230: tdc.finish(); // Write token desc file if necessary
231:
232: } finally {
233: BatchTokenList.setMaintainLAState(origMaintainLAState);
234: }
235: }
236:
237: private static String readFile(NbTestCase test, File f)
238: throws Exception {
239: FileReader r = new FileReader(f);
240: int fileLen = (int) f.length();
241: CharBuffer cb = CharBuffer.allocate(fileLen);
242: r.read(cb);
243: cb.rewind();
244: return cb.toString();
245: }
246:
247: private static final class TokenDescCompare implements CharSequence {
248:
249: private String input;
250:
251: private int inputIndex;
252:
253: private int textLength;
254:
255: private StringBuilder output;
256:
257: private File outputFile;
258:
259: private int[] lineBoundsIndexes = new int[2 * 3]; // last three lines [start,end]
260:
261: private String testName;
262:
263: TokenDescCompare(String input, File outputFile) {
264: this .input = input;
265: this .outputFile = outputFile;
266: Arrays.fill(lineBoundsIndexes, -1);
267: if (input == null)
268: output = new StringBuilder(100);
269: }
270:
271: public void compareLine(CharSequence text, int tokenIndex) {
272: if (input != null) {
273: textLength = text.length();
274: if (input.length() - inputIndex < textLength
275: || !TokenUtilities.equals(text, this )) {
276: StringBuilder msg = new StringBuilder(100);
277: msg.append("\nDump file ");
278: msg.append(outputFile);
279: msg.append(":\n");
280: msg.append(testName);
281: msg.append(":\n");
282: if (tokenIndex >= 0) {
283: msg
284: .append("Invalid token description in dump file (tokenIndex=");
285: msg.append(tokenIndex);
286: msg.append("):");
287: } else {
288: msg.append("Invalid text in dump file:");
289: }
290: msg.append("\n ");
291: msg.append(input.subSequence(inputIndex,
292: findEOL(inputIndex)));
293: msg.append("\nExpected:\n ");
294: msg.append(text);
295: msg.append("\nPrevious context:\n");
296: for (int i = 0; i < lineBoundsIndexes.length; i += 2) {
297: int start = lineBoundsIndexes[i];
298: if (start != -1) {
299: msg.append(" ");
300: msg.append(input.subSequence(start,
301: lineBoundsIndexes[i + 1]));
302: msg.append('\n');
303: }
304: }
305: NbTestCase.fail(msg.toString());
306:
307: }
308: System.arraycopy(lineBoundsIndexes, 2,
309: lineBoundsIndexes, 0,
310: lineBoundsIndexes.length - 2);
311: lineBoundsIndexes[lineBoundsIndexes.length - 2] = inputIndex;
312: inputIndex += textLength;
313: lineBoundsIndexes[lineBoundsIndexes.length - 1] = inputIndex;
314: inputIndex = skipEOL(inputIndex);
315:
316: } else {
317: output.append(text);
318: String ls = (String) System
319: .getProperty("line.separator"); // NOI18N
320: output.append(ls);
321: }
322: }
323:
324: public void finish() throws Exception {
325: if (input == null) {
326: if (!outputFile.createNewFile()) {
327: NbTestCase.fail("Cannot create file " + outputFile);
328: }
329: FileWriter fw = new FileWriter(outputFile);
330: try {
331: fw.write(output.toString());
332: } finally {
333: fw.close();
334: }
335: NbTestCase.fail("Created tokens dump file "
336: + outputFile + "\nPlease re-run the test.");
337:
338: } else {
339: if (inputIndex < input.length()) {
340: NbTestCase.fail("Some text left unread:"
341: + input.substring(inputIndex));
342: }
343: }
344: }
345:
346: public void setTestName(String testName) {
347: this .testName = testName;
348: }
349:
350: public char charAt(int index) {
351: CharSequenceUtilities.checkIndexValid(index, length());
352: return input.charAt(inputIndex + index);
353: }
354:
355: public int length() {
356: return textLength;
357: }
358:
359: public CharSequence subSequence(int start, int end) {
360: CharSequenceUtilities.checkIndexesValid(this , start, end);
361: return input
362: .substring(inputIndex + start, inputIndex + end);
363: }
364:
365: private int findEOL(int start) {
366: while (start < input.length()) {
367: switch (input.charAt(start)) {
368: case '\r':
369: case '\n':
370: return start;
371: }
372: start++;
373: }
374: return start;
375: }
376:
377: private int skipEOL(int index) {
378: if (index < input.length()) {
379: index++; // skip separator char
380: if (input.charAt(index - 1) == '\r')
381: if (index < input.length()
382: && input.charAt(index) == '\n')
383: index++; // CRLF
384: }
385: return index;
386: }
387:
388: }
389:
390: }
|