01: package org.acm.seguin.pmd.cpd;
02:
03: import org.acm.seguin.pmd.cpd.cppast.CPPParser;
04: import org.acm.seguin.pmd.cpd.cppast.CPPParserTokenManager;
05: import org.acm.seguin.pmd.cpd.cppast.Token;
06: import org.acm.seguin.pmd.cpd.cppast.TokenMgrError;
07:
08: import java.io.IOException;
09: import java.io.LineNumberReader;
10: import java.io.Reader;
11: import java.io.StringReader;
12: import java.util.ArrayList;
13: import java.util.List;
14:
15: public class CPPTokenizer implements Tokenizer {
16: protected String EOL = System.getProperty("line.separator", "\n");
17:
18: private static boolean initted;
19:
20: public void tokenize(SourceCode sourceCode, Tokens tokenEntries,
21: Reader input) throws IOException {
22: try {
23: // first get a snapshot of the code
24: List lines = new ArrayList();
25: StringBuffer sb = new StringBuffer();
26: LineNumberReader r = new LineNumberReader(input);
27: String currentLine;
28: while ((currentLine = r.readLine()) != null) {
29: lines.add(currentLine);
30: sb.append(currentLine);
31: sb.append(EOL);
32: }
33: sourceCode.setCode(lines);
34:
35: // now tokenize it
36: if (!initted) {
37: new CPPParser(new StringReader(sb.toString()));
38: initted = true;
39: }
40: CPPParser.ReInit(new StringReader(sb.toString()));
41: Token currToken = CPPParserTokenManager.getNextToken();
42: int count = 0;
43: while (currToken.image != "") {
44: count++;
45: tokenEntries.add(new TokenEntry(currToken.image, count,
46: sourceCode.getFileName(), currToken.beginLine));
47: currToken = CPPParserTokenManager.getNextToken();
48: }
49: tokenEntries.add(TokenEntry.EOF);
50: System.out.println("Added " + sourceCode.getFileName());
51: } catch (TokenMgrError err) {
52: System.out.println("Skipping " + sourceCode.getFileName()
53: + " due to parse error");
54: List emptyCode = new ArrayList();
55: emptyCode.add("");
56: sourceCode.setCode(emptyCode);
57: tokenEntries.add(TokenEntry.EOF);
58: }
59: }
60: }
|