01: /**
02: * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
03: */package net.sourceforge.pmd.cpd;
04:
05: import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
06: import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
07: import net.sourceforge.pmd.cpd.cppast.Token;
08: import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
09:
10: import java.io.StringReader;
11:
12: public class CPPTokenizer implements Tokenizer {
13: private static SimpleCharStream charStream;
14:
15: public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
16: StringBuffer sb = sourceCode.getCodeBuffer();
17: try {
18: if (charStream == null) {
19: charStream = new SimpleCharStream(new StringReader(sb
20: .toString()));
21: } else {
22: charStream.ReInit(new StringReader(sb.toString()));
23: }
24: CPPParserTokenManager.ReInit(charStream);
25: CPPParserTokenManager.setFileName(sourceCode.getFileName());
26: Token currToken = CPPParserTokenManager.getNextToken();
27: while (currToken.image.length() > 0) {
28: tokenEntries.add(new TokenEntry(currToken.image,
29: sourceCode.getFileName(), currToken.beginLine));
30: currToken = CPPParserTokenManager.getNextToken();
31: }
32: tokenEntries.add(TokenEntry.getEOF());
33: System.err.println("Added " + sourceCode.getFileName());
34: } catch (TokenMgrError err) {
35: err.printStackTrace();
36: System.err.println("Skipping " + sourceCode.getFileName()
37: + " due to parse error");
38: tokenEntries.add(TokenEntry.getEOF());
39: }
40: }
41: }
|