01: package org.acm.seguin.pmd.cpd;
02:
03: import net.sourceforge.jrefactory.io.JavaCharStream;
04: import net.sourceforge.jrefactory.parser.JavaParserTokenManager;
05: import net.sourceforge.jrefactory.parser.Token;
06: import org.acm.seguin.pmd.PMD;
07:
08: import java.io.IOException;
09: import java.io.LineNumberReader;
10: import java.io.Reader;
11: import java.io.StringReader;
12: import java.util.ArrayList;
13: import java.util.List;
14:
15: public class JavaTokenizer implements Tokenizer {
16:
17: public void tokenize(SourceCode tokens, Tokens tokenEntries,
18: Reader input) throws IOException {
19: // first get a snapshot of the code
20: List lines = new ArrayList();
21: StringBuffer sb = new StringBuffer();
22: LineNumberReader r = new LineNumberReader(input);
23: String currentLine;
24: while ((currentLine = r.readLine()) != null) {
25: lines.add(currentLine);
26: sb.append(currentLine);
27: sb.append(PMD.EOL);
28: }
29: tokens.setCode(lines);
30:
31: // now tokenize it
32: /*
33: I'm doing a sort of State pattern thing here where
34: this goes into "discarding" mode when it hits an import or package
35: keyword and goes back into "accumulate mode when it hits a semicolon.
36: This could probably be turned into some objects.
37: */
38: JavaCharStream javaStream = new JavaCharStream(
39: new StringReader(sb.toString()));
40: JavaParserTokenManager tokenMgr = new JavaParserTokenManager(
41: javaStream);
42: Token currToken = tokenMgr.getNextToken();
43: boolean discarding = false;
44: int count = 0;
45: while (currToken.image != "") {
46: if (currToken.image.equals("import")
47: || currToken.image.equals("package")) {
48: discarding = true;
49: currToken = tokenMgr.getNextToken();
50: continue;
51: }
52:
53: if (discarding && currToken.image.equals(";")) {
54: discarding = false;
55: }
56:
57: if (discarding) {
58: currToken = tokenMgr.getNextToken();
59: continue;
60: }
61:
62: if (!currToken.image.equals(";")) {
63: count++;
64: tokenEntries.add(new TokenEntry(currToken.image, count,
65: tokens.getFileName(), currToken.beginLine));
66: }
67:
68: currToken = tokenMgr.getNextToken();
69: }
70: tokenEntries.add(TokenEntry.EOF);
71: }
72: }
|