01: package fri.patterns.interpreter.parsergenerator.lexer.examples;
02:
03: import java.io.*;
04: import fri.patterns.interpreter.parsergenerator.Token;
05: import fri.patterns.interpreter.parsergenerator.Lexer;
06: import fri.patterns.interpreter.parsergenerator.lexer.*;
07: import fri.patterns.interpreter.parsergenerator.syntax.Syntax;
08: import fri.patterns.interpreter.parsergenerator.syntax.builder.SyntaxSeparation;
09:
10: /**
11: Sample showing how to build a Lexer from StandardLexerRules building blocks.
12: This Lexer scans C/Java stringdefs, chardefs, integer, float, identifiers, comments,
13: and everything else as "verbatim".
14:
15: @author (c) 2002, Fritz Ritzberger
16: */
17:
18: public abstract class ExampleLexer {
19: /** Test main. */
20: public static void main(String[] args) {
21: String[][] rules = {
22: { "token", "stringdef" },
23: { "token", "chardef" },
24: { "token", "identifier" },
25: { "token", "float" },
26: { "token", "integer" },
27: { "token", "verbatim" },
28: { "verbatim", "char", Token.BUTNOT, "space",
29: Token.BUTNOT, "newline", Token.BUTNOT,
30: "letter", Token.BUTNOT, "digit", Token.BUTNOT,
31: "stringdef", Token.BUTNOT, "chardef",
32: Token.BUTNOT, "comment" },
33: { "ignored", "comment" }, { "ignored", "spaces" },
34: { "ignored", "newlines" }, };
35: Syntax syntax = new Syntax(rules);
36:
37: try {
38: fri.util.TimeStopper ts = new fri.util.TimeStopper();
39:
40: SyntaxSeparation separation = new SyntaxSeparation(syntax);
41: LexerBuilder builder = new LexerBuilder(separation
42: .getLexerSyntax(), separation.getIgnoredSymbols());
43: Lexer lexer = builder.getLexer();
44: lexer.setDebug(true);
45: lexer.setTerminals(separation.getTokenSymbols());
46:
47: System.err.println("time to build lexer was: "
48: + ts.getInterval());
49:
50: InputStream in = ExampleLexer.class
51: .getResourceAsStream("ExampleLexer.java"); //Reader in = new FileReader(args[0]);
52: lexer.setInput(in);
53:
54: System.err.println("time to build input was: "
55: + ts.getInterval());
56:
57: Token t;
58: do {
59: t = lexer.getNextToken(null);
60: //System.err.println(ts.getInterval()+" was time to read token "+t.symbol+" -> "+t.text);
61:
62: if (t.symbol == null) {
63: lexer.dump(System.err);
64: throw new LexerException("Uninterpretable input!");
65: }
66: System.out.println(t.symbol + " " + ">" + t.text + "<");
67: } while (Token.isEpsilon(t) == false);
68:
69: System.err.println("time to scan input was: "
70: + ts.getTimeMillis());
71: } catch (Exception e) {
72: e.printStackTrace();
73: }
74:
75: // some complicated Java expressions, for test lexing this source file
76: char testChar1 = 'c';
77: char testChar2 = '\r';
78: char testChar3 = '\000';
79: char testChar4 = '\377';
80: String testString1 = "\\\"\\/* comment in string */";
81: String testString2 = "///*|<>@,;.:-_#'+*~´`ßäöü\"ÄÖÜ^°§%&/[][]{}";
82: float testFloat = 123.456E-12f;
83: double testDouble = .456d;
84: int testInt1 = 0X12aB;
85: int testInt2 = 07654;
86: System.err.println("Zahl 07654 ist: " + testInt2);
87: long testLong1 = 123456L;
88: }
89:
90: }
|