001: /*
002: [The "BSD licence"]
003: Copyright (c) 2005-2006 Terence Parr
004: All rights reserved.
005:
006: Redistribution and use in source and binary forms, with or without
007: modification, are permitted provided that the following conditions
008: are met:
009: 1. Redistributions of source code must retain the above copyright
010: notice, this list of conditions and the following disclaimer.
011: 2. Redistributions in binary form must reproduce the above copyright
012: notice, this list of conditions and the following disclaimer in the
013: documentation and/or other materials provided with the distribution.
014: 3. The name of the author may not be used to endorse or promote products
015: derived from this software without specific prior written permission.
016:
017: THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
018: IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
019: OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
020: IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
021: INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
022: NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
023: DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
024: THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
025: (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
026: THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
027: */
028: package org.antlr.test;
029:
030: public class TestLexer extends BaseTest {
031: protected boolean debug = false;
032:
033: /** Public default constructor used by TestRig */
034: public TestLexer() {
035: }
036:
037: public void testSetText() throws Exception {
038: // this must return A not I to the parser; calling a nonfragment rule
039: // from a nonfragment rule does not set the overall token.
040: String grammar = "grammar P;\n"
041: + "a : A {System.out.println(input);} ;\n"
042: + "A : '\\\\' 't' {setText(\"\t\");} ;\n"
043: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
044: String found = execParser("P.g", grammar, "PParser", "PLexer",
045: "a", "\\t", debug);
046: assertEquals("\t\n", found);
047: }
048:
049: public void testRefToRuleDoesNotSetTokenNorEmitAnother()
050: throws Exception {
051: // this must return A not I to the parser; calling a nonfragment rule
052: // from a nonfragment rule does not set the overall token.
053: String grammar = "grammar P;\n"
054: + "a : A EOF {System.out.println(input);} ;\n"
055: + "A : '-' I ;\n" + "I : '0'..'9'+ ;\n"
056: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
057: String found = execParser("P.g", grammar, "PParser", "PLexer",
058: "a", "-34", debug);
059: assertEquals("-34\n", found);
060: }
061:
062: public void testWeCanSetType() throws Exception {
063: String grammar = "grammar P;\n" + "tokens {X;}\n"
064: + "a : X EOF {System.out.println(input);} ;\n"
065: + "A : '-' I {$type = X;} ;\n" + "I : '0'..'9'+ ;\n"
066: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
067: String found = execParser("P.g", grammar, "PParser", "PLexer",
068: "a", "-34", debug);
069: assertEquals("-34\n", found);
070: }
071:
072: public void testRefToFragment() throws Exception {
073: // this must return A not I to the parser; calling a nonfragment rule
074: // from a nonfragment rule does not set the overall token.
075: String grammar = "grammar P;\n"
076: + "a : A {System.out.println(input);} ;\n"
077: + "A : '-' I ;\n" + "fragment I : '0'..'9'+ ;\n"
078: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
079: String found = execParser("P.g", grammar, "PParser", "PLexer",
080: "a", "-34", debug);
081: assertEquals("-34\n", found);
082: }
083:
084: public void testMultipleRefToFragment() throws Exception {
085: // this must return A not I to the parser; calling a nonfragment rule
086: // from a nonfragment rule does not set the overall token.
087: String grammar = "grammar P;\n"
088: + "a : A EOF {System.out.println(input);} ;\n"
089: + "A : I '.' I ;\n" + "fragment I : '0'..'9'+ ;\n"
090: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
091: String found = execParser("P.g", grammar, "PParser", "PLexer",
092: "a", "3.14159", debug);
093: assertEquals("3.14159\n", found);
094: }
095:
096: public void testLabelInSubrule() throws Exception {
097: // can we see v outside?
098: String grammar = "grammar P;\n"
099: + "a : A EOF ;\n"
100: + "A : 'hi' WS (v=I)? {$channel=0; System.out.println($v.text);} ;\n"
101: + "fragment I : '0'..'9'+ ;\n"
102: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
103: String found = execParser("P.g", grammar, "PParser", "PLexer",
104: "a", "hi 342", debug);
105: assertEquals("342\n", found);
106: }
107:
108: public void testRefToTokenInLexer() throws Exception {
109: String grammar = "grammar P;\n" + "a : A EOF ;\n"
110: + "A : I {System.out.println($I.text);} ;\n"
111: + "fragment I : '0'..'9'+ ;\n"
112: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
113: String found = execParser("P.g", grammar, "PParser", "PLexer",
114: "a", "342", debug);
115: assertEquals("342\n", found);
116: }
117:
118: public void testListLabelInLexer() throws Exception {
119: String grammar = "grammar P;\n"
120: + "a : A ;\n"
121: + "A : i+=I+ {for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n"
122: + "fragment I : '0'..'9'+ ;\n"
123: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
124: String found = execParser("P.g", grammar, "PParser", "PLexer",
125: "a", "33 297", debug);
126: assertEquals(" 33 297\n", found);
127: }
128:
129: public void testDupListRefInLexer() throws Exception {
130: String grammar = "grammar P;\n"
131: + "a : A ;\n"
132: + "A : i+=I WS i+=I {$channel=0; for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n"
133: + "fragment I : '0'..'9'+ ;\n"
134: + "WS : (' '|'\\n') {$channel=HIDDEN;} ;";
135: String found = execParser("P.g", grammar, "PParser", "PLexer",
136: "a", "33 297", debug);
137: assertEquals(" 33 297\n", found);
138: }
139:
140: public void testCharLabelInLexer() {
141: String grammar = "grammar T;\n" + "a : B ;\n"
142: + "B : x='a' {System.out.println((char)$x);} ;\n";
143: String found = execParser("T.g", grammar, "TParser", "TLexer",
144: "a", "a", debug);
145: assertEquals("a\n", found);
146: }
147:
148: public void testRepeatedLabelInLexer() {
149: String grammar = "lexer grammar t;\n" + "B : x='a' x='b' ;\n";
150: boolean found = rawGenerateAndBuildRecognizer("t.g", grammar,
151: null, "tLexer", false);
152: boolean expecting = true; // should be ok
153: assertEquals(expecting, found);
154: }
155:
156: public void testRepeatedRuleLabelInLexer() {
157: String grammar = "lexer grammar t;\n" + "B : x=A x=A ;\n"
158: + "fragment A : 'a' ;\n";
159: boolean found = rawGenerateAndBuildRecognizer("t.g", grammar,
160: null, "tLexer", false);
161: boolean expecting = true; // should be ok
162: assertEquals(expecting, found);
163: }
164:
165: public void testIsolatedEOTEdge() {
166: String grammar = "lexer grammar T;\n" + "QUOTED_CONTENT \n"
167: + " : 'q' (~'q')* (('x' 'q') )* 'q' ; \n";
168: boolean found = rawGenerateAndBuildRecognizer("T.g", grammar,
169: null, "TLexer", false);
170: boolean expecting = true; // should be ok
171: assertEquals(expecting, found);
172: }
173:
174: }
|