001: /*BEGIN_COPYRIGHT_BLOCK
002: *
003: * Copyright (c) 2001-2007, JavaPLT group at Rice University (javaplt@rice.edu)
004: * All rights reserved.
005: *
006: * Redistribution and use in source and binary forms, with or without
007: * modification, are permitted provided that the following conditions are met:
008: * * Redistributions of source code must retain the above copyright
009: * notice, this list of conditions and the following disclaimer.
010: * * Redistributions in binary form must reproduce the above copyright
011: * notice, this list of conditions and the following disclaimer in the
012: * documentation and/or other materials provided with the distribution.
013: * * Neither the names of DrJava, the JavaPLT group, Rice University, nor the
014: * names of its contributors may be used to endorse or promote products
015: * derived from this software without specific prior written permission.
016: *
017: * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
018: * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
019: * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
020: * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
021: * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
022: * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
023: * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
024: * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
025: * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
026: * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
027: * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
028: *
029: * This software is Open Source Initiative approved Open Source Software.
030: * Open Source Initative Approved is a trademark of the Open Source Initiative.
031: *
032: * This file is part of DrJava. Download the current version of this project
033: * from http://www.drjava.org/ or http://sourceforge.net/projects/drjava/
034: *
035: * END_COPYRIGHT_BLOCK*/
036:
037: package edu.rice.cs.util.sexp;
038:
039: import java.util.*;
040: import java.io.*;
041:
042: class Lexer extends StreamTokenizer {
043:
044: public HashMap<String, Tokens.SExpToken> wordTable = new HashMap<String, Tokens.SExpToken>();
045:
046: private Tokens.SExpToken buffer;
047:
048: public Lexer(File file) throws FileNotFoundException {
049: this (new BufferedReader(new FileReader(file)));
050: }
051:
052: public Lexer(Reader reader) {
053: super (new BufferedReader(reader));
054: initLexer();
055: }
056:
057: private void initLexer() {
058:
059: // configure StreamTokenizer portion of this
060: resetSyntax();
061: parseNumbers();
062: slashSlashComments(true);
063: wordChars('!', '\'');
064: wordChars('*', '~');
065: quoteChar('"');
066: ordinaryChars('(', ')');
067: whitespaceChars(0, ' ');
068: commentChar(';');
069:
070: initWordTable();
071: buffer = null; // buffer initially empty
072: }
073:
074: /** Skips through the input stream until an EOL is encountered */
075: public void flush() throws IOException {
076: eolIsSignificant(true);
077: while (nextToken() != TT_EOL)
078: ; // eat tokens until EOL
079: eolIsSignificant(false);
080: }
081:
082: /** Performs a nextToken() operation from StreamTokenizer except
083: * for throwing an unchecked LexingException instead of a checked IOException */
084: private int getToken() {
085: try {
086: int tokenType = nextToken();
087: return tokenType;
088: } catch (IOException e) {
089: throw new LexingException(
090: "Unable to read the data from the given input");
091: }
092: }
093:
094: /** Returns the next Tokens.SExpToken without consuming it */
095: public Tokens.SExpToken peek() {
096: if (buffer == null)
097: buffer = readToken();
098: return buffer;
099: }
100:
101: /** Reads the next Tokens.SExpToken from the input stream and consumes it;
102: * Returns the Tokens.SExpToken object representing this Tokens.SExpToken */
103: public Tokens.SExpToken readToken() {
104:
105: if (buffer != null) {
106: Tokens.SExpToken token = buffer;
107: buffer = null; // clear buffer
108: return token;
109: }
110:
111: int tokenType = getToken();
112: // Process the Tokens.SExpToken returned by StreamTokenizer
113: switch (tokenType) {
114: case TT_NUMBER:
115: return new Tokens.NumberToken(nval);
116:
117: case TT_WORD:
118: String s = sval.toLowerCase();
119: Tokens.SExpToken regToken = wordTable.get(s);
120: if (regToken == null)
121: return new Tokens.WordToken(sval);
122:
123: return regToken;
124:
125: case TT_EOF:
126: return null;
127: case '(':
128: return Tokens.LeftParenToken.ONLY;
129: case ')':
130: return Tokens.RightParenToken.ONLY;
131: case '"':
132: return new Tokens.QuotedTextToken(sval);
133: case '\\':
134: // int t = getToken();
135: // if (t == '"') {
136: // return new Tokens.WordToken("\"");
137: // }
138: // else if (t == '\\') {
139: // return new Tokens.WordToken("\\");
140: // }
141: // else if (t == ' ') {
142: // return new Tokens.WordToken(" ");
143: // }
144: // else if (t == 'n') {
145: // return new Tokens.WordToken("\n");
146: // }
147: // else if (t == 't') {
148: // return new Tokens.WordToken("\t");
149: // }
150: // else {
151: // pushBack();
152: return Tokens.BackSlashToken.ONLY;
153: // throw new SExpParseException("Invalid escape sequence: \\" + (char)t);
154:
155: default:
156: return new Tokens.WordToken("" + (char) tokenType);
157: }
158: }
159:
160: /** Initialize the word table used by the lexer to classify Tokens */
161: private void initWordTable() {
162: // initialize wordTable
163: wordTable.put("true", Tokens.BooleanToken.TRUE);
164: wordTable.put("false", Tokens.BooleanToken.FALSE);
165: }
166: }
|