001: package antlr;
002:
003: import java.util.List;
004: import java.util.ArrayList;
005:
006: /* ANTLR Translator Generator
007: * Project led by Terence Parr at http://www.cs.usfca.edu
008: * Software rights: http://www.antlr.org/license.html
009: */
010:
011: /**A Stream of Token objects fed to the parser from a Tokenizer that can
012: * be rewound via mark()/rewind() methods.
013: * <p>
014: * A dynamic array is used to buffer up all the input tokens. Normally,
015: * "k" tokens are stored in the buffer. More tokens may be stored during
016: * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
017: * Consumption of tokens is deferred. In other words, reading the next
018: * token is not done by conume(), but deferred until needed by LA or LT.
019: * <p>
020: *
021: * @see antlr.Token
022: * @see antlr.TokenQueue
023: */
024:
025: public class TokenBuffer {
026:
027: // Number of active markers
028: private int nMarkers = 0;
029:
030: /** The index into the tokens list of the current token (next token
031: * to consume).
032: */
033: private int p = 0;
034:
035: public static final int INITIAL_BUFFER_SIZE = 2048;
036:
037: /** Record every single token pulled from the source so we can reproduce
038: * chunks of it later.
039: */
040: private final List tokens = new ArrayList(INITIAL_BUFFER_SIZE);
041:
042: // type buffer data (created to improve performance of LA)
043: private int size = 0;
044: private int[] data = new int[INITIAL_BUFFER_SIZE];
045:
046: /** Create a token buffer */
047: public TokenBuffer(TokenStream input) {
048: // fill buffer
049: try {
050: int pos = 0;
051: Token t = input.nextToken();
052: while ((t != null) && (t.getType() != Token.EOF_TYPE)) {
053: tokens.add(t);
054: if (pos == data.length)
055: resizeData();
056: data[pos++] = t.getType();
057: t = input.nextToken();
058: }
059: size = pos;
060: } catch (TokenStreamException tse) {
061: System.err.println("tmp error: can't load tokens: " + tse);
062: }
063: }
064:
065: /** Reset the input buffer to empty state */
066: /*public final void reset() {
067: nMarkers = 0;
068: p = 0;
069: size = 0;
070: tokens.clear();
071: data = null;
072: }*/
073:
074: // double data size
075: private final void resizeData() {
076: int[] newdata = new int[data.length * 2]; // resize
077: System.arraycopy(data, 0, newdata, 0, data.length);
078: data = newdata;
079: }
080:
081: /** Mark another token for deferred consumption */
082: public final void consume() {
083: p++;
084: }
085:
086: /** return the Tokenizer (needed by ParseView) */
087: /*public TokenStream getInput() {
088: return input;
089: }*/
090:
091: /** Get a lookahead token value */
092: public final int LA(int i) {
093: int dataPos = p + i - 1;
094: if (dataPos >= size) {
095: return TokenImpl.EOF_TYPE;
096: }
097: return data[dataPos];
098: }
099:
100: /** Get a lookahead token */
101: public final Token LT(int i) {
102: if ((p + i - 1) >= tokens.size()) {
103: return TokenImpl.EOF_TOKEN;
104: }
105: return (Token) tokens.get(p + i - 1);
106: }
107:
108: /** Get token at absolute position (indexed from 0) */
109: /*public final Token get(int i) {
110: return (Token)tokens.get(i);
111: }*/
112:
113: /**Return an integer marker that can be used to rewind the buffer to
114: * its current state.
115: */
116: public final int mark() {
117: //System.out.println("Marking at " + p);
118: //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
119: nMarkers++;
120: return p;
121: }
122:
123: /** What token index are we at? Assume mark() done at start.
124: */
125: public final int index() {
126: return p;
127: }
128:
129: public final void seek(int position) {
130: p = position;
131: }
132:
133: /**Rewind the token buffer to a marker.
134: * @param marker Marker returned previously from mark()
135: */
136: public final void rewind(int marker) {
137: seek(marker);
138: nMarkers--;
139: //System.out.println("Rewinding to " + marker);
140: //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
141: }
142:
143: }
|