001: package antlr;
002:
003: /* ANTLR Translator Generator
004: * Project led by Terence Parr at http://www.jGuru.com
005: * Software rights: http://www.antlr.org/RIGHTS.html
006: *
007: * $Id: TokenBuffer.java,v 1.1 2004/01/21 19:26:34 rgrimm Exp $
008: */
009:
010: /**A Stream of Token objects fed to the parser from a Tokenizer that can
011: * be rewound via mark()/rewind() methods.
012: * <p>
013: * A dynamic array is used to buffer up all the input tokens. Normally,
014: * "k" tokens are stored in the buffer. More tokens may be stored during
015: * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
016: * Consumption of tokens is deferred. In other words, reading the next
017: * token is not done by conume(), but deferred until needed by LA or LT.
018: * <p>
019: *
020: * @see antlr.Token
021: * @see antlr.Tokenizer
022: * @see antlr.TokenQueue
023: */
024:
025: import java.io.IOException;
026:
027: public class TokenBuffer {
028:
029: // Token source
030: protected TokenStream input;
031:
032: // Number of active markers
033: int nMarkers = 0;
034:
035: // Additional offset used when markers are active
036: int markerOffset = 0;
037:
038: // Number of calls to consume() since last LA() or LT() call
039: int numToConsume = 0;
040:
041: // Circular queue
042: TokenQueue queue;
043:
044: /** Create a token buffer */
045: public TokenBuffer(TokenStream input_) {
046: input = input_;
047: queue = new TokenQueue(1);
048: }
049:
050: /** Reset the input buffer to empty state */
051: public final void reset() {
052: nMarkers = 0;
053: markerOffset = 0;
054: numToConsume = 0;
055: queue.reset();
056: }
057:
058: /** Mark another token for deferred consumption */
059: public final void consume() {
060: numToConsume++;
061: }
062:
063: /** Ensure that the token buffer is sufficiently full */
064: private final void fill(int amount) throws TokenStreamException {
065: syncConsume();
066: // Fill the buffer sufficiently to hold needed tokens
067: while (queue.nbrEntries < amount + markerOffset) {
068: // Append the next token
069: queue.append(input.nextToken());
070: }
071: }
072:
073: /** return the Tokenizer (needed by ParseView) */
074: public TokenStream getInput() {
075: return input;
076: }
077:
078: /** Get a lookahead token value */
079: public final int LA(int i) throws TokenStreamException {
080: fill(i);
081: return queue.elementAt(markerOffset + i - 1).type;
082: }
083:
084: /** Get a lookahead token */
085: public final Token LT(int i) throws TokenStreamException {
086: fill(i);
087: return queue.elementAt(markerOffset + i - 1);
088: }
089:
090: /**Return an integer marker that can be used to rewind the buffer to
091: * its current state.
092: */
093: public final int mark() {
094: syncConsume();
095: //System.out.println("Marking at " + markerOffset);
096: //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
097: nMarkers++;
098: return markerOffset;
099: }
100:
101: /**Rewind the token buffer to a marker.
102: * @param mark Marker returned previously from mark()
103: */
104: public final void rewind(int mark) {
105: syncConsume();
106: markerOffset = mark;
107: nMarkers--;
108: //System.out.println("Rewinding to " + mark);
109: //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
110: }
111:
112: /** Sync up deferred consumption */
113: private final void syncConsume() {
114: while (numToConsume > 0) {
115: if (nMarkers > 0) {
116: // guess mode -- leave leading tokens and bump offset.
117: markerOffset++;
118: } else {
119: // normal mode -- remove first token
120: queue.removeFirst();
121: }
122: numToConsume--;
123: }
124: }
125: }
|