001: /*
002: [The "BSD licence"]
003: Copyright (c) 2005-2006 Terence Parr
004: All rights reserved.
005:
006: Redistribution and use in source and binary forms, with or without
007: modification, are permitted provided that the following conditions
008: are met:
009: 1. Redistributions of source code must retain the above copyright
010: notice, this list of conditions and the following disclaimer.
011: 2. Redistributions in binary form must reproduce the above copyright
012: notice, this list of conditions and the following disclaimer in the
013: documentation and/or other materials provided with the distribution.
014: 3. The name of the author may not be used to endorse or promote products
015: derived from this software without specific prior written permission.
016:
017: THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
018: IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
019: OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
020: IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
021: INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
022: NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
023: DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
024: THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
025: (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
026: THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
027: */
028: package org.antlr.runtime;
029:
030: import java.util.*;
031:
032: /** Useful for dumping out the input stream after doing some
033: * augmentation or other manipulations.
034: *
035: * You can insert stuff, replace, and delete chunks. Note that the
036: * operations are done lazily--only if you convert the buffer to a
037: * String. This is very efficient because you are not moving data around
038: * all the time. As the buffer of tokens is converted to strings, the
039: * toString() method(s) check to see if there is an operation at the
040: * current index. If so, the operation is done and then normal String
041: * rendering continues on the buffer. This is like having multiple Turing
042: * machine instruction streams (programs) operating on a single input tape. :)
043: *
044: * Since the operations are done lazily at toString-time, operations do not
045: * screw up the token index values. That is, an insert operation at token
046: * index i does not change the index values for tokens i+1..n-1.
047: *
048: * Because operations never actually alter the buffer, you may always get
049: * the original token stream back without undoing anything. Since
050: * the instructions are queued up, you can easily simulate transactions and
051: * roll back any changes if there is an error just by removing instructions.
052: * For example,
053: *
054: * CharStream input = new ANTLRFileStream("input");
055: * TLexer lex = new TLexer(input);
056: * TokenRewriteStream tokens = new TokenRewriteStream(lex);
057: * T parser = new T(tokens);
058: * parser.startRule();
059: *
060: * Then in the rules, you can execute
061: * Token t,u;
062: * ...
063: * input.insertAfter(t, "text to put after t");}
064: * input.insertAfter(u, "text after u");}
065: * System.out.println(tokens.toString());
066: *
067: * Actually, you have to cast the 'input' to a TokenRewriteStream. :(
068: *
069: * You can also have multiple "instruction streams" and get multiple
070: * rewrites from a single pass over the input. Just name the instruction
071: * streams and use that name again when printing the buffer. This could be
072: * useful for generating a C file and also its header file--all from the
073: * same buffer:
074: *
075: * tokens.insertAfter("pass1", t, "text to put after t");}
076: * tokens.insertAfter("pass2", u, "text after u");}
077: * System.out.println(tokens.toString("pass1"));
078: * System.out.println(tokens.toString("pass2"));
079: *
080: * If you don't use named rewrite streams, a "default" stream is used as
081: * the first example shows.
082: */
083: public class TokenRewriteStream extends CommonTokenStream {
084: public static final String DEFAULT_PROGRAM_NAME = "default";
085: public static final int PROGRAM_INIT_SIZE = 100;
086: public static final int MIN_TOKEN_INDEX = 0;
087:
088: // Define the rewrite operation hierarchy
089:
090: static class RewriteOperation {
091: protected int index;
092: protected Object text;
093:
094: protected RewriteOperation(int index, Object text) {
095: this .index = index;
096: this .text = text;
097: }
098:
099: /** Execute the rewrite operation by possibly adding to the buffer.
100: * Return the index of the next token to operate on.
101: */
102: public int execute(StringBuffer buf) {
103: return index;
104: }
105:
106: public String toString() {
107: String opName = getClass().getName();
108: int $index = opName.indexOf('$');
109: opName = opName.substring($index + 1, opName.length());
110: return opName + "@" + index + '"' + text + '"';
111: }
112: }
113:
114: static class InsertBeforeOp extends RewriteOperation {
115: public InsertBeforeOp(int index, Object text) {
116: super (index, text);
117: }
118:
119: public int execute(StringBuffer buf) {
120: buf.append(text);
121: return index;
122: }
123: }
124:
125: /** TODO: make insertAfters append after each other.
126: static class InsertAfterOp extends InsertBeforeOp {
127: public InsertAfterOp(int index, String text) {
128: super(index,text);
129: }
130: }
131: */
132:
133: /** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
134: * instructions.
135: */
136: static class ReplaceOp extends RewriteOperation {
137: protected int lastIndex;
138:
139: public ReplaceOp(int from, int to, Object text) {
140: super (from, text);
141: lastIndex = to;
142: }
143:
144: public int execute(StringBuffer buf) {
145: if (text != null) {
146: buf.append(text);
147: }
148: return lastIndex + 1;
149: }
150: }
151:
152: static class DeleteOp extends ReplaceOp {
153: public DeleteOp(int from, int to) {
154: super (from, to, null);
155: }
156: }
157:
158: /** You may have multiple, named streams of rewrite operations.
159: * I'm calling these things "programs."
160: * Maps String (name) -> rewrite (List)
161: */
162: protected Map programs = null;
163:
164: /** Map String (program name) -> Integer index */
165: protected Map lastRewriteTokenIndexes = null;
166:
167: public TokenRewriteStream() {
168: init();
169: }
170:
171: protected void init() {
172: programs = new HashMap();
173: programs.put(DEFAULT_PROGRAM_NAME, new ArrayList(
174: PROGRAM_INIT_SIZE));
175: lastRewriteTokenIndexes = new HashMap();
176: }
177:
178: public TokenRewriteStream(TokenSource tokenSource) {
179: super (tokenSource);
180: init();
181: }
182:
183: public TokenRewriteStream(TokenSource tokenSource, int channel) {
184: super (tokenSource, channel);
185: init();
186: }
187:
188: public void rollback(int instructionIndex) {
189: rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
190: }
191:
192: /** Rollback the instruction stream for a program so that
193: * the indicated instruction (via instructionIndex) is no
194: * longer in the stream. UNTESTED!
195: */
196: public void rollback(String programName, int instructionIndex) {
197: List is = (List) programs.get(programName);
198: if (is != null) {
199: programs.put(programName, is.subList(MIN_TOKEN_INDEX,
200: instructionIndex));
201: }
202: }
203:
204: public void deleteProgram() {
205: deleteProgram(DEFAULT_PROGRAM_NAME);
206: }
207:
208: /** Reset the program so that no instructions exist */
209: public void deleteProgram(String programName) {
210: rollback(programName, MIN_TOKEN_INDEX);
211: }
212:
213: /** If op.index > lastRewriteTokenIndexes, just add to the end.
214: * Otherwise, do linear */
215: protected void addToSortedRewriteList(RewriteOperation op) {
216: addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
217: }
218:
219: /** Add an instruction to the rewrite instruction list ordered by
220: * the instruction number (use a binary search for efficiency).
221: * The list is ordered so that toString() can be done efficiently.
222: *
223: * When there are multiple instructions at the same index, the instructions
224: * must be ordered to ensure proper behavior. For example, a delete at
225: * index i must kill any replace operation at i. Insert-before operations
226: * must come before any replace / delete instructions. If there are
227: * multiple insert instructions for a single index, they are done in
228: * reverse insertion order so that "insert foo" then "insert bar" yields
229: * "foobar" in front rather than "barfoo". This is convenient because
230: * I can insert new InsertOp instructions at the index returned by
231: * the binary search. A ReplaceOp kills any previous replace op. Since
232: * delete is the same as replace with null text, i can check for
233: * ReplaceOp and cover DeleteOp at same time. :)
234: */
235: protected void addToSortedRewriteList(String programName,
236: RewriteOperation op) {
237: List rewrites = getProgram(programName);
238: //System.out.println("### add "+op+"; rewrites="+rewrites);
239: Comparator comparator = new Comparator() {
240: public int compare(Object o, Object o1) {
241: RewriteOperation a = (RewriteOperation) o;
242: RewriteOperation b = (RewriteOperation) o1;
243: if (a.index < b.index)
244: return -1;
245: if (a.index > b.index)
246: return 1;
247: return 0;
248: }
249: };
250: int pos = Collections.binarySearch(rewrites, op, comparator);
251: //System.out.println("bin search returns: pos="+pos);
252:
253: if (pos >= 0) {
254: // binarySearch does not guarantee first element when multiple
255: // are found. I must seach backwards for first op with op.index
256: for (; pos >= 0; pos--) {
257: RewriteOperation prevOp = (RewriteOperation) rewrites
258: .get(pos);
259: if (prevOp.index < op.index) {
260: break;
261: }
262: }
263: pos++; // pos points at first op before ops with op.index; go back up one
264: // now pos is the index in rewrites of first op with op.index
265: //System.out.println("first op with op.index: pos="+pos);
266:
267: // an instruction operating already on that index was found;
268: // make this one happen after all the others
269: //System.out.println("found instr for index="+op.index);
270: if (op instanceof ReplaceOp) {
271: boolean replaced = false;
272: int i;
273: // look for an existing replace
274: for (i = pos; i < rewrites.size(); i++) {
275: RewriteOperation prevOp = (RewriteOperation) rewrites
276: .get(pos);
277: if (prevOp.index != op.index) {
278: break;
279: }
280: if (prevOp instanceof ReplaceOp) {
281: rewrites.set(pos, op); // replace old with new
282: replaced = true;
283: break;
284: }
285: // keep going; must be an insert
286: }
287: if (!replaced) {
288: // add replace op to the end of all the inserts
289: rewrites.add(i, op);
290: }
291: } else {
292: // inserts are added in front of existing inserts
293: rewrites.add(pos, op);
294: }
295: } else {
296: //System.out.println("no instruction at pos=="+pos);
297: rewrites.add(-pos - 1, op);
298: }
299: //System.out.println("after, rewrites="+rewrites);
300: }
301:
302: public void insertAfter(Token t, Object text) {
303: insertAfter(DEFAULT_PROGRAM_NAME, t, text);
304: }
305:
306: public void insertAfter(int index, Object text) {
307: insertAfter(DEFAULT_PROGRAM_NAME, index, text);
308: }
309:
310: public void insertAfter(String programName, Token t, Object text) {
311: insertAfter(programName, t.getTokenIndex(), text);
312: }
313:
314: public void insertAfter(String programName, int index, Object text) {
315: // to insert after, just insert before next index (even if past end)
316: insertBefore(programName, index + 1, text);
317: //addToSortedRewriteList(programName, new InsertAfterOp(index,text));
318: }
319:
320: public void insertBefore(Token t, Object text) {
321: insertBefore(DEFAULT_PROGRAM_NAME, t, text);
322: }
323:
324: public void insertBefore(int index, Object text) {
325: insertBefore(DEFAULT_PROGRAM_NAME, index, text);
326: }
327:
328: public void insertBefore(String programName, Token t, Object text) {
329: insertBefore(programName, t.getTokenIndex(), text);
330: }
331:
332: public void insertBefore(String programName, int index, Object text) {
333: addToSortedRewriteList(programName, new InsertBeforeOp(index,
334: text));
335: }
336:
337: public void replace(int index, Object text) {
338: replace(DEFAULT_PROGRAM_NAME, index, index, text);
339: }
340:
341: public void replace(int from, int to, Object text) {
342: replace(DEFAULT_PROGRAM_NAME, from, to, text);
343: }
344:
345: public void replace(Token indexT, Object text) {
346: replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
347: }
348:
349: public void replace(Token from, Token to, Object text) {
350: replace(DEFAULT_PROGRAM_NAME, from, to, text);
351: }
352:
353: public void replace(String programName, int from, int to,
354: Object text) {
355: if (from > to || from < 0 || to < 0) {
356: return;
357: }
358: addToSortedRewriteList(programName, new ReplaceOp(from, to,
359: text));
360: /*
361: // replace from..to by deleting from..to-1 and then do a replace
362: // on last index
363: for (int i=from; i<to; i++) {
364: addToSortedRewriteList(new DeleteOp(i,i));
365: }
366: addToSortedRewriteList(new ReplaceOp(to, to, text));
367: */
368: }
369:
370: public void replace(String programName, Token from, Token to,
371: Object text) {
372: replace(programName, from.getTokenIndex(), to.getTokenIndex(),
373: text);
374: }
375:
376: public void delete(int index) {
377: delete(DEFAULT_PROGRAM_NAME, index, index);
378: }
379:
380: public void delete(int from, int to) {
381: delete(DEFAULT_PROGRAM_NAME, from, to);
382: }
383:
384: public void delete(Token indexT) {
385: delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
386: }
387:
388: public void delete(Token from, Token to) {
389: delete(DEFAULT_PROGRAM_NAME, from, to);
390: }
391:
392: public void delete(String programName, int from, int to) {
393: replace(programName, from, to, null);
394: }
395:
396: public void delete(String programName, Token from, Token to) {
397: replace(programName, from, to, null);
398: }
399:
400: public int getLastRewriteTokenIndex() {
401: return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
402: }
403:
404: protected int getLastRewriteTokenIndex(String programName) {
405: Integer I = (Integer) lastRewriteTokenIndexes.get(programName);
406: if (I == null) {
407: return -1;
408: }
409: return I.intValue();
410: }
411:
412: protected void setLastRewriteTokenIndex(String programName, int i) {
413: lastRewriteTokenIndexes.put(programName, new Integer(i));
414: }
415:
416: protected List getProgram(String name) {
417: List is = (List) programs.get(name);
418: if (is == null) {
419: is = initializeProgram(name);
420: }
421: return is;
422: }
423:
424: private List initializeProgram(String name) {
425: List is = new ArrayList(PROGRAM_INIT_SIZE);
426: programs.put(name, is);
427: return is;
428: }
429:
430: public String toOriginalString() {
431: return toOriginalString(MIN_TOKEN_INDEX, size() - 1);
432: }
433:
434: public String toOriginalString(int start, int end) {
435: StringBuffer buf = new StringBuffer();
436: for (int i = start; i >= MIN_TOKEN_INDEX && i <= end
437: && i < tokens.size(); i++) {
438: buf.append(get(i).getText());
439: }
440: return buf.toString();
441: }
442:
443: public String toString() {
444: return toString(MIN_TOKEN_INDEX, size() - 1);
445: }
446:
447: public String toString(String programName) {
448: return toString(programName, MIN_TOKEN_INDEX, size() - 1);
449: }
450:
451: public String toString(int start, int end) {
452: return toString(DEFAULT_PROGRAM_NAME, start, end);
453: }
454:
455: public String toString(String programName, int start, int end) {
456: List rewrites = (List) programs.get(programName);
457: if (rewrites == null || rewrites.size() == 0) {
458: return toOriginalString(start, end); // no instructions to execute
459: }
460: StringBuffer buf = new StringBuffer();
461:
462: /// Index of first rewrite we have not done
463: int rewriteOpIndex = 0;
464:
465: int tokenCursor = start;
466: while (tokenCursor >= MIN_TOKEN_INDEX && tokenCursor <= end
467: && tokenCursor < tokens.size()) {
468: //System.out.println("tokenCursor="+tokenCursor);
469: // execute instructions associated with this token index
470: if (rewriteOpIndex < rewrites.size()) {
471: RewriteOperation op = (RewriteOperation) rewrites
472: .get(rewriteOpIndex);
473:
474: // skip all ops at lower index
475: while (op.index < tokenCursor
476: && rewriteOpIndex < rewrites.size()) {
477: rewriteOpIndex++;
478: if (rewriteOpIndex < rewrites.size()) {
479: op = (RewriteOperation) rewrites
480: .get(rewriteOpIndex);
481: }
482: }
483:
484: // while we have ops for this token index, exec them
485: while (tokenCursor == op.index
486: && rewriteOpIndex < rewrites.size()) {
487: //System.out.println("execute "+op+" at instruction "+rewriteOpIndex);
488: tokenCursor = op.execute(buf);
489: //System.out.println("after execute tokenCursor = "+tokenCursor);
490: rewriteOpIndex++;
491: if (rewriteOpIndex < rewrites.size()) {
492: op = (RewriteOperation) rewrites
493: .get(rewriteOpIndex);
494: }
495: }
496: }
497: // dump the token at this index
498: if (tokenCursor <= end) {
499: buf.append(get(tokenCursor).getText());
500: tokenCursor++;
501: }
502: }
503: // now see if there are operations (append) beyond last token index
504: for (int opi = rewriteOpIndex; opi < rewrites.size(); opi++) {
505: RewriteOperation op = (RewriteOperation) rewrites.get(opi);
506: if (op.index >= size()) {
507: op.execute(buf); // must be insertions if after last token
508: }
509: //System.out.println("execute "+op+" at "+opi);
510: //op.execute(buf); // must be insertions if after last token
511: }
512:
513: return buf.toString();
514: }
515:
516: public String toDebugString() {
517: return toDebugString(MIN_TOKEN_INDEX, size() - 1);
518: }
519:
520: public String toDebugString(int start, int end) {
521: StringBuffer buf = new StringBuffer();
522: for (int i = start; i >= MIN_TOKEN_INDEX && i <= end
523: && i < tokens.size(); i++) {
524: buf.append(get(i));
525: }
526: return buf.toString();
527: }
528: }
|