001: /* ====================================================================
002: * The LateralNZ Software License, Version 1.0
003: *
004: * Copyright (c) 2003 LateralNZ. All rights reserved.
005: *
006: * Redistribution and use in source and binary forms, with or without
007: * modification, are permitted provided that the following conditions
008: * are met:
009: *
010: * 1. Redistributions of source code must retain the above copyright
011: * notice, this list of conditions and the following disclaimer.
012: *
013: * 2. Redistributions in binary form must reproduce the above copyright
014: * notice, this list of conditions and the following disclaimer in
015: * the documentation and/or other materials provided with the
016: * distribution.
017: *
018: * 3. The end-user documentation included with the redistribution,
019: * if any, must include the following acknowledgment:
020: * "This product includes software developed by
021: * LateralNZ (http://www.lateralnz.org/) and other third parties."
022: * Alternately, this acknowledgment may appear in the software itself,
023: * if and wherever such third-party acknowledgments normally appear.
024: *
025: * 4. The names "LateralNZ" must not be used to endorse or promote
026: * products derived from this software without prior written
027: * permission. For written permission, please
028: * contact oss@lateralnz.org.
029: *
030: * 5. Products derived from this software may not be called "Panther",
031: * or "Lateral" or "LateralNZ", nor may "PANTHER" or "LATERAL" or
032: * "LATERALNZ" appear in their name, without prior written
033: * permission of LateralNZ.
034: *
035: * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
036: * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
037: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
038: * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
039: * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
040: * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
041: * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
042: * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
043: * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
044: * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
045: * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
046: * SUCH DAMAGE.
047: * ====================================================================
048: *
049: * This software consists of voluntary contributions made by many
050: * individuals on behalf of LateralNZ. For more
051: * information on Lateral, please see http://www.lateralnz.com/ or
052: * http://www.lateralnz.org
053: *
054: */
055: package org.lateralnz.c3d.util;
056:
057: /**
058: * a custom string tokenizer (based on the standard Java StringTokenizer)
059: * that supports marking a position and returning to it. This is most
060: * useful in lexical analysis/parsing/etc.
061: */
062: public class Tokenizer {
063: private static final String EMPTY = "";
064:
065: private String delimiters;
066: private boolean rtnDelims;
067: private String returnDelimiters;
068:
069: private char chars[];
070:
071: int currentPos;
072:
073: private int maxPos;
074:
075: int[] mark = new int[100];
076: int markpos = 0;
077:
078: public Tokenizer(String text, String delimiters,
079: String returnDelimiters) {
080: setString(text);
081: this .delimiters = delimiters;
082: this .returnDelimiters = returnDelimiters;
083: this .rtnDelims = (returnDelimiters != null && !returnDelimiters
084: .equals(EMPTY));
085: }
086:
087: /**
088: * set the string to tokenize (a helper to save having to create
089: * this object every time)
090: */
091: public final void setString(String text) {
092: this .maxPos = text.length();
093: this .currentPos = 0;
094:
095: this .chars = new char[maxPos];
096: text.getChars(0, maxPos, this .chars, 0);
097: }
098:
099: /**
100: * clean up and shutdown
101: */
102: public final void destroy() {
103: chars = null;
104: }
105:
106: /**
107: * mark the current position
108: */
109: public final void mark() {
110: markpos++;
111: if (markpos >= mark.length) {
112: int[] newmark = new int[mark.length * 2];
113: System.arraycopy(mark, 0, newmark, 0, mark.length);
114: mark = newmark;
115: }
116: mark[markpos] = currentPos;
117: }
118:
119: /**
120: * return to the last marked position
121: */
122: public final void release() {
123: currentPos = mark[markpos];
124: markpos--;
125: }
126:
127: /**
128: * reset the position counter to start tokenizing at the beginning again
129: */
130: public final void reset() {
131: currentPos = 0;
132: }
133:
134: /**
135: * return true if there are more tokens
136: */
137: public final boolean hasMoreTokens() {
138: boolean rtn = false;
139: mark();
140: String s = nextToken();
141: if (s != null && !s.equals(EMPTY)) {
142: rtn = true;
143: }
144: release();
145:
146: return rtn;
147: }
148:
149: /**
150: * get the next token
151: */
152: public final String nextToken() {
153: StringBuffer rtn = new StringBuffer();
154:
155: while (currentPos < maxPos
156: && delimiters.indexOf(chars[currentPos]) >= 0) {
157: if (rtnDelims
158: && returnDelimiters.indexOf(chars[currentPos]) >= 0) {
159: rtn.append(chars[currentPos]);
160: currentPos++;
161: return rtn.toString();
162: }
163: currentPos++;
164: }
165:
166: while (currentPos < maxPos
167: && delimiters.indexOf(chars[currentPos]) < 0) {
168: rtn.append(chars[currentPos]);
169: currentPos++;
170: }
171:
172: /*
173: while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) >= 0) {
174: currentPos++;
175: }
176:
177: while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) < 0) {
178: rtn.append(chars[currentPos]);
179: currentPos++;
180: }
181: */
182: return rtn.toString();
183: }
184:
185: /**
186: * look at the next token without moving the position indicator.
187: * (basically the works out to mark, next token, release)
188: */
189: public final String peekNextToken() {
190: mark();
191: String rtn = nextToken();
192: release();
193: return rtn;
194: }
195:
196: }
|