|
| |
|
| java.lang.Object
 de.susebox.jtopas.AbstractTokenizer
de.susebox.jtopas.StandardTokenizer
| StandardTokenizer | | public class StandardTokenizer extends AbstractTokenizer implements Tokenizer,TokenizerPropertyListener(Code) | |
This is the mainstream
Tokenizer . It implements the
Tokenizer interface in a straightforward approach without too specialized parse
optimizations.
Beside the
Tokenizer interface, the class StandardTokenizer
provides some basic features for cascading (nested) tokenizers. Consider the usual
HTML pages found today in the WWW. Most of them are a mixture of regular HTML,
cascading style sheets (CSS) and embedded JavaScript. These different languages
use different syntaxes, so one needs varous tokenizers on the same input stream.
This
Tokenizer implementation is not synchronized. Take care when using
with multible threads.
See Also: Tokenizer
See Also: TokenizerProperties
author:
Heiko Blau |
| Method Summary | |
| public void | close()
Closing this tokenizer frees resources. | | protected DataProvider | getDataProvider(int startPos, int length)
Implements the abstract method of the base class. | | public int | getRangeStart()
This method returns the absolute offset in characters to the start of the
parsed stream. | | protected int | readMoreData()
This method organizes the input buffer. | | public void | setSource(TokenizerSource source)
Additionally to the common behaviour implemented in
#de.susebox.jtopas.AbstractTokenizer#setSource , this method ajusts
the state speicific to the StandardTokenizer class. |
| _inputBuffer | | protected char[] _inputBuffer(Code) | | This buffer holds the currently read data. Dont use a buffered reader, since
we do buffering here.
|
| close | | public void close()(Code) | | Closing this tokenizer frees resources.
|
| getDataProvider | | protected DataProvider getDataProvider(int startPos, int length)(Code) | | Implements the abstract method of the base class.
Parameters:
startPos - position in the input data
Parameters:
length - number of characters |
| getRangeStart | | public int getRangeStart()(Code) | | This method returns the absolute offset in characters to the start of the
parsed stream. See the method description in
Tokenizer .
the absolute offset of the current text window in characters from the start of the data source of the Tokenizer
See Also: StandardTokenizer.getReadPosition |
| setSource | | public void setSource(TokenizerSource source)(Code) | | Additionally to the common behaviour implemented in
#de.susebox.jtopas.AbstractTokenizer#setSource , this method ajusts
the state speicific to the StandardTokenizer class.
Parameters:
source - a TokenizerSource to read data from |
| Methods inherited from de.susebox.jtopas.AbstractTokenizer | public void addTokenizer(AbstractTokenizer tokenizer) throws TokenizerException(Code)(Java Doc)
protected void adjustLineAndColumn(int type, int length)(Code)(Java Doc)
public void changeParseFlags(int flags, int mask) throws TokenizerException(Code)(Java Doc)
public void close()(Code)(Java Doc)
protected int comparePrefix(int offset, String prefix, boolean noCase) throws TokenizerException(Code)(Java Doc)
protected int completeBlockComment(TokenizerProperty prop) throws TokenizerException(Code)(Java Doc)
protected TokenizerProperty completeBoundedToken(Token token) throws TokenizerException(Code)(Java Doc)
protected int completeLineComment(TokenizerProperty prop) throws TokenizerException(Code)(Java Doc)
protected int completeString(TokenizerProperty prop) throws TokenizerException(Code)(Java Doc)
protected int completeWhitespace() throws TokenizerException(Code)(Java Doc)
public String currentImage() throws TokenizerException(Code)(Java Doc)
public Token currentToken() throws TokenizerException(Code)(Java Doc)
public int currentlyAvailable()(Code)(Java Doc)
protected DataProvider getBaseDataProvider(int startPos, int length)(Code)(Java Doc)
protected AbstractTokenizer getBaseTokenizer()(Code)(Java Doc)
public char getChar(int pos) throws IndexOutOfBoundsException(Code)(Java Doc)
public int getColumnNumber()(Code)(Java Doc)
public int getCurrentColumn()(Code)(Java Doc)
public int getCurrentLine()(Code)(Java Doc)
abstract protected DataProvider getDataProvider(int startPos, int length)(Code)(Java Doc)
public de.susebox.jtopas.spi.KeywordHandler getKeywordHandler()(Code)(Java Doc)
public int getLineNumber()(Code)(Java Doc)
public int getParseFlags()(Code)(Java Doc)
public de.susebox.jtopas.spi.PatternHandler getPatternHandler()(Code)(Java Doc)
public int getReadPosition()(Code)(Java Doc)
public de.susebox.jtopas.spi.SeparatorHandler getSeparatorHandler()(Code)(Java Doc)
public de.susebox.jtopas.spi.SequenceHandler getSequenceHandler()(Code)(Java Doc)
public TokenizerSource getSource()(Code)(Java Doc)
public String getText(int start, int len) throws IndexOutOfBoundsException(Code)(Java Doc)
public TokenizerProperties getTokenizerProperties()(Code)(Java Doc)
public de.susebox.jtopas.spi.WhitespaceHandler getWhitespaceHandler()(Code)(Java Doc)
public boolean hasMoreToken()(Code)(Java Doc)
protected boolean isEOF(int offset) throws TokenizerException(Code)(Java Doc)
protected boolean isFlagSet(int flag)(Code)(Java Doc)
protected boolean isFlagSet(TokenizerProperty prop, int flag)(Code)(Java Doc)
protected TokenizerProperty isKeyword(int startingAtPos, int length) throws TokenizerException(Code)(Java Doc)
protected boolean isPattern(int offset, boolean freePatternOnly) throws TokenizerException(Code)(Java Doc)
protected boolean isSeparator(int offset) throws TokenizerException(Code)(Java Doc)
protected boolean isSpecialSequence(int offset) throws TokenizerException(Code)(Java Doc)
protected boolean isWhitespace(char testChar)(Code)(Java Doc)
protected boolean isWhitespace(int offset) throws TokenizerException(Code)(Java Doc)
public String nextImage() throws TokenizerException(Code)(Java Doc)
public Token nextToken() throws TokenizerException(Code)(Java Doc)
public void propertyChanged(TokenizerPropertyEvent event)(Code)(Java Doc)
public int readMore() throws TokenizerException(Code)(Java Doc)
abstract protected int readMoreData() throws TokenizerException(Code)(Java Doc)
protected int readMoreDataFromBase() throws TokenizerException(Code)(Java Doc)
protected int readWhitespaces(int startingAtPos, int maxChars) throws TokenizerException(Code)(Java Doc)
public void setKeywordHandler(de.susebox.jtopas.spi.KeywordHandler handler)(Code)(Java Doc)
public void setPatternHandler(de.susebox.jtopas.spi.PatternHandler handler)(Code)(Java Doc)
public void setReadPositionAbsolute(int position) throws IndexOutOfBoundsException(Code)(Java Doc)
public void setReadPositionRelative(int offset) throws IndexOutOfBoundsException(Code)(Java Doc)
public void setSeparatorHandler(de.susebox.jtopas.spi.SeparatorHandler handler)(Code)(Java Doc)
public void setSequenceHandler(de.susebox.jtopas.spi.SequenceHandler handler)(Code)(Java Doc)
public void setSource(TokenizerSource source)(Code)(Java Doc)
public void setSource(Reader reader)(Code)(Java Doc)
public void setTokenizerProperties(TokenizerProperties props) throws NullPointerException(Code)(Java Doc)
public void setWhitespaceHandler(de.susebox.jtopas.spi.WhitespaceHandler handler)(Code)(Java Doc)
protected String[] splitBlockComment(TokenizerProperty prop, String image)(Code)(Java Doc)
protected String[] splitIntoLines(String image)(Code)(Java Doc)
protected String[] splitString(TokenizerProperty prop, String image)(Code)(Java Doc)
public void switchTo(AbstractTokenizer tokenizer) throws TokenizerException(Code)(Java Doc)
protected void synchronizeAll() throws TokenizerException(Code)(Java Doc)
|
|
|
|