001: package net.xoetrope.builder.editor.syntaxhighlight;
002:
003: /*
004: * PythonTokenMarker.java - Python token marker
005: * Copyright (C) 1999 Jonathan Revusky
006: * Copyright (C) 1998, 1999 Slava Pestov
007: *
008: * You may use and modify this package for any purpose. Redistribution is
009: * permitted, in both source and binary form, provided that this notice
010: * remains intact in all source distributions of this package.
011: */
012:
013: import javax.swing.text.Segment;
014:
015: /**
016: * Python token marker.
017: *
018: * @author Jonathan Revusky
019: * @version $Id: PythonTokenMarker.java,v 1.22 2005/01/05 17:20:49 luano Exp $
020: */
021: public class PythonTokenMarker extends TokenMarker {
022: private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
023: private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
024:
025: public PythonTokenMarker() {
026: this .keywords = getKeywords();
027: }
028:
029: public byte markTokensImpl(byte token, Segment line, int lineIndex) {
030: char[] array = line.array;
031: int offset = line.offset;
032: lastOffset = offset;
033: lastKeyword = offset;
034: int length = line.count + offset;
035: boolean backslash = false;
036:
037: loop: for (int i = offset; i < length; i++) {
038: int i1 = (i + 1);
039:
040: char c = array[i];
041: if (c == '\\') {
042: backslash = !backslash;
043: continue;
044: }
045:
046: switch (token) {
047: case Token.NULL:
048: switch (c) {
049: case '#':
050: if (backslash)
051: backslash = false;
052: else {
053: doKeyword(line, i, c);
054: addToken(i - lastOffset, token);
055: addToken(length - i, Token.COMMENT1);
056: lastOffset = lastKeyword = length;
057: break loop;
058: }
059: break;
060: case '"':
061: doKeyword(line, i, c);
062: if (backslash)
063: backslash = false;
064: else {
065: addToken(i - lastOffset, token);
066: if (SyntaxUtilities.regionMatches(false, line,
067: i1, "\"\"")) {
068: token = TRIPLEQUOTE1;
069: } else {
070: token = Token.LITERAL1;
071: }
072: lastOffset = lastKeyword = i;
073: }
074: break;
075: case '\'':
076: doKeyword(line, i, c);
077: if (backslash)
078: backslash = false;
079: else {
080: addToken(i - lastOffset, token);
081: if (SyntaxUtilities.regionMatches(false, line,
082: i1, "''")) {
083: token = TRIPLEQUOTE2;
084: } else {
085: token = Token.LITERAL2;
086: }
087: lastOffset = lastKeyword = i;
088: }
089: break;
090: default:
091: backslash = false;
092: if (!Character.isLetterOrDigit(c) && c != '_')
093: doKeyword(line, i, c);
094: break;
095: }
096: break;
097: case Token.LITERAL1:
098: if (backslash)
099: backslash = false;
100: else if (c == '"') {
101: addToken(i1 - lastOffset, token);
102: token = Token.NULL;
103: lastOffset = lastKeyword = i1;
104: }
105: break;
106: case Token.LITERAL2:
107: if (backslash)
108: backslash = false;
109: else if (c == '\'') {
110: addToken(i1 - lastOffset, Token.LITERAL1);
111: token = Token.NULL;
112: lastOffset = lastKeyword = i1;
113: }
114: break;
115: case TRIPLEQUOTE1:
116: if (backslash)
117: backslash = false;
118: else if (SyntaxUtilities.regionMatches(false, line, i,
119: "\"\"\"")) {
120: addToken((i += 4) - lastOffset, Token.LITERAL1);
121: token = Token.NULL;
122: lastOffset = lastKeyword = i;
123: }
124: break;
125: case TRIPLEQUOTE2:
126: if (backslash)
127: backslash = false;
128: else if (SyntaxUtilities.regionMatches(false, line, i,
129: "'''")) {
130: addToken((i += 4) - lastOffset, Token.LITERAL1);
131: token = Token.NULL;
132: lastOffset = lastKeyword = i;
133: }
134: break;
135: default:
136: throw new InternalError("Invalid state: " + token);
137: }
138: }
139:
140: switch (token) {
141: case TRIPLEQUOTE1:
142: case TRIPLEQUOTE2:
143: addToken(length - lastOffset, Token.LITERAL1);
144: break;
145: case Token.NULL:
146: doKeyword(line, length, '\0');
147: default:
148: addToken(length - lastOffset, token);
149: break;
150: }
151:
152: return token;
153: }
154:
155: public static KeywordMap getKeywords() {
156: if (pyKeywords == null) {
157: pyKeywords = new KeywordMap(false);
158: pyKeywords.add("and", Token.KEYWORD3);
159: pyKeywords.add("not", Token.KEYWORD3);
160: pyKeywords.add("or", Token.KEYWORD3);
161: pyKeywords.add("if", Token.KEYWORD1);
162: pyKeywords.add("for", Token.KEYWORD1);
163: pyKeywords.add("assert", Token.KEYWORD1);
164: pyKeywords.add("break", Token.KEYWORD1);
165: pyKeywords.add("continue", Token.KEYWORD1);
166: pyKeywords.add("elif", Token.KEYWORD1);
167: pyKeywords.add("else", Token.KEYWORD1);
168: pyKeywords.add("except", Token.KEYWORD1);
169: pyKeywords.add("exec", Token.KEYWORD1);
170: pyKeywords.add("finally", Token.KEYWORD1);
171: pyKeywords.add("raise", Token.KEYWORD1);
172: pyKeywords.add("return", Token.KEYWORD1);
173: pyKeywords.add("try", Token.KEYWORD1);
174: pyKeywords.add("while", Token.KEYWORD1);
175: pyKeywords.add("def", Token.KEYWORD2);
176: pyKeywords.add("class", Token.KEYWORD2);
177: pyKeywords.add("del", Token.KEYWORD2);
178: pyKeywords.add("from", Token.KEYWORD2);
179: pyKeywords.add("global", Token.KEYWORD2);
180: pyKeywords.add("import", Token.KEYWORD2);
181: pyKeywords.add("in", Token.KEYWORD2);
182: pyKeywords.add("is", Token.KEYWORD2);
183: pyKeywords.add("lambda", Token.KEYWORD2);
184: pyKeywords.add("pass", Token.KEYWORD2);
185: pyKeywords.add("print", Token.KEYWORD2);
186: }
187: return pyKeywords;
188: }
189:
190: // private members
191: private static KeywordMap pyKeywords;
192:
193: private KeywordMap keywords;
194: private int lastOffset;
195: private int lastKeyword;
196:
197: private boolean doKeyword(Segment line, int i, char c) {
198: int i1 = i + 1;
199:
200: int len = i - lastKeyword;
201: byte id = keywords.lookup(line, lastKeyword, len);
202: if (id != Token.NULL) {
203: if (lastKeyword != lastOffset)
204: addToken(lastKeyword - lastOffset, Token.NULL);
205: addToken(len, id);
206: lastOffset = i;
207: }
208: lastKeyword = i1;
209: return false;
210: }
211: }
|