001: /*
002: * Copyright (C) 2005 - 2008 JasperSoft Corporation. All rights reserved.
003: * http://www.jaspersoft.com.
004: *
005: * Unless you have purchased a commercial license agreement from JasperSoft,
006: * the following license terms apply:
007: *
008: * This program is free software; you can redistribute it and/or modify
009: * it under the terms of the GNU General Public License version 2 as published by
010: * the Free Software Foundation.
011: *
012: * This program is distributed WITHOUT ANY WARRANTY; and without the
013: * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
014: * See the GNU General Public License for more details.
015: *
016: * You should have received a copy of the GNU General Public License
017: * along with this program; if not, see http://www.gnu.org/licenses/gpl.txt
018: * or write to:
019: *
020: * Free Software Foundation, Inc.,
021: * 59 Temple Place - Suite 330,
022: * Boston, MA USA 02111-1307
023: *
024: *
025: *
026: *
027: * PythonTokenMarker.java
028: *
029: */
030:
031: package org.syntax.jedit.tokenmarker;
032:
033: import org.syntax.jedit.*;
034: import javax.swing.text.Segment;
035:
036: /**
037: * Python token marker.
038: *
039: * @author Jonathan Revusky
040: * @version $Id: PythonTokenMarker.java 1167 2008-01-15 18:49:05Z gtoffoli $
041: */
042: public class PythonTokenMarker extends TokenMarker {
043: private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
044: private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;
045:
046: public PythonTokenMarker() {
047: this .keywords = getKeywords();
048: }
049:
050: public byte markTokensImpl(byte token, Segment line, int lineIndex) {
051: char[] array = line.array;
052: int offset = line.offset;
053: lastOffset = offset;
054: lastKeyword = offset;
055: int length = line.count + offset;
056: boolean backslash = false;
057:
058: loop: for (int i = offset; i < length; i++) {
059: int i1 = (i + 1);
060:
061: char c = array[i];
062: if (c == '\\') {
063: backslash = !backslash;
064: continue;
065: }
066:
067: switch (token) {
068: case Token.NULL:
069: switch (c) {
070: case '#':
071: if (backslash)
072: backslash = false;
073: else {
074: doKeyword(line, i, c);
075: addToken(i - lastOffset, token);
076: addToken(length - i, Token.COMMENT1);
077: lastOffset = lastKeyword = length;
078: break loop;
079: }
080: break;
081: case '"':
082: doKeyword(line, i, c);
083: if (backslash)
084: backslash = false;
085: else {
086: addToken(i - lastOffset, token);
087: if (SyntaxUtilities.regionMatches(false, line,
088: i1, "\"\"")) {
089: token = TRIPLEQUOTE1;
090: } else {
091: token = Token.LITERAL1;
092: }
093: lastOffset = lastKeyword = i;
094: }
095: break;
096: case '\'':
097: doKeyword(line, i, c);
098: if (backslash)
099: backslash = false;
100: else {
101: addToken(i - lastOffset, token);
102: if (SyntaxUtilities.regionMatches(false, line,
103: i1, "''")) {
104: token = TRIPLEQUOTE2;
105: } else {
106: token = Token.LITERAL2;
107: }
108: lastOffset = lastKeyword = i;
109: }
110: break;
111: default:
112: backslash = false;
113: if (!Character.isLetterOrDigit(c) && c != '_')
114: doKeyword(line, i, c);
115: break;
116: }
117: break;
118: case Token.LITERAL1:
119: if (backslash)
120: backslash = false;
121: else if (c == '"') {
122: addToken(i1 - lastOffset, token);
123: token = Token.NULL;
124: lastOffset = lastKeyword = i1;
125: }
126: break;
127: case Token.LITERAL2:
128: if (backslash)
129: backslash = false;
130: else if (c == '\'') {
131: addToken(i1 - lastOffset, Token.LITERAL1);
132: token = Token.NULL;
133: lastOffset = lastKeyword = i1;
134: }
135: break;
136: case TRIPLEQUOTE1:
137: if (backslash)
138: backslash = false;
139: else if (SyntaxUtilities.regionMatches(false, line, i,
140: "\"\"\"")) {
141: addToken((i += 4) - lastOffset, Token.LITERAL1);
142: token = Token.NULL;
143: lastOffset = lastKeyword = i;
144: }
145: break;
146: case TRIPLEQUOTE2:
147: if (backslash)
148: backslash = false;
149: else if (SyntaxUtilities.regionMatches(false, line, i,
150: "'''")) {
151: addToken((i += 4) - lastOffset, Token.LITERAL1);
152: token = Token.NULL;
153: lastOffset = lastKeyword = i;
154: }
155: break;
156: default:
157: throw new InternalError("Invalid state: " + token);
158: }
159: }
160:
161: switch (token) {
162: case TRIPLEQUOTE1:
163: case TRIPLEQUOTE2:
164: addToken(length - lastOffset, Token.LITERAL1);
165: break;
166: case Token.NULL:
167: doKeyword(line, length, '\0');
168: default:
169: addToken(length - lastOffset, token);
170: break;
171: }
172:
173: return token;
174: }
175:
176: public static KeywordMap getKeywords() {
177: if (pyKeywords == null) {
178: pyKeywords = new KeywordMap(false);
179: pyKeywords.add("and", Token.KEYWORD3);
180: pyKeywords.add("not", Token.KEYWORD3);
181: pyKeywords.add("or", Token.KEYWORD3);
182: pyKeywords.add("if", Token.KEYWORD1);
183: pyKeywords.add("for", Token.KEYWORD1);
184: pyKeywords.add("assert", Token.KEYWORD1);
185: pyKeywords.add("break", Token.KEYWORD1);
186: pyKeywords.add("continue", Token.KEYWORD1);
187: pyKeywords.add("elif", Token.KEYWORD1);
188: pyKeywords.add("else", Token.KEYWORD1);
189: pyKeywords.add("except", Token.KEYWORD1);
190: pyKeywords.add("exec", Token.KEYWORD1);
191: pyKeywords.add("finally", Token.KEYWORD1);
192: pyKeywords.add("raise", Token.KEYWORD1);
193: pyKeywords.add("return", Token.KEYWORD1);
194: pyKeywords.add("try", Token.KEYWORD1);
195: pyKeywords.add("while", Token.KEYWORD1);
196: pyKeywords.add("def", Token.KEYWORD2);
197: pyKeywords.add("class", Token.KEYWORD2);
198: pyKeywords.add("del", Token.KEYWORD2);
199: pyKeywords.add("from", Token.KEYWORD2);
200: pyKeywords.add("global", Token.KEYWORD2);
201: pyKeywords.add("import", Token.KEYWORD2);
202: pyKeywords.add("in", Token.KEYWORD2);
203: pyKeywords.add("is", Token.KEYWORD2);
204: pyKeywords.add("lambda", Token.KEYWORD2);
205: pyKeywords.add("pass", Token.KEYWORD2);
206: pyKeywords.add("print", Token.KEYWORD2);
207: }
208: return pyKeywords;
209: }
210:
211: // private members
212: private static KeywordMap pyKeywords;
213:
214: private KeywordMap keywords;
215: private int lastOffset;
216: private int lastKeyword;
217:
218: private boolean doKeyword(Segment line, int i, char c) {
219: int i1 = i + 1;
220:
221: int len = i - lastKeyword;
222: byte id = keywords.lookup(line, lastKeyword, len);
223: if (id != Token.NULL) {
224: if (lastKeyword != lastOffset)
225: addToken(lastKeyword - lastOffset, Token.NULL);
226: addToken(len, id);
227: lastOffset = i;
228: }
229: lastKeyword = i1;
230: return false;
231: }
232: }
|