001: /*=============================================================================
002: * Copyright Texas Instruments 2000. All Rights Reserved.
003: *
004: * This program is free software; you can redistribute it and/or
005: * modify it under the terms of the GNU Lesser General Public
006: * License as published by the Free Software Foundation; either
007: * version 2 of the License, or (at your option) any later version.
008: *
009: * This program is distributed in the hope that it will be useful,
010: * but WITHOUT ANY WARRANTY; without even the implied warranty of
011: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012: * Lesser General Public License for more details.
013: *
014: * You should have received a copy of the GNU Lesser General Public
015: * License along with this library; if not, write to the Free Software
016: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
017: *
018: * $ProjectHeader: OSCRIPT 0.155 Fri, 20 Dec 2002 18:34:22 -0800 rclark $
019: */
020:
021: package oscript;
022:
023: import java.util.*;
024:
025: import oscript.fs.*;
026: import oscript.data.*;
027: import oscript.exceptions.*;
028: import oscript.util.StackFrame;
029: import oscript.util.MemberTable;
030:
031: /**
032: * The regression test driver... basically finds all the .os files under the
033: * <i>test</i> subdirectory, and runs them in alphabetical order using the
034: * following algorithm:
035: * <pre>
036: * for( var itr=...; itr.hasNext(); )
037: * {
038: * var testfile = itr.next();
039: *
040: * // run test with interpreter:
041: * OscriptInterpreter.useCompiler(false);
042: * import testfile;
043: * OscriptInterpreter.flushNodeEvaluatorCache();
044: *
045: * // run test with compiler:
046: * OscriptInterpreter.useCompiler(true);
047: * import testfile;
048: * OscriptInterpreter.flushNodeEvaluatorCache();
049: * }
050: * </pre>
051: * The only reason the test driver is implemented as java is to minimize the
052: * dependence on the scripting engine working properly in order to run the
053: * tests.
054: * <p>
055: * Because the tests are run in alphabetical order, simpler test files should
056: * be named such that they run before more complex tests, so basic language
057: * features used by more complex tests are themselves tested first.
058: * <p>
059: * The functions <code>expect(n)</code>, <code>logError(str)</code> and
060: * <code>logInfo(str)</code> are provided for the individual tests to report
061: * the number of expected outputs, errors and status respectively.
062: * <p>
063: * The first thing an individual test should do is call <code>expect(n)</code>
064: * to indicate the number of expected <code>logInfo(str)</code> or
065: * <code>logError(str)</code> outputs. This is so the driver can catch
066: * situations where the test did not execute fully.
067: *
068: * @author Rob Clark (rob@ti.com)
069: * <!--$Format: " * @version $Revision$"$-->
070: * @version 1
071: */
072: class OscriptRegressionTestDriver {
073: /**
074: * The scope that the regression tests are imported into... it is in this
075: * scope that <code>logError(str)</code> and <code>logInfo(str)</code> are
076: * defined.
077: */
078: private static final Scope regressionScope = new BasicScope(
079: OscriptInterpreter.getGlobalScope());
080:
081: /**
082: * The name of the test currently running. Undefined value if not running
083: * a test.
084: */
085: private static String testName;
086:
087: /**
088: * Are we using the compiler for the current test. Undefined value if not
089: * running a test.
090: */
091: private static boolean useCompiler;
092:
093: /**
094: * The number of expected outputs for the current test.
095: */
096: private static int nexpected;
097:
098: /**
099: * The total number of outputs so far for the current test.
100: */
101: private static int noutputs;
102:
103: /**
104: * Total number of errors... this is reset before running the first test, and
105: * incremented any time that <code>logError(str)</code> is called.
106: */
107: private static int nerrs;
108:
109: /**
110: * Total number of expected outputs, ie. the number of "checks" performed
111: * by all the tests
112: */
113: private static int nchecks;
114:
115: /**
116: * Should we show <code>logInfo()</code> output in addition to <code>logError()</code>?
117: */
118: private static boolean verbose = System
119: .getProperty("oscript.test.verbose") != null;
120:
121: static {
122:
123: // expect:
124: {
125: final OString argNames[] = { OString.makeString("n") };
126:
127: regressionScope.createMember("expect", 0).opAssign(
128: new OBuiltinFunction(OString.makeString("expect"),
129: argNames) {
130: public Value callAsFunction(StackFrame sf,
131: MemberTable args) {
132: if ((args == null)
133: || (args.length() != argNames.length))
134: throw PackagedScriptObjectException
135: .makeExceptionWrapper(new OIllegalArgumentException(
136: "wrong number of args!"));
137:
138: expect((int) (args.referenceAt(0)
139: .castToExactNumber()));
140: return null;
141: }
142: });
143: }
144: // logError:
145: {
146: final OString argNames[] = { OString.makeString("str") };
147:
148: regressionScope.createMember("logError", 0).opAssign(
149: new OBuiltinFunction(
150: OString.makeString("logError"), argNames) {
151: public Value callAsFunction(StackFrame sf,
152: MemberTable args) {
153: if ((args == null)
154: || (args.length() != argNames.length))
155: throw PackagedScriptObjectException
156: .makeExceptionWrapper(new OIllegalArgumentException(
157: "wrong number of args!"));
158:
159: logError(args.referenceAt(0).castToString());
160: return null;
161: }
162: });
163: }
164:
165: // logInfo:
166: {
167: final OString argNames[] = { OString.makeString("str") };
168:
169: regressionScope.createMember("logInfo", 0).opAssign(
170: new OBuiltinFunction(OString.makeString("logInfo"),
171: argNames) {
172: public Value callAsFunction(StackFrame sf,
173: MemberTable args) {
174: if ((args == null)
175: || (args.length() != argNames.length))
176: throw PackagedScriptObjectException
177: .makeExceptionWrapper(new OIllegalArgumentException(
178: "wrong number of args!"));
179:
180: logInfo(args.referenceAt(0).castToString());
181: return null;
182: }
183: });
184: }
185:
186: }
187:
188: private synchronized static void expect(int n) {
189: nexpected = n;
190: noutputs = 0;
191: // note: nerrs & nchecks are cumulative, not per test, so it does not get reset!
192: }
193:
194: private synchronized static void logError(String str) {
195: noutputs++;
196: nerrs++;
197: nchecks++;
198: System.out.println("[E] " + getTestDescriptor() + ": " + str);
199: }
200:
201: private synchronized static void logInfo(String str) {
202: noutputs++;
203: nchecks++;
204: if (verbose)
205: System.out.println("[I] " + getTestDescriptor() + ": "
206: + str);
207: }
208:
209: private static String getTestDescriptor() {
210: return testName + "-" + (useCompiler ? "c" : "i");
211: }
212:
213: private static Set findTests(AbstractFile file, Set testSet)
214: throws java.io.IOException {
215: if (file.exists() && file.canRead()) {
216: if (file.isDirectory())
217: for (Iterator itr = AbstractFileSystem.children(
218: file.getPath()).iterator(); itr.hasNext();)
219: findTests((AbstractFile) (itr.next()), testSet);
220: else if (file.getExtension().equals("os")
221: && !file.getName().startsWith("."))
222: testSet.add(file);
223: }
224:
225: return testSet;
226: }
227:
228: private static void runRegressionTest(AbstractFile file,
229: boolean useCompiler) throws oscript.parser.ParseException,
230: java.io.IOException {
231: OscriptInterpreter
232: .useCompiler(OscriptRegressionTestDriver.useCompiler = useCompiler);
233: logInfo("starting");
234: expect(-1); // set to bogus value to detect case of test not calling expect()
235: OscriptInterpreter.eval(file, regressionScope);
236: if (noutputs != nexpected)
237: logError("wrong number of results, expected " + nexpected
238: + " but got " + noutputs);
239: OscriptInterpreter.flushNodeEvaluatorCache();
240: }
241:
242: /**
243: * The test driver... called via {@link BuiltinFunction} __runRegressionTests
244: * defined in {@link OscriptBuiltins} in order to minimize the parts of the
245: * scripting engine that we depend on in order to run the regression tests
246: * (ie, so we don't depend on the java-bridge code working).
247: *
248: * @return the number of erros
249: */
250: static synchronized int runRegressionTests() {
251: try {
252: nchecks = nerrs = 0;
253:
254: AbstractFile testRoot = OscriptInterpreter.resolve("tests",
255: false);
256: String testRootPath = testRoot.getPath();
257:
258: Set testSet = findTests(testRoot, new TreeSet(
259: new Comparator() {
260:
261: public int compare(Object o1, Object o2) {
262: return ((AbstractFile) o1).getPath()
263: .compareTo(
264: ((AbstractFile) o2)
265: .getPath());
266: }
267:
268: }));
269:
270: for (Iterator itr = testSet.iterator(); itr.hasNext();) {
271: AbstractFile file = (AbstractFile) (itr.next());
272:
273: String path = file.getPath();
274: testName = path.substring(testRootPath.length() + 1,
275: path.length() - 3);
276:
277: // run test with interpreter:
278: // runRegressionTest( file, false );
279:
280: // run test with compiler:
281: runRegressionTest(file, true);
282:
283: if (nerrs > 0) {
284: logInfo(testName + " failed! Bailing out!");
285: break;
286: }
287: }
288:
289: System.out.println(nerrs + " errors out of " + nchecks);
290: return nerrs;
291: } catch (java.io.IOException e) {
292: logError("parse error: " + e.getMessage());
293: return -1;
294: } catch (oscript.parser.ParseException e) {
295: logError("I/O error: " + e.getMessage());
296: return -2;
297: } catch (Throwable e) {
298: e.printStackTrace();
299: logError("unhandled exception: " + e);
300: return -3;
301: }
302: }
303: }
304:
305: /*
306: * Local Variables:
307: * tab-width: 2
308: * indent-tabs-mode: nil
309: * mode: java
310: * c-indentation-style: java
311: * c-basic-offset: 2
312: * eval: (c-set-offset 'substatement-open '0)
313: * eval: (c-set-offset 'case-label '+)
314: * eval: (c-set-offset 'inclass '+)
315: * eval: (c-set-offset 'inline-open '0)
316: * End:
317: */
|