001: /*
002:
003: Derby - Class org.apache.derbyTesting.functionTests.harness.Sed
004:
005: Licensed to the Apache Software Foundation (ASF) under one or more
006: contributor license agreements. See the NOTICE file distributed with
007: this work for additional information regarding copyright ownership.
008: The ASF licenses this file to You under the Apache License, Version 2.0
009: (the "License"); you may not use this file except in compliance with
010: the License. You may obtain a copy of the License at
011:
012: http://www.apache.org/licenses/LICENSE-2.0
013:
014: Unless required by applicable law or agreed to in writing, software
015: distributed under the License is distributed on an "AS IS" BASIS,
016: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017: See the License for the specific language governing permissions and
018: limitations under the License.
019:
020: */
021:
022: package org.apache.derbyTesting.functionTests.harness;
023:
024: /***
025: * Sed.java
026: *
027: * This is a version of "sed" in Java for the Derby Function Tests,
028: * written using the OROMatcher Perl5 regular expression classes.
029: * The substitutions/deletions are based on the original kornshell tests.
030: *
031: ***/
032:
033: import java.io.*;
034: import java.util.Vector;
035: import org.apache.oro.text.regex.*;
036: import java.util.Enumeration;
037: import java.util.Properties;
038: import java.util.StringTokenizer;
039: import org.apache.derbyTesting.functionTests.util.TestUtil;
040:
041: public class Sed {
042: private static final String SQL_EXCEPTION_FILTERED_SUBSTITUTION = "java.sql.SQLException:";
043:
044: public Sed() {
045: }
046:
047: public static void main(String[] args) throws Exception {
048: if (args == null || args.length != 2) {
049: System.err.println("Usage: Sed sourcefile targetfile");
050: System.exit(1);
051: }
052: File src = new File(args[0]);
053: File tgt = new File(args[1]);
054: new Sed().exec(src, tgt, null, false, false, false);
055: }
056:
057: // The arguments should be the names of the input and output files
058: public void exec(File srcFile, File dstFile, InputStream isSed,
059: boolean isJCC, boolean isI18N, boolean isJDBC4)
060: throws IOException {
061: String hostName = TestUtil.getHostName();
062:
063: // Vector for storing lines to be deleted
064: Vector deleteLines = new Vector();
065: deleteLines.addElement("^ij version.*$");
066: deleteLines
067: .addElement("^\\*\\*\\*\\* Test Run Started .* \\*\\*\\*\\*$");
068: deleteLines
069: .addElement("^\\*\\*\\*\\* Test Run Completed .* \\*\\*\\*\\*$");
070: deleteLines.addElement("^ELAPSED TIME = [0-9]* milliseconds$");
071: deleteLines.addElement("^\\^\\?$");
072: //deleteLines.addElement("^\\.$"); // originally to remove lines with a dot
073: deleteLines.addElement("^S.*ij> $");
074: deleteLines.addElement("^ *$");
075: deleteLines.addElement("^Server StackTrace:$");
076: deleteLines.addElement("^\\[ *$");
077: deleteLines.addElement("^\\] *$");
078: deleteLines.addElement("^\\[$");
079: deleteLines.addElement("^\\]$");
080: deleteLines.addElement("^<not available>\\]$");
081: deleteLines.addElement("^(.*at .*)\\(.*:[0-9].*\\)$");
082: deleteLines.addElement("^(.*at .*)\\(*.java\\)$");
083: deleteLines.addElement("^(.*at .*)\\(Compiled Code\\)$");
084: deleteLines.addElement("^(.*at .*)\\(Interpreted Code\\)$");
085: deleteLines.addElement("^(.*at .*)\\(Unknown Source\\)$");
086: deleteLines.addElement("^(.*at .*)\\(Native Method\\)$");
087: deleteLines.addElement("^\\tat $"); // rare case of incomplete stack trace line
088: deleteLines.addElement("optimizer estimated cost");
089: deleteLines.addElement("optimizer estimated row count");
090: deleteLines
091: .addElement("Using executables built for native_threads");
092: deleteLines.addElement("Estimate of memory used");
093: deleteLines.addElement("Size of merge runs");
094: deleteLines.addElement("Number of merge runs");
095: deleteLines.addElement("Sort type");
096: deleteLines.addElement("Optimization started at .*$");
097: deleteLines
098: .addElement("WARNING 02000: No row was found for FETCH, UPDATE or DELETE");
099: // deleteLines for stack traces from j9 jvm to match those above for other jvms
100: deleteLines.addElement("Stack trace:");
101: deleteLines.addElement("^.*java/.*\\<init\\>\\(.*\\)V");
102: deleteLines.addElement("^.*org/apache/derby/.*\\(.*\\).*$");
103: // next for j9 stack trace with jarfiles test run.
104: deleteLines.addElement("^.*java/.*\\(.*\\).*$");
105: deleteLines.addElement("^\\[.*db2jcc.jar\\] [0-9].[1-9] - .*$");
106: deleteLines
107: .addElement("^\\[.*db2jcc_license_c.jar\\] [1-9].[0-9] - .*$");
108: deleteLines.addElement("^XSDB.*$");
109:
110: // JUnit noise
111: deleteLines.addElement("^\\.*$");
112: deleteLines.addElement("^Time: [0-9].*$");
113: deleteLines.addElement("^OK \\(.*$");
114:
115: // Vectors for substitutions
116: Vector searchStrings = new Vector();
117: searchStrings.addElement("^Transaction:\\(.*\\) *\\|");
118: searchStrings.addElement("^Read [0-9]* of [0-9]* bytes$");
119: searchStrings.addElement("Directory .*connect.wombat.seg0");
120: // Filter for constraint names - bug 5622 - our internal constraint names are too long. To be db2 compatible, we have reworked them.
121: StringBuffer constraintNameFilter = new StringBuffer();
122: constraintNameFilter
123: .append("SQL[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]");
124: searchStrings.addElement(constraintNameFilter.toString());
125: // Filter for uuids
126: StringBuffer uuidFilter = new StringBuffer();
127: uuidFilter
128: .append("[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-");
129: uuidFilter.append("[0-9a-f][0-9a-f][0-9a-f][0-9a-f]-");
130: uuidFilter.append("[0-9a-f][0-9a-f][0-9a-f][0-9a-f]-");
131: uuidFilter.append("[0-9a-f][0-9a-f][0-9a-f][0-9a-f]-");
132: uuidFilter
133: .append("[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]");
134: searchStrings.addElement(uuidFilter.toString());
135: // Filter for timestamps
136: StringBuffer timestampFilter = new StringBuffer();
137: timestampFilter
138: .append("[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ");
139: timestampFilter
140: .append("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]* *");
141: searchStrings.addElement(timestampFilter.toString());
142: // 3 digit year
143: timestampFilter = new StringBuffer();
144: timestampFilter
145: .append("[0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ");
146: timestampFilter
147: .append("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]* *");
148: searchStrings.addElement(timestampFilter.toString());
149: // ibm13 year
150: timestampFilter = new StringBuffer();
151: timestampFilter.append("[0-9]-[0-9][0-9]-[0-9][0-9] ");
152: timestampFilter
153: .append("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]* *");
154: searchStrings.addElement(timestampFilter.toString());
155: // Filter remove transaction id's from deadlock messages
156: searchStrings.addElement("^ Waiting XID : {.*}");
157: searchStrings.addElement("^ Granted XID : .*$");
158: searchStrings.addElement("^The selected victim is XID : .*");
159: // Filters for build numbers
160: searchStrings.addElement("(beta - )\\(([0-9]*)\\)");
161: searchStrings.addElement("Level2CostEstimateImpl: .*");
162: // Filter for xa tests for the numbers representing the db name (it can change)
163: searchStrings
164: .addElement("^Transaction ([0-9])* : \\(([0-9]*)\\,([0-9a-f]*)\\,([0-9a-f]*)\\)");
165: // Filter for optimizer number for zindexesLevel1 test (due to a change in display width for the test)
166: searchStrings
167: .addElement("^Modifying access paths using optimizer .[0-9]*");
168: searchStrings.addElement("CDWS[0-9]*");
169: searchStrings.addElement("IXWS[0-9]*");
170: // for j9, to eliminate intermittent failures due to this problem in j9:
171: searchStrings.addElement("FAILED STACK MAP");
172: if (isJCC) {
173: searchStrings.addElement("[ ]*\\|");
174: searchStrings.addElement("^--*");
175: }
176:
177: //Filter to suppress absould paths in error message for roll forward recovery tests
178: searchStrings.addElement("Directory.*.wombat.already.exists");
179: searchStrings
180: .addElement("Directory.*.extinout/crwombatlog/log.*.exists");
181:
182: // Filter for "DB2ConnectionCorrelator" text that can be printed as
183: // part of some JCC error messages.
184: searchStrings
185: .addElement(" DB2ConnectionCorrelator: [0-9A-Z.]*");
186: // Filter for SAX exception name diffs between jvms.
187: searchStrings.addElement("org.xml.sax.SAX.*$");
188: // Filter out localhost, or hostName
189: searchStrings.addElement(hostName);
190:
191: if (isJDBC4) {
192: // Filters for the sql exception class names which appear in
193: // exception messages. These are different in JDBC3 and JDBC4.
194: searchStrings.addElement("java.sql.SQLDataException:");
195: searchStrings
196: .addElement("java.sql.SQLDataSetSyncException:");
197: searchStrings.addElement("java.sql.SQLException:");
198: searchStrings
199: .addElement("java.sql.SQLFeatureNotSupportedException:");
200: searchStrings
201: .addElement("java.sql.SQLIntegrityConstraintViolationException:");
202: searchStrings
203: .addElement("java.sql.SQLInvalidAuthorizationSpecException:");
204: searchStrings
205: .addElement("java.sql.SQLNonTransientConnectionException:");
206: searchStrings
207: .addElement("java.sql.SQLNonTransientException:");
208: searchStrings.addElement("java.sql.SQLRuntimeException:");
209: searchStrings
210: .addElement("java.sql.SQLSyntaxErrorException:");
211: searchStrings.addElement("java.sql.SQLTimeoutException:");
212: searchStrings
213: .addElement("java.sql.SQLTransactionRollbackException:");
214: searchStrings
215: .addElement("java.sql.SQLTransientConnectionException:");
216: searchStrings.addElement("java.sql.SQLTransientException:");
217:
218: // The JDBC4 error from the driver is a little chattier
219: searchStrings
220: .addElement("No suitable driver found for [0-9A-Za-z:]*");
221: searchStrings
222: .addElement("No suitable driver;[0-9A-Za-z:=]*");
223: searchStrings
224: .addElement("SQL Exception: No suitable driver");
225:
226: // Timestamp diagnostic looks a little different under jdk16
227: searchStrings.addElement("\\[\\.fffffffff\\]");
228: }
229:
230: Vector subStrings = new Vector();
231: subStrings.addElement("Transaction:(XXX)|");
232: subStrings.addElement("Read ... bytes");
233: subStrings.addElement("Directory DBLOCATION/seg0");
234: subStrings.addElement("xxxxGENERATED-IDxxxx");
235: subStrings.addElement("xxxxFILTERED-UUIDxxxx");
236: subStrings.addElement("xxxxxxFILTERED-TIMESTAMPxxxxx");
237: subStrings.addElement("xxxxxxFILTERED-TIMESTAMPxxxxx");
238: subStrings.addElement("xxxxxxFILTERED-TIMESTAMPxxxxx");
239: // remove transaction id's from deadlock messages
240: subStrings.addElement(" Waiting XID : {WWW,QQQ}");
241: subStrings.addElement(" Granted XID : {GGG.QQQ}...");
242: subStrings.addElement("The selected victim is XID : VVV");
243: // sub build numbers
244: subStrings.addElement("$1(xxXXxxFILTERED-BUILD-NUMBERxxXXxx)");
245: subStrings
246: .addElement("Level2CostEstimateImpl: xxXXxxFILTERED-INFORMATIONxxXXxx");
247: // sub for db name in xa tests (it can change)
248: subStrings
249: .addElement("Transaction $1 : ($2,FILTERED,FILTERED)");
250: // sub for optimizer number for zindexesLevel1 test
251: subStrings
252: .addElement("Modifying access paths using optimizer FILTERED_NUMBER");
253: subStrings.addElement("CDWSno");
254: subStrings.addElement("IXWSno");
255: // for j9, to eliminate intermittent failures due to this problem in j9:
256: subStrings.addElement("");
257: // for JCC replace multiple blanks with one blank to handle differences
258: // in display width
259: if (isJCC) {
260: subStrings.addElement(" |");
261: subStrings.addElement("-----");
262: }
263: subStrings
264: .addElement("Directory DBLOCATION/wombat already exists");
265: subStrings
266: .addElement("Directory 'extinout<sp>crwombatlog<sp>log' exists");
267: // ignore the 'DB2ConnectionCorrelator' thing altogether.
268: subStrings.addElement("");
269: // Filter for SAX exception name diffs between jvms.
270: subStrings.addElement("xxxFILTERED-SAX-EXCEPTIONxxx'.");
271: // Filter out localhost, or hostName
272: subStrings.addElement("xxxFILTERED_HOSTNAMExxx");
273:
274: if (isJDBC4) {
275: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
276: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
277: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
278: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
279: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
280: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
281: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
282: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
283: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
284: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
285: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
286: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
287: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
288: subStrings.addElement(SQL_EXCEPTION_FILTERED_SUBSTITUTION);
289:
290: subStrings.addElement("No suitable driver");
291: subStrings.addElement("No suitable driver");
292: subStrings
293: .addElement("java.sql.SQLException: No suitable driver");
294:
295: subStrings.addElement(".fffffffff");
296: }
297:
298: doWork(srcFile, dstFile, null, deleteLines, searchStrings,
299: subStrings, isSed, isI18N);
300:
301: } // end exec
302:
303: // This just does JCC changes on the output master file
304: public void execJCC(InputStream is, File dstFile)
305: throws IOException {
306: // Vector for storing lines to be deleted
307: Vector deleteLines = new Vector();
308:
309: // Vectors for substitutions
310: Vector searchStrings = new Vector();
311: searchStrings.addElement("[ ]*\\|");
312: searchStrings.addElement("^--*");
313:
314: Vector subStrings = new Vector();
315: // true and false show up as 1 and 0 in JCC.
316: //because they have no boolean support
317: subStrings.addElement(" |");
318: subStrings.addElement("-----");
319:
320: doWork(null, dstFile, is, deleteLines, searchStrings,
321: subStrings, null);
322:
323: }
324:
325: private void doWork(File srcFile, File dstFile, InputStream is,
326: Vector deleteLines, Vector searchStrings,
327: Vector subStrings, InputStream isSed) throws IOException {
328: doWork(srcFile, dstFile, is, deleteLines, searchStrings,
329: subStrings, isSed, false);
330: }
331:
332: private void doWork(File srcFile, File dstFile, InputStream is,
333: Vector deleteLines, Vector searchStrings,
334: Vector subStrings, InputStream isSed, boolean isI18N)
335: throws IOException {
336:
337: boolean lineDeleted = false;
338: PatternMatcher matcher;
339: Perl5Compiler pcompiler;
340: PatternMatcherInput input;
341: BufferedReader inFile;
342: PrintWriter outFile;
343: String result = "";
344: String regex;
345: Vector delPatternVector = new Vector();
346: Vector subPatternVector = new Vector();
347:
348: // ---------------------------------
349: // Try loading the sed properties if they exist (see jdbc_sed.properties as an example)
350: if (isSed != null) {
351: Properties sedp = new Properties();
352:
353: sedp.load(isSed);
354: for (Enumeration e = sedp.propertyNames(); e
355: .hasMoreElements();) {
356: String key = (String) e.nextElement();
357: if (key.equals("substitute")) {
358: String value = sedp.getProperty(key);
359: // value string contains a comma separated list of patterns
360: StringTokenizer st = new StringTokenizer(value, ",");
361: String patternName = "";
362: String patName = "";
363: String subName = "";
364: while (st.hasMoreTokens()) {
365: patternName = st.nextToken();
366: // pattern;substitute
367: StringTokenizer st2 = new StringTokenizer(
368: patternName, ";");
369: patName = st2.nextToken();
370: subName = st2.nextToken();
371: if (!patName.equals("") && !subName.equals("")) {
372: searchStrings.addElement(patName);
373: subStrings.addElement(subName);
374: }
375: //System.out.println("pattern = " + patName + " substitute " + subName);
376: }
377: } else if (key.equals("delete")) {
378: String value = sedp.getProperty(key);
379: // value string contains a comma separated list of patterns
380: StringTokenizer st = new StringTokenizer(value, ",");
381: String patternName = "";
382: while (st.hasMoreTokens()) {
383: patternName = st.nextToken();
384: deleteLines.addElement(patternName);
385: }
386: }
387: }
388: }
389: // ---------------------------------
390:
391: //Create Perl5Compiler and Perl5Matcher
392: pcompiler = new Perl5Compiler();
393: matcher = new Perl5Matcher();
394:
395: // Define the input and output files based on args
396: if (is == null && isI18N) {
397: // read UTF-8 encoded file
398: InputStream fs = new FileInputStream(srcFile);
399: inFile = new BufferedReader(new InputStreamReader(fs,
400: "UTF-8"));
401: } else if (is == null) {
402: // read the file using the default encoding
403: inFile = new BufferedReader(new FileReader(srcFile));
404: } else {
405: inFile = new BufferedReader(new InputStreamReader(is,
406: "UTF-8"));
407: }
408: outFile = new PrintWriter(new BufferedWriter(new FileWriter(
409: dstFile), 10000), true);
410:
411: // Attempt to compile the patterns for deletes
412: for (int i = 0; i < deleteLines.size(); i++) {
413: try {
414: regex = (String) deleteLines.elementAt(i);
415: //System.out.println("The pattern: " + regex);
416: Pattern pattern = pcompiler.compile(regex);
417: if (pattern == null)
418: System.out.println("pattern is null");
419: delPatternVector.addElement(pattern);
420: } catch (MalformedPatternException e) {
421: System.out.println("Bad pattern.");
422: System.out.println(e.getMessage());
423: }
424: }
425:
426: // Attempt to compile the patterns for substitutes
427: for (int i = 0; i < searchStrings.size(); i++) {
428: try {
429: regex = (String) searchStrings.elementAt(i);
430: //System.out.println("The pattern: " + regex);
431: Pattern pattern = pcompiler.compile(regex);
432: if (pattern == null)
433: System.out.println("pattern is null");
434: subPatternVector.addElement(pattern);
435: } catch (MalformedPatternException e) {
436: System.out.println("Bad pattern.");
437: System.out.println(e.getMessage());
438: }
439: }
440:
441: String str;
442: int j;
443: int lineCount = 0;
444: // Read the input file
445: while ((str = inFile.readLine()) != null) {
446: lineCount++;
447:
448: //System.out.println("***Line no: " + lineCount);
449: //System.out.println("***Line is: " + str);
450: lineDeleted = false;
451:
452: // First delete any nulls (Cafe 1.8 leaves nulls)
453: if (str.length() == 1) {
454: if (str.charAt(0) == (char) 0) {
455: // Skip this line, don't write it
456: //System.out.println("Skip this line...");
457: lineDeleted = true;
458: }
459: }
460:
461: // Now determine if & if so, replace, any non-ascii characters
462: // We do this because non-ascii characters in .sql files will
463: // result in different characters depending on encoding, and
464: // encoding may be different on different os's
465: if (isI18N) {
466: boolean hasNonAscii = false;
467: // check for any characters in the control range
468: for (int si = 0; si < str.length(); si++) {
469: char c = str.charAt(si);
470: if (c < (char) 0x20 || c >= (char) 0x7f) {
471: hasNonAscii = true;
472: break;
473: }
474: }
475:
476: if (hasNonAscii) {
477: StringBuffer sb = new StringBuffer();
478: for (int si = 0; si < str.length(); si++) {
479: char c = str.charAt(si);
480: if (c < (char) 0x20 || c >= (char) 0x7f) {
481: sb.append(' ');
482: // Encoded Character:> ... <
483: sb.append("EnC:>");
484: sb.append((int) str.charAt(si));
485: sb.append("< ");
486: } else
487: sb.append(c);
488: }
489: str = sb.toString();
490: }
491: }
492:
493: // Determine if this line should be deleted for delete pattern match
494: if (lineDeleted == false) {
495: for (j = 0; j < delPatternVector.size(); j++) {
496: if (matcher.contains(str,
497: (Pattern) delPatternVector.elementAt(j))) {
498: //System.out.println("***Match found to delete line***");
499: String tmpp = ((Pattern) delPatternVector
500: .elementAt(j)).getPattern();
501: //System.out.println("***Pattern is: " + tmpp);
502:
503: // In this case we are removing the line, so don't write it out
504: lineDeleted = true;
505: break;
506: }
507: }
508: }
509:
510: // Determine if any substitutions are needed
511: if (lineDeleted == false) {
512: Substitution substitution;
513: StringSubstitution strsub = new StringSubstitution("");
514: Perl5Substitution perlsub = new Perl5Substitution("");
515: boolean subDone = false;
516: for (j = 0; j < subPatternVector.size(); j++) {
517: input = new PatternMatcherInput(str);
518: Pattern patt = (Pattern) subPatternVector
519: .elementAt(j);
520: String pstr = patt.getPattern();
521: //System.out.println("Pattern string is " + pstr);
522: String sub = (String) subStrings.elementAt(j);
523: if (sub.indexOf("$") > 0) {
524: perlsub.setSubstitution(sub);
525: substitution = (Substitution) perlsub;
526: } else {
527: strsub.setSubstitution(sub);
528: substitution = (Substitution) strsub;
529: }
530: //System.out.println("Substitute str = " + sub);
531: if (matcher.contains(input, patt)) {
532: MatchResult mr = matcher.getMatch();
533: //System.out.println("***Match found for substitute***");
534: // In this case we do a substitute
535: result = Util.substitute(matcher, patt,
536: substitution, str, Util.SUBSTITUTE_ALL);
537: //System.out.println("New string: " + result);
538: //outFile.println(result);
539: str = result;
540: subDone = true;
541: }
542: }
543: if (subDone) {
544: //System.out.println("write the subbed line");
545: outFile.println(result);
546: } else {
547: //System.out.println("Write the str: " + str);
548: outFile.println(str);
549: outFile.flush();
550: }
551: }// end if
552: } // end while
553: inFile.close();
554: outFile.flush();
555: outFile.close();
556: }// end doWork
557: }
|