001: /*
002: * Copyright 2007 Google Inc.
003: *
004: * Licensed under the Apache License, Version 2.0 (the "License"); you may not
005: * use this file except in compliance with the License. You may obtain a copy of
006: * the License at
007: *
008: * http://www.apache.org/licenses/LICENSE-2.0
009: *
010: * Unless required by applicable law or agreed to in writing, software
011: * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
012: * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
013: * License for the specific language governing permissions and limitations under
014: * the License.
015: */
016: package com.google.gwt.junit.benchmarks;
017:
018: import com.google.gwt.core.ext.TreeLogger;
019: import com.google.gwt.core.ext.typeinfo.HasMetaData;
020: import com.google.gwt.core.ext.typeinfo.JClassType;
021: import com.google.gwt.core.ext.typeinfo.JMethod;
022: import com.google.gwt.core.ext.typeinfo.TypeOracle;
023: import com.google.gwt.dev.util.Util;
024: import com.google.gwt.junit.client.TestResults;
025: import com.google.gwt.junit.client.Trial;
026: import com.google.gwt.junit.rebind.BenchmarkGenerator;
027: import com.google.gwt.util.tools.Utility;
028:
029: import junit.framework.TestCase;
030:
031: import org.w3c.dom.Document;
032: import org.w3c.dom.Element;
033:
034: import java.io.File;
035: import java.io.FileOutputStream;
036: import java.io.IOException;
037: import java.text.BreakIterator;
038: import java.text.DateFormat;
039: import java.util.ArrayList;
040: import java.util.Date;
041: import java.util.HashMap;
042: import java.util.List;
043: import java.util.Map;
044: import java.util.regex.Matcher;
045: import java.util.regex.Pattern;
046:
047: import javax.xml.parsers.DocumentBuilder;
048: import javax.xml.parsers.DocumentBuilderFactory;
049: import javax.xml.parsers.ParserConfigurationException;
050:
051: /**
052: * Generates a detailed report that contains the results of all of the
053: * benchmark-related unit tests executed during a unit test session. The primary
054: * user of this class is {@link com.google.gwt.junit.JUnitShell}.
055: *
056: * The report is in XML format. To view the XML reports, use benchmarkViewer.
057: */
058: public class BenchmarkReport {
059:
060: /**
061: * Converts a set of test results for a single benchmark method into XML.
062: */
063: private class BenchmarkXml {
064:
065: private MetaData metaData;
066:
067: private List<TestResults> results;
068:
069: private TestCase test;
070:
071: BenchmarkXml(TestCase test, List<TestResults> results) {
072: this .test = test;
073: this .results = results;
074: Map<String, MetaData> methodMetaData = testMetaData
075: .get(test.getClass().toString());
076: metaData = methodMetaData.get(test.getName());
077: }
078:
079: Element toElement(Document doc) {
080: Element benchmark = doc.createElement("benchmark");
081: benchmark.setAttribute("class", test.getClass().getName());
082: benchmark.setAttribute("name", metaData.getTestName());
083: benchmark.setAttribute("description", metaData
084: .getTestDescription());
085:
086: String sourceCode = metaData.getSourceCode();
087: if (sourceCode != null) {
088: Element sourceCodeElement = doc
089: .createElement("source_code");
090: sourceCodeElement.appendChild(doc
091: .createTextNode(sourceCode));
092: benchmark.appendChild(sourceCodeElement);
093: }
094:
095: // TODO(tobyr): create target_code element
096:
097: for (TestResults result : results) {
098: benchmark.appendChild(toElement(doc, result));
099: }
100:
101: return benchmark;
102: }
103:
104: private Element toElement(Document doc, TestResults result) {
105: Element resultElement = doc.createElement("result");
106: resultElement.setAttribute("host", result.getHost());
107: resultElement.setAttribute("agent", result.getAgent());
108:
109: List<Trial> trials = result.getTrials();
110:
111: for (Trial trial : trials) {
112: Element trialElement = toElement(doc, trial);
113: resultElement.appendChild(trialElement);
114: }
115:
116: return resultElement;
117: }
118:
119: private Element toElement(Document doc, Trial trial) {
120: Element trialElement = doc.createElement("trial");
121:
122: Map<String, String> variables = trial.getVariables();
123:
124: for (Map.Entry<String, String> entry : variables.entrySet()) {
125: Object name = entry.getKey();
126: Object value = entry.getValue();
127: Element variableElement = doc.createElement("variable");
128: variableElement.setAttribute("name", name.toString());
129: variableElement.setAttribute("value", value.toString());
130: trialElement.appendChild(variableElement);
131: }
132:
133: trialElement.setAttribute("timing", String.valueOf(trial
134: .getRunTimeMillis()));
135:
136: Throwable exception = trial.getException();
137:
138: if (exception != null) {
139: Element exceptionElement = doc
140: .createElement("exception");
141: exceptionElement.appendChild(doc
142: .createTextNode(exception.toString()));
143: trialElement.appendChild(exceptionElement);
144: }
145:
146: return trialElement;
147: }
148: }
149:
150: /**
151: * Parses .java source files to get source code for methods.
152: */
153: private class Parser {
154:
155: /**
156: * Maps classes to the contents of their source files.
157: */
158: private Map<JClassType, String> classSources = new HashMap<JClassType, String>();
159:
160: /**
161: * Returns the source code for the method of the given name.
162: *
163: * @param method a not <code>null</code> method
164: * @return <code>null</code> if the source code for the method can not be
165: * located
166: */
167: public String getMethod(JMethod method) {
168: JClassType clazz = method.getEnclosingType();
169:
170: if (!classSources.containsKey(clazz)) {
171: char[] sourceContents = null;
172: File sourceFile = findSourceFile(clazz);
173: if (sourceFile != null) {
174: sourceContents = Util.readFileAsChars(sourceFile);
175: classSources.put(clazz, new String(sourceContents));
176: }
177:
178: if (sourceContents == null) {
179: classSources.put(clazz, null);
180: String msg = "An unknown I/O exception occured while trying to read "
181: + sourceFile.getAbsolutePath();
182: logger.log(TreeLogger.WARN, msg, null);
183: } else {
184: classSources.put(clazz, new String(sourceContents));
185: String msg = "BenchmarkReport read the contents of "
186: + sourceFile.getAbsolutePath();
187: TreeLogger branch = logger.branch(TreeLogger.DEBUG,
188: msg, null);
189: if (logger.isLoggable(TreeLogger.SPAM)) {
190: branch.log(TreeLogger.SPAM, new String(
191: sourceContents), null);
192: }
193: }
194: }
195:
196: String source = classSources.get(clazz);
197:
198: if (source == null) {
199: return source;
200: }
201:
202: try {
203: return source.substring(method.getDeclStart(), method
204: .getDeclEnd() + 1);
205: } catch (IndexOutOfBoundsException e) {
206: logger.log(TreeLogger.WARN, "Unable to parse "
207: + method.getName(), e);
208: // Have seen this happen when the compiler read the source using one
209: // character encoding, and then this Parser read it in a different
210: // encoding. I don't know if there are other cases in which this can
211: // occur.
212: return null;
213: }
214: }
215: }
216:
217: /**
218: * Converts an entire report into XML.
219: */
220: private class ReportXml {
221:
222: private Map<String, Element> categoryElementMap = new HashMap<String, Element>();
223:
224: private Date date = new Date();
225:
226: private String version = "unknown";
227:
228: Element toElement(Document doc) {
229: Element report = doc.createElement("gwt_benchmark_report");
230: String dateString = DateFormat.getDateTimeInstance()
231: .format(date);
232: report.setAttribute("date", dateString);
233: report.setAttribute("gwt_version", version);
234:
235: // Add each test result into the report.
236: // Add the category for the test result, if necessary.
237: for (Map.Entry<TestCase, List<TestResults>> entry : testResults
238: .entrySet()) {
239: TestCase test = entry.getKey();
240: List<TestResults> results = entry.getValue();
241: BenchmarkXml xml = new BenchmarkXml(test, results);
242: Element categoryElement = getCategoryElement(doc,
243: report, xml.metaData.getCategory()
244: .getClassName());
245: categoryElement.appendChild(xml.toElement(doc));
246: }
247:
248: return report;
249: }
250:
251: /**
252: * Locates or creates the category element by the specified name.
253: *
254: * @param doc The document to search
255: * @param report The report to which the category belongs
256: * @param name The name of the category
257: *
258: * @return The matching category element
259: */
260: private Element getCategoryElement(Document doc,
261: Element report, String name) {
262: Element e = categoryElementMap.get(name);
263:
264: if (e != null) {
265: return e;
266: }
267:
268: Element categoryElement = doc.createElement("category");
269: categoryElementMap.put(name, categoryElement);
270: CategoryImpl category = testCategories.get(name);
271: categoryElement.setAttribute("name", category.getName());
272: categoryElement.setAttribute("description", category
273: .getDescription());
274:
275: report.appendChild(categoryElement);
276:
277: return categoryElement;
278: }
279: }
280:
281: private static final String GWT_BENCHMARK_CATEGORY = "gwt.benchmark.category";
282:
283: private static final String GWT_BENCHMARK_DESCRIPTION = "gwt.benchmark.description";
284:
285: private static final String GWT_BENCHMARK_NAME = "gwt.benchmark.name";
286:
287: private static File findSourceFile(JClassType clazz) {
288: final char separator = File.separator.charAt(0);
289: String filePath = clazz.getPackage().getName().replace('.',
290: separator)
291: + separator + clazz.getSimpleSourceName() + ".java";
292: String[] paths = getClassPath();
293:
294: for (int i = 0; i < paths.length; ++i) {
295: File maybeSourceFile = new File(paths[i] + separator
296: + filePath);
297:
298: if (maybeSourceFile.exists()) {
299: return maybeSourceFile;
300: }
301: }
302:
303: return null;
304: }
305:
306: private static String[] getClassPath() {
307: String path = System.getProperty("java.class.path");
308: return path.split(File.pathSeparator);
309: }
310:
311: private static String getSimpleMetaData(HasMetaData hasMetaData,
312: String name) {
313: String[][] allValues = hasMetaData.getMetaData(name);
314:
315: if (allValues == null) {
316: return null;
317: }
318:
319: StringBuffer result = new StringBuffer();
320:
321: for (int i = 0; i < allValues.length; ++i) {
322: String[] values = allValues[i];
323: for (int j = 0; j < values.length; ++j) {
324: result.append(values[j]);
325: result.append(" ");
326: }
327: }
328:
329: String resultString = result.toString().trim();
330: return resultString.equals("") ? null : resultString;
331: }
332:
333: private TreeLogger logger;
334:
335: private Parser parser = new Parser();
336:
337: private Map<String, CategoryImpl> testCategories = new HashMap<String, CategoryImpl>();
338:
339: private Map<String, Map<String, MetaData>> testMetaData = new HashMap<String, Map<String, MetaData>>();
340:
341: private Map<TestCase, List<TestResults>> testResults = new HashMap<TestCase, List<TestResults>>();
342:
343: private TypeOracle typeOracle;
344:
345: public BenchmarkReport(TreeLogger logger) {
346: this .logger = logger;
347: }
348:
349: /**
350: * Adds the Benchmark to the report. All of the metadata about the benchmark
351: * (category, name, description, etc...) is recorded from the TypeOracle.
352: *
353: * @param benchmarkClass The benchmark class to record. Must not be
354: * <code>null</code>.
355: * @param typeOracle The <code>TypeOracle</code> for the compilation session
356: * must not be <code>null</code>.
357: */
358: public void addBenchmark(JClassType benchmarkClass,
359: TypeOracle typeOracle) {
360:
361: this .typeOracle = typeOracle;
362: String categoryType = getSimpleMetaData(benchmarkClass,
363: GWT_BENCHMARK_CATEGORY);
364:
365: Map<String, JMethod> zeroArgMethods = BenchmarkGenerator
366: .getNotOverloadedTestMethods(benchmarkClass);
367: Map<String, JMethod> parameterizedMethods = BenchmarkGenerator
368: .getParameterizedTestMethods(benchmarkClass,
369: TreeLogger.NULL);
370: List<JMethod> testMethods = new ArrayList<JMethod>(
371: zeroArgMethods.size() + parameterizedMethods.size());
372: testMethods.addAll(zeroArgMethods.values());
373: testMethods.addAll(parameterizedMethods.values());
374:
375: Map<String, MetaData> metaDataMap = testMetaData
376: .get(benchmarkClass.toString());
377: if (metaDataMap == null) {
378: metaDataMap = new HashMap<String, MetaData>();
379: testMetaData.put(benchmarkClass.toString(), metaDataMap);
380: }
381:
382: // Add all of the benchmark methods
383: for (JMethod method : testMethods) {
384: String methodName = method.getName();
385: String methodCategoryType = getSimpleMetaData(method,
386: GWT_BENCHMARK_CATEGORY);
387: if (methodCategoryType == null) {
388: methodCategoryType = categoryType;
389: }
390: CategoryImpl methodCategory = getCategory(methodCategoryType);
391:
392: String methodSource = parser.getMethod(method);
393: StringBuffer sourceBuffer = (methodSource == null) ? null
394: : new StringBuffer(methodSource);
395: StringBuffer summary = new StringBuffer();
396: StringBuffer comment = new StringBuffer();
397: getComment(sourceBuffer, summary, comment);
398:
399: MetaData metaData = new MetaData(benchmarkClass.toString(),
400: methodName, (sourceBuffer != null) ? sourceBuffer
401: .toString() : null, methodCategory,
402: methodName, summary.toString());
403: metaDataMap.put(methodName, metaData);
404: }
405: }
406:
407: public void addBenchmarkResults(TestCase test, TestResults results) {
408: List<TestResults> currentResults = testResults.get(test);
409: if (currentResults == null) {
410: currentResults = new ArrayList<TestResults>();
411: testResults.put(test, currentResults);
412: }
413: currentResults.add(results);
414: }
415:
416: /**
417: * Generates reports for all of the benchmarks which were added to the
418: * generator.
419: *
420: * @param outputPath The path to write the reports to.
421: * @throws ParserConfigurationException If an error occurs during xml parsing
422: * @throws IOException If anything goes wrong writing to outputPath
423: */
424: public void generate(String outputPath)
425: throws ParserConfigurationException, IOException {
426:
427: // Don't generate a new report if no tests were actually run.
428: if (testResults.size() == 0) {
429: return;
430: }
431:
432: DocumentBuilderFactory factory = DocumentBuilderFactory
433: .newInstance();
434: DocumentBuilder builder = factory.newDocumentBuilder();
435: Document doc = builder.newDocument();
436: doc.appendChild(new ReportXml().toElement(doc));
437: byte[] xmlBytes = Util.toXmlUtf8(doc);
438: FileOutputStream fos = null;
439: try {
440: fos = new FileOutputStream(outputPath);
441: fos.write(xmlBytes);
442: } finally {
443: Utility.close(fos);
444: }
445:
446: // TODO(bruce) The code below is commented out because of GWT Issue 958.
447:
448: // // TODO(tobyr) Looks like indenting is busted
449: // // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6296446
450: // // Not a big deal, since we don't intend to read the XML by hand anyway
451: // TransformerFactory transformerFactory = TransformerFactory.newInstance();
452: // // Think this can be used with JDK 1.5
453: // // transformerFactory.setAttribute( "indent-number", new Integer(2) );
454: // Transformer serializer = transformerFactory.newTransformer();
455: // serializer.setOutputProperty(OutputKeys.METHOD, "xml");
456: // serializer.setOutputProperty(OutputKeys.INDENT, "yes");
457: // serializer
458: // .setOutputProperty("{ http://xml.apache.org/xslt }indent-amount", "2");
459: // BufferedOutputStream docOut = new BufferedOutputStream(
460: // new FileOutputStream(outputPath));
461: // serializer.transform(new DOMSource(doc), new StreamResult(docOut));
462: // docOut.close();
463: }
464:
465: private CategoryImpl getCategory(String name) {
466: CategoryImpl c = testCategories.get(name);
467:
468: if (c != null) {
469: return c;
470: }
471:
472: String categoryName = "";
473: String categoryDescription = "";
474:
475: if (name != null) {
476: JClassType categoryType = typeOracle.findType(name);
477:
478: if (categoryType != null) {
479: categoryName = getSimpleMetaData(categoryType,
480: GWT_BENCHMARK_NAME);
481: categoryDescription = getSimpleMetaData(categoryType,
482: GWT_BENCHMARK_DESCRIPTION);
483: }
484: }
485:
486: c = new CategoryImpl(name, categoryName, categoryDescription);
487: testCategories.put(name, c);
488: return c;
489: }
490:
491: /**
492: * Parses out the JavaDoc comment from a string of source code. Returns the
493: * first sentence summary in <code>summary</code> and the body of the entire
494: * comment (including the summary) in <code>comment</code>.
495: *
496: * @param sourceCode The source code of a function, including its comment.
497: * Modified to remove leading whitespace.
498: * @param summary Modified to contain the first sentence of the comment.
499: * @param comment Modified to contain the entire comment.
500: */
501: private void getComment(StringBuffer sourceCode,
502: StringBuffer summary, StringBuffer comment) {
503:
504: if (sourceCode == null) {
505: return;
506: }
507:
508: summary.setLength(0);
509: comment.setLength(0);
510:
511: String regex = "/\\*\\*(.(?!}-\\*/))*\\*/";
512:
513: Pattern p = Pattern.compile(regex, Pattern.DOTALL);
514: Matcher m = p.matcher(sourceCode);
515:
516: // Early out if there is no javadoc comment.
517: if (!m.find()) {
518: return;
519: }
520:
521: String commentStr = m.group();
522:
523: p = Pattern.compile("(/\\*\\*\\s*)" + // The comment header
524: "(((\\s*\\**\\s*)([^\n\r]*)[\n\r]+)*)" // The comment body
525: );
526:
527: m = p.matcher(commentStr);
528:
529: if (!m.find()) {
530: return;
531: }
532:
533: String stripped = m.group(2);
534:
535: p = Pattern.compile("^\\p{Blank}*\\**\\p{Blank}*",
536: Pattern.MULTILINE);
537: String bareComment = p.matcher(stripped).replaceAll("");
538:
539: BreakIterator iterator = BreakIterator.getSentenceInstance();
540: iterator.setText(bareComment);
541: int firstSentenceEnd = iterator.next();
542: if (firstSentenceEnd == BreakIterator.DONE) {
543: summary.append(bareComment);
544: } else {
545: summary.append(bareComment.substring(0, firstSentenceEnd));
546: }
547:
548: comment.append(bareComment);
549:
550: // Measure the indentation width on the second line to infer what the
551: // first line indent should be.
552: p = Pattern.compile("[^\\r\\n]+[\\r\\n]+(\\s+)\\*",
553: Pattern.MULTILINE);
554: m = p.matcher(sourceCode);
555: int indentLen = 0;
556: if (m.find()) {
557: String indent = m.group(1);
558: indentLen = indent.length() - 1;
559: }
560: StringBuffer leadingIndent = new StringBuffer();
561: for (int i = 0; i < indentLen; ++i) {
562: leadingIndent.append(' ');
563: }
564:
565: // By inserting at 0 here, we are assuming that sourceCode begins with
566: // /**, which is actually a function of how JDT sees a declaration start.
567: // If in the future, you see bogus indentation here, it means that this
568: // assumption is bad.
569: sourceCode.insert(0, leadingIndent);
570: }
571: }
|