From ea34f9d33c82624657136bccdd519ea23bd5e8e2 Mon Sep 17 00:00:00 2001 From: baztian Date: Fri, 4 Oct 2013 17:34:06 +0200 Subject: [PATCH] Add --xml option for tests. Using https://github.com/danielfm/unittest-xml-reporting/releases/tag/1.7.0 to create xml output from unit tests. --- libs/xmlrunner/__init__.py | 467 +++++++++++++++++++++++++++++++++++++ libs/xmlrunner/version.py | 14 ++ test.py | 7 +- 3 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 libs/xmlrunner/__init__.py create mode 100644 libs/xmlrunner/version.py diff --git a/libs/xmlrunner/__init__.py b/libs/xmlrunner/__init__.py new file mode 100644 index 0000000..2ce86d5 --- /dev/null +++ b/libs/xmlrunner/__init__.py @@ -0,0 +1,467 @@ +# -*- coding: utf-8 -*- + +""" +This module provides the XMLTestRunner class, which is heavily based on the +default TextTestRunner. +""" + +import os +import sys +import time +try: + from unittest2.runner import TextTestRunner + from unittest2.runner import TextTestResult as _TextTestResult + from unittest2.result import TestResult +except ImportError: + from unittest import TestResult, _TextTestResult, TextTestRunner + +try: + # Removed in Python 3 + from cStringIO import StringIO +except ImportError: + from io import StringIO + + +# Allow version to be detected at runtime. +from .version import __version__, __version_info__ + + +class _DelegateIO(object): + """ + This class defines an object that captures whatever is written to + a stream or file. + """ + + def __init__(self, delegate): + self._captured = StringIO() + self.delegate = delegate + + def write(self, text): + self._captured.write(text) + self.delegate.write(text) + + def __getattr__(self, attr): + return getattr(self._captured, attr) + + +def testcase_name(test_method): + testcase = type(test_method) + + # Ignore module name if it is '__main__' + module = testcase.__module__ + '.' + if module == '__main__.': + module = '' + result = module + testcase.__name__ + return result + + +class _TestInfo(object): + """ + This class keeps useful information about the execution of a + test method. + """ + + # Possible test outcomes + (SUCCESS, FAILURE, ERROR, SKIP) = range(4) + + def __init__(self, test_result, test_method, outcome=SUCCESS, err=None): + self.test_result = test_result + self.test_method = test_method + self.outcome = outcome + self.elapsed_time = 0 + self.err = err + + self.test_description = self.test_result.getDescription(test_method) + self.test_exception_info = ( + '' if outcome in (self.SUCCESS, self.SKIP) + else self.test_result._exc_info_to_string( + self.err, test_method) + ) + + self.test_name = testcase_name(test_method) + self.test_id = test_method.id() + + def id(self): + return self.test_method.id() + + def test_finished(self): + """Save info that can only be calculated once a test has run. + """ + self.elapsed_time = \ + self.test_result.stop_time - self.test_result.start_time + + def get_description(self): + """ + Return a text representation of the test method. + """ + return self.test_description + + def get_error_info(self): + """ + Return a text representation of an exception thrown by a test + method. + """ + return self.test_exception_info + + +class _XMLTestResult(_TextTestResult): + """ + A test result class that can express test results in a XML report. + + Used by XMLTestRunner. + """ + def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, + elapsed_times=True): + _TextTestResult.__init__(self, stream, descriptions, verbosity) + self.successes = [] + self.callback = None + self.elapsed_times = elapsed_times + + def _prepare_callback(self, test_info, target_list, verbose_str, + short_str): + """ + Appends a _TestInfo to the given target list and sets a callback + method to be called by stopTest method. + """ + target_list.append(test_info) + + def callback(): + """Prints the test method outcome to the stream, as well as + the elapsed time. + """ + + test_info.test_finished() + + # Ignore the elapsed times for a more reliable unit testing + if not self.elapsed_times: + self.start_time = self.stop_time = 0 + + if self.showAll: + self.stream.writeln( + '%s (%.3fs)' % (verbose_str, test_info.elapsed_time) + ) + elif self.dots: + self.stream.write(short_str) + self.callback = callback + + def startTest(self, test): + """ + Called before execute each test method. + """ + self.start_time = time.time() + TestResult.startTest(self, test) + + if self.showAll: + self.stream.write(' ' + self.getDescription(test)) + self.stream.write(" ... ") + + def stopTest(self, test): + """ + Called after execute each test method. + """ + _TextTestResult.stopTest(self, test) + self.stop_time = time.time() + + if self.callback and callable(self.callback): + self.callback() + self.callback = None + + def addSuccess(self, test): + """ + Called when a test executes successfully. + """ + self._prepare_callback( + _TestInfo(self, test), self.successes, 'OK', '.' + ) + + def addFailure(self, test, err): + """ + Called when a test method fails. + """ + testinfo = _TestInfo(self, test, _TestInfo.ERROR, err) + self.errors.append(( + testinfo, + self._exc_info_to_string(err, test) + )) + self._prepare_callback(testinfo, [], 'FAIL', 'F') + + def addError(self, test, err): + """ + Called when a test method raises an error. + """ + testinfo = _TestInfo(self, test, _TestInfo.ERROR, err) + self.errors.append(( + testinfo, + self._exc_info_to_string(err, test) + )) + self._prepare_callback(testinfo, [], 'ERROR', 'E') + + def addSkip(self, test, reason): + """ + Called when a test method was skipped. + """ + testinfo = _TestInfo(self, test, _TestInfo.SKIP, reason) + self.skipped.append((testinfo, reason)) + self._prepare_callback(testinfo, [], 'SKIP', 'S') + + def printErrorList(self, flavour, errors): + """ + Writes information about the FAIL or ERROR to the stream. + """ + for test_info, error in errors: + self.stream.writeln(self.separator1) + self.stream.writeln( + '%s [%.3fs]: %s' % (flavour, test_info.elapsed_time, + test_info.get_description()) + ) + self.stream.writeln(self.separator2) + self.stream.writeln('%s' % test_info.get_error_info()) + + def _get_info_by_testcase(self, outsuffix): + """ + Organizes test results by TestCase module. This information is + used during the report generation, where a XML report will be created + for each TestCase. + """ + tests_by_testcase = {} + + for tests in (self.successes, self.failures, self.errors, self.skipped): + for test_info in tests: + if isinstance(test_info, tuple): + # This is a skipped, error or a failure test case + test_info = test_info[0] + testcase_name = test_info.test_name + if not testcase_name in tests_by_testcase: + tests_by_testcase[testcase_name] = [] + tests_by_testcase[testcase_name].append(test_info) + + return tests_by_testcase + + def _report_testsuite(suite_name, outsuffix, tests, xml_document): + """ + Appends the testsuite section to the XML document. + """ + testsuite = xml_document.createElement('testsuite') + xml_document.appendChild(testsuite) + + testsuite.setAttribute('name', "%s-%s" % (suite_name, outsuffix)) + testsuite.setAttribute('tests', str(len(tests))) + + testsuite.setAttribute( + 'time', '%.3f' % sum(map(lambda e: e.elapsed_time, tests)) + ) + failures = filter(lambda e: e.outcome == _TestInfo.FAILURE, tests) + testsuite.setAttribute('failures', str(len(list(failures)))) + + errors = filter(lambda e: e.outcome == _TestInfo.ERROR, tests) + testsuite.setAttribute('errors', str(len(list(errors)))) + + return testsuite + + _report_testsuite = staticmethod(_report_testsuite) + + def _test_method_name(test_id): + """ + Returns the test method name. + """ + return test_id.split('.')[-1] + + _test_method_name = staticmethod(_test_method_name) + + def _report_testcase(suite_name, test_result, xml_testsuite, xml_document): + """ + Appends a testcase section to the XML document. + """ + testcase = xml_document.createElement('testcase') + xml_testsuite.appendChild(testcase) + + testcase.setAttribute('classname', suite_name) + testcase.setAttribute( + 'name', _XMLTestResult._test_method_name(test_result.test_id) + ) + testcase.setAttribute('time', '%.3f' % test_result.elapsed_time) + + if (test_result.outcome != _TestInfo.SUCCESS): + elem_name = ('failure', 'error', 'skipped')[test_result.outcome - 1] + failure = xml_document.createElement(elem_name) + testcase.appendChild(failure) + if test_result.outcome != _TestInfo.SKIP: + failure.setAttribute('type', test_result.err[0].__name__) + failure.setAttribute('message', str(test_result.err[1])) + error_info = str(test_result.get_error_info()) + failureText = xml_document.createCDATASection(error_info) + failure.appendChild(failureText) + else: + failure.setAttribute('type', 'skip') + failure.setAttribute('message', test_result.err) + + + _report_testcase = staticmethod(_report_testcase) + + def _report_output(test_runner, xml_testsuite, xml_document): + """ + Appends the system-out and system-err sections to the XML document. + """ + systemout = xml_document.createElement('system-out') + xml_testsuite.appendChild(systemout) + + systemout_text = xml_document.createCDATASection(sys.stdout.getvalue()) + systemout.appendChild(systemout_text) + + systemerr = xml_document.createElement('system-err') + xml_testsuite.appendChild(systemerr) + + systemerr_text = xml_document.createCDATASection(sys.stderr.getvalue()) + systemerr.appendChild(systemerr_text) + + _report_output = staticmethod(_report_output) + + def generate_reports(self, test_runner): + """ + Generates the XML reports to a given XMLTestRunner object. + """ + from xml.dom.minidom import Document + all_results = self._get_info_by_testcase(test_runner.outsuffix) + + if (isinstance(test_runner.output, str) and not + os.path.exists(test_runner.output)): + os.makedirs(test_runner.output) + + for suite, tests in all_results.items(): + doc = Document() + + # Build the XML file + testsuite = _XMLTestResult._report_testsuite( + suite, test_runner.outsuffix, tests, doc + ) + for test in tests: + _XMLTestResult._report_testcase(suite, test, testsuite, doc) + _XMLTestResult._report_output(test_runner, testsuite, doc) + xml_content = doc.toprettyxml(indent='\t') + + if type(test_runner.output) is str: + report_file = open( + '%s%sTEST-%s-%s.xml' % ( + test_runner.output, os.sep, suite, + test_runner.outsuffix + ), 'w' + ) + try: + report_file.write(xml_content) + finally: + report_file.close() + else: + # Assume that test_runner.output is a stream + test_runner.output.write(xml_content) + + +class XMLTestRunner(TextTestRunner): + """ + A test runner class that outputs the results in JUnit like XML files. + """ + def __init__(self, output='.', outsuffix=None, stream=sys.stderr, + descriptions=True, verbosity=1, elapsed_times=True): + TextTestRunner.__init__(self, stream, descriptions, verbosity) + self.verbosity = verbosity + self.output = output + if outsuffix: + self.outsuffix = outsuffix + else: + self.outsuffix = time.strftime("%Y%m%d%H%M%S") + self.elapsed_times = elapsed_times + + def _make_result(self): + """ + Creates a TestResult object which will be used to store + information about the executed tests. + """ + return _XMLTestResult( + self.stream, self.descriptions, self.verbosity, self.elapsed_times + ) + + def _patch_standard_output(self): + """ + Replaces stdout and stderr streams with string-based streams + in order to capture the tests' output. + """ + sys.stdout = _DelegateIO(sys.stdout) + sys.stderr = _DelegateIO(sys.stderr) + + def _restore_standard_output(self): + """ + Restores stdout and stderr streams. + """ + sys.stdout = sys.stdout.delegate + sys.stderr = sys.stderr.delegate + + def run(self, test): + """ + Runs the given test case or test suite. + """ + try: + # Prepare the test execution + self._patch_standard_output() + result = self._make_result() + + # Print a nice header + self.stream.writeln() + self.stream.writeln('Running tests...') + self.stream.writeln(result.separator2) + + # Execute tests + start_time = time.time() + test(result) + stop_time = time.time() + time_taken = stop_time - start_time + + # Print results + result.printErrors() + self.stream.writeln(result.separator2) + run = result.testsRun + self.stream.writeln("Ran %d test%s in %.3fs" % ( + run, run != 1 and "s" or "", time_taken) + ) + self.stream.writeln() + + expectedFails = unexpectedSuccesses = skipped = 0 + try: + results = map(len, (result.expectedFailures, + result.unexpectedSuccesses, + result.skipped)) + except AttributeError: + pass + else: + expectedFails, unexpectedSuccesses, skipped = results + + # Error traces + infos = [] + if not result.wasSuccessful(): + self.stream.write("FAILED") + failed, errored = map(len, (result.failures, result.errors)) + if failed: + infos.append("failures={0}".format(failed)) + if errored: + infos.append("errors={0}".format(errored)) + else: + self.stream.write("OK") + + if skipped: + infos.append("skipped={0}".format(skipped)) + if expectedFails: + infos.append("expected failures={0}".format(expectedFails)) + if unexpectedSuccesses: + infos.append("unexpected successes={0}".fornat(unexpectedSuccesses)) + + if infos: + self.stream.writeln(" ({0})".format(", ".join(infos))) + else: + self.stream.write("\n") + + # Generate reports + self.stream.writeln() + self.stream.writeln('Generating XML reports...') + result.generate_reports(self) + finally: + self._restore_standard_output() + + return result diff --git a/libs/xmlrunner/version.py b/libs/xmlrunner/version.py new file mode 100644 index 0000000..9f2975f --- /dev/null +++ b/libs/xmlrunner/version.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" + xmlrunner.version + ~~~~~~~~~~~~~~~~~ + + Runtime library version information. + + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + :copyright: © 2013 Pedro Algarvio. + :license: LGPL, see LICENSE for more details. +""" + +__version_info__ = (1, 7, 0) +__version__ = '.'.join(map(str, __version_info__)) diff --git a/test.py b/test.py index d22122a..7046c47 100644 --- a/test.py +++ b/test.py @@ -19,7 +19,12 @@ import unittest2 def main(): suite = unittest2.loader.TestLoader().discover('src/test') - unittest2.TextTestRunner(verbosity=2).run(suite) + if len(sys.argv) > 1 and '--xml' in sys.argv: + import xmlrunner + runner = xmlrunner.XMLTestRunner(output='build/test-reports') + else: + runner = unittest2.TextTestRunner(verbosity=2) + runner.run(suite) if __name__ == '__main__': main()