"""Support for parametric tests in unittest.
:Author: Fernando Perez
Purpose
=======
Briefly, the main class in this module allows you to easily and cleanly
(without the gross name-mangling hacks that are normally needed) to write
unittest TestCase classes that have parametrized tests. That is, tests which
consist of multiple sub-tests that scan for example a parameter range, but
where you want each sub-test to:
* count as a separate test in the statistics.
* be run even if others in the group error out or fail.
The class offers a simple name-based convention to create such tests (see
simple example at the end), in one of two ways:
* Each sub-test in a group can be run fully independently, with the
setUp/tearDown methods being called each time.
* The whole group can be run with setUp/tearDown being called only once for the
group. This lets you conveniently reuse state that may be very expensive to
compute for multiple tests. Be careful not to corrupt it!!!
Caveats
=======
This code relies on implementation details of the unittest module (some key
methods are heavily modified versions of those, after copying them in). So it
may well break either if you make sophisticated use of the unittest APIs, or if
unittest itself changes in the future. I have only tested this with Python
2.5.
"""
__docformat__ = "restructuredtext en"
import unittest
class ParametricTestCase(unittest.TestCase):
"""TestCase subclass with support for parametric tests.
Subclasses of this class can implement test methods that return a list of
tests and arguments to call those with, to do parametric testing (often
also called 'data driven' testing."""
#: Prefix for tests with independent state. These methods will be run with
#: a separate setUp/tearDown call for each test in the group.
_indepParTestPrefix = 'testip'
#: Prefix for tests with shared state. These methods will be run with
#: a single setUp/tearDown call for the whole group. This is useful when
#: writing a group of tests for which the setup is expensive and one wants
#: to actually share that state. Use with care (especially be careful not
#: to mutate the state you are using, which will alter later tests).
_shareParTestPrefix = 'testsp'
def exec_test(self,test,args,result):
"""Execute a single test. Returns a success boolean"""
ok = False
try:
test(*args)
ok = True
except self.failureException:
result.addFailure(self, self._exc_info())
except KeyboardInterrupt:
raise
except:
result.addError(self, self._exc_info())
return ok
def set_testMethodDoc(self,doc):
self._testMethodDoc = doc
self._TestCase__testMethodDoc = doc
def get_testMethodDoc(self):
return self._testMethodDoc
testMethodDoc = property(fset=set_testMethodDoc, fget=get_testMethodDoc)
def get_testMethodName(self):
try:
return getattr(self,"_testMethodName")
except:
return getattr(self,"_TestCase__testMethodName")
testMethodName = property(fget=get_testMethodName)
def run_test(self, testInfo,result):
"""Run one test with arguments"""
test,args = testInfo[0],testInfo[1:]
# Reset the doc attribute to be the docstring of this particular test,
# so that in error messages it prints the actual test's docstring and
# not that of the test factory.
self.testMethodDoc = test.__doc__
result.startTest(self)
try:
try:
self.setUp()
except KeyboardInterrupt:
raise
except:
result.addError(self, self._exc_info())
return
ok = self.exec_test(test,args,result)
try:
self.tearDown()
except KeyboardInterrupt:
raise
except:
result.addError(self, self._exc_info())
ok = False
if ok: result.addSuccess(self)
finally:
result.stopTest(self)
def run_tests(self, tests,result):
"""Run many tests with a common setUp/tearDown.
The entire set of tests is run with a single setUp/tearDown call."""
try:
self.setUp()
except KeyboardInterrupt:
raise
except:
result.testsRun += 1
result.addError(self, self._exc_info())
return
saved_doc = self.testMethodDoc
try:
# Run all the tests specified
for testInfo in tests:
test,args = testInfo[0],testInfo[1:]
# Set the doc argument for this test. Note that even if we do
# this, the fail/error tracebacks still print the docstring for
# the parent factory, because they only generate the message at
# the end of the run, AFTER we've restored it. There is no way
# to tell the unittest system (without overriding a lot of
# stuff) to extract this information right away, the logic is
# hardcoded to pull it later, since unittest assumes it doesn't
# change.
self.testMethodDoc = test.__doc__
result.startTest(self)
ok = self.exec_test(test,args,result)
if ok: result.addSuccess(self)
finally:
# Restore docstring info and run tearDown once only.
self.testMethodDoc = saved_doc
try:
self.tearDown()
except KeyboardInterrupt:
raise
except:
result.addError(self, self._exc_info())
def run(self, result=None):
"""Test runner."""
#print
#print '*** run for method:',self._testMethodName # dbg
#print '*** doc:',self._testMethodDoc # dbg
if result is None: result = self.defaultTestResult()
# Independent tests: each gets its own setup/teardown
if self.testMethodName.startswith(self._indepParTestPrefix):
for t in getattr(self,self.testMethodName)():
self.run_test(t,result)
# Shared-state test: single setup/teardown for all
elif self.testMethodName.startswith(self._shareParTestPrefix):
tests = getattr(self,self.testMethodName,'runTest')()
self.run_tests(tests,result)
# Normal unittest Test methods
else:
unittest.TestCase.run(self,result)
#############################################################################
# Quick and dirty interactive example/test
if __name__ == '__main__':
class ExampleTestCase(ParametricTestCase):
#-------------------------------------------------------------------
# An instrumented setUp method so we can see when it gets called and
# how many times per instance
counter = 0
def setUp(self):
self.counter += 1
print 'setUp count: %2s for: %s' % (self.counter,
self.testMethodDoc)
#-------------------------------------------------------------------
# A standard test method, just like in the unittest docs.
def test_foo(self):
"""Normal test for feature foo."""
pass
#-------------------------------------------------------------------
# Testing methods that need parameters. These can NOT be named test*,
# since they would be picked up by unittest and called without
# arguments. Instead, call them anything else (I use tst*) and then
# load them via the factories below.
def tstX(self,i):
"Test feature X with parameters."
print 'tstX, i=',i
if i==1 or i==3:
# Test fails
self.fail('i is bad, bad: %s' % i)
def tstY(self,i):
"Test feature Y with parameters."
print 'tstY, i=',i
if i==1:
# Force an error
1/0
def tstXX(self,i,j):
"Test feature XX with parameters."
print 'tstXX, i=',i,'j=',j
if i==1:
# Test fails
self.fail('i is bad, bad: %s' % i)
def tstYY(self,i):
"Test feature YY with parameters."
print 'tstYY, i=',i
if i==2:
# Force an error
1/0
def tstZZ(self):
"""Test feature ZZ without parameters, needs multiple runs.
This could be a random test that you want to run multiple times."""
pass
#-------------------------------------------------------------------
# Parametric test factories that create the test groups to call the
# above tst* methods with their required arguments.
def testip(self):
"""Independent parametric test factory.
A separate setUp() call is made for each test returned by this
method.
You must return an iterable (list or generator is fine) containing
tuples with the actual method to be called as the first argument,
and the arguments for that call later."""
return [(self.tstX,i) for i in range(5)]
def testip2(self):
"""Another independent parametric test factory"""
return [(self.tstY,i) for i in range(5)]
def testip3(self):
"""Test factory combining different subtests.
This one shows how to assemble calls to different tests."""
return [(self.tstX,3),(self.tstX,9),(self.tstXX,4,10),
(self.tstZZ,),(self.tstZZ,)]
def testsp(self):
"""Shared parametric test factory
A single setUp() call is made for all the tests returned by this
method.
"""
return [(self.tstXX,i,i+1) for i in range(5)]
def testsp2(self):
"""Another shared parametric test factory"""
return [(self.tstYY,i) for i in range(5)]
def testsp3(self):
"""Another shared parametric test factory.
This one simply calls the same test multiple times, without any
arguments. Note that you must still return tuples, even if there
are no arguments."""
return [(self.tstZZ,) for i in range(10)]
# This test class runs normally under unittest's default runner
unittest.main()
|