unittest.py patch: add skipped test functionality

[Remy Blank]

Ok, here we go.

I added the possibility for tests using the unittest.py framework
to be skipped. Basically, I added two methods to TestCase:

TestCase.skip(msg): skips unconditionally
TestCase.skipIf(expr, msg): skips if expr is true

These can be called either in setUp() or in the test methods. I also
added reporting of skipped tests to TestResult, _TextTestResult and
TextTestRunner. If no tests are skipped, everything should be the
same as before.

I am using Python 2.3.3, so the changes are against the file in that
version. I attached the patch against the original
(unittest_skip.patch), a complete test suite for the new functionality
(testSkippedTest.py) and a usage example (SkippedTestDemo.py).

I would welcome any feedback about the idea, the implementation,
suggestions for improvements, fixes, etc.

I would also like to know if this would be a candidate for inclusion
into the unittest.py provided with Python. I suppose I should
contact Steve Purcell directly for that.

-- Remy



Quick usage example:

class ReadShadowTest(unittest.TestCase):
"""Read access to /etc/shadow"""
def testReadingAsRoot(self):
"""Reading /etc/shadow as root"""
self.skipIf(os.geteuid() != 0, "Must be root")
open("/etc/shadow").close()


The attached example (SkippedTestDemo.py) produces the following output:

$ ./SkippedTestDemo.py -v
Access to autoexec.bat ... SKIPPED (Only available on Windows)
Access to config.sys ... SKIPPED (Only available on Windows)
Reading /etc/shadow as root ... SKIPPED (Must be root)
Reading /etc/shadow as non-root ... ok

----------------------------------------------------------------------
Ran 4 tests in 0.004s

OK (skipped=3)

--- unittest.py.orig 2004-09-24 10:33:21.933370804 +0200
+++ unittest.py 2004-09-24 02:21:25.000000000 +0200
@@ -75,6 +75,8 @@
def _strclass(cls):
return "%s.%s" % (cls.__module__, cls.__name__)

+class SkipException(Exception): pass
+
class TestResult:
"""Holder for test result information.

@@ -89,6 +91,7 @@
def __init__(self):
self.failures = []
self.errors = []
+ self.skipped = []
self.testsRun = 0
self.shouldStop = 0

@@ -111,6 +114,11 @@
returned by sys.exc_info()."""
self.failures.append((test, self._exc_info_to_string(err)))

+ def addSkipped(self, test, err):
+ """Called when a test is skipped. 'err' is a tuble of values as
+ returned by sys.exc_info()."""
+ self.skipped.append((test, str(err[1])))
+
def addSuccess(self, test):
"Called when a test has completed successfully"
pass
@@ -128,9 +136,9 @@
return string.join(traceback.format_exception(*err), '')

def __repr__(self):
- return "<%s run=%i errors=%i failures=%i>" % \
+ return "<%s run=%i errors=%i failures=%i skipped=%i>" % \
(_strclass(self.__class__), self.testsRun, len(self.errors),
- len(self.failures))
+ len(self.failures), len(self.skipped))


class TestCase:
@@ -218,6 +226,9 @@
try:
try:
self.setUp()
+ except SkipException:
+ result.addSkipped(self, self.__exc_info())
+ return
except KeyboardInterrupt:
raise
except:
@@ -230,6 +241,8 @@
ok = 1
except self.failureException:
result.addFailure(self, self.__exc_info())
+ except SkipException:
+ result.addSkipped(self, self.__exc_info())
except KeyboardInterrupt:
raise
except:
@@ -346,6 +359,13 @@

assert_ = failUnless

+ def skip(self, msg=None):
+ """Skip the test, with the given message."""
+ raise SkipException, msg
+
+ def skipIf(self, expr, msg=None):
+ """Skip the test if the expression is true."""
+ if expr: raise SkipException, msg


class TestSuite:
@@ -623,6 +643,16 @@
elif self.dots:
self.stream.write('F')

+ def addSkipped(self, test, err):
+ TestResult.addSkipped(self, test, err)
+ if self.showAll:
+ msg = str(err[1])
+ if msg:
+ msg = " (" + msg + ")"
+ self.stream.writeln("SKIPPED" + msg)
+ elif self.dots:
+ self.stream.write('S')
+
def printErrors(self):
if self.dots or self.showAll:
self.stream.writeln()
@@ -666,15 +696,20 @@
self.stream.writeln()
if not result.wasSuccessful():
self.stream.write("FAILED (")
- failed, errored = map(len, (result.failures, result.errors))
+ failed, errored, skipped = map(len, (result.failures, result.errors, result.skipped))
if failed:
self.stream.write("failures=%d" % failed)
if errored:
if failed: self.stream.write(", ")
self.stream.write("errors=%d" % errored)
+ if skipped:
+ self.stream.write(", skipped=%d" % skipped)
self.stream.writeln(")")
else:
- self.stream.writeln("OK")
+ if result.skipped:
+ self.stream.writeln("OK (skipped=%d)" % len(result.skipped))
+ else:
+ self.stream.writeln("OK")
return result



#!/usr/bin/env python
"""\
testSkippedTest.py Test cases for adding skipped tests to unittest.py
Copyright (C) 2004 Remy Blank
"""

import unittest
import sys
from cStringIO import StringIO

# Dummy test cases
class SkipInTest(unittest.TestCase):
"""Tests skip in test methods"""
def testDoesntSkip(self):
"""Doesn't skip"""
self.assert_(True)

def testFails(self):
"""Fails"""
self.fail()

def testSkips(self):
"""Test skips"""
self.skip()

def testSkipsMessage(self):
"""Test skips with a message"""
self.skip("A message")


class SkipInSetUp(unittest.TestCase):
"""Tests skip in setUp()"""
def setUp(self):
super(SkipInSetUp, self).setUp()
self.skip()

def testDummy(self):
"""Never executed"""


# Test cases
class TestResultTest(unittest.TestCase):
"""Tests relating to TestResult"""
def testSkippedAttribute(self):
"""List of skipped tests"""
result = unittest.TestResult()
self.assertEqual([], result.skipped)
self.assertEqual("<unittest.TestResult run=0 errors=0 failures=0 skipped=0>", repr(result))

def testAddSkipped(self):
"""TestResult.addSkipped()"""
result = unittest.TestResult()
test = SkipInTest("testDoesntSkip")
try:
raise unittest.SkipException, "Skipped"
except:
result.addSkipped(test, sys.exc_info())

self.assert_(test is result.skipped[0][0])
self.assertEqual("Skipped", result.skipped[0][1])
self.assertEqual("<unittest.TestResult run=0 errors=0 failures=0 skipped=1>", repr(result))


class TestCaseTest(unittest.TestCase):
"""Tests relating to TestCase"""
def setUp(self):
super(TestCaseTest, self).setUp()
self.result = unittest.TestResult()

def testNoSkip(self):
"""Test case that doesn't skip"""
test = SkipInTest("testDoesntSkip")
test(self.result)

self.assertEqual(1, self.result.testsRun)
self.assertEqual([], self.result.skipped)

def testSkipInTest(self):
"""Test case that skips in test method"""
test = SkipInTest("testSkips")
test(self.result)

self.assertEqual(1, self.result.testsRun)
self.assertEqual([], self.result.failures)
self.assertEqual([], self.result.errors)
self.assert_(test is self.result.skipped[0][0])

def testSkipInSetUp(self):
"""Test case that skips in setUp()"""
test = SkipInSetUp("testDummy")
test(self.result)

self.assertEqual(1, self.result.testsRun)
self.assertEqual([], self.result.failures)
self.assertEqual([], self.result.errors)
self.assert_(test is self.result.skipped[0][0])


class SkipMethodsTest(unittest.TestCase):
"""Tests for skip() methods"""
def setUp(self):
super(SkipMethodsTest, self).setUp()
self.test = SkipInTest("testDoesntSkip")

def catchSkipException(self, callable, *args, **kwargs):
try:
callable(*args, **kwargs)
except unittest.SkipException, e:
return e
return None

def testSkip(self):
"""skip()"""
e = self.catchSkipException(self.test.skip)
self.assertNotEqual(None, e)

def testSkipMessage(self):
"""skip(message)"""
e = self.catchSkipException(self.test.skip, "A message")
self.assertNotEqual(None, e)
self.assertEqual("A message", str(e))

def testSkipIfFalse(self):
"""skipIf(False)"""
e = self.catchSkipException(self.test.skipIf, False)
self.assertEqual(None, e)

def testSkipIfTrue(self):
"""skipIf(True)"""
e = self.catchSkipException(self.test.skipIf, True)
self.assertNotEqual(None, e)

def testSkipIfTrueMessage(self):
"""skipIf(True, message)"""
e = self.catchSkipException(self.test.skipIf, True, "Another message")
self.assertNotEqual(None, e)
self.assertEqual("Another message", str(e))


class TextTestResultTest(unittest.TestCase):
"""Tests for _TextTestResult"""
def setUp(self):
super(TextTestResultTest, self).setUp()
self.out = unittest._WritelnDecorator(StringIO())

def testNoSkippingQuiet(self):
"""No skipping, verbosity=1"""
result = unittest._TextTestResult(self.out, True, 1)
test = SkipInTest("testDoesntSkip")
test(result)

self.assertEqual(".", self.out.getvalue())

def testNoSkippingVerbose(self):
"""No skipping, verbosity=2"""
result = unittest._TextTestResult(self.out, True, 2)
test = SkipInTest("testDoesntSkip")
test(result)

self.assertEqual("Doesn't skip ... ok\n", self.out.getvalue())

def testSkippingQuiet(self):
"""Skipping, verbosity=1"""
result = unittest._TextTestResult(self.out, True, 1)
test = SkipInTest("testSkips")
test(result)

self.assertEqual("S", self.out.getvalue())

def testSkippingVerbose(self):
"""Skipping, verbosity=2"""
result = unittest._TextTestResult(self.out, True, 2)
test = SkipInTest("testSkips")
test(result)

self.assertEqual("Test skips ... SKIPPED\n", self.out.getvalue())

def testSkippingMessageVerbose(self):
"""Skipping with a message, verbosity=2"""
result = unittest._TextTestResult(self.out, True, 2)
test = SkipInTest("testSkipsMessage")
test(result)

self.assertEqual("Test skips with a message ... SKIPPED (A message)\n", self.out.getvalue())


class TextTestRunnerTest(unittest.TestCase):
"""Tests for TextTestRunner"""
def setUp(self):
super(TextTestRunnerTest, self).setUp()
self.out = unittest._WritelnDecorator(StringIO())

def testOkNoSkips(self):
"""All tests OK, no skips"""
runner = unittest.TextTestRunner(self.out, True)
test = SkipInTest("testDoesntSkip")
runner.run(test)

self.assert_(self.out.getvalue().endswith("OK\n"))

def testOkSkip(self):
"""All tests OK, one skip"""
runner = unittest.TextTestRunner(self.out, True)
test = SkipInTest("testSkips")
runner.run(test)

self.assert_(self.out.getvalue().endswith("OK (skipped=1)\n"))

def testFailedNoSkip(self):
"""Failed test, no skips"""
runner = unittest.TextTestRunner(self.out, True)
test = SkipInTest("testFails")
runner.run(test)

self.assert_(self.out.getvalue().endswith("FAILED (failures=1)\n"))

def testFailedSkip(self):
"""Failed test, one skip"""
runner = unittest.TextTestRunner(self.out, True)
test = unittest.TestSuite((SkipInTest("testFails"), SkipInTest("testSkips")))
runner.run(test)

self.assert_(self.out.getvalue().endswith("FAILED (failures=1, skipped=1)\n"))


if __name__ == "__main__":
unittest.TestProgram(argv=sys.argv + ["TestResultTest", "TestCaseTest",
"SkipMethodsTest", "TextTestResultTest", "TextTestRunnerTest"])

#!/usr/bin/env python
"""\
SkippedTestDemo.py Demo usage for skipped tests in unittest.py
Copyright (C) 2004 Remy Blank
"""

import unittest
import os


class ReadShadowTest(unittest.TestCase):
"""Read access to /etc/shadow"""
def testReadingAsRoot(self):
"""Reading /etc/shadow as root"""
self.skipIf(os.geteuid() != 0, "Must be root")
open("/etc/shadow").close()

def testReadingAsUser(self):
"""Reading /etc/shadow as non-root"""
isRoot = os.geteuid() == 0
if isRoot: # Can run this test even as root
os.seteuid(500)
self.assertRaises(IOError, open, "/etc/shadow")
if isRoot:
os.seteuid(0)


class BootFileTest(unittest.TestCase):
"""Boot file access"""
def setUp(self):
super(BootFileTest, self).setUp()
self.skipIf(os.name != "nt", "Only available on Windows")

def testConfigSys(self):
"""Access to config.sys"""
open("c:\\config.sys").close()

def testAutoexecBat(self):
"""Access to autoexec.bat"""
open("c:\\autoexec.bat").close()


if __name__ == "__main__":
unittest.main()