From 9f57921467ea368be96ffc09c8c35e9acfcc41ed Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 24 Jul 2008 21:42:27 -0500 Subject: pactest: add test.expectfailure option for tests This will enable us to mark tests we know currently fail to differentiate them from those that we know should pass. Regressions should be easier to spot this way. Signed-off-by: Dan McGee --- pactest/pmenv.py | 31 ++++++++++++++++++++++++++----- pactest/pmtest.py | 1 + pactest/tests/fileconflict001.py | 2 ++ pactest/tests/fileconflict002.py | 2 ++ pactest/tests/fileconflict004.py | 2 ++ pactest/tests/sync403.py | 2 ++ pactest/tests/trans001.py | 2 ++ pactest/tests/upgrade046.py | 2 ++ pactest/tests/upgrade051.py | 2 ++ 9 files changed, 41 insertions(+), 5 deletions(-) diff --git a/pactest/pmenv.py b/pactest/pmenv.py index f2327f91..b86abe94 100755 --- a/pactest/pmenv.py +++ b/pactest/pmenv.py @@ -86,13 +86,25 @@ class pmenv: """ passed = 0 tpassed = [] + failed = 0 tfailed = [] + expectedfail = 0 + texpectedfail = [] + unexpectedpass = 0 + tunexpectedpass = [] for test in self.testcases: fail = test.result["fail"] - if fail == 0: + if fail == 0 and not test.expectfailure: passed += 1 tpassed.append(test) + elif fail != 0 and test.expectfailure: + expectedfail += 1 + texpectedfail.append(test) + elif fail == 0: # and not test.expectfail + unexpectedpass += 1 + tunexpectedpass.append(test) else: + failed += 1 tfailed.append(test) def _printtest(t): @@ -114,17 +126,26 @@ class pmenv: print "=========="*8 print "Results" print "----------"*8 + print " Passed:" for test in tpassed: _printtest(test) print "----------"*8 + print " Expected Failures:" + for test in texpectedfail: _printtest(test) + print "----------"*8 + print " Unexpected Passes:" + for test in tunexpectedpass: _printtest(test) + print "----------"*8 + print " Failed:" for test in tfailed: _printtest(test) print "----------"*8 total = len(self.testcases) - failed = total - passed - print "TOTAL = %3u" % total + print "Total = %3u" % total if total: - print "PASS = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total) - print "FAIL = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total) + print "Pass = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total) + print "Expected Fail = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total) + print "Unexpected Pass = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total) + print "Fail = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total) print "" if __name__ == "__main__": diff --git a/pactest/pmtest.py b/pactest/pmtest.py index e8f6fa8d..f31563b1 100755 --- a/pactest/pmtest.py +++ b/pactest/pmtest.py @@ -90,6 +90,7 @@ class pmtest: # Test rules self.rules = [] self.files = [] + self.expectfailure = False if os.path.isfile(self.name): execfile(self.name) diff --git a/pactest/tests/fileconflict001.py b/pactest/tests/fileconflict001.py index 4c2069ee..8c13911c 100644 --- a/pactest/tests/fileconflict001.py +++ b/pactest/tests/fileconflict001.py @@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2]) self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") + +self.expectfailure = True diff --git a/pactest/tests/fileconflict002.py b/pactest/tests/fileconflict002.py index c54f6daf..f70873ba 100644 --- a/pactest/tests/fileconflict002.py +++ b/pactest/tests/fileconflict002.py @@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2]) self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") + +self.expectfailure = True diff --git a/pactest/tests/fileconflict004.py b/pactest/tests/fileconflict004.py index a5347ccd..2396cedb 100644 --- a/pactest/tests/fileconflict004.py +++ b/pactest/tests/fileconflict004.py @@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0") self.addrule("PKG_EXIST=pkg1") self.addrule("PKG_VERSION=pkg1|2.0-1") self.addrule("FILE_TYPE=test|link") + +self.expectfailure = True diff --git a/pactest/tests/sync403.py b/pactest/tests/sync403.py index b8d01016..d8ab763c 100644 --- a/pactest/tests/sync403.py +++ b/pactest/tests/sync403.py @@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0") self.addrule("PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") self.addrule("PKG_EXIST=pkg3") + +self.expectfailure = True diff --git a/pactest/tests/trans001.py b/pactest/tests/trans001.py index b9889b14..b3d7883b 100644 --- a/pactest/tests/trans001.py +++ b/pactest/tests/trans001.py @@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("PKG_EXIST=pkg2") self.addrule("PKG_EXIST=pkg3") + +self.expectfailure = True diff --git a/pactest/tests/upgrade046.py b/pactest/tests/upgrade046.py index 60164b72..12390647 100644 --- a/pactest/tests/upgrade046.py +++ b/pactest/tests/upgrade046.py @@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy") self.addrule("FILE_MODIFIED=bin/foobar") self.addrule("FILE_EXIST=usr/share/file") self.addrule("FILE_MODIFIED=usr/share/file") + +self.expectfailure = True diff --git a/pactest/tests/upgrade051.py b/pactest/tests/upgrade051.py index e8c69eb9..4346c24d 100644 --- a/pactest/tests/upgrade051.py +++ b/pactest/tests/upgrade051.py @@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename() self.addrule("PACMAN_RETCODE=0") self.addrule("!PKG_EXIST=pkg1") self.addrule("PKG_EXIST=pkg2") + +self.expectfailure = True -- cgit v1.2.3-70-g09d2