summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2008-07-24 21:42:27 -0500
committerDan McGee <dan@archlinux.org>2008-07-24 22:16:29 -0500
commit9f57921467ea368be96ffc09c8c35e9acfcc41ed (patch)
tree4e043260b7d26f13d99c210c89d562a14011add9
parent7bc34ccde3b8860a4aaf8418996a72e8cc4dfae5 (diff)
pactest: add test.expectfailure option for tests
This will enable us to mark tests we know currently fail to differentiate them from those that we know should pass. Regressions should be easier to spot this way. Signed-off-by: Dan McGee <dan@archlinux.org>
-rwxr-xr-xpactest/pmenv.py31
-rwxr-xr-xpactest/pmtest.py1
-rw-r--r--pactest/tests/fileconflict001.py2
-rw-r--r--pactest/tests/fileconflict002.py2
-rw-r--r--pactest/tests/fileconflict004.py2
-rw-r--r--pactest/tests/sync403.py2
-rw-r--r--pactest/tests/trans001.py2
-rw-r--r--pactest/tests/upgrade046.py2
-rw-r--r--pactest/tests/upgrade051.py2
9 files changed, 41 insertions, 5 deletions
diff --git a/pactest/pmenv.py b/pactest/pmenv.py
index f2327f91..b86abe94 100755
--- a/pactest/pmenv.py
+++ b/pactest/pmenv.py
@@ -86,13 +86,25 @@ class pmenv:
"""
passed = 0
tpassed = []
+ failed = 0
tfailed = []
+ expectedfail = 0
+ texpectedfail = []
+ unexpectedpass = 0
+ tunexpectedpass = []
for test in self.testcases:
fail = test.result["fail"]
- if fail == 0:
+ if fail == 0 and not test.expectfailure:
passed += 1
tpassed.append(test)
+ elif fail != 0 and test.expectfailure:
+ expectedfail += 1
+ texpectedfail.append(test)
+ elif fail == 0: # and not test.expectfail
+ unexpectedpass += 1
+ tunexpectedpass.append(test)
else:
+ failed += 1
tfailed.append(test)
def _printtest(t):
@@ -114,17 +126,26 @@ class pmenv:
print "=========="*8
print "Results"
print "----------"*8
+ print " Passed:"
for test in tpassed: _printtest(test)
print "----------"*8
+ print " Expected Failures:"
+ for test in texpectedfail: _printtest(test)
+ print "----------"*8
+ print " Unexpected Passes:"
+ for test in tunexpectedpass: _printtest(test)
+ print "----------"*8
+ print " Failed:"
for test in tfailed: _printtest(test)
print "----------"*8
total = len(self.testcases)
- failed = total - passed
- print "TOTAL = %3u" % total
+ print "Total = %3u" % total
if total:
- print "PASS = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
- print "FAIL = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
+ print "Pass = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
+ print "Expected Fail = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total)
+ print "Unexpected Pass = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total)
+ print "Fail = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
print ""
if __name__ == "__main__":
diff --git a/pactest/pmtest.py b/pactest/pmtest.py
index e8f6fa8d..f31563b1 100755
--- a/pactest/pmtest.py
+++ b/pactest/pmtest.py
@@ -90,6 +90,7 @@ class pmtest:
# Test rules
self.rules = []
self.files = []
+ self.expectfailure = False
if os.path.isfile(self.name):
execfile(self.name)
diff --git a/pactest/tests/fileconflict001.py b/pactest/tests/fileconflict001.py
index 4c2069ee..8c13911c 100644
--- a/pactest/tests/fileconflict001.py
+++ b/pactest/tests/fileconflict001.py
@@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
self.addrule("PACMAN_RETCODE=1")
self.addrule("!PKG_EXIST=pkg1")
self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict002.py b/pactest/tests/fileconflict002.py
index c54f6daf..f70873ba 100644
--- a/pactest/tests/fileconflict002.py
+++ b/pactest/tests/fileconflict002.py
@@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
self.addrule("PACMAN_RETCODE=1")
self.addrule("!PKG_EXIST=pkg1")
self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict004.py b/pactest/tests/fileconflict004.py
index a5347ccd..2396cedb 100644
--- a/pactest/tests/fileconflict004.py
+++ b/pactest/tests/fileconflict004.py
@@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0")
self.addrule("PKG_EXIST=pkg1")
self.addrule("PKG_VERSION=pkg1|2.0-1")
self.addrule("FILE_TYPE=test|link")
+
+self.expectfailure = True
diff --git a/pactest/tests/sync403.py b/pactest/tests/sync403.py
index b8d01016..d8ab763c 100644
--- a/pactest/tests/sync403.py
+++ b/pactest/tests/sync403.py
@@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0")
self.addrule("PKG_EXIST=pkg1")
self.addrule("!PKG_EXIST=pkg2")
self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/trans001.py b/pactest/tests/trans001.py
index b9889b14..b3d7883b 100644
--- a/pactest/tests/trans001.py
+++ b/pactest/tests/trans001.py
@@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1")
self.addrule("!PKG_EXIST=pkg1")
self.addrule("PKG_EXIST=pkg2")
self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade046.py b/pactest/tests/upgrade046.py
index 60164b72..12390647 100644
--- a/pactest/tests/upgrade046.py
+++ b/pactest/tests/upgrade046.py
@@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy")
self.addrule("FILE_MODIFIED=bin/foobar")
self.addrule("FILE_EXIST=usr/share/file")
self.addrule("FILE_MODIFIED=usr/share/file")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade051.py b/pactest/tests/upgrade051.py
index e8c69eb9..4346c24d 100644
--- a/pactest/tests/upgrade051.py
+++ b/pactest/tests/upgrade051.py
@@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename()
self.addrule("PACMAN_RETCODE=0")
self.addrule("!PKG_EXIST=pkg1")
self.addrule("PKG_EXIST=pkg2")
+
+self.expectfailure = True