Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 90:1fb319ec33af
Skimming mode added (-k/--skim option)
In skimming mode, as soon as a single test case within a test group
is failed, the remaining test cases in the same group are skipped.
Bug fix and simply a bit of refactoring: TestCase.has_iofiles and
TestCase.has_ansfile are now defined (the meaning should be clear
from the names).
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Mon, 28 Feb 2011 15:32:22 +0000 |
| parents | 3ae6cb69e4ef |
| children | c62c9bfd614a |
| rev | line source |
|---|---|
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 2 |
| 43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 4 | |
| 21 | 5 from __future__ import division, with_statement |
| 6 | |
| 7 try: | |
| 8 from compat import * | |
| 9 import files, problem, config | |
| 10 except ImportError: | |
| 11 import __main__ | |
| 12 __main__.import_error(sys.exc_info()[1]) | |
| 13 else: | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
14 from __main__ import options |
| 21 | 15 |
| 16 import glob, re, sys, tempfile, time | |
| 17 from subprocess import Popen, PIPE, STDOUT | |
| 18 | |
| 19 import os | |
| 20 devnull = open(os.path.devnull, 'w+') | |
| 21 | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
22 if options.autotime: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
23 # This is really a dirty hack that assumes that sleep() does not spend |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
24 # the CPU time of the current process and that if clock() measures |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
25 # wall-clock time, then it is more precise than time() is. Both these |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
26 # assumptions are true on all platforms I have tested this on so far, |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
27 # but I am not aware of any guarantee that they will both be true |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
28 # on every other platform. |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
29 c = time.clock() |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
30 time.sleep(1) |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
31 c = time.clock() - c |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
32 if int(c + .5) == 1: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
33 clock = time.clock |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
34 else: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
35 clock = time.time |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
36 |
| 21 | 37 try: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
38 from win32 import * |
|
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
39 except Exception: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
40 from unix import * |
| 22 | 41 |
| 21 | 42 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 43 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 44 'NonZeroExitCode', 'CannotStartTestee', | |
| 45 'CannotStartValidator', 'CannotReadOutputFile', | |
|
81
24752db487c5
Fixed errors in the win32 module
Oleg Oshmyan <chortos@inbox.lv>
parents:
79
diff
changeset
|
46 'CannotReadInputFile', 'CannotReadAnswerFile', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
47 'MemoryLimitExceeded', 'CPUTimeLimitExceeded', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
48 'WallTimeLimitExceeded') |
| 21 | 49 |
| 50 | |
| 51 | |
| 52 # Exceptions | |
| 53 | |
| 54 class TestCaseNotPassed(Exception): __slots__ = () | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
55 class TestCaseSkipped(TestCaseNotPassed): __slots__ = () |
| 21 | 56 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
57 class CPUTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
58 class WallTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
59 class MemoryLimitExceeded(TestCaseNotPassed): __slots__ = () |
| 22 | 60 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 61 |
| 62 class WrongAnswer(TestCaseNotPassed): | |
| 63 __slots__ = 'comment' | |
| 64 def __init__(self, comment=''): | |
| 65 self.comment = comment | |
| 66 | |
| 67 class NonZeroExitCode(TestCaseNotPassed): | |
| 68 __slots__ = 'exitcode' | |
| 69 def __init__(self, exitcode): | |
| 70 self.exitcode = exitcode | |
| 71 | |
| 72 class ExceptionWrapper(TestCaseNotPassed): | |
| 73 __slots__ = 'upstream' | |
| 74 def __init__(self, upstream): | |
| 75 self.upstream = upstream | |
| 76 | |
| 77 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 78 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 79 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 80 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 81 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 82 | |
| 83 | |
| 84 | |
| 22 | 85 # Helper context managers |
| 86 | |
| 87 class CopyDeleting(object): | |
| 88 __slots__ = 'case', 'file', 'name' | |
| 89 | |
| 90 def __init__(self, case, file, name): | |
| 91 self.case = case | |
| 92 self.file = file | |
| 93 self.name = name | |
| 94 | |
| 95 def __enter__(self): | |
| 96 if self.name: | |
| 97 try: | |
| 98 self.file.copy(self.name) | |
| 99 except: | |
| 100 try: | |
| 101 self.__exit__(None, None, None) | |
| 102 except: | |
| 103 pass | |
| 104 raise | |
| 105 | |
| 106 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 107 if self.name: | |
| 108 self.case.files_to_delete.append(self.name) | |
| 109 | |
| 110 | |
| 111 class Copying(object): | |
| 112 __slots__ = 'file', 'name' | |
| 113 | |
| 114 def __init__(self, file, name): | |
| 115 self.file = file | |
| 116 self.name = name | |
| 117 | |
| 118 def __enter__(self): | |
| 119 if self.name: | |
| 120 self.file.copy(self.name) | |
| 121 | |
| 122 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 123 pass | |
| 124 | |
| 125 | |
| 126 | |
| 21 | 127 # Test case types |
| 16 | 128 |
| 129 class TestCase(object): | |
| 21 | 130 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
131 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
132 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
133 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
134 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
135 'time_limit_string') |
| 21 | 136 |
| 137 if ABCMeta: | |
| 138 __metaclass__ = ABCMeta | |
| 16 | 139 |
| 21 | 140 def __init__(case, prob, id, isdummy, points): |
| 16 | 141 case.problem = prob |
| 21 | 142 case.id = id |
| 143 case.isdummy = isdummy | |
| 144 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
145 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
146 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 147 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
148 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
149 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 150 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
151 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
152 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
153 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
154 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
155 case.wall_time_limit_string = '' |
| 21 | 156 if not isdummy: |
| 157 case.realinname = case.problem.config.testcaseinname | |
| 158 case.realoutname = case.problem.config.testcaseoutname | |
| 159 else: | |
| 160 case.realinname = case.problem.config.dummyinname | |
| 161 case.realoutname = case.problem.config.dummyoutname | |
| 162 | |
| 163 @abstractmethod | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
164 def test(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
165 raise NotImplementedError |
| 16 | 166 |
| 22 | 167 def __call__(case, callback): |
| 168 case.has_called_back = False | |
| 169 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
170 case.time_limit_string = case.wall_time_limit_string |
| 21 | 171 try: |
| 22 | 172 return case.test(callback) |
| 21 | 173 finally: |
| 22 | 174 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
175 if getattr(case, 'time_started', None) is None: |
| 22 | 176 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
177 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 178 case.time_stopped = now |
| 179 if not case.has_called_back: | |
| 180 callback() | |
| 21 | 181 case.cleanup() |
| 182 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
183 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
184 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
185 return False |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
186 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
187 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
188 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
189 return False |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
190 |
| 21 | 191 def cleanup(case): |
| 192 #if getattr(case, 'infile', None): | |
| 193 # case.infile.close() | |
| 194 #if getattr(case, 'outfile', None): | |
| 195 # case.outfile.close() | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
196 if getattr(case, 'process', None) and case.process.returncode is None: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
197 # Try KILLing after three unsuccessful TERM attempts in a row |
| 21 | 198 for i in range(3): |
| 199 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
200 terminate(case.process) |
| 21 | 201 except Exception: |
| 202 time.sleep(0) | |
| 203 case.process.poll() | |
| 204 else: | |
| 22 | 205 case.process.wait() |
| 21 | 206 break |
| 207 else: | |
| 208 # If killing the process is unsuccessful three times in a row, | |
| 209 # just silently stop trying | |
| 210 for i in range(3): | |
| 211 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
212 kill(case.process) |
| 21 | 213 except Exception: |
| 214 time.sleep(0) | |
| 215 case.process.poll() | |
| 216 else: | |
| 22 | 217 case.process.wait() |
| 21 | 218 break |
| 22 | 219 if case.files_to_delete: |
| 220 for name in case.files_to_delete: | |
| 221 try: | |
| 222 os.remove(name) | |
| 223 except Exception: | |
| 224 # It can't be helped | |
| 225 pass | |
| 21 | 226 |
| 227 def open_infile(case): | |
| 228 try: | |
| 229 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 230 except IOError: | |
| 231 e = sys.exc_info()[1] | |
| 232 raise CannotReadInputFile(e) | |
| 233 | |
| 234 def open_outfile(case): | |
| 235 try: | |
| 236 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 237 except IOError: | |
| 238 e = sys.exc_info()[1] | |
| 239 raise CannotReadAnswerFile(e) | |
| 240 | |
| 16 | 241 |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
242 class SkippedTestCase(TestCase): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
243 __slots__ = () |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
244 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
245 def test(case, callback): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
246 raise TestCaseSkipped |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
247 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
248 |
| 21 | 249 class ValidatedTestCase(TestCase): |
| 250 __slots__ = 'validator' | |
| 251 | |
| 252 def __init__(case, *args): | |
| 253 TestCase.__init__(case, *args) | |
| 254 if not case.problem.config.tester: | |
| 255 case.validator = None | |
| 256 else: | |
| 257 case.validator = case.problem.config.tester | |
| 258 | |
| 259 def validate(case, output): | |
| 260 if not case.validator: | |
| 261 # Compare the output with the reference output | |
| 262 case.open_outfile() | |
| 263 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
264 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
265 if refline is not None and not isinstance(refline, basestring): |
| 21 | 266 line = bytes(line, sys.getdefaultencoding()) |
| 267 if line != refline: | |
| 22 | 268 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
269 return 1 |
| 21 | 270 elif callable(case.validator): |
| 271 return case.validator(output) | |
| 272 else: | |
| 273 # Call the validator program | |
| 274 output.close() | |
| 23 | 275 if case.problem.config.ansname: |
| 276 case.open_outfile() | |
| 277 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
278 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
279 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
280 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
281 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 282 comment = case.process.communicate()[0].strip() |
| 26 | 283 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 284 if match: |
| 285 comment = comment[match.end():] | |
| 286 if not case.problem.config.maxexitcode: | |
| 287 if case.process.returncode: | |
| 288 raise WrongAnswer(comment) | |
| 289 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
290 return 1, comment |
| 21 | 291 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
292 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 293 |
| 294 | |
| 295 class BatchTestCase(ValidatedTestCase): | |
| 296 __slots__ = () | |
| 297 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
298 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
299 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
300 return (not case.problem.config.stdio or |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
301 case.validator and not callable(case.validator)) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
302 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
303 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
304 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
305 return case.validator and not callable(case.validator) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
306 |
| 22 | 307 def test(case, callback): |
| 21 | 308 case.open_infile() |
| 309 if case.problem.config.stdio: | |
| 54 | 310 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 311 # TODO: re-use the same file name if possible |
| 21 | 312 # FIXME: 2.5 lacks the delete parameter |
| 313 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 314 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
315 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 316 else: |
| 317 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
318 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
319 with contextmgr: |
|
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
320 with open(inputdatafname) as infile: |
| 83 | 321 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
322 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
323 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 22 | 324 raise NonZeroExitCode(case.process.returncode) |
| 325 callback() | |
| 326 case.has_called_back = True | |
| 327 outfile.seek(0) | |
| 328 return case.validate(outfile) | |
| 21 | 329 else: |
| 22 | 330 case.infile.copy(case.problem.config.inname) |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
331 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
332 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 333 raise NonZeroExitCode(case.process.returncode) |
| 22 | 334 callback() |
| 335 case.has_called_back = True | |
| 21 | 336 with open(case.problem.config.outname, 'rU') as output: |
| 337 return case.validate(output) | |
| 338 | |
| 339 | |
| 340 # This is the only test case type not executing any programs to be tested | |
| 341 class OutputOnlyTestCase(ValidatedTestCase): | |
| 342 __slots__ = () | |
| 343 def cleanup(case): pass | |
| 344 | |
| 345 class BestOutputTestCase(ValidatedTestCase): | |
| 346 __slots__ = () | |
| 347 | |
| 348 # This is the only test case type executing two programs simultaneously | |
| 349 class ReactiveTestCase(TestCase): | |
| 350 __slots__ = () | |
| 351 # The basic idea is to launch the program to be tested and the grader | |
| 352 # and to pipe their standard I/O from and to each other, | |
| 353 # and then to capture the grader's exit code and use it | |
| 26 | 354 # like the exit code of an output validator is used. |
| 21 | 355 |
| 356 | |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
357 class DummyTestContext(problem.TestGroup): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
358 __slots__ = () |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
359 def end(self): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
360 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
361 return 0, 0, self.log |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
362 |
| 21 | 363 def load_problem(prob, _types={'batch' : BatchTestCase, |
| 364 'outonly' : OutputOnlyTestCase, | |
| 365 'bestout' : BestOutputTestCase, | |
| 366 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
367 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
368 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
369 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
370 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
371 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
372 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
373 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
374 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
375 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
376 |
| 23 | 377 if options.legacy: |
| 378 prob.config.usegroups = False | |
| 58 | 379 newtests = [] |
| 23 | 380 for i, name in enumerate(prob.config.tests): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
381 # Same here; we'll need to iterate over them twice |
| 23 | 382 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
383 l = len(name) |
| 23 | 384 except Exception: |
| 385 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
386 name = tuple(name) |
| 23 | 387 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
388 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
389 l = len(name) |
| 58 | 390 if l > 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
391 prob.config.usegroups = True |
| 58 | 392 newtests.append(name) |
| 393 if prob.config.usegroups: | |
| 394 prob.config.tests = newtests | |
| 395 del newtests | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
396 |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
397 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
398 if isinstance(prob.config.pointmap, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
399 def getpoints(i, j, k=None): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
400 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
401 return prob.config.pointmap[i] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
402 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
403 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
404 return prob.config.pointmap[None] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
405 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
406 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
407 elif prob.config.usegroups: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
408 def getpoints(i, j, k): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
409 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
410 return prob.config.pointmap[k][j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
411 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
412 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
413 else: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
414 def getpoints(i, j): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
415 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
416 return prob.config.pointmap[j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
417 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
418 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
419 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
420 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
421 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
422 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
423 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
424 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 425 if prob.config.usegroups: |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
426 if not isinstance(prob.config.groupweight, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
427 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
428 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
429 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
430 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
431 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
89
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
432 if prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
433 yield DummyTestContext() |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
434 for i in prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
435 s = str(i).zfill(prob.config.paddummies) |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
436 yield _types[prob.config.kind](prob, s, True, 0) |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
437 yield problem.test_context_end |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
438 for k, group in enumerate(prob.config.tests): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
439 if not group: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
440 continue |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
441 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
442 case_type = _types[prob.config.kind] |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
443 for j, i in enumerate(group): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
444 s = str(i).zfill(prob.config.padtests) |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
445 if not (yield case_type(prob, s, False, getpoints(i, j, k))): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
446 if options.skim: |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
447 case_type = SkippedTestCase |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
448 else: |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
449 yield |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
450 yield problem.test_context_end |
| 16 | 451 else: |
| 452 for i in prob.config.tests: | |
| 21 | 453 s = str(i).zfill(prob.config.padtests) |
| 454 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 455 for i in prob.config.dummies: | |
| 456 s = str(i).zfill(prob.config.paddummies) | |
| 457 yield _types[prob.config.kind](prob, s, True, 0) | |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
458 for j, i in enumerate(prob.config.tests): |
| 21 | 459 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
460 yield _types[prob.config.kind](prob, s, False, getpoints(i, j)) |