Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 82:06356af50bf9
Finished testcases reorganization and CPU time limit implementation
We now have:
* Win32-specific code in the win32 module (including bug fixes),
* UNIX-specific and generic code in the unix module,
* a much cleaner testcases module,
* wait4-based resource limits working on Python 3 (this is a bug fix),
* no warning/error reported on non-Win32 when -x is not passed
but standard input does not come from a terminal,
* the maxtime configuration variable replaced with two new variables
named maxcputime and maxwalltime,
* CPU time reported if it can be determined unless an error occurs sooner
than it is determined (e. g. if the wall-clock time limit is exceeded),
* memory limits enforced even if Upreckon's forking already breaks them,
* CPU time limits and private virtual memory limits honoured on Win32,
* CPU time limits honoured on UNIX(-like) platforms supporting wait4
or getrusage,
* address space limits honoured on UNIX(-like) platforms supporting
setrlimit with RLIMIT_AS/RLIMIT_VMEM,
* resident set size limits honoured on UNIX(-like) platforms supporting
wait4.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Wed, 23 Feb 2011 23:35:27 +0000 |
| parents | 24752db487c5 |
| children | 37c4ad87583c |
| rev | line source |
|---|---|
| 21 | 1 #! /usr/bin/env python |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
2 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 3 |
| 43 | 4 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 5 | |
| 21 | 6 from __future__ import division, with_statement |
| 7 | |
| 8 try: | |
| 9 from compat import * | |
| 10 import files, problem, config | |
| 11 except ImportError: | |
| 12 import __main__ | |
| 13 __main__.import_error(sys.exc_info()[1]) | |
| 14 else: | |
| 15 from __main__ import clock, options | |
| 16 | |
| 17 import glob, re, sys, tempfile, time | |
| 18 from subprocess import Popen, PIPE, STDOUT | |
| 19 | |
| 20 import os | |
| 21 devnull = open(os.path.devnull, 'w+') | |
| 22 | |
| 23 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
24 from win32 import * |
|
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
25 except Exception: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
26 from unix import * |
| 22 | 27 |
| 21 | 28 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 29 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 30 'NonZeroExitCode', 'CannotStartTestee', | |
| 31 'CannotStartValidator', 'CannotReadOutputFile', | |
|
81
24752db487c5
Fixed errors in the win32 module
Oleg Oshmyan <chortos@inbox.lv>
parents:
79
diff
changeset
|
32 'CannotReadInputFile', 'CannotReadAnswerFile', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
33 'MemoryLimitExceeded', 'CPUTimeLimitExceeded', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
34 'WallTimeLimitExceeded') |
| 21 | 35 |
| 36 | |
| 37 | |
| 38 # Exceptions | |
| 39 | |
| 40 class TestCaseNotPassed(Exception): __slots__ = () | |
| 41 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
42 class CPUTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
43 class WallTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
44 class MemoryLimitExceeded(TestCaseNotPassed): __slots__ = () |
| 22 | 45 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 46 |
| 47 class WrongAnswer(TestCaseNotPassed): | |
| 48 __slots__ = 'comment' | |
| 49 def __init__(self, comment=''): | |
| 50 self.comment = comment | |
| 51 | |
| 52 class NonZeroExitCode(TestCaseNotPassed): | |
| 53 __slots__ = 'exitcode' | |
| 54 def __init__(self, exitcode): | |
| 55 self.exitcode = exitcode | |
| 56 | |
| 57 class ExceptionWrapper(TestCaseNotPassed): | |
| 58 __slots__ = 'upstream' | |
| 59 def __init__(self, upstream): | |
| 60 self.upstream = upstream | |
| 61 | |
| 62 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 63 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 64 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 65 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 66 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 67 | |
| 68 | |
| 69 | |
| 22 | 70 # Helper context managers |
| 71 | |
| 72 class CopyDeleting(object): | |
| 73 __slots__ = 'case', 'file', 'name' | |
| 74 | |
| 75 def __init__(self, case, file, name): | |
| 76 self.case = case | |
| 77 self.file = file | |
| 78 self.name = name | |
| 79 | |
| 80 def __enter__(self): | |
| 81 if self.name: | |
| 82 try: | |
| 83 self.file.copy(self.name) | |
| 84 except: | |
| 85 try: | |
| 86 self.__exit__(None, None, None) | |
| 87 except: | |
| 88 pass | |
| 89 raise | |
| 90 | |
| 91 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 92 if self.name: | |
| 93 self.case.files_to_delete.append(self.name) | |
| 94 | |
| 95 | |
| 96 class Copying(object): | |
| 97 __slots__ = 'file', 'name' | |
| 98 | |
| 99 def __init__(self, file, name): | |
| 100 self.file = file | |
| 101 self.name = name | |
| 102 | |
| 103 def __enter__(self): | |
| 104 if self.name: | |
| 105 self.file.copy(self.name) | |
| 106 | |
| 107 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 108 pass | |
| 109 | |
| 110 | |
| 111 | |
| 21 | 112 # Test case types |
| 16 | 113 |
| 114 class TestCase(object): | |
| 21 | 115 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
116 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
117 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
118 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
119 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
120 'time_limit_string') |
| 21 | 121 |
| 122 if ABCMeta: | |
| 123 __metaclass__ = ABCMeta | |
| 16 | 124 |
| 21 | 125 def __init__(case, prob, id, isdummy, points): |
| 16 | 126 case.problem = prob |
| 21 | 127 case.id = id |
| 128 case.isdummy = isdummy | |
| 129 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
130 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
131 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 132 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
133 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
134 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 135 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
136 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
137 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
138 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
139 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
140 case.wall_time_limit_string = '' |
| 21 | 141 if not isdummy: |
| 142 case.realinname = case.problem.config.testcaseinname | |
| 143 case.realoutname = case.problem.config.testcaseoutname | |
| 144 else: | |
| 145 case.realinname = case.problem.config.dummyinname | |
| 146 case.realoutname = case.problem.config.dummyoutname | |
| 147 | |
| 148 @abstractmethod | |
| 149 def test(case): raise NotImplementedError | |
| 16 | 150 |
| 22 | 151 def __call__(case, callback): |
| 152 case.has_called_back = False | |
| 153 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
154 case.time_limit_string = case.wall_time_limit_string |
| 21 | 155 try: |
| 22 | 156 return case.test(callback) |
| 21 | 157 finally: |
| 22 | 158 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
159 if getattr(case, 'time_started', None) is None: |
| 22 | 160 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
161 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 162 case.time_stopped = now |
| 163 if not case.has_called_back: | |
| 164 callback() | |
| 21 | 165 case.cleanup() |
| 166 | |
| 167 def cleanup(case): | |
| 168 #if getattr(case, 'infile', None): | |
| 169 # case.infile.close() | |
| 170 #if getattr(case, 'outfile', None): | |
| 171 # case.outfile.close() | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
172 if getattr(case, 'process', None) and case.process.returncode is None: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
173 # Try KILLing after three unsuccessful TERM attempts in a row |
| 21 | 174 for i in range(3): |
| 175 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
176 terminate(case.process) |
| 21 | 177 except Exception: |
| 178 time.sleep(0) | |
| 179 case.process.poll() | |
| 180 else: | |
| 22 | 181 case.process.wait() |
| 21 | 182 break |
| 183 else: | |
| 184 # If killing the process is unsuccessful three times in a row, | |
| 185 # just silently stop trying | |
| 186 for i in range(3): | |
| 187 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
188 kill(case.process) |
| 21 | 189 except Exception: |
| 190 time.sleep(0) | |
| 191 case.process.poll() | |
| 192 else: | |
| 22 | 193 case.process.wait() |
| 21 | 194 break |
| 22 | 195 if case.files_to_delete: |
| 196 for name in case.files_to_delete: | |
| 197 try: | |
| 198 os.remove(name) | |
| 199 except Exception: | |
| 200 # It can't be helped | |
| 201 pass | |
| 21 | 202 |
| 203 def open_infile(case): | |
| 204 try: | |
| 205 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 206 except IOError: | |
| 207 e = sys.exc_info()[1] | |
| 208 raise CannotReadInputFile(e) | |
| 209 | |
| 210 def open_outfile(case): | |
| 211 try: | |
| 212 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 213 except IOError: | |
| 214 e = sys.exc_info()[1] | |
| 215 raise CannotReadAnswerFile(e) | |
| 216 | |
| 16 | 217 |
| 21 | 218 class ValidatedTestCase(TestCase): |
| 219 __slots__ = 'validator' | |
| 220 | |
| 221 def __init__(case, *args): | |
| 222 TestCase.__init__(case, *args) | |
| 223 if not case.problem.config.tester: | |
| 224 case.validator = None | |
| 225 else: | |
| 226 case.validator = case.problem.config.tester | |
| 227 | |
| 228 def validate(case, output): | |
| 229 if not case.validator: | |
| 230 # Compare the output with the reference output | |
| 231 case.open_outfile() | |
| 232 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
233 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
234 if refline is not None and not isinstance(refline, basestring): |
| 21 | 235 line = bytes(line, sys.getdefaultencoding()) |
| 236 if line != refline: | |
| 22 | 237 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
238 return 1 |
| 21 | 239 elif callable(case.validator): |
| 240 return case.validator(output) | |
| 241 else: | |
| 242 # Call the validator program | |
| 243 output.close() | |
| 23 | 244 if case.problem.config.ansname: |
| 245 case.open_outfile() | |
| 246 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
247 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
248 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
249 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
250 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 251 comment = case.process.communicate()[0].strip() |
| 26 | 252 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 253 if match: |
| 254 comment = comment[match.end():] | |
| 255 if not case.problem.config.maxexitcode: | |
| 256 if case.process.returncode: | |
| 257 raise WrongAnswer(comment) | |
| 258 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
259 return 1, comment |
| 21 | 260 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
261 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 262 |
| 263 | |
| 264 class BatchTestCase(ValidatedTestCase): | |
| 265 __slots__ = () | |
| 266 | |
| 22 | 267 def test(case, callback): |
| 21 | 268 case.open_infile() |
| 269 case.time_started = None | |
| 270 if case.problem.config.stdio: | |
| 54 | 271 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 272 # TODO: re-use the same file name if possible |
| 21 | 273 # FIXME: 2.5 lacks the delete parameter |
| 274 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 275 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
276 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 277 else: |
| 278 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
279 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
280 with contextmgr: |
|
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
281 with open(inputdatafname) as infile: |
| 22 | 282 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
283 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
284 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 22 | 285 raise NonZeroExitCode(case.process.returncode) |
| 286 callback() | |
| 287 case.has_called_back = True | |
| 288 outfile.seek(0) | |
| 289 return case.validate(outfile) | |
| 21 | 290 else: |
| 22 | 291 case.infile.copy(case.problem.config.inname) |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
292 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
293 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 294 raise NonZeroExitCode(case.process.returncode) |
| 22 | 295 callback() |
| 296 case.has_called_back = True | |
| 21 | 297 with open(case.problem.config.outname, 'rU') as output: |
| 298 return case.validate(output) | |
| 299 | |
| 300 | |
| 301 # This is the only test case type not executing any programs to be tested | |
| 302 class OutputOnlyTestCase(ValidatedTestCase): | |
| 303 __slots__ = () | |
| 304 def cleanup(case): pass | |
| 305 | |
| 306 class BestOutputTestCase(ValidatedTestCase): | |
| 307 __slots__ = () | |
| 308 | |
| 309 # This is the only test case type executing two programs simultaneously | |
| 310 class ReactiveTestCase(TestCase): | |
| 311 __slots__ = () | |
| 312 # The basic idea is to launch the program to be tested and the grader | |
| 313 # and to pipe their standard I/O from and to each other, | |
| 314 # and then to capture the grader's exit code and use it | |
| 26 | 315 # like the exit code of an output validator is used. |
| 21 | 316 |
| 317 | |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
318 class DummyTestContext(problem.TestGroup): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
319 __slots__ = () |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
320 def end(self): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
321 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
322 return 0, 0, self.log |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
323 |
| 21 | 324 def load_problem(prob, _types={'batch' : BatchTestCase, |
| 325 'outonly' : OutputOnlyTestCase, | |
| 326 'bestout' : BestOutputTestCase, | |
| 327 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
328 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
329 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
330 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
331 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
332 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
333 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
334 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
335 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
336 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
337 |
| 23 | 338 if options.legacy: |
| 339 prob.config.usegroups = False | |
| 58 | 340 newtests = [] |
| 23 | 341 for i, name in enumerate(prob.config.tests): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
342 # Same here; we'll need to iterate over them twice |
| 23 | 343 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
344 l = len(name) |
| 23 | 345 except Exception: |
| 346 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
347 name = tuple(name) |
| 23 | 348 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
349 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
350 l = len(name) |
| 58 | 351 if l > 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
352 prob.config.usegroups = True |
| 58 | 353 newtests.append(name) |
| 354 if prob.config.usegroups: | |
| 355 prob.config.tests = newtests | |
| 356 del newtests | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
357 |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
358 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
359 if isinstance(prob.config.pointmap, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
360 def getpoints(i, j, k=None): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
361 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
362 return prob.config.pointmap[i] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
363 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
364 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
365 return prob.config.pointmap[None] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
366 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
367 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
368 elif prob.config.usegroups: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
369 def getpoints(i, j, k): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
370 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
371 return prob.config.pointmap[k][j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
372 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
373 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
374 else: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
375 def getpoints(i, j): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
376 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
377 return prob.config.pointmap[j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
378 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
379 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
380 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
381 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
382 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
383 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
384 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
385 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 386 if prob.config.usegroups: |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
387 if not isinstance(prob.config.groupweight, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
388 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
389 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
390 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
391 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
392 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
393 yield DummyTestContext() |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
394 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
395 s = str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
396 yield _types[prob.config.kind](prob, s, True, 0) |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
397 yield problem.test_context_end |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
398 for k, group in enumerate(prob.config.tests): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
399 if not group: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
400 continue |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
401 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
402 for j, i in enumerate(group): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
403 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
404 yield _types[prob.config.kind](prob, s, False, getpoints(i, j, k)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
405 yield problem.test_context_end |
| 16 | 406 else: |
| 407 for i in prob.config.tests: | |
| 21 | 408 s = str(i).zfill(prob.config.padtests) |
| 409 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 410 for i in prob.config.dummies: | |
| 411 s = str(i).zfill(prob.config.paddummies) | |
| 412 yield _types[prob.config.kind](prob, s, True, 0) | |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
413 for j, i in enumerate(prob.config.tests): |
| 21 | 414 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
415 yield _types[prob.config.kind](prob, s, False, getpoints(i, j)) |