Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 86:8cd7a732f2f3
Fixed a crash in the unix module
Bug fix: Upreckon no longer crashes on non-Win32 when the standard input
is not a console or Upreckon does not have the tools to catch key presses.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Fri, 25 Feb 2011 00:10:20 +0000 |
| parents | 741ae3391b61 |
| children | cd347cfca272 |
| rev | line source |
|---|---|
| 21 | 1 #! /usr/bin/env python |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
2 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 3 |
| 43 | 4 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 5 | |
| 21 | 6 from __future__ import division, with_statement |
| 7 | |
| 8 try: | |
| 9 from compat import * | |
| 10 import files, problem, config | |
| 11 except ImportError: | |
| 12 import __main__ | |
| 13 __main__.import_error(sys.exc_info()[1]) | |
| 14 else: | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
15 from __main__ import options |
| 21 | 16 |
| 17 import glob, re, sys, tempfile, time | |
| 18 from subprocess import Popen, PIPE, STDOUT | |
| 19 | |
| 20 import os | |
| 21 devnull = open(os.path.devnull, 'w+') | |
| 22 | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
23 if options.autotime: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
24 # This is really a dirty hack that assumes that sleep() does not spend |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
25 # the CPU time of the current process and that if clock() measures |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
26 # wall-clock time, then it is more precise than time() is. Both these |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
27 # assumptions are true on all platforms I have tested this on so far, |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
28 # but I am not aware of any guarantee that they will both be true |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
29 # on every other platform. |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
30 c = time.clock() |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
31 time.sleep(1) |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
32 c = time.clock() - c |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
33 if int(c + .5) == 1: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
34 clock = time.clock |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
35 else: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
36 clock = time.time |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
37 |
| 21 | 38 try: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
39 from win32 import * |
|
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
40 except Exception: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
41 from unix import * |
| 22 | 42 |
| 21 | 43 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 44 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 45 'NonZeroExitCode', 'CannotStartTestee', | |
| 46 'CannotStartValidator', 'CannotReadOutputFile', | |
|
81
24752db487c5
Fixed errors in the win32 module
Oleg Oshmyan <chortos@inbox.lv>
parents:
79
diff
changeset
|
47 'CannotReadInputFile', 'CannotReadAnswerFile', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
48 'MemoryLimitExceeded', 'CPUTimeLimitExceeded', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
49 'WallTimeLimitExceeded') |
| 21 | 50 |
| 51 | |
| 52 | |
| 53 # Exceptions | |
| 54 | |
| 55 class TestCaseNotPassed(Exception): __slots__ = () | |
| 56 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
57 class CPUTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
58 class WallTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
59 class MemoryLimitExceeded(TestCaseNotPassed): __slots__ = () |
| 22 | 60 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 61 |
| 62 class WrongAnswer(TestCaseNotPassed): | |
| 63 __slots__ = 'comment' | |
| 64 def __init__(self, comment=''): | |
| 65 self.comment = comment | |
| 66 | |
| 67 class NonZeroExitCode(TestCaseNotPassed): | |
| 68 __slots__ = 'exitcode' | |
| 69 def __init__(self, exitcode): | |
| 70 self.exitcode = exitcode | |
| 71 | |
| 72 class ExceptionWrapper(TestCaseNotPassed): | |
| 73 __slots__ = 'upstream' | |
| 74 def __init__(self, upstream): | |
| 75 self.upstream = upstream | |
| 76 | |
| 77 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 78 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 79 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 80 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 81 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 82 | |
| 83 | |
| 84 | |
| 22 | 85 # Helper context managers |
| 86 | |
| 87 class CopyDeleting(object): | |
| 88 __slots__ = 'case', 'file', 'name' | |
| 89 | |
| 90 def __init__(self, case, file, name): | |
| 91 self.case = case | |
| 92 self.file = file | |
| 93 self.name = name | |
| 94 | |
| 95 def __enter__(self): | |
| 96 if self.name: | |
| 97 try: | |
| 98 self.file.copy(self.name) | |
| 99 except: | |
| 100 try: | |
| 101 self.__exit__(None, None, None) | |
| 102 except: | |
| 103 pass | |
| 104 raise | |
| 105 | |
| 106 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 107 if self.name: | |
| 108 self.case.files_to_delete.append(self.name) | |
| 109 | |
| 110 | |
| 111 class Copying(object): | |
| 112 __slots__ = 'file', 'name' | |
| 113 | |
| 114 def __init__(self, file, name): | |
| 115 self.file = file | |
| 116 self.name = name | |
| 117 | |
| 118 def __enter__(self): | |
| 119 if self.name: | |
| 120 self.file.copy(self.name) | |
| 121 | |
| 122 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 123 pass | |
| 124 | |
| 125 | |
| 126 | |
| 21 | 127 # Test case types |
| 16 | 128 |
| 129 class TestCase(object): | |
| 21 | 130 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
131 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
132 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
133 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
134 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
135 'time_limit_string') |
| 21 | 136 |
| 137 if ABCMeta: | |
| 138 __metaclass__ = ABCMeta | |
| 16 | 139 |
| 21 | 140 def __init__(case, prob, id, isdummy, points): |
| 16 | 141 case.problem = prob |
| 21 | 142 case.id = id |
| 143 case.isdummy = isdummy | |
| 144 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
145 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
146 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 147 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
148 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
149 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 150 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
151 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
152 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
153 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
154 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
155 case.wall_time_limit_string = '' |
| 21 | 156 if not isdummy: |
| 157 case.realinname = case.problem.config.testcaseinname | |
| 158 case.realoutname = case.problem.config.testcaseoutname | |
| 159 else: | |
| 160 case.realinname = case.problem.config.dummyinname | |
| 161 case.realoutname = case.problem.config.dummyoutname | |
| 162 | |
| 163 @abstractmethod | |
| 164 def test(case): raise NotImplementedError | |
| 16 | 165 |
| 22 | 166 def __call__(case, callback): |
| 167 case.has_called_back = False | |
| 168 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
169 case.time_limit_string = case.wall_time_limit_string |
| 21 | 170 try: |
| 22 | 171 return case.test(callback) |
| 21 | 172 finally: |
| 22 | 173 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
174 if getattr(case, 'time_started', None) is None: |
| 22 | 175 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
176 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 177 case.time_stopped = now |
| 178 if not case.has_called_back: | |
| 179 callback() | |
| 21 | 180 case.cleanup() |
| 181 | |
| 182 def cleanup(case): | |
| 183 #if getattr(case, 'infile', None): | |
| 184 # case.infile.close() | |
| 185 #if getattr(case, 'outfile', None): | |
| 186 # case.outfile.close() | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
187 if getattr(case, 'process', None) and case.process.returncode is None: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
188 # Try KILLing after three unsuccessful TERM attempts in a row |
| 21 | 189 for i in range(3): |
| 190 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
191 terminate(case.process) |
| 21 | 192 except Exception: |
| 193 time.sleep(0) | |
| 194 case.process.poll() | |
| 195 else: | |
| 22 | 196 case.process.wait() |
| 21 | 197 break |
| 198 else: | |
| 199 # If killing the process is unsuccessful three times in a row, | |
| 200 # just silently stop trying | |
| 201 for i in range(3): | |
| 202 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
203 kill(case.process) |
| 21 | 204 except Exception: |
| 205 time.sleep(0) | |
| 206 case.process.poll() | |
| 207 else: | |
| 22 | 208 case.process.wait() |
| 21 | 209 break |
| 22 | 210 if case.files_to_delete: |
| 211 for name in case.files_to_delete: | |
| 212 try: | |
| 213 os.remove(name) | |
| 214 except Exception: | |
| 215 # It can't be helped | |
| 216 pass | |
| 21 | 217 |
| 218 def open_infile(case): | |
| 219 try: | |
| 220 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 221 except IOError: | |
| 222 e = sys.exc_info()[1] | |
| 223 raise CannotReadInputFile(e) | |
| 224 | |
| 225 def open_outfile(case): | |
| 226 try: | |
| 227 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 228 except IOError: | |
| 229 e = sys.exc_info()[1] | |
| 230 raise CannotReadAnswerFile(e) | |
| 231 | |
| 16 | 232 |
| 21 | 233 class ValidatedTestCase(TestCase): |
| 234 __slots__ = 'validator' | |
| 235 | |
| 236 def __init__(case, *args): | |
| 237 TestCase.__init__(case, *args) | |
| 238 if not case.problem.config.tester: | |
| 239 case.validator = None | |
| 240 else: | |
| 241 case.validator = case.problem.config.tester | |
| 242 | |
| 243 def validate(case, output): | |
| 244 if not case.validator: | |
| 245 # Compare the output with the reference output | |
| 246 case.open_outfile() | |
| 247 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
248 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
249 if refline is not None and not isinstance(refline, basestring): |
| 21 | 250 line = bytes(line, sys.getdefaultencoding()) |
| 251 if line != refline: | |
| 22 | 252 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
253 return 1 |
| 21 | 254 elif callable(case.validator): |
| 255 return case.validator(output) | |
| 256 else: | |
| 257 # Call the validator program | |
| 258 output.close() | |
| 23 | 259 if case.problem.config.ansname: |
| 260 case.open_outfile() | |
| 261 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
262 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
263 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
264 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
265 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 266 comment = case.process.communicate()[0].strip() |
| 26 | 267 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 268 if match: |
| 269 comment = comment[match.end():] | |
| 270 if not case.problem.config.maxexitcode: | |
| 271 if case.process.returncode: | |
| 272 raise WrongAnswer(comment) | |
| 273 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
274 return 1, comment |
| 21 | 275 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
276 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 277 |
| 278 | |
| 279 class BatchTestCase(ValidatedTestCase): | |
| 280 __slots__ = () | |
| 281 | |
| 22 | 282 def test(case, callback): |
| 21 | 283 case.open_infile() |
| 284 case.time_started = None | |
| 285 if case.problem.config.stdio: | |
| 54 | 286 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 287 # TODO: re-use the same file name if possible |
| 21 | 288 # FIXME: 2.5 lacks the delete parameter |
| 289 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 290 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
291 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 292 else: |
| 293 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
294 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
295 with contextmgr: |
|
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
296 with open(inputdatafname) as infile: |
| 83 | 297 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
298 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
299 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 22 | 300 raise NonZeroExitCode(case.process.returncode) |
| 301 callback() | |
| 302 case.has_called_back = True | |
| 303 outfile.seek(0) | |
| 304 return case.validate(outfile) | |
| 21 | 305 else: |
| 22 | 306 case.infile.copy(case.problem.config.inname) |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
307 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
308 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 309 raise NonZeroExitCode(case.process.returncode) |
| 22 | 310 callback() |
| 311 case.has_called_back = True | |
| 21 | 312 with open(case.problem.config.outname, 'rU') as output: |
| 313 return case.validate(output) | |
| 314 | |
| 315 | |
| 316 # This is the only test case type not executing any programs to be tested | |
| 317 class OutputOnlyTestCase(ValidatedTestCase): | |
| 318 __slots__ = () | |
| 319 def cleanup(case): pass | |
| 320 | |
| 321 class BestOutputTestCase(ValidatedTestCase): | |
| 322 __slots__ = () | |
| 323 | |
| 324 # This is the only test case type executing two programs simultaneously | |
| 325 class ReactiveTestCase(TestCase): | |
| 326 __slots__ = () | |
| 327 # The basic idea is to launch the program to be tested and the grader | |
| 328 # and to pipe their standard I/O from and to each other, | |
| 329 # and then to capture the grader's exit code and use it | |
| 26 | 330 # like the exit code of an output validator is used. |
| 21 | 331 |
| 332 | |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
333 class DummyTestContext(problem.TestGroup): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
334 __slots__ = () |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
335 def end(self): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
336 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
337 return 0, 0, self.log |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
338 |
| 21 | 339 def load_problem(prob, _types={'batch' : BatchTestCase, |
| 340 'outonly' : OutputOnlyTestCase, | |
| 341 'bestout' : BestOutputTestCase, | |
| 342 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
343 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
344 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
345 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
346 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
347 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
348 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
349 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
350 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
351 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
352 |
| 23 | 353 if options.legacy: |
| 354 prob.config.usegroups = False | |
| 58 | 355 newtests = [] |
| 23 | 356 for i, name in enumerate(prob.config.tests): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
357 # Same here; we'll need to iterate over them twice |
| 23 | 358 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
359 l = len(name) |
| 23 | 360 except Exception: |
| 361 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
362 name = tuple(name) |
| 23 | 363 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
364 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
365 l = len(name) |
| 58 | 366 if l > 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
367 prob.config.usegroups = True |
| 58 | 368 newtests.append(name) |
| 369 if prob.config.usegroups: | |
| 370 prob.config.tests = newtests | |
| 371 del newtests | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
372 |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
373 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
374 if isinstance(prob.config.pointmap, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
375 def getpoints(i, j, k=None): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
376 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
377 return prob.config.pointmap[i] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
378 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
379 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
380 return prob.config.pointmap[None] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
381 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
382 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
383 elif prob.config.usegroups: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
384 def getpoints(i, j, k): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
385 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
386 return prob.config.pointmap[k][j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
387 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
388 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
389 else: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
390 def getpoints(i, j): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
391 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
392 return prob.config.pointmap[j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
393 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
394 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
395 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
396 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
397 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
398 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
399 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
400 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 401 if prob.config.usegroups: |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
402 if not isinstance(prob.config.groupweight, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
403 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
404 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
405 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
406 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
407 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
408 yield DummyTestContext() |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
409 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
410 s = str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
411 yield _types[prob.config.kind](prob, s, True, 0) |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
412 yield problem.test_context_end |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
413 for k, group in enumerate(prob.config.tests): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
414 if not group: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
415 continue |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
416 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
417 for j, i in enumerate(group): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
418 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
419 yield _types[prob.config.kind](prob, s, False, getpoints(i, j, k)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
420 yield problem.test_context_end |
| 16 | 421 else: |
| 422 for i in prob.config.tests: | |
| 21 | 423 s = str(i).zfill(prob.config.padtests) |
| 424 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 425 for i in prob.config.dummies: | |
| 426 s = str(i).zfill(prob.config.paddummies) | |
| 427 yield _types[prob.config.kind](prob, s, True, 0) | |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
428 for j, i in enumerate(prob.config.tests): |
| 21 | 429 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
430 yield _types[prob.config.kind](prob, s, False, getpoints(i, j)) |