Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 55:c726235e49ee
Hopefully somewhat improve accuracy of run-time reporting
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Mon, 20 Dec 2010 23:03:28 +0200 |
| parents | 1914ae9cfdce |
| children | 693a938bdeee |
| rev | line source |
|---|---|
| 21 | 1 #! /usr/bin/env python |
| 16 | 2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
| 3 | |
| 43 | 4 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 5 | |
| 21 | 6 from __future__ import division, with_statement |
| 7 | |
| 8 try: | |
| 9 from compat import * | |
| 10 import files, problem, config | |
| 11 except ImportError: | |
| 12 import __main__ | |
| 13 __main__.import_error(sys.exc_info()[1]) | |
| 14 else: | |
| 15 from __main__ import clock, options | |
| 16 | |
| 17 import glob, re, sys, tempfile, time | |
| 18 from subprocess import Popen, PIPE, STDOUT | |
| 19 | |
| 20 import os | |
| 21 devnull = open(os.path.devnull, 'w+') | |
| 22 | |
| 23 try: | |
| 24 from signal import SIGTERM, SIGKILL | |
| 25 except ImportError: | |
| 26 SIGTERM = 15 | |
| 27 SIGKILL = 9 | |
| 28 | |
| 16 | 29 try: |
| 21 | 30 from _subprocess import TerminateProcess |
| 31 except ImportError: | |
| 32 # CPython 2.5 does define _subprocess.TerminateProcess even though it is | |
| 33 # not used in the subprocess module, but maybe something else does not | |
| 34 try: | |
| 35 import ctypes | |
| 36 TerminateProcess = ctypes.windll.kernel32.TerminateProcess | |
| 37 except (ImportError, AttributeError): | |
| 38 TerminateProcess = None | |
| 39 | |
| 22 | 40 |
| 41 # Do the hacky-wacky dark magic needed to catch presses of the Escape button. | |
| 42 # If only Python supported forcible termination of threads... | |
| 43 if not sys.stdin.isatty(): | |
| 44 canceled = init_canceled = lambda: False | |
| 45 pause = None | |
| 46 else: | |
| 47 try: | |
| 48 # Windows has select() too, but it is not the select() we want | |
| 49 import msvcrt | |
| 50 except ImportError: | |
| 51 try: | |
| 52 import select, termios, tty, atexit | |
| 53 except ImportError: | |
| 54 # It cannot be helped! | |
| 55 # Silently disable support for killing the program being tested | |
| 56 canceled = init_canceled = lambda: False | |
| 57 pause = None | |
| 58 else: | |
| 59 def cleanup(old=termios.tcgetattr(sys.stdin.fileno())): | |
| 60 termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old) | |
| 61 atexit.register(cleanup) | |
| 62 del cleanup | |
| 63 tty.setcbreak(sys.stdin.fileno()) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
64 def canceled(select=select.select, stdin=sys.stdin, read=sys.stdin.read): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
65 while select((stdin,), (), (), 0)[0]: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
66 if read(1) == '\33': |
| 22 | 67 return True |
| 68 return False | |
| 69 def init_canceled(): | |
| 70 while select.select((sys.stdin,), (), (), 0)[0]: | |
| 71 sys.stdin.read(1) | |
| 72 def pause(): | |
| 73 sys.stdin.read(1) | |
| 74 else: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
75 def canceled(kbhit=msvcrt.kbhit, getch=msvcrt.getch): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
76 while kbhit(): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
77 c = getch() |
| 22 | 78 if c == '\33': |
| 79 return True | |
| 80 elif c == '\0': | |
| 81 # Let's hope no-one is fiddling with this | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
82 getch() |
| 22 | 83 return False |
| 84 def init_canceled(): | |
| 85 while msvcrt.kbhit(): | |
| 86 msvcrt.getch() | |
| 87 def pause(): | |
| 88 msvcrt.getch() | |
| 89 | |
| 90 | |
| 21 | 91 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 92 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 93 'NonZeroExitCode', 'CannotStartTestee', | |
| 94 'CannotStartValidator', 'CannotReadOutputFile', | |
| 95 'CannotReadInputFile', 'CannotReadAnswerFile') | |
| 21 | 96 |
| 97 | |
| 98 | |
| 99 # Exceptions | |
| 100 | |
| 101 class TestCaseNotPassed(Exception): __slots__ = () | |
| 102 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
| 22 | 103 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 104 |
| 105 class WrongAnswer(TestCaseNotPassed): | |
| 106 __slots__ = 'comment' | |
| 107 def __init__(self, comment=''): | |
| 108 self.comment = comment | |
| 109 | |
| 110 class NonZeroExitCode(TestCaseNotPassed): | |
| 111 __slots__ = 'exitcode' | |
| 112 def __init__(self, exitcode): | |
| 113 self.exitcode = exitcode | |
| 114 | |
| 115 class ExceptionWrapper(TestCaseNotPassed): | |
| 116 __slots__ = 'upstream' | |
| 117 def __init__(self, upstream): | |
| 118 self.upstream = upstream | |
| 119 | |
| 120 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 121 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 122 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 123 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 124 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 125 | |
| 126 | |
| 127 | |
| 22 | 128 # Helper context managers |
| 129 | |
| 130 class CopyDeleting(object): | |
| 131 __slots__ = 'case', 'file', 'name' | |
| 132 | |
| 133 def __init__(self, case, file, name): | |
| 134 self.case = case | |
| 135 self.file = file | |
| 136 self.name = name | |
| 137 | |
| 138 def __enter__(self): | |
| 139 if self.name: | |
| 140 try: | |
| 141 self.file.copy(self.name) | |
| 142 except: | |
| 143 try: | |
| 144 self.__exit__(None, None, None) | |
| 145 except: | |
| 146 pass | |
| 147 raise | |
| 148 | |
| 149 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 150 if self.name: | |
| 151 self.case.files_to_delete.append(self.name) | |
| 152 | |
| 153 | |
| 154 class Copying(object): | |
| 155 __slots__ = 'file', 'name' | |
| 156 | |
| 157 def __init__(self, file, name): | |
| 158 self.file = file | |
| 159 self.name = name | |
| 160 | |
| 161 def __enter__(self): | |
| 162 if self.name: | |
| 163 self.file.copy(self.name) | |
| 164 | |
| 165 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 166 pass | |
| 167 | |
| 168 | |
| 169 | |
| 21 | 170 # Test case types |
| 16 | 171 |
| 172 class TestCase(object): | |
| 21 | 173 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
| 174 'process', 'time_started', 'time_stopped', 'time_limit_string', | |
| 22 | 175 'realinname', 'realoutname', 'maxtime', 'maxmemory', |
| 176 'has_called_back', 'files_to_delete') | |
| 21 | 177 |
| 178 if ABCMeta: | |
| 179 __metaclass__ = ABCMeta | |
| 16 | 180 |
| 21 | 181 def __init__(case, prob, id, isdummy, points): |
| 16 | 182 case.problem = prob |
| 21 | 183 case.id = id |
| 184 case.isdummy = isdummy | |
| 185 case.points = points | |
| 186 case.maxtime = case.problem.config.maxtime | |
| 187 case.maxmemory = case.problem.config.maxmemory | |
| 188 if case.maxtime: | |
| 189 case.time_limit_string = '/%.3f' % case.maxtime | |
| 190 else: | |
| 191 case.time_limit_string = '' | |
| 192 if not isdummy: | |
| 193 case.realinname = case.problem.config.testcaseinname | |
| 194 case.realoutname = case.problem.config.testcaseoutname | |
| 195 else: | |
| 196 case.realinname = case.problem.config.dummyinname | |
| 197 case.realoutname = case.problem.config.dummyoutname | |
| 198 | |
| 199 @abstractmethod | |
| 200 def test(case): raise NotImplementedError | |
| 16 | 201 |
| 22 | 202 def __call__(case, callback): |
| 203 case.has_called_back = False | |
| 204 case.files_to_delete = [] | |
| 21 | 205 try: |
| 22 | 206 return case.test(callback) |
| 21 | 207 finally: |
| 22 | 208 now = clock() |
| 209 if not getattr(case, 'time_started', None): | |
| 210 case.time_started = case.time_stopped = now | |
| 211 elif not getattr(case, 'time_stopped', None): | |
| 212 case.time_stopped = now | |
| 213 if not case.has_called_back: | |
| 214 callback() | |
| 21 | 215 case.cleanup() |
| 216 | |
| 217 def cleanup(case): | |
| 218 #if getattr(case, 'infile', None): | |
| 219 # case.infile.close() | |
| 220 #if getattr(case, 'outfile', None): | |
| 221 # case.outfile.close() | |
| 222 if getattr(case, 'process', None): | |
| 223 # Try killing after three unsuccessful TERM attempts in a row | |
| 224 # (except on Windows, where TERMing is killing) | |
| 225 for i in range(3): | |
| 226 try: | |
| 227 try: | |
| 228 case.process.terminate() | |
| 229 except AttributeError: | |
| 230 # Python 2.5 | |
| 231 if TerminateProcess and hasattr(proc, '_handle'): | |
| 232 # Windows API | |
| 233 TerminateProcess(proc._handle, 1) | |
| 234 else: | |
| 235 # POSIX | |
| 236 os.kill(proc.pid, SIGTERM) | |
| 237 except Exception: | |
| 238 time.sleep(0) | |
| 239 case.process.poll() | |
| 240 else: | |
| 22 | 241 case.process.wait() |
| 21 | 242 break |
| 243 else: | |
| 244 # If killing the process is unsuccessful three times in a row, | |
| 245 # just silently stop trying | |
| 246 for i in range(3): | |
| 247 try: | |
| 248 try: | |
| 249 case.process.kill() | |
| 250 except AttributeError: | |
| 251 # Python 2.5 | |
| 252 if TerminateProcess and hasattr(proc, '_handle'): | |
| 253 # Windows API | |
| 254 TerminateProcess(proc._handle, 1) | |
| 255 else: | |
| 256 # POSIX | |
| 257 os.kill(proc.pid, SIGKILL) | |
| 258 except Exception: | |
| 259 time.sleep(0) | |
| 260 case.process.poll() | |
| 261 else: | |
| 22 | 262 case.process.wait() |
| 21 | 263 break |
| 22 | 264 if case.files_to_delete: |
| 265 for name in case.files_to_delete: | |
| 266 try: | |
| 267 os.remove(name) | |
| 268 except Exception: | |
| 269 # It can't be helped | |
| 270 pass | |
| 21 | 271 |
| 272 def open_infile(case): | |
| 273 try: | |
| 274 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 275 except IOError: | |
| 276 e = sys.exc_info()[1] | |
| 277 raise CannotReadInputFile(e) | |
| 278 | |
| 279 def open_outfile(case): | |
| 280 try: | |
| 281 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 282 except IOError: | |
| 283 e = sys.exc_info()[1] | |
| 284 raise CannotReadAnswerFile(e) | |
| 285 | |
| 16 | 286 |
| 21 | 287 class ValidatedTestCase(TestCase): |
| 288 __slots__ = 'validator' | |
| 289 | |
| 290 def __init__(case, *args): | |
| 291 TestCase.__init__(case, *args) | |
| 292 if not case.problem.config.tester: | |
| 293 case.validator = None | |
| 294 else: | |
| 295 case.validator = case.problem.config.tester | |
| 296 | |
| 297 def validate(case, output): | |
| 298 if not case.validator: | |
| 299 # Compare the output with the reference output | |
| 300 case.open_outfile() | |
| 301 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
302 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
303 if refline is not None and not isinstance(refline, basestring): |
| 21 | 304 line = bytes(line, sys.getdefaultencoding()) |
| 305 if line != refline: | |
| 22 | 306 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
307 return 1 |
| 21 | 308 elif callable(case.validator): |
| 309 return case.validator(output) | |
| 310 else: | |
| 311 # Call the validator program | |
| 312 output.close() | |
| 23 | 313 if case.problem.config.ansname: |
| 314 case.open_outfile() | |
| 315 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
316 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
317 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
318 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
319 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 320 comment = case.process.communicate()[0].strip() |
| 26 | 321 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 322 if match: |
| 323 comment = comment[match.end():] | |
| 324 if not case.problem.config.maxexitcode: | |
| 325 if case.process.returncode: | |
| 326 raise WrongAnswer(comment) | |
| 327 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
328 return 1, comment |
| 21 | 329 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
330 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 331 |
| 332 | |
| 333 class BatchTestCase(ValidatedTestCase): | |
| 334 __slots__ = () | |
| 335 | |
| 22 | 336 def test(case, callback): |
| 337 init_canceled() | |
| 21 | 338 if sys.platform == 'win32' or not case.maxmemory: |
| 339 preexec_fn = None | |
| 340 else: | |
| 341 def preexec_fn(): | |
| 342 try: | |
| 343 import resource | |
| 344 maxmemory = int(case.maxmemory * 1048576) | |
| 345 resource.setrlimit(resource.RLIMIT_AS, (maxmemory, maxmemory)) | |
| 346 # I would also set a CPU time limit but I do not want the time | |
| 347 # that passes between the calls to fork and exec to be counted in | |
| 348 except MemoryError: | |
| 349 # We do not have enough memory for ourselves; | |
| 350 # let the parent know about this | |
| 351 raise | |
| 352 except Exception: | |
| 353 # Well, at least we tried | |
| 354 pass | |
| 355 case.open_infile() | |
| 356 case.time_started = None | |
| 357 if case.problem.config.stdio: | |
| 54 | 358 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 359 # TODO: re-use the same file name if possible |
| 21 | 360 # FIXME: 2.5 lacks the delete parameter |
| 361 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 362 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
363 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 364 else: |
| 365 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
366 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
367 with contextmgr: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
368 # FIXME: this U doesn't do anything good for the child process, does it? |
| 22 | 369 with open(inputdatafname, 'rU') as infile: |
| 370 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile: | |
| 21 | 371 try: |
| 22 | 372 try: |
| 373 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn) | |
| 374 except MemoryError: | |
| 375 # If there is not enough memory for the forked test.py, | |
| 376 # opt for silent dropping of the limit | |
| 26 | 377 # TODO: show a warning somewhere |
| 22 | 378 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
| 379 except OSError: | |
| 380 raise CannotStartTestee(sys.exc_info()[1]) | |
| 381 case.time_started = clock() | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
382 time_next_check = case.time_started + .15 |
| 22 | 383 if not case.maxtime: |
| 384 while True: | |
| 385 exitcode, now = case.process.poll(), clock() | |
| 386 if exitcode is not None: | |
| 387 case.time_stopped = now | |
| 388 break | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
389 # For some reason (probably Microsoft's fault), |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
390 # msvcrt.kbhit() is slow as hell |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
391 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
392 if now >= time_next_check: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
393 if canceled(): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
394 raise CanceledByUser |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
395 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
396 time_next_check = now + .15 |
|
55
c726235e49ee
Hopefully somewhat improve accuracy of run-time reporting
Oleg Oshmyan <chortos@inbox.lv>
parents:
54
diff
changeset
|
397 time.sleep(.001) |
| 22 | 398 else: |
| 399 time_end = case.time_started + case.maxtime | |
| 400 while True: | |
| 401 exitcode, now = case.process.poll(), clock() | |
| 402 if exitcode is not None: | |
| 403 case.time_stopped = now | |
| 404 break | |
| 405 elif now >= time_end: | |
| 406 raise TimeLimitExceeded | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
407 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
408 if now >= time_next_check: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
409 if canceled(): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
410 raise CanceledByUser |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
411 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
412 time_next_check = now + .15 |
|
55
c726235e49ee
Hopefully somewhat improve accuracy of run-time reporting
Oleg Oshmyan <chortos@inbox.lv>
parents:
54
diff
changeset
|
413 time.sleep(.001) |
| 22 | 414 if config.globalconf.force_zero_exitcode and case.process.returncode: |
| 415 raise NonZeroExitCode(case.process.returncode) | |
| 416 callback() | |
| 417 case.has_called_back = True | |
| 418 outfile.seek(0) | |
| 419 return case.validate(outfile) | |
| 21 | 420 else: |
| 22 | 421 case.infile.copy(case.problem.config.inname) |
| 21 | 422 try: |
| 423 try: | |
| 424 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn) | |
| 425 except MemoryError: | |
| 426 # If there is not enough memory for the forked test.py, | |
| 427 # opt for silent dropping of the limit | |
| 26 | 428 # TODO: show a warning somewhere |
| 21 | 429 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT) |
| 430 except OSError: | |
| 431 raise CannotStartTestee(sys.exc_info()[1]) | |
| 432 case.time_started = clock() | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
433 time_next_check = case.time_started + .15 |
| 21 | 434 if not case.maxtime: |
| 22 | 435 while True: |
| 436 exitcode, now = case.process.poll(), clock() | |
| 437 if exitcode is not None: | |
| 438 case.time_stopped = now | |
| 439 break | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
440 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
441 if now >= time_next_check: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
442 if canceled(): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
443 raise CanceledByUser |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
444 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
445 time_next_check = now + .15 |
|
55
c726235e49ee
Hopefully somewhat improve accuracy of run-time reporting
Oleg Oshmyan <chortos@inbox.lv>
parents:
54
diff
changeset
|
446 time.sleep(.001) |
| 21 | 447 else: |
| 448 time_end = case.time_started + case.maxtime | |
| 449 while True: | |
| 22 | 450 exitcode, now = case.process.poll(), clock() |
| 21 | 451 if exitcode is not None: |
| 452 case.time_stopped = now | |
| 453 break | |
| 454 elif now >= time_end: | |
| 22 | 455 raise TimeLimitExceeded |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
456 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
457 if now >= time_next_check: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
458 if canceled(): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
459 raise CanceledByUser |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
460 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
461 time_next_check = now + .15 |
|
55
c726235e49ee
Hopefully somewhat improve accuracy of run-time reporting
Oleg Oshmyan <chortos@inbox.lv>
parents:
54
diff
changeset
|
462 time.sleep(.001) |
| 21 | 463 if config.globalconf.force_zero_exitcode and case.process.returncode: |
| 464 raise NonZeroExitCode(case.process.returncode) | |
| 22 | 465 callback() |
| 466 case.has_called_back = True | |
| 21 | 467 with open(case.problem.config.outname, 'rU') as output: |
| 468 return case.validate(output) | |
| 469 | |
| 470 | |
| 471 # This is the only test case type not executing any programs to be tested | |
| 472 class OutputOnlyTestCase(ValidatedTestCase): | |
| 473 __slots__ = () | |
| 474 def cleanup(case): pass | |
| 475 | |
| 476 class BestOutputTestCase(ValidatedTestCase): | |
| 477 __slots__ = () | |
| 478 | |
| 479 # This is the only test case type executing two programs simultaneously | |
| 480 class ReactiveTestCase(TestCase): | |
| 481 __slots__ = () | |
| 482 # The basic idea is to launch the program to be tested and the grader | |
| 483 # and to pipe their standard I/O from and to each other, | |
| 484 # and then to capture the grader's exit code and use it | |
| 26 | 485 # like the exit code of an output validator is used. |
| 21 | 486 |
| 487 | |
| 488 def load_problem(prob, _types={'batch' : BatchTestCase, | |
| 489 'outonly' : OutputOnlyTestCase, | |
| 490 'bestout' : BestOutputTestCase, | |
| 491 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
492 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
493 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
494 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
495 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
496 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
497 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
498 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
499 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
500 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
501 |
| 23 | 502 if options.legacy: |
| 503 prob.config.usegroups = False | |
| 504 prob.config.tests = list(prob.config.tests) | |
| 505 for i, name in enumerate(prob.config.tests): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
506 # Same here; we'll need to iterate over them twice |
| 23 | 507 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
508 l = len(name) |
| 23 | 509 except Exception: |
| 510 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
511 name = tuple(name) |
| 23 | 512 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
513 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
514 l = len(name) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
515 if len(name) > 1: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
516 prob.config.usegroups = True |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
517 break |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
518 elif not len(name): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
519 prob.config.tests[i] = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
520 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
521 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
522 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
523 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
524 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
525 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 526 if prob.config.usegroups: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
527 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
528 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
529 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
530 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
531 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
532 s = str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
533 yield _types[prob.config.kind](prob, s, True, 0) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
534 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
535 yield problem.TestGroup() |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
536 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
537 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
538 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1))) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
539 yield problem.test_context_end |
| 16 | 540 else: |
| 541 for i in prob.config.tests: | |
| 21 | 542 s = str(i).zfill(prob.config.padtests) |
| 543 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 544 for i in prob.config.dummies: | |
| 545 s = str(i).zfill(prob.config.paddummies) | |
| 546 yield _types[prob.config.kind](prob, s, True, 0) | |
| 547 for i in prob.config.tests: | |
| 548 s = str(i).zfill(prob.config.padtests) | |
| 549 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1))) |