Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 99:9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
testcases.load_problem now always handles the True sent by Problem.test.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Wed, 02 Mar 2011 20:03:44 +0000 |
| parents | c62c9bfd614a |
| children | 4e6f231f055f |
| rev | line source |
|---|---|
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 2 |
| 43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 4 | |
| 21 | 5 from __future__ import division, with_statement |
| 6 | |
| 91 | 7 from compat import * |
| 8 import files, problem, config | |
| 9 from __main__ import options | |
| 21 | 10 |
| 11 import glob, re, sys, tempfile, time | |
| 12 from subprocess import Popen, PIPE, STDOUT | |
| 13 | |
| 14 import os | |
| 15 devnull = open(os.path.devnull, 'w+') | |
| 16 | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
17 if options.autotime: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
18 # This is really a dirty hack that assumes that sleep() does not spend |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
19 # the CPU time of the current process and that if clock() measures |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
20 # wall-clock time, then it is more precise than time() is. Both these |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
21 # assumptions are true on all platforms I have tested this on so far, |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
22 # but I am not aware of any guarantee that they will both be true |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
23 # on every other platform. |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
24 c = time.clock() |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
25 time.sleep(1) |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
26 c = time.clock() - c |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
27 if int(c + .5) == 1: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
28 clock = time.clock |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
29 else: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
30 clock = time.time |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
31 |
| 21 | 32 try: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
33 from win32 import * |
|
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
34 except Exception: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
35 from unix import * |
| 22 | 36 |
| 21 | 37 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 38 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 39 'NonZeroExitCode', 'CannotStartTestee', | |
| 40 'CannotStartValidator', 'CannotReadOutputFile', | |
|
81
24752db487c5
Fixed errors in the win32 module
Oleg Oshmyan <chortos@inbox.lv>
parents:
79
diff
changeset
|
41 'CannotReadInputFile', 'CannotReadAnswerFile', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
42 'MemoryLimitExceeded', 'CPUTimeLimitExceeded', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
43 'WallTimeLimitExceeded') |
| 21 | 44 |
| 45 | |
| 46 | |
| 47 # Exceptions | |
| 48 | |
| 49 class TestCaseNotPassed(Exception): __slots__ = () | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
50 class TestCaseSkipped(TestCaseNotPassed): __slots__ = () |
| 21 | 51 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
52 class CPUTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
53 class WallTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
54 class MemoryLimitExceeded(TestCaseNotPassed): __slots__ = () |
| 22 | 55 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 56 |
| 57 class WrongAnswer(TestCaseNotPassed): | |
| 58 __slots__ = 'comment' | |
| 59 def __init__(self, comment=''): | |
| 60 self.comment = comment | |
| 61 | |
| 62 class NonZeroExitCode(TestCaseNotPassed): | |
| 63 __slots__ = 'exitcode' | |
| 64 def __init__(self, exitcode): | |
| 65 self.exitcode = exitcode | |
| 66 | |
| 67 class ExceptionWrapper(TestCaseNotPassed): | |
| 68 __slots__ = 'upstream' | |
| 69 def __init__(self, upstream): | |
| 70 self.upstream = upstream | |
| 71 | |
| 72 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 73 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 74 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 75 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 76 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 77 | |
| 78 | |
| 79 | |
| 22 | 80 # Helper context managers |
| 81 | |
| 82 class CopyDeleting(object): | |
| 83 __slots__ = 'case', 'file', 'name' | |
| 84 | |
| 85 def __init__(self, case, file, name): | |
| 86 self.case = case | |
| 87 self.file = file | |
| 88 self.name = name | |
| 89 | |
| 90 def __enter__(self): | |
| 91 if self.name: | |
| 92 try: | |
| 93 self.file.copy(self.name) | |
| 94 except: | |
| 95 try: | |
| 96 self.__exit__(None, None, None) | |
| 97 except: | |
| 98 pass | |
| 99 raise | |
| 100 | |
| 101 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 102 if self.name: | |
| 103 self.case.files_to_delete.append(self.name) | |
| 104 | |
| 105 | |
| 106 class Copying(object): | |
| 107 __slots__ = 'file', 'name' | |
| 108 | |
| 109 def __init__(self, file, name): | |
| 110 self.file = file | |
| 111 self.name = name | |
| 112 | |
| 113 def __enter__(self): | |
| 114 if self.name: | |
| 115 self.file.copy(self.name) | |
| 116 | |
| 117 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 118 pass | |
| 119 | |
| 120 | |
| 121 | |
| 21 | 122 # Test case types |
| 16 | 123 |
| 124 class TestCase(object): | |
| 21 | 125 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
126 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
127 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
128 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
129 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
130 'time_limit_string') |
| 21 | 131 |
| 132 if ABCMeta: | |
| 133 __metaclass__ = ABCMeta | |
| 16 | 134 |
| 21 | 135 def __init__(case, prob, id, isdummy, points): |
| 16 | 136 case.problem = prob |
| 21 | 137 case.id = id |
| 138 case.isdummy = isdummy | |
| 139 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
140 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
141 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 142 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
143 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
144 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 145 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
146 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
147 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
148 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
149 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
150 case.wall_time_limit_string = '' |
| 21 | 151 if not isdummy: |
| 152 case.realinname = case.problem.config.testcaseinname | |
| 153 case.realoutname = case.problem.config.testcaseoutname | |
| 154 else: | |
| 155 case.realinname = case.problem.config.dummyinname | |
| 156 case.realoutname = case.problem.config.dummyoutname | |
| 157 | |
| 158 @abstractmethod | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
159 def test(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
160 raise NotImplementedError |
| 16 | 161 |
| 22 | 162 def __call__(case, callback): |
| 163 case.has_called_back = False | |
| 164 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
165 case.time_limit_string = case.wall_time_limit_string |
| 21 | 166 try: |
| 22 | 167 return case.test(callback) |
| 21 | 168 finally: |
| 22 | 169 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
170 if getattr(case, 'time_started', None) is None: |
| 22 | 171 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
172 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 173 case.time_stopped = now |
| 174 if not case.has_called_back: | |
| 175 callback() | |
| 21 | 176 case.cleanup() |
| 177 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
178 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
179 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
180 return False |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
181 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
182 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
183 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
184 return False |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
185 |
| 21 | 186 def cleanup(case): |
| 187 #if getattr(case, 'infile', None): | |
| 188 # case.infile.close() | |
| 189 #if getattr(case, 'outfile', None): | |
| 190 # case.outfile.close() | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
191 if getattr(case, 'process', None) and case.process.returncode is None: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
192 # Try KILLing after three unsuccessful TERM attempts in a row |
| 21 | 193 for i in range(3): |
| 194 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
195 terminate(case.process) |
| 21 | 196 except Exception: |
| 197 time.sleep(0) | |
| 198 case.process.poll() | |
| 199 else: | |
| 22 | 200 case.process.wait() |
| 21 | 201 break |
| 202 else: | |
| 203 # If killing the process is unsuccessful three times in a row, | |
| 204 # just silently stop trying | |
| 205 for i in range(3): | |
| 206 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
207 kill(case.process) |
| 21 | 208 except Exception: |
| 209 time.sleep(0) | |
| 210 case.process.poll() | |
| 211 else: | |
| 22 | 212 case.process.wait() |
| 21 | 213 break |
| 22 | 214 if case.files_to_delete: |
| 215 for name in case.files_to_delete: | |
| 216 try: | |
| 217 os.remove(name) | |
| 218 except Exception: | |
| 219 # It can't be helped | |
| 220 pass | |
| 21 | 221 |
| 222 def open_infile(case): | |
| 223 try: | |
| 224 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 225 except IOError: | |
| 226 e = sys.exc_info()[1] | |
| 227 raise CannotReadInputFile(e) | |
| 228 | |
| 229 def open_outfile(case): | |
| 230 try: | |
| 231 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 232 except IOError: | |
| 233 e = sys.exc_info()[1] | |
| 234 raise CannotReadAnswerFile(e) | |
| 235 | |
| 16 | 236 |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
237 class SkippedTestCase(TestCase): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
238 __slots__ = () |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
239 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
240 def test(case, callback): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
241 raise TestCaseSkipped |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
242 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
243 |
| 21 | 244 class ValidatedTestCase(TestCase): |
| 245 __slots__ = 'validator' | |
| 246 | |
| 247 def __init__(case, *args): | |
| 248 TestCase.__init__(case, *args) | |
| 249 if not case.problem.config.tester: | |
| 250 case.validator = None | |
| 251 else: | |
| 252 case.validator = case.problem.config.tester | |
| 253 | |
| 254 def validate(case, output): | |
| 255 if not case.validator: | |
| 256 # Compare the output with the reference output | |
| 257 case.open_outfile() | |
| 258 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
259 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
260 if refline is not None and not isinstance(refline, basestring): |
| 21 | 261 line = bytes(line, sys.getdefaultencoding()) |
| 262 if line != refline: | |
| 22 | 263 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
264 return 1 |
| 21 | 265 elif callable(case.validator): |
| 266 return case.validator(output) | |
| 267 else: | |
| 268 # Call the validator program | |
| 269 output.close() | |
| 23 | 270 if case.problem.config.ansname: |
| 271 case.open_outfile() | |
| 272 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
273 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
274 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
275 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
276 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 277 comment = case.process.communicate()[0].strip() |
| 26 | 278 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 279 if match: |
| 280 comment = comment[match.end():] | |
| 281 if not case.problem.config.maxexitcode: | |
| 282 if case.process.returncode: | |
| 283 raise WrongAnswer(comment) | |
| 284 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
285 return 1, comment |
| 21 | 286 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
287 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 288 |
| 289 | |
| 290 class BatchTestCase(ValidatedTestCase): | |
| 291 __slots__ = () | |
| 292 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
293 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
294 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
295 return (not case.problem.config.stdio or |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
296 case.validator and not callable(case.validator)) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
297 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
298 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
299 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
300 return case.validator and not callable(case.validator) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
301 |
| 22 | 302 def test(case, callback): |
| 21 | 303 case.open_infile() |
| 304 if case.problem.config.stdio: | |
| 54 | 305 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 306 # TODO: re-use the same file name if possible |
| 21 | 307 # FIXME: 2.5 lacks the delete parameter |
| 308 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 309 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
310 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 311 else: |
| 312 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
313 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
314 with contextmgr: |
|
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
315 with open(inputdatafname) as infile: |
| 83 | 316 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
317 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
318 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 22 | 319 raise NonZeroExitCode(case.process.returncode) |
| 320 callback() | |
| 321 case.has_called_back = True | |
| 322 outfile.seek(0) | |
| 323 return case.validate(outfile) | |
| 21 | 324 else: |
| 22 | 325 case.infile.copy(case.problem.config.inname) |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
326 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
327 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 328 raise NonZeroExitCode(case.process.returncode) |
| 22 | 329 callback() |
| 330 case.has_called_back = True | |
| 21 | 331 with open(case.problem.config.outname, 'rU') as output: |
| 332 return case.validate(output) | |
| 333 | |
| 334 | |
| 335 # This is the only test case type not executing any programs to be tested | |
| 336 class OutputOnlyTestCase(ValidatedTestCase): | |
| 337 __slots__ = () | |
| 338 def cleanup(case): pass | |
| 339 | |
| 340 class BestOutputTestCase(ValidatedTestCase): | |
| 341 __slots__ = () | |
| 342 | |
| 343 # This is the only test case type executing two programs simultaneously | |
| 344 class ReactiveTestCase(TestCase): | |
| 345 __slots__ = () | |
| 346 # The basic idea is to launch the program to be tested and the grader | |
| 347 # and to pipe their standard I/O from and to each other, | |
| 348 # and then to capture the grader's exit code and use it | |
| 26 | 349 # like the exit code of an output validator is used. |
| 21 | 350 |
| 351 | |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
352 class DummyTestContext(problem.TestGroup): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
353 __slots__ = () |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
354 def end(self): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
355 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
356 return 0, 0, self.log |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
357 |
| 21 | 358 def load_problem(prob, _types={'batch' : BatchTestCase, |
| 359 'outonly' : OutputOnlyTestCase, | |
| 360 'bestout' : BestOutputTestCase, | |
| 361 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
362 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
363 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
364 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
365 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
366 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
367 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
368 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
369 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
370 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
371 |
| 23 | 372 if options.legacy: |
| 373 prob.config.usegroups = False | |
| 58 | 374 newtests = [] |
| 23 | 375 for i, name in enumerate(prob.config.tests): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
376 # Same here; we'll need to iterate over them twice |
| 23 | 377 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
378 l = len(name) |
| 23 | 379 except Exception: |
| 380 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
381 name = tuple(name) |
| 23 | 382 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
383 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
384 l = len(name) |
| 58 | 385 if l > 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
386 prob.config.usegroups = True |
| 58 | 387 newtests.append(name) |
| 388 if prob.config.usegroups: | |
| 389 prob.config.tests = newtests | |
| 390 del newtests | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
391 |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
392 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
393 if isinstance(prob.config.pointmap, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
394 def getpoints(i, j, k=None): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
395 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
396 return prob.config.pointmap[i] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
397 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
398 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
399 return prob.config.pointmap[None] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
400 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
401 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
402 elif prob.config.usegroups: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
403 def getpoints(i, j, k): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
404 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
405 return prob.config.pointmap[k][j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
406 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
407 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
408 else: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
409 def getpoints(i, j): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
410 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
411 return prob.config.pointmap[j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
412 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
413 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
414 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
415 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
416 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
417 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
418 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
419 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 420 if prob.config.usegroups: |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
421 if not isinstance(prob.config.groupweight, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
422 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
423 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
424 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
425 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
426 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
89
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
427 if prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
428 yield DummyTestContext() |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
429 for i in prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
430 s = str(i).zfill(prob.config.paddummies) |
|
99
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
431 if (yield _types[prob.config.kind](prob, s, True, 0)): |
|
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
432 yield |
|
89
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
433 yield problem.test_context_end |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
434 for k, group in enumerate(prob.config.tests): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
435 if not group: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
436 continue |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
437 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
438 case_type = _types[prob.config.kind] |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
439 for j, i in enumerate(group): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
440 s = str(i).zfill(prob.config.padtests) |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
441 if not (yield case_type(prob, s, False, getpoints(i, j, k))): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
442 if options.skim: |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
443 case_type = SkippedTestCase |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
444 else: |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
445 yield |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
446 yield problem.test_context_end |
| 16 | 447 else: |
| 448 for i in prob.config.tests: | |
| 21 | 449 s = str(i).zfill(prob.config.padtests) |
| 450 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 451 for i in prob.config.dummies: | |
| 452 s = str(i).zfill(prob.config.paddummies) | |
|
99
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
453 if (yield _types[prob.config.kind](prob, s, True, 0)): |
|
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
454 yield |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
455 for j, i in enumerate(prob.config.tests): |
| 21 | 456 s = str(i).zfill(prob.config.padtests) |
|
99
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
457 if (yield _types[prob.config.kind](prob, s, False, getpoints(i, j))): |
|
9b41934b2e95
Fixed a major regression introduced in changeset 1fb319ec33af
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
458 yield |