Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/testcases.py @ 205:166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Callable validators now return three-tuples (number granted, bool correct,
str comment) instead of two-tuples (number granted, str comment). They are
still allowed to return single numbers.
Callable validators must now explicitly raise
upreckon.exceptions.WrongAnswer if they want the verdict to be Wrong
Answer rather than Partly Correct.
okexitcodemask specifies a bitmask ANDed with the exit code of the
external validator to get a boolean flag showing whether the answer is to
be marked as 'OK' rather than 'partly correct'. The bits covered by the
bitmask are reset to zeroes before devising the number of points granted
from the resulting number.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Wed, 17 Aug 2011 20:44:54 +0300 |
| parents | 00c80bba7f13 |
| children | 4edb6ef5a676 |
| rev | line source |
|---|---|
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 2 |
| 43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 4 | |
| 21 | 5 from __future__ import division, with_statement |
| 6 | |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
7 from .compat import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
8 from .exceptions import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
9 from . import files, config |
| 91 | 10 from __main__ import options |
| 21 | 11 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
12 import re, sys, tempfile |
| 21 | 13 from subprocess import Popen, PIPE, STDOUT |
| 14 | |
| 15 import os | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
16 devnull = open(os.path.devnull, 'w+b') |
| 21 | 17 |
|
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
18 class DummySignalIgnorer(object): |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
19 def __enter__(self): pass |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
20 def __exit__(self, exc_type, exc_value, traceback): pass |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
21 signal_ignorer = DummySignalIgnorer() |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
22 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
23 try: |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
24 from .win32 import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
25 except Exception: |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
26 from .unix import * |
| 21 | 27 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
28 __all__ = ('TestCase', 'SkippedTestCase', 'ValidatedTestCase', 'BatchTestCase', |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
29 'OutputOnlyTestCase') |
|
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
30 |
| 21 | 31 |
| 32 | |
| 22 | 33 # Helper context managers |
| 34 | |
| 35 class CopyDeleting(object): | |
| 36 __slots__ = 'case', 'file', 'name' | |
| 37 | |
| 38 def __init__(self, case, file, name): | |
| 39 self.case = case | |
| 40 self.file = file | |
| 41 self.name = name | |
| 42 | |
| 43 def __enter__(self): | |
| 44 if self.name: | |
| 45 try: | |
| 46 self.file.copy(self.name) | |
| 47 except: | |
| 48 try: | |
| 49 self.__exit__(None, None, None) | |
| 50 except: | |
| 51 pass | |
| 52 raise | |
| 53 | |
| 54 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 55 if self.name: | |
| 56 self.case.files_to_delete.append(self.name) | |
| 57 | |
| 58 | |
| 59 class Copying(object): | |
| 60 __slots__ = 'file', 'name' | |
| 61 | |
| 62 def __init__(self, file, name): | |
| 63 self.file = file | |
| 64 self.name = name | |
| 65 | |
| 66 def __enter__(self): | |
| 67 if self.name: | |
| 68 self.file.copy(self.name) | |
| 69 | |
| 70 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 71 pass | |
| 72 | |
| 73 | |
| 74 | |
| 21 | 75 # Test case types |
| 16 | 76 |
| 77 class TestCase(object): | |
| 21 | 78 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
79 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
80 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
81 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
82 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
83 'time_limit_string') |
|
106
aa0378ea1f93
Replaced constant properties with regular attributes
Oleg Oshmyan <chortos@inbox.lv>
parents:
105
diff
changeset
|
84 has_ansfile = has_iofiles = False |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
85 needs_realinname = True |
| 21 | 86 |
| 87 if ABCMeta: | |
| 88 __metaclass__ = ABCMeta | |
| 16 | 89 |
| 21 | 90 def __init__(case, prob, id, isdummy, points): |
| 16 | 91 case.problem = prob |
| 21 | 92 case.id = id |
| 93 case.isdummy = isdummy | |
| 94 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
95 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
96 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 97 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
98 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
99 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 100 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
101 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
102 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
103 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
104 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
105 case.wall_time_limit_string = '' |
| 21 | 106 if not isdummy: |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
107 if case.needs_realinname: |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
108 case.realinname = case.problem.config.testcaseinname |
| 21 | 109 case.realoutname = case.problem.config.testcaseoutname |
| 110 else: | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
111 if case.needs_realinname: |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
112 case.realinname = case.problem.config.dummyinname |
| 21 | 113 case.realoutname = case.problem.config.dummyoutname |
| 114 | |
| 115 @abstractmethod | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
116 def test(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
117 raise NotImplementedError |
| 16 | 118 |
| 22 | 119 def __call__(case, callback): |
| 120 case.has_called_back = False | |
| 121 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
122 case.time_limit_string = case.wall_time_limit_string |
| 21 | 123 try: |
| 22 | 124 return case.test(callback) |
| 21 | 125 finally: |
| 22 | 126 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
127 if getattr(case, 'time_started', None) is None: |
| 22 | 128 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
129 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 130 case.time_stopped = now |
| 131 if not case.has_called_back: | |
| 132 callback() | |
| 21 | 133 case.cleanup() |
| 134 | |
| 135 def cleanup(case): | |
|
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
136 # Note that native extensions clean up on their own |
|
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
137 # and never let this condition be satisfied |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
138 if getattr(case, 'process', None) and case.process.returncode is None: |
|
134
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
139 kill(case.process) |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
140 for name in case.files_to_delete: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
141 try: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
142 os.remove(name) |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
143 except OSError: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
144 # It can't be helped |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
145 pass |
| 21 | 146 |
| 147 def open_infile(case): | |
| 148 try: | |
| 149 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 150 except IOError: | |
| 151 e = sys.exc_info()[1] | |
| 152 raise CannotReadInputFile(e) | |
| 153 | |
| 154 def open_outfile(case): | |
| 155 try: | |
| 156 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 157 except IOError: | |
| 158 e = sys.exc_info()[1] | |
| 159 raise CannotReadAnswerFile(e) | |
| 160 | |
| 16 | 161 |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
162 class SkippedTestCase(TestCase): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
163 __slots__ = () |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
164 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
165 def test(case, callback): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
166 raise TestCaseSkipped |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
167 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
168 |
| 21 | 169 class ValidatedTestCase(TestCase): |
| 170 __slots__ = 'validator' | |
| 171 | |
| 172 def __init__(case, *args): | |
| 173 TestCase.__init__(case, *args) | |
| 174 if not case.problem.config.tester: | |
| 175 case.validator = None | |
| 176 else: | |
| 177 case.validator = case.problem.config.tester | |
| 178 | |
| 179 def validate(case, output): | |
| 180 if not case.validator: | |
| 181 # Compare the output with the reference output | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
182 buffer = refbuffer = crlfhalf = refcrlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
183 crlf = '\r\n'.encode('ascii') |
| 21 | 184 case.open_outfile() |
| 185 with case.outfile.open() as refoutput: | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
186 while True: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
187 data = output.read(4096 - len(buffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
188 refdata = refoutput.read(4096 - len(refbuffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
189 if not case.problem.config.binary: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
190 data, refdata = crlfhalf + data, refcrlfhalf + refdata |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
191 size, refsize = len(data), len(refdata) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
192 if data and data != crlfhalf and data[-1] == crlf[0]: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
193 size -= 1 |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
194 crlfhalf = data[-1:] |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
195 else: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
196 crlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
197 if refdata and refdata != refcrlfhalf and refdata[-1] == crlf[0]: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
198 refsize -= 1 |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
199 refcrlfhalf = refdata[-1:] |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
200 else: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
201 refcrlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
202 data = data[:size].replace(crlf, crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
203 data = data.replace(crlf[:1], crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
204 refdata = refdata[:refsize].replace(crlf, crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
205 refdata = refdata.replace(crlf[:1], crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
206 buffer += data |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
207 refbuffer += refdata |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
208 if not (buffer or refbuffer or crlfhalf or refcrlfhalf): |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
209 break |
|
180
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
210 elif not buffer and not crlfhalf or not refbuffer and not refcrlfhalf: |
|
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
211 raise WrongAnswer |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
212 size = min(len(buffer), len(refbuffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
213 if buffer[:size] != refbuffer[:size]: |
| 22 | 214 raise WrongAnswer |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
215 buffer, refbuffer = buffer[size:], refbuffer[size:] |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
216 return 1 |
| 21 | 217 elif callable(case.validator): |
| 218 return case.validator(output) | |
|
204
00c80bba7f13
Removed some long-unnoticed trailing whitespace
Oleg Oshmyan <chortos@inbox.lv>
parents:
200
diff
changeset
|
219 else: |
| 21 | 220 # Call the validator program |
| 221 output.close() | |
| 23 | 222 if case.problem.config.ansname: |
| 223 case.open_outfile() | |
| 224 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
225 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
226 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
227 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
228 raise CannotStartValidator(sys.exc_info()[1]) |
|
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
229 with signal_ignorer: |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
230 comment = case.process.communicate()[0].strip() |
| 26 | 231 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 232 if match: |
| 233 comment = comment[match.end():] | |
| 234 if not case.problem.config.maxexitcode: | |
| 235 if case.process.returncode: | |
| 236 raise WrongAnswer(comment) | |
| 237 else: | |
|
205
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
238 return 1, True, comment |
| 21 | 239 else: |
|
205
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
240 if case.problem.config.okexitcodeflag: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
241 correct = bool(case.process.returncode & case.problem.config.okexitcodeflag) |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
242 case.process.returncode &= ~case.problem.config.okexitcodeflag |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
243 else: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
244 correct = case.process.returncode >= case.problem.config.maxexitcode |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
245 if not correct and not case.process.returncode: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
246 raise WrongAnswer(comment) |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
247 return case.process.returncode / case.problem.config.maxexitcode, correct, comment |
| 21 | 248 |
| 249 | |
| 250 class BatchTestCase(ValidatedTestCase): | |
| 251 __slots__ = () | |
| 252 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
253 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
254 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
255 return (not case.problem.config.stdio or |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
256 case.validator and not callable(case.validator)) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
257 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
258 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
259 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
260 return case.validator and not callable(case.validator) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
261 |
| 22 | 262 def test(case, callback): |
| 21 | 263 case.open_infile() |
| 264 if case.problem.config.stdio: | |
| 54 | 265 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 266 # TODO: re-use the same file name if possible |
| 21 | 267 # FIXME: 2.5 lacks the delete parameter |
| 268 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 269 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
270 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 271 else: |
| 272 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
273 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
274 with contextmgr: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
275 with tempfile.TemporaryFile('w+b') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+b') as outfile: |
|
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
276 with open(inputdatafname) as infile: |
|
130
652028850ea4
Removed useless arguments to call() from testcases.BatchTestCase.test()
Oleg Oshmyan <chortos@inbox.lv>
parents:
128
diff
changeset
|
277 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull) |
|
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
278 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
|
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
279 raise NonZeroExitCode(case.process.returncode) |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
280 case.has_called_back = True |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
281 callback() |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
282 outfile.seek(0) |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
283 return case.validate(outfile) |
| 21 | 284 else: |
| 22 | 285 case.infile.copy(case.problem.config.inname) |
|
176
88e1e6786f67
Fixed crashing on stdio=False with _unix
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
286 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=devnull) |
|
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
287 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 288 raise NonZeroExitCode(case.process.returncode) |
|
103
4e6f231f055f
Fixed race condition resulting in calling back twice from TestCase.test
Oleg Oshmyan <chortos@inbox.lv>
parents:
99
diff
changeset
|
289 case.has_called_back = True |
| 22 | 290 callback() |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
291 try: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
292 output = open(case.problem.config.outname, 'rb') |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
293 except IOError: |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
294 raise CannotReadOutputFile(sys.exc_info()[1]) |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
295 with output as output: |
| 21 | 296 return case.validate(output) |
| 297 | |
| 298 | |
| 299 # This is the only test case type not executing any programs to be tested | |
| 300 class OutputOnlyTestCase(ValidatedTestCase): | |
| 301 __slots__ = () | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
302 needs_realinname = False |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
303 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
304 def cleanup(case): |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
305 pass |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
306 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
307 def test(case, callback): |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
308 case.time_stopped = case.time_started = 0 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
309 case.has_called_back = True |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
310 callback() |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
311 try: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
312 output = open(case.problem.config.outname.replace('$', case.id), 'rb') |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
313 except IOError: |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
314 raise CannotReadOutputFile(sys.exc_info()[1]) |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
315 with output as output: |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
316 return case.validate(output) |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
317 |
| 21 | 318 |
| 319 class BestOutputTestCase(ValidatedTestCase): | |
| 320 __slots__ = () | |
| 321 | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
322 |
| 21 | 323 # This is the only test case type executing two programs simultaneously |
| 324 class ReactiveTestCase(TestCase): | |
| 325 __slots__ = () | |
| 326 # The basic idea is to launch the program to be tested and the grader | |
| 327 # and to pipe their standard I/O from and to each other, | |
| 328 # and then to capture the grader's exit code and use it | |
| 26 | 329 # like the exit code of an output validator is used. |