Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/testcases.py @ 208:ede78fbd509a
Revamped files.File
This changes (simplifies) file search path:
* when searching for testconf, .../testconf.py is now given precendence
over .../tests/testconf.py;
* when searching inside archives, a/tests/b/c is now given precedence
over a/c.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Thu, 18 Aug 2011 02:20:24 +0300 |
| parents | 4edb6ef5a676 |
| children | ce1285728952 |
| rev | line source |
|---|---|
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 2 |
| 43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 4 | |
| 21 | 5 from __future__ import division, with_statement |
| 6 | |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
7 from .compat import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
8 from .exceptions import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
9 from . import files, config |
| 91 | 10 from __main__ import options |
| 21 | 11 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
12 import re, sys, tempfile |
| 21 | 13 from subprocess import Popen, PIPE, STDOUT |
| 14 | |
| 15 import os | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
16 devnull = open(os.path.devnull, 'w+b') |
| 21 | 17 |
|
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
18 class DummySignalIgnorer(object): |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
19 def __enter__(self): pass |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
20 def __exit__(self, exc_type, exc_value, traceback): pass |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
21 signal_ignorer = DummySignalIgnorer() |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
22 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
23 try: |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
24 from .win32 import * |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
25 except Exception: |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
26 from .unix import * |
| 21 | 27 |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
28 __all__ = ('TestCase', 'SkippedTestCase', 'ValidatedTestCase', 'BatchTestCase', |
|
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
29 'OutputOnlyTestCase') |
|
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
30 |
| 21 | 31 |
| 32 | |
| 22 | 33 # Helper context managers |
| 34 | |
| 35 class CopyDeleting(object): | |
| 36 __slots__ = 'case', 'file', 'name' | |
| 37 | |
| 38 def __init__(self, case, file, name): | |
| 39 self.case = case | |
| 40 self.file = file | |
| 41 self.name = name | |
| 42 | |
| 43 def __enter__(self): | |
| 44 if self.name: | |
| 45 try: | |
| 46 self.file.copy(self.name) | |
| 47 except: | |
| 48 try: | |
| 49 self.__exit__(None, None, None) | |
| 50 except: | |
| 51 pass | |
| 52 raise | |
| 53 | |
| 54 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 55 if self.name: | |
| 56 self.case.files_to_delete.append(self.name) | |
| 57 | |
| 58 | |
| 59 class Copying(object): | |
| 60 __slots__ = 'file', 'name' | |
| 61 | |
| 62 def __init__(self, file, name): | |
| 63 self.file = file | |
| 64 self.name = name | |
| 65 | |
| 66 def __enter__(self): | |
| 67 if self.name: | |
| 68 self.file.copy(self.name) | |
| 69 | |
| 70 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 71 pass | |
| 72 | |
| 73 | |
| 74 | |
| 21 | 75 # Test case types |
| 16 | 76 |
| 77 class TestCase(object): | |
| 21 | 78 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
79 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
80 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
81 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
82 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
83 'time_limit_string') |
|
106
aa0378ea1f93
Replaced constant properties with regular attributes
Oleg Oshmyan <chortos@inbox.lv>
parents:
105
diff
changeset
|
84 has_ansfile = has_iofiles = False |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
85 needs_realinname = True |
| 21 | 86 |
| 87 if ABCMeta: | |
| 88 __metaclass__ = ABCMeta | |
| 16 | 89 |
| 21 | 90 def __init__(case, prob, id, isdummy, points): |
| 16 | 91 case.problem = prob |
| 21 | 92 case.id = id |
| 93 case.isdummy = isdummy | |
| 94 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
95 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
96 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 97 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
98 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
99 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 100 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
101 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
102 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
103 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
104 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
105 case.wall_time_limit_string = '' |
| 21 | 106 if not isdummy: |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
107 if case.needs_realinname: |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
108 case.realinname = case.problem.config.testcaseinname |
| 21 | 109 case.realoutname = case.problem.config.testcaseoutname |
| 110 else: | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
111 if case.needs_realinname: |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
112 case.realinname = case.problem.config.dummyinname |
| 21 | 113 case.realoutname = case.problem.config.dummyoutname |
| 114 | |
| 115 @abstractmethod | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
116 def test(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
117 raise NotImplementedError |
| 16 | 118 |
| 22 | 119 def __call__(case, callback): |
| 120 case.has_called_back = False | |
| 121 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
122 case.time_limit_string = case.wall_time_limit_string |
| 21 | 123 try: |
| 22 | 124 return case.test(callback) |
| 21 | 125 finally: |
| 22 | 126 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
127 if getattr(case, 'time_started', None) is None: |
| 22 | 128 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
129 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 130 case.time_stopped = now |
| 131 if not case.has_called_back: | |
| 132 callback() | |
| 21 | 133 case.cleanup() |
| 134 | |
| 135 def cleanup(case): | |
|
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
136 # Note that native extensions clean up on their own |
|
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
137 # and never let this condition be satisfied |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
138 if getattr(case, 'process', None) and case.process.returncode is None: |
|
134
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
139 kill(case.process) |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
140 for name in case.files_to_delete: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
141 try: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
142 os.remove(name) |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
143 except OSError: |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
144 # It can't be helped |
|
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
145 pass |
| 21 | 146 |
| 147 def open_infile(case): | |
| 148 try: | |
| 208 | 149 case.infile = files.File.from_virtual_path('/'.join((case.problem.name, case.realinname.replace('$', case.id))), False, 'test data') |
| 21 | 150 except IOError: |
| 151 e = sys.exc_info()[1] | |
| 152 raise CannotReadInputFile(e) | |
| 153 | |
| 154 def open_outfile(case): | |
| 155 try: | |
| 208 | 156 case.outfile = files.File.from_virtual_path('/'.join((case.problem.name, case.realoutname.replace('$', case.id))), False, 'test data') |
| 21 | 157 except IOError: |
| 158 e = sys.exc_info()[1] | |
| 159 raise CannotReadAnswerFile(e) | |
| 160 | |
| 16 | 161 |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
162 class SkippedTestCase(TestCase): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
163 __slots__ = () |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
164 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
165 def test(case, callback): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
166 raise TestCaseSkipped |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
167 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
168 |
| 21 | 169 class ValidatedTestCase(TestCase): |
| 170 __slots__ = 'validator' | |
| 171 | |
| 172 def __init__(case, *args): | |
| 173 TestCase.__init__(case, *args) | |
| 174 if not case.problem.config.tester: | |
| 175 case.validator = None | |
| 176 else: | |
| 177 case.validator = case.problem.config.tester | |
| 178 | |
| 179 def validate(case, output): | |
| 180 if not case.validator: | |
| 181 # Compare the output with the reference output | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
182 buffer = refbuffer = crlfhalf = refcrlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
183 crlf = '\r\n'.encode('ascii') |
| 21 | 184 case.open_outfile() |
| 185 with case.outfile.open() as refoutput: | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
186 while True: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
187 data = output.read(4096 - len(buffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
188 refdata = refoutput.read(4096 - len(refbuffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
189 if not case.problem.config.binary: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
190 data, refdata = crlfhalf + data, refcrlfhalf + refdata |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
191 size, refsize = len(data), len(refdata) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
192 if data and data != crlfhalf and data[-1] == crlf[0]: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
193 size -= 1 |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
194 crlfhalf = data[-1:] |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
195 else: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
196 crlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
197 if refdata and refdata != refcrlfhalf and refdata[-1] == crlf[0]: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
198 refsize -= 1 |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
199 refcrlfhalf = refdata[-1:] |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
200 else: |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
201 refcrlfhalf = ''.encode() |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
202 data = data[:size].replace(crlf, crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
203 data = data.replace(crlf[:1], crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
204 refdata = refdata[:refsize].replace(crlf, crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
205 refdata = refdata.replace(crlf[:1], crlf[1:]) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
206 buffer += data |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
207 refbuffer += refdata |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
208 if not (buffer or refbuffer or crlfhalf or refcrlfhalf): |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
209 break |
|
180
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
210 elif not buffer and not crlfhalf or not refbuffer and not refcrlfhalf: |
|
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
211 raise WrongAnswer |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
212 size = min(len(buffer), len(refbuffer)) |
|
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
213 if buffer[:size] != refbuffer[:size]: |
| 22 | 214 raise WrongAnswer |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
215 buffer, refbuffer = buffer[size:], refbuffer[size:] |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
216 return 1 |
| 21 | 217 elif callable(case.validator): |
| 218 return case.validator(output) | |
|
204
00c80bba7f13
Removed some long-unnoticed trailing whitespace
Oleg Oshmyan <chortos@inbox.lv>
parents:
200
diff
changeset
|
219 else: |
| 21 | 220 # Call the validator program |
| 221 output.close() | |
| 23 | 222 if case.problem.config.ansname: |
| 223 case.open_outfile() | |
| 224 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
225 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
226 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
227 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
228 raise CannotStartValidator(sys.exc_info()[1]) |
|
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
229 with signal_ignorer: |
|
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
230 comment = case.process.communicate()[0].strip() |
|
206
4edb6ef5a676
'Incorrect' is now stripped from external validator output
Oleg Oshmyan <chortos@inbox.lv>
parents:
205
diff
changeset
|
231 match = re.match(r'(?i)(?:ok|(?:(?:in)?correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 232 if match: |
| 233 comment = comment[match.end():] | |
| 234 if not case.problem.config.maxexitcode: | |
| 235 if case.process.returncode: | |
| 236 raise WrongAnswer(comment) | |
| 237 else: | |
|
205
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
238 return 1, True, comment |
| 21 | 239 else: |
|
205
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
240 if case.problem.config.okexitcodeflag: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
241 correct = bool(case.process.returncode & case.problem.config.okexitcodeflag) |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
242 case.process.returncode &= ~case.problem.config.okexitcodeflag |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
243 else: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
244 correct = case.process.returncode >= case.problem.config.maxexitcode |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
245 if not correct and not case.process.returncode: |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
246 raise WrongAnswer(comment) |
|
166a23999bf7
Added confvar okexitcodemask; changed the validator protocol
Oleg Oshmyan <chortos@inbox.lv>
parents:
204
diff
changeset
|
247 return case.process.returncode / case.problem.config.maxexitcode, correct, comment |
| 21 | 248 |
| 249 | |
| 250 class BatchTestCase(ValidatedTestCase): | |
| 251 __slots__ = () | |
| 252 | |
|
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
253 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
254 def has_iofiles(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
255 return (not case.problem.config.stdio or |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
256 case.validator and not callable(case.validator)) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
257 |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
258 @property |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
259 def has_ansfile(case): |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
260 return case.validator and not callable(case.validator) |
|
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
261 |
| 22 | 262 def test(case, callback): |
| 21 | 263 case.open_infile() |
| 264 if case.problem.config.stdio: | |
| 54 | 265 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 266 # TODO: re-use the same file name if possible |
| 21 | 267 # FIXME: 2.5 lacks the delete parameter |
| 268 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 269 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
270 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 271 else: |
| 272 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
273 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
274 with contextmgr: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
275 with tempfile.TemporaryFile('w+b') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+b') as outfile: |
|
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
276 with open(inputdatafname) as infile: |
|
130
652028850ea4
Removed useless arguments to call() from testcases.BatchTestCase.test()
Oleg Oshmyan <chortos@inbox.lv>
parents:
128
diff
changeset
|
277 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull) |
|
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
278 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
|
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
279 raise NonZeroExitCode(case.process.returncode) |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
280 case.has_called_back = True |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
281 callback() |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
282 outfile.seek(0) |
|
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
283 return case.validate(outfile) |
| 21 | 284 else: |
| 22 | 285 case.infile.copy(case.problem.config.inname) |
|
176
88e1e6786f67
Fixed crashing on stdio=False with _unix
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
286 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=devnull) |
|
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
287 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 288 raise NonZeroExitCode(case.process.returncode) |
|
103
4e6f231f055f
Fixed race condition resulting in calling back twice from TestCase.test
Oleg Oshmyan <chortos@inbox.lv>
parents:
99
diff
changeset
|
289 case.has_called_back = True |
| 22 | 290 callback() |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
291 try: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
292 output = open(case.problem.config.outname, 'rb') |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
293 except IOError: |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
294 raise CannotReadOutputFile(sys.exc_info()[1]) |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
295 with output as output: |
| 21 | 296 return case.validate(output) |
| 297 | |
| 298 | |
| 299 # This is the only test case type not executing any programs to be tested | |
| 300 class OutputOnlyTestCase(ValidatedTestCase): | |
| 301 __slots__ = () | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
302 needs_realinname = False |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
303 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
304 def cleanup(case): |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
305 pass |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
306 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
307 def test(case, callback): |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
308 case.time_stopped = case.time_started = 0 |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
309 case.has_called_back = True |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
310 callback() |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
311 try: |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
312 output = open(case.problem.config.outname.replace('$', case.id), 'rb') |
|
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
313 except IOError: |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
314 raise CannotReadOutputFile(sys.exc_info()[1]) |
|
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
315 with output as output: |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
316 return case.validate(output) |
|
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
317 |
| 21 | 318 |
| 319 class BestOutputTestCase(ValidatedTestCase): | |
| 320 __slots__ = () | |
| 321 | |
|
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
322 |
| 21 | 323 # This is the only test case type executing two programs simultaneously |
| 324 class ReactiveTestCase(TestCase): | |
| 325 __slots__ = () | |
| 326 # The basic idea is to launch the program to be tested and the grader | |
| 327 # and to pipe their standard I/O from and to each other, | |
| 328 # and then to capture the grader's exit code and use it | |
| 26 | 329 # like the exit code of an output validator is used. |