Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/testcases.py @ 204:00c80bba7f13
Removed some long-unnoticed trailing whitespace
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Wed, 17 Aug 2011 18:48:21 +0300 |
parents | fa81289ee407 |
children | 166a23999bf7 |
rev | line source |
---|---|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
16 | 2 |
43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
4 | |
21 | 5 from __future__ import division, with_statement |
6 | |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
7 from .compat import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
8 from .exceptions import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
9 from . import files, config |
91 | 10 from __main__ import options |
21 | 11 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
12 import re, sys, tempfile |
21 | 13 from subprocess import Popen, PIPE, STDOUT |
14 | |
15 import os | |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
16 devnull = open(os.path.devnull, 'w+b') |
21 | 17 |
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
18 class DummySignalIgnorer(object): |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
19 def __enter__(self): pass |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
20 def __exit__(self, exc_type, exc_value, traceback): pass |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
21 signal_ignorer = DummySignalIgnorer() |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
22 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
23 try: |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
24 from .win32 import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
25 except Exception: |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
26 from .unix import * |
21 | 27 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
28 __all__ = ('TestCase', 'SkippedTestCase', 'ValidatedTestCase', 'BatchTestCase', |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
29 'OutputOnlyTestCase') |
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
30 |
21 | 31 |
32 | |
22 | 33 # Helper context managers |
34 | |
35 class CopyDeleting(object): | |
36 __slots__ = 'case', 'file', 'name' | |
37 | |
38 def __init__(self, case, file, name): | |
39 self.case = case | |
40 self.file = file | |
41 self.name = name | |
42 | |
43 def __enter__(self): | |
44 if self.name: | |
45 try: | |
46 self.file.copy(self.name) | |
47 except: | |
48 try: | |
49 self.__exit__(None, None, None) | |
50 except: | |
51 pass | |
52 raise | |
53 | |
54 def __exit__(self, exc_type, exc_val, exc_tb): | |
55 if self.name: | |
56 self.case.files_to_delete.append(self.name) | |
57 | |
58 | |
59 class Copying(object): | |
60 __slots__ = 'file', 'name' | |
61 | |
62 def __init__(self, file, name): | |
63 self.file = file | |
64 self.name = name | |
65 | |
66 def __enter__(self): | |
67 if self.name: | |
68 self.file.copy(self.name) | |
69 | |
70 def __exit__(self, exc_type, exc_val, exc_tb): | |
71 pass | |
72 | |
73 | |
74 | |
21 | 75 # Test case types |
16 | 76 |
77 class TestCase(object): | |
21 | 78 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
79 'process', 'time_started', 'time_stopped', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
80 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
81 'maxmemory', 'has_called_back', 'files_to_delete', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
82 'cpu_time_limit_string', 'wall_time_limit_string', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
83 'time_limit_string') |
106
aa0378ea1f93
Replaced constant properties with regular attributes
Oleg Oshmyan <chortos@inbox.lv>
parents:
105
diff
changeset
|
84 has_ansfile = has_iofiles = False |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
85 needs_realinname = True |
21 | 86 |
87 if ABCMeta: | |
88 __metaclass__ = ABCMeta | |
16 | 89 |
21 | 90 def __init__(case, prob, id, isdummy, points): |
16 | 91 case.problem = prob |
21 | 92 case.id = id |
93 case.isdummy = isdummy | |
94 case.points = points | |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
95 case.maxcputime = case.problem.config.maxcputime |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
96 case.maxwalltime = case.problem.config.maxwalltime |
21 | 97 case.maxmemory = case.problem.config.maxmemory |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
98 if case.maxcputime: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
99 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
21 | 100 else: |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
101 case.cpu_time_limit_string = '' |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
102 if case.maxwalltime: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
103 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
104 else: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
105 case.wall_time_limit_string = '' |
21 | 106 if not isdummy: |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
107 if case.needs_realinname: |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
108 case.realinname = case.problem.config.testcaseinname |
21 | 109 case.realoutname = case.problem.config.testcaseoutname |
110 else: | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
111 if case.needs_realinname: |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
112 case.realinname = case.problem.config.dummyinname |
21 | 113 case.realoutname = case.problem.config.dummyoutname |
114 | |
115 @abstractmethod | |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
116 def test(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
117 raise NotImplementedError |
16 | 118 |
22 | 119 def __call__(case, callback): |
120 case.has_called_back = False | |
121 case.files_to_delete = [] | |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
122 case.time_limit_string = case.wall_time_limit_string |
21 | 123 try: |
22 | 124 return case.test(callback) |
21 | 125 finally: |
22 | 126 now = clock() |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
127 if getattr(case, 'time_started', None) is None: |
22 | 128 case.time_started = case.time_stopped = now |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
129 elif getattr(case, 'time_stopped', None) is None: |
22 | 130 case.time_stopped = now |
131 if not case.has_called_back: | |
132 callback() | |
21 | 133 case.cleanup() |
134 | |
135 def cleanup(case): | |
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
136 # Note that native extensions clean up on their own |
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
137 # and never let this condition be satisfied |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
138 if getattr(case, 'process', None) and case.process.returncode is None: |
134
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
139 kill(case.process) |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
140 for name in case.files_to_delete: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
141 try: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
142 os.remove(name) |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
143 except OSError: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
144 # It can't be helped |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
145 pass |
21 | 146 |
147 def open_infile(case): | |
148 try: | |
149 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
150 except IOError: | |
151 e = sys.exc_info()[1] | |
152 raise CannotReadInputFile(e) | |
153 | |
154 def open_outfile(case): | |
155 try: | |
156 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
157 except IOError: | |
158 e = sys.exc_info()[1] | |
159 raise CannotReadAnswerFile(e) | |
160 | |
16 | 161 |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
162 class SkippedTestCase(TestCase): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
163 __slots__ = () |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
164 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
165 def test(case, callback): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
166 raise TestCaseSkipped |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
167 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
168 |
21 | 169 class ValidatedTestCase(TestCase): |
170 __slots__ = 'validator' | |
171 | |
172 def __init__(case, *args): | |
173 TestCase.__init__(case, *args) | |
174 if not case.problem.config.tester: | |
175 case.validator = None | |
176 else: | |
177 case.validator = case.problem.config.tester | |
178 | |
179 def validate(case, output): | |
180 if not case.validator: | |
181 # Compare the output with the reference output | |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
182 buffer = refbuffer = crlfhalf = refcrlfhalf = ''.encode() |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
183 crlf = '\r\n'.encode('ascii') |
21 | 184 case.open_outfile() |
185 with case.outfile.open() as refoutput: | |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
186 while True: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
187 data = output.read(4096 - len(buffer)) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
188 refdata = refoutput.read(4096 - len(refbuffer)) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
189 if not case.problem.config.binary: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
190 data, refdata = crlfhalf + data, refcrlfhalf + refdata |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
191 size, refsize = len(data), len(refdata) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
192 if data and data != crlfhalf and data[-1] == crlf[0]: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
193 size -= 1 |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
194 crlfhalf = data[-1:] |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
195 else: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
196 crlfhalf = ''.encode() |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
197 if refdata and refdata != refcrlfhalf and refdata[-1] == crlf[0]: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
198 refsize -= 1 |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
199 refcrlfhalf = refdata[-1:] |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
200 else: |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
201 refcrlfhalf = ''.encode() |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
202 data = data[:size].replace(crlf, crlf[1:]) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
203 data = data.replace(crlf[:1], crlf[1:]) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
204 refdata = refdata[:refsize].replace(crlf, crlf[1:]) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
205 refdata = refdata.replace(crlf[:1], crlf[1:]) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
206 buffer += data |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
207 refbuffer += refdata |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
208 if not (buffer or refbuffer or crlfhalf or refcrlfhalf): |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
209 break |
180
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
210 elif not buffer and not crlfhalf or not refbuffer and not refcrlfhalf: |
760d38ee86d6
Fixed hanging in the built-in output validator when output lengths differ
Oleg Oshmyan <chortos@inbox.lv>
parents:
176
diff
changeset
|
211 raise WrongAnswer |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
212 size = min(len(buffer), len(refbuffer)) |
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
213 if buffer[:size] != refbuffer[:size]: |
22 | 214 raise WrongAnswer |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
215 buffer, refbuffer = buffer[size:], refbuffer[size:] |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
216 return 1 |
21 | 217 elif callable(case.validator): |
218 return case.validator(output) | |
204
00c80bba7f13
Removed some long-unnoticed trailing whitespace
Oleg Oshmyan <chortos@inbox.lv>
parents:
200
diff
changeset
|
219 else: |
21 | 220 # Call the validator program |
221 output.close() | |
23 | 222 if case.problem.config.ansname: |
223 case.open_outfile() | |
224 case.outfile.copy(case.problem.config.ansname) | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
225 try: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
226 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
227 except OSError: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
228 raise CannotStartValidator(sys.exc_info()[1]) |
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
229 with signal_ignorer: |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
230 comment = case.process.communicate()[0].strip() |
26 | 231 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
21 | 232 if match: |
233 comment = comment[match.end():] | |
234 if not case.problem.config.maxexitcode: | |
235 if case.process.returncode: | |
236 raise WrongAnswer(comment) | |
237 else: | |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
238 return 1, comment |
21 | 239 else: |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
240 return case.process.returncode / case.problem.config.maxexitcode, comment |
21 | 241 |
242 | |
243 class BatchTestCase(ValidatedTestCase): | |
244 __slots__ = () | |
245 | |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
246 @property |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
247 def has_iofiles(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
248 return (not case.problem.config.stdio or |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
249 case.validator and not callable(case.validator)) |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
250 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
251 @property |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
252 def has_ansfile(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
253 return case.validator and not callable(case.validator) |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
254 |
22 | 255 def test(case, callback): |
21 | 256 case.open_infile() |
257 if case.problem.config.stdio: | |
54 | 258 if options.erase and not case.validator or not case.problem.config.inname: |
22 | 259 # TODO: re-use the same file name if possible |
21 | 260 # FIXME: 2.5 lacks the delete parameter |
261 with tempfile.NamedTemporaryFile(delete=False) as f: | |
22 | 262 inputdatafname = f.name |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
263 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
21 | 264 else: |
265 inputdatafname = case.problem.config.inname | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
266 contextmgr = Copying(case.infile, inputdatafname) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
267 with contextmgr: |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
268 with tempfile.TemporaryFile('w+b') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+b') as outfile: |
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
269 with open(inputdatafname) as infile: |
130
652028850ea4
Removed useless arguments to call() from testcases.BatchTestCase.test()
Oleg Oshmyan <chortos@inbox.lv>
parents:
128
diff
changeset
|
270 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull) |
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
271 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
163
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
272 raise NonZeroExitCode(case.process.returncode) |
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
273 case.has_called_back = True |
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
274 callback() |
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
275 outfile.seek(0) |
8198aa2ed20d
Some broken output validators on Win32 no longer get access-denied errors
Oleg Oshmyan <chortos@inbox.lv>
parents:
146
diff
changeset
|
276 return case.validate(outfile) |
21 | 277 else: |
22 | 278 case.infile.copy(case.problem.config.inname) |
176
88e1e6786f67
Fixed crashing on stdio=False with _unix
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
279 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=devnull) |
200
fa81289ee407
force_zero_exitcode is now a problem-specific configuration variable
Oleg Oshmyan <chortos@inbox.lv>
parents:
180
diff
changeset
|
280 if case.problem.config.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
21 | 281 raise NonZeroExitCode(case.process.returncode) |
103
4e6f231f055f
Fixed race condition resulting in calling back twice from TestCase.test
Oleg Oshmyan <chortos@inbox.lv>
parents:
99
diff
changeset
|
282 case.has_called_back = True |
22 | 283 callback() |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
284 try: |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
285 output = open(case.problem.config.outname, 'rb') |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
286 except IOError: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
287 raise CannotReadOutputFile(sys.exc_info()[1]) |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
288 with output as output: |
21 | 289 return case.validate(output) |
290 | |
291 | |
292 # This is the only test case type not executing any programs to be tested | |
293 class OutputOnlyTestCase(ValidatedTestCase): | |
294 __slots__ = () | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
295 needs_realinname = False |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
296 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
297 def cleanup(case): |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
298 pass |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
299 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
300 def test(case, callback): |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
301 case.time_stopped = case.time_started = 0 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
302 case.has_called_back = True |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
303 callback() |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
304 try: |
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
163
diff
changeset
|
305 output = open(case.problem.config.outname.replace('$', case.id), 'rb') |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
306 except IOError: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
307 raise CannotReadOutputFile(sys.exc_info()[1]) |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
308 with output as output: |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
309 return case.validate(output) |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
310 |
21 | 311 |
312 class BestOutputTestCase(ValidatedTestCase): | |
313 __slots__ = () | |
314 | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
315 |
21 | 316 # This is the only test case type executing two programs simultaneously |
317 class ReactiveTestCase(TestCase): | |
318 __slots__ = () | |
319 # The basic idea is to launch the program to be tested and the grader | |
320 # and to pipe their standard I/O from and to each other, | |
321 # and then to capture the grader's exit code and use it | |
26 | 322 # like the exit code of an output validator is used. |