Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/testcases.py @ 146:d5b6708c1955
Distutils support, reorganization and cleaning up
* Removed command-line options -t and -u.
* Reorganized code:
o all modules are now in package upreckon;
o TestCaseNotPassed and its descendants now live in a separate
module exceptions;
o load_problem now lives in module problem.
* Commented out mentions of command-line option -c in --help.
* Added a distutils-based setup.py.
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Sat, 28 May 2011 14:24:25 +0100 |
parents | testcases.py@ed4035661b85 |
children | 93bf6b333c99 |
rev | line source |
---|---|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
16 | 2 |
43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
4 | |
21 | 5 from __future__ import division, with_statement |
6 | |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
7 from .compat import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
8 from .exceptions import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
9 from . import files, config |
91 | 10 from __main__ import options |
21 | 11 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
12 import re, sys, tempfile |
21 | 13 from subprocess import Popen, PIPE, STDOUT |
14 | |
15 import os | |
16 devnull = open(os.path.devnull, 'w+') | |
17 | |
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
18 class DummySignalIgnorer(object): |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
19 def __enter__(self): pass |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
20 def __exit__(self, exc_type, exc_value, traceback): pass |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
21 signal_ignorer = DummySignalIgnorer() |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
22 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
23 try: |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
24 from .win32 import * |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
25 except Exception: |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
26 from .unix import * |
21 | 27 |
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
28 __all__ = ('TestCase', 'SkippedTestCase', 'ValidatedTestCase', 'BatchTestCase', |
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
136
diff
changeset
|
29 'OutputOnlyTestCase') |
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
30 |
21 | 31 |
32 | |
22 | 33 # Helper context managers |
34 | |
35 class CopyDeleting(object): | |
36 __slots__ = 'case', 'file', 'name' | |
37 | |
38 def __init__(self, case, file, name): | |
39 self.case = case | |
40 self.file = file | |
41 self.name = name | |
42 | |
43 def __enter__(self): | |
44 if self.name: | |
45 try: | |
46 self.file.copy(self.name) | |
47 except: | |
48 try: | |
49 self.__exit__(None, None, None) | |
50 except: | |
51 pass | |
52 raise | |
53 | |
54 def __exit__(self, exc_type, exc_val, exc_tb): | |
55 if self.name: | |
56 self.case.files_to_delete.append(self.name) | |
57 | |
58 | |
59 class Copying(object): | |
60 __slots__ = 'file', 'name' | |
61 | |
62 def __init__(self, file, name): | |
63 self.file = file | |
64 self.name = name | |
65 | |
66 def __enter__(self): | |
67 if self.name: | |
68 self.file.copy(self.name) | |
69 | |
70 def __exit__(self, exc_type, exc_val, exc_tb): | |
71 pass | |
72 | |
73 | |
74 | |
21 | 75 # Test case types |
16 | 76 |
77 class TestCase(object): | |
21 | 78 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
79 'process', 'time_started', 'time_stopped', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
80 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
81 'maxmemory', 'has_called_back', 'files_to_delete', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
82 'cpu_time_limit_string', 'wall_time_limit_string', |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
83 'time_limit_string') |
106
aa0378ea1f93
Replaced constant properties with regular attributes
Oleg Oshmyan <chortos@inbox.lv>
parents:
105
diff
changeset
|
84 has_ansfile = has_iofiles = False |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
85 needs_realinname = True |
21 | 86 |
87 if ABCMeta: | |
88 __metaclass__ = ABCMeta | |
16 | 89 |
21 | 90 def __init__(case, prob, id, isdummy, points): |
16 | 91 case.problem = prob |
21 | 92 case.id = id |
93 case.isdummy = isdummy | |
94 case.points = points | |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
95 case.maxcputime = case.problem.config.maxcputime |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
96 case.maxwalltime = case.problem.config.maxwalltime |
21 | 97 case.maxmemory = case.problem.config.maxmemory |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
98 if case.maxcputime: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
99 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
21 | 100 else: |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
101 case.cpu_time_limit_string = '' |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
102 if case.maxwalltime: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
103 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
104 else: |
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
105 case.wall_time_limit_string = '' |
21 | 106 if not isdummy: |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
107 if case.needs_realinname: |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
108 case.realinname = case.problem.config.testcaseinname |
21 | 109 case.realoutname = case.problem.config.testcaseoutname |
110 else: | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
111 if case.needs_realinname: |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
112 case.realinname = case.problem.config.dummyinname |
21 | 113 case.realoutname = case.problem.config.dummyoutname |
114 | |
115 @abstractmethod | |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
116 def test(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
117 raise NotImplementedError |
16 | 118 |
22 | 119 def __call__(case, callback): |
120 case.has_called_back = False | |
121 case.files_to_delete = [] | |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
122 case.time_limit_string = case.wall_time_limit_string |
21 | 123 try: |
22 | 124 return case.test(callback) |
21 | 125 finally: |
22 | 126 now = clock() |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
127 if getattr(case, 'time_started', None) is None: |
22 | 128 case.time_started = case.time_stopped = now |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
129 elif getattr(case, 'time_stopped', None) is None: |
22 | 130 case.time_stopped = now |
131 if not case.has_called_back: | |
132 callback() | |
21 | 133 case.cleanup() |
134 | |
135 def cleanup(case): | |
136
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
136 # Note that native extensions clean up on their own |
ed4035661b85
Added a C implementation of the unix module (called _unix)
Oleg Oshmyan <chortos@inbox.lv>
parents:
134
diff
changeset
|
137 # and never let this condition be satisfied |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
138 if getattr(case, 'process', None) and case.process.returncode is None: |
134
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
139 kill(case.process) |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
140 for name in case.files_to_delete: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
141 try: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
142 os.remove(name) |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
143 except OSError: |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
144 # It can't be helped |
e84f33a60a5c
Moved process killing logic into platform-specific modules
Oleg Oshmyan <chortos@inbox.lv>
parents:
130
diff
changeset
|
145 pass |
21 | 146 |
147 def open_infile(case): | |
148 try: | |
149 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
150 except IOError: | |
151 e = sys.exc_info()[1] | |
152 raise CannotReadInputFile(e) | |
153 | |
154 def open_outfile(case): | |
155 try: | |
156 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
157 except IOError: | |
158 e = sys.exc_info()[1] | |
159 raise CannotReadAnswerFile(e) | |
160 | |
16 | 161 |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
162 class SkippedTestCase(TestCase): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
163 __slots__ = () |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
164 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
165 def test(case, callback): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
166 raise TestCaseSkipped |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
167 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
168 |
21 | 169 class ValidatedTestCase(TestCase): |
170 __slots__ = 'validator' | |
171 | |
172 def __init__(case, *args): | |
173 TestCase.__init__(case, *args) | |
174 if not case.problem.config.tester: | |
175 case.validator = None | |
176 else: | |
177 case.validator = case.problem.config.tester | |
178 | |
179 def validate(case, output): | |
180 if not case.validator: | |
181 # Compare the output with the reference output | |
182 case.open_outfile() | |
183 with case.outfile.open() as refoutput: | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
184 for line, refline in zip_longest(output, refoutput): |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
185 if refline is not None and not isinstance(refline, basestring): |
21 | 186 line = bytes(line, sys.getdefaultencoding()) |
187 if line != refline: | |
22 | 188 raise WrongAnswer |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
189 return 1 |
21 | 190 elif callable(case.validator): |
191 return case.validator(output) | |
192 else: | |
193 # Call the validator program | |
194 output.close() | |
23 | 195 if case.problem.config.ansname: |
196 case.open_outfile() | |
197 case.outfile.copy(case.problem.config.ansname) | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
198 try: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
199 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
200 except OSError: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
201 raise CannotStartValidator(sys.exc_info()[1]) |
108
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
202 with signal_ignorer: |
218b8c28549c
Fixed a crash due to SIGCHLD interrupting validator output pipe reads
Oleg Oshmyan <chortos@inbox.lv>
parents:
106
diff
changeset
|
203 comment = case.process.communicate()[0].strip() |
26 | 204 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
21 | 205 if match: |
206 comment = comment[match.end():] | |
207 if not case.problem.config.maxexitcode: | |
208 if case.process.returncode: | |
209 raise WrongAnswer(comment) | |
210 else: | |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
211 return 1, comment |
21 | 212 else: |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
213 return case.process.returncode / case.problem.config.maxexitcode, comment |
21 | 214 |
215 | |
216 class BatchTestCase(ValidatedTestCase): | |
217 __slots__ = () | |
218 | |
90
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
219 @property |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
220 def has_iofiles(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
221 return (not case.problem.config.stdio or |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
222 case.validator and not callable(case.validator)) |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
223 |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
224 @property |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
225 def has_ansfile(case): |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
226 return case.validator and not callable(case.validator) |
1fb319ec33af
Skimming mode added (-k/--skim option)
Oleg Oshmyan <chortos@inbox.lv>
parents:
89
diff
changeset
|
227 |
22 | 228 def test(case, callback): |
21 | 229 case.open_infile() |
230 if case.problem.config.stdio: | |
54 | 231 if options.erase and not case.validator or not case.problem.config.inname: |
22 | 232 # TODO: re-use the same file name if possible |
21 | 233 # FIXME: 2.5 lacks the delete parameter |
234 with tempfile.NamedTemporaryFile(delete=False) as f: | |
22 | 235 inputdatafname = f.name |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
236 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
21 | 237 else: |
238 inputdatafname = case.problem.config.inname | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
239 contextmgr = Copying(case.infile, inputdatafname) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
240 with contextmgr: |
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
241 with open(inputdatafname) as infile: |
83 | 242 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile: |
130
652028850ea4
Removed useless arguments to call() from testcases.BatchTestCase.test()
Oleg Oshmyan <chortos@inbox.lv>
parents:
128
diff
changeset
|
243 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull) |
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
244 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
22 | 245 raise NonZeroExitCode(case.process.returncode) |
103
4e6f231f055f
Fixed race condition resulting in calling back twice from TestCase.test
Oleg Oshmyan <chortos@inbox.lv>
parents:
99
diff
changeset
|
246 case.has_called_back = True |
22 | 247 callback() |
248 outfile.seek(0) | |
249 return case.validate(outfile) | |
21 | 250 else: |
22 | 251 case.infile.copy(case.problem.config.inname) |
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
252 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
253 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
21 | 254 raise NonZeroExitCode(case.process.returncode) |
103
4e6f231f055f
Fixed race condition resulting in calling back twice from TestCase.test
Oleg Oshmyan <chortos@inbox.lv>
parents:
99
diff
changeset
|
255 case.has_called_back = True |
22 | 256 callback() |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
257 try: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
258 output = open(case.problem.config.outname, 'rU') |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
259 except IOError: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
260 raise CannotReadOutputFile(sys.exc_info()[1]) |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
261 with output as output: |
21 | 262 return case.validate(output) |
263 | |
264 | |
265 # This is the only test case type not executing any programs to be tested | |
266 class OutputOnlyTestCase(ValidatedTestCase): | |
267 __slots__ = () | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
268 needs_realinname = False |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
269 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
270 def cleanup(case): |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
271 pass |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
272 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
273 def test(case, callback): |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
274 case.time_stopped = case.time_started = 0 |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
275 case.has_called_back = True |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
276 callback() |
105
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
277 try: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
278 output = open(case.problem.config.outname.replace('$', case.id), 'rU') |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
279 except IOError: |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
280 raise CannotReadOutputFile(sys.exc_info()[1]) |
9f922b11c98a
Absent output files no longer crash Upreckon
Oleg Oshmyan <chortos@inbox.lv>
parents:
104
diff
changeset
|
281 with output as output: |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
282 return case.validate(output) |
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
283 |
21 | 284 |
285 class BestOutputTestCase(ValidatedTestCase): | |
286 __slots__ = () | |
287 | |
104
8f46e84922f9
Output-only problems are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
103
diff
changeset
|
288 |
21 | 289 # This is the only test case type executing two programs simultaneously |
290 class ReactiveTestCase(TestCase): | |
291 __slots__ = () | |
292 # The basic idea is to launch the program to be tested and the grader | |
293 # and to pipe their standard I/O from and to each other, | |
294 # and then to capture the grader's exit code and use it | |
26 | 295 # like the exit code of an output validator is used. |