Mercurial > ~astiob > upreckon > hgweb

comparison upreckon/testcases.py @ 146:d5b6708c1955

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Distutils support, reorganization and cleaning up * Removed command-line options -t and -u. * Reorganized code: o all modules are now in package upreckon; o TestCaseNotPassed and its descendants now live in a separate module exceptions; o load_problem now lives in module problem. * Commented out mentions of command-line option -c in --help. * Added a distutils-based setup.py.
author Oleg Oshmyan <chortos@inbox.lv>
date Sat, 28 May 2011 14:24:25 +0100
parents testcases.py@ed4035661b85
children 93bf6b333c99
comparison
equal deleted inserted replaced
145:d2c266c8d820 146:d5b6708c1955
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv>
2
3 # TODO: copy the ansfile if not options.erase even if no validator is used
4
5 from __future__ import division, with_statement
6
7 from .compat import *
8 from .exceptions import *
9 from . import files, config
10 from __main__ import options
11
12 import re, sys, tempfile
13 from subprocess import Popen, PIPE, STDOUT
14
15 import os
16 devnull = open(os.path.devnull, 'w+')
17
18 class DummySignalIgnorer(object):
19 def __enter__(self): pass
20 def __exit__(self, exc_type, exc_value, traceback): pass
21 signal_ignorer = DummySignalIgnorer()
22
23 try:
24 from .win32 import *
25 except Exception:
26 from .unix import *
27
28 __all__ = ('TestCase', 'SkippedTestCase', 'ValidatedTestCase', 'BatchTestCase',
29 'OutputOnlyTestCase')
30
31
32
33 # Helper context managers
34
35 class CopyDeleting(object):
36 __slots__ = 'case', 'file', 'name'
37
38 def __init__(self, case, file, name):
39 self.case = case
40 self.file = file
41 self.name = name
42
43 def __enter__(self):
44 if self.name:
45 try:
46 self.file.copy(self.name)
47 except:
48 try:
49 self.__exit__(None, None, None)
50 except:
51 pass
52 raise
53
54 def __exit__(self, exc_type, exc_val, exc_tb):
55 if self.name:
56 self.case.files_to_delete.append(self.name)
57
58
59 class Copying(object):
60 __slots__ = 'file', 'name'
61
62 def __init__(self, file, name):
63 self.file = file
64 self.name = name
65
66 def __enter__(self):
67 if self.name:
68 self.file.copy(self.name)
69
70 def __exit__(self, exc_type, exc_val, exc_tb):
71 pass
72
73
74
75 # Test case types
76
77 class TestCase(object):
78 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points',
79 'process', 'time_started', 'time_stopped',
80 'realinname', 'realoutname', 'maxcputime', 'maxwalltime',
81 'maxmemory', 'has_called_back', 'files_to_delete',
82 'cpu_time_limit_string', 'wall_time_limit_string',
83 'time_limit_string')
84 has_ansfile = has_iofiles = False
85 needs_realinname = True
86
87 if ABCMeta:
88 __metaclass__ = ABCMeta
89
90 def __init__(case, prob, id, isdummy, points):
91 case.problem = prob
92 case.id = id
93 case.isdummy = isdummy
94 case.points = points
95 case.maxcputime = case.problem.config.maxcputime
96 case.maxwalltime = case.problem.config.maxwalltime
97 case.maxmemory = case.problem.config.maxmemory
98 if case.maxcputime:
99 case.cpu_time_limit_string = '/%.3f' % case.maxcputime
100 else:
101 case.cpu_time_limit_string = ''
102 if case.maxwalltime:
103 case.wall_time_limit_string = '/%.3f' % case.maxwalltime
104 else:
105 case.wall_time_limit_string = ''
106 if not isdummy:
107 if case.needs_realinname:
108 case.realinname = case.problem.config.testcaseinname
109 case.realoutname = case.problem.config.testcaseoutname
110 else:
111 if case.needs_realinname:
112 case.realinname = case.problem.config.dummyinname
113 case.realoutname = case.problem.config.dummyoutname
114
115 @abstractmethod
116 def test(case):
117 raise NotImplementedError
118
119 def __call__(case, callback):
120 case.has_called_back = False
121 case.files_to_delete = []
122 case.time_limit_string = case.wall_time_limit_string
123 try:
124 return case.test(callback)
125 finally:
126 now = clock()
127 if getattr(case, 'time_started', None) is None:
128 case.time_started = case.time_stopped = now
129 elif getattr(case, 'time_stopped', None) is None:
130 case.time_stopped = now
131 if not case.has_called_back:
132 callback()
133 case.cleanup()
134
135 def cleanup(case):
136 # Note that native extensions clean up on their own
137 # and never let this condition be satisfied
138 if getattr(case, 'process', None) and case.process.returncode is None:
139 kill(case.process)
140 for name in case.files_to_delete:
141 try:
142 os.remove(name)
143 except OSError:
144 # It can't be helped
145 pass
146
147 def open_infile(case):
148 try:
149 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id))))
150 except IOError:
151 e = sys.exc_info()[1]
152 raise CannotReadInputFile(e)
153
154 def open_outfile(case):
155 try:
156 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id))))
157 except IOError:
158 e = sys.exc_info()[1]
159 raise CannotReadAnswerFile(e)
160
161
162 class SkippedTestCase(TestCase):
163 __slots__ = ()
164
165 def test(case, callback):
166 raise TestCaseSkipped
167
168
169 class ValidatedTestCase(TestCase):
170 __slots__ = 'validator'
171
172 def __init__(case, *args):
173 TestCase.__init__(case, *args)
174 if not case.problem.config.tester:
175 case.validator = None
176 else:
177 case.validator = case.problem.config.tester
178
179 def validate(case, output):
180 if not case.validator:
181 # Compare the output with the reference output
182 case.open_outfile()
183 with case.outfile.open() as refoutput:
184 for line, refline in zip_longest(output, refoutput):
185 if refline is not None and not isinstance(refline, basestring):
186 line = bytes(line, sys.getdefaultencoding())
187 if line != refline:
188 raise WrongAnswer
189 return 1
190 elif callable(case.validator):
191 return case.validator(output)
192 else:
193 # Call the validator program
194 output.close()
195 if case.problem.config.ansname:
196 case.open_outfile()
197 case.outfile.copy(case.problem.config.ansname)
198 try:
199 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1)
200 except OSError:
201 raise CannotStartValidator(sys.exc_info()[1])
202 with signal_ignorer:
203 comment = case.process.communicate()[0].strip()
204 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment)
205 if match:
206 comment = comment[match.end():]
207 if not case.problem.config.maxexitcode:
208 if case.process.returncode:
209 raise WrongAnswer(comment)
210 else:
211 return 1, comment
212 else:
213 return case.process.returncode / case.problem.config.maxexitcode, comment
214
215
216 class BatchTestCase(ValidatedTestCase):
217 __slots__ = ()
218
219 @property
220 def has_iofiles(case):
221 return (not case.problem.config.stdio or
222 case.validator and not callable(case.validator))
223
224 @property
225 def has_ansfile(case):
226 return case.validator and not callable(case.validator)
227
228 def test(case, callback):
229 case.open_infile()
230 if case.problem.config.stdio:
231 if options.erase and not case.validator or not case.problem.config.inname:
232 # TODO: re-use the same file name if possible
233 # FIXME: 2.5 lacks the delete parameter
234 with tempfile.NamedTemporaryFile(delete=False) as f:
235 inputdatafname = f.name
236 contextmgr = CopyDeleting(case, case.infile, inputdatafname)
237 else:
238 inputdatafname = case.problem.config.inname
239 contextmgr = Copying(case.infile, inputdatafname)
240 with contextmgr:
241 with open(inputdatafname) as infile:
242 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile:
243 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull)
244 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0:
245 raise NonZeroExitCode(case.process.returncode)
246 case.has_called_back = True
247 callback()
248 outfile.seek(0)
249 return case.validate(outfile)
250 else:
251 case.infile.copy(case.problem.config.inname)
252 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT)
253 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0:
254 raise NonZeroExitCode(case.process.returncode)
255 case.has_called_back = True
256 callback()
257 try:
258 output = open(case.problem.config.outname, 'rU')
259 except IOError:
260 raise CannotReadOutputFile(sys.exc_info()[1])
261 with output as output:
262 return case.validate(output)
263
264
265 # This is the only test case type not executing any programs to be tested
266 class OutputOnlyTestCase(ValidatedTestCase):
267 __slots__ = ()
268 needs_realinname = False
269
270 def cleanup(case):
271 pass
272
273 def test(case, callback):
274 case.time_stopped = case.time_started = 0
275 case.has_called_back = True
276 callback()
277 try:
278 output = open(case.problem.config.outname.replace('$', case.id), 'rU')
279 except IOError:
280 raise CannotReadOutputFile(sys.exc_info()[1])
281 with output as output:
282 return case.validate(output)
283
284
285 class BestOutputTestCase(ValidatedTestCase):
286 __slots__ = ()
287
288
289 # This is the only test case type executing two programs simultaneously
290 class ReactiveTestCase(TestCase):
291 __slots__ = ()
292 # The basic idea is to launch the program to be tested and the grader
293 # and to pipe their standard I/O from and to each other,
294 # and then to capture the grader's exit code and use it
295 # like the exit code of an output validator is used.