Mercurial > ~astiob > upreckon > hgweb
annotate test.py @ 9:ed90b375d197
Award maxexitcode points by default
The default number of points allocated per test case is now maxexitcode if it is set and is an integer (so that all given points are integers as well).
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Sun, 14 Feb 2010 00:51:27 +0000 |
| parents | b0034b18f942 |
| children | c87ec78f1fae |
| rev | line source |
|---|---|
| 0 | 1 #! /usr/bin/python |
| 3 | 2 # Copyright (c) 2009-2010 Chortos-2 <chortos@inbox.lv> |
| 0 | 3 |
| 4 import os, sys, shutil, time, subprocess, filecmp, optparse, signal, tempfile, tarfile, zipfile | |
| 5 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
6 parser = optparse.OptionParser(version='test.py 1.21.0 (SVN)', usage='usage: %prog [options] [problem names] [[path' + os.path.sep + 'to' + os.path.sep + ']solution-app] [test case numbers]\n\nTest case numbers can be specified in plain text or as a Python expression\nif there is only one positional argument.\n\nOnly problem names listed in testconf.py are recognized.') |
| 0 | 7 parser.add_option('-e', '--exclude', dest='exclude', action='append', help='test case number(s) to exclude, as a Python expression; multiple -e options can be supplied') |
| 8 parser.add_option('-c', '--cleanup', dest='clean', action='store_true', default=False, help='delete the copies of input/output files and exit') | |
| 9 parser.add_option('-s', '--save-io', dest='erase', action='store_false', default=True, help='do not delete the copies of input/output files after the last test case; create copies of input files and store output in files even if the solution uses standard I/O; delete the stored input/output files if the solution uses standard I/O and the -c/--cleanup option is specified') | |
| 4 | 10 parser.add_option('-m', '--copy-io', dest='copyonly', action='store_true', default=False, help='only create a copy of the input/output files of the last test case for manual testing; to delete them, use options -cs or -cm') |
| 0 | 11 parser.add_option('-x', '--auto-exit', dest='pause', action='store_false', default=True, help='do not wait for a key to be pressed when finished testing') |
| 12 parser.add_option('-p', '--python', action='store_true', default=False, help='always parse all positional arguments as a single Python expression (including the first argument even if it names an executable file)') | |
| 13 parser.add_option('-t', '--detect-time', dest='autotime', action='store_true', default=False, help='spend a second detecting the most precise time measurement function') | |
| 14 | |
| 15 options, args = parser.parse_args() | |
| 16 parser.destroy() | |
| 17 del parser | |
| 18 | |
| 19 globals1 = set(globals()) | |
| 20 | |
| 21 # Initialize some configuration variables with default values | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
22 tasknames = (os.path.curdir,) |
| 0 | 23 maxtime = 0 |
| 24 tests = () | |
| 25 dummies = () | |
| 26 testsexcluded = () | |
| 27 padwithzeroestolength = 0 | |
| 28 taskweight = 100 | |
| 29 pointmap = {} | |
| 30 stdio = False | |
| 31 dummyinname = '' | |
| 32 dummyoutname = '' | |
| 33 tester = '' | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
34 maxexitcode = 0 |
| 0 | 35 |
| 36 def exectestconf_helper(name): | |
| 37 if os.path.isfile('tests.tar'): | |
| 38 f = tarfile.open('tests.tar') | |
| 39 try: | |
| 40 exec f.extractfile(name).read() in globals() | |
| 41 f.close() | |
| 42 return True | |
| 43 except KeyError: | |
| 44 f.close() | |
| 45 if os.path.isfile('tests.zip'): | |
| 46 f = zipfile.ZipFile('tests.zip') | |
| 47 try: | |
| 48 exec f.open(name, 'rU').read() in globals() | |
| 49 f.close() | |
| 50 return True | |
| 51 except KeyError: | |
| 52 f.close() | |
| 53 if os.path.isfile('tests.tgz'): | |
| 54 f = tarfile.open('tests.tgz') | |
| 55 try: | |
| 56 exec f.extractfile(name).read() in globals() | |
| 57 f.close() | |
| 58 return True | |
| 59 except KeyError: | |
| 60 f.close() | |
| 61 if os.path.isfile('tests.tar.gz'): | |
| 62 f = tarfile.open('tests.tar.gz') | |
| 63 try: | |
| 64 exec f.extractfile(name).read() in globals() | |
| 65 f.close() | |
| 66 return True | |
| 67 except KeyError: | |
| 68 f.close() | |
| 69 if os.path.isfile('tests.tbz2'): | |
| 70 f = tarfile.open('tests.tbz2') | |
| 71 try: | |
| 72 exec f.extractfile(name).read() in globals() | |
| 73 f.close() | |
| 74 return True | |
| 75 except KeyError: | |
| 76 f.close() | |
| 77 if os.path.isfile('tests.tar.bz2'): | |
| 78 f = tarfile.open('tests.tar.bz2') | |
| 79 try: | |
| 80 exec f.extractfile(name).read() in globals() | |
| 81 f.close() | |
| 82 return True | |
| 83 except KeyError: | |
| 84 f.close() | |
| 85 return False | |
| 86 | |
| 87 try: | |
| 88 execfile('testconf.py') | |
| 89 except IOError, error: | |
| 90 exc_info = sys.exc_info()[2] | |
| 91 try: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
92 execfile(os.path.join('tests', 'testconf.py')) |
| 0 | 93 except IOError: |
| 94 if not exectestconf_helper('testconf.py'): | |
| 95 raise IOError, (error.errno, 'The configuration file is missing', error.filename), exc_info | |
| 96 del exc_info | |
| 97 | |
| 98 globals2 = set(globals()) | |
| 99 globals2.remove('globals1') | |
| 100 globals2 -= globals1 | |
| 101 del globals1 | |
| 102 | |
| 103 shared = {} | |
| 104 g = globals() | |
| 105 for k in globals2: | |
| 106 shared[k] = g[k] | |
| 107 | |
| 108 newtasknames = [] | |
| 109 while len(args) and args[0] in tasknames: | |
| 110 newtasknames.append(args[0]) | |
| 111 del args[0] | |
| 112 if len(newtasknames): | |
| 113 tasknames = newtasknames | |
| 114 | |
| 115 scoresumoveralltasks = 0 | |
| 116 scoremaxoveralltasks = 0 | |
| 117 ntasks = 0 | |
| 118 nfulltasks = 0 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
119 cwd = '' # At any time this is either '' or taskname |
| 0 | 120 |
| 121 if options.autotime: | |
| 122 c = time.clock() | |
| 123 time.sleep(1) | |
| 124 c = time.clock() - c | |
| 125 if int(c + .99999) == 1: | |
| 126 clock = time.clock | |
| 127 else: | |
| 128 clock = time.time | |
| 129 elif os.name == 'nt': | |
| 130 clock = time.clock | |
| 131 else: | |
| 132 clock = time.time | |
| 133 | |
| 134 if options.copyonly: | |
| 135 options.erase = False | |
| 136 | |
| 137 def existstestcase_helper(name): | |
| 138 if os.path.isfile('tests.tar'): | |
| 139 f = tarfile.open('tests.tar') | |
| 140 try: | |
| 141 f.getmember(name) | |
| 142 f.close() | |
| 143 return True | |
| 144 except KeyError: | |
| 145 f.close() | |
| 146 if os.path.isfile('tests.zip'): | |
| 147 f = zipfile.ZipFile('tests.zip') | |
| 148 try: | |
| 149 f.getinfo(name) | |
| 150 f.close() | |
| 151 return True | |
| 152 except KeyError: | |
| 153 f.close() | |
| 154 if os.path.isfile('tests.tgz'): | |
| 155 f = tarfile.open('tests.tgz') | |
| 156 try: | |
| 157 f.getmember(name) | |
| 158 f.close() | |
| 159 return True | |
| 160 except KeyError: | |
| 161 f.close() | |
| 162 if os.path.isfile('tests.tar.gz'): | |
| 163 f = tarfile.open('tests.tar.gz') | |
| 164 try: | |
| 165 f.getmember(name) | |
| 166 f.close() | |
| 167 return True | |
| 168 except KeyError: | |
| 169 f.close() | |
| 170 if os.path.isfile('tests.tbz2'): | |
| 171 f = tarfile.open('tests.tbz2') | |
| 172 try: | |
| 173 f.getmember(name) | |
| 174 f.close() | |
| 175 return True | |
| 176 except KeyError: | |
| 177 f.close() | |
| 178 if os.path.isfile('tests.tar.bz2'): | |
| 179 f = tarfile.open('tests.tar.bz2') | |
| 180 try: | |
| 181 f.getmember(name) | |
| 182 f.close() | |
| 183 return True | |
| 184 except KeyError: | |
| 185 f.close() | |
| 186 return False | |
| 187 | |
| 188 def existstestcase(name): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
189 if os.path.isfile(os.path.join('tests', taskname, name)) or os.path.isfile(os.path.join('tests', name)): |
| 0 | 190 return True |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
191 if cwd and (os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)) or os.path.isfile(os.path.join(oldcwd, 'tests', name))): |
| 0 | 192 return True |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
193 if existstestcase_helper(os.path.join(taskname, name)) or existstestcase_helper(name): |
| 0 | 194 return True |
| 195 if cwd: | |
| 196 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
197 if existstestcase_helper(os.path.join(cwd, name)) or existstestcase_helper(name): |
| 0 | 198 os.chdir(cwd) |
| 199 return True | |
| 200 os.chdir(cwd) | |
| 201 return False | |
| 202 | |
| 203 def opentestcase_helper(name): | |
| 204 if os.path.isfile('tests.tar'): | |
| 205 f = tarfile.open('tests.tar') | |
| 206 try: | |
| 207 c = f.extractfile(name) | |
| 208 return c | |
| 209 except KeyError: | |
| 210 f.close() | |
| 211 if os.path.isfile('tests.zip'): | |
| 212 f = zipfile.ZipFile('tests.zip') | |
| 213 try: | |
| 214 c = f.open(name, 'rU') | |
| 215 f.close() | |
| 216 return c | |
| 217 except KeyError: | |
| 218 f.close() | |
| 219 if os.path.isfile('tests.tgz'): | |
| 220 f = tarfile.open('tests.tgz') | |
| 221 try: | |
| 222 c = f.extractfile(name) | |
| 223 return c | |
| 224 except KeyError: | |
| 225 f.close() | |
| 226 if os.path.isfile('tests.tar.gz'): | |
| 227 f = tarfile.open('tests.tar.gz') | |
| 228 try: | |
| 229 c = f.extractfile(name) | |
| 230 return c | |
| 231 except KeyError: | |
| 232 f.close() | |
| 233 if os.path.isfile('tests.tbz2'): | |
| 234 f = tarfile.open('tests.tbz2') | |
| 235 try: | |
| 236 c = f.extractfile(name) | |
| 237 return c | |
| 238 except KeyError: | |
| 239 f.close() | |
| 240 if os.path.isfile('tests.tar.bz2'): | |
| 241 f = tarfile.open('tests.tar.bz2') | |
| 242 try: | |
| 243 c = f.extractfile(name) | |
| 244 return c | |
| 245 except KeyError: | |
| 246 f.close() | |
| 247 return None | |
| 248 | |
| 249 def opentestcase(name): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
250 if os.path.isfile(os.path.join('tests', taskname, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
251 return open(os.path.join('tests', taskname, name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
252 elif os.path.isfile(os.path.join('tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
253 return open(os.path.join('tests', name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
254 f = opentestcase_helper(os.path.join(taskname, name)) |
| 0 | 255 if not f: |
| 256 f = opentestcase_helper(name) | |
| 257 if f: | |
| 258 return f | |
| 259 if cwd: | |
| 3 | 260 if os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)): |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
261 return open(os.path.join(oldcwd, 'tests', cwd, name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
262 elif os.path.isfile(os.path.join(oldcwd, 'tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
263 return open(os.path.join(oldcwd, 'tests', name), 'rU') |
| 0 | 264 os.chdir(oldcwd) |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
265 f = opentestcase_helper(os.path.join(cwd, name)) |
| 0 | 266 if not f: |
| 267 f = opentestcase_helper(name) | |
| 268 os.chdir(cwd) | |
| 269 if f: | |
| 270 return f | |
| 271 raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found' | |
| 272 | |
| 273 def copytestcase_helper(name, target): | |
| 274 if os.path.isfile('tests.tar'): | |
| 275 f = tarfile.open('tests.tar') | |
| 276 try: | |
| 277 m = f.getmember(name) | |
| 278 m.name = target | |
| 279 f.extract(m) | |
| 280 f.close() | |
| 281 return True | |
| 282 except KeyError: | |
| 283 f.close() | |
| 284 if os.path.isfile('tests.zip'): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
285 if not os.path.isabs(target): |
| 0 | 286 f = zipfile.ZipFile('tests.zip') |
| 287 m = f.getinfo(name) | |
| 288 try: | |
| 289 m.filename = target | |
| 290 f.extract(m) | |
| 291 f.close() | |
| 292 return True | |
| 293 except KeyError: | |
| 294 f.close() | |
| 295 else: | |
| 296 oldcwd = os.getcwdu() | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
297 os.chdir('/') # FIXME: portability? |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
298 f = zipfile.ZipFile(os.path.join(oldcwd, 'tests.zip')) |
| 0 | 299 try: |
| 300 m = f.getinfo(name) | |
| 301 m.filename = target[1:] | |
| 302 f.extract(m) | |
| 303 f.close() | |
| 304 os.chdir(oldcwd) | |
| 305 return True | |
| 306 except KeyError: | |
| 307 f.close() | |
| 308 os.chdir(oldwcd) | |
| 309 if os.path.isfile('tests.tgz'): | |
| 310 f = tarfile.open('tests.tgz') | |
| 311 try: | |
| 312 m = f.getmember(name) | |
| 313 m.name = target | |
| 314 f.extract(m) | |
| 315 f.close() | |
| 316 return True | |
| 317 except KeyError: | |
| 318 f.close() | |
| 319 if os.path.isfile('tests.tar.gz'): | |
| 320 f = tarfile.open('tests.tar.gz') | |
| 321 try: | |
| 322 m = f.getmember(name) | |
| 323 m.name = target | |
| 324 f.extract(m) | |
| 325 f.close() | |
| 326 return True | |
| 327 except KeyError: | |
| 328 f.close() | |
| 329 if os.path.isfile('tests.tbz2'): | |
| 330 f = tarfile.open('tests.tbz2') | |
| 331 try: | |
| 332 m = f.getmember(name) | |
| 333 m.name = target | |
| 334 f.extract(m) | |
| 335 f.close() | |
| 336 return True | |
| 337 except KeyError: | |
| 338 f.close() | |
| 339 if os.path.isfile('tests.tar.bz2'): | |
| 340 f = tarfile.open('tests.tar.bz2') | |
| 341 try: | |
| 342 m = f.getmember(name) | |
| 343 m.name = target | |
| 344 f.extract(m) | |
| 345 f.close() | |
| 346 return True | |
| 347 except KeyError: | |
| 348 f.close() | |
| 349 return False | |
| 350 | |
| 351 def copytestcase(name, target): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
352 if os.path.isfile(os.path.join('tests', taskname, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
353 shutil.copyfile(os.path.join('tests', taskname, name), target) |
| 0 | 354 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
355 elif os.path.isfile(os.path.join('tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
356 shutil.copyfile(os.path.join('tests', name), target) |
| 0 | 357 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
358 if copytestcase_helper(os.path.join(taskname, name), target) or copytestcase_helper(name, target): |
| 0 | 359 return |
| 360 if cwd: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
361 if os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
362 shutil.copyfile(os.path.join(oldcwd, 'tests', cwd, name), target) |
| 0 | 363 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
364 elif os.path.isfile(os.path.join(oldcwd, 'tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
365 shutil.copyfile(os.path.join(oldcwd, 'tests', name), target) |
| 0 | 366 return |
| 367 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
368 if copytestcase_helper(os.path.join(cwd, name), target) or copytestcase_helper(name, target): |
| 0 | 369 os.chdir(cwd) |
| 370 return | |
| 371 os.chdir(cwd) | |
| 372 raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found' | |
| 373 | |
| 374 # Always chdir if the directory exists but use any existing config | |
| 375 def chdir_and_exec_testconf(): | |
| 376 global cwd | |
| 377 cwd = '' | |
| 378 if os.path.isdir(taskname): | |
| 379 os.chdir(taskname) | |
| 3 | 380 if taskname != os.path.curdir: |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
381 cwd = taskname |
| 0 | 382 try: |
| 383 execfile('testconf.py', globals()) | |
| 384 return | |
| 385 except IOError: | |
| 386 pass | |
| 387 if not cwd: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
388 if os.path.isfile(os.path.join('tests', taskname, 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
389 execfile(os.path.join('tests', taskname, 'testconf.py'), globals()) |
| 0 | 390 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
391 if os.path.isfile(os.path.join('tests', 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
392 execfile(os.path.join('tests', 'testconf.py'), globals()) |
| 0 | 393 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
394 if exectestconf_helper(os.path.join(taskname, 'testconf.py')) or exectestconf_helper('testconf.py'): |
| 0 | 395 return |
| 396 if cwd: | |
| 397 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
398 if os.path.isfile(os.path.join('tests', cwd, 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
399 execfile(os.path.join('tests', cwd, 'testconf.py'), globals()) |
| 0 | 400 os.chdir(cwd) |
| 401 return | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
402 if os.path.isfile(os.path.join('tests', 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
403 execfile(os.path.join('tests', 'testconf.py'), globals()) |
| 0 | 404 os.chdir(cwd) |
| 405 return | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
406 if exectestconf_helper(os.path.join(cwd, 'testconf.py')) or exectestconf_helper('testconf.py'): |
| 0 | 407 os.chdir(cwd) |
| 408 return | |
| 409 if os.path.isfile('testconf.py'): | |
| 410 execfile('testconf.py', globals()) | |
| 411 os.chdir(cwd) | |
| 412 return | |
| 413 os.chdir(cwd) | |
| 414 elif os.path.isfile('testconf.py'): | |
| 415 execfile('testconf.py', globals()) | |
| 416 return | |
| 417 raise KeyError, 'The configuration file for task ' + taskname + ' is missing' | |
| 418 | |
| 419 try: | |
| 420 name | |
| 421 namedefined = True | |
| 422 except Exception: | |
| 423 namedefined = False | |
| 424 | |
| 425 for taskname in tasknames: | |
| 426 if ntasks: | |
| 427 print | |
| 428 | |
| 429 try: | |
| 430 if len(tasknames) > 1: | |
| 431 print taskname | |
| 432 except Exception: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
433 if taskname != os.path.curdir or ntasks: |
| 0 | 434 print taskname |
| 435 | |
| 436 try: del inname | |
| 437 except NameError: pass | |
| 438 try: del outname | |
| 439 except NameError: pass | |
| 440 try: del ansname | |
| 441 except NameError: pass | |
| 442 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
443 if not namedefined and taskname != os.path.curdir: |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
444 name = os.path.join(os.path.curdir, taskname) |
| 0 | 445 for k in shared: |
| 446 g[k] = shared[k] | |
| 447 | |
| 448 oldcwd = os.getcwdu() | |
| 449 chdir_and_exec_testconf() | |
| 450 | |
| 451 if options.clean: | |
| 452 try: | |
| 453 if not stdio or tester: | |
| 454 if not tester: | |
| 455 inname | |
| 456 outname | |
| 457 if tester: | |
| 458 ansname | |
| 459 except NameError, error: | |
| 460 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 461 if not options.erase: | |
| 462 try: | |
| 463 inname = inname.replace('%', taskname) | |
| 464 except NameError: | |
| 465 inname = taskname + '.in' | |
| 466 try: | |
| 467 outname = outname.replace('%', taskname) | |
| 468 except NameError: | |
| 469 outname = taskname + '.out' | |
| 470 try: | |
| 471 ansname = ansname.replace('%', taskname) | |
| 472 except NameError: | |
| 473 ansname = taskname + '.ans' | |
| 4 | 474 elif not stdio or tester or not options.erase: |
| 0 | 475 inname = inname.replace('%', taskname) |
| 476 outname = outname.replace('%', taskname) | |
| 477 if tester: | |
| 478 ansname = ansname.replace('%', taskname) | |
| 479 if not stdio or tester or not options.erase: | |
| 480 if os.path.exists(inname): os.remove(inname) | |
| 481 if os.path.exists(outname): os.remove(outname) | |
| 482 if (tester or not options.erase) and ansname: | |
| 483 if os.path.exists(ansname): os.remove(ansname) | |
| 484 continue | |
| 485 | |
| 486 try: | |
| 487 name | |
| 488 except NameError, error: | |
| 489 if str(error).count('name') == 1: | |
| 490 raise NameError, 'configuration ' + str(error), sys.exc_info()[2] | |
| 491 else: | |
| 492 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 493 | |
| 494 try: | |
| 495 if not stdio: | |
| 496 inname | |
| 497 outname | |
| 498 testcaseinname | |
| 499 if tester: | |
| 500 outname | |
| 501 if ansname: | |
| 502 testcaseoutname | |
| 503 else: | |
| 504 testcaseoutname | |
| 505 except NameError, error: | |
| 506 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 507 | |
| 508 if not options.erase: | |
| 509 try: | |
| 510 inname | |
| 511 except NameError: | |
| 512 inname = taskname + '.in' | |
| 513 try: | |
| 514 outname | |
| 515 except NameError: | |
| 516 outname = taskname + '.out' | |
| 517 try: | |
| 518 ansname | |
| 519 except NameError: | |
| 520 ansname = taskname + '.ans' | |
| 521 | |
| 522 if options.pause: | |
| 523 try: | |
| 524 pause | |
| 525 except NameError, error: | |
| 526 if os.name == 'posix': | |
| 527 pause = 'read -s -n 1' | |
| 3 | 528 print 'Configuration ' + str(error).replace('name ', 'variable ') + '; it was devised automatically but the choice might be incorrect, so test.py might exit immediately after the testing is completed.' |
| 0 | 529 elif os.name == 'nt': |
| 530 pause = 'pause' | |
| 531 else: | |
| 532 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ') + ' and cannot be devised automatically', sys.exc_info()[2] | |
| 533 | |
| 534 if not dummyinname: | |
| 535 dummyinname = testcaseinname | |
| 536 if not dummyoutname and (not tester or ansname): | |
| 537 dummyoutname = testcaseoutname | |
| 538 | |
| 539 dummyinname = dummyinname.replace('%', taskname) | |
| 540 dummyoutname = dummyoutname.replace('%', taskname) | |
| 541 testcaseinname = testcaseinname.replace('%', taskname) | |
| 542 if not stdio or not options.erase: | |
| 543 inname = inname.replace('%', taskname) | |
| 544 outname = outname.replace('%', taskname) | |
| 545 try: | |
| 546 ansname = ansname.replace('%', taskname) | |
| 547 except NameError: | |
| 548 pass | |
| 549 if tester: | |
| 550 try: inname = inname.replace('%', taskname) | |
| 551 except NameError: pass | |
| 552 outname = outname.replace('%', taskname) | |
| 553 if ansname: | |
| 554 ansname = ansname.replace('%', taskname) | |
| 555 testcaseoutname = testcaseoutname.replace('%', taskname) | |
| 556 else: | |
| 557 testcaseoutname = testcaseoutname.replace('%', taskname) | |
| 558 | |
| 559 if isinstance(padwithzeroestolength, tuple): | |
| 560 padwithzeroestolength, paddummieswithzeroestolength = padwithzeroestolength | |
| 561 else: | |
| 562 paddummieswithzeroestolength = padwithzeroestolength | |
| 563 | |
| 564 if options.python: | |
| 565 dummies = () | |
| 566 s = ' '.join(args) | |
| 567 tests = eval(s) | |
| 568 try: | |
| 569 tests.__iter__ | |
| 570 except AttributeError: | |
| 571 tests = (tests,) | |
| 572 elif len(args): | |
| 573 if os.path.exists(args[0]): | |
| 574 name = args[0] | |
| 575 del args[0] | |
| 576 if len(args) > 1: | |
| 577 dummies = () | |
| 578 tests = args | |
| 579 elif len(args): | |
| 580 dummies = () | |
| 581 s = args[0] | |
| 582 if len(s) < padwithzeroestolength: | |
| 583 s = s.zfill(padwithzeroestolength) | |
| 584 if existstestcase(testcaseinname.replace('$', s)): | |
| 585 tests = (s,) | |
| 586 else: | |
| 587 try: | |
| 588 tests = eval(args[0]) | |
| 589 try: | |
| 590 tests.__iter__ | |
| 591 except AttributeError: | |
| 592 tests = (tests,) | |
| 593 except Exception: | |
| 594 tests = (s,) | |
| 595 | |
| 596 if options.exclude: | |
| 597 testsexcluded = [] | |
| 598 for i in options.exclude: | |
| 599 v = eval(i) | |
| 600 try: | |
| 601 testsexcluded.extend(v) | |
| 602 except TypeError: | |
| 603 testsexcluded.append(v) | |
| 604 | |
| 605 # Windows doesn't like paths beginning with .\ and not ending with an extension | |
| 606 name = os.path.normcase(name) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
607 if os.name == 'nt' and name.startswith('.\\'): |
| 0 | 608 name = name[2:] |
| 609 | |
| 610 newpointmap = {} | |
| 611 | |
| 612 for i in pointmap: | |
| 613 try: | |
| 614 for j in i: | |
| 615 newpointmap[j] = pointmap[i] | |
| 616 except TypeError: | |
| 617 newpointmap[i] = pointmap[i] | |
| 618 | |
| 619 pointmap = newpointmap | |
| 620 | |
|
9
ed90b375d197
Award maxexitcode points by default
Oleg Oshmyan <chortos@inbox.lv>
parents:
6
diff
changeset
|
621 if not tester: |
|
ed90b375d197
Award maxexitcode points by default
Oleg Oshmyan <chortos@inbox.lv>
parents:
6
diff
changeset
|
622 maxexitcode = 0 |
|
ed90b375d197
Award maxexitcode points by default
Oleg Oshmyan <chortos@inbox.lv>
parents:
6
diff
changeset
|
623 |
| 0 | 624 if maxtime > 0: |
| 625 strmaxtime = '/%.3f' % maxtime | |
| 626 else: | |
| 627 strmaxtime = '' | |
| 628 | |
| 629 padoutputtolength = 0 | |
| 630 ntests = [] | |
| 631 | |
| 632 for j in dummies: | |
| 633 try: | |
| 634 j.__iter__ | |
| 635 except AttributeError: | |
| 636 j = (j,) | |
| 637 ntests.append((j, True)) | |
| 638 for i in j: | |
| 639 s = str(i) | |
| 640 if len(s) < paddummieswithzeroestolength: | |
| 641 s = s.zfill(paddummieswithzeroestolength) | |
| 642 s = 'sample ' + s | |
| 643 if padoutputtolength < len(s): | |
| 644 padoutputtolength = len(s) | |
| 645 | |
| 646 for j in tests: | |
| 647 try: | |
| 648 j.__iter__ | |
| 649 except AttributeError: | |
| 650 j = (j,) | |
| 651 ntests.append((j, False)) | |
| 652 for i in j: | |
| 653 s = str(i) | |
| 654 if len(s) < padwithzeroestolength: | |
| 655 s = s.zfill(padwithzeroestolength) | |
| 656 if padoutputtolength < len(s): | |
| 657 padoutputtolength = len(s) | |
| 658 | |
| 659 tests = ntests | |
| 660 score = maxpoints = ncorrect = ntotal = ncorrectvalued = nvalued = 0 | |
| 661 | |
| 662 if options.copyonly: | |
| 663 j, isdummy = tests[-1] | |
| 664 if isdummy: | |
| 665 realinname = dummyinname | |
| 666 realoutname = dummyoutname | |
| 667 else: | |
| 668 realinname = testcaseinname | |
| 669 realoutname = testcaseoutname | |
| 670 for i in j: | |
| 671 if i in testsexcluded and not isdummy: | |
| 672 continue | |
| 673 s = str(i) | |
| 674 if isdummy: | |
| 675 if len(s) < paddummieswithzeroestolength: | |
| 676 s = s.zfill(paddummieswithzeroestolength) | |
| 677 else: | |
| 678 if len(s) < padwithzeroestolength: | |
| 679 s = s.zfill(padwithzeroestolength) | |
| 680 copytestcase(realinname.replace('$', s), inname) | |
| 681 if ansname: | |
| 682 copytestcase(realoutname.replace('$', s), ansname) | |
| 683 continue | |
| 684 | |
| 685 for j, isdummy in tests: | |
| 686 ncorrectgrp = 0 | |
| 687 ntotalgrp = 0 | |
| 688 scoregrp = 0 | |
| 689 maxpointsgrp = 0 | |
| 690 if isdummy: | |
| 691 realinname = dummyinname | |
| 692 realoutname = dummyoutname | |
| 693 else: | |
| 694 realinname = testcaseinname | |
| 695 realoutname = testcaseoutname | |
| 696 for i in j: | |
| 697 if i in testsexcluded and not isdummy: | |
| 698 continue | |
| 699 ntotalgrp += 1 | |
| 700 s = str(i) | |
| 701 if isdummy: | |
| 702 npoints = 0 | |
| 703 if len(s) < paddummieswithzeroestolength: | |
| 704 s = s.zfill(paddummieswithzeroestolength) | |
| 705 spref = 'sample ' | |
| 706 else: | |
|
9
ed90b375d197
Award maxexitcode points by default
Oleg Oshmyan <chortos@inbox.lv>
parents:
6
diff
changeset
|
707 npoints = pointmap.get(None, maxexitcode if maxexitcode and isinstance(maxexitcode, int) else 1) |
| 0 | 708 npoints = pointmap.get(i, npoints) |
| 709 maxpointsgrp += npoints | |
| 710 if npoints: | |
| 711 nvalued += 1 | |
| 712 if len(s) < padwithzeroestolength: | |
| 713 s = s.zfill(padwithzeroestolength) | |
| 714 spref = '' | |
| 715 print ' ' * (padoutputtolength - len(spref + s)) + spref + s + ':', | |
| 716 sys.stdout.flush() | |
| 717 outputdata = open(os.devnull, 'w') | |
| 718 if stdio: | |
| 719 f = tempfile.NamedTemporaryFile(delete=False) | |
| 720 inputdatafname = f.name | |
| 721 f.close() | |
| 722 copytestcase(realinname.replace('$', s), inputdatafname) | |
| 723 inputdata = open(inputdatafname, 'rU') | |
| 724 if options.erase: | |
| 725 tempoutput = tempfile.TemporaryFile('w+') | |
| 726 else: | |
| 727 tempoutput = open(outname, 'w+') | |
| 728 try: | |
| 729 proc = subprocess.Popen(name, stdin=inputdata, stdout=tempoutput, stderr=outputdata, universal_newlines=True) | |
| 730 except OSError, error: | |
| 731 raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 732 else: | |
| 733 if os.path.exists(outname): | |
| 734 os.remove(outname) | |
| 735 copytestcase(realinname.replace('$', s), inname) | |
| 736 try: | |
| 737 proc = subprocess.Popen(name, stdin=outputdata, stdout=outputdata, stderr=outputdata, universal_newlines=True) | |
| 738 except OSError, error: | |
| 739 raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 740 cl = clock() | |
| 741 if maxtime > 0: | |
| 742 while 1: | |
| 743 proc.poll() | |
| 744 elapsed = clock() - cl | |
| 745 if proc.returncode == None: | |
| 746 if elapsed >= maxtime: | |
| 747 print '%.3f%s s, 0/%d, time limit exceeded' % (elapsed, strmaxtime, npoints) | |
| 748 sys.stdout.flush() | |
| 749 while proc.returncode == None: | |
| 750 try: | |
| 751 proc.terminate() | |
| 752 except OSError: | |
| 753 pass | |
| 754 except AttributeError: | |
| 755 try: | |
| 756 os.kill(proc.pid, signal.SIGTERM) | |
| 757 except Exception: | |
| 758 pass | |
| 759 proc.poll() | |
| 760 outputdata.close() | |
| 761 if stdio: | |
| 762 tempoutput.close() | |
| 763 break | |
| 764 else: | |
| 765 print '%.3f%s s,' % (elapsed, strmaxtime), | |
| 766 sys.stdout.flush() | |
| 767 elapsed = 0 | |
| 768 if stdio: | |
| 769 tempoutput.seek(0) | |
| 770 lines = tempoutput.readlines() | |
| 771 tempoutput.close() | |
| 772 break | |
| 773 if elapsed >= maxtime: | |
| 774 continue | |
| 775 else: | |
| 776 data = proc.communicate() | |
| 777 elapsed = clock() - cl | |
| 778 print '%.3f%s s,' % (elapsed, strmaxtime), | |
| 779 sys.stdout.flush() | |
| 780 if stdio: | |
| 781 tempoutput.seek(0) | |
| 782 lines = tempoutput.readlines() | |
| 783 tempoutput.close() | |
| 784 outputdata.close() | |
| 785 if stdio: | |
| 786 inputdata.close() | |
| 787 try: | |
| 788 os.unlink(inputdatafname) | |
| 789 except Exception: | |
| 790 pass | |
| 791 if proc.returncode > 0: | |
| 792 print '0/%d, non-zero return code %d' % (npoints, proc.returncode) | |
| 793 sys.stdout.flush() | |
| 794 elif proc.returncode < 0: | |
| 795 print '0/%d, terminated by signal %d' % (npoints, -proc.returncode) | |
| 796 sys.stdout.flush() | |
| 797 else: | |
| 798 if not tester: | |
| 799 if stdio: | |
| 800 outputdata = opentestcase(realoutname.replace('$', s)) | |
| 801 r = 0 | |
| 802 data = outputdata.read().splitlines(True) | |
| 803 if len(lines) != len(data): | |
| 804 r = 1 | |
| 805 else: | |
| 806 for i in zip(lines, data): | |
| 807 if i[0] != i[1]: | |
| 808 r = 1 | |
| 809 break | |
| 810 outputdata.close() | |
| 811 else: | |
| 812 try: | |
| 813 inputdata = open(outname, 'rU') | |
| 814 except IOError: | |
| 815 print '0/%g, output file not created or not readable' % npoints | |
| 816 sys.stdout.flush() | |
| 817 r = None | |
| 818 else: | |
| 819 outputdata = opentestcase(realoutname.replace('$', s)) | |
| 820 r = 0 | |
| 821 lines = inputdata.readlines() | |
| 822 data = outputdata.read().splitlines(True) | |
| 823 if len(lines) != len(data): | |
| 824 r = 1 | |
| 825 else: | |
| 826 for i in zip(lines, data): | |
| 827 if i[0] != i[1]: | |
| 828 r = 1 | |
| 829 break | |
| 830 inputdata.close() | |
| 831 outputdata.close() | |
| 832 else: | |
| 833 if ansname: | |
| 834 copytestcase(realoutname.replace('$', s), ansname) | |
| 835 if stdio: | |
| 836 try: copytestcase(realinname.replace('$', s), inname) | |
| 837 except NameError: pass | |
| 838 outputdata = open(outname, 'w') | |
| 839 outputdata.writelines(lines) | |
| 840 outputdata.close() | |
| 841 try: | |
| 842 proc = subprocess.Popen(tester, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) | |
| 843 except OSError, error: | |
| 844 raise OSError, 'The tester application cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 845 data = proc.communicate() | |
| 846 r = proc.returncode | |
| 847 if tester and data[0]: | |
| 848 data = ''.join((' (', data[0].strip(), ')')) | |
| 849 else: | |
| 850 data = '' | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
851 if not maxexitcode and r or maxexitcode and not r: |
| 0 | 852 print '0/%g, wrong answer%s' % (npoints, data) |
| 853 sys.stdout.flush() | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
854 elif not maxexitcode and r == 0 or maxexitcode and r >= maxexitcode: |
| 0 | 855 print '%g/%g, OK%s' % (npoints, npoints, data) |
| 856 sys.stdout.flush() | |
| 857 scoregrp += npoints | |
| 858 ncorrectgrp += 1 | |
| 859 if npoints: | |
| 860 ncorrectvalued += 1 | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
861 elif maxexitcode and r != None: |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
862 actualpoints = npoints*r/maxexitcode if not npoints*r%maxexitcode else float(npoints*r)/maxexitcode |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
863 print '%g/%g, partly OK%s' % (actualpoints, npoints, data) |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
864 sys.stdout.flush() |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
865 scoregrp += actualpoints |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
866 ncorrectgrp += 1 |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
867 if npoints: |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
868 ncorrectvalued += 1 |
| 0 | 869 if ntotalgrp: |
|
6
b0034b18f942
maxexitcode now gets on with test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
5
diff
changeset
|
870 if ncorrectgrp < ntotalgrp: |
| 0 | 871 scoregrp = 0 |
| 872 if ntotalgrp > 1: | |
| 873 print 'Group total: %d/%d tests; %g/%g points' % (ncorrectgrp, ntotalgrp, scoregrp, maxpointsgrp) | |
| 874 sys.stdout.flush() | |
| 875 ncorrect += ncorrectgrp | |
| 876 ntotal += ntotalgrp | |
| 877 score += scoregrp | |
| 878 maxpoints += maxpointsgrp | |
| 879 | |
| 880 if options.erase: | |
| 881 if not stdio or tester: | |
| 882 if os.path.exists(inname): os.remove(inname) | |
| 883 if os.path.exists(outname): os.remove(outname) | |
| 884 if tester and ansname: | |
| 885 if os.path.exists(ansname): os.remove(ansname) | |
| 886 elif stdio: | |
| 887 copytestcase(realinname.replace('$', s), inname) | |
| 888 copytestcase(realoutname.replace('$', s), ansname) | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
889 actualpoints = (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0 |
| 0 | 890 if nvalued != ntotal: |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
891 print 'Grand total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, ncorrectvalued, nvalued, score, maxpoints, actualpoints, taskweight) |
| 0 | 892 else: |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
893 print 'Grand total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, score, maxpoints, actualpoints, taskweight) |
| 0 | 894 |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
895 scoresumoveralltasks += actualpoints |
| 0 | 896 scoremaxoveralltasks += taskweight |
| 897 ntasks += 1 | |
| 898 nfulltasks += int((score == maxpoints) if maxpoints else (taskweight == 0)) | |
| 899 | |
| 900 os.chdir(oldcwd) | |
| 901 | |
| 902 if options.clean or options.copyonly: | |
| 903 sys.exit() | |
| 904 | |
| 905 if ntasks != 1: | |
| 906 print | |
| 907 print 'Grand grand total: %g/%g weighted points; %d/%d problems solved fully' % (scoresumoveralltasks, scoremaxoveralltasks, nfulltasks, ntasks) | |
| 908 | |
| 909 if options.pause: | |
| 910 print 'Press any key to exit... ', | |
| 911 sys.stdout.flush() | |
| 912 os.system(pause + ' >' + os.devnull) |