Mercurial > ~astiob > upreckon > hgweb
annotate 2.00/problem.py @ 40:af9c45708987
Cemented a decision previously being unsure about
The mere presense of the tasknames configuration variable now always makes problem names to be printed.
This is not new, but the old behaviour (only printing names if we test more than one problem), previously commented out, has now been removed altogether.
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Sun, 05 Dec 2010 14:34:24 +0100 |
| parents | 2b459f9743b4 |
| children |
| rev | line source |
|---|---|
| 21 | 1 #! /usr/bin/env python |
| 16 | 2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
| 3 | |
| 21 | 4 from __future__ import division, with_statement |
| 5 | |
| 6 try: | |
| 7 from compat import * | |
| 8 import config, testcases | |
| 9 except ImportError: | |
| 10 import __main__ | |
| 11 __main__.import_error(sys.exc_info()[1]) | |
| 12 else: | |
| 22 | 13 from __main__ import clock, options |
| 21 | 14 |
| 22 | 15 import os, re, sys |
| 21 | 16 |
| 16 | 17 try: |
| 21 | 18 import signal |
| 19 except ImportError: | |
| 20 signalnames = () | |
| 21 else: | |
| 22 # Construct a cache of all signal names available on the current | |
| 23 # platform. Prefer names from the UNIX standards over other versions. | |
| 24 unixnames = frozenset(('HUP', 'INT', 'QUIT', 'ILL', 'ABRT', 'FPE', 'KILL', 'SEGV', 'PIPE', 'ALRM', 'TERM', 'USR1', 'USR2', 'CHLD', 'CONT', 'STOP', 'TSTP', 'TTIN', 'TTOU', 'BUS', 'POLL', 'PROF', 'SYS', 'TRAP', 'URG', 'VTALRM', 'XCPU', 'XFSZ')) | |
| 25 signalnames = {} | |
| 26 for name in dir(signal): | |
| 27 if re.match('SIG[A-Z]+$', name): | |
| 28 value = signal.__dict__[name] | |
| 22 | 29 if isinstance(value, int) and (value not in signalnames or name[3:] in unixnames): |
| 21 | 30 signalnames[value] = name |
| 31 del unixnames | |
| 16 | 32 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
33 __all__ = 'Problem', 'TestContext', 'test_context_end', 'TestGroup' |
| 21 | 34 |
| 26 | 35 def strerror(e): |
| 36 s = getattr(e, 'strerror') | |
| 37 if not s: s = str(e) | |
| 38 return ' (%s%s)' % (s[0].lower(), s[1:]) if s else '' | |
| 21 | 39 |
| 40 class Cache(object): | |
| 41 def __init__(self, mydict): | |
| 42 self.__dict__ = mydict | |
| 16 | 43 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
44 class TestContext(object): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
45 pass |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
46 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
47 test_context_end = object() |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
48 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
49 class TestGroup(TestContext): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
50 __slots__ = 'case', 'log', 'correct', 'allcorrect', 'real', 'max', 'ntotal', 'nvalued', 'ncorrect', 'ncorrectvalued' |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
51 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
52 def __init__(self): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
53 self.real = self.max = self.ntotal = self.nvalued = self.ncorrect = self.ncorrectvalued = 0 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
54 self.allcorrect = True |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
55 self.log = [] |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
56 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
57 def case_start(self, case): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
58 self.case = case |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
59 self.correct = False |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
60 self.ntotal += 1 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
61 self.max += case.points |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
62 if case.points: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
63 self.nvalued += 1 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
64 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
65 def case_correct(self): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
66 self.correct = True |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
67 self.ncorrect += 1 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
68 if self.case.points: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
69 self.ncorrectvalued += 1 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
70 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
71 def case_end(self, granted): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
72 self.log.append((self.case, self.correct, granted)) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
73 self.real += granted |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
74 del self.case |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
75 if not self.correct: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
76 self.allcorrect = False |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
77 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
78 def end(self): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
79 say('Group total: %d/%d tests; %d/%d points' % (self.ncorrect, self.ntotal, self.real if self.allcorrect else 0, self.max)) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
80 # No real need to flush stdout, as it will anyway be flushed in a moment, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
81 # when either the problem total or the next test case's ID is printed |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
82 if self.allcorrect: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
83 return self.log |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
84 else: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
85 return ((case, correct, 0) for case, correct, granted in self.log) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
86 |
| 16 | 87 class Problem(object): |
| 88 __slots__ = 'name', 'config', 'cache', 'testcases' | |
| 89 | |
| 90 def __init__(prob, name): | |
| 91 if not isinstance(name, basestring): | |
| 92 # This shouldn't happen, of course | |
| 21 | 93 raise TypeError('Problem() argument 1 must be string, not ' + type(name).__name__) |
| 16 | 94 prob.name = name |
| 21 | 95 prob.config = config.load_problem(name) |
| 96 if not getattr(prob.config, 'kind', None): prob.config.kind = 'batch' | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
97 prob.cache = Cache({'padoutput': 0}) |
| 21 | 98 prob.testcases = testcases.load_problem(prob) |
| 99 | |
| 100 # TODO | |
| 101 def build(prob): | |
| 102 raise NotImplementedError | |
| 16 | 103 |
| 104 def test(prob): | |
| 23 | 105 case = None |
| 22 | 106 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
107 contexts = [TestGroup()] |
| 22 | 108 for case in prob.testcases: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
109 if case is test_context_end: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
110 for case, correct, granted in contexts.pop().end(): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
111 contexts[-1].case_start(case) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
112 if correct: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
113 contexts[-1].case_correct() |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
114 contexts[-1].case_end(granted) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
115 continue |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
116 elif isinstance(case, TestContext): |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
117 contexts.append(case) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
118 continue |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
119 contexts[-1].case_start(case) |
| 22 | 120 granted = 0 |
| 121 id = str(case.id) | |
| 122 if case.isdummy: | |
| 123 id = 'sample ' + id | |
| 124 say('%*s: ' % (prob.cache.padoutput, id), end='') | |
| 125 sys.stdout.flush() | |
| 126 try: | |
| 127 granted = case(lambda: (say('%7.3f%s s, ' % (case.time_stopped - case.time_started, case.time_limit_string), end=''), sys.stdout.flush())) | |
| 128 except testcases.CanceledByUser: | |
| 129 verdict = 'canceled by the user' | |
| 130 except testcases.TimeLimitExceeded: | |
| 131 verdict = 'time limit exceeded' | |
| 132 except testcases.WrongAnswer: | |
| 133 e = sys.exc_info()[1] | |
| 134 if e.comment: | |
| 135 verdict = 'wrong answer (%s)' % e.comment | |
| 136 else: | |
| 137 verdict = 'wrong answer' | |
| 138 except testcases.NonZeroExitCode: | |
| 139 e = sys.exc_info()[1] | |
| 140 if e.exitcode < 0: | |
| 141 if sys.platform == 'win32': | |
| 142 verdict = 'terminated with error 0x%X' % (e.exitcode + 0x100000000) | |
| 143 elif -e.exitcode in signalnames: | |
| 144 verdict = 'terminated by signal %d (%s)' % (-e.exitcode, signalnames[-e.exitcode]) | |
| 145 else: | |
| 146 verdict = 'terminated by signal %d' % -e.exitcode | |
| 21 | 147 else: |
| 22 | 148 verdict = 'non-zero return code %d' % e.exitcode |
| 149 except testcases.CannotStartTestee: | |
| 26 | 150 verdict = 'cannot launch the program to test%s' % strerror(sys.exc_info()[1].upstream) |
| 22 | 151 except testcases.CannotStartValidator: |
| 26 | 152 verdict = 'cannot launch the validator%s' % strerror(sys.exc_info()[1].upstream) |
| 22 | 153 except testcases.CannotReadOutputFile: |
| 26 | 154 verdict = 'cannot read the output file%s' % strerror(sys.exc_info()[1].upstream) |
| 22 | 155 except testcases.CannotReadInputFile: |
| 26 | 156 verdict = 'cannot read the input file%s' % strerror(sys.exc_info()[1].upstream) |
| 22 | 157 except testcases.CannotReadAnswerFile: |
| 26 | 158 verdict = 'cannot read the reference output file%s' % strerror(sys.exc_info()[1].upstream) |
| 22 | 159 except testcases.TestCaseNotPassed: |
| 26 | 160 verdict = 'unspecified reason [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) |
| 22 | 161 #except Exception: |
| 26 | 162 # verdict = 'unknown error [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) |
| 21 | 163 else: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
164 try: |
| 22 | 165 granted, comment = granted |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
166 except TypeError: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
167 comment = '' |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
168 else: |
| 22 | 169 if comment: |
| 170 comment = ' (%s)' % comment | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
171 if granted >= 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
172 contexts[-1].case_correct() |
| 22 | 173 verdict = 'OK' + comment |
| 174 elif not granted: | |
| 175 verdict = 'wrong answer' + comment | |
| 176 else: | |
| 177 verdict = 'partly correct' + comment | |
| 26 | 178 granted *= case.points |
| 22 | 179 say('%g/%g, %s' % (granted, case.points, verdict)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
180 contexts[-1].case_end(granted) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
181 weighted = contexts[0].real * prob.config.taskweight / contexts[0].max if contexts[0].max else 0 |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
182 if contexts[0].nvalued != contexts[0].ntotal: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
183 say('Problem total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (contexts[0].ncorrect, contexts[0].ntotal, contexts[0].ncorrectvalued, contexts[0].nvalued, contexts[0].real, contexts[0].max, weighted, prob.config.taskweight)) |
| 21 | 184 else: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
185 say('Problem total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (contexts[0].ncorrect, contexts[0].ntotal, contexts[0].real, contexts[0].max, weighted, prob.config.taskweight)) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
186 sys.stdout.flush() |
| 22 | 187 return weighted, prob.config.taskweight |
| 188 finally: | |
| 23 | 189 if options.erase and (not prob.config.stdio or case and case.validator): |
| 22 | 190 for var in 'in', 'out': |
| 191 name = getattr(prob.config, var + 'name') | |
| 192 if name: | |
| 193 try: | |
| 194 os.remove(name) | |
| 195 except Exception: | |
| 196 pass | |
| 197 if case.validator and not callable(case.validator): | |
| 198 if prob.config.ansname: | |
| 199 try: | |
| 200 os.remove(prob.config.ansname) | |
| 201 except Exception: | |
| 202 pass |