Mercurial > ~astiob > upreckon > hgweb

comparison problem.py @ 76:0e5ae28e0b2b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Points are now weighted on a test context basis In particular, this has allowed for simple extensions to the format of testconf to award points to whole test groups without at the same time compromising the future ability of giving partial score for correct but slow solutions. Specifically, the groupweight configuration variable has been added and normally has the format {groupindex: points} where groupindex is the group's index in the tests configuration variable. The backwards incompatible change is that test contexts are no longer guaranteed to learn the score awarded or the maximum possible score for every test case and may instead be notified about them in batches. In other news, the pointmap and groupweight configuration variables can (now) be given as sequences in addition to mappings. (Technically, the distinction currently made is dict versus everything else.) Items of a sequence pointmap/groupweight correspond directly to the test cases/ groups defined in the tests configuration variable; in particular, when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be written as pointmap=tests=[1],[2,3]. Missing items are handled in the same way in which they are handled when the variable is a mapping. Note that the items of groupweight correspond to whole test groups rather than individual test cases. In other news again, the wording of problem total lines has been changed from '<unweighted> points; weighted score: <weighted>' to '<weighted> points (<unweighted> before weighting)', and group total lines now properly report fractional numbers of points (this is a bug fix).
author Oleg Oshmyan <chortos@inbox.lv>
date Sat, 08 Jan 2011 16:03:35 +0200
parents 007f7eb6fb2b
children 69eadc60f4e2
comparison
equal deleted inserted replaced
75:007f7eb6fb2b 76:0e5ae28e0b2b
50 pass 50 pass
51 51
52 test_context_end = object() 52 test_context_end = object()
53 53
54 class TestGroup(TestContext): 54 class TestGroup(TestContext):
55 __slots__ = 'case', 'log', 'correct', 'allcorrect', 'real', 'max', 'ntotal', 'nvalued', 'ncorrect', 'ncorrectvalued' 55 __slots__ = 'points', 'case', 'log', 'correct', 'allcorrect', 'real', 'max', 'ntotal', 'nvalued', 'ncorrect', 'ncorrectvalued'
56 56
57 def __init__(self): 57 def __init__(self, points=None):
58 self.points = points
58 self.real = self.max = self.ntotal = self.nvalued = self.ncorrect = self.ncorrectvalued = 0 59 self.real = self.max = self.ntotal = self.nvalued = self.ncorrect = self.ncorrectvalued = 0
59 self.allcorrect = True 60 self.allcorrect = True
60 self.log = [] 61 self.log = []
61 62
62 def case_start(self, case): 63 def case_start(self, case):
63 self.case = case 64 self.case = case
64 self.correct = False 65 self.correct = False
65 self.ntotal += 1 66 self.ntotal += 1
66 self.max += case.points
67 if case.points: 67 if case.points:
68 self.nvalued += 1 68 self.nvalued += 1
69 69
70 def case_correct(self): 70 def case_correct(self):
71 self.correct = True 71 self.correct = True
72 self.ncorrect += 1 72 self.ncorrect += 1
73 if self.case.points: 73 if self.case.points:
74 self.ncorrectvalued += 1 74 self.ncorrectvalued += 1
75 75
76 def case_end(self, granted): 76 def case_end(self):
77 self.log.append((self.case, self.correct, granted)) 77 self.log.append((self.case, self.correct))
78 self.real += granted
79 del self.case 78 del self.case
80 if not self.correct: 79 if not self.correct:
81 self.allcorrect = False 80 self.allcorrect = False
82 81
82 def score(self, real, max):
83 self.real += real
84 self.max += max
85
83 def end(self): 86 def end(self):
84 say('Group total: %d/%d tests; %d/%d points' % (self.ncorrect, self.ntotal, self.real if self.allcorrect else 0, self.max)) 87 if not self.allcorrect:
88 self.real = 0
89 if self.points is not None and self.points != self.max:
90 max, weighted = self.points, self.real * self.points / self.max if self.max else 0
91 before_weighting = ' (%g/%g before weighting)' % (self.real, self.max)
92 else:
93 max, weighted = self.max, self.real
94 before_weighting = ''
95 say('Group total: %d/%d tests, %g/%g points%s' % (self.ncorrect, self.ntotal, weighted, max, before_weighting))
85 # No real need to flush stdout, as it will anyway be flushed in a moment, 96 # No real need to flush stdout, as it will anyway be flushed in a moment,
86 # when either the problem total or the next test case's ID is printed 97 # when either the problem total or the next test case's ID is printed
87 if self.allcorrect: 98 return weighted, max, self.log
88 return self.log
89 else:
90 return ((case, correct, 0) for case, correct, granted in self.log)
91 99
92 class Problem(object): 100 class Problem(object):
93 __slots__ = 'name', 'config', 'cache', 'testcases' 101 __slots__ = 'name', 'config', 'cache', 'testcases'
94 102
95 def __init__(prob, name): 103 def __init__(prob, name):
109 case = None 117 case = None
110 try: 118 try:
111 contexts = deque((TestGroup(),)) 119 contexts = deque((TestGroup(),))
112 for case in prob.testcases: 120 for case in prob.testcases:
113 if case is test_context_end: 121 if case is test_context_end:
114 for case, correct, granted in contexts.pop().end(): 122 real, max, log = contexts.pop().end()
123 for case, correct in log:
115 contexts[-1].case_start(case) 124 contexts[-1].case_start(case)
116 if correct: 125 if correct:
117 contexts[-1].case_correct() 126 contexts[-1].case_correct()
118 contexts[-1].case_end(granted) 127 contexts[-1].case_end()
128 contexts[-1].score(real, max)
119 continue 129 continue
120 elif isinstance(case, TestContext): 130 elif isinstance(case, TestContext):
121 contexts.append(case) 131 contexts.append(case)
122 continue 132 continue
123 contexts[-1].case_start(case) 133 contexts[-1].case_start(case)
179 verdict = 'wrong answer' + comment 189 verdict = 'wrong answer' + comment
180 else: 190 else:
181 verdict = 'partly correct' + comment 191 verdict = 'partly correct' + comment
182 granted *= case.points 192 granted *= case.points
183 say('%g/%g, %s' % (granted, case.points, verdict)) 193 say('%g/%g, %s' % (granted, case.points, verdict))
184 contexts[-1].case_end(granted) 194 contexts[-1].case_end()
195 contexts[-1].score(granted, case.points)
185 weighted = contexts[0].real * prob.config.taskweight / contexts[0].max if contexts[0].max else 0 196 weighted = contexts[0].real * prob.config.taskweight / contexts[0].max if contexts[0].max else 0
197 before_weighting = valued = ''
198 if prob.config.taskweight != contexts[0].max:
199 before_weighting = ' (%g/%g before weighting)' % (contexts[0].real, contexts[0].max)
186 if contexts[0].nvalued != contexts[0].ntotal: 200 if contexts[0].nvalued != contexts[0].ntotal:
187 say('Problem total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (contexts[0].ncorrect, contexts[0].ntotal, contexts[0].ncorrectvalued, contexts[0].nvalued, contexts[0].real, contexts[0].max, weighted, prob.config.taskweight)) 201 valued = ' (%d/%d valued)' % (contexts[0].ncorrectvalued, contexts[0].nvalued)
188 else: 202 say('Problem total: %d/%d tests%s, %g/%g points%s' % (contexts[0].ncorrect, contexts[0].ntotal, valued, weighted, prob.config.taskweight, before_weighting))
189 say('Problem total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (contexts[0].ncorrect, contexts[0].ntotal, contexts[0].real, contexts[0].max, weighted, prob.config.taskweight))
190 sys.stdout.flush() 203 sys.stdout.flush()
191 return weighted, prob.config.taskweight 204 return weighted, prob.config.taskweight
192 finally: 205 finally:
193 if options.erase and (not prob.config.stdio or case and case.validator): 206 if options.erase and (not prob.config.stdio or case and case.validator):
194 for var in 'in', 'out': 207 for var in 'in', 'out':