Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/files.py @ 193:a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Specify match='regexp', and your tests and dummies will be treated
as regular expressions describing test case identifiers. Every file that
is in a suitable location and whose name matches {testcase,dummy}inname
and the given regexp will be treated as a file with test case input data.
You are free to use backreferences in the regexps, but group numbering
starts at two rather than one.
If you want test groups, you can get them magically created for you
by putting a part of the test ID in a group in the regexp sense
and specifying the tests variable as a pair consisting of the regexp
itself and the number of this regexp group (remember group numbers start
at two).
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Thu, 11 Aug 2011 23:20:52 +0300 |
| parents | 0d657576b1ac |
| children | 8c30a2c8a09e |
| rev | line source |
|---|---|
| 16 | 1 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
| 2 | |
| 21 | 3 """File access routines and classes with support for archives.""" |
| 4 | |
| 5 from __future__ import division, with_statement | |
| 6 | |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
133
diff
changeset
|
7 from .compat import * |
|
193
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
8 import contextlib, itertools, os, posixpath, re, shutil, sys |
| 21 | 9 |
| 10 # You don't need to know about anything else. | |
|
193
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
11 __all__ = 'File', 'regexp' |
| 21 | 12 |
| 13 # In these two variables, use full stops no matter what os.extsep is; | |
| 14 # all full stops will be converted to os.extsep on the fly | |
| 15 archives = 'tests.tar', 'tests.zip', 'tests.tgz', 'tests.tar.gz', 'tests.tbz2', 'tests.tar.bz2' | |
| 16 formats = {} | |
| 17 | |
| 18 class Archive(object): | |
| 19 __slots__ = 'file' | |
| 20 | |
| 21 if ABCMeta: | |
| 22 __metaclass__ = ABCMeta | |
| 23 | |
| 24 def __new__(cls, path): | |
| 25 """ | |
| 26 Create a new instance of the archive class corresponding | |
| 27 to the file name in the given path. | |
| 28 """ | |
| 29 if cls is not Archive: | |
| 30 return object.__new__(cls) | |
| 31 else: | |
| 32 # Do this by hand rather than through os.path.splitext | |
| 33 # because we support multi-dotted file name extensions | |
| 34 ext = path.partition(os.path.extsep)[2] | |
| 35 while ext: | |
| 36 if ext in formats: | |
| 37 return formats[ext](path) | |
| 38 ext = ext.partition(os.path.extsep)[2] | |
| 39 raise LookupError("unsupported archive file name extension in file name '%s'" % filename) | |
| 40 | |
| 41 @abstractmethod | |
| 42 def __init__(self, path): raise NotImplementedError | |
| 43 | |
| 44 @abstractmethod | |
| 45 def extract(self, name, target): raise NotImplementedError | |
| 16 | 46 |
| 21 | 47 try: |
| 48 import tarfile | |
| 31 | 49 except ImportError: |
| 50 TarArchive = None | |
| 51 else: | |
| 21 | 52 class TarArchive(Archive): |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
53 __slots__ = '_namelist' |
| 21 | 54 |
| 55 def __init__(self, path): | |
| 56 self.file = tarfile.open(path) | |
| 57 | |
| 58 def extract(self, name, target): | |
| 59 member = self.file.getmember(name) | |
| 60 member.name = target | |
| 61 self.file.extract(member) | |
| 62 | |
| 63 def open(self, name): | |
| 64 return self.file.extractfile(name) | |
| 65 | |
| 66 def exists(self, queried_name): | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
67 if not hasattr(self, '_namelist'): |
| 21 | 68 names = set() |
| 69 for name in self.file.getnames(): | |
| 70 cutname = name | |
| 71 while cutname: | |
| 72 names.add(cutname) | |
| 73 cutname = cutname.rpartition('/')[0] | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
74 self._namelist = frozenset(names) |
|
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
75 return queried_name in self._namelist |
| 21 | 76 |
| 77 def __enter__(self): | |
| 78 if hasattr(self.file, '__enter__'): | |
| 79 self.file.__enter__() | |
| 80 return self | |
| 81 | |
| 82 def __exit__(self, exc_type, exc_value, traceback): | |
| 83 if hasattr(self.file, '__exit__'): | |
| 84 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 85 elif exc_type is None: | |
| 86 self.file.close() | |
| 87 else: | |
| 88 # This code was shamelessly copied from tarfile.py of Python 2.7 | |
| 89 if not self.file._extfileobj: | |
| 90 self.file.fileobj.close() | |
| 91 self.file.closed = True | |
| 92 | |
| 93 formats['tar'] = formats['tgz'] = formats['tar.gz'] = formats['tbz2'] = formats['tar.bz2'] = TarArchive | |
| 94 | |
| 95 try: | |
| 96 import zipfile | |
| 31 | 97 except ImportError: |
| 98 ZipArchive = None | |
| 99 else: | |
| 21 | 100 class ZipArchive(Archive): |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
101 __slots__ = '_namelist' |
| 21 | 102 |
| 103 def __init__(self, path): | |
| 104 self.file = zipfile.ZipFile(path) | |
| 105 | |
| 106 def extract(self, name, target): | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
107 member = self.file.getinfo(name) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
108 # FIXME: 2.5 lacks ZipFile.extract |
| 21 | 109 if os.path.isabs(target): |
| 110 # To my knowledge, this is as portable as it gets | |
| 111 path = os.path.join(os.path.splitdrive(target)[0], os.path.sep) | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
112 member.filename = os.path.relpath(target, path) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
113 self.file.extract(member, path) |
| 21 | 114 else: |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
115 member.filename = os.path.relpath(target) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
116 self.file.extract(member) |
| 21 | 117 |
| 118 def open(self, name): | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
155
diff
changeset
|
119 return self.file.open(name, 'r') |
| 21 | 120 |
| 121 def exists(self, queried_name): | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
122 if not hasattr(self, '_namelist'): |
| 21 | 123 names = set() |
| 124 for name in self.file.namelist(): | |
| 125 cutname = name | |
| 126 while cutname: | |
| 127 names.add(cutname) | |
| 128 cutname = cutname.rpartition('/')[0] | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
129 self._namelist = frozenset(names) |
|
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
130 return queried_name in self._namelist |
| 21 | 131 |
| 132 def __enter__(self): | |
| 133 if hasattr(self.file, '__enter__'): | |
| 134 self.file.__enter__() | |
| 135 return self | |
| 136 | |
| 137 def __exit__(self, exc_type, exc_value, traceback): | |
| 138 if hasattr(self.file, '__exit__'): | |
| 139 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 140 else: | |
| 141 return self.file.close() | |
| 142 | |
| 143 formats['zip'] = ZipArchive | |
| 144 | |
| 145 # Remove unsupported archive formats and replace full stops | |
| 146 # with the platform-dependent file name extension separator | |
| 147 def issupported(filename, formats=formats): | |
| 148 ext = filename.partition('.')[2] | |
| 149 while ext: | |
| 150 if ext in formats: return True | |
| 151 ext = ext.partition('.')[2] | |
| 152 return False | |
| 153 archives = [filename.replace('.', os.path.extsep) for filename in filter(issupported, archives)] | |
| 154 formats = dict((item[0].replace('.', os.path.extsep), item[1]) for item in items(formats)) | |
| 155 | |
| 156 open_archives = {} | |
| 157 | |
| 158 def open_archive(path): | |
| 159 if path in open_archives: | |
| 160 return open_archives[path] | |
| 161 else: | |
| 162 open_archives[path] = archive = Archive(path) | |
| 163 return archive | |
| 16 | 164 |
| 21 | 165 class File(object): |
| 166 __slots__ = 'virtual_path', 'real_path', 'full_real_path', 'archive' | |
| 167 | |
| 168 def __init__(self, virtpath, allow_root=False, msg='test data'): | |
| 169 self.virtual_path = virtpath | |
| 170 self.archive = None | |
| 171 if not self.realize_path('', tuple(comp.replace('.', os.path.extsep) for comp in virtpath.split('/')), allow_root): | |
| 172 raise IOError("%s file '%s' could not be found" % (msg, virtpath)) | |
| 173 | |
| 174 def realize_path(self, root, virtpath, allow_root=False, hastests=False): | |
| 175 if root and not os.path.exists(root): | |
| 176 return False | |
| 177 if len(virtpath) > 1: | |
| 178 if self.realize_path(os.path.join(root, virtpath[0]), virtpath[1:], allow_root, hastests): | |
| 179 return True | |
| 180 elif not hastests: | |
| 181 if self.realize_path(os.path.join(root, 'tests'), virtpath, allow_root, True): | |
| 182 return True | |
| 183 for archive in archives: | |
| 184 path = os.path.join(root, archive) | |
| 185 if os.path.exists(path): | |
| 186 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 187 return True | |
| 23 | 188 if self.realize_path(root, virtpath[1:], allow_root, hastests): |
| 21 | 189 return True |
| 190 else: | |
| 191 if not hastests: | |
| 192 path = os.path.join(root, 'tests', virtpath[0]) | |
| 193 if os.path.exists(path): | |
| 194 self.full_real_path = self.real_path = path | |
| 195 return True | |
| 196 for archive in archives: | |
| 197 path = os.path.join(root, archive) | |
| 198 if os.path.exists(path): | |
| 199 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 200 return True | |
| 201 if hastests or allow_root: | |
| 202 path = os.path.join(root, virtpath[0]) | |
| 203 if os.path.exists(path): | |
| 204 self.full_real_path = self.real_path = path | |
| 205 return True | |
| 206 return False | |
| 207 | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
208 def realize_path_archive(self, archive, root, virtpath, archpath, hastests=False): |
| 21 | 209 if root and not archive.exists(root): |
| 210 return False | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
211 path = posixpath.join(root, virtpath[0]) |
| 21 | 212 if len(virtpath) > 1: |
| 213 if self.realize_path_archive(archive, path, virtpath[1:], archpath): | |
| 214 return True | |
| 215 elif self.realize_path_archive(archive, root, virtpath[1:], archpath): | |
| 216 return True | |
| 217 else: | |
| 218 if archive.exists(path): | |
| 219 self.archive = archive | |
| 220 self.real_path = path | |
| 221 self.full_real_path = os.path.join(archpath, *path.split('/')) | |
| 222 return True | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
223 if not hastests: |
|
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
224 if self.realize_path_archive(archive, posixpath.join(root, 'tests'), virtpath, archpath, True): |
|
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
225 return True |
| 21 | 226 return False |
| 227 | |
| 228 def open(self): | |
| 229 if self.archive: | |
| 230 file = self.archive.open(self.real_path) | |
| 231 if hasattr(file, '__exit__'): | |
| 232 return file | |
| 233 else: | |
| 234 return contextlib.closing(file) | |
| 235 else: | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
155
diff
changeset
|
236 return open(self.real_path, 'rb') |
| 21 | 237 |
| 238 def copy(self, target): | |
| 239 if self.archive: | |
| 240 self.archive.extract(self.real_path, target) | |
| 241 else: | |
|
193
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
242 shutil.copy(self.real_path, target) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
243 |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
244 def regexp(pattern, hastests=False, droplast=False): |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
245 # FIXME: add test archives |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
246 if not pattern: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
247 yield os.curdir, '' |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
248 return |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
249 dirname, basename = posixpath.split(pattern) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
250 if hastests: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
251 dirnames = regexp(dirname, True) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
252 else: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
253 dirnames = itertools.chain(regexp(dirname), regexp(posixpath.join(dirname, 'tests'), True, True)) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
254 reobj = re.compile(pattern + '$', re.UNICODE) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
255 for dirname, vdirname in dirnames: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
256 try: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
257 names = os.listdir(dirname) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
258 except OSError: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
259 continue |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
260 for name in names: |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
261 path = os.path.join(dirname, name) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
262 vpath = posixpath.join(vdirname, name) |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
263 if re.match(reobj, vpath): |
|
a76cdc26ba9d
Added conf. var. match and match='regexp' for non-archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
178
diff
changeset
|
264 yield path, vpath if not droplast else vdirname |