Mercurial > ~astiob > upreckon > hgweb
annotate upreckon/files.py @ 178:0d657576b1ac
tests directories are now searched for within archives
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Mon, 20 Jun 2011 17:52:09 +0300 |
| parents | e0b2fbd7ebe0 |
| children | a76cdc26ba9d |
| rev | line source |
|---|---|
| 16 | 1 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
| 2 | |
| 21 | 3 """File access routines and classes with support for archives.""" |
| 4 | |
| 5 from __future__ import division, with_statement | |
| 6 | |
|
146
d5b6708c1955
Distutils support, reorganization and cleaning up
Oleg Oshmyan <chortos@inbox.lv>
parents:
133
diff
changeset
|
7 from .compat import * |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
8 import contextlib, os, posixpath, shutil, sys |
| 21 | 9 |
| 10 # You don't need to know about anything else. | |
| 11 __all__ = 'File', | |
| 12 | |
| 13 # In these two variables, use full stops no matter what os.extsep is; | |
| 14 # all full stops will be converted to os.extsep on the fly | |
| 15 archives = 'tests.tar', 'tests.zip', 'tests.tgz', 'tests.tar.gz', 'tests.tbz2', 'tests.tar.bz2' | |
| 16 formats = {} | |
| 17 | |
| 18 class Archive(object): | |
| 19 __slots__ = 'file' | |
| 20 | |
| 21 if ABCMeta: | |
| 22 __metaclass__ = ABCMeta | |
| 23 | |
| 24 def __new__(cls, path): | |
| 25 """ | |
| 26 Create a new instance of the archive class corresponding | |
| 27 to the file name in the given path. | |
| 28 """ | |
| 29 if cls is not Archive: | |
| 30 return object.__new__(cls) | |
| 31 else: | |
| 32 # Do this by hand rather than through os.path.splitext | |
| 33 # because we support multi-dotted file name extensions | |
| 34 ext = path.partition(os.path.extsep)[2] | |
| 35 while ext: | |
| 36 if ext in formats: | |
| 37 return formats[ext](path) | |
| 38 ext = ext.partition(os.path.extsep)[2] | |
| 39 raise LookupError("unsupported archive file name extension in file name '%s'" % filename) | |
| 40 | |
| 41 @abstractmethod | |
| 42 def __init__(self, path): raise NotImplementedError | |
| 43 | |
| 44 @abstractmethod | |
| 45 def extract(self, name, target): raise NotImplementedError | |
| 16 | 46 |
| 21 | 47 try: |
| 48 import tarfile | |
| 31 | 49 except ImportError: |
| 50 TarArchive = None | |
| 51 else: | |
| 21 | 52 class TarArchive(Archive): |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
53 __slots__ = '_namelist' |
| 21 | 54 |
| 55 def __init__(self, path): | |
| 56 self.file = tarfile.open(path) | |
| 57 | |
| 58 def extract(self, name, target): | |
| 59 member = self.file.getmember(name) | |
| 60 member.name = target | |
| 61 self.file.extract(member) | |
| 62 | |
| 63 def open(self, name): | |
| 64 return self.file.extractfile(name) | |
| 65 | |
| 66 def exists(self, queried_name): | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
67 if not hasattr(self, '_namelist'): |
| 21 | 68 names = set() |
| 69 for name in self.file.getnames(): | |
| 70 cutname = name | |
| 71 while cutname: | |
| 72 names.add(cutname) | |
| 73 cutname = cutname.rpartition('/')[0] | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
74 self._namelist = frozenset(names) |
|
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
75 return queried_name in self._namelist |
| 21 | 76 |
| 77 def __enter__(self): | |
| 78 if hasattr(self.file, '__enter__'): | |
| 79 self.file.__enter__() | |
| 80 return self | |
| 81 | |
| 82 def __exit__(self, exc_type, exc_value, traceback): | |
| 83 if hasattr(self.file, '__exit__'): | |
| 84 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 85 elif exc_type is None: | |
| 86 self.file.close() | |
| 87 else: | |
| 88 # This code was shamelessly copied from tarfile.py of Python 2.7 | |
| 89 if not self.file._extfileobj: | |
| 90 self.file.fileobj.close() | |
| 91 self.file.closed = True | |
| 92 | |
| 93 formats['tar'] = formats['tgz'] = formats['tar.gz'] = formats['tbz2'] = formats['tar.bz2'] = TarArchive | |
| 94 | |
| 95 try: | |
| 96 import zipfile | |
| 31 | 97 except ImportError: |
| 98 ZipArchive = None | |
| 99 else: | |
| 21 | 100 class ZipArchive(Archive): |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
101 __slots__ = '_namelist' |
| 21 | 102 |
| 103 def __init__(self, path): | |
| 104 self.file = zipfile.ZipFile(path) | |
| 105 | |
| 106 def extract(self, name, target): | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
107 member = self.file.getinfo(name) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
108 # FIXME: 2.5 lacks ZipFile.extract |
| 21 | 109 if os.path.isabs(target): |
| 110 # To my knowledge, this is as portable as it gets | |
| 111 path = os.path.join(os.path.splitdrive(target)[0], os.path.sep) | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
112 member.filename = os.path.relpath(target, path) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
113 self.file.extract(member, path) |
| 21 | 114 else: |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
115 member.filename = os.path.relpath(target) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
116 self.file.extract(member) |
| 21 | 117 |
| 118 def open(self, name): | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
155
diff
changeset
|
119 return self.file.open(name, 'r') |
| 21 | 120 |
| 121 def exists(self, queried_name): | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
122 if not hasattr(self, '_namelist'): |
| 21 | 123 names = set() |
| 124 for name in self.file.namelist(): | |
| 125 cutname = name | |
| 126 while cutname: | |
| 127 names.add(cutname) | |
| 128 cutname = cutname.rpartition('/')[0] | |
|
132
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
129 self._namelist = frozenset(names) |
|
cdd0f970d112
Fixed several small bugs in the files module
Oleg Oshmyan <chortos@inbox.lv>
parents:
98
diff
changeset
|
130 return queried_name in self._namelist |
| 21 | 131 |
| 132 def __enter__(self): | |
| 133 if hasattr(self.file, '__enter__'): | |
| 134 self.file.__enter__() | |
| 135 return self | |
| 136 | |
| 137 def __exit__(self, exc_type, exc_value, traceback): | |
| 138 if hasattr(self.file, '__exit__'): | |
| 139 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 140 else: | |
| 141 return self.file.close() | |
| 142 | |
| 143 formats['zip'] = ZipArchive | |
| 144 | |
| 145 # Remove unsupported archive formats and replace full stops | |
| 146 # with the platform-dependent file name extension separator | |
| 147 def issupported(filename, formats=formats): | |
| 148 ext = filename.partition('.')[2] | |
| 149 while ext: | |
| 150 if ext in formats: return True | |
| 151 ext = ext.partition('.')[2] | |
| 152 return False | |
| 153 archives = [filename.replace('.', os.path.extsep) for filename in filter(issupported, archives)] | |
| 154 formats = dict((item[0].replace('.', os.path.extsep), item[1]) for item in items(formats)) | |
| 155 | |
| 156 open_archives = {} | |
| 157 | |
| 158 def open_archive(path): | |
| 159 if path in open_archives: | |
| 160 return open_archives[path] | |
| 161 else: | |
| 162 open_archives[path] = archive = Archive(path) | |
| 163 return archive | |
| 16 | 164 |
| 21 | 165 class File(object): |
| 166 __slots__ = 'virtual_path', 'real_path', 'full_real_path', 'archive' | |
| 167 | |
| 168 def __init__(self, virtpath, allow_root=False, msg='test data'): | |
| 169 self.virtual_path = virtpath | |
| 170 self.archive = None | |
| 171 if not self.realize_path('', tuple(comp.replace('.', os.path.extsep) for comp in virtpath.split('/')), allow_root): | |
| 172 raise IOError("%s file '%s' could not be found" % (msg, virtpath)) | |
| 173 | |
| 174 def realize_path(self, root, virtpath, allow_root=False, hastests=False): | |
| 175 if root and not os.path.exists(root): | |
| 176 return False | |
| 177 if len(virtpath) > 1: | |
| 178 if self.realize_path(os.path.join(root, virtpath[0]), virtpath[1:], allow_root, hastests): | |
| 179 return True | |
| 180 elif not hastests: | |
| 181 if self.realize_path(os.path.join(root, 'tests'), virtpath, allow_root, True): | |
| 182 return True | |
| 183 for archive in archives: | |
| 184 path = os.path.join(root, archive) | |
| 185 if os.path.exists(path): | |
| 186 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 187 return True | |
| 23 | 188 if self.realize_path(root, virtpath[1:], allow_root, hastests): |
| 21 | 189 return True |
| 190 else: | |
| 191 if not hastests: | |
| 192 path = os.path.join(root, 'tests', virtpath[0]) | |
| 193 if os.path.exists(path): | |
| 194 self.full_real_path = self.real_path = path | |
| 195 return True | |
| 196 for archive in archives: | |
| 197 path = os.path.join(root, archive) | |
| 198 if os.path.exists(path): | |
| 199 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 200 return True | |
| 201 if hastests or allow_root: | |
| 202 path = os.path.join(root, virtpath[0]) | |
| 203 if os.path.exists(path): | |
| 204 self.full_real_path = self.real_path = path | |
| 205 return True | |
| 206 return False | |
| 207 | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
208 def realize_path_archive(self, archive, root, virtpath, archpath, hastests=False): |
| 21 | 209 if root and not archive.exists(root): |
| 210 return False | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
211 path = posixpath.join(root, virtpath[0]) |
| 21 | 212 if len(virtpath) > 1: |
| 213 if self.realize_path_archive(archive, path, virtpath[1:], archpath): | |
| 214 return True | |
| 215 elif self.realize_path_archive(archive, root, virtpath[1:], archpath): | |
| 216 return True | |
| 217 else: | |
| 218 if archive.exists(path): | |
| 219 self.archive = archive | |
| 220 self.real_path = path | |
| 221 self.full_real_path = os.path.join(archpath, *path.split('/')) | |
| 222 return True | |
|
178
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
223 if not hastests: |
|
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
224 if self.realize_path_archive(archive, posixpath.join(root, 'tests'), virtpath, archpath, True): |
|
0d657576b1ac
tests directories are now searched for within archives
Oleg Oshmyan <chortos@inbox.lv>
parents:
174
diff
changeset
|
225 return True |
| 21 | 226 return False |
| 227 | |
| 228 def open(self): | |
| 229 if self.archive: | |
| 230 file = self.archive.open(self.real_path) | |
| 231 if hasattr(file, '__exit__'): | |
| 232 return file | |
| 233 else: | |
| 234 return contextlib.closing(file) | |
| 235 else: | |
|
174
e0b2fbd7ebe0
Improved built-in output validator; added conf. var. binary
Oleg Oshmyan <chortos@inbox.lv>
parents:
155
diff
changeset
|
236 return open(self.real_path, 'rb') |
| 21 | 237 |
| 238 def copy(self, target): | |
| 239 if self.archive: | |
| 240 self.archive.extract(self.real_path, target) | |
| 241 else: | |
| 242 shutil.copy(self.real_path, target) |