Mercurial > ~astiob > upreckon > hgweb
annotate files.py @ 124:19c42a3cd962
Very fast testees no longer break Popen on some UNIX systems on Python 2.6+
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Sun, 24 Apr 2011 20:05:20 +0100 |
| parents | 62a96d51bf94 |
| children | cdd0f970d112 |
| rev | line source |
|---|---|
| 16 | 1 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
| 2 | |
| 21 | 3 """File access routines and classes with support for archives.""" |
| 4 | |
| 5 from __future__ import division, with_statement | |
| 6 | |
| 91 | 7 from compat import * |
| 21 | 8 import contextlib, os, shutil, sys |
| 9 | |
| 10 # You don't need to know about anything else. | |
| 11 __all__ = 'File', | |
| 12 | |
| 13 # In these two variables, use full stops no matter what os.extsep is; | |
| 14 # all full stops will be converted to os.extsep on the fly | |
| 15 archives = 'tests.tar', 'tests.zip', 'tests.tgz', 'tests.tar.gz', 'tests.tbz2', 'tests.tar.bz2' | |
| 16 formats = {} | |
| 17 | |
| 18 class Archive(object): | |
| 19 __slots__ = 'file' | |
| 20 | |
| 21 if ABCMeta: | |
| 22 __metaclass__ = ABCMeta | |
| 23 | |
| 24 def __new__(cls, path): | |
| 25 """ | |
| 26 Create a new instance of the archive class corresponding | |
| 27 to the file name in the given path. | |
| 28 """ | |
| 29 if cls is not Archive: | |
| 30 return object.__new__(cls) | |
| 31 else: | |
| 32 # Do this by hand rather than through os.path.splitext | |
| 33 # because we support multi-dotted file name extensions | |
| 34 ext = path.partition(os.path.extsep)[2] | |
| 35 while ext: | |
| 36 if ext in formats: | |
| 37 return formats[ext](path) | |
| 38 ext = ext.partition(os.path.extsep)[2] | |
| 39 raise LookupError("unsupported archive file name extension in file name '%s'" % filename) | |
| 40 | |
| 41 @abstractmethod | |
| 42 def __init__(self, path): raise NotImplementedError | |
| 43 | |
| 44 @abstractmethod | |
| 45 def extract(self, name, target): raise NotImplementedError | |
| 46 | |
| 47 def __del__(self): | |
| 48 del self.file | |
| 16 | 49 |
| 21 | 50 try: |
| 51 import tarfile | |
| 31 | 52 except ImportError: |
| 53 TarArchive = None | |
| 54 else: | |
| 21 | 55 class TarArchive(Archive): |
| 56 __slots__ = '__namelist' | |
| 57 | |
| 58 def __init__(self, path): | |
| 59 self.file = tarfile.open(path) | |
| 60 | |
| 61 def extract(self, name, target): | |
| 62 member = self.file.getmember(name) | |
| 63 member.name = target | |
| 64 self.file.extract(member) | |
| 65 | |
| 66 # TODO: somehow automagically emulate universal line break support | |
| 67 def open(self, name): | |
| 68 return self.file.extractfile(name) | |
| 69 | |
| 70 def exists(self, queried_name): | |
| 71 if not hasattr(self, '__namelist'): | |
| 72 names = set() | |
| 73 for name in self.file.getnames(): | |
| 74 cutname = name | |
| 75 while cutname: | |
| 76 names.add(cutname) | |
| 77 cutname = cutname.rpartition('/')[0] | |
| 78 self.__namelist = frozenset(names) | |
| 79 return queried_name in self.__namelist | |
| 80 | |
| 81 def __enter__(self): | |
| 82 if hasattr(self.file, '__enter__'): | |
| 83 self.file.__enter__() | |
| 84 return self | |
| 85 | |
| 86 def __exit__(self, exc_type, exc_value, traceback): | |
| 87 if hasattr(self.file, '__exit__'): | |
| 88 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 89 elif exc_type is None: | |
| 90 self.file.close() | |
| 91 else: | |
| 92 # This code was shamelessly copied from tarfile.py of Python 2.7 | |
| 93 if not self.file._extfileobj: | |
| 94 self.file.fileobj.close() | |
| 95 self.file.closed = True | |
| 96 | |
| 97 formats['tar'] = formats['tgz'] = formats['tar.gz'] = formats['tbz2'] = formats['tar.bz2'] = TarArchive | |
| 98 | |
| 99 try: | |
| 100 import zipfile | |
| 31 | 101 except ImportError: |
| 102 ZipArchive = None | |
| 103 else: | |
| 21 | 104 class ZipArchive(Archive): |
| 105 __slots__ = '__namelist' | |
| 106 | |
| 107 def __init__(self, path): | |
| 108 self.file = zipfile.ZipFile(path) | |
| 109 | |
| 110 def extract(self, name, target): | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
111 member = self.file.getinfo(name) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
112 # FIXME: 2.5 lacks ZipFile.extract |
| 21 | 113 if os.path.isabs(target): |
| 114 # To my knowledge, this is as portable as it gets | |
| 115 path = os.path.join(os.path.splitdrive(target)[0], os.path.sep) | |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
116 member.filename = os.path.relpath(target, path) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
117 self.file.extract(member, path) |
| 21 | 118 else: |
|
98
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
119 member.filename = os.path.relpath(target) |
|
62a96d51bf94
Fixed ZipArchive.extract with relative paths on Windows
Oleg Oshmyan <chortos@inbox.lv>
parents:
91
diff
changeset
|
120 self.file.extract(member) |
| 21 | 121 |
| 122 def open(self, name): | |
| 123 return self.file.open(name, 'rU') | |
| 124 | |
| 125 def exists(self, queried_name): | |
| 126 if not hasattr(self, '__namelist'): | |
| 127 names = set() | |
| 128 for name in self.file.namelist(): | |
| 129 cutname = name | |
| 130 while cutname: | |
| 131 names.add(cutname) | |
| 132 cutname = cutname.rpartition('/')[0] | |
| 133 self.__namelist = frozenset(names) | |
| 134 return queried_name in self.__namelist | |
| 135 | |
| 136 def __enter__(self): | |
| 137 if hasattr(self.file, '__enter__'): | |
| 138 self.file.__enter__() | |
| 139 return self | |
| 140 | |
| 141 def __exit__(self, exc_type, exc_value, traceback): | |
| 142 if hasattr(self.file, '__exit__'): | |
| 143 return self.file.__exit__(exc_type, exc_value, traceback) | |
| 144 else: | |
| 145 return self.file.close() | |
| 146 | |
| 147 formats['zip'] = ZipArchive | |
| 148 | |
| 149 # Remove unsupported archive formats and replace full stops | |
| 150 # with the platform-dependent file name extension separator | |
| 151 def issupported(filename, formats=formats): | |
| 152 ext = filename.partition('.')[2] | |
| 153 while ext: | |
| 154 if ext in formats: return True | |
| 155 ext = ext.partition('.')[2] | |
| 156 return False | |
| 157 archives = [filename.replace('.', os.path.extsep) for filename in filter(issupported, archives)] | |
| 158 formats = dict((item[0].replace('.', os.path.extsep), item[1]) for item in items(formats)) | |
| 159 | |
| 160 open_archives = {} | |
| 161 | |
| 162 def open_archive(path): | |
| 163 if path in open_archives: | |
| 164 return open_archives[path] | |
| 165 else: | |
| 166 open_archives[path] = archive = Archive(path) | |
| 167 return archive | |
| 16 | 168 |
| 21 | 169 class File(object): |
| 170 __slots__ = 'virtual_path', 'real_path', 'full_real_path', 'archive' | |
| 171 | |
| 172 def __init__(self, virtpath, allow_root=False, msg='test data'): | |
| 173 self.virtual_path = virtpath | |
| 174 self.archive = None | |
| 175 if not self.realize_path('', tuple(comp.replace('.', os.path.extsep) for comp in virtpath.split('/')), allow_root): | |
| 176 raise IOError("%s file '%s' could not be found" % (msg, virtpath)) | |
| 177 | |
| 178 def realize_path(self, root, virtpath, allow_root=False, hastests=False): | |
| 179 if root and not os.path.exists(root): | |
| 180 return False | |
| 181 if len(virtpath) > 1: | |
| 182 if self.realize_path(os.path.join(root, virtpath[0]), virtpath[1:], allow_root, hastests): | |
| 183 return True | |
| 184 elif not hastests: | |
| 185 if self.realize_path(os.path.join(root, 'tests'), virtpath, allow_root, True): | |
| 186 return True | |
| 187 for archive in archives: | |
| 188 path = os.path.join(root, archive) | |
| 189 if os.path.exists(path): | |
| 190 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 191 return True | |
| 23 | 192 if self.realize_path(root, virtpath[1:], allow_root, hastests): |
| 21 | 193 return True |
| 194 else: | |
| 195 if not hastests: | |
| 196 path = os.path.join(root, 'tests', virtpath[0]) | |
| 197 if os.path.exists(path): | |
| 198 self.full_real_path = self.real_path = path | |
| 199 return True | |
| 200 for archive in archives: | |
| 201 path = os.path.join(root, archive) | |
| 202 if os.path.exists(path): | |
| 203 if self.realize_path_archive(open_archive(path), '', virtpath, path): | |
| 204 return True | |
| 205 if hastests or allow_root: | |
| 206 path = os.path.join(root, virtpath[0]) | |
| 207 if os.path.exists(path): | |
| 208 self.full_real_path = self.real_path = path | |
| 209 return True | |
| 210 return False | |
| 211 | |
| 212 def realize_path_archive(self, archive, root, virtpath, archpath): | |
| 213 if root and not archive.exists(root): | |
| 214 return False | |
| 215 if root: path = ''.join((root, '/', virtpath[0])) | |
| 216 else: path = virtpath[0] | |
| 217 if len(virtpath) > 1: | |
| 218 if self.realize_path_archive(archive, path, virtpath[1:], archpath): | |
| 219 return True | |
| 220 elif self.realize_path_archive(archive, root, virtpath[1:], archpath): | |
| 221 return True | |
| 222 else: | |
| 223 if archive.exists(path): | |
| 224 self.archive = archive | |
| 225 self.real_path = path | |
| 226 self.full_real_path = os.path.join(archpath, *path.split('/')) | |
| 227 return True | |
| 228 return False | |
| 229 | |
| 230 def open(self): | |
| 231 if self.archive: | |
| 232 file = self.archive.open(self.real_path) | |
| 233 if hasattr(file, '__exit__'): | |
| 234 return file | |
| 235 else: | |
| 236 return contextlib.closing(file) | |
| 237 else: | |
| 54 | 238 return open(self.real_path, 'rU') |
| 21 | 239 |
| 240 def copy(self, target): | |
| 241 if self.archive: | |
| 242 self.archive.extract(self.real_path, target) | |
| 243 else: | |
| 244 shutil.copy(self.real_path, target) |