| # Copyright 1998-2009 Gentoo Foundation |
| # Distributed under the terms of the GNU General Public License v2 |
| # $Id$ |
| |
| __all__ = ["PreservedLibsRegistry", "LinkageMap", |
| "vardbapi", "vartree", "dblink"] + \ |
| ["write_contents", "tar_contents"] |
| |
| import portage |
| portage.proxy.lazyimport.lazyimport(globals(), |
| 'portage.checksum:perform_md5', |
| 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ |
| 'use_reduce,paren_reduce', |
| 'portage.elog:elog_process', |
| 'portage.elog.filtering:filter_mergephases,filter_unmergephases', |
| 'portage.locks:lockdir,unlockdir', |
| 'portage.output:bold,colorize', |
| 'portage.update:fixdbentries', |
| 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ |
| 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ |
| 'grabfile,grabdict,normalize_path,new_protect_filename,getlibpaths', |
| 'portage.versions:best,catpkgsplit,catsplit,pkgcmp,pkgsplit', |
| ) |
| |
| from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ |
| PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH |
| from portage.data import portage_gid, portage_uid, secpass |
| from portage.dbapi import dbapi |
| from portage.exception import CommandNotFound, \ |
| InvalidData, InvalidPackageName, \ |
| FileNotFound, PermissionDenied, UnsupportedAPIException |
| |
| from portage import listdir, dep_expand, digraph, flatten, key_expand, \ |
| doebuild_environment, doebuild, env_update, prepare_build_dirs, \ |
| abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey |
| |
| from portage.cache.mappings import slot_dict_class |
| |
| import os, re, shutil, stat, errno, copy, subprocess |
| import logging |
| import shlex |
| import sys |
| from itertools import izip |
| |
| try: |
| import cPickle as pickle |
| except ImportError: |
| import pickle |
| |
| class PreservedLibsRegistry(object): |
| """ This class handles the tracking of preserved library objects """ |
| def __init__(self, filename, autocommit=True): |
| """ @param filename: absolute path for saving the preserved libs records |
| @type filename: String |
| @param autocommit: determines if the file is written after every update |
| @type autocommit: Boolean |
| """ |
| self._filename = filename |
| self._autocommit = autocommit |
| self.load() |
| self.pruneNonExisting() |
| |
| def load(self): |
| """ Reload the registry data from file """ |
| self._data = None |
| try: |
| self._data = pickle.load(open(self._filename, 'rb')) |
| except (ValueError, pickle.UnpicklingError), e: |
| writemsg_level("!!! Error loading '%s': %s\n" % \ |
| (self._filename, e), level=logging.ERROR, noiselevel=-1) |
| except (EOFError, IOError), e: |
| if isinstance(e, EOFError) or e.errno == errno.ENOENT: |
| pass |
| elif e.errno == PermissionDenied.errno: |
| raise PermissionDenied(self._filename) |
| else: |
| raise |
| if self._data is None: |
| self._data = {} |
| self._data_orig = self._data.copy() |
| def store(self): |
| """ Store the registry data to file. No need to call this if autocommit |
| was enabled. |
| """ |
| if os.environ.get("SANDBOX_ON") == "1" or \ |
| self._data == self._data_orig: |
| return |
| try: |
| f = atomic_ofstream(self._filename, 'wb') |
| pickle.dump(self._data, f, protocol=2) |
| f.close() |
| except EnvironmentError, e: |
| if e.errno != PermissionDenied.errno: |
| writemsg("!!! %s %s\n" % (e, self._filename), noiselevel=-1) |
| else: |
| self._data_orig = self._data.copy() |
| |
| def register(self, cpv, slot, counter, paths): |
| """ Register new objects in the registry. If there is a record with the |
| same packagename (internally derived from cpv) and slot it is |
| overwritten with the new data. |
| @param cpv: package instance that owns the objects |
| @type cpv: CPV (as String) |
| @param slot: the value of SLOT of the given package instance |
| @type slot: String |
| @param counter: vdb counter value for the package instace |
| @type counter: Integer |
| @param paths: absolute paths of objects that got preserved during an update |
| @type paths: List |
| """ |
| cp = "/".join(catpkgsplit(cpv)[:2]) |
| cps = cp+":"+slot |
| if len(paths) == 0 and cps in self._data \ |
| and self._data[cps][0] == cpv and int(self._data[cps][1]) == int(counter): |
| del self._data[cps] |
| elif len(paths) > 0: |
| self._data[cps] = (cpv, counter, paths) |
| if self._autocommit: |
| self.store() |
| |
| def unregister(self, cpv, slot, counter): |
| """ Remove a previous registration of preserved objects for the given package. |
| @param cpv: package instance whose records should be removed |
| @type cpv: CPV (as String) |
| @param slot: the value of SLOT of the given package instance |
| @type slot: String |
| """ |
| self.register(cpv, slot, counter, []) |
| |
| def pruneNonExisting(self): |
| """ Remove all records for objects that no longer exist on the filesystem. """ |
| for cps in self._data.keys(): |
| cpv, counter, paths = self._data[cps] |
| paths = [f for f in paths if os.path.exists(f)] |
| if len(paths) > 0: |
| self._data[cps] = (cpv, counter, paths) |
| else: |
| del self._data[cps] |
| if self._autocommit: |
| self.store() |
| |
| def hasEntries(self): |
| """ Check if this registry contains any records. """ |
| return len(self._data) > 0 |
| |
| def getPreservedLibs(self): |
| """ Return a mapping of packages->preserved objects. |
| @returns mapping of package instances to preserved objects |
| @rtype Dict cpv->list-of-paths |
| """ |
| rValue = {} |
| for cps in self._data: |
| rValue[self._data[cps][0]] = self._data[cps][2] |
| return rValue |
| |
| class LinkageMap(object): |
| |
| """Models dynamic linker dependencies.""" |
| |
| _needed_aux_key = "NEEDED.ELF.2" |
| _soname_map_class = slot_dict_class( |
| ("consumers", "providers"), prefix="") |
| |
| def __init__(self, vardbapi): |
| self._dbapi = vardbapi |
| self._root = self._dbapi.root |
| self._libs = {} |
| self._obj_properties = {} |
| self._obj_key_cache = {} |
| self._defpath = set() |
| self._path_key_cache = {} |
| |
| def _clear_cache(self): |
| self._libs.clear() |
| self._obj_properties.clear() |
| self._obj_key_cache.clear() |
| self._defpath.clear() |
| self._path_key_cache.clear() |
| |
| def _path_key(self, path): |
| key = self._path_key_cache.get(path) |
| if key is None: |
| key = self._ObjectKey(path, self._root) |
| self._path_key_cache[path] = key |
| return key |
| |
| def _obj_key(self, path): |
| key = self._obj_key_cache.get(path) |
| if key is None: |
| key = self._ObjectKey(path, self._root) |
| self._obj_key_cache[path] = key |
| return key |
| |
| class _ObjectKey(object): |
| |
| """Helper class used as _obj_properties keys for objects.""" |
| |
| __slots__ = ("__weakref__", "_key") |
| |
| def __init__(self, obj, root): |
| """ |
| This takes a path to an object. |
| |
| @param object: path to a file |
| @type object: string (example: '/usr/bin/bar') |
| |
| """ |
| self._key = self._generate_object_key(obj, root) |
| |
| def __hash__(self): |
| return hash(self._key) |
| |
| def __eq__(self, other): |
| return self._key == other._key |
| |
| def _generate_object_key(self, obj, root): |
| """ |
| Generate object key for a given object. |
| |
| @param object: path to a file |
| @type object: string (example: '/usr/bin/bar') |
| @rtype: 2-tuple of types (long, int) if object exists. string if |
| object does not exist. |
| @return: |
| 1. 2-tuple of object's inode and device from a stat call, if object |
| exists. |
| 2. realpath of object if object does not exist. |
| |
| """ |
| abs_path = os.path.join(root, obj.lstrip(os.sep)) |
| try: |
| object_stat = os.stat(abs_path) |
| except OSError: |
| # Use the realpath as the key if the file does not exists on the |
| # filesystem. |
| return os.path.realpath(abs_path) |
| # Return a tuple of the device and inode. |
| return (object_stat.st_dev, object_stat.st_ino) |
| |
| def file_exists(self): |
| """ |
| Determine if the file for this key exists on the filesystem. |
| |
| @rtype: Boolean |
| @return: |
| 1. True if the file exists. |
| 2. False if the file does not exist or is a broken symlink. |
| |
| """ |
| return isinstance(self._key, tuple) |
| |
| class _LibGraphNode(_ObjectKey): |
| __slots__ = ("alt_paths",) |
| |
| def __init__(self, obj, root): |
| LinkageMap._ObjectKey.__init__(self, obj, root) |
| self.alt_paths = set() |
| |
| def __str__(self): |
| return str(sorted(self.alt_paths)) |
| |
| def rebuild(self, exclude_pkgs=None, include_file=None): |
| """ |
| Raises CommandNotFound if there are preserved libs |
| and the scanelf binary is not available. |
| """ |
| root = self._root |
| root_len = len(root) - 1 |
| self._clear_cache() |
| self._defpath.update(getlibpaths(self._root)) |
| libs = self._libs |
| obj_key_cache = self._obj_key_cache |
| obj_properties = self._obj_properties |
| |
| lines = [] |
| |
| # Data from include_file is processed first so that it |
| # overrides any data from previously installed files. |
| if include_file is not None: |
| lines += grabfile(include_file) |
| |
| aux_keys = [self._needed_aux_key] |
| for cpv in self._dbapi.cpv_all(): |
| if exclude_pkgs is not None and cpv in exclude_pkgs: |
| continue |
| lines += self._dbapi.aux_get(cpv, aux_keys)[0].split('\n') |
| # Cache NEEDED.* files avoid doing excessive IO for every rebuild. |
| self._dbapi.flush_cache() |
| |
| # have to call scanelf for preserved libs here as they aren't |
| # registered in NEEDED.ELF.2 files |
| if self._dbapi.plib_registry and self._dbapi.plib_registry.getPreservedLibs(): |
| args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"] |
| for items in self._dbapi.plib_registry.getPreservedLibs().values(): |
| args.extend(os.path.join(root, x.lstrip("." + os.sep)) \ |
| for x in items) |
| try: |
| proc = subprocess.Popen(args, stdout=subprocess.PIPE) |
| except EnvironmentError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| raise CommandNotFound(args[0]) |
| else: |
| for l in proc.stdout: |
| l = l[3:].rstrip("\n") |
| if not l: |
| continue |
| fields = l.split(";") |
| if len(fields) < 5: |
| writemsg_level("\nWrong number of fields " + \ |
| "returned from scanelf: %s\n\n" % (l,), |
| level=logging.ERROR, noiselevel=-1) |
| continue |
| fields[1] = fields[1][root_len:] |
| lines.append(";".join(fields)) |
| proc.wait() |
| |
| for l in lines: |
| l = l.rstrip("\n") |
| if not l: |
| continue |
| fields = l.split(";") |
| if len(fields) < 5: |
| writemsg_level("\nWrong number of fields " + \ |
| "in %s: %s\n\n" % (self._needed_aux_key, l), |
| level=logging.ERROR, noiselevel=-1) |
| continue |
| arch = fields[0] |
| obj = fields[1] |
| soname = fields[2] |
| path = set([normalize_path(x) \ |
| for x in filter(None, fields[3].replace( |
| "${ORIGIN}", os.path.dirname(obj)).replace( |
| "$ORIGIN", os.path.dirname(obj)).split(":"))]) |
| needed = filter(None, fields[4].split(",")) |
| |
| obj_key = self._obj_key(obj) |
| indexed = True |
| myprops = obj_properties.get(obj_key) |
| if myprops is None: |
| indexed = False |
| myprops = (arch, needed, path, soname, set()) |
| obj_properties[obj_key] = myprops |
| # All object paths are added into the obj_properties tuple. |
| myprops[4].add(obj) |
| |
| # Don't index the same file more that once since only one |
| # set of data can be correct and therefore mixing data |
| # may corrupt the index (include_file overrides previously |
| # installed). |
| if indexed: |
| continue |
| |
| arch_map = libs.get(arch) |
| if arch_map is None: |
| arch_map = {} |
| libs[arch] = arch_map |
| if soname: |
| soname_map = arch_map.get(soname) |
| if soname_map is None: |
| soname_map = self._soname_map_class( |
| providers=set(), consumers=set()) |
| arch_map[soname] = soname_map |
| soname_map.providers.add(obj_key) |
| for needed_soname in needed: |
| soname_map = arch_map.get(needed_soname) |
| if soname_map is None: |
| soname_map = self._soname_map_class( |
| providers=set(), consumers=set()) |
| arch_map[needed_soname] = soname_map |
| soname_map.consumers.add(obj_key) |
| |
| def listBrokenBinaries(self, debug=False): |
| """ |
| Find binaries and their needed sonames, which have no providers. |
| |
| @param debug: Boolean to enable debug output |
| @type debug: Boolean |
| @rtype: dict (example: {'/usr/bin/foo': set(['libbar.so'])}) |
| @return: The return value is an object -> set-of-sonames mapping, where |
| object is a broken binary and the set consists of sonames needed by |
| object that have no corresponding libraries to fulfill the dependency. |
| |
| """ |
| class _LibraryCache(object): |
| |
| """ |
| Caches properties associated with paths. |
| |
| The purpose of this class is to prevent multiple instances of |
| _ObjectKey for the same paths. |
| |
| """ |
| |
| def __init__(cache_self): |
| cache_self.cache = {} |
| |
| def get(cache_self, obj): |
| """ |
| Caches and returns properties associated with an object. |
| |
| @param obj: absolute path (can be symlink) |
| @type obj: string (example: '/usr/lib/libfoo.so') |
| @rtype: 4-tuple with types |
| (string or None, string or None, 2-tuple, Boolean) |
| @return: 4-tuple with the following components: |
| 1. arch as a string or None if it does not exist, |
| 2. soname as a string or None if it does not exist, |
| 3. obj_key as 2-tuple, |
| 4. Boolean representing whether the object exists. |
| (example: ('libfoo.so.1', (123L, 456L), True)) |
| |
| """ |
| if obj in cache_self.cache: |
| return cache_self.cache[obj] |
| else: |
| obj_key = self._obj_key(obj) |
| # Check that the library exists on the filesystem. |
| if obj_key.file_exists(): |
| # Get the arch and soname from LinkageMap._obj_properties if |
| # it exists. Otherwise, None. |
| arch, _, _, soname, _ = \ |
| self._obj_properties.get(obj_key, (None,)*5) |
| return cache_self.cache.setdefault(obj, \ |
| (arch, soname, obj_key, True)) |
| else: |
| return cache_self.cache.setdefault(obj, \ |
| (None, None, obj_key, False)) |
| |
| rValue = {} |
| cache = _LibraryCache() |
| providers = self.listProviders() |
| |
| # Iterate over all obj_keys and their providers. |
| for obj_key, sonames in providers.items(): |
| arch, _, path, _, objs = self._obj_properties[obj_key] |
| path = path.union(self._defpath) |
| # Iterate over each needed soname and the set of library paths that |
| # fulfill the soname to determine if the dependency is broken. |
| for soname, libraries in sonames.items(): |
| # validLibraries is used to store libraries, which satisfy soname, |
| # so if no valid libraries are found, the soname is not satisfied |
| # for obj_key. If unsatisfied, objects associated with obj_key |
| # must be emerged. |
| validLibraries = set() |
| # It could be the case that the library to satisfy the soname is |
| # not in the obj's runpath, but a symlink to the library is (eg |
| # libnvidia-tls.so.1 in nvidia-drivers). Also, since LinkageMap |
| # does not catalog symlinks, broken or missing symlinks may go |
| # unnoticed. As a result of these cases, check that a file with |
| # the same name as the soname exists in obj's runpath. |
| # XXX If we catalog symlinks in LinkageMap, this could be improved. |
| for directory in path: |
| cachedArch, cachedSoname, cachedKey, cachedExists = \ |
| cache.get(os.path.join(directory, soname)) |
| # Check that this library provides the needed soname. Doing |
| # this, however, will cause consumers of libraries missing |
| # sonames to be unnecessarily emerged. (eg libmix.so) |
| if cachedSoname == soname and cachedArch == arch: |
| validLibraries.add(cachedKey) |
| if debug and cachedKey not in \ |
| set(map(self._obj_key_cache.get, libraries)): |
| # XXX This is most often due to soname symlinks not in |
| # a library's directory. We could catalog symlinks in |
| # LinkageMap to avoid checking for this edge case here. |
| print "Found provider outside of findProviders:", \ |
| os.path.join(directory, soname), "->", \ |
| self._obj_properties[cachedKey][4], libraries |
| # A valid library has been found, so there is no need to |
| # continue. |
| break |
| if debug and cachedArch == arch and \ |
| cachedKey in self._obj_properties: |
| print "Broken symlink or missing/bad soname:", \ |
| os.path.join(directory, soname), '->', \ |
| self._obj_properties[cachedKey], "with soname", \ |
| cachedSoname, "but expecting", soname |
| # This conditional checks if there are no libraries to satisfy the |
| # soname (empty set). |
| if not validLibraries: |
| for obj in objs: |
| rValue.setdefault(obj, set()).add(soname) |
| # If no valid libraries have been found by this point, then |
| # there are no files named with the soname within obj's runpath, |
| # but if there are libraries (from the providers mapping), it is |
| # likely that soname symlinks or the actual libraries are |
| # missing or broken. Thus those libraries are added to rValue |
| # in order to emerge corrupt library packages. |
| for lib in libraries: |
| rValue.setdefault(lib, set()).add(soname) |
| if debug: |
| if not os.path.isfile(lib): |
| print "Missing library:", lib |
| else: |
| print "Possibly missing symlink:", \ |
| os.path.join(os.path.dirname(lib), soname) |
| return rValue |
| |
| def listProviders(self): |
| """ |
| Find the providers for all object keys in LinkageMap. |
| |
| @rtype: dict (example: |
| {(123L, 456L): {'libbar.so': set(['/lib/libbar.so.1.5'])}}) |
| @return: The return value is an object key -> providers mapping, where |
| providers is a mapping of soname -> set-of-library-paths returned |
| from the findProviders method. |
| |
| """ |
| rValue = {} |
| if not self._libs: |
| self.rebuild() |
| # Iterate over all object keys within LinkageMap. |
| for obj_key in self._obj_properties: |
| rValue.setdefault(obj_key, self.findProviders(obj_key)) |
| return rValue |
| |
| def isMasterLink(self, obj): |
| """ |
| Determine whether an object is a master link. |
| |
| @param obj: absolute path to an object |
| @type obj: string (example: '/usr/bin/foo') |
| @rtype: Boolean |
| @return: |
| 1. True if obj is a master link |
| 2. False if obj is not a master link |
| |
| """ |
| basename = os.path.basename(obj) |
| obj_key = self._obj_key(obj) |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s (%s) not in object list" % (obj_key, obj)) |
| soname = self._obj_properties[obj_key][3] |
| return (len(basename) < len(soname)) |
| |
| def listLibraryObjects(self): |
| """ |
| Return a list of library objects. |
| |
| Known limitation: library objects lacking an soname are not included. |
| |
| @rtype: list of strings |
| @return: list of paths to all providers |
| |
| """ |
| rValue = [] |
| if not self._libs: |
| self.rebuild() |
| for arch_map in self._libs.itervalues(): |
| for soname_map in arch_map.itervalues(): |
| for obj_key in soname_map.providers: |
| rValue.extend(self._obj_properties[obj_key][4]) |
| return rValue |
| |
| def getSoname(self, obj): |
| """ |
| Return the soname associated with an object. |
| |
| @param obj: absolute path to an object |
| @type obj: string (example: '/usr/bin/bar') |
| @rtype: string |
| @return: soname as a string |
| |
| """ |
| if not self._libs: |
| self.rebuild() |
| if isinstance(obj, self._ObjectKey): |
| obj_key = obj |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s not in object list" % obj_key) |
| return self._obj_properties[obj_key][3] |
| if obj not in self._obj_key_cache: |
| raise KeyError("%s not in object list" % obj) |
| return self._obj_properties[self._obj_key_cache[obj]][3] |
| |
| def findProviders(self, obj): |
| """ |
| Find providers for an object or object key. |
| |
| This method may be called with a key from _obj_properties. |
| |
| In some cases, not all valid libraries are returned. This may occur when |
| an soname symlink referencing a library is in an object's runpath while |
| the actual library is not. We should consider cataloging symlinks within |
| LinkageMap as this would avoid those cases and would be a better model of |
| library dependencies (since the dynamic linker actually searches for |
| files named with the soname in the runpaths). |
| |
| @param obj: absolute path to an object or a key from _obj_properties |
| @type obj: string (example: '/usr/bin/bar') or _ObjectKey |
| @rtype: dict (example: {'libbar.so': set(['/lib/libbar.so.1.5'])}) |
| @return: The return value is a soname -> set-of-library-paths, where |
| set-of-library-paths satisfy soname. |
| |
| """ |
| rValue = {} |
| |
| if not self._libs: |
| self.rebuild() |
| |
| # Determine the obj_key from the arguments. |
| if isinstance(obj, self._ObjectKey): |
| obj_key = obj |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s not in object list" % obj_key) |
| else: |
| obj_key = self._obj_key(obj) |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s (%s) not in object list" % (obj_key, obj)) |
| |
| arch, needed, path, _, _ = self._obj_properties[obj_key] |
| path_keys = set(self._path_key(x) for x in path.union(self._defpath)) |
| for soname in needed: |
| rValue[soname] = set() |
| if arch not in self._libs or soname not in self._libs[arch]: |
| continue |
| # For each potential provider of the soname, add it to rValue if it |
| # resides in the obj's runpath. |
| for provider_key in self._libs[arch][soname].providers: |
| providers = self._obj_properties[provider_key][4] |
| for provider in providers: |
| if self._path_key(os.path.dirname(provider)) in path_keys: |
| rValue[soname].add(provider) |
| return rValue |
| |
| def findConsumers(self, obj): |
| """ |
| Find consumers of an object or object key. |
| |
| This method may be called with a key from _obj_properties. If this |
| method is going to be called with an object key, to avoid not catching |
| shadowed libraries, do not pass new _ObjectKey instances to this method. |
| Instead pass the obj as a string. |
| |
| In some cases, not all consumers are returned. This may occur when |
| an soname symlink referencing a library is in an object's runpath while |
| the actual library is not. For example, this problem is noticeable for |
| binutils since it's libraries are added to the path via symlinks that |
| are gemerated in the /usr/$CHOST/lib/ directory by binutils-config. |
| Failure to recognize consumers of these symlinks makes preserve-libs |
| fail to preserve binutils libs that are needed by these unrecognized |
| consumers. |
| |
| Note that library consumption via dlopen (common for kde plugins) is |
| currently undetected. However, it is possible to use the |
| corresponding libtool archive (*.la) files to detect such consumers |
| (revdep-rebuild is able to detect them). |
| |
| @param obj: absolute path to an object or a key from _obj_properties |
| @type obj: string (example: '/usr/bin/bar') or _ObjectKey |
| @rtype: set of strings (example: set(['/bin/foo', '/usr/bin/bar'])) |
| @return: The return value is a soname -> set-of-library-paths, where |
| set-of-library-paths satisfy soname. |
| |
| """ |
| rValue = set() |
| |
| if not self._libs: |
| self.rebuild() |
| |
| # Determine the obj_key and the set of objects matching the arguments. |
| if isinstance(obj, self._ObjectKey): |
| obj_key = obj |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s not in object list" % obj_key) |
| objs = self._obj_properties[obj_key][4] |
| else: |
| objs = set([obj]) |
| obj_key = self._obj_key(obj) |
| if obj_key not in self._obj_properties: |
| raise KeyError("%s (%s) not in object list" % (obj_key, obj)) |
| |
| # If there is another version of this lib with the |
| # same soname and the master link points to that |
| # other version, this lib will be shadowed and won't |
| # have any consumers. |
| if not isinstance(obj, self._ObjectKey): |
| soname = self._obj_properties[obj_key][3] |
| master_link = os.path.join(self._root, |
| os.path.dirname(obj).lstrip(os.path.sep), soname) |
| try: |
| master_st = os.stat(master_link) |
| obj_st = os.stat(obj) |
| except OSError: |
| pass |
| else: |
| if (obj_st.st_dev, obj_st.st_ino) != \ |
| (master_st.st_dev, master_st.st_ino): |
| return set() |
| |
| # Determine the directory(ies) from the set of objects. |
| objs_dir_keys = set(self._path_key(os.path.dirname(x)) for x in objs) |
| defpath_keys = set(self._path_key(x) for x in self._defpath) |
| |
| arch, _, _, soname, _ = self._obj_properties[obj_key] |
| if arch in self._libs and soname in self._libs[arch]: |
| # For each potential consumer, add it to rValue if an object from the |
| # arguments resides in the consumer's runpath. |
| for consumer_key in self._libs[arch][soname].consumers: |
| _, _, path, _, consumer_objs = \ |
| self._obj_properties[consumer_key] |
| path_keys = defpath_keys.union(self._path_key(x) for x in path) |
| if objs_dir_keys.intersection(path_keys): |
| rValue.update(consumer_objs) |
| return rValue |
| |
| class vardbapi(dbapi): |
| |
| _excluded_dirs = ["CVS", "lost+found"] |
| _excluded_dirs = [re.escape(x) for x in _excluded_dirs] |
| _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ |
| "|".join(_excluded_dirs) + r')$') |
| |
| _aux_cache_version = "1" |
| _owners_cache_version = "1" |
| |
| # Number of uncached packages to trigger cache update, since |
| # it's wasteful to update it for every vdb change. |
| _aux_cache_threshold = 5 |
| |
| _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') |
| _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') |
| |
| def __init__(self, root, categories=None, settings=None, vartree=None): |
| """ |
| The categories parameter is unused since the dbapi class |
| now has a categories property that is generated from the |
| available packages. |
| """ |
| self.root = root[:] |
| |
| #cache for category directory mtimes |
| self.mtdircache = {} |
| |
| #cache for dependency checks |
| self.matchcache = {} |
| |
| #cache for cp_list results |
| self.cpcache = {} |
| |
| self.blockers = None |
| if settings is None: |
| from portage import settings |
| self.settings = settings |
| if vartree is None: |
| from portage import db |
| vartree = db[root]["vartree"] |
| self.vartree = vartree |
| self._aux_cache_keys = set( |
| ["CHOST", "COUNTER", "DEPEND", "DESCRIPTION", |
| "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", |
| "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", |
| "repository", "RESTRICT" , "SLOT", "USE"]) |
| self._aux_cache_obj = None |
| self._aux_cache_filename = os.path.join(self.root, |
| CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle") |
| self._counter_path = os.path.join(root, |
| CACHE_PATH.lstrip(os.path.sep), "counter") |
| |
| try: |
| self.plib_registry = PreservedLibsRegistry( |
| os.path.join(self.root, PRIVATE_PATH, "preserved_libs_registry")) |
| except PermissionDenied: |
| # apparently this user isn't allowed to access PRIVATE_PATH |
| self.plib_registry = None |
| |
| self.linkmap = LinkageMap(self) |
| self._owners = self._owners_db(self) |
| |
| def getpath(self, mykey, filename=None): |
| rValue = os.path.join(self.root, VDB_PATH, mykey) |
| if filename != None: |
| rValue = os.path.join(rValue, filename) |
| return rValue |
| |
| def cpv_exists(self, mykey): |
| "Tells us whether an actual ebuild exists on disk (no masking)" |
| return os.path.exists(self.getpath(mykey)) |
| |
| def cpv_counter(self, mycpv): |
| "This method will grab the COUNTER. Returns a counter value." |
| try: |
| return long(self.aux_get(mycpv, ["COUNTER"])[0]) |
| except (KeyError, ValueError): |
| pass |
| writemsg_level(("portage: COUNTER for %s was corrupted; " + \ |
| "resetting to value of 0\n") % (mycpv,), |
| level=logging.ERROR, noiselevel=-1) |
| return 0 |
| |
| def _counter_hash(self): |
| try: |
| from hashlib import md5 as new_hash |
| except ImportError: |
| from md5 import new as new_hash |
| h = new_hash() |
| aux_keys = ["COUNTER"] |
| cpv_list = self.cpv_all() |
| cpv_list.sort() |
| for cpv in cpv_list: |
| try: |
| counter, = self.aux_get(cpv, aux_keys) |
| except KeyError: |
| continue |
| h.update(counter) |
| return h.hexdigest() |
| |
| def cpv_inject(self, mycpv): |
| "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" |
| os.makedirs(self.getpath(mycpv)) |
| counter = self.counter_tick(self.root, mycpv=mycpv) |
| # write local package counter so that emerge clean does the right thing |
| write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter)) |
| |
| def isInjected(self, mycpv): |
| if self.cpv_exists(mycpv): |
| if os.path.exists(self.getpath(mycpv, filename="INJECTED")): |
| return True |
| if not os.path.exists(self.getpath(mycpv, filename="CONTENTS")): |
| return True |
| return False |
| |
| def move_ent(self, mylist): |
| origcp = mylist[1] |
| newcp = mylist[2] |
| |
| # sanity check |
| for cp in [origcp, newcp]: |
| if not (isvalidatom(cp) and isjustname(cp)): |
| raise InvalidPackageName(cp) |
| origmatches = self.match(origcp, use_cache=0) |
| moves = 0 |
| if not origmatches: |
| return moves |
| for mycpv in origmatches: |
| mycpsplit = catpkgsplit(mycpv) |
| mynewcpv = newcp + "-" + mycpsplit[2] |
| mynewcat = newcp.split("/")[0] |
| if mycpsplit[3] != "r0": |
| mynewcpv += "-" + mycpsplit[3] |
| mycpsplit_new = catpkgsplit(mynewcpv) |
| origpath = self.getpath(mycpv) |
| if not os.path.exists(origpath): |
| continue |
| moves += 1 |
| if not os.path.exists(self.getpath(mynewcat)): |
| #create the directory |
| os.makedirs(self.getpath(mynewcat)) |
| newpath = self.getpath(mynewcpv) |
| if os.path.exists(newpath): |
| #dest already exists; keep this puppy where it is. |
| continue |
| _movefile(origpath, newpath, mysettings=self.settings) |
| |
| # We need to rename the ebuild now. |
| old_pf = catsplit(mycpv)[1] |
| new_pf = catsplit(mynewcpv)[1] |
| if new_pf != old_pf: |
| try: |
| os.rename(os.path.join(newpath, old_pf + ".ebuild"), |
| os.path.join(newpath, new_pf + ".ebuild")) |
| except EnvironmentError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") |
| write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") |
| fixdbentries([mylist], newpath) |
| return moves |
| |
| def cp_list(self, mycp, use_cache=1): |
| mysplit=catsplit(mycp) |
| if mysplit[0] == '*': |
| mysplit[0] = mysplit[0][1:] |
| try: |
| mystat = os.stat(self.getpath(mysplit[0]))[stat.ST_MTIME] |
| except OSError: |
| mystat = 0 |
| if use_cache and mycp in self.cpcache: |
| cpc = self.cpcache[mycp] |
| if cpc[0] == mystat: |
| return cpc[1][:] |
| cat_dir = self.getpath(mysplit[0]) |
| try: |
| dir_list = os.listdir(cat_dir) |
| except EnvironmentError, e: |
| if e.errno == PermissionDenied.errno: |
| raise PermissionDenied(cat_dir) |
| del e |
| dir_list = [] |
| |
| returnme = [] |
| for x in dir_list: |
| if self._excluded_dirs.match(x) is not None: |
| continue |
| ps = pkgsplit(x) |
| if not ps: |
| self.invalidentry(os.path.join(self.getpath(mysplit[0]), x)) |
| continue |
| if len(mysplit) > 1: |
| if ps[0] == mysplit[1]: |
| returnme.append(mysplit[0]+"/"+x) |
| self._cpv_sort_ascending(returnme) |
| if use_cache: |
| self.cpcache[mycp] = [mystat, returnme[:]] |
| elif mycp in self.cpcache: |
| del self.cpcache[mycp] |
| return returnme |
| |
| def cpv_all(self, use_cache=1): |
| """ |
| Set use_cache=0 to bypass the portage.cachedir() cache in cases |
| when the accuracy of mtime staleness checks should not be trusted |
| (generally this is only necessary in critical sections that |
| involve merge or unmerge of packages). |
| """ |
| returnme = [] |
| basepath = os.path.join(self.root, VDB_PATH) + os.path.sep |
| |
| if use_cache: |
| from portage import listdir |
| else: |
| def listdir(p, **kwargs): |
| try: |
| return [x for x in os.listdir(p) \ |
| if os.path.isdir(os.path.join(p, x))] |
| except EnvironmentError, e: |
| if e.errno == PermissionDenied.errno: |
| raise PermissionDenied(p) |
| del e |
| return [] |
| |
| for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1): |
| if self._excluded_dirs.match(x) is not None: |
| continue |
| if not self._category_re.match(x): |
| continue |
| for y in listdir(basepath + x, EmptyOnError=1, dirsonly=1): |
| if self._excluded_dirs.match(y) is not None: |
| continue |
| subpath = x + "/" + y |
| # -MERGING- should never be a cpv, nor should files. |
| try: |
| if catpkgsplit(subpath) is None: |
| self.invalidentry(self.getpath(subpath)) |
| continue |
| except InvalidData: |
| self.invalidentry(self.getpath(subpath)) |
| continue |
| returnme.append(subpath) |
| |
| return returnme |
| |
| def cp_all(self, use_cache=1): |
| mylist = self.cpv_all(use_cache=use_cache) |
| d={} |
| for y in mylist: |
| if y[0] == '*': |
| y = y[1:] |
| try: |
| mysplit = catpkgsplit(y) |
| except InvalidData: |
| self.invalidentry(self.getpath(y)) |
| continue |
| if not mysplit: |
| self.invalidentry(self.getpath(y)) |
| continue |
| d[mysplit[0]+"/"+mysplit[1]] = None |
| return d.keys() |
| |
| def checkblockers(self, origdep): |
| pass |
| |
| def _clear_cache(self): |
| self.mtdircache.clear() |
| self.matchcache.clear() |
| self.cpcache.clear() |
| self._aux_cache_obj = None |
| |
| def _add(self, pkg_dblink): |
| self._clear_pkg_cache(pkg_dblink) |
| |
| def _remove(self, pkg_dblink): |
| self._clear_pkg_cache(pkg_dblink) |
| |
| def _clear_pkg_cache(self, pkg_dblink): |
| # Due to 1 second mtime granularity in <python-2.5, mtime checks |
| # are not always sufficient to invalidate vardbapi caches. Therefore, |
| # the caches need to be actively invalidated here. |
| self.mtdircache.pop(pkg_dblink.cat, None) |
| self.matchcache.pop(pkg_dblink.cat, None) |
| self.cpcache.pop(pkg_dblink.mysplit[0], None) |
| from portage import dircache |
| dircache.pop(pkg_dblink.dbcatdir, None) |
| |
| def match(self, origdep, use_cache=1): |
| "caching match function" |
| mydep = dep_expand( |
| origdep, mydb=self, use_cache=use_cache, settings=self.settings) |
| mykey = dep_getkey(mydep) |
| mycat = catsplit(mykey)[0] |
| if not use_cache: |
| if mycat in self.matchcache: |
| del self.mtdircache[mycat] |
| del self.matchcache[mycat] |
| return list(self._iter_match(mydep, |
| self.cp_list(mydep.cp, use_cache=use_cache))) |
| try: |
| curmtime = os.stat(self.root+VDB_PATH+"/"+mycat).st_mtime |
| except (IOError, OSError): |
| curmtime=0 |
| |
| if mycat not in self.matchcache or \ |
| self.mtdircache[mycat] != curmtime: |
| # clear cache entry |
| self.mtdircache[mycat] = curmtime |
| self.matchcache[mycat] = {} |
| if mydep not in self.matchcache[mycat]: |
| mymatch = list(self._iter_match(mydep, |
| self.cp_list(mydep.cp, use_cache=use_cache))) |
| self.matchcache[mycat][mydep] = mymatch |
| return self.matchcache[mycat][mydep][:] |
| |
| def findname(self, mycpv): |
| return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild") |
| |
| def flush_cache(self): |
| """If the current user has permission and the internal aux_get cache has |
| been updated, save it to disk and mark it unmodified. This is called |
| by emerge after it has loaded the full vdb for use in dependency |
| calculations. Currently, the cache is only written if the user has |
| superuser privileges (since that's required to obtain a lock), but all |
| users have read access and benefit from faster metadata lookups (as |
| long as at least part of the cache is still valid).""" |
| if self._aux_cache is not None and \ |
| len(self._aux_cache["modified"]) >= self._aux_cache_threshold and \ |
| secpass >= 2: |
| self._owners.populate() # index any unindexed contents |
| valid_nodes = set(self.cpv_all()) |
| for cpv in self._aux_cache["packages"].keys(): |
| if cpv not in valid_nodes: |
| del self._aux_cache["packages"][cpv] |
| del self._aux_cache["modified"] |
| try: |
| f = atomic_ofstream(self._aux_cache_filename, 'wb') |
| pickle.dump(self._aux_cache, f, protocol=2) |
| f.close() |
| apply_secpass_permissions( |
| self._aux_cache_filename, gid=portage_gid, mode=0644) |
| except (IOError, OSError), e: |
| pass |
| self._aux_cache["modified"] = set() |
| |
| @property |
| def _aux_cache(self): |
| if self._aux_cache_obj is None: |
| self._aux_cache_init() |
| return self._aux_cache_obj |
| |
| def _aux_cache_init(self): |
| aux_cache = None |
| open_kwargs = {} |
| if sys.hexversion >= 0x3000000: |
| # Buffered io triggers extreme performance issues in |
| # Unpickler.load() (problem observed with python-3.0.1). |
| # Unfortunately, performance is still poor relative to |
| # python-2.x, but buffering makes it much worse. |
| open_kwargs["buffering"] = 0 |
| try: |
| f = open(self._aux_cache_filename, mode='rb', **open_kwargs) |
| mypickle = pickle.Unpickler(f) |
| try: |
| mypickle.find_global = None |
| except AttributeError: |
| # TODO: If py3k, override Unpickler.find_class(). |
| pass |
| aux_cache = mypickle.load() |
| f.close() |
| del f |
| except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError), e: |
| if isinstance(e, pickle.UnpicklingError): |
| writemsg("!!! Error loading '%s': %s\n" % \ |
| (self._aux_cache_filename, str(e)), noiselevel=-1) |
| del e |
| |
| if not aux_cache or \ |
| not isinstance(aux_cache, dict) or \ |
| aux_cache.get("version") != self._aux_cache_version or \ |
| not aux_cache.get("packages"): |
| aux_cache = {"version": self._aux_cache_version} |
| aux_cache["packages"] = {} |
| |
| owners = aux_cache.get("owners") |
| if owners is not None: |
| if not isinstance(owners, dict): |
| owners = None |
| elif "version" not in owners: |
| owners = None |
| elif owners["version"] != self._owners_cache_version: |
| owners = None |
| elif "base_names" not in owners: |
| owners = None |
| elif not isinstance(owners["base_names"], dict): |
| owners = None |
| |
| if owners is None: |
| owners = { |
| "base_names" : {}, |
| "version" : self._owners_cache_version |
| } |
| aux_cache["owners"] = owners |
| |
| aux_cache["modified"] = set() |
| self._aux_cache_obj = aux_cache |
| |
| def aux_get(self, mycpv, wants): |
| """This automatically caches selected keys that are frequently needed |
| by emerge for dependency calculations. The cached metadata is |
| considered valid if the mtime of the package directory has not changed |
| since the data was cached. The cache is stored in a pickled dict |
| object with the following format: |
| |
| {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} |
| |
| If an error occurs while loading the cache pickle or the version is |
| unrecognized, the cache will simple be recreated from scratch (it is |
| completely disposable). |
| """ |
| cache_these_wants = self._aux_cache_keys.intersection(wants) |
| for x in wants: |
| if self._aux_cache_keys_re.match(x) is not None: |
| cache_these_wants.add(x) |
| |
| if not cache_these_wants: |
| return self._aux_get(mycpv, wants) |
| |
| cache_these = set(self._aux_cache_keys) |
| cache_these.update(cache_these_wants) |
| |
| mydir = self.getpath(mycpv) |
| mydir_stat = None |
| try: |
| mydir_stat = os.stat(mydir) |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| raise KeyError(mycpv) |
| mydir_mtime = long(mydir_stat.st_mtime) |
| pkg_data = self._aux_cache["packages"].get(mycpv) |
| pull_me = cache_these.union(wants) |
| mydata = {"_mtime_" : mydir_mtime} |
| cache_valid = False |
| cache_incomplete = False |
| cache_mtime = None |
| metadata = None |
| if pkg_data is not None: |
| if not isinstance(pkg_data, tuple) or len(pkg_data) != 2: |
| pkg_data = None |
| else: |
| cache_mtime, metadata = pkg_data |
| if not isinstance(cache_mtime, (long, int)) or \ |
| not isinstance(metadata, dict): |
| pkg_data = None |
| |
| if pkg_data: |
| cache_mtime, metadata = pkg_data |
| cache_valid = cache_mtime == mydir_mtime |
| if cache_valid: |
| mydata.update(metadata) |
| pull_me.difference_update(mydata) |
| |
| if pull_me: |
| # pull any needed data and cache it |
| aux_keys = list(pull_me) |
| for k, v in izip(aux_keys, |
| self._aux_get(mycpv, aux_keys, st=mydir_stat)): |
| mydata[k] = v |
| if not cache_valid or cache_these.difference(metadata): |
| cache_data = {} |
| if cache_valid and metadata: |
| cache_data.update(metadata) |
| for aux_key in cache_these: |
| cache_data[aux_key] = mydata[aux_key] |
| self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) |
| self._aux_cache["modified"].add(mycpv) |
| if not mydata['SLOT']: |
| # Empty slot triggers InvalidAtom exceptions when generating slot |
| # atoms for packages, so translate it to '0' here. |
| mydata['SLOT'] = '0' |
| return [mydata[x] for x in wants] |
| |
| def _aux_get(self, mycpv, wants, st=None): |
| mydir = self.getpath(mycpv) |
| if st is None: |
| try: |
| st = os.stat(mydir) |
| except OSError, e: |
| if e.errno == errno.ENOENT: |
| raise KeyError(mycpv) |
| elif e.errno == PermissionDenied.errno: |
| raise PermissionDenied(mydir) |
| else: |
| raise |
| if not stat.S_ISDIR(st.st_mode): |
| raise KeyError(mycpv) |
| results = [] |
| for x in wants: |
| if x == "_mtime_": |
| results.append(long(st.st_mtime)) |
| continue |
| try: |
| myf = open(os.path.join(mydir, x), "r") |
| try: |
| myd = myf.read() |
| finally: |
| myf.close() |
| # Preserve \n for metadata that is known to |
| # contain multiple lines. |
| if self._aux_multi_line_re.match(x) is None: |
| myd = " ".join(myd.split()) |
| except IOError: |
| myd = "" |
| if x == "EAPI" and not myd: |
| results.append("0") |
| else: |
| results.append(myd) |
| return results |
| |
| def aux_update(self, cpv, values): |
| cat, pkg = catsplit(cpv) |
| mylink = dblink(cat, pkg, self.root, self.settings, |
| treetype="vartree", vartree=self.vartree) |
| if not mylink.exists(): |
| raise KeyError(cpv) |
| for k, v in values.iteritems(): |
| if v: |
| mylink.setfile(k, v) |
| else: |
| try: |
| os.unlink(os.path.join(self.getpath(cpv), k)) |
| except EnvironmentError: |
| pass |
| |
| def counter_tick(self, myroot, mycpv=None): |
| return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv) |
| |
| def get_counter_tick_core(self, myroot, mycpv=None): |
| """ |
| Use this method to retrieve the counter instead |
| of having to trust the value of a global counter |
| file that can lead to invalid COUNTER |
| generation. When cache is valid, the package COUNTER |
| files are not read and we rely on the timestamp of |
| the package directory to validate cache. The stat |
| calls should only take a short time, so performance |
| is sufficient without having to rely on a potentially |
| corrupt global counter file. |
| |
| The global counter file located at |
| $CACHE_PATH/counter serves to record the |
| counter of the last installed package and |
| it also corresponds to the total number of |
| installation actions that have occurred in |
| the history of this package database. |
| """ |
| cp_list = self.cp_list |
| max_counter = 0 |
| for cp in self.cp_all(): |
| for cpv in cp_list(cp): |
| try: |
| counter = int(self.aux_get(cpv, ["COUNTER"])[0]) |
| except (KeyError, OverflowError, ValueError): |
| continue |
| if counter > max_counter: |
| max_counter = counter |
| |
| new_vdb = False |
| counter = -1 |
| try: |
| cfile = open(self._counter_path, "r") |
| except EnvironmentError, e: |
| new_vdb = not bool(self.cpv_all()) |
| if not new_vdb: |
| writemsg("!!! Unable to read COUNTER file: '%s'\n" % \ |
| self._counter_path, noiselevel=-1) |
| writemsg("!!! %s\n" % str(e), noiselevel=-1) |
| del e |
| else: |
| try: |
| try: |
| counter = long(cfile.readline().strip()) |
| finally: |
| cfile.close() |
| except (OverflowError, ValueError), e: |
| writemsg("!!! COUNTER file is corrupt: '%s'\n" % \ |
| self._counter_path, noiselevel=-1) |
| writemsg("!!! %s\n" % str(e), noiselevel=-1) |
| del e |
| |
| # We must ensure that we return a counter |
| # value that is at least as large as the |
| # highest one from the installed packages, |
| # since having a corrupt value that is too low |
| # can trigger incorrect AUTOCLEAN behavior due |
| # to newly installed packages having lower |
| # COUNTERs than the previous version in the |
| # same slot. |
| if counter > max_counter: |
| max_counter = counter |
| |
| if counter < 0 and not new_vdb: |
| writemsg("!!! Initializing COUNTER to " + \ |
| "value of %d\n" % max_counter, noiselevel=-1) |
| |
| return max_counter + 1 |
| |
| def counter_tick_core(self, myroot, incrementing=1, mycpv=None): |
| "This method will grab the next COUNTER value and record it back to the global file. Returns new counter value." |
| counter = self.get_counter_tick_core(myroot, mycpv=mycpv) - 1 |
| if incrementing: |
| #increment counter |
| counter += 1 |
| # update new global counter file |
| write_atomic(self._counter_path, str(counter)) |
| return counter |
| |
| def _dblink(self, cpv): |
| category, pf = catsplit(cpv) |
| return dblink(category, pf, self.root, |
| self.settings, vartree=self.vartree, treetype="vartree") |
| |
| def removeFromContents(self, pkg, paths, relative_paths=True): |
| """ |
| @param pkg: cpv for an installed package |
| @type pkg: string |
| @param paths: paths of files to remove from contents |
| @type paths: iterable |
| """ |
| if not hasattr(pkg, "getcontents"): |
| pkg = self._dblink(pkg) |
| root = self.root |
| root_len = len(root) - 1 |
| new_contents = pkg.getcontents().copy() |
| removed = 0 |
| |
| for filename in paths: |
| filename = normalize_path(filename) |
| if relative_paths: |
| relative_filename = filename |
| else: |
| relative_filename = filename[root_len:] |
| contents_key = pkg._match_contents(relative_filename, root) |
| if contents_key: |
| del new_contents[contents_key] |
| removed += 1 |
| |
| if removed: |
| f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) |
| write_contents(new_contents, root, f) |
| f.close() |
| pkg._clear_contents_cache() |
| |
| class _owners_cache(object): |
| """ |
| This class maintains an hash table that serves to index package |
| contents by mapping the basename of file to a list of possible |
| packages that own it. This is used to optimize owner lookups |
| by narrowing the search down to a smaller number of packages. |
| """ |
| try: |
| from hashlib import md5 as _new_hash |
| except ImportError: |
| from md5 import new as _new_hash |
| |
| _hash_bits = 16 |
| _hex_chars = int(_hash_bits / 4) |
| |
| def __init__(self, vardb): |
| self._vardb = vardb |
| |
| def add(self, cpv): |
| root_len = len(self._vardb.root) |
| contents = self._vardb._dblink(cpv).getcontents() |
| pkg_hash = self._hash_pkg(cpv) |
| if not contents: |
| # Empty path is a code used to represent empty contents. |
| self._add_path("", pkg_hash) |
| for x in contents: |
| self._add_path(x[root_len:], pkg_hash) |
| self._vardb._aux_cache["modified"].add(cpv) |
| |
| def _add_path(self, path, pkg_hash): |
| """ |
| Empty path is a code that represents empty contents. |
| """ |
| if path: |
| name = os.path.basename(path.rstrip(os.path.sep)) |
| if not name: |
| return |
| else: |
| name = path |
| name_hash = self._hash_str(name) |
| base_names = self._vardb._aux_cache["owners"]["base_names"] |
| pkgs = base_names.get(name_hash) |
| if pkgs is None: |
| pkgs = {} |
| base_names[name_hash] = pkgs |
| pkgs[pkg_hash] = None |
| |
| def _hash_str(self, s): |
| h = self._new_hash() |
| # Always use a constant utf_8 encoding here, since |
| # the "default" encoding can change. |
| h.update(s) |
| h = h.hexdigest() |
| h = h[-self._hex_chars:] |
| h = int(h, 16) |
| return h |
| |
| def _hash_pkg(self, cpv): |
| counter, mtime = self._vardb.aux_get( |
| cpv, ["COUNTER", "_mtime_"]) |
| try: |
| counter = int(counter) |
| except ValueError: |
| counter = 0 |
| return (cpv, counter, mtime) |
| |
| class _owners_db(object): |
| |
| def __init__(self, vardb): |
| self._vardb = vardb |
| |
| def populate(self): |
| self._populate() |
| |
| def _populate(self): |
| owners_cache = vardbapi._owners_cache(self._vardb) |
| cached_hashes = set() |
| base_names = self._vardb._aux_cache["owners"]["base_names"] |
| |
| # Take inventory of all cached package hashes. |
| for name, hash_values in base_names.items(): |
| if not isinstance(hash_values, dict): |
| del base_names[name] |
| continue |
| cached_hashes.update(hash_values) |
| |
| # Create sets of valid package hashes and uncached packages. |
| uncached_pkgs = set() |
| hash_pkg = owners_cache._hash_pkg |
| valid_pkg_hashes = set() |
| for cpv in self._vardb.cpv_all(): |
| hash_value = hash_pkg(cpv) |
| valid_pkg_hashes.add(hash_value) |
| if hash_value not in cached_hashes: |
| uncached_pkgs.add(cpv) |
| |
| # Cache any missing packages. |
| for cpv in uncached_pkgs: |
| owners_cache.add(cpv) |
| |
| # Delete any stale cache. |
| stale_hashes = cached_hashes.difference(valid_pkg_hashes) |
| if stale_hashes: |
| for base_name_hash, bucket in base_names.items(): |
| for hash_value in stale_hashes.intersection(bucket): |
| del bucket[hash_value] |
| if not bucket: |
| del base_names[base_name_hash] |
| |
| return owners_cache |
| |
| def get_owners(self, path_iter): |
| """ |
| @return the owners as a dblink -> set(files) mapping. |
| """ |
| owners = {} |
| for owner, f in self.iter_owners(path_iter): |
| owned_files = owners.get(owner) |
| if owned_files is None: |
| owned_files = set() |
| owners[owner] = owned_files |
| owned_files.add(f) |
| return owners |
| |
| def getFileOwnerMap(self, path_iter): |
| owners = self.get_owners(path_iter) |
| file_owners = {} |
| for pkg_dblink, files in owners.iteritems(): |
| for f in files: |
| owner_set = file_owners.get(f) |
| if owner_set is None: |
| owner_set = set() |
| file_owners[f] = owner_set |
| owner_set.add(pkg_dblink) |
| return file_owners |
| |
| def iter_owners(self, path_iter): |
| """ |
| Iterate over tuples of (dblink, path). In order to avoid |
| consuming too many resources for too much time, resources |
| are only allocated for the duration of a given iter_owners() |
| call. Therefore, to maximize reuse of resources when searching |
| for multiple files, it's best to search for them all in a single |
| call. |
| """ |
| |
| owners_cache = self._populate() |
| |
| vardb = self._vardb |
| root = vardb.root |
| hash_pkg = owners_cache._hash_pkg |
| hash_str = owners_cache._hash_str |
| base_names = self._vardb._aux_cache["owners"]["base_names"] |
| |
| dblink_cache = {} |
| |
| def dblink(cpv): |
| x = dblink_cache.get(cpv) |
| if x is None: |
| x = self._vardb._dblink(cpv) |
| dblink_cache[cpv] = x |
| return x |
| |
| for path in path_iter: |
| name = os.path.basename(path.rstrip(os.path.sep)) |
| if not name: |
| continue |
| |
| name_hash = hash_str(name) |
| pkgs = base_names.get(name_hash) |
| if pkgs is not None: |
| for hash_value in pkgs: |
| if not isinstance(hash_value, tuple) or \ |
| len(hash_value) != 3: |
| continue |
| cpv, counter, mtime = hash_value |
| if not isinstance(cpv, basestring): |
| continue |
| try: |
| current_hash = hash_pkg(cpv) |
| except KeyError: |
| continue |
| |
| if current_hash != hash_value: |
| continue |
| if dblink(cpv).isowner(path, root): |
| yield dblink(cpv), path |
| |
| class vartree(object): |
| "this tree will scan a var/db/pkg database located at root (passed to init)" |
| def __init__(self, root="/", virtual=None, clone=None, categories=None, |
| settings=None): |
| if clone: |
| writemsg("vartree.__init__(): deprecated " + \ |
| "use of clone parameter\n", noiselevel=-1) |
| self.root = clone.root[:] |
| self.dbapi = copy.deepcopy(clone.dbapi) |
| self.populated = 1 |
| from portage import config |
| self.settings = config(clone=clone.settings) |
| else: |
| self.root = root[:] |
| if settings is None: |
| from portage import settings |
| self.settings = settings # for key_expand calls |
| if categories is None: |
| categories = settings.categories |
| self.dbapi = vardbapi(self.root, categories=categories, |
| settings=settings, vartree=self) |
| self.populated = 1 |
| |
| def getpath(self, mykey, filename=None): |
| return self.dbapi.getpath(mykey, filename=filename) |
| |
| def zap(self, mycpv): |
| return |
| |
| def inject(self, mycpv): |
| return |
| |
| def get_provide(self, mycpv): |
| myprovides = [] |
| mylines = None |
| try: |
| mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE", "USE"]) |
| if mylines: |
| myuse = myuse.split() |
| mylines = flatten(use_reduce(paren_reduce(mylines), uselist=myuse)) |
| for myprovide in mylines: |
| mys = catpkgsplit(myprovide) |
| if not mys: |
| mys = myprovide.split("/") |
| myprovides += [mys[0] + "/" + mys[1]] |
| return myprovides |
| except SystemExit, e: |
| raise |
| except Exception, e: |
| mydir = os.path.join(self.root, VDB_PATH, mycpv) |
| writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir, |
| noiselevel=-1) |
| if mylines: |
| writemsg("Possibly Invalid: '%s'\n" % str(mylines), |
| noiselevel=-1) |
| writemsg("Exception: %s\n\n" % str(e), noiselevel=-1) |
| return [] |
| |
| def get_all_provides(self): |
| myprovides = {} |
| for node in self.getallcpv(): |
| for mykey in self.get_provide(node): |
| if mykey in myprovides: |
| myprovides[mykey] += [node] |
| else: |
| myprovides[mykey] = [node] |
| return myprovides |
| |
| def dep_bestmatch(self, mydep, use_cache=1): |
| "compatibility method -- all matches, not just visible ones" |
| #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi)) |
| mymatch = best(self.dbapi.match( |
| dep_expand(mydep, mydb=self.dbapi, settings=self.settings), |
| use_cache=use_cache)) |
| if mymatch is None: |
| return "" |
| else: |
| return mymatch |
| |
| def dep_match(self, mydep, use_cache=1): |
| "compatibility method -- we want to see all matches, not just visible ones" |
| #mymatch = match(mydep,self.dbapi) |
| mymatch = self.dbapi.match(mydep, use_cache=use_cache) |
| if mymatch is None: |
| return [] |
| else: |
| return mymatch |
| |
| def exists_specific(self, cpv): |
| return self.dbapi.cpv_exists(cpv) |
| |
| def getallcpv(self): |
| """temporary function, probably to be renamed --- Gets a list of all |
| category/package-versions installed on the system.""" |
| return self.dbapi.cpv_all() |
| |
| def getallnodes(self): |
| """new behavior: these are all *unmasked* nodes. There may or may not be available |
| masked package for nodes in this nodes list.""" |
| return self.dbapi.cp_all() |
| |
| def exists_specific_cat(self, cpv, use_cache=1): |
| cpv = key_expand(cpv, mydb=self.dbapi, use_cache=use_cache, |
| settings=self.settings) |
| a = catpkgsplit(cpv) |
| if not a: |
| return 0 |
| mylist = listdir(self.getpath(a[0]), EmptyOnError=1) |
| for x in mylist: |
| b = pkgsplit(x) |
| if not b: |
| self.dbapi.invalidentry(self.getpath(a[0], filename=x)) |
| continue |
| if a[1] == b[0]: |
| return 1 |
| return 0 |
| |
| def getebuildpath(self, fullpackage): |
| cat, package = catsplit(fullpackage) |
| return self.getpath(fullpackage, filename=package+".ebuild") |
| |
| def getnode(self, mykey, use_cache=1): |
| mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, |
| settings=self.settings) |
| if not mykey: |
| return [] |
| mysplit = catsplit(mykey) |
| mydirlist = listdir(self.getpath(mysplit[0]),EmptyOnError=1) |
| returnme = [] |
| for x in mydirlist: |
| mypsplit = pkgsplit(x) |
| if not mypsplit: |
| self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) |
| continue |
| if mypsplit[0] == mysplit[1]: |
| appendme = [mysplit[0]+"/"+x, [mysplit[0], mypsplit[0], mypsplit[1], mypsplit[2]]] |
| returnme.append(appendme) |
| return returnme |
| |
| |
| def getslot(self, mycatpkg): |
| "Get a slot for a catpkg; assume it exists." |
| try: |
| return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] |
| except KeyError: |
| return "" |
| |
| def hasnode(self, mykey, use_cache): |
| """Does the particular node (cat/pkg key) exist?""" |
| mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, |
| settings=self.settings) |
| mysplit = catsplit(mykey) |
| mydirlist = listdir(self.getpath(mysplit[0]), EmptyOnError=1) |
| for x in mydirlist: |
| mypsplit = pkgsplit(x) |
| if not mypsplit: |
| self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) |
| continue |
| if mypsplit[0] == mysplit[1]: |
| return 1 |
| return 0 |
| |
| def populate(self): |
| self.populated=1 |
| |
| class dblink(object): |
| """ |
| This class provides an interface to the installed package database |
| At present this is implemented as a text backend in /var/db/pkg. |
| """ |
| |
| import re |
| _normalize_needed = re.compile(r'.*//.*|^[^/]|.+/$|(^|.*/)\.\.?(/.*|$)') |
| _contents_split_counts = { |
| "dev": 2, |
| "dir": 2, |
| "fif": 2, |
| "obj": 4, |
| "sym": 5 |
| } |
| |
| # When looping over files for merge/unmerge, temporarily yield to the |
| # scheduler each time this many files are processed. |
| _file_merge_yield_interval = 20 |
| |
| def __init__(self, cat, pkg, myroot, mysettings, treetype=None, |
| vartree=None, blockers=None, scheduler=None): |
| """ |
| Creates a DBlink object for a given CPV. |
| The given CPV may not be present in the database already. |
| |
| @param cat: Category |
| @type cat: String |
| @param pkg: Package (PV) |
| @type pkg: String |
| @param myroot: Typically ${ROOT} |
| @type myroot: String (Path) |
| @param mysettings: Typically portage.config |
| @type mysettings: An instance of portage.config |
| @param treetype: one of ['porttree','bintree','vartree'] |
| @type treetype: String |
| @param vartree: an instance of vartree corresponding to myroot. |
| @type vartree: vartree |
| """ |
| |
| self.cat = cat |
| self.pkg = pkg |
| self.mycpv = self.cat + "/" + self.pkg |
| self.mysplit = list(catpkgsplit(self.mycpv)[1:]) |
| self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0]) |
| self.treetype = treetype |
| if vartree is None: |
| from portage import db |
| vartree = db[myroot]["vartree"] |
| self.vartree = vartree |
| self._blockers = blockers |
| self._scheduler = scheduler |
| |
| self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH)) |
| self.dbcatdir = self.dbroot+"/"+cat |
| self.dbpkgdir = self.dbcatdir+"/"+pkg |
| self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg |
| self.dbdir = self.dbpkgdir |
| |
| self._lock_vdb = None |
| |
| self.settings = mysettings |
| self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" |
| |
| self.myroot=myroot |
| protect_obj = ConfigProtect(myroot, |
| shlex.split(mysettings.get("CONFIG_PROTECT", "")), |
| shlex.split(mysettings.get("CONFIG_PROTECT_MASK", ""))) |
| self.updateprotect = protect_obj.updateprotect |
| self.isprotected = protect_obj.isprotected |
| self._installed_instance = None |
| self.contentscache = None |
| self._contents_inodes = None |
| self._contents_basenames = None |
| self._linkmap_broken = False |
| self._md5_merge_map = {} |
| |
| def lockdb(self): |
| if self._lock_vdb: |
| raise AssertionError("Lock already held.") |
| # At least the parent needs to exist for the lock file. |
| ensure_dirs(self.dbroot) |
| self._lock_vdb = lockdir(self.dbroot) |
| |
| def unlockdb(self): |
| if self._lock_vdb: |
| unlockdir(self._lock_vdb) |
| self._lock_vdb = None |
| |
| def getpath(self): |
| "return path to location of db information (for >>> informational display)" |
| return self.dbdir |
| |
| def exists(self): |
| "does the db entry exist? boolean." |
| return os.path.exists(self.dbdir) |
| |
| def delete(self): |
| """ |
| Remove this entry from the database |
| """ |
| if not os.path.exists(self.dbdir): |
| return |
| |
| # Check validity of self.dbdir before attempting to remove it. |
| if not self.dbdir.startswith(self.dbroot): |
| writemsg("portage.dblink.delete(): invalid dbdir: %s\n" % \ |
| self.dbdir, noiselevel=-1) |
| return |
| |
| shutil.rmtree(self.dbdir) |
| self.vartree.dbapi._remove(self) |
| |
| def clearcontents(self): |
| """ |
| For a given db entry (self), erase the CONTENTS values. |
| """ |
| if os.path.exists(self.dbdir+"/CONTENTS"): |
| os.unlink(self.dbdir+"/CONTENTS") |
| |
| def _clear_contents_cache(self): |
| self.contentscache = None |
| self._contents_inodes = None |
| self._contents_basenames = None |
| |
| def getcontents(self): |
| """ |
| Get the installed files of a given package (aka what that package installed) |
| """ |
| contents_file = os.path.join(self.dbdir, "CONTENTS") |
| if self.contentscache is not None: |
| return self.contentscache |
| pkgfiles = {} |
| try: |
| myc = open(contents_file,"r") |
| except EnvironmentError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| self.contentscache = pkgfiles |
| return pkgfiles |
| mylines = myc.readlines() |
| myc.close() |
| null_byte = "\0" |
| normalize_needed = self._normalize_needed |
| contents_split_counts = self._contents_split_counts |
| myroot = self.myroot |
| if myroot == os.path.sep: |
| myroot = None |
| pos = 0 |
| errors = [] |
| for pos, line in enumerate(mylines): |
| if null_byte in line: |
| # Null bytes are a common indication of corruption. |
| errors.append((pos + 1, "Null byte found in CONTENTS entry")) |
| continue |
| line = line.rstrip("\n") |
| # Split on " " so that even file paths that |
| # end with spaces can be handled. |
| mydat = line.split(" ") |
| entry_type = mydat[0] # empty string if line is empty |
| correct_split_count = contents_split_counts.get(entry_type) |
| if correct_split_count and len(mydat) > correct_split_count: |
| # Apparently file paths contain spaces, so reassemble |
| # the split have the correct_split_count. |
| newsplit = [entry_type] |
| spaces_total = len(mydat) - correct_split_count |
| if entry_type == "sym": |
| try: |
| splitter = mydat.index("->", 2, len(mydat) - 2) |
| except ValueError: |
| errors.append((pos + 1, "Unrecognized CONTENTS entry")) |
| continue |
| spaces_in_path = splitter - 2 |
| spaces_in_target = spaces_total - spaces_in_path |
| newsplit.append(" ".join(mydat[1:splitter])) |
| newsplit.append("->") |
| target_end = splitter + spaces_in_target + 2 |
| newsplit.append(" ".join(mydat[splitter + 1:target_end])) |
| newsplit.extend(mydat[target_end:]) |
| else: |
| path_end = spaces_total + 2 |
| newsplit.append(" ".join(mydat[1:path_end])) |
| newsplit.extend(mydat[path_end:]) |
| mydat = newsplit |
| |
| # we do this so we can remove from non-root filesystems |
| # (use the ROOT var to allow maintenance on other partitions) |
| try: |
| if normalize_needed.match(mydat[1]): |
| mydat[1] = normalize_path(mydat[1]) |
| if not mydat[1].startswith(os.path.sep): |
| mydat[1] = os.path.sep + mydat[1] |
| if myroot: |
| mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep)) |
| if mydat[0] == "obj": |
| #format: type, mtime, md5sum |
| pkgfiles[mydat[1]] = [mydat[0], mydat[3], mydat[2]] |
| elif mydat[0] == "dir": |
| #format: type |
| pkgfiles[mydat[1]] = [mydat[0]] |
| elif mydat[0] == "sym": |
| #format: type, mtime, dest |
| pkgfiles[mydat[1]] = [mydat[0], mydat[4], mydat[3]] |
| elif mydat[0] == "dev": |
| #format: type |
| pkgfiles[mydat[1]] = [mydat[0]] |
| elif mydat[0]=="fif": |
| #format: type |
| pkgfiles[mydat[1]] = [mydat[0]] |
| else: |
| errors.append((pos + 1, "Unrecognized CONTENTS entry")) |
| except (KeyError, IndexError): |
| errors.append((pos + 1, "Unrecognized CONTENTS entry")) |
| if errors: |
| writemsg("!!! Parse error in '%s'\n" % contents_file, noiselevel=-1) |
| for pos, e in errors: |
| writemsg("!!! line %d: %s\n" % (pos, e), noiselevel=-1) |
| self.contentscache = pkgfiles |
| return pkgfiles |
| |
| def unmerge(self, pkgfiles=None, trimworld=1, cleanup=1, |
| ldpath_mtimes=None, others_in_slot=None): |
| """ |
| Calls prerm |
| Unmerges a given package (CPV) |
| calls postrm |
| calls cleanrm |
| calls env_update |
| |
| @param pkgfiles: files to unmerge (generally self.getcontents() ) |
| @type pkgfiles: Dictionary |
| @param trimworld: Remove CPV from world file if True, not if False |
| @type trimworld: Boolean |
| @param cleanup: cleanup to pass to doebuild (see doebuild) |
| @type cleanup: Boolean |
| @param ldpath_mtimes: mtimes to pass to env_update (see env_update) |
| @type ldpath_mtimes: Dictionary |
| @param others_in_slot: all dblink instances in this slot, excluding self |
| @type others_in_slot: list |
| @rtype: Integer |
| @returns: |
| 1. os.EX_OK if everything went well. |
| 2. return code of the failed phase (for prerm, postrm, cleanrm) |
| |
| Notes: |
| The caller must ensure that lockdb() and unlockdb() are called |
| before and after this method. |
| """ |
| showMessage = self._display_merge |
| if self.vartree.dbapi._categories is not None: |
| self.vartree.dbapi._categories = None |
| # When others_in_slot is supplied, the security check has already been |
| # done for this slot, so it shouldn't be repeated until the next |
| # replacement or unmerge operation. |
| if others_in_slot is None: |
| slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] |
| slot_matches = self.vartree.dbapi.match( |
| "%s:%s" % (dep_getkey(self.mycpv), slot)) |
| others_in_slot = [] |
| for cur_cpv in slot_matches: |
| if cur_cpv == self.mycpv: |
| continue |
| others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], |
| self.vartree.root, self.settings, vartree=self.vartree, |
| treetype="vartree")) |
| |
| retval = self._security_check([self] + others_in_slot) |
| if retval: |
| return retval |
| |
| contents = self.getcontents() |
| # Now, don't assume that the name of the ebuild is the same as the |
| # name of the dir; the package may have been moved. |
| myebuildpath = None |
| ebuild_phase = "prerm" |
| log_path = None |
| mystuff = os.listdir(self.dbdir) |
| for x in mystuff: |
| if x.endswith(".ebuild"): |
| myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") |
| if x[:-7] != self.pkg: |
| # Clean up after vardbapi.move_ent() breakage in |
| # portage versions before 2.1.2 |
| os.rename(os.path.join(self.dbdir, x), myebuildpath) |
| write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") |
| break |
| |
| self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi) |
| if myebuildpath: |
| try: |
| doebuild_environment(myebuildpath, "prerm", self.myroot, |
| self.settings, 0, 0, self.vartree.dbapi) |
| except UnsupportedAPIException, e: |
| # Sometimes this happens due to corruption of the EAPI file. |
| writemsg("!!! FAILED prerm: %s\n" % \ |
| os.path.join(self.dbdir, "EAPI"), noiselevel=-1) |
| writemsg("%s\n" % str(e), noiselevel=-1) |
| myebuildpath = None |
| else: |
| catdir = os.path.dirname(self.settings["PORTAGE_BUILDDIR"]) |
| ensure_dirs(os.path.dirname(catdir), uid=portage_uid, |
| gid=portage_gid, mode=070, mask=0) |
| |
| builddir_lock = None |
| catdir_lock = None |
| scheduler = self._scheduler |
| retval = -1 |
| try: |
| if myebuildpath: |
| catdir_lock = lockdir(catdir) |
| ensure_dirs(catdir, |
| uid=portage_uid, gid=portage_gid, |
| mode=070, mask=0) |
| builddir_lock = lockdir( |
| self.settings["PORTAGE_BUILDDIR"]) |
| try: |
| unlockdir(catdir_lock) |
| finally: |
| catdir_lock = None |
| |
| prepare_build_dirs(self.myroot, self.settings, 1) |
| log_path = self.settings.get("PORTAGE_LOG_FILE") |
| |
| if scheduler is None: |
| retval = doebuild(myebuildpath, ebuild_phase, self.myroot, |
| self.settings, cleanup=cleanup, use_cache=0, |
| mydbapi=self.vartree.dbapi, tree=self.treetype, |
| vartree=self.vartree) |
| else: |
| retval = scheduler.dblinkEbuildPhase( |
| self, self.vartree.dbapi, myebuildpath, ebuild_phase) |
| |
| # XXX: Decide how to handle failures here. |
| if retval != os.EX_OK: |
| writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1) |
| |
| self._unmerge_pkgfiles(pkgfiles, others_in_slot) |
| self._clear_contents_cache() |
| |
| # Remove the registration of preserved libs for this pkg instance |
| plib_registry = self.vartree.dbapi.plib_registry |
| plib_registry.unregister(self.mycpv, self.settings["SLOT"], |
| self.vartree.dbapi.cpv_counter(self.mycpv)) |
| |
| if myebuildpath: |
| ebuild_phase = "postrm" |
| if scheduler is None: |
| retval = doebuild(myebuildpath, ebuild_phase, self.myroot, |
| self.settings, use_cache=0, tree=self.treetype, |
| mydbapi=self.vartree.dbapi, vartree=self.vartree) |
| else: |
| retval = scheduler.dblinkEbuildPhase( |
| self, self.vartree.dbapi, myebuildpath, ebuild_phase) |
| |
| # XXX: Decide how to handle failures here. |
| if retval != os.EX_OK: |
| writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1) |
| |
| # Skip this if another package in the same slot has just been |
| # merged on top of this package, since the other package has |
| # already called LinkageMap.rebuild() and passed it's NEEDED file |
| # in as an argument. |
| if not others_in_slot: |
| self._linkmap_rebuild(exclude_pkgs=(self.mycpv,)) |
| |
| # remove preserved libraries that don't have any consumers left |
| cpv_lib_map = self._find_unused_preserved_libs() |
| if cpv_lib_map: |
| self._remove_preserved_libs(cpv_lib_map) |
| for cpv, removed in cpv_lib_map.iteritems(): |
| if not self.vartree.dbapi.cpv_exists(cpv): |
| for dblnk in others_in_slot: |
| if dblnk.mycpv == cpv: |
| # This one just got merged so it doesn't |
| # register with cpv_exists() yet. |
| self.vartree.dbapi.removeFromContents( |
| dblnk, removed) |
| break |
| continue |
| self.vartree.dbapi.removeFromContents(cpv, removed) |
| else: |
| # Prune any preserved libs that may have |
| # been unmerged with this package. |
| self.vartree.dbapi.plib_registry.pruneNonExisting() |
| |
| finally: |
| if builddir_lock: |
| try: |
| if myebuildpath: |
| if retval != os.EX_OK: |
| msg_lines = [] |
| msg = ("The '%s' " % ebuild_phase) + \ |
| ("phase of the '%s' package " % self.mycpv) + \ |
| ("has failed with exit value %s." % retval) |
| from textwrap import wrap |
| msg_lines.extend(wrap(msg, 72)) |
| msg_lines.append("") |
| |
| ebuild_name = os.path.basename(myebuildpath) |
| ebuild_dir = os.path.dirname(myebuildpath) |
| msg = "The problem occurred while executing " + \ |
| ("the ebuild file named '%s' " % ebuild_name) + \ |
| ("located in the '%s' directory. " \ |
| % ebuild_dir) + \ |
| "If necessary, manually remove " + \ |
| "the environment.bz2 file and/or the " + \ |
| "ebuild file located in that directory." |
| msg_lines.extend(wrap(msg, 72)) |
| msg_lines.append("") |
| |
| msg = "Removal " + \ |
| "of the environment.bz2 file is " + \ |
| "preferred since it may allow the " + \ |
| "removal phases to execute successfully. " + \ |
| "The ebuild will be " + \ |
| "sourced and the eclasses " + \ |
| "from the current portage tree will be used " + \ |
| "when necessary. Removal of " + \ |
| "the ebuild file will cause the " + \ |
| "pkg_prerm() and pkg_postrm() removal " + \ |
| "phases to be skipped entirely." |
| msg_lines.extend(wrap(msg, 72)) |
| |
| self._eerror(ebuild_phase, msg_lines) |
| |
| # process logs created during pre/postrm |
| elog_process(self.mycpv, self.settings, phasefilter=filter_unmergephases) |
| if retval == os.EX_OK: |
| if scheduler is None: |
| doebuild(myebuildpath, "cleanrm", self.myroot, |
| self.settings, tree="vartree", |
| mydbapi=self.vartree.dbapi, |
| vartree=self.vartree) |
| else: |
| scheduler.dblinkEbuildPhase( |
| self, self.vartree.dbapi, |
| myebuildpath, "cleanrm") |
| finally: |
| unlockdir(builddir_lock) |
| try: |
| if myebuildpath and not catdir_lock: |
| # Lock catdir for removal if empty. |
| catdir_lock = lockdir(catdir) |
| finally: |
| if catdir_lock: |
| try: |
| os.rmdir(catdir) |
| except OSError, e: |
| if e.errno not in (errno.ENOENT, |
| errno.ENOTEMPTY, errno.EEXIST): |
| raise |
| del e |
| unlockdir(catdir_lock) |
| |
| if log_path is not None and os.path.exists(log_path): |
| # Restore this since it gets lost somewhere above and it |
| # needs to be set for _display_merge() to be able to log. |
| # Note that the log isn't necessarily supposed to exist |
| # since if PORT_LOGDIR is unset then it's a temp file |
| # so it gets cleaned above. |
| self.settings["PORTAGE_LOG_FILE"] = log_path |
| else: |
| self.settings.pop("PORTAGE_LOG_FILE", None) |
| |
| env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes, |
| contents=contents, env=self.settings.environ(), |
| writemsg_level=self._display_merge) |
| return os.EX_OK |
| |
| def _display_merge(self, msg, level=0, noiselevel=0): |
| if not self._verbose and noiselevel >= 0 and level < logging.WARN: |
| return |
| if self._scheduler is not None: |
| self._scheduler.dblinkDisplayMerge(self, msg, |
| level=level, noiselevel=noiselevel) |
| return |
| writemsg_level(msg, level=level, noiselevel=noiselevel) |
| |
| def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): |
| """ |
| |
| Unmerges the contents of a package from the liveFS |
| Removes the VDB entry for self |
| |
| @param pkgfiles: typically self.getcontents() |
| @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] } |
| @param others_in_slot: all dblink instances in this slot, excluding self |
| @type others_in_slot: list |
| @rtype: None |
| """ |
| |
| showMessage = self._display_merge |
| scheduler = self._scheduler |
| |
| if not pkgfiles: |
| showMessage("No package files given... Grabbing a set.\n") |
| pkgfiles = self.getcontents() |
| |
| if others_in_slot is None: |
| others_in_slot = [] |
| slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] |
| slot_matches = self.vartree.dbapi.match( |
| "%s:%s" % (dep_getkey(self.mycpv), slot)) |
| for cur_cpv in slot_matches: |
| if cur_cpv == self.mycpv: |
| continue |
| others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], |
| self.vartree.root, self.settings, |
| vartree=self.vartree, treetype="vartree")) |
| |
| dest_root = normalize_path(self.vartree.root).rstrip(os.path.sep) + \ |
| os.path.sep |
| dest_root_len = len(dest_root) - 1 |
| |
| conf_mem_file = os.path.join(dest_root, CONFIG_MEMORY_FILE) |
| cfgfiledict = grabdict(conf_mem_file) |
| stale_confmem = [] |
| |
| unmerge_orphans = "unmerge-orphans" in self.settings.features |
| |
| if pkgfiles: |
| self.updateprotect() |
| mykeys = pkgfiles.keys() |
| mykeys.sort() |
| mykeys.reverse() |
| |
| #process symlinks second-to-last, directories last. |
| mydirs = [] |
| ignored_unlink_errnos = ( |
| errno.EBUSY, errno.ENOENT, |
| errno.ENOTDIR, errno.EISDIR) |
| ignored_rmdir_errnos = ( |
| errno.EEXIST, errno.ENOTEMPTY, |
| errno.EBUSY, errno.ENOENT, |
| errno.ENOTDIR, errno.EISDIR) |
| modprotect = os.path.join(self.vartree.root, "lib/modules/") |
| |
| def unlink(file_name, lstatobj): |
| if bsd_chflags: |
| if lstatobj.st_flags != 0: |
| bsd_chflags.lchflags(file_name, 0) |
| parent_name = os.path.dirname(file_name) |
| # Use normal stat/chflags for the parent since we want to |
| # follow any symlinks to the real parent directory. |
| pflags = os.stat(parent_name).st_flags |
| if pflags != 0: |
| bsd_chflags.chflags(parent_name, 0) |
| try: |
| if not stat.S_ISLNK(lstatobj.st_mode): |
| # Remove permissions to ensure that any hardlinks to |
| # suid/sgid files are rendered harmless. |
| os.chmod(file_name, 0) |
| os.unlink(file_name) |
| finally: |
| if bsd_chflags and pflags != 0: |
| # Restore the parent flags we saved before unlinking |
| bsd_chflags.chflags(parent_name, pflags) |
| |
| def show_unmerge(zing, desc, file_type, file_name): |
| showMessage("%s %s %s %s\n" % \ |
| (zing, desc.ljust(8), file_type, file_name)) |
| for i, objkey in enumerate(mykeys): |
| |
| if scheduler is not None and \ |
| 0 == i % self._file_merge_yield_interval: |
| scheduler.scheduleYield() |
| |
| obj = normalize_path(objkey) |
| file_data = pkgfiles[objkey] |
| file_type = file_data[0] |
| statobj = None |
| try: |
| statobj = os.stat(obj) |
| except OSError: |
| pass |
| lstatobj = None |
| try: |
| lstatobj = os.lstat(obj) |
| except (OSError, AttributeError): |
| pass |
| islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) |
| if lstatobj is None: |
| show_unmerge("---", "!found", file_type, obj) |
| continue |
| if obj.startswith(dest_root): |
| relative_path = obj[dest_root_len:] |
| is_owned = False |
| for dblnk in others_in_slot: |
| if dblnk.isowner(relative_path, dest_root): |
| is_owned = True |
| break |
| if is_owned: |
| # A new instance of this package claims the file, so |
| # don't unmerge it. |
| show_unmerge("---", "replaced", file_type, obj) |
| continue |
| elif relative_path in cfgfiledict: |
| stale_confmem.append(relative_path) |
| # next line includes a tweak to protect modules from being unmerged, |
| # but we don't protect modules from being overwritten if they are |
| # upgraded. We effectively only want one half of the config protection |
| # functionality for /lib/modules. For portage-ng both capabilities |
| # should be able to be independently specified. |
| if obj.startswith(modprotect): |
| show_unmerge("---", "cfgpro", file_type, obj) |
| continue |
| |
| # Don't unlink symlinks to directories here since that can |
| # remove /lib and /usr/lib symlinks. |
| if unmerge_orphans and \ |
| lstatobj and not stat.S_ISDIR(lstatobj.st_mode) and \ |
| not (islink and statobj and stat.S_ISDIR(statobj.st_mode)) and \ |
| not self.isprotected(obj): |
| try: |
| unlink(obj, lstatobj) |
| except EnvironmentError, e: |
| if e.errno not in ignored_unlink_errnos: |
| raise |
| del e |
| show_unmerge("<<<", "", file_type, obj) |
| continue |
| |
| lmtime = str(lstatobj[stat.ST_MTIME]) |
| if (pkgfiles[objkey][0] not in ("dir", "fif", "dev")) and (lmtime != pkgfiles[objkey][1]): |
| show_unmerge("---", "!mtime", file_type, obj) |
| continue |
| |
| if pkgfiles[objkey][0] == "dir": |
| if statobj is None or not stat.S_ISDIR(statobj.st_mode): |
| show_unmerge("---", "!dir", file_type, obj) |
| continue |
| mydirs.append(obj) |
| elif pkgfiles[objkey][0] == "sym": |
| if not islink: |
| show_unmerge("---", "!sym", file_type, obj) |
| continue |
| # Go ahead and unlink symlinks to directories here when |
| # they're actually recorded as symlinks in the contents. |
| # Normally, symlinks such as /lib -> lib64 are not recorded |
| # as symlinks in the contents of a package. If a package |
| # installs something into ${D}/lib/, it is recorded in the |
| # contents as a directory even if it happens to correspond |
| # to a symlink when it's merged to the live filesystem. |
| try: |
| unlink(obj, lstatobj) |
| show_unmerge("<<<", "", file_type, obj) |
| except (OSError, IOError),e: |
| if e.errno not in ignored_unlink_errnos: |
| raise |
| del e |
| show_unmerge("!!!", "", file_type, obj) |
| elif pkgfiles[objkey][0] == "obj": |
| if statobj is None or not stat.S_ISREG(statobj.st_mode): |
| show_unmerge("---", "!obj", file_type, obj) |
| continue |
| mymd5 = None |
| try: |
| mymd5 = perform_md5(obj, calc_prelink=1) |
| except FileNotFound, e: |
| # the file has disappeared between now and our stat call |
| show_unmerge("---", "!obj", file_type, obj) |
| continue |
| |
| # string.lower is needed because db entries used to be in upper-case. The |
| # string.lower allows for backwards compatibility. |
| if mymd5 != pkgfiles[objkey][2].lower(): |
| show_unmerge("---", "!md5", file_type, obj) |
| continue |
| try: |
| unlink(obj, lstatobj) |
| except (OSError, IOError), e: |
| if e.errno not in ignored_unlink_errnos: |
| raise |
| del e |
| show_unmerge("<<<", "", file_type, obj) |
| elif pkgfiles[objkey][0] == "fif": |
| if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): |
| show_unmerge("---", "!fif", file_type, obj) |
| continue |
| show_unmerge("---", "", file_type, obj) |
| elif pkgfiles[objkey][0] == "dev": |
| show_unmerge("---", "", file_type, obj) |
| |
| mydirs.sort() |
| mydirs.reverse() |
| |
| for obj in mydirs: |
| try: |
| if bsd_chflags: |
| lstatobj = os.lstat(obj) |
| if lstatobj.st_flags != 0: |
| bsd_chflags.lchflags(obj, 0) |
| parent_name = os.path.dirname(obj) |
| # Use normal stat/chflags for the parent since we want to |
| # follow any symlinks to the real parent directory. |
| pflags = os.stat(parent_name).st_flags |
| if pflags != 0: |
| bsd_chflags.chflags(parent_name, 0) |
| try: |
| os.rmdir(obj) |
| finally: |
| if bsd_chflags and pflags != 0: |
| # Restore the parent flags we saved before unlinking |
| bsd_chflags.chflags(parent_name, pflags) |
| show_unmerge("<<<", "", "dir", obj) |
| except EnvironmentError, e: |
| if e.errno not in ignored_rmdir_errnos: |
| raise |
| if e.errno != errno.ENOENT: |
| show_unmerge("---", "!empty", "dir", obj) |
| del e |
| |
| # Remove stale entries from config memory. |
| if stale_confmem: |
| for filename in stale_confmem: |
| del cfgfiledict[filename] |
| writedict(cfgfiledict, conf_mem_file) |
| |
| #remove self from vartree database so that our own virtual gets zapped if we're the last node |
| self.vartree.zap(self.mycpv) |
| |
| def isowner(self, filename, destroot): |
| """ |
| Check if a file belongs to this package. This may |
| result in a stat call for the parent directory of |
| every installed file, since the inode numbers are |
| used to work around the problem of ambiguous paths |
| caused by symlinked directories. The results of |
| stat calls are cached to optimize multiple calls |
| to this method. |
| |
| @param filename: |
| @type filename: |
| @param destroot: |
| @type destroot: |
| @rtype: Boolean |
| @returns: |
| 1. True if this package owns the file. |
| 2. False if this package does not own the file. |
| """ |
| return bool(self._match_contents(filename, destroot)) |
| |
| def _match_contents(self, filename, destroot): |
| """ |
| The matching contents entry is returned, which is useful |
| since the path may differ from the one given by the caller, |
| due to symlinks. |
| |
| @rtype: String |
| @return: the contents entry corresponding to the given path, or False |
| if the file is not owned by this package. |
| """ |
| |
| destfile = normalize_path( |
| os.path.join(destroot, filename.lstrip(os.path.sep))) |
| |
| pkgfiles = self.getcontents() |
| if pkgfiles and destfile in pkgfiles: |
| return destfile |
| if pkgfiles: |
| basename = os.path.basename(destfile) |
| if self._contents_basenames is None: |
| self._contents_basenames = set( |
| os.path.basename(x) for x in pkgfiles) |
| if basename not in self._contents_basenames: |
| # This is a shortcut that, in most cases, allows us to |
| # eliminate this package as an owner without the need |
| # to examine inode numbers of parent directories. |
| return False |
| |
| # Use stat rather than lstat since we want to follow |
| # any symlinks to the real parent directory. |
| parent_path = os.path.dirname(destfile) |
| try: |
| parent_stat = os.stat(parent_path) |
| except EnvironmentError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| return False |
| if self._contents_inodes is None: |
| self._contents_inodes = {} |
| parent_paths = set() |
| for x in pkgfiles: |
| p_path = os.path.dirname(x) |
| if p_path in parent_paths: |
| continue |
| parent_paths.add(p_path) |
| try: |
| s = os.stat(p_path) |
| except OSError: |
| pass |
| else: |
| inode_key = (s.st_dev, s.st_ino) |
| # Use lists of paths in case multiple |
| # paths reference the same inode. |
| p_path_list = self._contents_inodes.get(inode_key) |
| if p_path_list is None: |
| p_path_list = [] |
| self._contents_inodes[inode_key] = p_path_list |
| if p_path not in p_path_list: |
| p_path_list.append(p_path) |
| p_path_list = self._contents_inodes.get( |
| (parent_stat.st_dev, parent_stat.st_ino)) |
| if p_path_list: |
| for p_path in p_path_list: |
| x = os.path.join(p_path, basename) |
| if x in pkgfiles: |
| return x |
| |
| return False |
| |
| def _linkmap_rebuild(self, **kwargs): |
| if self._linkmap_broken: |
| return |
| try: |
| self.vartree.dbapi.linkmap.rebuild(**kwargs) |
| except CommandNotFound, e: |
| self._linkmap_broken = True |
| self._display_merge("!!! Disabling preserve-libs " + \ |
| "due to error: Command Not Found: %s\n" % (e,), |
| level=logging.ERROR, noiselevel=-1) |
| |
| def _find_libs_to_preserve(self): |
| """ |
| Get set of relative paths for libraries to be preserved. The file |
| paths are selected from self._installed_instance.getcontents(). |
| """ |
| if self._linkmap_broken or not \ |
| (self._installed_instance is not None and \ |
| "preserve-libs" in self.settings.features): |
| return None |
| |
| linkmap = self.vartree.dbapi.linkmap |
| installed_instance = self._installed_instance |
| old_contents = installed_instance.getcontents() |
| root = self.myroot |
| root_len = len(root) - 1 |
| lib_graph = digraph() |
| path_node_map = {} |
| |
| def path_to_node(path): |
| node = path_node_map.get(path) |
| if node is None: |
| node = LinkageMap._LibGraphNode(path, root) |
| alt_path_node = lib_graph.get(node) |
| if alt_path_node is not None: |
| node = alt_path_node |
| node.alt_paths.add(path) |
| path_node_map[path] = node |
| return node |
| |
| consumer_map = {} |
| provider_nodes = set() |
| # Create provider nodes and add them to the graph. |
| for f_abs in old_contents: |
| f = f_abs[root_len:] |
| if self.isowner(f, root): |
| continue |
| try: |
| consumers = linkmap.findConsumers(f) |
| except KeyError: |
| continue |
| if not consumers: |
| continue |
| provider_node = path_to_node(f) |
| lib_graph.add(provider_node, None) |
| provider_nodes.add(provider_node) |
| consumer_map[provider_node] = consumers |
| |
| # Create consumer nodes and add them to the graph. |
| # Note that consumers can also be providers. |
| for provider_node, consumers in consumer_map.iteritems(): |
| for c in consumers: |
| if self.isowner(c, root): |
| continue |
| consumer_node = path_to_node(c) |
| if installed_instance.isowner(c, root) and \ |
| consumer_node not in provider_nodes: |
| # This is not a provider, so it will be uninstalled. |
| continue |
| lib_graph.add(provider_node, consumer_node) |
| |
| # Locate nodes which should be preserved. They consist of all |
| # providers that are reachable from consumers that are not |
| # providers themselves. |
| preserve_nodes = set() |
| for consumer_node in lib_graph.root_nodes(): |
| if consumer_node in provider_nodes: |
| continue |
| # Preserve all providers that are reachable from this consumer. |
| node_stack = lib_graph.child_nodes(consumer_node) |
| while node_stack: |
| provider_node = node_stack.pop() |
| if provider_node in preserve_nodes: |
| continue |
| preserve_nodes.add(provider_node) |
| node_stack.extend(lib_graph.child_nodes(provider_node)) |
| |
| preserve_paths = set() |
| for preserve_node in preserve_nodes: |
| # Make sure that at least one of the paths is not a symlink. |
| # This prevents symlinks from being erroneously preserved by |
| # themselves when the old instance installed symlinks that |
| # the new instance does not install. |
| have_lib = False |
| for f in preserve_node.alt_paths: |
| f_abs = os.path.join(root, f.lstrip(os.sep)) |
| try: |
| if stat.S_ISREG(os.lstat(f_abs).st_mode): |
| have_lib = True |
| break |
| except OSError: |
| continue |
| |
| if have_lib: |
| preserve_paths.update(preserve_node.alt_paths) |
| |
| return preserve_paths |
| |
| def _add_preserve_libs_to_contents(self, preserve_paths): |
| """ |
| Preserve libs returned from _find_libs_to_preserve(). |
| """ |
| |
| if not preserve_paths: |
| return |
| |
| showMessage = self._display_merge |
| root = self.myroot |
| |
| # Copy contents entries from the old package to the new one. |
| new_contents = self.getcontents().copy() |
| old_contents = self._installed_instance.getcontents() |
| for f in sorted(preserve_paths): |
| f_abs = os.path.join(root, f.lstrip(os.sep)) |
| contents_entry = old_contents.get(f_abs) |
| if contents_entry is None: |
| # This will probably never happen, but it might if one of the |
| # paths returned from findConsumers() refers to one of the libs |
| # that should be preserved yet the path is not listed in the |
| # contents. Such a path might belong to some other package, so |
| # it shouldn't be preserved here. |
| showMessage(("!!! File '%s' will not be preserved " + \ |
| "due to missing contents entry\n") % (f_abs,), |
| level=logging.ERROR, noiselevel=-1) |
| preserve_paths.remove(f) |
| continue |
| new_contents[f_abs] = contents_entry |
| obj_type = contents_entry[0] |
| showMessage(">>> needed %s %s\n" % (obj_type, f_abs), |
| noiselevel=-1) |
| # Add parent directories to contents if necessary. |
| parent_dir = os.path.dirname(f_abs) |
| while len(parent_dir) > len(root): |
| new_contents[parent_dir] = ["dir"] |
| prev = parent_dir |
| parent_dir = os.path.dirname(parent_dir) |
| if prev == parent_dir: |
| break |
| outfile = atomic_ofstream(os.path.join(self.dbtmpdir, "CONTENTS")) |
| write_contents(new_contents, root, outfile) |
| outfile.close() |
| self._clear_contents_cache() |
| |
| def _find_unused_preserved_libs(self): |
| """ |
| Find preserved libraries that don't have any consumers left. |
| """ |
| |
| if self._linkmap_broken: |
| return {} |
| |
| # Since preserved libraries can be consumers of other preserved |
| # libraries, use a graph to track consumer relationships. |
| plib_dict = self.vartree.dbapi.plib_registry.getPreservedLibs() |
| lib_graph = digraph() |
| preserved_nodes = set() |
| preserved_paths = set() |
| path_cpv_map = {} |
| path_node_map = {} |
| root = self.myroot |
| |
| def path_to_node(path): |
| node = path_node_map.get(path) |
| if node is None: |
| node = LinkageMap._LibGraphNode(path, root) |
| alt_path_node = lib_graph.get(node) |
| if alt_path_node is not None: |
| node = alt_path_node |
| node.alt_paths.add(path) |
| path_node_map[path] = node |
| return node |
| |
| linkmap = self.vartree.dbapi.linkmap |
| for cpv, plibs in plib_dict.iteritems(): |
| for f in plibs: |
| path_cpv_map[f] = cpv |
| preserved_node = path_to_node(f) |
| if not preserved_node.file_exists(): |
| continue |
| lib_graph.add(preserved_node, None) |
| preserved_paths.add(f) |
| preserved_nodes.add(preserved_node) |
| for c in self.vartree.dbapi.linkmap.findConsumers(f): |
| consumer_node = path_to_node(c) |
| if not consumer_node.file_exists(): |
| continue |
| # Note that consumers may also be providers. |
| lib_graph.add(preserved_node, consumer_node) |
| |
| # Eliminate consumers having providers with the same soname as an |
| # installed library that is not preserved. This eliminates |
| # libraries that are erroneously preserved due to a move from one |
| # directory to another. |
| provider_cache = {} |
| for preserved_node in preserved_nodes: |
| soname = linkmap.getSoname(preserved_node) |
| for consumer_node in lib_graph.parent_nodes(preserved_node): |
| if consumer_node in preserved_nodes: |
| continue |
| providers = provider_cache.get(consumer_node) |
| if providers is None: |
| providers = linkmap.findProviders(consumer_node) |
| provider_cache[consumer_node] = providers |
| providers = providers.get(soname) |
| if providers is None: |
| continue |
| for provider in providers: |
| if provider in preserved_paths: |
| continue |
| provider_node = path_to_node(provider) |
| if not provider_node.file_exists(): |
| continue |
| if provider_node in preserved_nodes: |
| continue |
| # An alternative provider seems to be |
| # installed, so drop this edge. |
| lib_graph.remove_edge(preserved_node, consumer_node) |
| break |
| |
| cpv_lib_map = {} |
| while not lib_graph.empty(): |
| root_nodes = preserved_nodes.intersection(lib_graph.root_nodes()) |
| if not root_nodes: |
| break |
| lib_graph.difference_update(root_nodes) |
| unlink_list = set() |
| for node in root_nodes: |
| unlink_list.update(node.alt_paths) |
| unlink_list = sorted(unlink_list) |
| for obj in unlink_list: |
| cpv = path_cpv_map.get(obj) |
| if cpv is None: |
| # This means that a symlink is in the preserved libs |
| # registry, but the actual lib it points to is not. |
| self._display_merge("!!! symlink to lib is preserved, " + \ |
| "but not the lib itself:\n!!! '%s'\n" % (obj,), |
| level=logging.ERROR, noiselevel=-1) |
| continue |
| removed = cpv_lib_map.get(cpv) |
| if removed is None: |
| removed = set() |
| cpv_lib_map[cpv] = removed |
| removed.add(obj) |
| |
| return cpv_lib_map |
| |
| def _remove_preserved_libs(self, cpv_lib_map): |
| """ |
| Remove files returned from _find_unused_preserved_libs(). |
| """ |
| |
| files_to_remove = set() |
| for files in cpv_lib_map.itervalues(): |
| files_to_remove.update(files) |
| files_to_remove = sorted(files_to_remove) |
| showMessage = self._display_merge |
| root = self.myroot |
| |
| parent_dirs = set() |
| for obj in files_to_remove: |
| obj = os.path.join(root, obj.lstrip(os.sep)) |
| parent_dirs.add(os.path.dirname(obj)) |
| if os.path.islink(obj): |
| obj_type = "sym" |
| else: |
| obj_type = "obj" |
| try: |
| os.unlink(obj) |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| else: |
| showMessage("<<< !needed %s %s\n" % (obj_type, obj), |
| noiselevel=-1) |
| |
| # Remove empty parent directories if possible. |
| while parent_dirs: |
| x = parent_dirs.pop() |
| while True: |
| try: |
| os.rmdir(x) |
| except OSError: |
| break |
| prev = x |
| x = os.path.dirname(x) |
| if x == prev: |
| break |
| |
| self.vartree.dbapi.plib_registry.pruneNonExisting() |
| |
| def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): |
| collision_ignore = set([normalize_path(myignore) for myignore in \ |
| shlex.split(self.settings.get("COLLISION_IGNORE", ""))]) |
| |
| # For collisions with preserved libraries, the current package |
| # will assume ownership and the libraries will be unregistered. |
| plib_dict = self.vartree.dbapi.plib_registry.getPreservedLibs() |
| plib_cpv_map = {} |
| plib_paths = set() |
| for cpv, paths in plib_dict.iteritems(): |
| plib_paths.update(paths) |
| for f in paths: |
| plib_cpv_map[f] = cpv |
| plib_inodes = self._lstat_inode_map(plib_paths) |
| plib_collisions = {} |
| |
| showMessage = self._display_merge |
| scheduler = self._scheduler |
| stopmerge = False |
| collisions = [] |
| destroot = normalize_path(destroot).rstrip(os.path.sep) + \ |
| os.path.sep |
| showMessage(" %s checking %d files for package collisions\n" % \ |
| (colorize("GOOD", "*"), len(mycontents))) |
| for i, f in enumerate(mycontents): |
| if i % 1000 == 0 and i != 0: |
| showMessage("%d files checked ...\n" % i) |
| |
| if scheduler is not None and \ |
| 0 == i % self._file_merge_yield_interval: |
| scheduler.scheduleYield() |
| |
| dest_path = normalize_path( |
| os.path.join(destroot, f.lstrip(os.path.sep))) |
| try: |
| dest_lstat = os.lstat(dest_path) |
| except EnvironmentError, e: |
| if e.errno == errno.ENOENT: |
| del e |
| continue |
| elif e.errno == errno.ENOTDIR: |
| del e |
| # A non-directory is in a location where this package |
| # expects to have a directory. |
| dest_lstat = None |
| parent_path = dest_path |
| while len(parent_path) > len(destroot): |
| parent_path = os.path.dirname(parent_path) |
| try: |
| dest_lstat = os.lstat(parent_path) |
| break |
| except EnvironmentError, e: |
| if e.errno != errno.ENOTDIR: |
| raise |
| del e |
| if not dest_lstat: |
| raise AssertionError( |
| "unable to find non-directory " + \ |
| "parent for '%s'" % dest_path) |
| dest_path = parent_path |
| f = os.path.sep + dest_path[len(destroot):] |
| if f in collisions: |
| continue |
| else: |
| raise |
| if f[0] != "/": |
| f="/"+f |
| |
| plibs = plib_inodes.get((dest_lstat.st_dev, dest_lstat.st_ino)) |
| if plibs: |
| for path in plibs: |
| cpv = plib_cpv_map[path] |
| paths = plib_collisions.get(cpv) |
| if paths is None: |
| paths = set() |
| plib_collisions[cpv] = paths |
| paths.add(path) |
| # The current package will assume ownership and the |
| # libraries will be unregistered, so exclude this |
| # path from the normal collisions. |
| continue |
| |
| isowned = False |
| full_path = os.path.join(destroot, f.lstrip(os.path.sep)) |
| for ver in mypkglist: |
| if ver.isowner(f, destroot): |
| isowned = True |
| break |
| if not isowned and self.isprotected(full_path): |
| isowned = True |
| if not isowned: |
| stopmerge = True |
| if collision_ignore: |
| if f in collision_ignore: |
| stopmerge = False |
| else: |
| for myignore in collision_ignore: |
| if f.startswith(myignore + os.path.sep): |
| stopmerge = False |
| break |
| if stopmerge: |
| collisions.append(f) |
| return collisions, plib_collisions |
| |
| def _lstat_inode_map(self, path_iter): |
| """ |
| Use lstat to create a map of the form: |
| {(st_dev, st_ino) : set([path1, path2, ...])} |
| Multiple paths may reference the same inode due to hardlinks. |
| All lstat() calls are relative to self.myroot. |
| """ |
| root = self.myroot |
| inode_map = {} |
| for f in path_iter: |
| path = os.path.join(root, f.lstrip(os.sep)) |
| try: |
| st = os.lstat(path) |
| except OSError, e: |
| if e.errno not in (errno.ENOENT, errno.ENOTDIR): |
| raise |
| del e |
| continue |
| key = (st.st_dev, st.st_ino) |
| paths = inode_map.get(key) |
| if paths is None: |
| paths = set() |
| inode_map[key] = paths |
| paths.add(f) |
| return inode_map |
| |
| def _security_check(self, installed_instances): |
| if not installed_instances: |
| return 0 |
| |
| showMessage = self._display_merge |
| scheduler = self._scheduler |
| |
| file_paths = set() |
| for dblnk in installed_instances: |
| file_paths.update(dblnk.getcontents()) |
| inode_map = {} |
| real_paths = set() |
| for i, path in enumerate(file_paths): |
| |
| if scheduler is not None and \ |
| 0 == i % self._file_merge_yield_interval: |
| scheduler.scheduleYield() |
| |
| try: |
| s = os.lstat(path) |
| except OSError, e: |
| if e.errno not in (errno.ENOENT, errno.ENOTDIR): |
| raise |
| del e |
| continue |
| if not stat.S_ISREG(s.st_mode): |
| continue |
| path = os.path.realpath(path) |
| if path in real_paths: |
| continue |
| real_paths.add(path) |
| if s.st_nlink > 1 and \ |
| s.st_mode & (stat.S_ISUID | stat.S_ISGID): |
| k = (s.st_dev, s.st_ino) |
| inode_map.setdefault(k, []).append((path, s)) |
| suspicious_hardlinks = [] |
| for path_list in inode_map.itervalues(): |
| path, s = path_list[0] |
| if len(path_list) == s.st_nlink: |
| # All hardlinks seem to be owned by this package. |
| continue |
| suspicious_hardlinks.append(path_list) |
| if not suspicious_hardlinks: |
| return 0 |
| |
| msg = [] |
| msg.append("suid/sgid file(s) " + \ |
| "with suspicious hardlink(s):") |
| msg.append("") |
| for path_list in suspicious_hardlinks: |
| for path, s in path_list: |
| msg.append("\t%s" % path) |
| msg.append("") |
| msg.append("See the Gentoo Security Handbook " + \ |
| "guide for advice on how to proceed.") |
| |
| self._eerror("preinst", msg) |
| |
| return 1 |
| |
| def _eqawarn(self, phase, lines): |
| from portage.elog.messages import eqawarn as _eqawarn |
| if self._scheduler is None: |
| for l in lines: |
| _eqawarn(l, phase=phase, key=self.settings.mycpv) |
| else: |
| self._scheduler.dblinkElog(self, |
| phase, _eqawarn, lines) |
| |
| def _eerror(self, phase, lines): |
| from portage.elog.messages import eerror as _eerror |
| if self._scheduler is None: |
| for l in lines: |
| _eerror(l, phase=phase, key=self.settings.mycpv) |
| else: |
| self._scheduler.dblinkElog(self, |
| phase, _eerror, lines) |
| |
| def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, |
| mydbapi=None, prev_mtimes=None): |
| """ |
| |
| This function does the following: |
| |
| calls self._preserve_libs if FEATURES=preserve-libs |
| calls self._collision_protect if FEATURES=collision-protect |
| calls doebuild(mydo=pkg_preinst) |
| Merges the package to the livefs |
| unmerges old version (if required) |
| calls doebuild(mydo=pkg_postinst) |
| calls env_update |
| calls elog_process |
| |
| @param srcroot: Typically this is ${D} |
| @type srcroot: String (Path) |
| @param destroot: Path to merge to (usually ${ROOT}) |
| @type destroot: String (Path) |
| @param inforoot: root of the vardb entry ? |
| @type inforoot: String (Path) |
| @param myebuild: path to the ebuild that we are processing |
| @type myebuild: String (Path) |
| @param mydbapi: dbapi which is handed to doebuild. |
| @type mydbapi: portdbapi instance |
| @param prev_mtimes: { Filename:mtime } mapping for env_update |
| @type prev_mtimes: Dictionary |
| @rtype: Boolean |
| @returns: |
| 1. 0 on success |
| 2. 1 on failure |
| |
| secondhand is a list of symlinks that have been skipped due to their target |
| not existing; we will merge these symlinks at a later time. |
| """ |
| |
| showMessage = self._display_merge |
| scheduler = self._scheduler |
| |
| srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep |
| destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep |
| |
| if not os.path.isdir(srcroot): |
| showMessage("!!! Directory Not Found: D='%s'\n" % srcroot, |
| level=logging.ERROR, noiselevel=-1) |
| return 1 |
| |
| slot = '' |
| for var_name in ('CHOST', 'SLOT'): |
| try: |
| val = open(os.path.join(inforoot, var_name)).readline().strip() |
| except EnvironmentError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| val = '' |
| |
| if var_name == 'SLOT': |
| slot = val |
| |
| if not slot.strip(): |
| slot = self.settings.get(var_name, '') |
| if not slot.strip(): |
| showMessage("!!! SLOT is undefined\n", |
| level=logging.ERROR, noiselevel=-1) |
| return 1 |
| write_atomic(os.path.join(inforoot, var_name), slot + '\n') |
| |
| if val != self.settings.get(var_name, ''): |
| self._eqawarn('preinst', |
| ["QA Notice: Expected %s='%s', got '%s'\n" % \ |
| (var_name, self.settings.get(var_name, ''), val)]) |
| |
| def eerror(lines): |
| self._eerror("preinst", lines) |
| |
| if not os.path.exists(self.dbcatdir): |
| os.makedirs(self.dbcatdir) |
| |
| otherversions = [] |
| for v in self.vartree.dbapi.cp_list(self.mysplit[0]): |
| otherversions.append(v.split("/")[1]) |
| |
| cp = self.mysplit[0] |
| slot_atom = "%s:%s" % (cp, slot) |
| |
| # filter any old-style virtual matches |
| slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) \ |
| if cpv_getkey(cpv) == cp] |
| |
| if self.mycpv not in slot_matches and \ |
| self.vartree.dbapi.cpv_exists(self.mycpv): |
| # handle multislot or unapplied slotmove |
| slot_matches.append(self.mycpv) |
| |
| others_in_slot = [] |
| from portage import config |
| for cur_cpv in slot_matches: |
| # Clone the config in case one of these has to be unmerged since |
| # we need it to have private ${T} etc... for things like elog. |
| others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], |
| self.vartree.root, config(clone=self.settings), |
| vartree=self.vartree, treetype="vartree", |
| scheduler=self._scheduler)) |
| |
| retval = self._security_check(others_in_slot) |
| if retval: |
| return retval |
| |
| if slot_matches: |
| # Used by self.isprotected(). |
| max_dblnk = None |
| max_counter = -1 |
| for dblnk in others_in_slot: |
| cur_counter = self.vartree.dbapi.cpv_counter(dblnk.mycpv) |
| if cur_counter > max_counter: |
| max_counter = cur_counter |
| max_dblnk = dblnk |
| self._installed_instance = max_dblnk |
| |
| myfilelist = [] |
| mylinklist = [] |
| paths_with_newlines = [] |
| srcroot_len = len(srcroot) |
| def onerror(e): |
| raise |
| for parent, dirs, files in os.walk(srcroot, onerror=onerror): |
| for f in files: |
| file_path = os.path.join(parent, f) |
| relative_path = file_path[srcroot_len:] |
| |
| if "\n" in relative_path: |
| paths_with_newlines.append(relative_path) |
| |
| file_mode = os.lstat(file_path).st_mode |
| if stat.S_ISREG(file_mode): |
| myfilelist.append(relative_path) |
| elif stat.S_ISLNK(file_mode): |
| # Note: os.walk puts symlinks to directories in the "dirs" |
| # list and it does not traverse them since that could lead |
| # to an infinite recursion loop. |
| mylinklist.append(relative_path) |
| |
| if paths_with_newlines: |
| msg = [] |
| msg.append("This package installs one or more files containing") |
| msg.append("a newline (\\n) character:") |
| msg.append("") |
| paths_with_newlines.sort() |
| for f in paths_with_newlines: |
| msg.append("\t/%s" % (f.replace("\n", "\\n"))) |
| msg.append("") |
| msg.append("package %s NOT merged" % self.mycpv) |
| msg.append("") |
| eerror(msg) |
| return 1 |
| |
| # If there are no files to merge, and an installed package in the same |
| # slot has files, it probably means that something went wrong. |
| if self.settings.get("PORTAGE_PACKAGE_EMPTY_ABORT") == "1" and \ |
| not myfilelist and not mylinklist and others_in_slot: |
| installed_files = None |
| for other_dblink in others_in_slot: |
| installed_files = other_dblink.getcontents() |
| if not installed_files: |
| continue |
| from textwrap import wrap |
| wrap_width = 72 |
| msg = [] |
| d = ( |
| self.mycpv, |
| other_dblink.mycpv |
| ) |
| msg.extend(wrap(("The '%s' package will not install " + \ |
| "any files, but the currently installed '%s'" + \ |
| " package has the following files: ") % d, wrap_width)) |
| msg.append("") |
| msg.extend(sorted(installed_files)) |
| msg.append("") |
| msg.append("package %s NOT merged" % self.mycpv) |
| msg.append("") |
| msg.extend(wrap( |
| ("Manually run `emerge --unmerge =%s` " % \ |
| other_dblink.mycpv) + "if you really want to " + \ |
| "remove the above files. Set " + \ |
| "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " + \ |
| "/etc/make.conf if you do not want to " + \ |
| "abort in cases like this.", |
| wrap_width)) |
| eerror(msg) |
| if installed_files: |
| return 1 |
| |
| # check for package collisions |
| blockers = None |
| if self._blockers is not None: |
| # This is only supposed to be called when |
| # the vdb is locked, like it is here. |
| blockers = self._blockers() |
| if blockers is None: |
| blockers = [] |
| collisions, plib_collisions = \ |
| self._collision_protect(srcroot, destroot, |
| others_in_slot + blockers, myfilelist + mylinklist) |
| |
| # Make sure the ebuild environment is initialized and that ${T}/elog |
| # exists for logging of collision-protect eerror messages. |
| if myebuild is None: |
| myebuild = os.path.join(inforoot, self.pkg + ".ebuild") |
| doebuild_environment(myebuild, "preinst", destroot, |
| self.settings, 0, 0, mydbapi) |
| prepare_build_dirs(destroot, self.settings, cleanup) |
| |
| if collisions: |
| collision_protect = "collision-protect" in self.settings.features |
| protect_owned = "protect-owned" in self.settings.features |
| msg = "This package will overwrite one or more files that" + \ |
| " may belong to other packages (see list below)." |
| if not (collision_protect or protect_owned): |
| msg += " Add either \"collision-protect\" or" + \ |
| " \"protect-owned\" to FEATURES in" + \ |
| " make.conf if you would like the merge to abort" + \ |
| " in cases like this. See the make.conf man page for" + \ |
| " more information about these features." |
| if self.settings.get("PORTAGE_QUIET") != "1": |
| msg += " You can use a command such as" + \ |
| " `portageq owners / <filename>` to identify the" + \ |
| " installed package that owns a file. If portageq" + \ |
| " reports that only one package owns a file then do NOT" + \ |
| " file a bug report. A bug report is only useful if it" + \ |
| " identifies at least two or more packages that are known" + \ |
| " to install the same file(s)." + \ |
| " If a collision occurs and you" + \ |
| " can not explain where the file came from then you" + \ |
| " should simply ignore the collision since there is not" + \ |
| " enough information to determine if a real problem" + \ |
| " exists. Please do NOT file a bug report at" + \ |
| " http://bugs.gentoo.org unless you report exactly which" + \ |
| " two packages install the same file(s). Once again," + \ |
| " please do NOT file a bug report unless you have" + \ |
| " completely understood the above message." |
| |
| self.settings["EBUILD_PHASE"] = "preinst" |
| from textwrap import wrap |
| msg = wrap(msg, 70) |
| if collision_protect: |
| msg.append("") |
| msg.append("package %s NOT merged" % self.settings.mycpv) |
| msg.append("") |
| msg.append("Detected file collision(s):") |
| msg.append("") |
| |
| for f in collisions: |
| msg.append("\t%s" % \ |
| os.path.join(destroot, f.lstrip(os.path.sep))) |
| |
| eerror(msg) |
| |
| msg = [] |
| msg.append("") |
| msg.append("Searching all installed" + \ |
| " packages for file collisions...") |
| msg.append("") |
| msg.append("Press Ctrl-C to Stop") |
| msg.append("") |
| eerror(msg) |
| |
| owners = self.vartree.dbapi._owners.get_owners(collisions) |
| self.vartree.dbapi.flush_cache() |
| |
| for pkg, owned_files in owners.iteritems(): |
| cpv = pkg.mycpv |
| msg = [] |
| msg.append("%s" % cpv) |
| for f in sorted(owned_files): |
| msg.append("\t%s" % os.path.join(destroot, |
| f.lstrip(os.path.sep))) |
| msg.append("") |
| eerror(msg) |
| |
| if not owners: |
| eerror(["None of the installed" + \ |
| " packages claim the file(s).", ""]) |
| |
| # The explanation about the collision and how to solve |
| # it may not be visible via a scrollback buffer, especially |
| # if the number of file collisions is large. Therefore, |
| # show a summary at the end. |
| if collision_protect: |
| msg = "Package '%s' NOT merged due to file collisions." % \ |
| self.settings.mycpv |
| elif protect_owned and owners: |
| msg = "Package '%s' NOT merged due to file collisions." % \ |
| self.settings.mycpv |
| else: |
| msg = "Package '%s' merged despite file collisions." % \ |
| self.settings.mycpv |
| msg += " If necessary, refer to your elog " + \ |
| "messages for the whole content of the above message." |
| eerror(wrap(msg, 70)) |
| |
| if collision_protect or (protect_owned and owners): |
| return 1 |
| |
| # The merge process may move files out of the image directory, |
| # which causes invalidation of the .installed flag. |
| try: |
| os.unlink(os.path.join( |
| os.path.dirname(normalize_path(srcroot)), ".installed")) |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| |
| self.dbdir = self.dbtmpdir |
| self.delete() |
| ensure_dirs(self.dbtmpdir) |
| |
| # run preinst script |
| if scheduler is None: |
| showMessage(">>> Merging %s to %s\n" % (self.mycpv, destroot)) |
| a = doebuild(myebuild, "preinst", destroot, self.settings, |
| use_cache=0, tree=self.treetype, mydbapi=mydbapi, |
| vartree=self.vartree) |
| else: |
| a = scheduler.dblinkEbuildPhase( |
| self, mydbapi, myebuild, "preinst") |
| |
| # XXX: Decide how to handle failures here. |
| if a != os.EX_OK: |
| showMessage("!!! FAILED preinst: "+str(a)+"\n", |
| level=logging.ERROR, noiselevel=-1) |
| return a |
| |
| # copy "info" files (like SLOT, CFLAGS, etc.) into the database |
| for x in os.listdir(inforoot): |
| self.copyfile(inforoot+"/"+x) |
| |
| # write local package counter for recording |
| counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) |
| lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w") |
| lcfile.write(str(counter)) |
| lcfile.close() |
| |
| # open CONTENTS file (possibly overwriting old one) for recording |
| outfile = open(os.path.join(self.dbtmpdir, "CONTENTS"),"w") |
| |
| self.updateprotect() |
| |
| #if we have a file containing previously-merged config file md5sums, grab it. |
| conf_mem_file = os.path.join(destroot, CONFIG_MEMORY_FILE) |
| cfgfiledict = grabdict(conf_mem_file) |
| cfgfiledict_orig = cfgfiledict.copy() |
| if "NOCONFMEM" in self.settings: |
| cfgfiledict["IGNORE"]=1 |
| else: |
| cfgfiledict["IGNORE"]=0 |
| |
| # Always behave like --noconfmem is enabled for downgrades |
| # so that people who don't know about this option are less |
| # likely to get confused when doing upgrade/downgrade cycles. |
| pv_split = catpkgsplit(self.mycpv)[1:] |
| for other in others_in_slot: |
| if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0: |
| cfgfiledict["IGNORE"] = 1 |
| break |
| |
| # Don't bump mtimes on merge since some application require |
| # preservation of timestamps. This means that the unmerge phase must |
| # check to see if file belongs to an installed instance in the same |
| # slot. |
| mymtime = None |
| |
| # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change) |
| prevmask = os.umask(0) |
| secondhand = [] |
| |
| # we do a first merge; this will recurse through all files in our srcroot but also build up a |
| # "second hand" of symlinks to merge later |
| if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime): |
| return 1 |
| |
| # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are |
| # broken symlinks. We'll merge them too. |
| lastlen = 0 |
| while len(secondhand) and len(secondhand)!=lastlen: |
| # clear the thirdhand. Anything from our second hand that |
| # couldn't get merged will be added to thirdhand. |
| |
| thirdhand = [] |
| if self.mergeme(srcroot, destroot, outfile, thirdhand, |
| secondhand, cfgfiledict, mymtime): |
| return 1 |
| |
| #swap hands |
| lastlen = len(secondhand) |
| |
| # our thirdhand now becomes our secondhand. It's ok to throw |
| # away secondhand since thirdhand contains all the stuff that |
| # couldn't be merged. |
| secondhand = thirdhand |
| |
| if len(secondhand): |
| # force merge of remaining symlinks (broken or circular; oh well) |
| if self.mergeme(srcroot, destroot, outfile, None, |
| secondhand, cfgfiledict, mymtime): |
| return 1 |
| self._md5_merge_map.clear() |
| |
| #restore umask |
| os.umask(prevmask) |
| |
| #if we opened it, close it |
| outfile.flush() |
| outfile.close() |
| |
| # write out our collection of md5sums |
| cfgfiledict.pop("IGNORE", None) |
| if cfgfiledict != cfgfiledict_orig: |
| ensure_dirs(os.path.dirname(conf_mem_file), |
| gid=portage_gid, mode=02750, mask=02) |
| writedict(cfgfiledict, conf_mem_file) |
| |
| # These caches are populated during collision-protect and the data |
| # they contain is now invalid. It's very important to invalidate |
| # the contents_inodes cache so that FEATURES=unmerge-orphans |
| # doesn't unmerge anything that belongs to this package that has |
| # just been merged. |
| for dblnk in others_in_slot: |
| dblnk._clear_contents_cache() |
| self._clear_contents_cache() |
| |
| linkmap = self.vartree.dbapi.linkmap |
| self._linkmap_rebuild(include_file=os.path.join(inforoot, |
| linkmap._needed_aux_key)) |
| |
| # Preserve old libs if they are still in use |
| preserve_paths = self._find_libs_to_preserve() |
| if preserve_paths: |
| self._add_preserve_libs_to_contents(preserve_paths) |
| |
| # If portage is reinstalling itself, remove the old |
| # version now since we want to use the temporary |
| # PORTAGE_BIN_PATH that will be removed when we return. |
| reinstall_self = False |
| if self.myroot == "/" and \ |
| match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]): |
| reinstall_self = True |
| |
| if scheduler is None: |
| def emerge_log(msg): |
| pass |
| else: |
| emerge_log = scheduler.dblinkEmergeLog |
| |
| autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" |
| |
| if autoclean: |
| emerge_log(" >>> AUTOCLEAN: %s" % (slot_atom,)) |
| |
| others_in_slot.append(self) # self has just been merged |
| for dblnk in list(others_in_slot): |
| if dblnk is self: |
| continue |
| if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self): |
| continue |
| showMessage(">>> Safely unmerging already-installed instance...\n") |
| emerge_log(" === Unmerging... (%s)" % (dblnk.mycpv,)) |
| others_in_slot.remove(dblnk) # dblnk will unmerge itself now |
| dblnk._linkmap_broken = self._linkmap_broken |
| unmerge_rval = dblnk.unmerge(trimworld=0, |
| ldpath_mtimes=prev_mtimes, others_in_slot=others_in_slot) |
| |
| if unmerge_rval == os.EX_OK: |
| emerge_log(" >>> unmerge success: %s" % (dblnk.mycpv,)) |
| else: |
| emerge_log(" !!! unmerge FAILURE: %s" % (dblnk.mycpv,)) |
| |
| # TODO: Check status and abort if necessary. |
| dblnk.delete() |
| showMessage(">>> Original instance of package unmerged safely.\n") |
| |
| if len(others_in_slot) > 1: |
| showMessage(colorize("WARN", "WARNING:") |
| + " AUTOCLEAN is disabled. This can cause serious" |
| + " problems due to overlapping packages.\n", |
| level=logging.WARN, noiselevel=-1) |
| |
| # We hold both directory locks. |
| self.dbdir = self.dbpkgdir |
| self.delete() |
| _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) |
| |
| # keep track of the libs we preserved |
| if preserve_paths: |
| self.vartree.dbapi.plib_registry.register(self.mycpv, |
| slot, counter, sorted(preserve_paths)) |
| |
| # Check for file collisions with blocking packages |
| # and remove any colliding files from their CONTENTS |
| # since they now belong to this package. |
| self._clear_contents_cache() |
| contents = self.getcontents() |
| destroot_len = len(destroot) - 1 |
| for blocker in blockers: |
| self.vartree.dbapi.removeFromContents(blocker, iter(contents), |
| relative_paths=False) |
| |
| # Unregister any preserved libs that this package has overwritten |
| # and update the contents of the packages that owned them. |
| plib_registry = self.vartree.dbapi.plib_registry |
| plib_dict = plib_registry.getPreservedLibs() |
| for cpv, paths in plib_collisions.iteritems(): |
| if cpv not in plib_dict: |
| continue |
| if cpv == self.mycpv: |
| continue |
| try: |
| slot, counter = self.vartree.dbapi.aux_get( |
| cpv, ["SLOT", "COUNTER"]) |
| except KeyError: |
| continue |
| remaining = [f for f in plib_dict[cpv] if f not in paths] |
| plib_registry.register(cpv, slot, counter, remaining) |
| self.vartree.dbapi.removeFromContents(cpv, paths) |
| |
| self.vartree.dbapi._add(self) |
| contents = self.getcontents() |
| |
| #do postinst script |
| self.settings["PORTAGE_UPDATE_ENV"] = \ |
| os.path.join(self.dbpkgdir, "environment.bz2") |
| self.settings.backup_changes("PORTAGE_UPDATE_ENV") |
| try: |
| if scheduler is None: |
| a = doebuild(myebuild, "postinst", destroot, self.settings, |
| use_cache=0, tree=self.treetype, mydbapi=mydbapi, |
| vartree=self.vartree) |
| if a == os.EX_OK: |
| showMessage(">>> %s %s\n" % (self.mycpv, "merged.")) |
| else: |
| a = scheduler.dblinkEbuildPhase( |
| self, mydbapi, myebuild, "postinst") |
| finally: |
| self.settings.pop("PORTAGE_UPDATE_ENV", None) |
| |
| # XXX: Decide how to handle failures here. |
| if a != os.EX_OK: |
| showMessage("!!! FAILED postinst: "+str(a)+"\n", |
| level=logging.ERROR, noiselevel=-1) |
| return a |
| |
| downgrade = False |
| for v in otherversions: |
| if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0: |
| downgrade = True |
| |
| #update environment settings, library paths. DO NOT change symlinks. |
| env_update(makelinks=(not downgrade), |
| target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, |
| contents=contents, env=self.settings.environ(), |
| writemsg_level=self._display_merge) |
| |
| # For gcc upgrades, preserved libs have to be removed after the |
| # the library path has been updated. |
| self._linkmap_rebuild() |
| cpv_lib_map = self._find_unused_preserved_libs() |
| if cpv_lib_map: |
| self._remove_preserved_libs(cpv_lib_map) |
| for cpv, removed in cpv_lib_map.iteritems(): |
| if not self.vartree.dbapi.cpv_exists(cpv): |
| continue |
| self.vartree.dbapi.removeFromContents(cpv, removed) |
| |
| return os.EX_OK |
| |
| def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): |
| """ |
| |
| This function handles actual merging of the package contents to the livefs. |
| It also handles config protection. |
| |
| @param srcroot: Where are we copying files from (usually ${D}) |
| @type srcroot: String (Path) |
| @param destroot: Typically ${ROOT} |
| @type destroot: String (Path) |
| @param outfile: File to log operations to |
| @type outfile: File Object |
| @param secondhand: A set of items to merge in pass two (usually |
| or symlinks that point to non-existing files that may get merged later) |
| @type secondhand: List |
| @param stufftomerge: Either a diretory to merge, or a list of items. |
| @type stufftomerge: String or List |
| @param cfgfiledict: { File:mtime } mapping for config_protected files |
| @type cfgfiledict: Dictionary |
| @param thismtime: The current time (typically long(time.time()) |
| @type thismtime: Long |
| @rtype: None or Boolean |
| @returns: |
| 1. True on failure |
| 2. None otherwise |
| |
| """ |
| |
| showMessage = self._display_merge |
| writemsg = self._display_merge |
| scheduler = self._scheduler |
| |
| from os.path import sep, join |
| srcroot = normalize_path(srcroot).rstrip(sep) + sep |
| destroot = normalize_path(destroot).rstrip(sep) + sep |
| |
| # this is supposed to merge a list of files. There will be 2 forms of argument passing. |
| if isinstance(stufftomerge, basestring): |
| #A directory is specified. Figure out protection paths, listdir() it and process it. |
| mergelist = os.listdir(join(srcroot, stufftomerge)) |
| offset = stufftomerge |
| else: |
| mergelist = stufftomerge |
| offset = "" |
| |
| for i, x in enumerate(mergelist): |
| |
| if scheduler is not None and \ |
| 0 == i % self._file_merge_yield_interval: |
| scheduler.scheduleYield() |
| |
| mysrc = join(srcroot, offset, x) |
| mydest = join(destroot, offset, x) |
| # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") |
| myrealdest = join(sep, offset, x) |
| # stat file once, test using S_* macros many times (faster that way) |
| mystat = os.lstat(mysrc) |
| mymode = mystat[stat.ST_MODE] |
| # handy variables; mydest is the target object on the live filesystems; |
| # mysrc is the source object in the temporary install dir |
| try: |
| mydstat = os.lstat(mydest) |
| mydmode = mydstat.st_mode |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| #dest file doesn't exist |
| mydstat = None |
| mydmode = None |
| |
| if stat.S_ISLNK(mymode): |
| # we are merging a symbolic link |
| myabsto = abssymlink(mysrc) |
| if myabsto.startswith(srcroot): |
| myabsto = myabsto[len(srcroot):] |
| myabsto = myabsto.lstrip(sep) |
| myto = os.readlink(mysrc) |
| if self.settings and self.settings["D"]: |
| if myto.startswith(self.settings["D"]): |
| myto = myto[len(self.settings["D"]):] |
| # myrealto contains the path of the real file to which this symlink points. |
| # we can simply test for existence of this file to see if the target has been merged yet |
| myrealto = normalize_path(os.path.join(destroot, myabsto)) |
| if mydmode!=None: |
| #destination exists |
| if not stat.S_ISLNK(mydmode): |
| if stat.S_ISDIR(mydmode): |
| # directory in the way: we can't merge a symlink over a directory |
| # we won't merge this, continue with next file... |
| continue |
| |
| if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): |
| # Kill file blocking installation of symlink to dir #71787 |
| pass |
| elif self.isprotected(mydest): |
| # Use md5 of the target in ${D} if it exists... |
| try: |
| newmd5 = perform_md5(join(srcroot, myabsto)) |
| except FileNotFound: |
| # Maybe the target is merged already. |
| try: |
| newmd5 = perform_md5(myrealto) |
| except FileNotFound: |
| newmd5 = None |
| mydest = new_protect_filename(mydest, newmd5=newmd5) |
| |
| # if secondhand is None it means we're operating in "force" mode and should not create a second hand. |
| if (secondhand != None) and (not os.path.exists(myrealto)): |
| # either the target directory doesn't exist yet or the target file doesn't exist -- or |
| # the target is a broken symlink. We will add this file to our "second hand" and merge |
| # it later. |
| secondhand.append(mysrc[len(srcroot):]) |
| continue |
| # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly |
| mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) |
| if mymtime != None: |
| showMessage(">>> %s -> %s\n" % (mydest, myto)) |
| outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") |
| else: |
| showMessage("!!! Failed to move file.\n", |
| level=logging.ERROR, noiselevel=-1) |
| showMessage("!!! %s -> %s\n" % (mydest, myto), |
| level=logging.ERROR, noiselevel=-1) |
| return 1 |
| elif stat.S_ISDIR(mymode): |
| # we are merging a directory |
| if mydmode != None: |
| # destination exists |
| |
| if bsd_chflags: |
| # Save then clear flags on dest. |
| dflags = mydstat.st_flags |
| if dflags != 0: |
| bsd_chflags.lchflags(mydest, 0) |
| |
| if not os.access(mydest, os.W_OK): |
| pkgstuff = pkgsplit(self.pkg) |
| writemsg("\n!!! Cannot write to '"+mydest+"'.\n", noiselevel=-1) |
| writemsg("!!! Please check permissions and directories for broken symlinks.\n") |
| writemsg("!!! You may start the merge process again by using ebuild:\n") |
| writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n") |
| writemsg("!!! And finish by running this: env-update\n\n") |
| return 1 |
| |
| if stat.S_ISDIR(mydmode) or \ |
| (stat.S_ISLNK(mydmode) and os.path.isdir(mydest)): |
| # a symlink to an existing directory will work for us; keep it: |
| showMessage("--- %s/\n" % mydest) |
| if bsd_chflags: |
| bsd_chflags.lchflags(mydest, dflags) |
| else: |
| # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. |
| if movefile(mydest, mydest+".backup", mysettings=self.settings) is None: |
| return 1 |
| showMessage("bak %s %s.backup\n" % (mydest, mydest), |
| level=logging.ERROR, noiselevel=-1) |
| #now create our directory |
| if self.settings.selinux_enabled(): |
| import selinux |
| sid = selinux.get_sid(mysrc) |
| selinux.secure_mkdir(mydest,sid) |
| else: |
| os.mkdir(mydest) |
| if bsd_chflags: |
| bsd_chflags.lchflags(mydest, dflags) |
| os.chmod(mydest, mystat[0]) |
| os.chown(mydest, mystat[4], mystat[5]) |
| showMessage(">>> %s/\n" % mydest) |
| else: |
| #destination doesn't exist |
| if self.settings.selinux_enabled(): |
| import selinux |
| sid = selinux.get_sid(mysrc) |
| selinux.secure_mkdir(mydest, sid) |
| else: |
| os.mkdir(mydest) |
| os.chmod(mydest, mystat[0]) |
| os.chown(mydest, mystat[4], mystat[5]) |
| showMessage(">>> %s/\n" % mydest) |
| outfile.write("dir "+myrealdest+"\n") |
| # recurse and merge this directory |
| if self.mergeme(srcroot, destroot, outfile, secondhand, |
| join(offset, x), cfgfiledict, thismtime): |
| return 1 |
| elif stat.S_ISREG(mymode): |
| # we are merging a regular file |
| mymd5 = perform_md5(mysrc, calc_prelink=1) |
| # calculate config file protection stuff |
| mydestdir = os.path.dirname(mydest) |
| moveme = 1 |
| zing = "!!!" |
| mymtime = None |
| protected = self.isprotected(mydest) |
| if mydmode != None: |
| # destination file exists |
| if stat.S_ISDIR(mydmode): |
| # install of destination is blocked by an existing directory with the same name |
| moveme = 0 |
| showMessage("!!! %s\n" % mydest, |
| level=logging.ERROR, noiselevel=-1) |
| elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): |
| cfgprot = 0 |
| # install of destination is blocked by an existing regular file, |
| # or by a symlink to an existing regular file; |
| # now, config file management may come into play. |
| # we only need to tweak mydest if cfg file management is in play. |
| if protected: |
| # we have a protection path; enable config file management. |
| destmd5 = perform_md5(mydest, calc_prelink=1) |
| if mymd5 == destmd5: |
| #file already in place; simply update mtimes of destination |
| moveme = 1 |
| else: |
| if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: |
| """ An identical update has previously been |
| merged. Skip it unless the user has chosen |
| --noconfmem.""" |
| moveme = cfgfiledict["IGNORE"] |
| cfgprot = cfgfiledict["IGNORE"] |
| if not moveme: |
| zing = "---" |
| mymtime = long(mystat.st_mtime) |
| else: |
| moveme = 1 |
| cfgprot = 1 |
| if moveme: |
| # Merging a new file, so update confmem. |
| cfgfiledict[myrealdest] = [mymd5] |
| elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: |
| """A previously remembered update has been |
| accepted, so it is removed from confmem.""" |
| del cfgfiledict[myrealdest] |
| if cfgprot: |
| mydest = new_protect_filename(mydest, newmd5=mymd5) |
| |
| # whether config protection or not, we merge the new file the |
| # same way. Unless moveme=0 (blocking directory) |
| if moveme: |
| # Do not hardlink files unless they are in the same |
| # directory, since otherwise tar may not be able to |
| # extract a tarball of the resulting hardlinks due to |
| # 'Invalid cross-device link' errors (depends on layout of |
| # mount points). Also, don't hardlink zero-byte files since |
| # it doesn't save any space, and don't hardlink |
| # CONFIG_PROTECTed files since config files shouldn't be |
| # hardlinked to eachother (for example, shadow installs |
| # several identical config files inside /etc/pam.d/). |
| parent_dir = os.path.dirname(myrealdest) |
| hardlink_key = (parent_dir, mymd5, mystat.st_size, |
| mystat.st_mode, mystat.st_uid, mystat.st_gid) |
| |
| hardlink_candidates = None |
| if not protected and mystat.st_size != 0: |
| hardlink_candidates = self._md5_merge_map.get(hardlink_key) |
| if hardlink_candidates is None: |
| hardlink_candidates = [] |
| self._md5_merge_map[hardlink_key] = hardlink_candidates |
| |
| mymtime = movefile(mysrc, mydest, newmtime=thismtime, |
| sstat=mystat, mysettings=self.settings, |
| hardlink_candidates=hardlink_candidates) |
| if mymtime is None: |
| return 1 |
| if hardlink_candidates is not None: |
| hardlink_candidates.append(mydest) |
| zing = ">>>" |
| |
| if mymtime != None: |
| outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") |
| showMessage("%s %s\n" % (zing,mydest)) |
| else: |
| # we are merging a fifo or device node |
| zing = "!!!" |
| if mydmode is None: |
| # destination doesn't exist |
| if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None: |
| zing = ">>>" |
| else: |
| return 1 |
| if stat.S_ISFIFO(mymode): |
| outfile.write("fif %s\n" % myrealdest) |
| else: |
| outfile.write("dev %s\n" % myrealdest) |
| showMessage(zing + " " + mydest + "\n") |
| |
| def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, |
| mydbapi=None, prev_mtimes=None): |
| """ |
| If portage is reinstalling itself, create temporary |
| copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order |
| to avoid relying on the new versions which may be |
| incompatible. Register an atexit hook to clean up the |
| temporary directories. Pre-load elog modules here since |
| we won't be able to later if they get unmerged (happens |
| when namespace changes). |
| """ |
| if self.vartree.dbapi._categories is not None: |
| self.vartree.dbapi._categories = None |
| if self.myroot == "/" and \ |
| match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]) and \ |
| not self.vartree.dbapi.cpv_exists(self.mycpv): |
| settings = self.settings |
| base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"]) |
| from tempfile import mkdtemp |
| |
| # Make the temp directory inside PORTAGE_TMPDIR since, unlike |
| # /tmp, it can't be mounted with the "noexec" option. |
| base_path_tmp = mkdtemp("", "._portage_reinstall_.", |
| settings["PORTAGE_TMPDIR"]) |
| from portage.process import atexit_register |
| atexit_register(shutil.rmtree, base_path_tmp) |
| dir_perms = 0755 |
| for subdir in "bin", "pym": |
| var_name = "PORTAGE_%s_PATH" % subdir.upper() |
| var_orig = settings[var_name] |
| var_new = os.path.join(base_path_tmp, subdir) |
| settings[var_name] = var_new |
| settings.backup_changes(var_name) |
| shutil.copytree(var_orig, var_new, symlinks=True) |
| os.chmod(var_new, dir_perms) |
| os.chmod(base_path_tmp, dir_perms) |
| # This serves so pre-load the modules. |
| elog_process(self.mycpv, self.settings, |
| phasefilter=filter_mergephases) |
| |
| return self._merge(mergeroot, inforoot, |
| myroot, myebuild=myebuild, cleanup=cleanup, |
| mydbapi=mydbapi, prev_mtimes=prev_mtimes) |
| |
| def _merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, |
| mydbapi=None, prev_mtimes=None): |
| retval = -1 |
| self.lockdb() |
| try: |
| self.vartree.dbapi.plib_registry.load() |
| self.vartree.dbapi.plib_registry.pruneNonExisting() |
| retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, |
| cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) |
| |
| # Process ebuild logfiles |
| elog_process(self.mycpv, self.settings, phasefilter=filter_mergephases) |
| if retval == os.EX_OK and "noclean" not in self.settings.features: |
| if myebuild is None: |
| myebuild = os.path.join(inforoot, self.pkg + ".ebuild") |
| |
| if self._scheduler is None: |
| doebuild(myebuild, "clean", myroot, |
| self.settings, tree=self.treetype, |
| mydbapi=mydbapi, vartree=self.vartree) |
| else: |
| self._scheduler.dblinkEbuildPhase( |
| self, mydbapi, myebuild, "clean") |
| finally: |
| self.vartree.dbapi.linkmap._clear_cache() |
| self.unlockdb() |
| return retval |
| |
| def getstring(self,name): |
| "returns contents of a file with whitespace converted to spaces" |
| if not os.path.exists(self.dbdir+"/"+name): |
| return "" |
| myfile = open(self.dbdir+"/"+name,"r") |
| mydata = myfile.read().split() |
| myfile.close() |
| return " ".join(mydata) |
| |
| def copyfile(self,fname): |
| shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) |
| |
| def getfile(self,fname): |
| if not os.path.exists(self.dbdir+"/"+fname): |
| return "" |
| myfile = open(self.dbdir+"/"+fname,"r") |
| mydata = myfile.read() |
| myfile.close() |
| return mydata |
| |
| def setfile(self,fname,data): |
| write_atomic(os.path.join(self.dbdir, fname), data) |
| |
| def getelements(self,ename): |
| if not os.path.exists(self.dbdir+"/"+ename): |
| return [] |
| myelement = open(self.dbdir+"/"+ename,"r") |
| mylines = myelement.readlines() |
| myreturn = [] |
| for x in mylines: |
| for y in x[:-1].split(): |
| myreturn.append(y) |
| myelement.close() |
| return myreturn |
| |
| def setelements(self,mylist,ename): |
| myelement = open(self.dbdir+"/"+ename,"w") |
| for x in mylist: |
| myelement.write(x+"\n") |
| myelement.close() |
| |
| def isregular(self): |
| "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" |
| return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) |
| |
| def write_contents(contents, root, f): |
| """ |
| Write contents to any file like object. The file will be left open. |
| """ |
| root_len = len(root) - 1 |
| for filename in sorted(contents): |
| entry_data = contents[filename] |
| entry_type = entry_data[0] |
| relative_filename = filename[root_len:] |
| if entry_type == "obj": |
| entry_type, mtime, md5sum = entry_data |
| line = "%s %s %s %s\n" % \ |
| (entry_type, relative_filename, md5sum, mtime) |
| elif entry_type == "sym": |
| entry_type, mtime, link = entry_data |
| line = "%s %s -> %s %s\n" % \ |
| (entry_type, relative_filename, link, mtime) |
| else: # dir, dev, fif |
| line = "%s %s\n" % (entry_type, relative_filename) |
| f.write(line) |
| |
| def tar_contents(contents, root, tar, protect=None, onProgress=None): |
| from portage.util import normalize_path |
| import tarfile |
| root = normalize_path(root).rstrip(os.path.sep) + os.path.sep |
| id_strings = {} |
| maxval = len(contents) |
| curval = 0 |
| if onProgress: |
| onProgress(maxval, 0) |
| paths = contents.keys() |
| paths.sort() |
| for path in paths: |
| curval += 1 |
| try: |
| lst = os.lstat(path) |
| except OSError, e: |
| if e.errno != errno.ENOENT: |
| raise |
| del e |
| if onProgress: |
| onProgress(maxval, curval) |
| continue |
| contents_type = contents[path][0] |
| if path.startswith(root): |
| arcname = path[len(root):] |
| else: |
| raise ValueError("invalid root argument: '%s'" % root) |
| live_path = path |
| if 'dir' == contents_type and \ |
| not stat.S_ISDIR(lst.st_mode) and \ |
| os.path.isdir(live_path): |
| # Even though this was a directory in the original ${D}, it exists |
| # as a symlink to a directory in the live filesystem. It must be |
| # recorded as a real directory in the tar file to ensure that tar |
| # can properly extract it's children. |
| live_path = os.path.realpath(live_path) |
| tarinfo = tar.gettarinfo(live_path, arcname) |
| |
| if stat.S_ISREG(lst.st_mode): |
| # break hardlinks due to bug #185305 |
| tarinfo.type = tarfile.REGTYPE |
| if protect and protect(path): |
| # Create an empty file as a place holder in order to avoid |
| # potential collision-protect issues. |
| tarinfo.size = 0 |
| tar.addfile(tarinfo) |
| else: |
| f = open(path) |
| try: |
| tar.addfile(tarinfo, f) |
| finally: |
| f.close() |
| else: |
| tar.addfile(tarinfo) |
| if onProgress: |
| onProgress(maxval, curval) |