| #!/usr/bin/python -b |
| # Copyright 1999-2021 Gentoo Authors |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| import argparse |
| import errno |
| import math |
| import signal |
| import subprocess |
| import sys |
| |
| from os import path as osp |
| if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): |
| sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "lib")) |
| import portage |
| portage._internal_caller = True |
| from portage import os |
| from portage import xpak |
| from portage.dbapi.dep_expand import dep_expand |
| from portage.dep import Atom, use_reduce |
| from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData, |
| InvalidDependString, PackageSetNotFound, PermissionDenied) |
| from portage.util import ensure_dirs, shlex_split, varexpand, _xattr |
| xattr = _xattr.xattr |
| from portage._sets import load_default_config, SETPREFIX |
| from portage.process import find_binary |
| from portage.util.compression_probe import _compressors |
| from portage.util._eventloop.global_event_loop import global_event_loop |
| |
| |
| def quickpkg_atom(options, infos, arg, eout): |
| settings = portage.settings |
| eroot = portage.settings['EROOT'] |
| trees = portage.db[eroot] |
| vartree = trees["vartree"] |
| vardb = vartree.dbapi |
| bintree = trees["bintree"] |
| |
| include_config = options.include_config == "y" |
| include_unmodified_config = options.include_unmodified_config == "y" |
| fix_metadata_keys = ["PF", "CATEGORY"] |
| |
| try: |
| atom = dep_expand(arg, mydb=vardb, settings=vartree.settings) |
| except AmbiguousPackageName as e: |
| # Multiple matches thrown from cpv_expand |
| eout.eerror("Please use a more specific atom: %s" % \ |
| " ".join(e.args[0])) |
| del e |
| infos["missing"].append(arg) |
| return 1 |
| except (InvalidAtom, InvalidData): |
| eout.eerror("Invalid atom: %s" % (arg,)) |
| infos["missing"].append(arg) |
| return 1 |
| if atom[:1] == '=' and arg[:1] != '=': |
| # dep_expand() allows missing '=' but it's really invalid |
| eout.eerror("Invalid atom: %s" % (arg,)) |
| infos["missing"].append(arg) |
| return 1 |
| |
| matches = vardb.match(atom) |
| pkgs_for_arg = 0 |
| retval = 0 |
| for cpv in matches: |
| excluded_config_files = [] |
| dblnk = vardb._dblink(cpv) |
| have_lock = False |
| |
| if "__PORTAGE_INHERIT_VARDB_LOCK" not in settings: |
| try: |
| dblnk.lockdb() |
| have_lock = True |
| except PermissionDenied: |
| pass |
| |
| try: |
| if not dblnk.exists(): |
| # unmerged by a concurrent process |
| continue |
| iuse, use, restrict = vardb.aux_get(cpv, |
| ["IUSE","USE","RESTRICT"]) |
| iuse = [ x.lstrip("+-") for x in iuse.split() ] |
| use = use.split() |
| try: |
| restrict = use_reduce(restrict, uselist=use, flat=True) |
| except InvalidDependString as e: |
| eout.eerror("Invalid RESTRICT metadata " + \ |
| "for '%s': %s; skipping" % (cpv, str(e))) |
| del e |
| continue |
| if "bindist" in iuse and "bindist" not in use: |
| eout.ewarn("%s: package was emerged with USE=-bindist!" % cpv) |
| eout.ewarn("%s: it might not be legal to redistribute this." % cpv) |
| elif "bindist" in restrict: |
| eout.ewarn("%s: package has RESTRICT=bindist!" % cpv) |
| eout.ewarn("%s: it might not be legal to redistribute this." % cpv) |
| eout.ebegin("Building package for %s" % cpv) |
| pkgs_for_arg += 1 |
| existing_metadata = dict(zip(fix_metadata_keys, |
| vardb.aux_get(cpv, fix_metadata_keys))) |
| category, pf = portage.catsplit(cpv) |
| required_metadata = {} |
| required_metadata["CATEGORY"] = category |
| required_metadata["PF"] = pf |
| update_metadata = {} |
| for k, v in required_metadata.items(): |
| if v != existing_metadata[k]: |
| update_metadata[k] = v |
| if update_metadata: |
| vardb.aux_update(cpv, update_metadata) |
| xpdata = xpak.xpak(dblnk.dbdir) |
| binpkg_tmpfile = os.path.join(bintree.pkgdir, |
| cpv + ".tbz2." + str(portage.getpid())) |
| ensure_dirs(os.path.dirname(binpkg_tmpfile)) |
| binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2") |
| try: |
| compression = _compressors[binpkg_compression] |
| except KeyError as e: |
| if binpkg_compression: |
| eout.eerror("Invalid or unsupported compression method: %s" % e.args[0]) |
| return 1 |
| # Empty BINPKG_COMPRESS disables compression. |
| binpkg_compression = 'none' |
| compression = { |
| 'compress': 'cat', |
| 'package': 'sys-apps/coreutils', |
| } |
| try: |
| compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0] |
| except IndexError as e: |
| eout.eerror("Invalid or unsupported compression method: %s" % e.args[0]) |
| return 1 |
| if find_binary(compression_binary) is None: |
| missing_package = compression["package"] |
| eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package)) |
| return 1 |
| cmd = shlex_split(varexpand(compression["compress"], mydict=settings)) |
| # Filter empty elements that make Popen fail |
| cmd = [x for x in cmd if x != ""] |
| with open(binpkg_tmpfile, "wb") as fobj: |
| proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj) |
| excluded_config_files = dblnk.quickpkg(proc.stdin, |
| include_config=include_config, |
| include_unmodified_config=include_unmodified_config) |
| proc.stdin.close() |
| if proc.wait() != os.EX_OK: |
| eout.eend(1) |
| eout.eerror("Compressor failed for package %s" % cpv) |
| retval |= 1 |
| try: |
| os.unlink(binpkg_tmpfile) |
| except OSError as e: |
| if e.errno not in (errno.ENOENT, errno.ESTALE): |
| raise |
| continue |
| xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata) |
| finally: |
| if have_lock: |
| dblnk.unlockdb() |
| pkg_info = bintree.inject(cpv, filename=binpkg_tmpfile) |
| # The pkg_info value ensures that the following getname call |
| # returns the correct path when FEATURES=binpkg-multi-instance |
| # is enabled, but fallback to cpv in case the inject call |
| # returned None due to some kind of failure. |
| binpkg_path = bintree.getname(pkg_info or cpv) |
| try: |
| s = os.stat(binpkg_path) |
| except OSError: |
| s = None |
| |
| if s is None or pkg_info is None: |
| # Sanity check, shouldn't happen normally. |
| eout.eend(1) |
| eout.eerror("Failed to create package: '%s'" % binpkg_path) |
| retval |= 1 |
| else: |
| eout.eend(0) |
| infos["successes"].append((cpv, s.st_size)) |
| infos["config_files_excluded"] += len(excluded_config_files) |
| for filename in excluded_config_files: |
| eout.ewarn("Excluded config: '%s'" % filename) |
| if not pkgs_for_arg: |
| eout.eerror("Could not find anything " + \ |
| "to match '%s'; skipping" % arg) |
| infos["missing"].append(arg) |
| retval |= 1 |
| return retval |
| |
| def quickpkg_set(options, infos, arg, eout): |
| eroot = portage.settings['EROOT'] |
| trees = portage.db[eroot] |
| vartree = trees["vartree"] |
| |
| settings = vartree.settings |
| settings._init_dirs() |
| setconfig = load_default_config(settings, trees) |
| sets = setconfig.getSets() |
| |
| set_name = arg[1:] |
| if not set_name in sets: |
| eout.eerror("Package set not found: '%s'; skipping" % (arg,)) |
| infos["missing"].append(arg) |
| return 1 |
| |
| try: |
| atoms = setconfig.getSetAtoms(set_name) |
| except PackageSetNotFound as e: |
| eout.eerror("Failed to process package set '%s' because " % set_name + |
| "it contains the non-existent package set '%s'; skipping" % e) |
| infos["missing"].append(arg) |
| return 1 |
| retval = os.EX_OK |
| for atom in atoms: |
| retval |= quickpkg_atom(options, infos, atom, eout) |
| return retval |
| |
| |
| def quickpkg_extended_atom(options, infos, atom, eout): |
| eroot = portage.settings['EROOT'] |
| trees = portage.db[eroot] |
| vartree = trees["vartree"] |
| vardb = vartree.dbapi |
| |
| require_metadata = atom.slot or atom.repo |
| atoms = [] |
| for cpv in vardb.cpv_all(): |
| cpv_atom = Atom("=%s" % cpv) |
| |
| if atom == "*/*": |
| atoms.append(cpv_atom) |
| continue |
| |
| if not portage.match_from_list(atom, [cpv]): |
| continue |
| |
| if require_metadata: |
| try: |
| cpv = vardb._pkg_str(cpv, atom.repo) |
| except (KeyError, InvalidData): |
| continue |
| if not portage.match_from_list(atom, [cpv]): |
| continue |
| |
| atoms.append(cpv_atom) |
| |
| for atom in atoms: |
| quickpkg_atom(options, infos, atom, eout) |
| |
| |
| def quickpkg_main(options, args, eout): |
| eroot = portage.settings['EROOT'] |
| trees = portage.db[eroot] |
| bintree = trees["bintree"] |
| |
| try: |
| ensure_dirs(bintree.pkgdir) |
| except portage.exception.PortageException: |
| pass |
| if not os.access(bintree.pkgdir, os.W_OK): |
| eout.eerror("No write access to '%s'" % bintree.pkgdir) |
| return errno.EACCES |
| |
| if 'xattr' in portage.settings.features and not _xattr.XATTRS_WORKS: |
| eout.eerror("No xattr support library was found, " |
| "so xattrs will not be preserved!") |
| portage.settings.unlock() |
| portage.settings.features.remove('xattr') |
| portage.settings.lock() |
| |
| infos = {} |
| infos["successes"] = [] |
| infos["missing"] = [] |
| infos["config_files_excluded"] = 0 |
| for arg in args: |
| if arg[0] == SETPREFIX: |
| quickpkg_set(options, infos, arg, eout) |
| continue |
| try: |
| atom = Atom(arg, allow_wildcard=True, allow_repo=True) |
| except (InvalidAtom, InvalidData): |
| # maybe it's valid but missing category (requires dep_expand) |
| quickpkg_atom(options, infos, arg, eout) |
| else: |
| if atom.extended_syntax: |
| quickpkg_extended_atom(options, infos, atom, eout) |
| else: |
| quickpkg_atom(options, infos, atom, eout) |
| |
| if not infos["successes"]: |
| eout.eerror("No packages found") |
| return 1 |
| print() |
| eout.einfo("Packages now in '%s':" % bintree.pkgdir) |
| units = {10:'K', 20:'M', 30:'G', 40:'T', |
| 50:'P', 60:'E', 70:'Z', 80:'Y'} |
| for cpv, size in infos["successes"]: |
| if not size: |
| # avoid OverflowError in math.log() |
| size_str = "0" |
| else: |
| power_of_2 = math.log(size, 2) |
| power_of_2 = 10*(power_of_2//10) |
| unit = units.get(power_of_2) |
| if unit: |
| size = float(size)/(2**power_of_2) |
| size_str = "%.1f" % size |
| if len(size_str) > 4: |
| # emulate `du -h`, don't show too many sig figs |
| size_str = str(int(size)) |
| size_str += unit |
| else: |
| size_str = str(size) |
| eout.einfo("%s: %s" % (cpv, size_str)) |
| if infos["config_files_excluded"]: |
| print() |
| eout.ewarn("Excluded config files: %d" % infos["config_files_excluded"]) |
| eout.ewarn("See --help if you would like to include config files.") |
| if infos["missing"]: |
| print() |
| eout.ewarn("The following packages could not be found:") |
| eout.ewarn(" ".join(infos["missing"])) |
| return 2 |
| return os.EX_OK |
| |
| if __name__ == "__main__": |
| usage = "quickpkg [options] <list of package atoms or package sets>" |
| parser = argparse.ArgumentParser(usage=usage) |
| parser.add_argument("--umask", |
| default="0077", |
| help="umask used during package creation (default is 0077)") |
| parser.add_argument("--ignore-default-opts", |
| action="store_true", |
| help="do not use the QUICKPKG_DEFAULT_OPTS environment variable") |
| parser.add_argument("--include-config", |
| choices=["y","n"], |
| default="n", |
| metavar="<y|n>", |
| help="include all files protected by CONFIG_PROTECT (as a security precaution, default is 'n')") |
| parser.add_argument("--include-unmodified-config", |
| choices=["y","n"], |
| default="n", |
| metavar="<y|n>", |
| help="include files protected by CONFIG_PROTECT that have not been modified since installation (as a security precaution, default is 'n')") |
| options, args = parser.parse_known_args(sys.argv[1:]) |
| if not options.ignore_default_opts: |
| default_opts = shlex_split( |
| portage.settings.get("QUICKPKG_DEFAULT_OPTS", "")) |
| options, args = parser.parse_known_args(default_opts + sys.argv[1:]) |
| if not args: |
| parser.error("no packages atoms given") |
| try: |
| umask = int(options.umask, 8) |
| except ValueError: |
| parser.error("invalid umask: %s" % options.umask) |
| # We need to ensure a sane umask for the packages that will be created. |
| old_umask = os.umask(umask) |
| eout = portage.output.EOutput() |
| def sigwinch_handler(signum, frame): |
| lines, eout.term_columns = portage.output.get_term_size() |
| signal.signal(signal.SIGWINCH, sigwinch_handler) |
| try: |
| retval = quickpkg_main(options, args, eout) |
| finally: |
| os.umask(old_umask) |
| signal.signal(signal.SIGWINCH, signal.SIG_DFL) |
| global_event_loop().close() |
| sys.exit(retval) |