| # Copyright 1999-2021 Gentoo Authors |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| import functools |
| |
| import _emerge.emergelog |
| from _emerge.EbuildPhase import EbuildPhase |
| from _emerge.BinpkgFetcher import BinpkgFetcher |
| from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor |
| from _emerge.CompositeTask import CompositeTask |
| from _emerge.BinpkgVerifier import BinpkgVerifier |
| from _emerge.EbuildMerge import EbuildMerge |
| from _emerge.EbuildBuildDir import EbuildBuildDir |
| from _emerge.SpawnProcess import SpawnProcess |
| from portage.eapi import eapi_exports_replace_vars |
| from portage.util import ensure_dirs |
| from portage.util._async.AsyncTaskFuture import AsyncTaskFuture |
| import portage |
| from portage import os |
| from portage import shutil |
| from portage import _encodings |
| from portage import _unicode_decode |
| from portage import _unicode_encode |
| import io |
| import logging |
| |
| |
| class Binpkg(CompositeTask): |
| |
| __slots__ = ( |
| "find_blockers", |
| "ldpath_mtimes", |
| "logger", |
| "opts", |
| "pkg", |
| "pkg_count", |
| "prefetcher", |
| "settings", |
| "world_atom", |
| ) + ( |
| "_bintree", |
| "_build_dir", |
| "_build_prefix", |
| "_ebuild_path", |
| "_fetched_pkg", |
| "_image_dir", |
| "_infloc", |
| "_pkg_path", |
| "_tree", |
| "_verify", |
| ) |
| |
| def _writemsg_level(self, msg, level=0, noiselevel=0): |
| self.scheduler.output( |
| msg, |
| level=level, |
| noiselevel=noiselevel, |
| log_path=self.settings.get("PORTAGE_LOG_FILE"), |
| ) |
| |
| def _start(self): |
| |
| pkg = self.pkg |
| settings = self.settings |
| settings.setcpv(pkg) |
| self._tree = "bintree" |
| self._bintree = self.pkg.root_config.trees[self._tree] |
| self._verify = not self.opts.pretend |
| |
| # Use realpath like doebuild_environment() does, since we assert |
| # that this path is literally identical to PORTAGE_BUILDDIR. |
| dir_path = os.path.join( |
| os.path.realpath(settings["PORTAGE_TMPDIR"]), |
| "portage", |
| pkg.category, |
| pkg.pf, |
| ) |
| self._image_dir = os.path.join(dir_path, "image") |
| self._infloc = os.path.join(dir_path, "build-info") |
| self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") |
| settings["EBUILD"] = self._ebuild_path |
| portage.doebuild_environment( |
| self._ebuild_path, "setup", settings=self.settings, db=self._bintree.dbapi |
| ) |
| if dir_path != self.settings["PORTAGE_BUILDDIR"]: |
| raise AssertionError( |
| "'%s' != '%s'" % (dir_path, self.settings["PORTAGE_BUILDDIR"]) |
| ) |
| self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) |
| settings.configdict["pkg"]["EMERGE_FROM"] = "binary" |
| settings.configdict["pkg"]["MERGE_TYPE"] = "binary" |
| |
| if eapi_exports_replace_vars(settings["EAPI"]): |
| vardb = self.pkg.root_config.trees["vartree"].dbapi |
| settings["REPLACING_VERSIONS"] = " ".join( |
| set( |
| portage.versions.cpv_getversion(x) |
| for x in vardb.match(self.pkg.slot_atom) |
| + vardb.match("=" + self.pkg.cpv) |
| ) |
| ) |
| |
| # The prefetcher has already completed or it |
| # could be running now. If it's running now, |
| # wait for it to complete since it holds |
| # a lock on the file being fetched. The |
| # portage.locks functions are only designed |
| # to work between separate processes. Since |
| # the lock is held by the current process, |
| # use the scheduler and fetcher methods to |
| # synchronize with the fetcher. |
| prefetcher = self.prefetcher |
| if prefetcher is None: |
| pass |
| elif prefetcher.isAlive() and prefetcher.poll() is None: |
| |
| if not self.background: |
| fetch_log = os.path.join( |
| _emerge.emergelog._emerge_log_dir, "emerge-fetch.log" |
| ) |
| msg = ( |
| "Fetching in the background:", |
| prefetcher.pkg_path, |
| "To view fetch progress, run in another terminal:", |
| "tail -f %s" % fetch_log, |
| ) |
| out = portage.output.EOutput() |
| for l in msg: |
| out.einfo(l) |
| |
| self._current_task = prefetcher |
| prefetcher.addExitListener(self._prefetch_exit) |
| return |
| |
| self._prefetch_exit(prefetcher) |
| |
| def _prefetch_exit(self, prefetcher): |
| |
| if self._was_cancelled(): |
| self.wait() |
| return |
| |
| if not (self.opts.pretend or self.opts.fetchonly): |
| self._start_task( |
| AsyncTaskFuture(future=self._build_dir.async_lock()), |
| self._start_fetcher, |
| ) |
| else: |
| self._start_fetcher() |
| |
| def _start_fetcher(self, lock_task=None): |
| if lock_task is not None: |
| self._assert_current(lock_task) |
| if lock_task.cancelled: |
| self._default_final_exit(lock_task) |
| return |
| |
| lock_task.future.result() |
| # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). |
| portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) |
| # If necessary, discard old log so that we don't |
| # append to it. |
| self._build_dir.clean_log() |
| |
| pkg = self.pkg |
| pkg_count = self.pkg_count |
| fetcher = None |
| |
| if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): |
| |
| fetcher = BinpkgFetcher( |
| background=self.background, |
| logfile=self.settings.get("PORTAGE_LOG_FILE"), |
| pkg=self.pkg, |
| pretend=self.opts.pretend, |
| scheduler=self.scheduler, |
| ) |
| |
| msg = " --- (%s of %s) Fetching Binary (%s::%s)" % ( |
| pkg_count.curval, |
| pkg_count.maxval, |
| pkg.cpv, |
| fetcher.pkg_path, |
| ) |
| short_msg = "emerge: (%s of %s) %s Fetch" % ( |
| pkg_count.curval, |
| pkg_count.maxval, |
| pkg.cpv, |
| ) |
| self.logger.log(msg, short_msg=short_msg) |
| |
| # Allow the Scheduler's fetch queue to control the |
| # number of concurrent fetchers. |
| fetcher.addExitListener(self._fetcher_exit) |
| self._task_queued(fetcher) |
| self.scheduler.fetch.schedule(fetcher) |
| return |
| |
| self._fetcher_exit(fetcher) |
| |
| def _fetcher_exit(self, fetcher): |
| |
| # The fetcher only has a returncode when |
| # --getbinpkg is enabled. |
| if fetcher is not None: |
| self._fetched_pkg = fetcher.pkg_path |
| if self._default_exit(fetcher) != os.EX_OK: |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| if self.opts.pretend: |
| self._current_task = None |
| self.returncode = os.EX_OK |
| self.wait() |
| return |
| |
| verifier = None |
| if self._verify: |
| if self._fetched_pkg: |
| path = self._fetched_pkg |
| else: |
| path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv) |
| logfile = self.settings.get("PORTAGE_LOG_FILE") |
| verifier = BinpkgVerifier( |
| background=self.background, |
| logfile=logfile, |
| pkg=self.pkg, |
| scheduler=self.scheduler, |
| _pkg_path=path, |
| ) |
| self._start_task(verifier, self._verifier_exit) |
| return |
| |
| self._verifier_exit(verifier) |
| |
| def _verifier_exit(self, verifier): |
| if verifier is not None and self._default_exit(verifier) != os.EX_OK: |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| logger = self.logger |
| pkg = self.pkg |
| pkg_count = self.pkg_count |
| |
| if self._fetched_pkg: |
| pkg_path = self._bintree.getname( |
| self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), |
| allocate_new=False, |
| ) |
| else: |
| pkg_path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv) |
| |
| # This gives bashrc users an opportunity to do various things |
| # such as remove binary packages after they're installed. |
| if pkg_path is not None: |
| self.settings["PORTAGE_BINPKG_FILE"] = pkg_path |
| self._pkg_path = pkg_path |
| |
| logfile = self.settings.get("PORTAGE_LOG_FILE") |
| if logfile is not None and os.path.isfile(logfile): |
| # Remove fetch log after successful fetch. |
| try: |
| os.unlink(logfile) |
| except OSError: |
| pass |
| |
| if self.opts.fetchonly: |
| self._current_task = None |
| self.returncode = os.EX_OK |
| self.wait() |
| return |
| |
| msg = " === (%s of %s) Merging Binary (%s::%s)" % ( |
| pkg_count.curval, |
| pkg_count.maxval, |
| pkg.cpv, |
| pkg_path, |
| ) |
| short_msg = "emerge: (%s of %s) %s Merge Binary" % ( |
| pkg_count.curval, |
| pkg_count.maxval, |
| pkg.cpv, |
| ) |
| logger.log(msg, short_msg=short_msg) |
| |
| phase = "clean" |
| settings = self.settings |
| ebuild_phase = EbuildPhase( |
| background=self.background, |
| phase=phase, |
| scheduler=self.scheduler, |
| settings=settings, |
| ) |
| |
| self._start_task(ebuild_phase, self._clean_exit) |
| |
| def _clean_exit(self, clean_phase): |
| if self._default_exit(clean_phase) != os.EX_OK: |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| self._start_task( |
| AsyncTaskFuture(future=self._unpack_metadata(loop=self.scheduler)), |
| self._unpack_metadata_exit, |
| ) |
| |
| async def _unpack_metadata(self, loop=None): |
| |
| dir_path = self.settings["PORTAGE_BUILDDIR"] |
| |
| infloc = self._infloc |
| pkg = self.pkg |
| pkg_path = self._pkg_path |
| |
| dir_mode = 0o755 |
| for mydir in (dir_path, self._image_dir, infloc): |
| portage.util.ensure_dirs( |
| mydir, |
| uid=portage.data.portage_uid, |
| gid=portage.data.portage_gid, |
| mode=dir_mode, |
| ) |
| |
| # This initializes PORTAGE_LOG_FILE. |
| portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) |
| self._writemsg_level(">>> Extracting info\n") |
| |
| await self._bintree.dbapi.unpack_metadata( |
| self.settings, infloc, loop=self.scheduler |
| ) |
| check_missing_metadata = ("CATEGORY", "PF") |
| for k, v in zip( |
| check_missing_metadata, |
| self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata), |
| ): |
| if v: |
| continue |
| elif k == "CATEGORY": |
| v = pkg.category |
| elif k == "PF": |
| v = pkg.pf |
| else: |
| continue |
| |
| f = io.open( |
| _unicode_encode( |
| os.path.join(infloc, k), encoding=_encodings["fs"], errors="strict" |
| ), |
| mode="w", |
| encoding=_encodings["content"], |
| errors="backslashreplace", |
| ) |
| try: |
| f.write(_unicode_decode(v + "\n")) |
| finally: |
| f.close() |
| |
| # Store the md5sum in the vdb. |
| if pkg_path is not None: |
| (md5sum,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["MD5"]) |
| if not md5sum: |
| md5sum = portage.checksum.perform_md5(pkg_path) |
| with io.open( |
| _unicode_encode( |
| os.path.join(infloc, "BINPKGMD5"), |
| encoding=_encodings["fs"], |
| errors="strict", |
| ), |
| mode="w", |
| encoding=_encodings["content"], |
| errors="strict", |
| ) as f: |
| f.write(_unicode_decode("{}\n".format(md5sum))) |
| |
| env_extractor = BinpkgEnvExtractor( |
| background=self.background, scheduler=self.scheduler, settings=self.settings |
| ) |
| env_extractor.start() |
| await env_extractor.async_wait() |
| if env_extractor.returncode != os.EX_OK: |
| raise portage.exception.PortageException( |
| "failed to extract environment for {}".format(self.pkg.cpv) |
| ) |
| |
| def _unpack_metadata_exit(self, unpack_metadata): |
| if self._default_exit(unpack_metadata) != os.EX_OK: |
| unpack_metadata.future.result() |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| setup_phase = EbuildPhase( |
| background=self.background, |
| phase="setup", |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ) |
| |
| setup_phase.addExitListener(self._setup_exit) |
| self._task_queued(setup_phase) |
| self.scheduler.scheduleSetup(setup_phase) |
| |
| def _setup_exit(self, setup_phase): |
| if self._default_exit(setup_phase) != os.EX_OK: |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) |
| self._start_task( |
| AsyncTaskFuture( |
| future=self._bintree.dbapi.unpack_contents( |
| self.settings, self._image_dir, loop=self.scheduler |
| ) |
| ), |
| self._unpack_contents_exit, |
| ) |
| |
| def _unpack_contents_exit(self, unpack_contents): |
| if self._default_exit(unpack_contents) != os.EX_OK: |
| unpack_contents.future.result() |
| self._writemsg_level( |
| "!!! Error Extracting '%s'\n" % self._pkg_path, |
| noiselevel=-1, |
| level=logging.ERROR, |
| ) |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| try: |
| with io.open( |
| _unicode_encode( |
| os.path.join(self._infloc, "EPREFIX"), |
| encoding=_encodings["fs"], |
| errors="strict", |
| ), |
| mode="r", |
| encoding=_encodings["repo.content"], |
| errors="replace", |
| ) as f: |
| self._build_prefix = f.read().rstrip("\n") |
| except IOError: |
| self._build_prefix = "" |
| |
| if self._build_prefix == self.settings["EPREFIX"]: |
| ensure_dirs(self.settings["ED"]) |
| self._current_task = None |
| self.returncode = os.EX_OK |
| self.wait() |
| return |
| |
| env = self.settings.environ() |
| env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] |
| chpathtool = SpawnProcess( |
| args=[ |
| portage._python_interpreter, |
| os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), |
| self.settings["D"], |
| self._build_prefix, |
| self.settings["EPREFIX"], |
| ], |
| background=self.background, |
| env=env, |
| scheduler=self.scheduler, |
| logfile=self.settings.get("PORTAGE_LOG_FILE"), |
| ) |
| self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) |
| self._start_task(chpathtool, self._chpathtool_exit) |
| |
| def _chpathtool_exit(self, chpathtool): |
| if self._final_exit(chpathtool) != os.EX_OK: |
| self._writemsg_level( |
| "!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],), |
| noiselevel=-1, |
| level=logging.ERROR, |
| ) |
| self._async_unlock_builddir(returncode=self.returncode) |
| return |
| |
| # We want to install in "our" prefix, not the binary one |
| with io.open( |
| _unicode_encode( |
| os.path.join(self._infloc, "EPREFIX"), |
| encoding=_encodings["fs"], |
| errors="strict", |
| ), |
| mode="w", |
| encoding=_encodings["repo.content"], |
| errors="strict", |
| ) as f: |
| f.write(self.settings["EPREFIX"] + "\n") |
| |
| # Move the files to the correct location for merge. |
| image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"], "image_tmp") |
| build_d = os.path.join( |
| self.settings["D"], self._build_prefix.lstrip(os.sep) |
| ).rstrip(os.sep) |
| if not os.path.isdir(build_d): |
| # Assume this is a virtual package or something. |
| shutil.rmtree(self._image_dir) |
| ensure_dirs(self.settings["ED"]) |
| else: |
| os.rename(build_d, image_tmp_dir) |
| if build_d != self._image_dir: |
| shutil.rmtree(self._image_dir) |
| ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) |
| os.rename(image_tmp_dir, self.settings["ED"]) |
| |
| self.wait() |
| |
| def _async_unlock_builddir(self, returncode=None): |
| """ |
| Release the lock asynchronously, and if a returncode parameter |
| is given then set self.returncode and notify exit listeners. |
| """ |
| if self.opts.pretend or self.opts.fetchonly: |
| if returncode is not None: |
| self.returncode = returncode |
| self._async_wait() |
| return |
| if returncode is not None: |
| # The returncode will be set after unlock is complete. |
| self.returncode = None |
| portage.elog.elog_process(self.pkg.cpv, self.settings) |
| self._start_task( |
| AsyncTaskFuture(future=self._build_dir.async_unlock()), |
| functools.partial(self._unlock_builddir_exit, returncode=returncode), |
| ) |
| |
| def _unlock_builddir_exit(self, unlock_task, returncode=None): |
| self._assert_current(unlock_task) |
| if unlock_task.cancelled and returncode is not None: |
| self._default_final_exit(unlock_task) |
| return |
| |
| # Normally, async_unlock should not raise an exception here. |
| unlock_task.future.cancelled() or unlock_task.future.result() |
| if returncode is not None: |
| self.returncode = returncode |
| self._async_wait() |
| |
| def create_install_task(self): |
| task = EbuildMerge( |
| exit_hook=self._install_exit, |
| find_blockers=self.find_blockers, |
| ldpath_mtimes=self.ldpath_mtimes, |
| logger=self.logger, |
| pkg=self.pkg, |
| pkg_count=self.pkg_count, |
| pkg_path=self._pkg_path, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| tree=self._tree, |
| world_atom=self.world_atom, |
| ) |
| return task |
| |
| def _install_exit(self, task): |
| """ |
| @returns: Future, result is the returncode from an |
| EbuildBuildDir.async_unlock() task |
| """ |
| self.settings.pop("PORTAGE_BINPKG_FILE", None) |
| if ( |
| task.returncode == os.EX_OK |
| and "binpkg-logs" not in self.settings.features |
| and self.settings.get("PORTAGE_LOG_FILE") |
| ): |
| try: |
| os.unlink(self.settings["PORTAGE_LOG_FILE"]) |
| except OSError: |
| pass |
| self._async_unlock_builddir() |
| if self._current_task is None: |
| result = self.scheduler.create_future() |
| self.scheduler.call_soon(result.set_result, os.EX_OK) |
| else: |
| result = self._current_task.async_wait() |
| return result |
