| # Copyright 1999-2021 Gentoo Authors |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| import functools |
| import gzip |
| import io |
| import sys |
| import tempfile |
| |
| from _emerge.AsynchronousLock import AsynchronousLock |
| from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor |
| from _emerge.MiscFunctionsProcess import MiscFunctionsProcess |
| from _emerge.EbuildProcess import EbuildProcess |
| from _emerge.CompositeTask import CompositeTask |
| from _emerge.PackagePhase import PackagePhase |
| from _emerge.TaskSequence import TaskSequence |
| from portage.package.ebuild._ipc.QueryCommand import QueryCommand |
| from portage.util._dyn_libs.soname_deps_qa import ( |
| _get_all_provides, |
| _get_unresolved_soname_deps, |
| ) |
| from portage.package.ebuild.prepare_build_dirs import ( |
| _prepare_workdir, |
| _prepare_fake_distdir, |
| _prepare_fake_filesdir, |
| ) |
| from portage.util import writemsg, ensure_dirs |
| from portage.util._async.AsyncTaskFuture import AsyncTaskFuture |
| from portage.util._async.BuildLogger import BuildLogger |
| from portage.util.futures import asyncio |
| from portage.util.futures.executor.fork import ForkExecutor |
| |
| try: |
| from portage.xml.metadata import MetaDataXML |
| except (SystemExit, KeyboardInterrupt): |
| raise |
| except (ImportError, SystemError, RuntimeError, Exception): |
| # broken or missing xml support |
| # https://bugs.python.org/issue14988 |
| MetaDataXML = None |
| |
| import portage |
| |
| portage.proxy.lazyimport.lazyimport( |
| globals(), |
| "portage.elog:messages@elog_messages", |
| "portage.package.ebuild.doebuild:_check_build_log," |
| + "_post_phase_cmds,_post_phase_userpriv_perms," |
| + "_post_phase_emptydir_cleanup," |
| + "_post_src_install_soname_symlinks," |
| + "_post_src_install_uid_fix,_postinst_bsdflags," |
| + "_post_src_install_write_metadata," |
| + "_preinst_bsdflags", |
| "portage.util.futures.unix_events:_set_nonblocking", |
| ) |
| from portage import os |
| from portage import _encodings |
| from portage import _unicode_encode |
| |
| |
| class EbuildPhase(CompositeTask): |
| |
| __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",) |
| |
| # FEATURES displayed prior to setup phase |
| _features_display = ( |
| "ccache", |
| "compressdebug", |
| "distcc", |
| "fakeroot", |
| "installsources", |
| "keeptemp", |
| "keepwork", |
| "network-sandbox", |
| "network-sandbox-proxy", |
| "nostrip", |
| "preserve-libs", |
| "sandbox", |
| "selinux", |
| "sesandbox", |
| "splitdebug", |
| "suidctl", |
| "test", |
| "userpriv", |
| "usersandbox", |
| ) |
| |
| # Locked phases |
| _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") |
| |
| def _start(self): |
| future = asyncio.ensure_future(self._async_start(), loop=self.scheduler) |
| self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) |
| |
| async def _async_start(self): |
| |
| need_builddir = self.phase not in EbuildProcess._phases_without_builddir |
| |
| if need_builddir: |
| phase_completed_file = os.path.join( |
| self.settings["PORTAGE_BUILDDIR"], ".%sed" % self.phase.rstrip("e") |
| ) |
| if not os.path.exists(phase_completed_file): |
| # If the phase is really going to run then we want |
| # to eliminate any stale elog messages that may |
| # exist from a previous run. |
| try: |
| os.unlink(os.path.join(self.settings["T"], "logging", self.phase)) |
| except OSError: |
| pass |
| ensure_dirs(os.path.join(self.settings["PORTAGE_BUILDDIR"], "empty")) |
| |
| if self.phase in ("nofetch", "pretend", "setup"): |
| |
| use = self.settings.get("PORTAGE_BUILT_USE") |
| if use is None: |
| use = self.settings["PORTAGE_USE"] |
| |
| maint_str = "" |
| upstr_str = "" |
| metadata_xml_path = os.path.join( |
| os.path.dirname(self.settings["EBUILD"]), "metadata.xml" |
| ) |
| if MetaDataXML is not None and os.path.isfile(metadata_xml_path): |
| herds_path = os.path.join( |
| self.settings["PORTDIR"], "metadata/herds.xml" |
| ) |
| try: |
| metadata_xml = MetaDataXML(metadata_xml_path, herds_path) |
| maint_str = metadata_xml.format_maintainer_string() |
| upstr_str = metadata_xml.format_upstream_string() |
| except SyntaxError: |
| maint_str = "<invalid metadata.xml>" |
| |
| msg = [] |
| msg.append("Package: %s" % self.settings.mycpv) |
| if self.settings.get("PORTAGE_REPO_NAME"): |
| msg.append("Repository: %s" % self.settings["PORTAGE_REPO_NAME"]) |
| if maint_str: |
| msg.append("Maintainer: %s" % maint_str) |
| if upstr_str: |
| msg.append("Upstream: %s" % upstr_str) |
| |
| msg.append("USE: %s" % use) |
| relevant_features = [] |
| enabled_features = self.settings.features |
| for x in self._features_display: |
| if x in enabled_features: |
| relevant_features.append(x) |
| if relevant_features: |
| msg.append("FEATURES: %s" % " ".join(relevant_features)) |
| |
| # Force background=True for this header since it's intended |
| # for the log and it doesn't necessarily need to be visible |
| # elsewhere. |
| await self._elog("einfo", msg, background=True) |
| |
| if self.phase == "package": |
| if "PORTAGE_BINPKG_TMPFILE" not in self.settings: |
| self.settings["PORTAGE_BINPKG_TMPFILE"] = ( |
| os.path.join( |
| self.settings["PKGDIR"], |
| self.settings["CATEGORY"], |
| self.settings["PF"], |
| ) |
| + ".tbz2" |
| ) |
| |
| def _async_start_exit(self, task): |
| task.future.cancelled() or task.future.result() |
| if self._default_exit(task) != os.EX_OK: |
| self.wait() |
| return |
| |
| if self.phase in ("pretend", "prerm"): |
| env_extractor = BinpkgEnvExtractor( |
| background=self.background, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ) |
| if env_extractor.saved_env_exists(): |
| self._start_task(env_extractor, self._env_extractor_exit) |
| return |
| # If the environment.bz2 doesn't exist, then ebuild.sh will |
| # source the ebuild as a fallback. |
| |
| self._start_lock() |
| |
| def _env_extractor_exit(self, env_extractor): |
| if self._default_exit(env_extractor) != os.EX_OK: |
| self.wait() |
| return |
| |
| self._start_lock() |
| |
| def _start_lock(self): |
| if ( |
| self.phase in self._locked_phases |
| and "ebuild-locks" in self.settings.features |
| ): |
| eroot = self.settings["EROOT"] |
| lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild") |
| if os.access(os.path.dirname(lock_path), os.W_OK): |
| self._ebuild_lock = AsynchronousLock( |
| path=lock_path, scheduler=self.scheduler |
| ) |
| self._start_task(self._ebuild_lock, self._lock_exit) |
| return |
| |
| self._start_ebuild() |
| |
| def _lock_exit(self, ebuild_lock): |
| if self._default_exit(ebuild_lock) != os.EX_OK: |
| self.wait() |
| return |
| self._start_ebuild() |
| |
| def _get_log_path(self): |
| # Don't open the log file during the clean phase since the |
| # open file can result in an nfs lock on $T/build.log which |
| # prevents the clean phase from removing $T. |
| logfile = None |
| if ( |
| self.phase not in ("clean", "cleanrm") |
| and self.settings.get("PORTAGE_BACKGROUND") != "subprocess" |
| ): |
| logfile = self.settings.get("PORTAGE_LOG_FILE") |
| return logfile |
| |
| def _start_ebuild(self): |
| if self.phase == "package": |
| self._start_task( |
| PackagePhase( |
| actionmap=self.actionmap, |
| background=self.background, |
| fd_pipes=self.fd_pipes, |
| logfile=self._get_log_path(), |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ), |
| self._ebuild_exit, |
| ) |
| return |
| |
| if self.phase == "unpack": |
| alist = self.settings.configdict["pkg"].get("A", "").split() |
| _prepare_fake_distdir(self.settings, alist) |
| _prepare_fake_filesdir(self.settings) |
| |
| fd_pipes = self.fd_pipes |
| if fd_pipes is None: |
| if not self.background and self.phase == "nofetch": |
| # All the pkg_nofetch output goes to stderr since |
| # it's considered to be an error message. |
| fd_pipes = {1: sys.__stderr__.fileno()} |
| |
| ebuild_process = EbuildProcess( |
| actionmap=self.actionmap, |
| background=self.background, |
| fd_pipes=fd_pipes, |
| logfile=self._get_log_path(), |
| phase=self.phase, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ) |
| |
| self._start_task(ebuild_process, self._ebuild_exit) |
| |
| def _ebuild_exit(self, ebuild_process): |
| self._assert_current(ebuild_process) |
| if self._ebuild_lock is None: |
| self._ebuild_exit_unlocked(ebuild_process) |
| else: |
| self._start_task( |
| AsyncTaskFuture(future=self._ebuild_lock.async_unlock()), |
| functools.partial(self._ebuild_exit_unlocked, ebuild_process), |
| ) |
| |
| def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None): |
| if unlock_task is not None: |
| self._assert_current(unlock_task) |
| if unlock_task.cancelled: |
| self._default_final_exit(unlock_task) |
| return |
| |
| # Normally, async_unlock should not raise an exception here. |
| unlock_task.future.result() |
| |
| fail = False |
| if ebuild_process.returncode != os.EX_OK: |
| self.returncode = ebuild_process.returncode |
| if self.phase == "test" and "test-fail-continue" in self.settings.features: |
| # mark test phase as complete (bug #452030) |
| try: |
| open( |
| _unicode_encode( |
| os.path.join(self.settings["PORTAGE_BUILDDIR"], ".tested"), |
| encoding=_encodings["fs"], |
| errors="strict", |
| ), |
| "wb", |
| ).close() |
| except OSError: |
| pass |
| else: |
| fail = True |
| |
| if not fail: |
| self.returncode = None |
| |
| logfile = self._get_log_path() |
| |
| if self.phase == "install": |
| out = io.StringIO() |
| _check_build_log(self.settings, out=out) |
| msg = out.getvalue() |
| self.scheduler.output(msg, log_path=logfile) |
| |
| if fail: |
| self._die_hooks() |
| return |
| |
| settings = self.settings |
| _post_phase_userpriv_perms(settings) |
| _post_phase_emptydir_cleanup(settings) |
| |
| if self.phase == "unpack": |
| # Bump WORKDIR timestamp, in case tar gave it a timestamp |
| # that will interfere with distfiles / WORKDIR timestamp |
| # comparisons as reported in bug #332217. Also, fix |
| # ownership since tar can change that too. |
| os.utime(settings["WORKDIR"], None) |
| _prepare_workdir(settings) |
| elif self.phase == "install": |
| out = io.StringIO() |
| _post_src_install_write_metadata(settings) |
| _post_src_install_uid_fix(settings, out) |
| msg = out.getvalue() |
| if msg: |
| self.scheduler.output(msg, log_path=logfile) |
| elif self.phase == "preinst": |
| _preinst_bsdflags(settings) |
| elif self.phase == "postinst": |
| _postinst_bsdflags(settings) |
| |
| post_phase_cmds = _post_phase_cmds.get(self.phase) |
| if post_phase_cmds is not None: |
| if logfile is not None and self.phase in ("install",): |
| # Log to a temporary file, since the code we are running |
| # reads PORTAGE_LOG_FILE for QA checks, and we want to |
| # avoid annoying "gzip: unexpected end of file" messages |
| # when FEATURES=compress-build-logs is enabled. |
| fd, logfile = tempfile.mkstemp() |
| os.close(fd) |
| post_phase = _PostPhaseCommands( |
| background=self.background, |
| commands=post_phase_cmds, |
| elog=self._elog, |
| fd_pipes=self.fd_pipes, |
| logfile=logfile, |
| phase=self.phase, |
| scheduler=self.scheduler, |
| settings=settings, |
| ) |
| self._start_task(post_phase, self._post_phase_exit) |
| return |
| |
| # this point is not reachable if there was a failure and |
| # we returned for die_hooks above, so returncode must |
| # indicate success (especially if ebuild_process.returncode |
| # is unsuccessful and test-fail-continue came into play) |
| self.returncode = os.EX_OK |
| self._current_task = None |
| self.wait() |
| |
| def _post_phase_exit(self, post_phase): |
| |
| self._assert_current(post_phase) |
| |
| log_path = None |
| if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": |
| log_path = self.settings.get("PORTAGE_LOG_FILE") |
| |
| if post_phase.logfile is not None and post_phase.logfile != log_path: |
| # We were logging to a temp file (see above), so append |
| # temp file to main log and remove temp file. |
| self._append_temp_log(post_phase.logfile, log_path) |
| |
| if self._final_exit(post_phase) != os.EX_OK: |
| writemsg("!!! post %s failed; exiting.\n" % self.phase, noiselevel=-1) |
| self._die_hooks() |
| return |
| |
| self._current_task = None |
| self.wait() |
| return |
| |
| def _append_temp_log(self, temp_log, log_path): |
| |
| temp_file = open( |
| _unicode_encode(temp_log, encoding=_encodings["fs"], errors="strict"), "rb" |
| ) |
| |
| log_file, log_file_real = self._open_log(log_path) |
| |
| for line in temp_file: |
| log_file.write(line) |
| |
| temp_file.close() |
| log_file.close() |
| if log_file_real is not log_file: |
| log_file_real.close() |
| os.unlink(temp_log) |
| |
| def _open_log(self, log_path): |
| |
| f = open( |
| _unicode_encode(log_path, encoding=_encodings["fs"], errors="strict"), |
| mode="ab", |
| ) |
| f_real = f |
| |
| if log_path.endswith(".gz"): |
| f = gzip.GzipFile(filename="", mode="ab", fileobj=f) |
| |
| return (f, f_real) |
| |
| def _die_hooks(self): |
| self.returncode = None |
| phase = "die_hooks" |
| die_hooks = MiscFunctionsProcess( |
| background=self.background, |
| commands=[phase], |
| phase=phase, |
| logfile=self._get_log_path(), |
| fd_pipes=self.fd_pipes, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ) |
| self._start_task(die_hooks, self._die_hooks_exit) |
| |
| def _die_hooks_exit(self, die_hooks): |
| if ( |
| self.phase != "clean" |
| and "noclean" not in self.settings.features |
| and "fail-clean" in self.settings.features |
| ): |
| self._default_exit(die_hooks) |
| self._fail_clean() |
| return |
| self._final_exit(die_hooks) |
| self.returncode = 1 |
| self.wait() |
| |
| def _fail_clean(self): |
| self.returncode = None |
| portage.elog.elog_process(self.settings.mycpv, self.settings) |
| phase = "clean" |
| clean_phase = EbuildPhase( |
| background=self.background, |
| fd_pipes=self.fd_pipes, |
| phase=phase, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| ) |
| self._start_task(clean_phase, self._fail_clean_exit) |
| |
| def _fail_clean_exit(self, clean_phase): |
| self._final_exit(clean_phase) |
| self.returncode = 1 |
| self.wait() |
| |
| async def _elog(self, elog_funcname, lines, background=None): |
| if background is None: |
| background = self.background |
| out = io.StringIO() |
| phase = self.phase |
| elog_func = getattr(elog_messages, elog_funcname) |
| global_havecolor = portage.output.havecolor |
| try: |
| portage.output.havecolor = self.settings.get( |
| "NOCOLOR", "false" |
| ).lower() in ("no", "false") |
| for line in lines: |
| elog_func(line, phase=phase, key=self.settings.mycpv, out=out) |
| finally: |
| portage.output.havecolor = global_havecolor |
| msg = out.getvalue() |
| if msg: |
| build_logger = None |
| try: |
| log_file = None |
| log_path = None |
| if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": |
| log_path = self.settings.get("PORTAGE_LOG_FILE") |
| if log_path: |
| build_logger = BuildLogger( |
| env=self.settings.environ(), |
| log_path=log_path, |
| log_filter_file=self.settings.get( |
| "PORTAGE_LOG_FILTER_FILE_CMD" |
| ), |
| scheduler=self.scheduler, |
| ) |
| build_logger.start() |
| _set_nonblocking(build_logger.stdin.fileno()) |
| log_file = build_logger.stdin |
| |
| await self.scheduler.async_output( |
| msg, log_file=log_file, background=background |
| ) |
| |
| if build_logger is not None: |
| build_logger.stdin.close() |
| await build_logger.async_wait() |
| except asyncio.CancelledError: |
| if build_logger is not None: |
| build_logger.cancel() |
| raise |
| |
| |
| class _PostPhaseCommands(CompositeTask): |
| |
| __slots__ = ("commands", "elog", "fd_pipes", "logfile", "phase", "settings") |
| |
| def _start(self): |
| if isinstance(self.commands, list): |
| cmds = [({}, self.commands)] |
| else: |
| cmds = list(self.commands) |
| |
| if "selinux" not in self.settings.features: |
| cmds = [ |
| (kwargs, commands) |
| for kwargs, commands in cmds |
| if not kwargs.get("selinux_only") |
| ] |
| |
| tasks = TaskSequence() |
| for kwargs, commands in cmds: |
| # Select args intended for MiscFunctionsProcess. |
| kwargs = dict( |
| (k, v) for k, v in kwargs.items() if k in ("ld_preload_sandbox",) |
| ) |
| tasks.add( |
| MiscFunctionsProcess( |
| background=self.background, |
| commands=commands, |
| fd_pipes=self.fd_pipes, |
| logfile=self.logfile, |
| phase=self.phase, |
| scheduler=self.scheduler, |
| settings=self.settings, |
| **kwargs |
| ) |
| ) |
| |
| self._start_task(tasks, self._commands_exit) |
| |
| def _commands_exit(self, task): |
| |
| if self._default_exit(task) != os.EX_OK: |
| self._async_wait() |
| return |
| |
| if self.phase == "install": |
| out = io.StringIO() |
| _post_src_install_soname_symlinks(self.settings, out) |
| msg = out.getvalue() |
| if msg: |
| self.scheduler.output( |
| msg, log_path=self.settings.get("PORTAGE_LOG_FILE") |
| ) |
| |
| if "qa-unresolved-soname-deps" in self.settings.features: |
| # This operates on REQUIRES metadata generated by the above function call. |
| future = asyncio.ensure_future( |
| self._soname_deps_qa(), loop=self.scheduler |
| ) |
| # If an unexpected exception occurs, then this will raise it. |
| future.add_done_callback( |
| lambda future: future.cancelled() or future.result() |
| ) |
| self._start_task( |
| AsyncTaskFuture(future=future), self._default_final_exit |
| ) |
| else: |
| self._default_final_exit(task) |
| else: |
| self._default_final_exit(task) |
| |
| async def _soname_deps_qa(self): |
| |
| vardb = QueryCommand.get_db()[self.settings["EROOT"]]["vartree"].dbapi |
| |
| all_provides = await self.scheduler.run_in_executor( |
| ForkExecutor(loop=self.scheduler), _get_all_provides, vardb |
| ) |
| |
| unresolved = _get_unresolved_soname_deps( |
| os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info"), all_provides |
| ) |
| |
| if unresolved: |
| unresolved.sort() |
| qa_msg = ["QA Notice: Unresolved soname dependencies:"] |
| qa_msg.append("") |
| qa_msg.extend( |
| "\t%s: %s" % (filename, " ".join(sorted(soname_deps))) |
| for filename, soname_deps in unresolved |
| ) |
| qa_msg.append("") |
| await self.elog("eqawarn", qa_msg) |