blob: 12326fffda9f3d701f97e182253134ab235555c7 [file] [log] [blame]
# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
import gzip
import io
import sys
import tempfile
from _emerge.AsynchronousLock import AsynchronousLock
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
from _emerge.PackagePhase import PackagePhase
from _emerge.TaskSequence import TaskSequence
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
from portage.util._dyn_libs.soname_deps_qa import (
_get_all_provides,
_get_unresolved_soname_deps,
)
from portage.package.ebuild.prepare_build_dirs import (
_prepare_workdir,
_prepare_fake_distdir,
_prepare_fake_filesdir,
)
from portage.util import writemsg, ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
try:
from portage.xml.metadata import MetaDataXML
except (SystemExit, KeyboardInterrupt):
raise
except (ImportError, SystemError, RuntimeError, Exception):
# broken or missing xml support
# https://bugs.python.org/issue14988
MetaDataXML = None
import portage
portage.proxy.lazyimport.lazyimport(
globals(),
"portage.elog:messages@elog_messages",
"portage.package.ebuild.doebuild:_check_build_log,"
+ "_post_phase_cmds,_post_phase_userpriv_perms,"
+ "_post_phase_emptydir_cleanup,"
+ "_post_src_install_soname_symlinks,"
+ "_post_src_install_uid_fix,_postinst_bsdflags,"
+ "_post_src_install_write_metadata,"
+ "_preinst_bsdflags",
"portage.util.futures.unix_events:_set_nonblocking",
)
from portage import os
from portage import _encodings
from portage import _unicode_encode
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",)
# FEATURES displayed prior to setup phase
_features_display = (
"ccache",
"compressdebug",
"distcc",
"fakeroot",
"installsources",
"keeptemp",
"keepwork",
"network-sandbox",
"network-sandbox-proxy",
"nostrip",
"preserve-libs",
"sandbox",
"selinux",
"sesandbox",
"splitdebug",
"suidctl",
"test",
"userpriv",
"usersandbox",
)
# Locked phases
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
future = asyncio.ensure_future(self._async_start(), loop=self.scheduler)
self._start_task(AsyncTaskFuture(future=future), self._async_start_exit)
async def _async_start(self):
need_builddir = self.phase not in EbuildProcess._phases_without_builddir
if need_builddir:
phase_completed_file = os.path.join(
self.settings["PORTAGE_BUILDDIR"], ".%sed" % self.phase.rstrip("e")
)
if not os.path.exists(phase_completed_file):
# If the phase is really going to run then we want
# to eliminate any stale elog messages that may
# exist from a previous run.
try:
os.unlink(os.path.join(self.settings["T"], "logging", self.phase))
except OSError:
pass
ensure_dirs(os.path.join(self.settings["PORTAGE_BUILDDIR"], "empty"))
if self.phase in ("nofetch", "pretend", "setup"):
use = self.settings.get("PORTAGE_BUILT_USE")
if use is None:
use = self.settings["PORTAGE_USE"]
maint_str = ""
upstr_str = ""
metadata_xml_path = os.path.join(
os.path.dirname(self.settings["EBUILD"]), "metadata.xml"
)
if MetaDataXML is not None and os.path.isfile(metadata_xml_path):
herds_path = os.path.join(
self.settings["PORTDIR"], "metadata/herds.xml"
)
try:
metadata_xml = MetaDataXML(metadata_xml_path, herds_path)
maint_str = metadata_xml.format_maintainer_string()
upstr_str = metadata_xml.format_upstream_string()
except SyntaxError:
maint_str = "<invalid metadata.xml>"
msg = []
msg.append("Package: %s" % self.settings.mycpv)
if self.settings.get("PORTAGE_REPO_NAME"):
msg.append("Repository: %s" % self.settings["PORTAGE_REPO_NAME"])
if maint_str:
msg.append("Maintainer: %s" % maint_str)
if upstr_str:
msg.append("Upstream: %s" % upstr_str)
msg.append("USE: %s" % use)
relevant_features = []
enabled_features = self.settings.features
for x in self._features_display:
if x in enabled_features:
relevant_features.append(x)
if relevant_features:
msg.append("FEATURES: %s" % " ".join(relevant_features))
# Force background=True for this header since it's intended
# for the log and it doesn't necessarily need to be visible
# elsewhere.
await self._elog("einfo", msg, background=True)
if self.phase == "package":
if "PORTAGE_BINPKG_TMPFILE" not in self.settings:
self.settings["PORTAGE_BINPKG_TMPFILE"] = (
os.path.join(
self.settings["PKGDIR"],
self.settings["CATEGORY"],
self.settings["PF"],
)
+ ".tbz2"
)
def _async_start_exit(self, task):
task.future.cancelled() or task.future.result()
if self._default_exit(task) != os.EX_OK:
self.wait()
return
if self.phase in ("pretend", "prerm"):
env_extractor = BinpkgEnvExtractor(
background=self.background,
scheduler=self.scheduler,
settings=self.settings,
)
if env_extractor.saved_env_exists():
self._start_task(env_extractor, self._env_extractor_exit)
return
# If the environment.bz2 doesn't exist, then ebuild.sh will
# source the ebuild as a fallback.
self._start_lock()
def _env_extractor_exit(self, env_extractor):
if self._default_exit(env_extractor) != os.EX_OK:
self.wait()
return
self._start_lock()
def _start_lock(self):
if (
self.phase in self._locked_phases
and "ebuild-locks" in self.settings.features
):
eroot = self.settings["EROOT"]
lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild")
if os.access(os.path.dirname(lock_path), os.W_OK):
self._ebuild_lock = AsynchronousLock(
path=lock_path, scheduler=self.scheduler
)
self._start_task(self._ebuild_lock, self._lock_exit)
return
self._start_ebuild()
def _lock_exit(self, ebuild_lock):
if self._default_exit(ebuild_lock) != os.EX_OK:
self.wait()
return
self._start_ebuild()
def _get_log_path(self):
# Don't open the log file during the clean phase since the
# open file can result in an nfs lock on $T/build.log which
# prevents the clean phase from removing $T.
logfile = None
if (
self.phase not in ("clean", "cleanrm")
and self.settings.get("PORTAGE_BACKGROUND") != "subprocess"
):
logfile = self.settings.get("PORTAGE_LOG_FILE")
return logfile
def _start_ebuild(self):
if self.phase == "package":
self._start_task(
PackagePhase(
actionmap=self.actionmap,
background=self.background,
fd_pipes=self.fd_pipes,
logfile=self._get_log_path(),
scheduler=self.scheduler,
settings=self.settings,
),
self._ebuild_exit,
)
return
if self.phase == "unpack":
alist = self.settings.configdict["pkg"].get("A", "").split()
_prepare_fake_distdir(self.settings, alist)
_prepare_fake_filesdir(self.settings)
fd_pipes = self.fd_pipes
if fd_pipes is None:
if not self.background and self.phase == "nofetch":
# All the pkg_nofetch output goes to stderr since
# it's considered to be an error message.
fd_pipes = {1: sys.__stderr__.fileno()}
ebuild_process = EbuildProcess(
actionmap=self.actionmap,
background=self.background,
fd_pipes=fd_pipes,
logfile=self._get_log_path(),
phase=self.phase,
scheduler=self.scheduler,
settings=self.settings,
)
self._start_task(ebuild_process, self._ebuild_exit)
def _ebuild_exit(self, ebuild_process):
self._assert_current(ebuild_process)
if self._ebuild_lock is None:
self._ebuild_exit_unlocked(ebuild_process)
else:
self._start_task(
AsyncTaskFuture(future=self._ebuild_lock.async_unlock()),
functools.partial(self._ebuild_exit_unlocked, ebuild_process),
)
def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None):
if unlock_task is not None:
self._assert_current(unlock_task)
if unlock_task.cancelled:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.result()
fail = False
if ebuild_process.returncode != os.EX_OK:
self.returncode = ebuild_process.returncode
if self.phase == "test" and "test-fail-continue" in self.settings.features:
# mark test phase as complete (bug #452030)
try:
open(
_unicode_encode(
os.path.join(self.settings["PORTAGE_BUILDDIR"], ".tested"),
encoding=_encodings["fs"],
errors="strict",
),
"wb",
).close()
except OSError:
pass
else:
fail = True
if not fail:
self.returncode = None
logfile = self._get_log_path()
if self.phase == "install":
out = io.StringIO()
_check_build_log(self.settings, out=out)
msg = out.getvalue()
self.scheduler.output(msg, log_path=logfile)
if fail:
self._die_hooks()
return
settings = self.settings
_post_phase_userpriv_perms(settings)
_post_phase_emptydir_cleanup(settings)
if self.phase == "unpack":
# Bump WORKDIR timestamp, in case tar gave it a timestamp
# that will interfere with distfiles / WORKDIR timestamp
# comparisons as reported in bug #332217. Also, fix
# ownership since tar can change that too.
os.utime(settings["WORKDIR"], None)
_prepare_workdir(settings)
elif self.phase == "install":
out = io.StringIO()
_post_src_install_write_metadata(settings)
_post_src_install_uid_fix(settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=logfile)
elif self.phase == "preinst":
_preinst_bsdflags(settings)
elif self.phase == "postinst":
_postinst_bsdflags(settings)
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
if logfile is not None and self.phase in ("install",):
# Log to a temporary file, since the code we are running
# reads PORTAGE_LOG_FILE for QA checks, and we want to
# avoid annoying "gzip: unexpected end of file" messages
# when FEATURES=compress-build-logs is enabled.
fd, logfile = tempfile.mkstemp()
os.close(fd)
post_phase = _PostPhaseCommands(
background=self.background,
commands=post_phase_cmds,
elog=self._elog,
fd_pipes=self.fd_pipes,
logfile=logfile,
phase=self.phase,
scheduler=self.scheduler,
settings=settings,
)
self._start_task(post_phase, self._post_phase_exit)
return
# this point is not reachable if there was a failure and
# we returned for die_hooks above, so returncode must
# indicate success (especially if ebuild_process.returncode
# is unsuccessful and test-fail-continue came into play)
self.returncode = os.EX_OK
self._current_task = None
self.wait()
def _post_phase_exit(self, post_phase):
self._assert_current(post_phase)
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
if post_phase.logfile is not None and post_phase.logfile != log_path:
# We were logging to a temp file (see above), so append
# temp file to main log and remove temp file.
self._append_temp_log(post_phase.logfile, log_path)
if self._final_exit(post_phase) != os.EX_OK:
writemsg("!!! post %s failed; exiting.\n" % self.phase, noiselevel=-1)
self._die_hooks()
return
self._current_task = None
self.wait()
return
def _append_temp_log(self, temp_log, log_path):
temp_file = open(
_unicode_encode(temp_log, encoding=_encodings["fs"], errors="strict"), "rb"
)
log_file, log_file_real = self._open_log(log_path)
for line in temp_file:
log_file.write(line)
temp_file.close()
log_file.close()
if log_file_real is not log_file:
log_file_real.close()
os.unlink(temp_log)
def _open_log(self, log_path):
f = open(
_unicode_encode(log_path, encoding=_encodings["fs"], errors="strict"),
mode="ab",
)
f_real = f
if log_path.endswith(".gz"):
f = gzip.GzipFile(filename="", mode="ab", fileobj=f)
return (f, f_real)
def _die_hooks(self):
self.returncode = None
phase = "die_hooks"
die_hooks = MiscFunctionsProcess(
background=self.background,
commands=[phase],
phase=phase,
logfile=self._get_log_path(),
fd_pipes=self.fd_pipes,
scheduler=self.scheduler,
settings=self.settings,
)
self._start_task(die_hooks, self._die_hooks_exit)
def _die_hooks_exit(self, die_hooks):
if (
self.phase != "clean"
and "noclean" not in self.settings.features
and "fail-clean" in self.settings.features
):
self._default_exit(die_hooks)
self._fail_clean()
return
self._final_exit(die_hooks)
self.returncode = 1
self.wait()
def _fail_clean(self):
self.returncode = None
portage.elog.elog_process(self.settings.mycpv, self.settings)
phase = "clean"
clean_phase = EbuildPhase(
background=self.background,
fd_pipes=self.fd_pipes,
phase=phase,
scheduler=self.scheduler,
settings=self.settings,
)
self._start_task(clean_phase, self._fail_clean_exit)
def _fail_clean_exit(self, clean_phase):
self._final_exit(clean_phase)
self.returncode = 1
self.wait()
async def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = self.settings.get(
"NOCOLOR", "false"
).lower() in ("no", "false")
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
build_logger = None
try:
log_file = None
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
if log_path:
build_logger = BuildLogger(
env=self.settings.environ(),
log_path=log_path,
log_filter_file=self.settings.get(
"PORTAGE_LOG_FILTER_FILE_CMD"
),
scheduler=self.scheduler,
)
build_logger.start()
_set_nonblocking(build_logger.stdin.fileno())
log_file = build_logger.stdin
await self.scheduler.async_output(
msg, log_file=log_file, background=background
)
if build_logger is not None:
build_logger.stdin.close()
await build_logger.async_wait()
except asyncio.CancelledError:
if build_logger is not None:
build_logger.cancel()
raise
class _PostPhaseCommands(CompositeTask):
__slots__ = ("commands", "elog", "fd_pipes", "logfile", "phase", "settings")
def _start(self):
if isinstance(self.commands, list):
cmds = [({}, self.commands)]
else:
cmds = list(self.commands)
if "selinux" not in self.settings.features:
cmds = [
(kwargs, commands)
for kwargs, commands in cmds
if not kwargs.get("selinux_only")
]
tasks = TaskSequence()
for kwargs, commands in cmds:
# Select args intended for MiscFunctionsProcess.
kwargs = dict(
(k, v) for k, v in kwargs.items() if k in ("ld_preload_sandbox",)
)
tasks.add(
MiscFunctionsProcess(
background=self.background,
commands=commands,
fd_pipes=self.fd_pipes,
logfile=self.logfile,
phase=self.phase,
scheduler=self.scheduler,
settings=self.settings,
**kwargs
)
)
self._start_task(tasks, self._commands_exit)
def _commands_exit(self, task):
if self._default_exit(task) != os.EX_OK:
self._async_wait()
return
if self.phase == "install":
out = io.StringIO()
_post_src_install_soname_symlinks(self.settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE")
)
if "qa-unresolved-soname-deps" in self.settings.features:
# This operates on REQUIRES metadata generated by the above function call.
future = asyncio.ensure_future(
self._soname_deps_qa(), loop=self.scheduler
)
# If an unexpected exception occurs, then this will raise it.
future.add_done_callback(
lambda future: future.cancelled() or future.result()
)
self._start_task(
AsyncTaskFuture(future=future), self._default_final_exit
)
else:
self._default_final_exit(task)
else:
self._default_final_exit(task)
async def _soname_deps_qa(self):
vardb = QueryCommand.get_db()[self.settings["EROOT"]]["vartree"].dbapi
all_provides = await self.scheduler.run_in_executor(
ForkExecutor(loop=self.scheduler), _get_all_provides, vardb
)
unresolved = _get_unresolved_soname_deps(
os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info"), all_provides
)
if unresolved:
unresolved.sort()
qa_msg = ["QA Notice: Unresolved soname dependencies:"]
qa_msg.append("")
qa_msg.extend(
"\t%s: %s" % (filename, " ".join(sorted(soname_deps)))
for filename, soname_deps in unresolved
)
qa_msg.append("")
await self.elog("eqawarn", qa_msg)