Fix bugs in install hook for sbom package info
Errors of install hook will not cause build failure
so errors are written to the output file to be parsed
by build-executor service.
Files formatted by depot-tools/black.
BUG=b/254334533
TEST=presubmit
RELEASE_NOTE=None
Change-Id: I486b1f4c1aee4438ec16ed57c9a5f1c12dc2d280
Reviewed-on: https://cos-review.googlesource.com/c/third_party/platform/crosutils/+/40388
Tested-by: Cusky Presubmit Bot <presubmit@cos-infra-prod.iam.gserviceaccount.com>
Reviewed-by: Vaibhav Rustagi <vaibhavrustagi@google.com>
diff --git a/hooks/install/gen-sbom-package-info.py b/hooks/install/gen-sbom-package-info.py
index 59e5ffd..05bce2a 100755
--- a/hooks/install/gen-sbom-package-info.py
+++ b/hooks/install/gen-sbom-package-info.py
@@ -20,21 +20,28 @@
from sbom_info_lib import download_url
from sbom_info_lib import go_dep
from sbom_info_lib import licenses
+from chromite.lib import osutils
SBOM_INFO_FILE_NAME = "sbom-pkg-info"
class SbomPackageInfo:
- def __init__(self, url, license, go_dep):
- self.download_url = url
- self.licenses = license
- self.go_dep = go_dep
+ def __init__(self):
+ self.download_url = ""
+ self.licenses = ""
+ self.go_dep = ""
+ self.err = ""
def write_to_build_info(self, build_info_dir):
- with open(f"{build_info_dir}/{SBOM_INFO_FILE_NAME}", "w") as f:
- f.write(f"download-url:{self.download_url}\n")
- f.write(f"licenses:{self.licenses}\n")
- f.write(f"go-dep:{self.go_dep}\n")
+ content = (
+ f"download-url:{self.download_url}\n"
+ + f"licenses:{self.licenses}\n"
+ + f"go-dep:{self.go_dep}\n"
+ + f"err:{self.err}\n"
+ )
+ osutils.WriteFile(
+ f"{build_info_dir}/{SBOM_INFO_FILE_NAME}", content, makedirs=True
+ )
class SBOMPkgInfoError(Exception):
@@ -42,22 +49,45 @@
super().__init__(msg)
+def read_build_info(build_info_dir):
+ with open(os.path.join(build_info_dir, "repository"), "r") as f:
+ repository = f.read().strip()
+ with open(os.path.join(build_info_dir, "CATEGORY"), "r") as f:
+ category = f.read().strip()
+ with open(os.path.join(build_info_dir, "PF"), "r") as f:
+ pf = f.read().strip()
+ return repository, category, pf
+
+
def main():
+ sbom_pkg_info = SbomPackageInfo()
package_dir = os.getenv("PORTAGE_BUILDDIR")
build_info_dir = os.path.join(package_dir, "build-info")
- package_name = os.path.basename(package_dir)
- ebuild = os.path.join(build_info_dir, package_name + ".ebuild")
- url = download_url.get_download_url(build_info_dir, ebuild)
- sbom_pkg_info = SbomPackageInfo(
- url,
- licenses.get_licenses(build_info_dir),
- go_dep.get_go_dep(url, build_info_dir),
- )
- if not sbom_pkg_info.download_url and "private-overlays" not in ebuild:
- raise SBOMPkgInfoError(f"download url not found")
- if not sbom_pkg_info.licenses:
- raise SBOMPkgInfoError(f"license not found")
- sbom_pkg_info.write_to_build_info(build_info_dir)
+ private = False
+ try:
+ package_name = os.path.basename(package_dir)
+ ebuild = os.path.join(build_info_dir, package_name + ".ebuild")
+ repository, category, pf = read_build_info(build_info_dir)
+ if "private" in repository:
+ # Skip private packages.
+ private = True
+ return
+ sbom_pkg_info.download_url = download_url.get_download_url(
+ ebuild, repository, category, pf
+ )
+ sbom_pkg_info.licenses = licenses.get_licenses(build_info_dir)
+ sbom_pkg_info.go_dep = go_dep.get_go_dep(
+ sbom_pkg_info.download_url, build_info_dir
+ )
+ if not sbom_pkg_info.download_url:
+ raise SBOMPkgInfoError(f"download url not found")
+ if not sbom_pkg_info.licenses:
+ sbom_pkg_info.licenses = "NONE"
+ except Exception as e:
+ sbom_pkg_info.err = repr(e)
+ finally:
+ if not private:
+ sbom_pkg_info.write_to_build_info(build_info_dir)
if __name__ == "__main__":
diff --git a/hooks/install/sbom_info_lib/download_url.py b/hooks/install/sbom_info_lib/download_url.py
index 9df3912..0173ffc 100644
--- a/hooks/install/sbom_info_lib/download_url.py
+++ b/hooks/install/sbom_info_lib/download_url.py
@@ -46,6 +46,7 @@
"LLVM_HASH",
"CROS_WORKON_REPO",
"GNOME_ORG_MODULE",
+ "GRUB2_COMMIT",
}
# For packages whose package names are hard to parse or not defined in ebuilds.
PN_REPLACE_DICT = {
@@ -82,7 +83,7 @@
CROS_HOMEPAGE = "HOMEPAGE"
GOB_REPO_DICT = {
"project-lakitu": "https://cos.googlesource.com/cos/overlays/board-overlays/+/master/project-lakitu/",
- "chromiumos-overlay": "https://cos.googlesource.com/third_party/overlays/chromiumos-overlay/+/master/",
+ "chromiumos": "https://cos.googlesource.com/third_party/overlays/chromiumos-overlay/+/master/",
"portage-stable": "https://cos.googlesource.com/third_party/overlays/portage-stable/+/master/",
"eclass-overlay": "https://cos.googlesource.com/third_party/overlays/eclass-overlay/+/master/",
}
@@ -94,6 +95,9 @@
"sys-devel/autoconf-wrapper",
"sys-devel/automake-wrapper",
"dev-python/namespace-zope",
+ "chromeos-base/chromeos-bsp-test-root-lakitu",
+ "dev-python/namespace-jaraco",
+ "dev-python/namespace-google",
# project-lakitu
"app-admin/cgroup-helper",
"app-admin/extensions-manager",
@@ -106,9 +110,11 @@
"chromeos-base/chromeos-bsp-lakitu-common",
"chromeos-base/chromeos-firewall-init-lakitu",
"chromeos-base/chromeos-init-systemd",
+ "chromeos-base/chromeos-init-systemd-dev",
"chromeos-base/cloud-audit-config",
"chromeos-base/cloud-filesystem-init",
"chromeos-base/cloud-network-init",
+ "dev-util/cos-dev-tools",
"net-misc/chrony-config",
"sys-apps/loadpin-trigger",
"sys-apps/system-sysdaemons",
@@ -120,6 +126,9 @@
"chromeos-base/update-policy-embedded",
"dev-util/glib-utils",
"chromeos-base/openssh-server-init",
+ "chromeos-base/autotest-all",
+ "chromeos-base/autotest-client",
+ "chromeos-base/chromeos-ssh-testkeys",
}
@@ -233,19 +242,19 @@
cros_commit = parse_var_from_env(CROS_COMMIT)
if not cros_repo:
cros_repo = [CROS_GIT_HOST_URL] * len(cros_proj)
- if len(cros_proj) != len(cros_commit):
- return res
for i in range(len(cros_proj)):
uri = os.path.join(cros_repo[i], cros_proj[i])
if not is_uri_valid(uri):
continue
- if cros_subtree and cros_subtree[i]:
+ if not cros_commit:
+ res.append(uri)
+ elif cros_subtree and cros_subtree[i]:
subtrees = cros_subtree[i].split(" ")
for subtree in subtrees:
res.append(f"{uri}@{cros_commit[i]}#{subtree}")
else:
res.append(f"{uri}@{cros_commit[i]}")
- return res
+ return ",".join(res)
def get_gcs_name_from_src_uri(regex, content):
@@ -276,9 +285,9 @@
break
gnome_pn = os.getenv(GNOME_PN)
if gnome_pn:
- gcs_names.add(f'{gnome_pn}-{os.getenv["PV"]}')
+ gcs_names.add(f'{gnome_pn}-{os.getenv("PV")}.tar.xz')
gcs_names_src = get_gcs_name_from_src_uri(REGEX_SRC_URI, content)
- if gcs_names:
+ if gcs_names_src:
gcs_names.update(gcs_names_src)
else:
gcs_names.update(get_gcs_name_from_src_uri(REGEX_SRC_URI_PLUS, content))
@@ -330,7 +339,7 @@
def search_homepage():
homepage = os.getenv(CROS_HOMEPAGE)
- if "chromium.googlesource.com" in homepage and is_uri_valid(homepage):
+ if homepage and "chromium.googlesource.com" in homepage and is_uri_valid(homepage):
commit = os.getenv(CROS_COMMIT)
if commit:
return f"{homepage}@{commit}"
@@ -361,18 +370,7 @@
os.environ[var] = ""
-def read_build_info(build_info_dir):
- with open(os.path.join(build_info_dir, "repository"), "r") as f:
- repository = f.read().strip()
- with open(os.path.join(build_info_dir, "CATEGORY"), "r") as f:
- category = f.read().strip()
- with open(os.path.join(build_info_dir, "PF"), "r") as f:
- pf = f.read().strip()
- return repository, category, pf
-
-
-def get_download_url(build_info_dir, ebuild):
- repository, category, pf = read_build_info(build_info_dir)
+def get_download_url(ebuild, repository, category, pf):
if repository == "private-overlays":
return ""
os.environ["CROS_GIT_HOST_URL"] = CROS_GIT_HOST_URL
diff --git a/hooks/install/sbom_info_lib/go_dep.py b/hooks/install/sbom_info_lib/go_dep.py
index da05c9c..79429a1 100644
--- a/hooks/install/sbom_info_lib/go_dep.py
+++ b/hooks/install/sbom_info_lib/go_dep.py
@@ -34,6 +34,8 @@
if url.startswith("gs://"):
subprocess.run(["gsutil", "cp", url, filepath])
else:
+ if not url.startswith("http"):
+ return ""
if url.startswith("https://github.com"):
url = f'{url.replace("@","/archive/")}.tar.gz'
else:
@@ -45,10 +47,14 @@
def get_go_dep(download_url, build_info_dir):
res = set()
+ if not download_url:
+ return res
for url in download_url.split(","):
if url.endswith(".gn"):
continue
filepath = download_src_code(url, build_info_dir)
+ if not filepath:
+ return res
try:
t = tarfile.open(filepath, "r:gz")
for filename in t.getnames():
@@ -65,6 +71,6 @@
if dep:
res.add(dep)
except:
- print(f"{url} is not a .gz file.")
+ pass
os.remove(filepath)
return ",".join(res)
