Merge "Merge cos-sdk into ToT" into main
diff --git a/hooks/install/gen-sbom-package-info.py b/hooks/install/gen-sbom-package-info.py
index 59e5ffd..05bce2a 100755
--- a/hooks/install/gen-sbom-package-info.py
+++ b/hooks/install/gen-sbom-package-info.py
@@ -20,21 +20,28 @@
from sbom_info_lib import download_url
from sbom_info_lib import go_dep
from sbom_info_lib import licenses
+from chromite.lib import osutils
SBOM_INFO_FILE_NAME = "sbom-pkg-info"
class SbomPackageInfo:
- def __init__(self, url, license, go_dep):
- self.download_url = url
- self.licenses = license
- self.go_dep = go_dep
+ def __init__(self):
+ self.download_url = ""
+ self.licenses = ""
+ self.go_dep = ""
+ self.err = ""
def write_to_build_info(self, build_info_dir):
- with open(f"{build_info_dir}/{SBOM_INFO_FILE_NAME}", "w") as f:
- f.write(f"download-url:{self.download_url}\n")
- f.write(f"licenses:{self.licenses}\n")
- f.write(f"go-dep:{self.go_dep}\n")
+ content = (
+ f"download-url:{self.download_url}\n"
+ + f"licenses:{self.licenses}\n"
+ + f"go-dep:{self.go_dep}\n"
+ + f"err:{self.err}\n"
+ )
+ osutils.WriteFile(
+ f"{build_info_dir}/{SBOM_INFO_FILE_NAME}", content, makedirs=True
+ )
class SBOMPkgInfoError(Exception):
@@ -42,22 +49,45 @@
super().__init__(msg)
+def read_build_info(build_info_dir):
+ with open(os.path.join(build_info_dir, "repository"), "r") as f:
+ repository = f.read().strip()
+ with open(os.path.join(build_info_dir, "CATEGORY"), "r") as f:
+ category = f.read().strip()
+ with open(os.path.join(build_info_dir, "PF"), "r") as f:
+ pf = f.read().strip()
+ return repository, category, pf
+
+
def main():
+ sbom_pkg_info = SbomPackageInfo()
package_dir = os.getenv("PORTAGE_BUILDDIR")
build_info_dir = os.path.join(package_dir, "build-info")
- package_name = os.path.basename(package_dir)
- ebuild = os.path.join(build_info_dir, package_name + ".ebuild")
- url = download_url.get_download_url(build_info_dir, ebuild)
- sbom_pkg_info = SbomPackageInfo(
- url,
- licenses.get_licenses(build_info_dir),
- go_dep.get_go_dep(url, build_info_dir),
- )
- if not sbom_pkg_info.download_url and "private-overlays" not in ebuild:
- raise SBOMPkgInfoError(f"download url not found")
- if not sbom_pkg_info.licenses:
- raise SBOMPkgInfoError(f"license not found")
- sbom_pkg_info.write_to_build_info(build_info_dir)
+ private = False
+ try:
+ package_name = os.path.basename(package_dir)
+ ebuild = os.path.join(build_info_dir, package_name + ".ebuild")
+ repository, category, pf = read_build_info(build_info_dir)
+ if "private" in repository:
+ # Skip private packages.
+ private = True
+ return
+ sbom_pkg_info.download_url = download_url.get_download_url(
+ ebuild, repository, category, pf
+ )
+ sbom_pkg_info.licenses = licenses.get_licenses(build_info_dir)
+ sbom_pkg_info.go_dep = go_dep.get_go_dep(
+ sbom_pkg_info.download_url, build_info_dir
+ )
+ if not sbom_pkg_info.download_url:
+ raise SBOMPkgInfoError(f"download url not found")
+ if not sbom_pkg_info.licenses:
+ sbom_pkg_info.licenses = "NONE"
+ except Exception as e:
+ sbom_pkg_info.err = repr(e)
+ finally:
+ if not private:
+ sbom_pkg_info.write_to_build_info(build_info_dir)
if __name__ == "__main__":
diff --git a/hooks/install/sbom_info_lib/download_url.py b/hooks/install/sbom_info_lib/download_url.py
index 9df3912..37b3999 100644
--- a/hooks/install/sbom_info_lib/download_url.py
+++ b/hooks/install/sbom_info_lib/download_url.py
@@ -23,6 +23,7 @@
"gs://chromeos-mirror/gentoo/distfiles/",
"gs://chromeos-localmirror/distfiles/",
]
+GCS_FILE_NOT_FOUND = "One or more URLs matched no objects"
# An allow-list for variables parsed in an ebuild file.
EBUILD_VARS = {
"MY_P",
@@ -46,6 +47,7 @@
"LLVM_HASH",
"CROS_WORKON_REPO",
"GNOME_ORG_MODULE",
+ "GRUB2_COMMIT",
}
# For packages whose package names are hard to parse or not defined in ebuilds.
PN_REPLACE_DICT = {
@@ -82,7 +84,7 @@
CROS_HOMEPAGE = "HOMEPAGE"
GOB_REPO_DICT = {
"project-lakitu": "https://cos.googlesource.com/cos/overlays/board-overlays/+/master/project-lakitu/",
- "chromiumos-overlay": "https://cos.googlesource.com/third_party/overlays/chromiumos-overlay/+/master/",
+ "chromiumos": "https://cos.googlesource.com/third_party/overlays/chromiumos-overlay/+/master/",
"portage-stable": "https://cos.googlesource.com/third_party/overlays/portage-stable/+/master/",
"eclass-overlay": "https://cos.googlesource.com/third_party/overlays/eclass-overlay/+/master/",
}
@@ -94,6 +96,9 @@
"sys-devel/autoconf-wrapper",
"sys-devel/automake-wrapper",
"dev-python/namespace-zope",
+ "chromeos-base/chromeos-bsp-test-root-lakitu",
+ "dev-python/namespace-jaraco",
+ "dev-python/namespace-google",
# project-lakitu
"app-admin/cgroup-helper",
"app-admin/extensions-manager",
@@ -106,9 +111,11 @@
"chromeos-base/chromeos-bsp-lakitu-common",
"chromeos-base/chromeos-firewall-init-lakitu",
"chromeos-base/chromeos-init-systemd",
+ "chromeos-base/chromeos-init-systemd-dev",
"chromeos-base/cloud-audit-config",
"chromeos-base/cloud-filesystem-init",
"chromeos-base/cloud-network-init",
+ "dev-util/cos-dev-tools",
"net-misc/chrony-config",
"sys-apps/loadpin-trigger",
"sys-apps/system-sysdaemons",
@@ -120,6 +127,9 @@
"chromeos-base/update-policy-embedded",
"dev-util/glib-utils",
"chromeos-base/openssh-server-init",
+ "chromeos-base/autotest-all",
+ "chromeos-base/autotest-client",
+ "chromeos-base/chromeos-ssh-testkeys",
}
@@ -233,19 +243,19 @@
cros_commit = parse_var_from_env(CROS_COMMIT)
if not cros_repo:
cros_repo = [CROS_GIT_HOST_URL] * len(cros_proj)
- if len(cros_proj) != len(cros_commit):
- return res
for i in range(len(cros_proj)):
uri = os.path.join(cros_repo[i], cros_proj[i])
if not is_uri_valid(uri):
continue
- if cros_subtree and cros_subtree[i]:
+ if not cros_commit:
+ res.append(uri)
+ elif cros_subtree and cros_subtree[i]:
subtrees = cros_subtree[i].split(" ")
for subtree in subtrees:
res.append(f"{uri}@{cros_commit[i]}#{subtree}")
else:
res.append(f"{uri}@{cros_commit[i]}")
- return res
+ return ",".join(res)
def get_gcs_name_from_src_uri(regex, content):
@@ -276,9 +286,9 @@
break
gnome_pn = os.getenv(GNOME_PN)
if gnome_pn:
- gcs_names.add(f'{gnome_pn}-{os.getenv["PV"]}')
+ gcs_names.add(f'{gnome_pn}-{os.getenv("PV")}.tar.xz')
gcs_names_src = get_gcs_name_from_src_uri(REGEX_SRC_URI, content)
- if gcs_names:
+ if gcs_names_src:
gcs_names.update(gcs_names_src)
else:
gcs_names.update(get_gcs_name_from_src_uri(REGEX_SRC_URI_PLUS, content))
@@ -293,7 +303,7 @@
res = subprocess.run(
["gsutil", "ls", link], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
- if res.stderr:
+ if res.stderr and GCS_FILE_NOT_FOUND in res.stderr.decode("utf-8"):
continue
else:
return res.stdout.decode("utf-8").rstrip()
@@ -330,7 +340,7 @@
def search_homepage():
homepage = os.getenv(CROS_HOMEPAGE)
- if "chromium.googlesource.com" in homepage and is_uri_valid(homepage):
+ if homepage and "chromium.googlesource.com" in homepage and is_uri_valid(homepage):
commit = os.getenv(CROS_COMMIT)
if commit:
return f"{homepage}@{commit}"
@@ -361,18 +371,7 @@
os.environ[var] = ""
-def read_build_info(build_info_dir):
- with open(os.path.join(build_info_dir, "repository"), "r") as f:
- repository = f.read().strip()
- with open(os.path.join(build_info_dir, "CATEGORY"), "r") as f:
- category = f.read().strip()
- with open(os.path.join(build_info_dir, "PF"), "r") as f:
- pf = f.read().strip()
- return repository, category, pf
-
-
-def get_download_url(build_info_dir, ebuild):
- repository, category, pf = read_build_info(build_info_dir)
+def get_download_url(ebuild, repository, category, pf):
if repository == "private-overlays":
return ""
os.environ["CROS_GIT_HOST_URL"] = CROS_GIT_HOST_URL
diff --git a/hooks/install/sbom_info_lib/go_dep.py b/hooks/install/sbom_info_lib/go_dep.py
index da05c9c..79429a1 100644
--- a/hooks/install/sbom_info_lib/go_dep.py
+++ b/hooks/install/sbom_info_lib/go_dep.py
@@ -34,6 +34,8 @@
if url.startswith("gs://"):
subprocess.run(["gsutil", "cp", url, filepath])
else:
+ if not url.startswith("http"):
+ return ""
if url.startswith("https://github.com"):
url = f'{url.replace("@","/archive/")}.tar.gz'
else:
@@ -45,10 +47,14 @@
def get_go_dep(download_url, build_info_dir):
res = set()
+ if not download_url:
+ return res
for url in download_url.split(","):
if url.endswith(".gn"):
continue
filepath = download_src_code(url, build_info_dir)
+ if not filepath:
+ return res
try:
t = tarfile.open(filepath, "r:gz")
for filename in t.getnames():
@@ -65,6 +71,6 @@
if dep:
res.add(dep)
except:
- print(f"{url} is not a .gz file.")
+ pass
os.remove(filepath)
return ",".join(res)
