hooks/install: Use go-licenses to get go dependencies for SBOM
In order to find licenses for each Go dependency,
https://github.com/google/go-licenses is used.
go-licenses is able to fetch collection of libraries
used by the package, directly or transitively. So
we only need to find which directory contains a
main function (not necessarily main.go) and then
run the rool on it to get dependencies and their
licenses of all executables.
This will give us more dependencies than needed
because we sometimes don't compile all the executables.
But more is better than less.
BUG=b/254334533
TEST=presubmite
RELEASE_NOTE=None
Change-Id: I84510ef2ff7b2511b33ca2140b953859f1ea89cc
Reviewed-on: https://cos-review.googlesource.com/c/third_party/platform/crosutils/+/42887
Tested-by: Cusky Presubmit Bot <presubmit@cos-infra-prod.iam.gserviceaccount.com>
Reviewed-on: https://cos-review.googlesource.com/c/third_party/platform/crosutils/+/44367
Reviewed-by: Robert Kolchmeyer <rkolchmeyer@google.com>
diff --git a/hooks/install/sbom_info_lib/go-licenses-template b/hooks/install/sbom_info_lib/go-licenses-template
new file mode 100644
index 0000000..402dbc0
--- /dev/null
+++ b/hooks/install/sbom_info_lib/go-licenses-template
@@ -0,0 +1,3 @@
+{{ range . }}
+{{ .Name }}@{{ .Version }}@{{ .LicenseName }}
+{{ end }}
\ No newline at end of file
diff --git a/hooks/install/sbom_info_lib/go_dep.py b/hooks/install/sbom_info_lib/go_dep.py
index 805eb3f..dc2810a 100644
--- a/hooks/install/sbom_info_lib/go_dep.py
+++ b/hooks/install/sbom_info_lib/go_dep.py
@@ -10,40 +10,95 @@
# GNU General Public License for more details.
#
-# This script is used to find go dependencies
-# of a go pacakge. It reads 'go.mod', 'vendor.mod'
-# or 'vendor.conf' in the source code.
+# This script uses https://github.com/google/go-licenses to find go dependencies.
import os
-import re
-from chromite.lib import cros_build_lib
+import subprocess
+from sbom_info_lib import licenses
+from sbom_info_lib import license_data
-# REGEX_GO_MOD_DEP finds
-# `require (
-# go-pkg1 v1.2.3
-# go-pkg2 v4.5.6 ...
-# )` in go.mod or other mod file.
-REGEX_GO_MOD_DEP = "require \((.*?)\)"
-GO_MOD_DEP_FILE = ["go.mod", "vendor.mod", "vendor.conf"]
+# Max number of processes running in parallel for go-licenses.
+PROCESS_LIMIT = 32
+GO_LICENSES_TEMPLATE = (
+ "/mnt/host/source/src/scripts/hooks/install/sbom_info_lib/go-licenses-template"
+)
+GO_LICENSES_BIN = "/usr/bin/go-licenses"
+
+
+def shell_find(path, name):
+ return subprocess.run(
+ ["find", path, "-type", "f", "-name", name],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+
+def find_go_file(path):
+ return shell_find(path, "*\\.go").stdout
def get_go_dep(src_path):
+ # Check if package contains go file.
+ go_files = find_go_file(src_path)
+ if not go_files:
+ return []
+
res = set()
- args = ["find", src_path, "-type", "f"]
- result = cros_build_lib.run(args, stdout=True, encoding="utf-8")
- files = result.stdout.splitlines()
- for name in files:
- basename = os.path.basename(name)
- if basename not in GO_MOD_DEP_FILE:
+ target_path_list = []
+ go_files_list = go_files.decode("utf-8").split("\n")
+ for go_file_path in go_files_list:
+ if not go_file_path:
continue
- f = open(name, "r")
- content = f.read()
- match = re.findall(REGEX_GO_MOD_DEP, content, re.DOTALL)
- for req in match:
- deps = req.strip().split("\n")
- for dep in deps:
- # remove comments.
- dep = dep.split("//")[0].strip()
- if dep:
- res.add(dep)
+ # go-licenses can fetch collection of libraries
+ # used by the package, directly or transitively.
+ # So only need to run it on main module.
+ with open(go_file_path, "r") as f:
+ if "func main()" not in f.read():
+ continue
+ target_path_list.append(os.path.dirname(go_file_path))
+ start_idx = 0
+ while start_idx < len(target_path_list):
+ p_list = [
+ subprocess.Popen(
+ [
+ GO_LICENSES_BIN,
+ "report",
+ f"{dir_path}/.",
+ "--template",
+ GO_LICENSES_TEMPLATE,
+ ],
+ cwd=dir_path,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ )
+ for dir_path in target_path_list[
+ start_idx : min(start_idx + PROCESS_LIMIT, len(target_path_list))
+ ]
+ ]
+ start_idx += PROCESS_LIMIT
+ for p in p_list:
+ output_bytes, _ = p.communicate()
+ output = output_bytes.decode("utf-8")
+ if not output:
+ continue
+ for line in output.splitlines():
+ if not line:
+ continue
+ # Output format: [<pkg_name>@<pkg_version>@<license_name>]
+ parts = line.split("@")
+ version = parts[1]
+ license = parts[2]
+ # Top level package e.g. google-guest-agent and packages in
+ # vendor/ have version/license "Unknown". They are not needed.
+ if version == "Unknown" or license == "Unknown":
+ continue
+ if license in licenses.LICENSE_MAP:
+ license = licenses.LICENSE_MAP[license]
+ parts[2] = license
+ line = "@".join(parts)
+ elif license not in license_data.SPDX_LICENSES:
+ raise licenses.UnknownLicenseError(
+ f"unknown Go dependency license: {license} in {line}"
+ )
+ res.add(line)
return list(res)
diff --git a/hooks/install/sbom_info_lib/licenses.py b/hooks/install/sbom_info_lib/licenses.py
index c2d9c8e..d430a0a 100644
--- a/hooks/install/sbom_info_lib/licenses.py
+++ b/hooks/install/sbom_info_lib/licenses.py
@@ -29,8 +29,10 @@
"FDL-1.1": "GFDL-1.1-only",
"FDL-1.2": "GFDL-1.2-only",
"GPL-2": "GPL-2.0-only",
+ "GPL-2.0": "GPL-2.0-only",
"GPL-2+": "GPL-2.0-or-later",
"GPL-3": "GPL-3.0-only",
+ "GPL-3.0": "GPL-3.0-only",
"GPL-3+": "GPL-3.0-or-later",
"LGPL-2": "LGPL-2.0-only",
"LGPL-2.1": "LGPL-2.1-only",
