lakitu: create package information file in the image
BUG=b/153744290
TEST=Ran "create_pkg_info_unittest.py". Also see TEST section of
http://go/cos-rev-i/c/cos/autotest-lakitu/+/6120
RELEASE_NOTE=None
Change-Id: I3b88692e82a384ec1afc92750e1cf906fa5139f8
diff --git a/overlay-lakitu/scripts/board_specific_setup.sh b/overlay-lakitu/scripts/board_specific_setup.sh
index cafdd0f..3e001d2 100644
--- a/overlay-lakitu/scripts/board_specific_setup.sh
+++ b/overlay-lakitu/scripts/board_specific_setup.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -12,6 +14,14 @@
# Don't install symbol table for kdump kernel.
INSTALL_MASK+=" /boot/kdump/System.map-*"
+TEMP_PACKAGE_LIST=""
+
+cleanup_temp_package_list() {
+ if [[ -e "${TEMP_PACKAGE_LIST}" ]]; then
+ rm -f "${TEMP_PACKAGE_LIST}"
+ fi
+}
+
# build_image script calls board_setup on the pristine base image.
board_make_image_bootable() {
local -r image="$1"
@@ -23,6 +33,49 @@
fi
}
+create_package_info() {
+ trap cleanup_temp_package_list EXIT
+ TEMP_PACKAGE_LIST="$(mktemp -t package-list.XXXXXXXXXX)"
+
+ # The "emerge" command below generates the list of packages that
+ # virtual/target-os depends on. Its results look like
+ #
+ # ...
+ # [binary R ] app-arch/gzip-1.9 to /build/lakitu/
+ # [binary R ] dev-libs/popt-1.16-r2 to /build/lakitu/
+ # [binary R ] app-emulation/docker-credential-helpers-0.6.3-r1 to /build/lakitu/
+ # ...
+ #
+ # This command line is similar to what ListInstalledPackage function (in
+ # chromite/licensing/licenses_lib.py) does.
+ #
+ # The following "grep" command filters out extra messages to leave the package
+ # list only.
+ #
+ # And the "sed" command extracts the category name, the package name, and the
+ # version from each line. With that, the example above is converted to
+ #
+ # ...
+ # app-arch/gzip-1.9
+ # dev-libs/popt-1.16-r2
+ # app-emulation/docker-credential-helpers-0.6.3-r1
+ # ...
+ "emerge-${BOARD}" \
+ --with-bdeps=n --with-bdeps-auto=n --usepkgonly --emptytree --pretend \
+ --color=n virtual/target-os | \
+ grep --color=never "^\[" | \
+ sed -E 's/\[[^]]+R[^]]+\] (.+) to \/build\/.*/\1/' \
+ > "${TEMP_PACKAGE_LIST}"
+
+ local -r script_root="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+ sudo "${script_root}/create_pkg_info.py" \
+ --input="${TEMP_PACKAGE_LIST}" \
+ --output="${root_fs_dir}"/etc/cos-package-info.json
+
+ cleanup_temp_package_list
+ trap - EXIT
+}
+
write_toolchain_path() {
local -r cros_overlay="/mnt/host/source/src/third_party/chromiumos-overlay"
local -r sdk_ver_file="${cros_overlay}/chromeos/binhost/host/sdk_version.conf"
@@ -37,13 +90,14 @@
sudo tee "${BUILD_DIR}/toolchain_path" > /dev/null
}
-# Moves the given rootfs_file to the given artifact location. The directory
-# containing the rootfs_file is deleted if it becomes empty after this move.
-# If the rootfs_file doesn't exist, put an empty file at the given artifact
-# location.
-export_image_artifact() {
- local rootfs_file="$1"
- local artifact="$2"
+# Moves the given rootfs file (a relative path to "root_fs_dir") to the given
+# artifact location (a relative path to "BUILD_DIR"). The directory containing
+# the rootfs file is deleted if it becomes empty after this move. If the
+# rootfs file doesn't exist, this function puts an empty file at the given
+# artifact location.
+move_for_artifact() {
+ local rootfs_file="${root_fs_dir}/$1"
+ local artifact="${BUILD_DIR}/$2"
if [[ ! -f "${rootfs_file}" ]]; then
touch "${artifact}"
return
@@ -55,64 +109,28 @@
fi
}
-write_toolchain_env() {
- # Create toolchain_env file in BUILD_DIR so that it can be exported
- # as an artifact.
- local artifact="${BUILD_DIR}/toolchain_env"
-
- # File from which kernel compiler information will be copied
- # This file is deleted after copying content to artifact
- local toolchain_env_file="${root_fs_dir}/etc/toolchain_env"
-
- # Copy kernel compiler info to BUILD artifact
- if [[ -f "${toolchain_env_file}" ]]; then
- cp "${toolchain_env_file}" "${artifact}"
- # Remove toolchain_env from image
- sudo rm "${toolchain_env_file}"
- else
- touch "${artifact}"
- fi
+# Creates toolchain_env file in BUILD_DIR so that it can be exported as an
+# artifact.
+export_toolchain_env() {
+ # File that has kernel compiler information.
+ move_for_artifact "etc/toolchain_env" "toolchain_env"
}
-write_kernel_info() {
- # Create kernel_info file in BUILD_DIR so that it can be exported
- # as an artifact.
- local build_artifact="${BUILD_DIR}/kernel_info"
-
- # File from which kernel information will be copied.
- # This file is deleted after copying content to artifact.
- local kernel_info_file="${root_fs_dir}/etc/kernel_info"
-
- # Copy kernel_info to BUILD artifact.
- if [[ -f "${kernel_info_file}" ]]; then
- cp "${kernel_info_file}" "${build_artifact}"
- # Remove kernel_info file from image.
- sudo rm "${kernel_info_file}"
- else
- touch "${build_artifact}"
- fi
+# Creates kernel_info file in BUILD_DIR so that it can be exported as an
+# artifact.
+export_kernel_info() {
+ # File with kernel information.
+ move_for_artifact "etc/kernel_info" "kernel_info"
}
-write_kernel_commit() {
- # Create kernel_commit file in BUILD_DIR so that it can be exported
- # as an artifact.
- local build_artifact="${BUILD_DIR}/kernel_commit"
-
- # File from which kernel commit will be copied.
- # This file is deleted after copying content to artifact.
- local kernel_commit_file="${root_fs_dir}/etc/kernel_commit"
-
- # Copy kernel_commit to BUILD artifact.
- if [[ -f "${kernel_commit_file}" ]]; then
- cp "${kernel_commit_file}" "${build_artifact}"
- # Remove kernel_commit file from image.
- sudo rm "${kernel_commit_file}"
- else
- touch "${build_artifact}"
- fi
+# Creates kernel_commit file in BUILD_DIR so that it can be exported as an
+# artifact.
+export_kernel_commit() {
+ # File with kernel commit ID.
+ move_for_artifact "etc/kernel_commit" "kernel_commit"
}
-# Export default GPU driver version file as an artifact.
+# Exports default GPU driver version file as an artifact.
export_gpu_default_version() {
local -r script_root="$1"
local -r default_driver_file="${script_root}/gpu_default_version"
@@ -126,16 +144,15 @@
# end of building base image.
board_finalize_base_image() {
local -r script_root="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+ create_package_info
write_toolchain_path
- export_image_artifact \
- "${root_fs_dir}/opt/google/src/kernel-src.tar.gz" \
- "${BUILD_DIR}/kernel-src.tar.gz"
- export_image_artifact \
- "${root_fs_dir}/opt/google/src/kernel-headers.tgz" \
- "${BUILD_DIR}/kernel-headers.tgz"
- write_toolchain_env
- write_kernel_info
- write_kernel_commit
+ move_for_artifact "opt/google/src/kernel-src.tar.gz" \
+ "kernel-src.tar.gz"
+ move_for_artifact "opt/google/src/kernel-headers.tgz" \
+ "kernel-headers.tgz"
+ export_toolchain_env
+ export_kernel_info
+ export_kernel_commit
cp "${BOARD_ROOT}/usr/lib/debug/boot/vmlinux" "${BUILD_DIR}/vmlinux"
export_gpu_default_version "${script_root}"
diff --git a/overlay-lakitu/scripts/create_pkg_info.py b/overlay-lakitu/scripts/create_pkg_info.py
new file mode 100755
index 0000000..423545a
--- /dev/null
+++ b/overlay-lakitu/scripts/create_pkg_info.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Create the package information file in JSON format.
+
+To get the package information, use the library at
+https://cos.googlesource.com/cos/tools/+/refs/heads/master/src/pkg/cos/pkg_info.go
+instead of accessing the JSON file directly.
+
+For information on the JSON format, see http://go/cos-package-list-design .
+"""
+
+from __future__ import print_function
+
+import collections
+import getopt
+import json
+import re
+import sys
+
+
+# The following regular expresssions are used for extracting the category name,
+# the package name, the package version without the revision, and the package
+# revision from a string that looks like
+#
+# category_name/package_name-1.2.3.4_beta1_rc2-r1
+#
+# These regular expressions follow the Gentoo package manager specification at
+# https://www.gentoo.org/proj/en/qa/pms.xml . They are similar to what SplitPV
+# function (in chromite/lib/portage_util.py) uses.
+
+CATEGORY_RE = r'(?P<category>\w[\w\+\.\-]*)'
+PACKAGE_RE = r'(?P<package>\w[\w+-]*)'
+VERSION_NUMLETTER_RE = r'\d+(\.\d+)*[a-z]?'
+VERSION_SUFFIX_RE = r'_(pre|p|beta|alpha|rc)\d*'
+VERSION_NO_REV_RE = r'(?P<version_no_rev>%s(%s)*)' % (VERSION_NUMLETTER_RE,
+ VERSION_SUFFIX_RE)
+REVISION_RE = r'-r(?P<revision>\d+)'
+CPV_RE = r'^%s\/%s-%s(%s)?$' % (CATEGORY_RE, PACKAGE_RE,
+ VERSION_NO_REV_RE, REVISION_RE)
+
+
+def PrintHelp():
+ print('usage: create_pkg_list.py --input=<input file> --output=<output file>')
+
+
+def CreateList(input_lines):
+ cpv_re = re.compile(CPV_RE, re.VERBOSE)
+ package_list = []
+ for line in input_lines:
+ match = cpv_re.match(line.strip())
+ if match is not None:
+ package_list.append(match.groupdict())
+ return package_list
+
+
+def WriteJson(package_list, output_file):
+ installed_packages = []
+ for p in package_list:
+ package_info = collections.OrderedDict(
+ [('category', p['category']),
+ ('name', p['package']),
+ ('version', p['version_no_rev'])
+ ]
+ )
+ if 'revision' in p and p['revision'] is not None:
+ package_info['revision'] = p['revision']
+
+ installed_packages.append(package_info)
+
+ result = {}
+ result['installedPackages'] = installed_packages
+ json.dump(result, output_file, indent=4)
+
+ return 0
+
+
+def main(argv):
+ input_fn = ''
+ output_fn = ''
+
+ try:
+ opts, args_left = getopt.getopt(argv, '', ['help', 'input=', 'output='])
+ except getopt.GetoptError:
+ PrintHelp()
+ return -1
+ if len(args_left) != 0:
+ PrintHelp()
+ return -1
+
+ for opt, arg in opts:
+ if opt == '--help':
+ PrintHelp()
+ elif opt == '--input':
+ input_fn = arg
+ elif opt == '--output':
+ output_fn = arg
+
+ if input_fn == '' or output_fn == '':
+ PrintHelp()
+ return -1
+
+ input_lines = []
+ try:
+ with open(input_fn) as input_file:
+ input_lines = input_file.readlines()
+ except OSError:
+ print('error: Failed to open input file: %s' % input_fn)
+ return -1
+ if len(input_lines) == 0:
+ print('error: No input lines')
+ return -1
+
+ package_list = CreateList(input_lines)
+ if len(package_list) == 0:
+ print('error: Empty package list')
+ return -1
+
+ try:
+ with open(output_fn, 'w') as output_file:
+ ret = WriteJson(package_list, output_file)
+ if ret != 0:
+ print('error: Failed to write package list')
+ return ret
+ except OSError:
+ print('error: Failed to open input file: %s' % input_fn)
+ return -1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/overlay-lakitu/scripts/create_pkg_info_unittest.py b/overlay-lakitu/scripts/create_pkg_info_unittest.py
new file mode 100755
index 0000000..25204b9
--- /dev/null
+++ b/overlay-lakitu/scripts/create_pkg_info_unittest.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for create_pkg_info."""
+
+from __future__ import print_function
+
+import io
+import unittest
+
+import create_pkg_info
+
+
+class CreatePkgInfoTest(unittest.TestCase):
+ """All the tests."""
+
+ INPUT_LINES = [
+ 'app-arch/gzip-1.9',
+ 'dev-libs/popt-1.16-r2',
+ 'app-emulation/docker-credential-helpers-0.6.3-r1',
+ '_not.real-category1+/_not-real_package1-12.34.56.78',
+ ' _not.real-category1+/_not-real_package2-12.34.56.78-r26 ',
+ ' _not.real-category1+/_not-real_package3-12.34.56.78_rc3 ',
+ ' _not.real-category1+/_not-real_package4-12.34.56.78_rc3-r26 ',
+ ' _not.real-category1+/_not-real_package5-12.34.56.78_pre2_rc3-r26 ',
+ '_not.real-category2+/_not-real_package1-12.34.56.78q',
+ '_not.real-category2+/_not-real_package2-12.34.56.78q-r26',
+ '_not.real-category2+/_not-real_package3-12.34.56.78q_rc3',
+ '_not.real-category2+/_not-real_package4-12.34.56.78q_rc3-r26',
+ '_not.real-category2+/_not-real_package5-12.34.56.78q_pre2_rc3-r26'
+ ]
+
+ EXPECTED_LIST = [
+ {'category': 'app-arch', 'package': 'gzip', 'version_no_rev': '1.9',
+ 'revision': None},
+ {'category': 'dev-libs', 'package': 'popt', 'version_no_rev': '1.16',
+ 'revision': '2'},
+ {'category': 'app-emulation', 'package': 'docker-credential-helpers',
+ 'version_no_rev': '0.6.3', 'revision': '1'},
+ {'category': '_not.real-category1+', 'package': '_not-real_package1',
+ 'version_no_rev': '12.34.56.78', 'revision': None},
+ {'category': '_not.real-category1+', 'package': '_not-real_package2',
+ 'version_no_rev': '12.34.56.78', 'revision': '26'},
+ {'category': '_not.real-category1+', 'package': '_not-real_package3',
+ 'version_no_rev': '12.34.56.78_rc3', 'revision': None},
+ {'category': '_not.real-category1+', 'package': '_not-real_package4',
+ 'version_no_rev': '12.34.56.78_rc3', 'revision': '26'},
+ {'category': '_not.real-category1+', 'package': '_not-real_package5',
+ 'version_no_rev': '12.34.56.78_pre2_rc3', 'revision': '26'},
+ {'category': '_not.real-category2+', 'package': '_not-real_package1',
+ 'version_no_rev': '12.34.56.78q', 'revision': None},
+ {'category': '_not.real-category2+', 'package': '_not-real_package2',
+ 'version_no_rev': '12.34.56.78q', 'revision': '26'},
+ {'category': '_not.real-category2+', 'package': '_not-real_package3',
+ 'version_no_rev': '12.34.56.78q_rc3', 'revision': None},
+ {'category': '_not.real-category2+', 'package': '_not-real_package4',
+ 'version_no_rev': '12.34.56.78q_rc3', 'revision': '26'},
+ {'category': '_not.real-category2+', 'package': '_not-real_package5',
+ 'version_no_rev': '12.34.56.78q_pre2_rc3', 'revision': '26'}
+ ]
+
+ EXPECTED_JSON = """{
+ "installedPackages": [
+ {
+ "category": "app-arch",
+ "name": "gzip",
+ "version": "1.9"
+ },
+ {
+ "category": "dev-libs",
+ "name": "popt",
+ "version": "1.16",
+ "revision": "2"
+ },
+ {
+ "category": "app-emulation",
+ "name": "docker-credential-helpers",
+ "version": "0.6.3",
+ "revision": "1"
+ },
+ {
+ "category": "_not.real-category1+",
+ "name": "_not-real_package1",
+ "version": "12.34.56.78"
+ },
+ {
+ "category": "_not.real-category1+",
+ "name": "_not-real_package2",
+ "version": "12.34.56.78",
+ "revision": "26"
+ },
+ {
+ "category": "_not.real-category1+",
+ "name": "_not-real_package3",
+ "version": "12.34.56.78_rc3"
+ },
+ {
+ "category": "_not.real-category1+",
+ "name": "_not-real_package4",
+ "version": "12.34.56.78_rc3",
+ "revision": "26"
+ },
+ {
+ "category": "_not.real-category1+",
+ "name": "_not-real_package5",
+ "version": "12.34.56.78_pre2_rc3",
+ "revision": "26"
+ },
+ {
+ "category": "_not.real-category2+",
+ "name": "_not-real_package1",
+ "version": "12.34.56.78q"
+ },
+ {
+ "category": "_not.real-category2+",
+ "name": "_not-real_package2",
+ "version": "12.34.56.78q",
+ "revision": "26"
+ },
+ {
+ "category": "_not.real-category2+",
+ "name": "_not-real_package3",
+ "version": "12.34.56.78q_rc3"
+ },
+ {
+ "category": "_not.real-category2+",
+ "name": "_not-real_package4",
+ "version": "12.34.56.78q_rc3",
+ "revision": "26"
+ },
+ {
+ "category": "_not.real-category2+",
+ "name": "_not-real_package5",
+ "version": "12.34.56.78q_pre2_rc3",
+ "revision": "26"
+ }
+ ]
+}"""
+
+ def __init__(self, *args, **kwargs):
+ unittest.TestCase.__init__(self, *args, **kwargs)
+ self.maxDiff = None
+
+ def testCreateList(self):
+ self.assertEqual(
+ create_pkg_info.CreateList(self.INPUT_LINES),
+ self.EXPECTED_LIST)
+
+ def testWriteJson(self):
+ mock_file = io.StringIO()
+ create_pkg_info.WriteJson(self.EXPECTED_LIST, mock_file)
+ self.assertEqual(mock_file.getvalue(), self.EXPECTED_JSON)
+ mock_file.close()
+
+
+if __name__ == '__main__':
+ unittest.main()
