| #!/usr/bin/python3 |
| # |
| # Copyright (C) 2016 The Android Open-Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| from __future__ import print_function |
| |
| import argparse |
| import atexit |
| import getpass |
| import hashlib |
| import itertools |
| import logging |
| import os |
| import re |
| import shutil |
| import string |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| import xml.etree.cElementTree as ElementTree |
| import zipfile |
| |
| import lib.build_artifact_fetcher |
| import lib.util |
| |
| |
| _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| |
| _EXPECTED_TARGET_PRODUCTS = { |
| '^x86': ('cheets_x86', 'cheets_x86_64', 'bertha_x86', 'bertha_x86_64'), |
| '^arm': ('cheets_arm', 'bertha_arm',), |
| '^aarch64$': ('cheets_arm', 'bertha_arm',), |
| } |
| _ANDROID_ROOT = '/opt/google/containers/android' |
| _ANDROID_ROOT_STATEFUL = os.path.join('/usr/local', |
| os.path.relpath(_ANDROID_ROOT, '/')) |
| _CONTAINER_INSTANCE_ROOT_WILDCARD = '/run/containers/android*' |
| _CONTAINER_ROOT = os.path.join(_ANDROID_ROOT, 'rootfs', 'root') |
| _RSYNC_COMMAND = ['rsync', '--inplace', '-v', '--progress'] |
| _SCP_COMMAND = ['scp'] |
| |
| _BUILD_FILENAME = string.Template('${product}-img-${build_id}.zip') |
| _BUILD_TARGET = string.Template('${product}-${build_variant}') |
| |
| _CHROMEOS_ARC_ANDROID_SDK_VERSION = 'CHROMEOS_ARC_ANDROID_SDK_VERSION=' |
| |
| _GENERIC_DEVICE = 'generic_%(arch)s_%(product)s' |
| _RO_BUILD_TYPE = 'ro.build.type=' |
| _RO_BUILD_VERSION_SDK = 'ro.build.version.sdk=' |
| _RO_PRODUCT_DEVICE = 'ro.product.device=' |
| |
| _ANDROID_REL_KEY_SIGNATURE_SUBSTRING = ( |
| '55b390dd7fdb9418631895d5f759f30112687ff621410c069308a') |
| _APK_KEY_DEBUG = 'debug-key' |
| _APK_KEY_RELEASE = 'release-key' |
| _APK_KEY_UNKNOWN = 'unknown' |
| _GMS_CORE_PACKAGE_NAME = 'com.google.android.gms' |
| |
| _ANDROID_SDK_MAPPING = { |
| 23: "MNC (API 23)", |
| 24: "NYC (API 24)", |
| 25: "NYC_MR1 (API 25)", |
| 26: "OC (API 26)", |
| } |
| |
| # Bytes per Megabyte. |
| _MB = 1024**2 |
| |
| |
| class RemoteProxy(object): |
| """Proxy class to run command line on the remote test device.""" |
| |
| def __init__(self, remote, dryrun): |
| self._remote = remote |
| self._dryrun = dryrun |
| self._sync_command = ( |
| _RSYNC_COMMAND if self._has_rsync_on_remote_device() else _SCP_COMMAND) |
| |
| def check_call(self, remote_command): |
| """Runs |remote_command| on the remote test device via ssh.""" |
| command = self.get_ssh_commandline(remote_command) |
| lib.util.check_call(dryrun=self._dryrun, *command) |
| |
| def check_output(self, remote_command): |
| """Runs |remote_command| on the remote test device via ssh, and returns |
| its output.""" |
| command = self.get_ssh_commandline(remote_command) |
| return lib.util.check_output(dryrun=self._dryrun, *command) |
| |
| def sync(self, file_list, dest_dir): |
| """Copies |file_list| to the |dest_dir| on the remote test device.""" |
| target = 'root@%s:%s' % (self._remote, dest_dir) |
| command = self._sync_command + file_list + [target] |
| lib.util.check_call(dryrun=self._dryrun, *command) |
| |
| def read(self, src_path): |
| """Gets the contents of |src_path| from the remote test device.""" |
| return self.check_output('/bin/cat %s' % src_path) |
| |
| def write(self, contents, dest_path): |
| """Writes |contents| into |dest_path| on the remote test device.""" |
| with tempfile.NamedTemporaryFile(mode='w') as f: |
| f.write(contents) |
| self.push(f.name, dest_path) |
| |
| def push(self, source_path, dest_path): |
| """Pushes |source_path| on the host, to |dest_path| on the remote test |
| device. |
| |
| Args: |
| source_path: Host file path to be pushed. |
| dest_path: Path to the destination location on the remote test device. |
| """ |
| target = 'root@%s:%s' % (self._remote, dest_path) |
| command = _SCP_COMMAND + [source_path, target] |
| lib.util.check_call(dryrun=self._dryrun, *command) |
| |
| def pull(self, source_path, dest_path): |
| """Pulls |source_path| from the remote test device, to |dest_path| on the |
| host. |
| |
| Args: |
| source_path: Remote test device file path to be pulled. |
| dest_path: Path to the destination location on the host. |
| """ |
| target = 'root@%s:%s' % (self._remote, source_path) |
| command = _SCP_COMMAND + [target, dest_path] |
| return lib.util.check_call(dryrun=self._dryrun, *command) |
| |
| def get_ssh_commandline(self, remote_command): |
| return ['ssh', 'root@' + self._remote, remote_command] |
| |
| def _has_rsync_on_remote_device(self): |
| command = self.get_ssh_commandline('which rsync') |
| logging.debug('Calling: %s', lib.util.get_command_str(command)) |
| # Always return true for --dryrun. |
| return self._dryrun or subprocess.call(command) == 0 |
| |
| |
| class TemporaryDirectory(object): |
| """A context object that has a temporary directory with the same lifetime.""" |
| |
| def __init__(self): |
| self.name = None |
| |
| def __enter__(self): |
| self.name = tempfile.mkdtemp() |
| return self |
| |
| def __exit__(self, exception_type, exception_value, traceback): |
| lib.util.check_call('rm', '-rf', self.name, sudo=True) |
| |
| |
| class MountWrapper(object): |
| """A context object that mounts an image during the lifetime.""" |
| |
| def __init__(self, image_path, mountpoint): |
| self._image_path = image_path |
| self._mountpoint = mountpoint |
| |
| def __enter__(self): |
| lib.util.check_call('/bin/mount', '-o', 'loop', self._image_path, |
| self._mountpoint, sudo=True) |
| return self |
| |
| def __exit__(self, exception_type, exception_value, traceback): |
| try: |
| lib.util.check_call('/bin/umount', self._mountpoint, sudo=True) |
| except Exception: |
| if not exception_type: |
| raise |
| # Instead of propagate the exception, record the one from exit body. |
| logging.exception('Failed to umount ' + self._mountpoint) |
| |
| |
| def _is_shift_guid_needed(root): |
| """Returns True if the UID/GID in needs to be shifted.""" |
| # Check just the root directory. If it is root, then shift is needed. |
| # TODO(b/65117245): Remove this, when build-time UID/GID shift is completed. |
| return os.stat(root).st_uid == 0 |
| |
| |
| class Simg2img(object): |
| """Wrapper class of simg2img""" |
| |
| def __init__(self, simg2img_path, dryrun): |
| self._path = simg2img_path |
| self._dryrun = dryrun |
| |
| def convert(self, src, dest): |
| """Converts the image to the raw image by simg2img command line. |
| |
| If |dryrun| is set, does not execute the commandline. |
| """ |
| lib.util.check_call(self._path, src, dest, dryrun=self._dryrun) |
| |
| |
| def _get_remote_device_vendor_image(remote_proxy, target_dir, dryrun, |
| unsquashfs_path): |
| """Fetches the contents of the remote vendor partition into target_dir. |
| |
| Args: |
| remote_proxy: RemoteProxy instance for the remote test device. |
| target_dir: Local directory where the vendor contents will be written. |
| This directory should already exist. |
| dryrun: If set, this function doesn't download anything. |
| unsquashfs_path: Path to the unsquashfs binary. Needed if the device |
| has a squashfs vendor image. |
| """ |
| if dryrun: |
| logging.debug('Skipping vendor download') |
| return |
| |
| remote_vendor_image = os.path.join(_ANDROID_ROOT, 'vendor.raw.img') |
| with tempfile.NamedTemporaryFile(prefix='vendor_', suffix='.raw.img', |
| mode='w') as local_image: |
| remote_proxy.pull(remote_vendor_image, local_image.name) |
| _extract_files(unsquashfs_path, target_dir, local_image.name, []) |
| |
| def _verify_machine_arch(remote_proxy, target_product, dryrun): |
| """Verifies if the data being pushed is build for the target architecture. |
| |
| Args: |
| remote_proxy: RemoteProxy instance for the remote test device. |
| target_product: Target product name of the image being pushed. This is |
| usually set by "lunch" command. E.g. "cheets_x86" or "cheets_arm". |
| dryrun: If set, this function assumes the machine architectures match. |
| |
| Raises: |
| AssertionError: If the pushing image does not match to the remote test |
| device. |
| """ |
| if dryrun: |
| logging.debug('Pretending machine architectures match') |
| return |
| remote_arch = remote_proxy.check_output('uname -m') |
| for arch_pattern, expected_set in _EXPECTED_TARGET_PRODUCTS.items(): |
| if re.search(arch_pattern, remote_arch): |
| expected = itertools.chain.from_iterable( |
| (expected, 'aosp_' + expected, 'sdk_google_' + expected) |
| for expected in expected_set) |
| assert target_product in expected, ( |
| ('Architecture mismatch: Deploying \'%s\' to \'%s\' seems incorrect.' |
| % (target_product, remote_arch))) |
| return |
| logging.warning('Unknown remote machine type \'%s\'. Skipping ' |
| 'architecture sanity check.', remote_arch) |
| |
| |
| def _convert_images(simg2img, out, product, push_vendor_image, |
| mksquashfs_path, unsquashfs_path, |
| shift_uid_py_path, dryrun=False, loglevel='INFO', |
| vendor_merge_dir=None): |
| """Converts the images being pushed to the raw images. |
| |
| Returns: |
| A tuple of (large_file_list, file_list). Each list consists of paths of |
| converted files. |
| """ |
| result = [] |
| result_large = [] |
| |
| system_raw_img = os.path.join(out, 'system.raw.img') |
| simg2img.convert(os.path.join(out, 'system.img'), system_raw_img) |
| file_contexts_path = None |
| if not dryrun: |
| with ContainerImageEditor(mksquashfs_path, unsquashfs_path, system_raw_img, |
| '/', out, loglevel=loglevel) as e: |
| file_contexts_path = e.file_contexts_path |
| if 'x86' in product: |
| logging.debug('Creating \'system/lib/arm\' dir and houdini symlinks ' |
| 'in the system image') |
| # Create system/lib/arm dir |
| dir_name = os.path.join(e.tmp_dir_name, 'system/lib/arm') |
| logging.debug('Creating directory: %s', dir_name) |
| lib.util.check_call('mkdir', '-p', dir_name, sudo=True) |
| # Create a symlink: system/bin/houdini --> /vendor/bin/houdini |
| lib.util.check_call('ln', '-sf', '/vendor/bin/houdini', |
| os.path.join(e.tmp_dir_name, 'system/bin/houdini'), |
| sudo=True) |
| # Create a symlink: |
| # system/lib/libhoudini.so --> /vendor/lib/libhoudini.so |
| lib.util.check_call('ln', '-sf', '/vendor/lib/libhoudini.so', |
| os.path.join(e.tmp_dir_name, |
| 'system/lib/libhoudini.so'), |
| sudo=True) |
| |
| # TODO(b/76094710): Remove the system libdrm.so until we have a version |
| # compatible with the Chrome OS version. |
| for lib_path in ('system/lib/libdrm.so', 'system/lib64/libdrm.so'): |
| rm_path = os.path.join(e.tmp_dir_name, lib_path) |
| lib.util.check_call('rm', '-f', rm_path, sudo=True) |
| |
| # TODO(b/65117245): This needs to be part of the build. |
| # Shift the UIDs/GIDs. |
| if _is_shift_guid_needed(e.tmp_dir_name): |
| logging.info('Detected the UIDs/GIDs are not shifted in system.img. ' |
| 'Shifting...') |
| lib.util.check_call(shift_uid_py_path, e.tmp_dir_name, sudo=True) |
| |
| result_large.append(system_raw_img) |
| |
| if push_vendor_image: |
| vendor_raw_img = os.path.join(out, 'vendor.raw.img') |
| simg2img.convert(os.path.join(out, 'vendor.img'), vendor_raw_img) |
| # TODO(b/65117245): This needs to be part of the build. |
| if not dryrun: |
| with ContainerImageEditor(mksquashfs_path, unsquashfs_path, |
| vendor_raw_img, 'vendor', out, |
| file_contexts_path=file_contexts_path, |
| loglevel=loglevel) as e: |
| # TODO(b/76094710): Remove the Android-built vendor libdrm.so until we |
| # have a version compatible with the Chrome OS version. |
| for lib_path in ('lib/libdrm.so', 'lib64/libdrm.so'): |
| rm_path = os.path.join(e.tmp_dir_name, lib_path) |
| lib.util.check_call('rm', '-f', rm_path, sudo=True) |
| |
| # Merge files from the downloaded vendor image. Only files not |
| # present in the Android build output are added. |
| logging.info('Merging device vendor partition into new image.') |
| lib.util.check_call('rsync', '--ignore-existing', '--whole-file', |
| '--verbose', '--recursive', '--links', '--perms', |
| '--inplace', vendor_merge_dir + '/', |
| e.tmp_dir_name, sudo=True) |
| lib.util.check_call('chmod', '0755', e.tmp_dir_name, sudo=True) |
| |
| # Remove the build fingerprint from the vendor build.prop (if it |
| # exists) to avoid vendor signature mismatches. The same is done in |
| # the Chrome OS side. |
| lib.util.check_call('/bin/sed', '-i', |
| r'/^ro\.vendor\.build\.fingerprint=/d', |
| os.path.join(e.tmp_dir_name, 'build.prop'), |
| sudo=True) |
| |
| # Shift the UIDs/GIDs. |
| if _is_shift_guid_needed(e.tmp_dir_name): |
| logging.info('Detected the UIDs/GIDs are not shifted in vendor.img. ' |
| 'Shifting...') |
| lib.util.check_call(shift_uid_py_path, e.tmp_dir_name, |
| sudo=True) |
| result.append(vendor_raw_img) |
| |
| return (result_large, result) |
| |
| |
| def _update_build_fingerprint(remote_proxy, build_fingerprint): |
| """Updates CHROMEOS_ARC_VERSION in /etc/lsb-release. |
| |
| Args: |
| remote_proxy: RemoteProxy instance connected to the test device. |
| build_fingerprint: The version code which should be embedded into |
| /etc/lsb-release. |
| """ |
| if not build_fingerprint: |
| logging.warning( |
| 'Skipping version update. ARC version will be reported incorrectly') |
| return |
| |
| # Replace the ARC version on disk with what we're pushing there. |
| logging.info('Updating CHROMEOS_ARC_VERSION...') |
| remote_proxy.check_call(' '.join([ |
| '/bin/sed', '-i', |
| # Note: we assume build_fingerprint does not contain any char which |
| # needs to be escaped. |
| r'"s/^\(CHROMEOS_ARC_VERSION=\).*/\1%(_BUILD_FINGERPRINT)s/"', |
| '/etc/lsb-release' |
| ]) % {'_BUILD_FINGERPRINT': build_fingerprint}) |
| |
| |
| def _get_remote_device_android_sdk_version(remote_proxy, dryrun): |
| """ Returns the Android SDK version on the remote device. |
| |
| Args: |
| remote_proxy: RemoteProxy instance for the remote test device. |
| dryrun: If set, this function assumes Android SDK version is 1. |
| """ |
| if dryrun: |
| logging.debug('Pretending target device\'s Android SDK version is 1') |
| return 1 |
| try: |
| line = remote_proxy.check_output( |
| 'grep ^%s /etc/lsb-release' % |
| _CHROMEOS_ARC_ANDROID_SDK_VERSION).strip() |
| except subprocess.CalledProcessError: |
| logging.exception('Failed to inspect /etc/lsb-release remotely') |
| return None |
| |
| if not line.startswith(_CHROMEOS_ARC_ANDROID_SDK_VERSION): |
| logging.warning('Failed to find the correct string format.\n' |
| 'Expected format: "%s"\nActual string: "%s"', |
| _CHROMEOS_ARC_ANDROID_SDK_VERSION, line) |
| return None |
| |
| android_sdk_version = int( |
| line[len(_CHROMEOS_ARC_ANDROID_SDK_VERSION):].strip()) |
| logging.debug('Target device\'s Android SDK version: %d', |
| android_sdk_version) |
| return android_sdk_version |
| |
| |
| def _verify_android_sdk_version(remote_proxy, provider, dryrun): |
| """Verifies if the Android SDK versions of the pushing image and the test |
| device are the same. |
| |
| Args: |
| remote_proxy: RemoteProxy instance for the remote test device. |
| provider: Android image provider. |
| dryrun: If set, this function assumes Android SDK versions match. |
| |
| Raises: |
| AssertionError: If the Android SDK version of pushing image does not |
| match the Android SDK version on the remote test device. |
| """ |
| if dryrun: |
| logging.debug('Pretending Android SDK versions match') |
| return |
| logging.debug('New image\'s Android SDK version: %d', |
| provider.get_build_version_sdk()) |
| |
| device_android_sdk_version = _get_remote_device_android_sdk_version( |
| remote_proxy, dryrun) |
| |
| if device_android_sdk_version is None: |
| if not boolean_prompt('Unable to determine the target device\'s Android ' |
| 'SDK version. Continue?', default=False): |
| sys.exit(1) |
| else: |
| if device_android_sdk_version != provider.get_build_version_sdk(): |
| if not boolean_prompt('Android SDK versions do not match. The target ' |
| 'device has {}, while the new image is {}. ' |
| 'Continue?'.format( |
| _android_sdk_version_to_string( |
| device_android_sdk_version), |
| _android_sdk_version_to_string( |
| provider.get_build_version_sdk())), |
| default=False): |
| sys.exit(1) |
| |
| |
| def _android_sdk_version_to_string(android_sdk_version): |
| """Converts the |android_sdk_version| to a human readable string |
| |
| Args: |
| android_sdk_version: The Android SDK version number as a string |
| """ |
| return _ANDROID_SDK_MAPPING.get( |
| android_sdk_version, |
| 'Unknown SDK Version (API {})'.format(android_sdk_version)) |
| |
| |
| def _get_selinux_file_contexts_contents(out): |
| """Returns the final contents of the SELinux file_contexts file.""" |
| contents = [] |
| for filename in ('file_contexts', 'plat_file_contexts', |
| 'nonplat_file_contexts', 'vendor_file_contexts'): |
| path = os.path.join(out, 'root', filename) |
| # Some files are always expected to be missing due to not being present in |
| # the branch. |
| if not os.path.exists(path): |
| logging.debug('Skipping %s since it is missing', path) |
| continue |
| with open(path, 'r') as f: |
| contents.append(f.read()) |
| return '\n'.join(contents) |
| |
| |
| def _is_selinux_file_contexts_updated(remote_proxy, out, chromeos_file_contexts, |
| dryrun): |
| """Returns True if SELinux file_contexts is updated.""" |
| if dryrun: |
| logging.debug('Pretending file_contexts is not updated in dryrun mode') |
| return False |
| if chromeos_file_contexts: |
| # If |chromeos_file_contexts| is explicitly specified, push it. |
| return True |
| remote_file_contexts_sha1, _ = remote_proxy.check_output( |
| 'sha1sum /etc/selinux/arc/contexts/files/android_file_contexts').split() |
| file_contexts_contents = _get_selinux_file_contexts_contents(out) |
| # hashlib expects bytes, not str. |
| host_file_contexts_sha1 = hashlib.sha1( |
| file_contexts_contents.encode('utf-8')).hexdigest() |
| return remote_file_contexts_sha1 != host_file_contexts_sha1 |
| |
| |
| def _update_selinux_file_contexts(remote_proxy, chromeos_file_contexts, out): |
| """Updates the selinux file_contexts file.""" |
| android_file_contexts_contents = _get_selinux_file_contexts_contents(out) |
| remote_proxy.write(android_file_contexts_contents, |
| '/etc/selinux/arc/contexts/files/android_file_contexts') |
| file_contexts_contents = [] |
| for line in android_file_contexts_contents.split('\n'): |
| line = line.strip() |
| if not line or line.startswith('#'): |
| continue |
| file_contexts_contents.append( |
| '%s/rootfs/root%s' % (_ANDROID_ROOT, line)) |
| |
| remote_file_contexts_path = '/etc/selinux/arc/contexts/files/file_contexts' |
| if chromeos_file_contexts: |
| with open(chromeos_file_contexts, 'r') as f: |
| chromeos_file_contexts_contents = f.read() |
| else: |
| remote_file_contexts_contents = remote_proxy.read(remote_file_contexts_path) |
| try: |
| # This string comes from |
| # private-overlays/project-cheets-private/chromeos-base/\ |
| # android-container-<VERSION>/files/chromeos_file_contexts |
| header_idx = remote_file_contexts_contents.index( |
| '# Chrome OS file contexts') |
| except ValueError: |
| # The header was missing. Will concat the whole file. |
| logging.warning('Could not find Chrome OS header in %s. ' |
| 'Will use the whole file', remote_file_contexts_path) |
| header_idx = 0 |
| chromeos_file_contexts_contents = remote_file_contexts_contents[header_idx:] |
| file_contexts_contents.append(chromeos_file_contexts_contents) |
| remote_proxy.write('\n'.join(file_contexts_contents), |
| remote_file_contexts_path) |
| |
| |
| def _is_selinux_policy_updated(remote_proxy, out, dryrun): |
| """Returns True if SELinux policy is updated.""" |
| if dryrun: |
| logging.debug('Pretending sepolicy is not updated in dryrun mode') |
| return False |
| remote_sepolicy_sha1, _ = remote_proxy.check_output( |
| 'sha1sum /etc/selinux/arc/policy/policy.30').split() |
| with open(os.path.join(out, 'root', 'sepolicy'), 'rb') as f: |
| host_sepolicy_sha1 = hashlib.sha1(f.read()).hexdigest() |
| return remote_sepolicy_sha1 != host_sepolicy_sha1 |
| |
| |
| def _update_selinux_policy(remote_proxy, out): |
| """Updates the selinux policy file.""" |
| remote_proxy.push(os.path.join(out, 'root', 'sepolicy'), |
| '/etc/selinux/arc/policy/policy.30') |
| |
| |
| def _remount_rootfs_as_writable(remote_proxy): |
| """Remounts root file system to make it writable.""" |
| remote_proxy.check_call('mount -o remount,rw /') |
| |
| |
| def _get_free_space(remote_proxy): |
| """Gets the number of free bytes in the root partition.""" |
| return int(remote_proxy.check_output( |
| 'echo $(( ' |
| ' $(df --output=avail --local --block-size 1 / | tail -n1) + ' |
| ' $(du --bytes /opt/google/containers/android/system.raw.img | ' |
| ' awk \'{print $1}\') ' |
| '))')) |
| |
| |
| def _update_container_config(remote_proxy, container_config): |
| """Updates the container config file.""" |
| remote_proxy.push(container_config, |
| '/opt/google/containers/android/config.json') |
| |
| |
| def boolean_prompt(prompt, default=True, true_value='yes', false_value='no', |
| prolog=None): |
| """Helper function for processing boolean choice prompts. |
| |
| Args: |
| prompt: The question to present to the user. |
| default: Boolean to return if the user just presses enter. |
| true_value: The text to display that represents a True returned. |
| false_value: The text to display that represents a False returned. |
| prolog: The text to display before prompt. |
| |
| Returns: |
| True or False. |
| """ |
| true_value, false_value = true_value.lower(), false_value.lower() |
| true_text, false_text = true_value, false_value |
| if true_value == false_value: |
| raise ValueError('true_value and false_value must differ: got %r' |
| % true_value) |
| |
| if default: |
| true_text = true_text[0].upper() + true_text[1:] |
| else: |
| false_text = false_text[0].upper() + false_text[1:] |
| |
| prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text)) |
| |
| if prolog: |
| prompt = ('\n%s\n%s' % (prolog, prompt)) |
| |
| while True: |
| try: |
| response = input(prompt).lower() |
| except EOFError: |
| # If the user hits CTRL+D, or stdin is disabled, use the default. |
| print(file=sys.stderr) |
| response = None |
| except KeyboardInterrupt: |
| # If the user hits CTRL+C, just exit the process. |
| print(file=sys.stderr) |
| print('CTRL+C detected; exiting', file=sys.stderr) |
| raise |
| |
| if not response: |
| return default |
| if true_value.startswith(response): |
| if not false_value.startswith(response): |
| return True |
| # common prefix between the two... |
| elif false_value.startswith(response): |
| return False |
| |
| |
| def _disable_rootfs_verification(force, remote_proxy): |
| make_dev_ssd_path = \ |
| '/usr/libexec/debugd/helpers/dev_features_rootfs_verification' |
| make_dev_ssd_command = remote_proxy.get_ssh_commandline(make_dev_ssd_path) |
| logging.info('Detected that the device has rootfs verification enabled.') |
| logging.info('This script can remove the rootfs verification using `%s`, ' |
| 'which requires that the device is rebooted afterwards.', |
| lib.util.get_command_str(make_dev_ssd_command)) |
| if not force: |
| logging.info('Automatically remove rootfs verification and skip this ' |
| 'prompt by specifying --force.') |
| if not boolean_prompt('Remove rootfs verification?', default=False): |
| return False |
| remote_proxy.check_call(make_dev_ssd_path) |
| reboot_time = time.time() |
| remote_proxy.check_call('reboot') |
| logging.debug('Waiting up to 10 seconds for the machine to reboot') |
| for _ in range(10): |
| time.sleep(1) |
| try: |
| device_boot_time = remote_proxy.check_output('grep btime /proc/stat | ' + |
| 'cut -d" " -f2') |
| if int(device_boot_time) >= reboot_time: |
| return True |
| except subprocess.CalledProcessError: |
| pass |
| logging.error('Failed to detect whether the device had successfully ' |
| 'rebooted') |
| return False |
| |
| |
| def _stop_ui(remote_proxy): |
| remote_proxy.check_call('\n'.join([ |
| # Unmount the container root/vendor and root if necessary. This also |
| # stops UI. |
| 'stop ui || true', |
| 'stop arc-system-mount || true', |
| ])) |
| |
| |
| class ImageUpdateMode(object): |
| """Context object to manage remote host writable status.""" |
| |
| def __init__(self, remote_proxy, is_selinux_policy_updated, push_to_stateful, |
| clobber_data, force): |
| self._remote_proxy = remote_proxy |
| self._is_selinux_policy_updated = is_selinux_policy_updated |
| self._push_to_stateful = push_to_stateful |
| self._clobber_data = clobber_data |
| self._force = force |
| |
| def __enter__(self): |
| logging.info('Setting up ChromeOS device to image-writable...') |
| |
| if self._clobber_data: |
| self._remote_proxy.check_call( |
| 'if [ -e %(ANDROID_ROOT_WILDCARD)s/root/data ]; then' |
| ' find %(ANDROID_ROOT_WILDCARD)s/root/data' |
| ' %(ANDROID_ROOT_WILDCARD)s/root/cache -mindepth 1 -delete;' |
| ' kill -9 `cat %(ANDROID_ROOT_WILDCARD)s/container.pid`;' |
| 'fi' % {'ANDROID_ROOT_WILDCARD': _CONTAINER_INSTANCE_ROOT_WILDCARD}) |
| |
| _stop_ui(self._remote_proxy) |
| try: |
| _remount_rootfs_as_writable(self._remote_proxy) |
| except subprocess.CalledProcessError: |
| if not _disable_rootfs_verification(self._force, self._remote_proxy): |
| raise |
| _stop_ui(self._remote_proxy) |
| # Try to remount rootfs as writable. Bail out if it fails this time. |
| _remount_rootfs_as_writable(self._remote_proxy) |
| self._remote_proxy.check_call('\n'.join([ |
| # Delete the image file if it is a symlink. |
| 'if [ -L %(_ANDROID_ROOT)s/system.raw.img ]; then' |
| ' rm %(_ANDROID_ROOT)s/system.raw.img;' |
| 'fi', |
| ]) % {'_ANDROID_ROOT': _ANDROID_ROOT}) |
| if self._push_to_stateful: |
| self._remote_proxy.check_call('\n'.join([ |
| # Create the destination directory in the stateful partition. |
| 'mkdir -p %(_ANDROID_ROOT_STATEFUL)s', |
| ]) % {'_ANDROID_ROOT_STATEFUL': _ANDROID_ROOT_STATEFUL}) |
| |
| def __exit__(self, exc_type, exc_value, traceback): |
| if self._push_to_stateful: |
| # Push the image to _ANDROID_ROOT_STATEFUL instead of _ANDROID_ROOT. |
| # Create a symlink so that arc-system-mount can handle it. |
| self._remote_proxy.check_call('\n'.join([ |
| 'ln -sf %(_ANDROID_ROOT_STATEFUL)s/system.raw.img ' |
| ' %(_ANDROID_ROOT)s/system.raw.img', |
| ]) % {'_ANDROID_ROOT': _ANDROID_ROOT, |
| '_ANDROID_ROOT_STATEFUL': _ANDROID_ROOT_STATEFUL}) |
| |
| if self._is_selinux_policy_updated: |
| logging.info('*** SELinux policy updated. ***') |
| else: |
| logging.info('*** SELinux policy is not updated. Restarting ui. ***') |
| try: |
| self._remote_proxy.check_call('\n'.join([ |
| # Make the whole invocation fail if any individual command does. |
| 'set -e', |
| |
| # Remount the root file system to readonly. |
| 'mount -o remount,ro /', |
| |
| # Restart UI. |
| 'start ui', |
| |
| # Mount the updated {system,vendor}.raw.img. This will also trigger |
| # android-ureadahead once it's done and should remove the packfile. |
| 'start arc-system-mount', |
| ])) |
| return |
| except Exception: |
| # The above commands are just an optimization to avoid having to reboot |
| # every single time an image is pushed, which saves 6-10s. If any of |
| # them fail, the only safe thing to do is reboot the device. |
| logging.exception('Failed to cleanly restart ui, fall back to reboot') |
| |
| logging.info('*** Reboot required. ***') |
| try: |
| self._remote_proxy.check_call('reboot') |
| except Exception: |
| if exc_type is None: |
| raise |
| # If the body block of a with statement also raises an error, here we |
| # just log the exception, so that the main exception will be propagated |
| # to the caller properly. |
| logging.exception('Failed to reboot the device') |
| |
| |
| class PreserveTimestamps(object): |
| """Context object to modify a file but preserve the original timestamp.""" |
| |
| def __init__(self, path): |
| self.path = path |
| self._original_timestamp = None |
| |
| def __enter__(self): |
| # Save the original timestamp |
| self._original_timestamp = os.stat(self.path) |
| return self |
| |
| def __exit__(self, exception_type, exception_value, traceback): |
| # Apply the original timestamp |
| os.utime(self.path, (self._original_timestamp.st_atime, |
| self._original_timestamp.st_mtime)) |
| |
| |
| def _download_artifact(out_dir, build_id, product, build_variant): |
| mapping = dict(build_id=build_id, product=product) |
| uploaded_filename = _BUILD_FILENAME.substitute(mapping) |
| filename = os.path.join(out_dir, uploaded_filename) |
| fetcher = lib.build_artifact_fetcher.BuildArtifactFetcher( |
| lib.util.get_product_arch(product), build_variant, build_id) |
| fetcher.fetch(uploaded_filename, filename) |
| return filename |
| |
| |
| def _extract_files(unsquashfs_path, out_dir, system_raw_img, paths): |
| with TemporaryDirectory() as tmp_dir: |
| if lib.util.get_image_type(system_raw_img) is 'squashfs': |
| # Note: passing an empty set of paths will extract everything |
| # because the *[] expansion will produce no args. This is intentional. |
| lib.util.check_call( |
| unsquashfs_path, '-d', tmp_dir.name, '-no-progress', '-f', |
| '-no-xattrs', system_raw_img, *[path[0] for path in paths], |
| sudo=True) |
| _extract_files_helper(tmp_dir.name, out_dir, paths) |
| else: |
| with MountWrapper(system_raw_img, tmp_dir.name): |
| _extract_files_helper(tmp_dir.name, out_dir, paths) |
| |
| |
| def _extract_files_helper(source_root, destination_root, paths): |
| # Passing an empty set of paths should extract everything. |
| if not paths: |
| paths = [('./', './')] |
| |
| for source, destination in paths: |
| source = os.path.join(source_root, source) |
| # Some files are always expected to be missing due to not being |
| # present in the branch. |
| if not os.path.exists(source): |
| logging.debug('Skipping %s since it is missing', source) |
| continue |
| destination = os.path.join(destination_root, destination) |
| if not os.path.exists(os.path.dirname(destination)): |
| os.makedirs(os.path.dirname(destination)) |
| # 'sudo' is needed due to b/65117245. Android P shifts ugids of extracted |
| # files. |
| lib.util.check_call('cp', '-r', source, destination, sudo=True) |
| lib.util.check_call('chown', '-R', getpass.getuser(), destination, |
| sudo=True) |
| |
| |
| def _extract_artifact(simg2img, unsquashfs_path, out_dir, filename): |
| with zipfile.ZipFile(filename, 'r') as z: |
| z.extract('system.img', out_dir) |
| z.extract('vendor.img', out_dir) |
| # Note that the same simg2img conversion is performed again for system.img |
| # later, but the extra run is acceptable (<2s). If this is important, we |
| # could try to change the program flow. |
| simg2img.convert(os.path.join(out_dir, 'system.img'), |
| os.path.join(out_dir, 'system.raw.img')) |
| _extract_files(unsquashfs_path, out_dir, |
| os.path.join(out_dir, 'system.raw.img'), |
| [('sepolicy', 'root/sepolicy'), |
| ('file_contexts', 'root/file_contexts'), |
| ('plat_file_contexts', 'root/plat_file_contexts'), |
| ('nonplat_file_contexts', 'root/nonplat_file_contexts'), |
| ('vendor_file_contexts', 'root/vendor_file_contexts'), |
| ('system/build.prop', 'build.prop')]) |
| |
| |
| def _make_tempdir_deleted_on_exit(): |
| d = tempfile.mkdtemp() |
| atexit.register(shutil.rmtree, d, ignore_errors=True) |
| return d |
| |
| |
| def _detect_cert_inconsistency(force, remote_proxy, new_variant, dryrun): |
| """Prompt to ask for deleting data based on detected situation (best effort). |
| |
| Detection is only accurate for active session, so it won't fix other |
| profiles. |
| |
| As GMS apps are signed with different key between user and non-user build, |
| the container won't run correctly if old key has been registered in /data. |
| """ |
| if dryrun: |
| return False |
| |
| # Get current build variant on device. |
| cmd = 'grep %s %s' % (_RO_BUILD_TYPE, |
| os.path.join(_CONTAINER_ROOT, 'system/build.prop')) |
| try: |
| line = remote_proxy.check_output(cmd).strip() |
| except subprocess.CalledProcessError: |
| # Catch any error to avoid blocking the push. |
| logging.exception('Failed to inspect build property remotely') |
| return False |
| device_variant = line[len(_RO_BUILD_TYPE):] |
| |
| device_apk_key = _APK_KEY_UNKNOWN |
| try: |
| device_apk_key = _get_remote_device_apk_key(remote_proxy) |
| except Exception as e: |
| logging.warning('There was an error getting the remote device APK ' |
| 'key signature %s. Assuming APK key signature is ' |
| '\'unknown\'', e) |
| |
| logging.debug('device apk key: %s; build variant: %s -> %s', device_apk_key, |
| device_variant, new_variant) |
| |
| # GMS signature in /data is inconsistent with the new build. |
| is_inconsistent = ( |
| (device_apk_key == _APK_KEY_RELEASE and new_variant != 'user') or |
| (device_apk_key == _APK_KEY_DEBUG and new_variant == 'user')) |
| |
| if is_inconsistent: |
| new_apk_key = _APK_KEY_RELEASE if new_variant == 'user' else _APK_KEY_DEBUG |
| logging.info('Detected apk signature change (%s -> %s[%s]) on current ' |
| 'user.', device_apk_key, new_apk_key, new_variant) |
| if force: |
| logging.info('Deleting /data and /cache.') |
| return True |
| logging.info('Automatically delete and skip this prompt by specifying ' |
| '--force.') |
| return boolean_prompt('Delete /data and /cache?', default=True) |
| |
| # Switching from/to user build. |
| if (device_variant == 'user') != (new_variant == 'user'): |
| logging.warning('\n\n** You are switching build variant (%s -> %s). If ' |
| 'you have ever run with the old image, make sure to wipe ' |
| 'out /data first before starting the container. **\n', |
| device_variant, new_variant) |
| return False |
| |
| |
| def _get_remote_device_apk_key(remote_proxy): |
| """Retrieves the APK key signature of the remote test device. |
| |
| Args: |
| remote_proxy: RemoteProxy instance for the remote test device. |
| """ |
| remote_packages_xml = os.path.join(_CONTAINER_INSTANCE_ROOT_WILDCARD, |
| 'root/data/system/packages.xml') |
| with TemporaryDirectory() as tmp_dir: |
| host_packages_xml = os.path.join(tmp_dir.name, 'packages.xml') |
| remote_proxy.pull(remote_packages_xml, host_packages_xml) |
| return _get_apk_key_from_xml(host_packages_xml) |
| |
| |
| def _get_apk_key_from_xml(xml_file): |
| """Parses |xml_file| to determine the APK key signature. |
| |
| Args: |
| xml_file: The XML file to parse. |
| """ |
| if not os.path.exists(xml_file): |
| logging.warning('XML file doesn\'t exist: %s', xml_file) |
| return _APK_KEY_UNKNOWN |
| |
| root = ElementTree.parse(xml_file).getroot() |
| gms_core_elements = root.findall('package[@name=\'%s\']' |
| % _GMS_CORE_PACKAGE_NAME) |
| assert len(gms_core_elements) == 1, ('Invalid number of GmsCore package ' |
| 'elements. Expected: 1 Actual: %d' |
| % len(gms_core_elements)) |
| gms_core_element = gms_core_elements[0] |
| sigs_element = gms_core_element.find('sigs') |
| assert sigs_element, ('Unable to find the |sigs| tag under the GmsCore ' |
| 'package tag.') |
| sigs_count_attribute = int(sigs_element.get('count')) |
| assert sigs_count_attribute == 1, ('Invalid signature count. Expected: 1 ' |
| 'Actual: %d' % sigs_count_attribute) |
| cert_element = sigs_element.find('cert') |
| gms_core_cert_index = int(cert_element.get('index', -1)) |
| logging.debug('GmsCore cert index: %d', gms_core_cert_index) |
| if gms_core_cert_index == -1: |
| logging.warning('Invalid cert index (%d)', gms_core_cert_index) |
| return _APK_KEY_UNKNOWN |
| |
| cert_key = cert_element.get('key') |
| if cert_key: |
| return _get_android_key_type_from_cert_key(cert_key) |
| |
| # The GmsCore package element for |cert| contains the cert index, but not the |
| # cert key. Find its the matching cert key. |
| for cert_element in root.findall('package/sigs/cert'): |
| cert_index = int(cert_element.get('index')) |
| cert_key = cert_element.get('key') |
| if cert_key and cert_index == gms_core_cert_index: |
| return _get_android_key_type_from_cert_key(cert_key) |
| logging.warning('Unable to find a cert key matching index %d', cert_index) |
| return _APK_KEY_UNKNOWN |
| |
| |
| def _get_android_key_type_from_cert_key(cert_key): |
| """Returns |_APK_KEY_RELEASE| if |cert_key| contains the Android release key |
| signature substring, otherwise it returns |_APK_KEY_DEBUG|.""" |
| if _ANDROID_REL_KEY_SIGNATURE_SUBSTRING in cert_key: |
| return _APK_KEY_RELEASE |
| else: |
| return _APK_KEY_DEBUG |
| |
| |
| def _find_build_property(line, build_property_name): |
| """Returns the value that matches |build_property_name| in |line|.""" |
| if line.startswith(build_property_name): |
| return line[len(build_property_name):].strip() |
| return None |
| |
| |
| class ContainerImageEditor(object): |
| """A context object that allows edits to the Android container image""" |
| |
| def __init__(self, mksquashfs_path, unsquashfs_path, image_path, mount_point, |
| out_dir, file_contexts_path=None, loglevel='INFO'): |
| self._mksquashfs_path = mksquashfs_path |
| self._unsquashfs_path = unsquashfs_path |
| self._image_path = image_path |
| self._mount_point = mount_point |
| self._out_dir = out_dir |
| self._is_squashfs = False |
| self._mount_wrapper = None |
| self.file_contexts_path = file_contexts_path |
| self._loglevel = loglevel |
| # Since we shift UIDs/GIDs of all extracted files there, we are going to |
| # need sudo permission to remove the temporary directory. shutil doesn't |
| # have the ability to run as sudo, so not using TemporaryDirectory() here. |
| self.tmp_dir_name = tempfile.mkdtemp(prefix='ContainerImageEditor_') |
| |
| def __enter__(self): |
| self._is_squashfs = lib.util.get_image_type(self._image_path) is 'squashfs' |
| logging.debug('Is the file system squashfs? %s', self._is_squashfs) |
| |
| if self._is_squashfs: |
| # Extract the files. |
| lib.util.check_call(self._unsquashfs_path, '-force', '-d', |
| self.tmp_dir_name, self._image_path, sudo=True) |
| |
| self._update_file_context_path() |
| if not os.path.exists(self.file_contexts_path): |
| raise EnvironmentError('"%s" not found.' % self.file_contexts_path) |
| else: |
| self._mount_wrapper = MountWrapper(self._image_path, self.tmp_dir_name) |
| self._mount_wrapper.__enter__() |
| self._update_file_context_path() |
| |
| return self |
| |
| def __exit__(self, *args): |
| try: |
| if self._is_squashfs: |
| # Re-compress the files back to raw.img. |
| with tempfile.NamedTemporaryFile( |
| prefix='android_fs_config_') as fs_config: |
| # Running this script as an external process since it needs sudo to |
| # read some of the attributes of some protected files. |
| lib.util.check_call( |
| os.path.join(_SCRIPT_DIR, 'generate_android_fs_config_file.py'), |
| '--loglevel=%s' % self._loglevel, '--output=%s' % fs_config.name, |
| '--mount-point=%s' % self._mount_point, self.tmp_dir_name, |
| sudo=True) |
| lib.util.check_call( |
| self._mksquashfs_path, self.tmp_dir_name, self._image_path, |
| '-no-progress', '-comp', 'gzip', '-no-exports', '-noappend', |
| '-mount-point', self._mount_point, '-product-out', self._out_dir, |
| '-context-file', self.file_contexts_path, '-no-recovery', |
| '-no-fragments', '-no-duplicates', '-b', '131072', '-t', '0', |
| '-fs-config-file', fs_config.name, |
| '-android-fs-config', sudo=True) |
| else: |
| self._mount_wrapper.__exit__(*args) |
| finally: |
| lib.util.check_call('rm', '-rf', self.tmp_dir_name, sudo=True) |
| |
| def _update_file_context_path(self): |
| if self.file_contexts_path: |
| logging.debug('file_contexts_path is already set') |
| return |
| self.file_contexts_path = os.path.join(_make_tempdir_deleted_on_exit(), |
| 'file_contexts') |
| file_contexts_path = os.path.join(self.tmp_dir_name, 'file_contexts') |
| if os.path.exists(file_contexts_path): |
| logging.debug('Found file_contexts in image') |
| lib.util.check_call('cp', file_contexts_path, self.file_contexts_path) |
| return |
| |
| file_contexts_paths = [] |
| # TODO(R66): Remove nonplat_file_contexts after R66 hits stable in favor of |
| # vendor_file_contexts. |
| candidates = ('plat_file_contexts', 'nonplat_file_contexts', |
| 'vendor_file_contexts') |
| for candidate in candidates: |
| candidate_path = os.path.join(self.tmp_dir_name, candidate) |
| if not os.path.exists(candidate_path): |
| continue |
| file_contexts_paths.append(candidate_path) |
| if len(file_contexts_paths) < 2: |
| raise EnvironmentError('Unable to find all the file_contexts files in ' |
| 'the image. Tried %r.' % (candidates,)) |
| logging.debug('Combining %s files', file_contexts_paths) |
| with TemporaryDirectory() as tmp_dir: |
| merged_file_contexts_path = os.path.join(tmp_dir.name, 'file_contexts') |
| with open(merged_file_contexts_path, 'w') as outfile: |
| for file_contexts_path in file_contexts_paths: |
| logging.debug('Writing %s to %s', file_contexts_path, |
| merged_file_contexts_path) |
| with open(file_contexts_path) as infile: |
| for line in infile: |
| outfile.write(line) |
| lib.util.check_call('cp', merged_file_contexts_path, |
| self.file_contexts_path) |
| |
| |
| class BaseProvider(object): |
| """Base class of image provider. |
| |
| Subclass should provide a directory with images in it. |
| """ |
| |
| def __init__(self): |
| self._build_variant = None |
| self._build_version_sdk = None |
| |
| def prepare(self): |
| """Subclass should prepare image in its implementation. |
| |
| Subclass must return the (image directory, product, fingerprint) tuple. |
| Product is a string like "cheets_arm". Fingerprint is the string that |
| will be updated to CHROMEOS_ARC_VERSION in /etc/lsb-release. |
| """ |
| raise NotImplementedError() |
| |
| def get_build_variant(self): |
| """ Returns the extracted build variant.""" |
| return self._build_variant |
| |
| def get_build_version_sdk(self): |
| """ Returns the extracted Android SDK version.""" |
| return self._build_version_sdk |
| |
| def read_build_prop_file(self, build_prop_file, remove_file=True): |
| """Gets variant and SDK information from build property file. |
| |
| This reads the specified build property file, and extracts the |
| "ro.build.variant" and "ro.build.version.sdk" fields. It optionally |
| deletes |build_prop_file| when done. |
| |
| Args: |
| build_prop_file: The fully qualified path to the build.prop file. |
| remove_file: Removes the |build_prop_file| when done. (default=True) |
| """ |
| logging.debug('Reading build prop file: %s', build_prop_file) |
| with open(build_prop_file, 'r') as f: |
| for line in f: |
| if self._build_version_sdk is None: |
| value = _find_build_property(line, _RO_BUILD_VERSION_SDK) |
| if value is not None: |
| self._build_version_sdk = int(value) |
| if self._build_variant is None: |
| value = _find_build_property(line, _RO_BUILD_TYPE) |
| if value is not None: |
| self._build_variant = value |
| if self._build_variant and self._build_version_sdk: |
| break |
| if remove_file: |
| logging.info('Deleting prop file: %s...', build_prop_file) |
| os.remove(build_prop_file) |
| |
| |
| class PrebuiltProvider(BaseProvider): |
| """A provider to provides prebuilt image from Android builder.""" |
| |
| def __init__(self, product, build_variant, build_id, simg2img, |
| unsquashfs_path): |
| super(PrebuiltProvider, self).__init__() |
| self._product = product |
| self._build_variant = build_variant |
| self._build_id = build_id |
| self._simg2img = simg2img |
| self._unsquashfs_path = unsquashfs_path |
| |
| def prepare(self): |
| fingerprint = '_'.join([self._product, self._build_variant, |
| self._build_id]) |
| |
| out_dir = _make_tempdir_deleted_on_exit() |
| filename = _download_artifact(out_dir, self._build_id, self._product, |
| self._build_variant) |
| _extract_artifact(self._simg2img, self._unsquashfs_path, out_dir, filename) |
| |
| build_prop_file = os.path.join(out_dir, 'build.prop') |
| self.read_build_prop_file(build_prop_file) |
| return out_dir, self._product, fingerprint |
| |
| |
| class LocalPrebuiltProvider(BaseProvider): |
| """A provider that provides prebuilt image from a local file.""" |
| |
| def __init__(self, prebuilt_file, simg2img, unsquashfs_path): |
| super(LocalPrebuiltProvider, self).__init__() |
| self._prebuilt_file = prebuilt_file |
| self._simg2img = simg2img |
| self._unsquashfs_path = unsquashfs_path |
| |
| def prepare(self): |
| out_dir = _make_tempdir_deleted_on_exit() |
| logging.debug('Extracting artifacts') |
| _extract_artifact(self._simg2img, self._unsquashfs_path, out_dir, |
| self._prebuilt_file) |
| |
| build_prop_file = os.path.join(out_dir, 'build.prop') |
| self.read_build_prop_file(build_prop_file) |
| if self._build_variant is None: |
| self._build_variant = 'user' # default to non-eng |
| |
| m = re.match(r'((?:bertha|cheets)_\w+)-img-P?\d+\.zip', |
| os.path.basename(self._prebuilt_file)) |
| if not m: |
| sys.exit('Unrecognized file name of prebuilt image archive.') |
| product = m.group(1) |
| |
| fingerprint = os.path.splitext(os.path.basename(self._prebuilt_file))[0] |
| return out_dir, product, fingerprint |
| |
| |
| class LocalBuildProvider(BaseProvider): |
| """A provider that provides local built image.""" |
| |
| def __init__(self, build_fingerprint, skip_build_prop_update, |
| added_build_properties, dryrun): |
| super(LocalBuildProvider, self).__init__() |
| self._build_fingerprint = build_fingerprint |
| self._skip_build_prop_update = skip_build_prop_update |
| self._added_build_properties = added_build_properties |
| self._dryrun = dryrun |
| expected_env = ('TARGET_BUILD_VARIANT', 'TARGET_PRODUCT', 'OUT') |
| if not all(var in os.environ for var in expected_env): |
| sys.exit('Did you run lunch?') |
| self._build_variant = os.environ.get('TARGET_BUILD_VARIANT') |
| self._target_product = os.environ.get('TARGET_PRODUCT') |
| self._out_dir = os.environ.get('OUT') |
| |
| # Create the generic device name by extracting the architecture type |
| # from the target product. |
| generic_device = _GENERIC_DEVICE % dict( |
| arch=lib.util.get_product_arch(self._target_product), |
| product=lib.util.get_product_name(self._target_product)) |
| new_prop_value = '%s%s' % (_RO_PRODUCT_DEVICE, generic_device) |
| if new_prop_value not in self._added_build_properties: |
| self._added_build_properties.append(new_prop_value) |
| |
| def prepare(self): |
| # Use build fingerprint if set. Otherwise, read it from the text file. |
| build_fingerprint = self._build_fingerprint |
| if not build_fingerprint: |
| fingerprint_filepath = os.path.join(self._out_dir, |
| 'build_fingerprint.txt') |
| if os.path.isfile(fingerprint_filepath): |
| with open(fingerprint_filepath) as f: |
| build_fingerprint = f.read().strip().replace('/', '_') |
| |
| # Find the absolute path of build.prop. |
| build_prop_file = os.path.join(self._out_dir, 'system/build.prop') |
| if not self._skip_build_prop_update: |
| self._update_local_build_prop_file(build_prop_file) |
| self.read_build_prop_file(build_prop_file, False) |
| return self._out_dir, self._target_product, build_fingerprint |
| |
| def _update_local_build_prop_file(self, build_prop_file): |
| """Updates build.prop of the local prebuilt image.""" |
| |
| if not build_prop_file: |
| logging.warning('Skipping. build_prop_file was not specified.') |
| return |
| |
| file_updated = False |
| for prop in self._added_build_properties: |
| key_and_value = prop.split('=') |
| # Check whether the property is already in build.prop. |
| try: |
| current_prop_value = lib.util.check_output('grep', |
| '%s=' % (key_and_value[0]), |
| build_prop_file).strip() |
| except subprocess.CalledProcessError: |
| # grep did not find a match |
| current_prop_value = None |
| |
| if not current_prop_value: |
| file_updated = True |
| logging.info('Appending "%s" to build.prop...', prop) |
| |
| with PreserveTimestamps(build_prop_file) as f: |
| lib.util.check_call( |
| '/bin/sed', '-i', |
| r'$a%s' % prop, |
| f.path) |
| elif prop != current_prop_value: |
| file_updated = True |
| logging.info('Setting "%s" to "%s" in build.prop...', |
| key_and_value[0], key_and_value[1]) |
| with PreserveTimestamps(build_prop_file) as f: |
| # Make the changes to build.prop |
| lib.util.check_call( |
| '/bin/sed', '-i', |
| r's/^\(%(_KEY)s=\).*/\1%(_VALUE)s/' |
| % {'_KEY': key_and_value[0], '_VALUE': key_and_value[1]}, |
| f.path) |
| |
| if not file_updated: |
| logging.info('build.prop does not need to be updated.') |
| return |
| |
| logging.info('Recreating the system image with the updated build.prop ' + |
| 'file...') |
| system_dir = os.path.join(self._out_dir, 'system') |
| system_image_info_file = os.path.join( |
| self._out_dir, |
| 'obj/PACKAGING/systemimage_intermediates/system_image_info.txt') |
| system_image_file = os.path.join(self._out_dir, 'system.img') |
| if self._dryrun: |
| return |
| with PreserveTimestamps(system_image_file) as f: |
| # Recreate system.img |
| lib.util.check_call( |
| './build/tools/releasetools/build_image.py', |
| system_dir, |
| system_image_info_file, |
| f.path, |
| system_dir) |
| |
| |
| class NullProvider(BaseProvider): |
| """ Provider used for dry runs """ |
| |
| def __init__(self): |
| super(NullProvider, self).__init__() |
| self._build_variant = 'user' |
| self._build_version_sdk = 1 |
| |
| def prepare(self): |
| return ('<dir>', '<product>', '<fingerprint>') |
| |
| |
| def _parse_prebuilt(param): |
| m = re.search( |
| r'^((?:bertha|cheets)_(?:arm|x86|x86_64))/(user|userdebug|eng)/(P?\d+)$', |
| param) |
| if not m: |
| sys.exit('Invalid format of --use-prebuilt') |
| return m.group(1), m.group(2), m.group(3) |
| |
| |
| def _default_tool_path(tool_name, allow_host=False): |
| # Automatically resolve tool path from the out/ directory if possible. |
| dirs_to_find = [] |
| if 'ANDROID_HOST_OUT' in os.environ: |
| dirs_to_find.append(os.path.join(os.environ.get('ANDROID_HOST_OUT'), |
| 'bin')) |
| dirs_to_find.append(_SCRIPT_DIR) |
| if allow_host and 'PATH' in os.environ: |
| dirs_to_find += os.environ.get('PATH').split(os.pathsep) |
| |
| for dir_name in dirs_to_find: |
| path = os.path.join(dir_name, tool_name) |
| if os.path.isfile(path) and os.access(path, os.X_OK): |
| return path |
| return None |
| |
| |
| def _default_shift_uid_py_path(): |
| # Automatically resolve shift_uid.py path if possible. |
| shift_uid_path = os.path.join(_SCRIPT_DIR, 'shift_uid.py') |
| if os.path.isfile(shift_uid_path): |
| return shift_uid_path |
| try: |
| shift_uid_path = os.path.join( |
| lib.util.find_repo_root(), |
| 'vendor/google_devices/bertha/scripts/shift_uid.py') |
| if os.path.isfile(shift_uid_path): |
| return shift_uid_path |
| except ValueError as e: |
| logging.exception('Failed to find shift_uid.py path: %s', e.output) |
| return None |
| |
| |
| def _resolve_args(args): |
| if not args.simg2img_path: |
| sys.exit('Cannot determine the path of simg2img') |
| if not args.mksquashfs_path: |
| sys.exit('Cannot determine the path of mksquashfs') |
| if not args.unsquashfs_path: |
| sys.exit('Cannot determine the path of unsquashfs. You may need to ' |
| 'install it with sudo apt install squashfs-tools.') |
| if (args.chromeos_file_contexts and |
| not os.path.isfile(args.chromeos_file_contexts)): |
| sys.exit('Chrome OS context file %s does not exist' % |
| args.chromeos_file_contexts) |
| if (args.container_config and |
| not os.path.isfile(args.container_config)): |
| sys.exit('The container config file %s does not exist' % |
| args.container_config) |
| if ((args.added_build_properties or args.enable_assistant_prop) and |
| (args.use_prebuilt or args.prebuilt_file)): |
| sys.exit('Cannot use "add-build-property" or "enable-assistant-prop" with ' |
| 'prebuilt images') |
| |
| |
| def _parse_args(): |
| """Parses the arguments.""" |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| description='Push image to Chromebook', |
| epilog="""Examples: |
| |
| To push from local build |
| $ %(prog)s <remote> |
| |
| To push from Android build prebuilt |
| $ %(prog)s --use-prebuilt cheets_arm/eng/123456 <remote> |
| |
| To push from local prebuilt |
| $ %(prog)s --use-prebuilt-file path/to/cheets_arm-img-123456.zip <remote> |
| """) |
| parser.add_argument( |
| '--push-vendor-image', action='store_true', help='Push vendor image') |
| parser.add_argument( |
| '--use-prebuilt', metavar='PRODUCT/BUILD_VARIANT/BUILD_ID', |
| type=_parse_prebuilt, |
| help='Push prebuilt image instead. Example value: cheets_arm/eng/123456') |
| parser.add_argument( |
| '--use-prebuilt-file', dest='prebuilt_file', metavar='<path>', |
| help='The downloaded image path') |
| parser.add_argument( |
| '--build-fingerprint', default=os.environ.get('BUILD_FINGERPRINT'), |
| help='If set, embed this fingerprint data to the /etc/lsb-release ' |
| 'as CHROMEOS_ARC_VERSION value.') |
| parser.add_argument( |
| '--dryrun', action='store_true', |
| help='Do not execute subprocesses.') |
| parser.add_argument( |
| '--loglevel', default='INFO', |
| choices=('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'), |
| help='Logging level.') |
| parser.add_argument( |
| '--simg2img-path', default=_default_tool_path('simg2img', |
| allow_host=True), |
| help='Executable path of simg2img') |
| parser.add_argument( |
| '--mksquashfs-path', default=_default_tool_path('mksquashfs'), |
| help='Executable path of mksquashfs') |
| parser.add_argument( |
| '--unsquashfs-path', default=_default_tool_path('unsquashfs'), |
| help='Executable path of unsquashfs') |
| parser.add_argument( |
| '--shift-uid-py-path', default=_default_shift_uid_py_path(), |
| help='Executable path of shift_uid.py') |
| parser.add_argument( |
| '--force', action='store_true', |
| help=('Skip all prompts (i.e., for disabling of rootfs verification). ' |
| 'This may result in the target machine being rebooted')) |
| parser.add_argument( |
| '--try-clobber-data', action='store_true', |
| help='If currently logged in, also clobber /data and /cache') |
| parser.add_argument( |
| '--skip_build_prop_update', action='store_true', |
| help=('Do not change ro.product.device to "generic_cheets" for local ' |
| 'builds')) |
| parser.add_argument( |
| '--push-to-stateful-partition', action='store_true', |
| help=('Place the system.raw.img on the stateful partition instead of /. ' |
| 'This is always used for builds that do not fit on /.')) |
| parser.add_argument( |
| '--add-build-property', action='append', default=[], |
| dest='added_build_properties', |
| help=('Update build.prop with the given property e.g. ' |
| 'some.property=true. If the given property exists, it will ' |
| 'update the property. Otherwise it will append the given ' |
| 'property to the end.')) |
| parser.add_argument( |
| '--enable-assistant-prop', action='store_true', |
| help=('Update build.prop with ro.opa.eligible_device=true, ' |
| 'this is required to run assistant on ChromeOS.')) |
| parser.add_argument( |
| '--chromeos-file-contexts', dest='chromeos_file_contexts', |
| metavar='<file>', |
| help=('If specified, the file is used (instead of DUT\'s) when the ' |
| 'script reconstructs file_contexts.')) |
| parser.add_argument( |
| '--container-config', dest='container_config', metavar='<file>', |
| help=('If specified, the file is pushed to the DUT as ' |
| '/opt/google/containers/android/config.json')) |
| parser.add_argument( |
| 'remote', |
| help=('The target test device. This is passed to ssh command etc., ' |
| 'so IP or the name registered in your .ssh/config file can be ' |
| 'accepted.')) |
| args = parser.parse_args() |
| |
| _resolve_args(args) |
| return args |
| |
| |
| def main(): |
| # Set up arguments. |
| args = _parse_args() |
| logging.basicConfig(level=getattr(logging, args.loglevel)) |
| |
| if args.enable_assistant_prop: |
| args.added_build_properties.append('ro.opa.eligible_device=true') |
| |
| simg2img = Simg2img(args.simg2img_path, args.dryrun) |
| |
| # Prepare local source. A preparer is responsible to return an directory |
| # that contains necessary files to push. It also needs to return metadata |
| # like product (e.g. cheets_arm) and a build fingerprint. |
| if args.dryrun: |
| provider = NullProvider() |
| elif args.use_prebuilt: |
| product, build_variant, build_id = args.use_prebuilt |
| # pylint has a false negative of redefined-variable-type with conditionals: |
| # https://github.com/PyCQA/pylint/issues/799 |
| # pylint: disable=redefined-variable-type |
| provider = PrebuiltProvider(product, build_variant, build_id, simg2img, |
| args.unsquashfs_path) |
| elif args.prebuilt_file: |
| provider = LocalPrebuiltProvider(args.prebuilt_file, simg2img, |
| args.unsquashfs_path) |
| else: |
| provider = LocalBuildProvider(args.build_fingerprint, |
| args.skip_build_prop_update, |
| args.added_build_properties, |
| args.dryrun) |
| |
| # Actually prepare the files to push. |
| out, product, fingerprint = provider.prepare() |
| |
| # Update the image. |
| remote_proxy = RemoteProxy(args.remote, args.dryrun) |
| _verify_android_sdk_version(remote_proxy, provider, args.dryrun) |
| _verify_machine_arch(remote_proxy, product, args.dryrun) |
| |
| if args.try_clobber_data: |
| clobber_data = True |
| else: |
| clobber_data = _detect_cert_inconsistency( |
| args.force, remote_proxy, provider.get_build_variant(), args.dryrun) |
| |
| with TemporaryDirectory() as vendor_merge_dir: |
| if args.push_vendor_image: |
| logging.info('Fetching existing vendor partition contents...') |
| _get_remote_device_vendor_image(remote_proxy, vendor_merge_dir.name, |
| args.dryrun, args.unsquashfs_path) |
| |
| logging.info('Converting images to raw images...') |
| (large_image_list, image_list) = _convert_images( |
| simg2img, out, product, args.push_vendor_image, |
| args.mksquashfs_path, args.unsquashfs_path, args.shift_uid_py_path, |
| dryrun=args.dryrun, loglevel=args.loglevel, |
| vendor_merge_dir=vendor_merge_dir.name) |
| |
| is_selinux_policy_updated = _is_selinux_policy_updated(remote_proxy, out, |
| args.dryrun) |
| is_selinux_file_contexts_updated = _is_selinux_file_contexts_updated( |
| remote_proxy, out, args.chromeos_file_contexts, args.dryrun) |
| total_bytes = sum(os.stat(filename).st_size for filename in large_image_list) |
| free_bytes = _get_free_space(remote_proxy) |
| push_to_stateful = (args.push_to_stateful_partition or |
| total_bytes >= free_bytes) |
| |
| if not args.push_to_stateful_partition and push_to_stateful: |
| logging.info('Pushing image to stateful partition ' |
| 'since it does not fit on / (%.2f MiB, %.2f free MiB).', |
| float(total_bytes) / _MB, float(free_bytes) / _MB) |
| |
| with ImageUpdateMode(remote_proxy, is_selinux_policy_updated, |
| push_to_stateful, clobber_data, args.force): |
| # TODO(74923319): Clean up the arc-setup-env path. |
| setup_env_path = remote_proxy.check_output(' '.join([ |
| 'find', '/etc', '-name', 'arc-setup-env'])).strip() |
| is_debuggable = provider.get_build_variant() != 'user' |
| remote_proxy.check_call(' '.join([ |
| '/bin/sed', '-i', |
| r'"s/^\(export ANDROID_DEBUGGABLE=\).*/\1%(_IS_DEBUGGABLE)d/"', |
| setup_env_path |
| ]) % {'_IS_DEBUGGABLE': is_debuggable}) |
| |
| logging.info('Syncing image files to ChromeOS...') |
| if large_image_list: |
| remote_proxy.sync(large_image_list, |
| _ANDROID_ROOT_STATEFUL if push_to_stateful else |
| _ANDROID_ROOT) |
| if image_list: |
| remote_proxy.sync(image_list, _ANDROID_ROOT) |
| _update_build_fingerprint(remote_proxy, fingerprint) |
| if is_selinux_file_contexts_updated: |
| _update_selinux_file_contexts(remote_proxy, args.chromeos_file_contexts, |
| out) |
| if is_selinux_policy_updated: |
| _update_selinux_policy(remote_proxy, out) |
| |
| if args.container_config: |
| _update_container_config(remote_proxy, args.container_config) |
| |
| |
| if __name__ == '__main__': |
| main() |