| #!/bin/bash |
| # |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| # Shared library for searching and choosing CrOS images from a large collection |
| # of archived versions. |
| # |
| # This library deals with CrOS versions in four forms: |
| # |
| # version string: e.g. "R12-3456.7.8" |
| # Completely specifies a CrOS version as release, major, minor and revision |
| # numbers. The revision number may be followed by a '-' or '.' and arbitrary |
| # characters, which will be ignored (e.g. "R12-3456.7.8-a9-b123"). |
| # |
| # version pattern: e.g. "R12-3456", "3456.7", etc. |
| # Specifies a partial CrOS version by omitting some of the numbers. For |
| # example, all of the following patterns would match "R12-3456.7.8": |
| # '', R12, R12-3456, R12-3456.7, R12-3456.7.8, 3456, 3456.7, 3456.7.8 |
| # |
| # numeric version: e.g. "12 3456 7 8" |
| # A version written as four space-separated integers corresponding to the |
| # release, major, minor and revision numbers. The last integer may be |
| # followed by another space and arbitrary characters, which will be ignored. |
| # |
| # numeric pattern: e.g. "12 3456 -1 -1" |
| # A version pattern written as four space-separated integers corresponding to |
| # the release, major, minor and revision numbers. Unrestricted fields are |
| # indicated by negative values. The last integer may be followed by another |
| # space and arbitrary characters, which will be ignored. |
| |
| ### GENERIC FUNCTIONS |
| |
| # Usage: cros_archive_get_numeric_version version |
| # Generates a numeric version for the specified version string. |
| # |
| # Writes the numeric version to stdout and returns success. If the version |
| # string is malformed, will produce no output and return failure. |
| cros_archive_get_numeric_version() { |
| local arg_{rel,ver,maj,min,rev,tail} |
| |
| # BUG(cwolfe): |
| # For some reason, attaching the next IFS to its read results in the IFS |
| # being ignored during cros_archive_filter_url_version_newest. This only |
| # occurs when run in the chroot (bash 4.2.20) and not on my workstation |
| # (bash 4.1.5). Hacking it to work for the moment, and need to figure out |
| # whether I've discovered a weird feature or a bash bug. |
| # |
| # Absent the bug, the next four lines should be: |
| # IFS='-' read -r arg_{rel,ver,tail} <<<"$1" |
| local old_ifs="${IFS}" |
| IFS='-' |
| read -r arg_{rel,ver,tail} <<<"$1" |
| IFS="${old_ifs}" |
| |
| IFS='.' read -r arg_{maj,min,rev,tail} <<<"${arg_ver}" |
| arg_rel=${arg_rel#R} |
| if [[ -z ${arg_rel} || -z ${arg_maj} || |
| -z ${arg_min} || -z ${arg_rev} ]]; then |
| return 1 # Malformed version |
| fi |
| |
| printf '%d %d %d %d\n' "${arg_rel}" "${arg_maj}" "${arg_min}" "${arg_rev}" |
| } |
| |
| # Usage: cros_archive_get_numeric_pattern version |
| # Generates a numeric pattern corresponding to the specified version pattern. |
| # |
| # Writes the numeric pattern to stdout and returns success. If the version |
| # pattern is malformed or contains extra characters, will produce no output and |
| # return failure. |
| cros_archive_get_numeric_pattern() { |
| local arg_{rel,ver,maj,min,rev,tail} |
| |
| IFS='-' read -r arg_{rel,ver,tail} <<<"$1" |
| if [[ -n ${arg_tail} ]]; then |
| return 1 # Malformed pattern: extra characters |
| fi |
| if [[ ${arg_rel:0:1} == 'R' ]]; then |
| arg_rel=${arg_rel#R} # Strip the 'R' prefix |
| elif [[ -n ${arg_rel} ]]; then |
| arg_ver=${arg_rel} # Actually got a version with no release. |
| arg_rel= |
| fi |
| if [[ -n ${arg_ver} ]]; then |
| IFS='.' read -r arg_{maj,min,rev,tail} <<<"${arg_ver}" |
| if [[ -n ${arg_tail} ]]; then |
| return 1 # Malformed pattern: extra characters |
| fi |
| fi |
| |
| printf '%d %d %d %d\n' \ |
| "${arg_rel:--1}" "${arg_maj:--1}" "${arg_min:--1}" "${arg_rev:--1}" |
| } |
| |
| # Usage: cros_archive_get_wildcard_pattern version |
| # Generates a wildcard pattern that will match the specified numeric version. |
| # |
| # Writes the wildcard pattern to stdout and returns success. If the numeric |
| # version is malformed, will produce no output and return failure. |
| cros_archive_get_wildcard_pattern() { |
| local arg_{rel,maj,min,rev} |
| read -r arg_{rel,maj,min,rev} <<<"$1" |
| |
| # Replace any negative fields with '*'. |
| [[ ${arg_rel} -ge 0 ]] || arg_rel='*' |
| [[ ${arg_maj} -ge 0 ]] || arg_maj='*' |
| [[ ${arg_min} -ge 0 ]] || arg_min='*' |
| [[ ${arg_rev} -ge 0 ]] || arg_rev='*' |
| |
| printf "R%s-%s.%s.%s\n" "${arg_rel}" "${arg_maj}" "${arg_min}" "${arg_rev}" |
| } |
| |
| # Usage: cros_archive_test_numeric_version_matches pattern version |
| # Tests whether a numeric pattern matches a numeric version. |
| # |
| # Returns success if the pattern matches the version and failure otherwise. |
| cros_archive_test_numeric_version_matches() { |
| local -a pattern version |
| read -ra pattern <<<"$1" |
| read -ra version <<<"$2" |
| if [[ ${#pattern[@]} -lt 4 || ${#version[@]} -lt 4 ]]; then |
| return 1 # Malformed pattern or version |
| fi |
| |
| local i |
| for i in {0..3}; do |
| if [[ ${pattern[$i]} -ge 0 && ${pattern[$i]} -ne ${version[$i]} ]]; then |
| return 1 # Version does not match pattern |
| fi |
| done |
| return 0 # Version matches pattern |
| } |
| |
| # Usage: cros_archive_test_numeric_version_newer first second |
| # Tests whether the first numeric version is newer than the second. |
| # |
| # Returns success if the first numeric version is newer than the second, |
| # and failure otherwise. |
| cros_archive_test_numeric_version_newer() { |
| local -a first second |
| read -ra first <<<"$1" |
| read -ra second <<<"$2" |
| if [[ ${#first[@]} -lt 4 || ${#second[@]} -lt 4 ]]; then |
| return 1 # Malformed version |
| fi |
| |
| local i |
| for i in {0..3}; do |
| if [[ ${first[$i]} -ne ${second[$i]} ]]; then |
| if [[ ${first[$i]} -gt ${second[$i]} ]]; then |
| return 0 # First is indeed newer then second. |
| else |
| return 1 # First is older than second. |
| fi |
| fi |
| done |
| return 1 # First and second are equal. |
| } |
| |
| # Usage: cros_archive_filter_numeric_version_matches pattern |
| # Filters a stream of numeric versions, keeping those which match the specified |
| # numeric pattern. |
| # |
| # Reads lines from stdin. Each line must be a numeric version. If a line |
| # matches the pattern it will be written to stdout in its entirety, otherwise |
| # the line will be discarded. |
| # |
| # Returns success at the end if input. |
| cros_archive_filter_numeric_version_matches() { |
| local pattern="$1" |
| local version |
| while read -r version; do |
| if cros_archive_test_numeric_version_matches "${pattern}" "${version}"; then |
| printf '%s\n' "${version}" |
| fi |
| done |
| } |
| |
| # Usage: cros_archive_filter_numeric_version_newest |
| # Filters a stream of numeric versions to choose the newest. |
| # |
| # Reads lines from stdin. Each line must be a numeric version. Once the end of |
| # input has been reached, the line containing the newest numeric version will |
| # be written to stdout in its entirety. Will not produce any output if no |
| # versions have been read. |
| # |
| # Returns success unless an error was encountered, even if no versions were |
| # processed. |
| cros_archive_filter_numeric_version_newest() { |
| local best |
| local curr |
| while read -r curr; do |
| if [[ -z ${best} ]] || |
| cros_archive_test_numeric_version_newer "${curr}" "${best}"; then |
| best="${curr}" |
| fi |
| done |
| if [[ -n ${best} ]]; then |
| printf '%s\n' "${best}" |
| fi |
| } |
| |
| # Usage: cros_archive_filter_url_version_newest regexp [pattern] |
| # Filters a stream of URLs to choose the one containing the newest version |
| # string. Optionally discards versions which do not match the specified numeric |
| # pattern. |
| # |
| # The regexp must contain a single parenthesized subexpression that selects the |
| # version string from a URL. Any URL which does not match this regexp or does |
| # not record a parenthesized subexpression will be discarded. |
| # |
| # Reads lines from stdin, each of which must consist of a single URL. Once the |
| # end of input has been reached, the URL containing the newest version string |
| # will be written to stdout. |
| # |
| # Returns success unless an error was encountered, even if no URLs were found. |
| cros_archive_filter_url_version_newest() { |
| local regexp="$1" |
| local pattern="${2:--1 -1 -1 -1}" |
| |
| get_numeric_result() { |
| local input |
| while read -r input; do |
| if [[ ! ( ${input} =~ ^${regexp}$ ) || ${#BASH_REMATCH[@]} -lt 1 ]]; then |
| continue |
| fi |
| local version=$(cros_archive_get_numeric_version "${BASH_REMATCH[1]}") |
| printf '%s %s\n' "${version}" "${input}" |
| done | |
| cros_archive_filter_numeric_version_matches "${pattern}" | |
| cros_archive_filter_numeric_version_newest |
| } |
| |
| local -a result |
| if ! read -a result < <(get_numeric_result); then |
| return 0 # No URL available |
| fi |
| |
| # Drop the first four elements in the result array, as they contain the |
| # numeric version. |
| printf '%s\n' "${result[*]:4}" |
| } |
| |
| ### GOOGLE STORAGE SPECIFIC FUNCTIONS |
| # |
| # This code addresses the problem of getting images from a GS archive in two |
| # stages: it generates a set of wildcards passed to "gsutil ls", and then uses |
| # a regexp to parse the version numbers out of the list. These two steps are |
| # necessary because gsutil supports only simple glob-style wildcards and can |
| # not do version-number ordering of the results. Fortunately the processing |
| # can be run in parallel to the "gsutil ls", so the overhead is small. |
| # |
| |
| # Usage: cros_archive_gs_get_url_wildcards channel board [pattern] |
| # Gets one or more URL wildcard for a specified channel and board which will |
| # include at least the archives corresponding to the specified numeric pattern. |
| # |
| # The 'archive' virtual channel is loaded from gs://chromeos-image-archive. |
| # All other channels will be requested from gs://chromeos-releases. |
| cros_archive_gs_get_url_wildcards() { |
| local channel="$1" |
| local board="$2" |
| local pattern=$(cros_archive_get_wildcard_pattern "${3:--1 -1 -1 -1}") |
| |
| # Add a '*' after the pattern, as some of the uploaded paths (but not all) |
| # contain additional numbers for attempt and build like "-a0-b1234". To |
| # avoid doubling '*'s, also remove any trailing '*' from the pattern. |
| pattern="${pattern%\*}*" |
| |
| if [[ ${channel} == 'archive' ]]; then |
| # Most boards use a -release builder, so output that URL first. |
| local suffix |
| for suffix in release full; do |
| printf 'gs://chromeos-image-archive/%s-%s/%s/image.zip\n' \ |
| "${board}" "${suffix}" "${pattern}" |
| done |
| else |
| printf 'gs://chromeos-releases/%s/%s/%s/ChromeOS-%s-%s.zip\n' \ |
| "${channel}-channel" "${board}" "${pattern#R*-}" "${pattern}" \ |
| "${board}" |
| fi |
| } |
| |
| # Usage: cros_archive_gs_get_url_regexp channel board |
| # Gets a regexp which can be used to extract the version from URLs associated |
| # with the specified channel and board. |
| # |
| # The 'archive' virtual channel is loaded from gs://chromeos-image-archive. |
| # All other channels will be requested from gs://chromeos-releases. |
| cros_archive_gs_get_url_regexp() { |
| local channel="$1" |
| local board="$2" |
| local pattern=$(cros_archive_get_wildcard_pattern "${3:--1 -1 -1 -1}") |
| |
| # The version_regexp picks out and records two dash-separated components that |
| # look like a CrOS version. In some cases it will be followed by another dash |
| # and additional information, so we need to be a little careful about what it |
| # matches. |
| local version_regexp='(R[^/-]*-[^/-]*)' |
| if [[ ${channel} == 'archive' ]]; then |
| printf 'gs://chromeos-image-archive/%s[^/]*/%s[^/]*/image.zip\n' \ |
| "${board}" "${version_regexp}" |
| else |
| printf 'gs://chromeos-releases/%s/%s/[^/]*/ChromeOS-%s[^/]*-%s.zip\n' \ |
| "${channel}-channel" "${board}" "${version_regexp}" "${board}" |
| fi |
| } |
| |
| # Usage: cros_archive_gsutil_ls_filter |
| # Gets multiple directory listings from gsutil ls. |
| # |
| # Reads lines from stdin. Each line must be a gs:// URL, possibly containing |
| # wildcards. Will write zero or more lines to stdout containing the URLs which |
| # were returned from the server. |
| # |
| # Errors from gsutil related to there being no matching URLs will be ignored. |
| # Returns success unless an error was encountered, even if no URLs were found. |
| # |
| # Environment: |
| # GSUTIL: name of gsutil program (default is 'gsutil') |
| cros_archive_gsutil_ls_filter() { |
| local url |
| while read -r url; do |
| # Run gsutil with its stdout passed through and its stderr recorded. |
| local rc=0 |
| exec 3>&1 |
| local error=$( ${GSUTIL:-gsutil} ls "${url}" 2>&1 1>&3 3>&- ) || rc=$? |
| exec 3>&- |
| |
| # Ignore errors that only indicate no matches were found. This behavior |
| # may vary between different versions of gsutil, so we should not assume |
| # an empty result sets non-zero rc. |
| if [[ ${rc} -ne 0 && ${error} != 'No matches for'* ]]; then |
| printf 'gsutil: %s\n' "${error}" >&2 |
| return 1 |
| fi |
| done |
| } |
| |
| # Usage: cros_archive_gs_list channel board [pattern] |
| # Gets the list of URLs matching the specified channel and board which will |
| # satisfy the specified numeric pattern. |
| # |
| # This command calls the Google Storage backend to get a list of available |
| # files. |
| # |
| # Returns success unless an error was encountered, even if no URLs were found. |
| cros_archive_gs_list() { |
| local channel="$1" |
| local board="$2" |
| local pattern=$(cros_archive_get_numeric_pattern "$3") |
| |
| local regexp=$(cros_archive_gs_get_url_regexp "${channel}" "${board}") |
| |
| cros_archive_gs_get_url_wildcards "${channel}" "${board}" "${pattern}" | |
| cros_archive_gsutil_ls_filter | |
| cros_archive_filter_url_version_newest "${regexp}" "${pattern}" |
| } |
| |
| # Usage: cros_archive_gs_get_url_version channel board url |
| # Gets the version string contained in the image URL for a particular channel |
| # and board. |
| # |
| # Writes the version string to stdout and returns success. If the URL does not |
| # match the pattern expected for the channel and board will produce no output |
| # and return failure. |
| cros_archive_gs_get_url_version() { |
| local channel="$1" |
| local board="$2" |
| local url="$3" |
| |
| local regexp=$(cros_archive_gs_get_url_regexp "${channel}" "${board}") |
| |
| if [[ ! ( ${url} =~ ^${regexp}$ ) || ${#BASH_REMATCH[@]} -lt 1 ]]; then |
| return 1 |
| fi |
| echo "${BASH_REMATCH[1]}" |
| } |