dev: add cros_archive.sh library to search archived images
Fetching images by version requires the ability to search in the
gs:// archives. Reading LATEST directly will be faster, but does
not help when bisecting problems.
This library supports parsing and formatting of CrOS versions and
several functions specific to the current organization of
gs://chromeos-image-archive and gs://chromeos-releases. It is
used by the cros_fetch_image script that I hope to get uploaded
shortly.
BUG=chromium-os:27665
TEST=unittests and gsutil cp the generated URLs.
Change-Id: I532591b17ce75c182bef9bf9c6fb6711bb812bb9
Reviewed-on: https://gerrit.chromium.org/gerrit/21517
Reviewed-by: Chris Wolfe <cwolfe@chromium.org>
Tested-by: Chris Wolfe <cwolfe@chromium.org>
Commit-Ready: Chris Wolfe <cwolfe@chromium.org>
diff --git a/host/lib/cros_archive.sh b/host/lib/cros_archive.sh
new file mode 100644
index 0000000..1a9078b
--- /dev/null
+++ b/host/lib/cros_archive.sh
@@ -0,0 +1,380 @@
+#!/bin/bash
+#
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Shared library for searching and choosing CrOS images from a large collection
+# of archived versions.
+#
+# This library deals with CrOS versions in four forms:
+#
+# version string: e.g. "R12-3456.7.8"
+# Completely specifies a CrOS version as release, major, minor and revision
+# numbers. The revision number may be followed by a '-' or '.' and arbitrary
+# characters, which will be ignored (e.g. "R12-3456.7.8-a9-b123").
+#
+# version pattern: e.g. "R12-3456", "3456.7", etc.
+# Specifies a partial CrOS version by omitting some of the numbers. For
+# example, all of the following patterns would match "R12-3456.7.8":
+# '', R12, R12-3456, R12-3456.7, R12-3456.7.8, 3456, 3456.7, 3456.7.8
+#
+# numeric version: e.g. "12 3456 7 8"
+# A version written as four space-separated integers corresponding to the
+# release, major, minor and revision numbers. The last integer may be
+# followed by another space and arbitrary characters, which will be ignored.
+#
+# numeric pattern: e.g. "12 3456 -1 -1"
+# A version pattern written as four space-separated integers corresponding to
+# the release, major, minor and revision numbers. Unrestricted fields are
+# indicated by negative values. The last integer may be followed by another
+# space and arbitrary characters, which will be ignored.
+
+### GENERIC FUNCTIONS
+
+# Usage: cros_archive_get_numeric_version version
+# Generates a numeric version for the specified version string.
+#
+# Writes the numeric version to stdout and returns success. If the version
+# string is malformed, will produce no output and return failure.
+cros_archive_get_numeric_version() {
+ local arg_{rel,ver,maj,min,rev,tail}
+
+ # BUG(cwolfe):
+ # For some reason, attaching the next IFS to its read results in the IFS
+ # being ignored during cros_archive_filter_url_version_newest. This only
+ # occurs when run in the chroot (bash 4.2.20) and not on my workstation
+ # (bash 4.1.5). Hacking it to work for the moment, and need to figure out
+ # whether I've discovered a weird feature or a bash bug.
+ #
+ # Absent the bug, the next four lines should be:
+ # IFS='-' read -r arg_{rel,ver,tail} <<<"$1"
+ local old_ifs="${IFS}"
+ IFS='-'
+ read -r arg_{rel,ver,tail} <<<"$1"
+ IFS="${old_ifs}"
+
+ IFS='.' read -r arg_{maj,min,rev,tail} <<<"${arg_ver}"
+ arg_rel=${arg_rel#R}
+ if [[ -z ${arg_rel} || -z ${arg_maj} ||
+ -z ${arg_min} || -z ${arg_rev} ]]; then
+ return 1 # Malformed version
+ fi
+
+ printf '%d %d %d %d\n' "${arg_rel}" "${arg_maj}" "${arg_min}" "${arg_rev}"
+}
+
+# Usage: cros_archive_get_numeric_pattern version
+# Generates a numeric pattern corresponding to the specified version pattern.
+#
+# Writes the numeric pattern to stdout and returns success. If the version
+# pattern is malformed or contains extra characters, will produce no output and
+# return failure.
+cros_archive_get_numeric_pattern() {
+ local arg_{rel,ver,maj,min,rev,tail}
+
+ IFS='-' read -r arg_{rel,ver,tail} <<<"$1"
+ if [[ -n ${arg_tail} ]]; then
+ return 1 # Malformed pattern: extra characters
+ fi
+ if [[ ${arg_rel:0:1} == 'R' ]]; then
+ arg_rel=${arg_rel#R} # Strip the 'R' prefix
+ elif [[ -n ${arg_rel} ]]; then
+ arg_ver=${arg_rel} # Actually got a version with no release.
+ arg_rel=
+ fi
+ if [[ -n ${arg_ver} ]]; then
+ IFS='.' read -r arg_{maj,min,rev,tail} <<<"${arg_ver}"
+ if [[ -n ${arg_tail} ]]; then
+ return 1 # Malformed pattern: extra characters
+ fi
+ fi
+
+ printf '%d %d %d %d\n' \
+ "${arg_rel:--1}" "${arg_maj:--1}" "${arg_min:--1}" "${arg_rev:--1}"
+}
+
+# Usage: cros_archive_get_wildcard_pattern version
+# Generates a wildcard pattern that will match the specified numeric version.
+#
+# Writes the wildcard pattern to stdout and returns success. If the numeric
+# version is malformed, will produce no output and return failure.
+cros_archive_get_wildcard_pattern() {
+ local arg_{rel,maj,min,rev}
+ read -r arg_{rel,maj,min,rev} <<<"$1"
+
+ # Replace any negative fields with '*'.
+ [[ ${arg_rel} -ge 0 ]] || arg_rel='*'
+ [[ ${arg_maj} -ge 0 ]] || arg_maj='*'
+ [[ ${arg_min} -ge 0 ]] || arg_min='*'
+ [[ ${arg_rev} -ge 0 ]] || arg_rev='*'
+
+ printf "R%s-%s.%s.%s\n" "${arg_rel}" "${arg_maj}" "${arg_min}" "${arg_rev}"
+}
+
+# Usage: cros_archive_test_numeric_version_matches pattern version
+# Tests whether a numeric pattern matches a numeric version.
+#
+# Returns success if the pattern matches the version and failure otherwise.
+cros_archive_test_numeric_version_matches() {
+ local -a pattern version
+ read -ra pattern <<<"$1"
+ read -ra version <<<"$2"
+ if [[ ${#pattern[@]} -lt 4 || ${#version[@]} -lt 4 ]]; then
+ return 1 # Malformed pattern or version
+ fi
+
+ local i
+ for i in {0..3}; do
+ if [[ ${pattern[$i]} -ge 0 && ${pattern[$i]} -ne ${version[$i]} ]]; then
+ return 1 # Version does not match pattern
+ fi
+ done
+ return 0 # Version matches pattern
+}
+
+# Usage: cros_archive_test_numeric_version_newer first second
+# Tests whether the first numeric version is newer than the second.
+#
+# Returns success if the first numeric version is newer than the second,
+# and failure otherwise.
+cros_archive_test_numeric_version_newer() {
+ local -a first second
+ read -ra first <<<"$1"
+ read -ra second <<<"$2"
+ if [[ ${#first[@]} -lt 4 || ${#second[@]} -lt 4 ]]; then
+ return 1 # Malformed version
+ fi
+
+ local i
+ for i in {0..3}; do
+ if [[ ${first[$i]} -ne ${second[$i]} ]]; then
+ if [[ ${first[$i]} -gt ${second[$i]} ]]; then
+ return 0 # First is indeed newer then second.
+ else
+ return 1 # First is older than second.
+ fi
+ fi
+ done
+ return 1 # First and second are equal.
+}
+
+# Usage: cros_archive_filter_numeric_version_matches pattern
+# Filters a stream of numeric versions, keeping those which match the specified
+# numeric pattern.
+#
+# Reads lines from stdin. Each line must be a numeric version. If a line
+# matches the pattern it will be written to stdout in its entirety, otherwise
+# the line will be discarded.
+#
+# Returns success at the end if input.
+cros_archive_filter_numeric_version_matches() {
+ local pattern="$1"
+ local version
+ while read -r version; do
+ if cros_archive_test_numeric_version_matches "${pattern}" "${version}"; then
+ printf '%s\n' "${version}"
+ fi
+ done
+}
+
+# Usage: cros_archive_filter_numeric_version_newest
+# Filters a stream of numeric versions to choose the newest.
+#
+# Reads lines from stdin. Each line must be a numeric version. Once the end of
+# input has been reached, the line containing the newest numeric version will
+# be written to stdout in its entirety. Will not produce any output if no
+# versions have been read.
+#
+# Returns success unless an error was encountered, even if no versions were
+# processed.
+cros_archive_filter_numeric_version_newest() {
+ local best
+ local curr
+ while read -r curr; do
+ if [[ -z ${best} ]] ||
+ cros_archive_test_numeric_version_newer "${curr}" "${best}"; then
+ best="${curr}"
+ fi
+ done
+ if [[ -n ${best} ]]; then
+ printf '%s\n' "${best}"
+ fi
+}
+
+# Usage: cros_archive_filter_url_version_newest regexp [pattern]
+# Filters a stream of URLs to choose the one containing the newest version
+# string. Optionally discards versions which do not match the specified numeric
+# pattern.
+#
+# The regexp must contain a single parenthesized subexpression that selects the
+# version string from a URL. Any URL which does not match this regexp or does
+# not record a parenthesized subexpression will be discarded.
+#
+# Reads lines from stdin, each of which must consist of a single URL. Once the
+# end of input has been reached, the URL containing the newest version string
+# will be written to stdout.
+#
+# Returns success unless an error was encountered, even if no URLs were found.
+cros_archive_filter_url_version_newest() {
+ local regexp="$1"
+ local pattern="${2:--1 -1 -1 -1}"
+
+ get_numeric_result() {
+ local input
+ while read -r input; do
+ if [[ ! ( ${input} =~ ^${regexp}$ ) || ${#BASH_REMATCH[@]} -lt 1 ]]; then
+ continue
+ fi
+ local version=$(cros_archive_get_numeric_version "${BASH_REMATCH[1]}")
+ printf '%s %s\n' "${version}" "${input}"
+ done |
+ cros_archive_filter_numeric_version_matches "${pattern}" |
+ cros_archive_filter_numeric_version_newest
+ }
+
+ local -a result
+ if ! read -a result < <(get_numeric_result); then
+ return 0 # No URL available
+ fi
+
+ # Drop the first four elements in the result array, as they contain the
+ # numeric version.
+ printf '%s\n' "${result[*]:4}"
+}
+
+### GOOGLE STORAGE SPECIFIC FUNCTIONS
+#
+# This code addresses the problem of getting images from a GS archive in two
+# stages: it generates a set of wildcards passed to "gsutil ls", and then uses
+# a regexp to parse the version numbers out of the list. These two steps are
+# necessary because gsutil supports only simple glob-style wildcards and can
+# not do version-number ordering of the results. Fortunately the processing
+# can be run in parallel to the "gsutil ls", so the overhead is small.
+#
+
+# Usage: cros_archive_gs_get_url_wildcards channel board [pattern]
+# Gets one or more URL wildcard for a specified channel and board which will
+# include at least the archives corresponding to the specified numeric pattern.
+#
+# The 'archive' virtual channel is loaded from gs://chromeos-image-archive.
+# All other channels will be requested from gs://chromeos-releases.
+cros_archive_gs_get_url_wildcards() {
+ local channel="$1"
+ local board="$2"
+ local pattern=$(cros_archive_get_wildcard_pattern "${3:--1 -1 -1 -1}")
+
+ # Add a '*' after the pattern, as some of the uploaded paths (but not all)
+ # contain additional numbers for attempt and build like "-a0-b1234". To
+ # avoid doubling '*'s, also remove any trailing '*' from the pattern.
+ pattern="${pattern%\*}*"
+
+ if [[ ${channel} == 'archive' ]]; then
+ # Most boards use a -release builder, so output that URL first.
+ local suffix
+ for suffix in release full; do
+ printf 'gs://chromeos-image-archive/%s-%s/%s/image.zip\n' \
+ "${board}" "${suffix}" "${pattern}"
+ done
+ else
+ printf 'gs://chromeos-releases/%s/%s/%s/ChromeOS-%s-%s.zip\n' \
+ "${channel}-channel" "${board}" "${pattern#R*-}" "${pattern}" \
+ "${board}"
+ fi
+}
+
+# Usage: cros_archive_gs_get_url_regexp channel board
+# Gets a regexp which can be used to extract the version from URLs associated
+# with the specified channel and board.
+#
+# The 'archive' virtual channel is loaded from gs://chromeos-image-archive.
+# All other channels will be requested from gs://chromeos-releases.
+cros_archive_gs_get_url_regexp() {
+ local channel="$1"
+ local board="$2"
+ local pattern=$(cros_archive_get_wildcard_pattern "${3:--1 -1 -1 -1}")
+
+ # The version_regexp picks out and records two dash-separated components that
+ # look like a CrOS version. In some cases it will be followed by another dash
+ # and additional information, so we need to be a little careful about what it
+ # matches.
+ local version_regexp='(R[^/-]*-[^/-]*)'
+ if [[ ${channel} == 'archive' ]]; then
+ printf 'gs://chromeos-image-archive/%s[^/]*/%s[^/]*/image.zip\n' \
+ "${board}" "${version_regexp}"
+ else
+ printf 'gs://chromeos-releases/%s/%s/[^/]*/ChromeOS-%s[^/]*-%s.zip\n' \
+ "${channel}-channel" "${board}" "${version_regexp}" "${board}"
+ fi
+}
+
+# Usage: cros_archive_gsutil_ls_filter
+# Gets multiple directory listings from gsutil ls.
+#
+# Reads lines from stdin. Each line must be a gs:// URL, possibly containing
+# wildcards. Will write zero or more lines to stdout containing the URLs which
+# were returned from the server.
+#
+# Errors from gsutil related to there being no matching URLs will be ignored.
+# Returns success unless an error was encountered, even if no URLs were found.
+#
+# Environment:
+# GSUTIL: name of gsutil program (default is 'gsutil')
+cros_archive_gsutil_ls_filter() {
+ local url
+ while read -r url; do
+ # Run gsutil with its stdout passed through and its stderr recorded.
+ local rc=0
+ exec 3>&1
+ local error=$( ${GSUTIL:-gsutil} ls "${url}" 2>&1 1>&3 3>&- ) || rc=$?
+ exec 3>&-
+
+ # Ignore errors that only indicate no matches were found. This behavior
+ # may vary between different versions of gsutil, so we should not assume
+ # an empty result sets non-zero rc.
+ if [[ ${rc} -ne 0 && ${error} != 'No matches for'* ]]; then
+ printf 'gsutil: %s\n' "${error}" >&2
+ return 1
+ fi
+ done
+}
+
+# Usage: cros_archive_gs_list channel board [pattern]
+# Gets the list of URLs matching the specified channel and board which will
+# satisfy the specified numeric pattern.
+#
+# This command calls the Google Storage backend to get a list of available
+# files.
+#
+# Returns success unless an error was encountered, even if no URLs were found.
+cros_archive_gs_list() {
+ local channel="$1"
+ local board="$2"
+ local pattern=$(cros_archive_get_numeric_pattern "$3")
+
+ local regexp=$(cros_archive_gs_get_url_regexp "${channel}" "${board}")
+
+ cros_archive_gs_get_url_wildcards "${channel}" "${board}" "${pattern}" |
+ cros_archive_gsutil_ls_filter |
+ cros_archive_filter_url_version_newest "${regexp}" "${pattern}"
+}
+
+# Usage: cros_archive_gs_get_url_version channel board url
+# Gets the version string contained in the image URL for a particular channel
+# and board.
+#
+# Writes the version string to stdout and returns success. If the URL does not
+# match the pattern expected for the channel and board will produce no output
+# and return failure.
+cros_archive_gs_get_url_version() {
+ local channel="$1"
+ local board="$2"
+ local url="$3"
+
+ local regexp=$(cros_archive_gs_get_url_regexp "${channel}" "${board}")
+
+ if [[ ! ( ${url} =~ ^${regexp}$ ) || ${#BASH_REMATCH[@]} -lt 1 ]]; then
+ return 1
+ fi
+ echo "${BASH_REMATCH[1]}"
+}
diff --git a/host/lib/cros_archive_unittest.sh b/host/lib/cros_archive_unittest.sh
new file mode 100755
index 0000000..961c660
--- /dev/null
+++ b/host/lib/cros_archive_unittest.sh
@@ -0,0 +1,379 @@
+#!/bin/bash
+#
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+
+set -o errexit
+set -o pipefail
+
+. "$(dirname "$(readlink -f "$0")")"/cros_archive.sh
+
+### TESTING FUNCTIONS
+
+total_passed=0
+total_failed=0
+
+expect() {
+ local caller_line="${BASH_SOURCE[1]}:${BASH_LINENO[0]}"
+ local output='*'
+ local status='0'
+ unset OPTIND
+ while getopts ':o:s:' OPTOPT; do
+ case ${OPTOPT} in
+ 'o') output=${OPTARG} ;;
+ 's') status=${OPTARG} ;;
+ '?') echo "expect: error processing option ${OPTARG}" \
+ "(called from ${caller_line})"
+ exit 2
+ ;;
+ esac
+ done
+ shift $(( OPTIND - 1 ))
+ local cmd=( "$@" )
+
+ local actual_output
+ local actual_status=0
+ actual_output=$( "${cmd[@]}" ) || actual_status=$?
+
+ local wrong_status
+ local wrong_output
+ if [[ ${status} != '*' && ${actual_status} -ne ${status} ]]; then
+ wrong_status='yes'
+ fi
+ if [[ ${output} != '*' && ${actual_output} != "${output}" ]]; then
+ wrong_output='yes'
+ fi
+
+ if [[ -n ${wrong_status} || -n ${wrong_output} ]]; then
+ echo "FAIL at ${caller_line}: ${cmd[@]}"
+ if [[ -n ${wrong_status} ]]; then
+ echo " exit status was ${actual_status}, expected ${status}"
+ fi
+ if [[ -n ${wrong_output} ]]; then
+ echo " the command produced unexpected content on stdout"
+ echo " expected: '${output}'"
+ echo " received: '${actual_output}'"
+ fi
+ echo
+ : $(( ++total_failed ))
+ else
+ : $(( ++total_passed ))
+ fi
+}
+
+### TESTS
+
+### cros_archive_get_numeric_version
+
+expect -o '12 3456 7 8' \
+ cros_archive_get_numeric_version 'R12-3456.7.8'
+
+expect -o '12 3456 7 8' \
+ cros_archive_get_numeric_version 'R12-3456.7.8-a9-b123'
+
+### cros_archive_get_numeric_pattern
+
+expect -o '-1 -1 -1 -1' \
+ cros_archive_get_numeric_pattern ''
+
+expect -o '12 -1 -1 -1' \
+ cros_archive_get_numeric_pattern 'R12'
+
+expect -o '12 3456 -1 -1' \
+ cros_archive_get_numeric_pattern 'R12-3456'
+
+expect -o '12 3456 7 -1' \
+ cros_archive_get_numeric_pattern 'R12-3456.7'
+
+expect -o '12 3456 7 8' \
+ cros_archive_get_numeric_pattern 'R12-3456.7.8'
+
+expect -o '-1 3456 -1 -1' \
+ cros_archive_get_numeric_pattern '3456'
+
+expect -o '-1 3456 7 -1' \
+ cros_archive_get_numeric_pattern '3456.7'
+
+expect -o '-1 3456 7 8' \
+ cros_archive_get_numeric_pattern '3456.7.8'
+
+### cros_archive_get_wildcard_pattern
+
+expect -o 'R*-*.*.*' \
+ cros_archive_get_wildcard_pattern '-1 -1 -1 -1'
+
+expect -o 'R12-*.*.*' \
+ cros_archive_get_wildcard_pattern '12 -1 -1 -1'
+
+expect -o 'R12-3456.*.*' \
+ cros_archive_get_wildcard_pattern '12 3456 -1 -1'
+
+expect -o 'R12-3456.7.*' \
+ cros_archive_get_wildcard_pattern '12 3456 7 -1'
+
+expect -o 'R12-3456.7.8' \
+ cros_archive_get_wildcard_pattern '12 3456 7 8'
+
+expect -o 'R*-3456.*.*' \
+ cros_archive_get_wildcard_pattern '-1 3456 -1 -1'
+
+expect -o 'R*-3456.7.*' \
+ cros_archive_get_wildcard_pattern '-1 3456 7 -1'
+
+expect -o 'R*-3456.7.8' \
+ cros_archive_get_wildcard_pattern '-1 3456 7 8'
+
+### cros_archive_test_numeric_version_matches
+
+expect cros_archive_test_numeric_version_matches \
+ '-1 -1 -1 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '12 -1 -1 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '12 3456 -1 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '12 3456 7 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '12 3456 7 8' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '-1 3456 -1 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '-1 3456 7 -1' '12 3456 7 8'
+
+expect cros_archive_test_numeric_version_matches \
+ '-1 3456 7 8' '12 3456 7 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '12 -1 -1 -1' '99 3456 7 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '12 3456 -1 -1' '12 9999 7 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '12 3456 7 -1' '12 3456 9 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '12 3456 7 8' '12 3456 7 9'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '-1 3456 -1 -1' '12 9999 7 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '-1 3456 7 -1' '12 3456 9 8'
+
+expect -s 1 cros_archive_test_numeric_version_matches \
+ '-1 3456 7 8' '12 3456 7 9'
+
+### cros_archive_test_numeric_version_newer
+
+expect cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '11 3456 7 8'
+
+expect cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3455 7 8'
+
+expect cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3456 6 8'
+
+expect cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3456 7 7'
+
+expect -s 1 cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3456 7 8'
+
+expect -s 1 cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '13 1 1 1'
+
+expect -s 1 cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3457 1 1'
+
+expect -s 1 cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3456 8 1'
+
+expect -s 1 cros_archive_test_numeric_version_newer \
+ '12 3456 7 8' '12 3456 7 9'
+
+### cros_archive_filter_numeric_version_matches
+
+numeric_version_list="\
+12 3456 7 8
+11 3456 7 8
+13 3456 7 8
+12 3455 7 8
+12 3457 7 8
+12 3456 6 8
+12 3456 8 8
+12 3456 7 7
+12 3456 7 9
+"
+
+expect -o $'12 3456 7 8' \
+ cros_archive_filter_numeric_version_matches '12 3456 7 8' \
+ <<<"${numeric_version_list}"
+
+expect -o $'12 3456 7 8\n11 3456 7 8\n13 3456 7 8' \
+ cros_archive_filter_numeric_version_matches '-1 3456 7 8' \
+ <<<"${numeric_version_list}"
+
+expect -o $'12 3456 7 8\n12 3455 7 8\n12 3457 7 8' \
+ cros_archive_filter_numeric_version_matches '12 -1 7 8' \
+ <<<"${numeric_version_list}"
+
+expect -o $'12 3456 7 8\n12 3456 6 8\n12 3456 8 8' \
+ cros_archive_filter_numeric_version_matches '12 3456 -1 8' \
+ <<<"${numeric_version_list}"
+
+expect -o $'12 3456 7 8\n12 3456 7 7\n12 3456 7 9' \
+ cros_archive_filter_numeric_version_matches '12 3456 7 -1' \
+ <<<"${numeric_version_list}"
+
+### cros_archive_filter_numeric_version_newest
+
+expect -o '13 3456 7 8' \
+ cros_archive_filter_numeric_version_newest \
+ <<<"${numeric_version_list}"
+
+numeric_version_list="\
+1 1 1 1
+1 1 1 0
+1 1 0 0
+1 0 0 0
+0 0 0 0
+"
+
+expect -o '1 1 1 1' \
+ cros_archive_filter_numeric_version_newest \
+ <<<"${numeric_version_list}"
+
+### cros_archive_filter_url_version_newest
+
+url_regexp='http://example.net/(.*)/image.zip'
+
+expect -o '' \
+ cros_archive_filter_url_version_newest "${url_regexp}" \
+ <<<""
+
+url_list="\
+http://example.net/R12-3456.7.8/image.zip
+http://example.net/R11-3456.7.8/image.zip
+http://example.net/R13-3456.7.8/image.zip
+http://example.net/R12-3455.7.8/image.zip
+http://example.net/R12-3457.7.8/image.zip
+http://example.net/R12-3456.6.8/image.zip
+http://example.net/R12-3456.8.8/image.zip
+http://example.net/R12-3456.7.7/image.zip
+http://example.net/R12-3456.7.9/image.zip"
+
+expect -o "\
+http://example.net/R13-3456.7.8/image.zip" \
+ cros_archive_filter_url_version_newest "${url_regexp}" \
+ <<<"${url_list}"
+
+url_list="\
+http://example.net/R1-1.1.1/image.zip
+http://example.net/R1-1.1.0/image.zip
+http://example.net/R1-1.0.0/image.zip
+http://example.net/R1-0.0.0/image.zip
+http://example.net/R0-0.0.0/image.zip"
+
+expect -o 'http://example.net/R1-1.1.1/image.zip' \
+ cros_archive_filter_url_version_newest "${url_regexp}" \
+ <<<"${url_list}"
+
+### cros_archive_gs_get_url_wildcards
+
+expect -o "\
+gs://chromeos-image-archive/x86-generic-release/R*-*.*.*/image.zip
+gs://chromeos-image-archive/x86-generic-full/R*-*.*.*/image.zip" \
+ cros_archive_gs_get_url_wildcards 'archive' 'x86-generic'
+
+expect_base='gs://chromeos-releases/canary-channel/x86-alex'
+expect -o "${expect_base}/*.*.*/ChromeOS-R*-*.*.*-x86-alex.zip" \
+ cros_archive_gs_get_url_wildcards 'canary' 'x86-alex'
+
+### cros_archive_gs_get_url_regexp
+
+url_list="\
+gs://chromeos-image-archive/x86-generic-full/R18-1660.0.0-a1-b1626/image.zip
+gs://chromeos-image-archive/x86-generic-full/R20-2223.0.0-a1-b3069/image.zip"
+regexp=$(cros_archive_gs_get_url_regexp 'archive' 'x86-generic')
+expect -o "${url_list}" grep -E "${regexp}" <<<"${url_list}"
+
+expect_base='gs://chromeos-releases/canary-channel'
+url_list="\
+${expect_base}/x86-alex/1590.2.0/ChromeOS-R18-1590.2.0-a1-b18-x86-alex.zip
+${expect_base}/x86-alex/2223.0.0/ChromeOS-R20-2223.0.0-x86-alex.zip"
+regexp=$(cros_archive_gs_get_url_regexp 'canary' 'x86-alex')
+expect -o "${url_list}" grep -E "${regexp}" <<<"${url_list}"
+
+### cros_archive_gs_list
+
+archive_base='gs://chromeos-image-archive/x86-generic-full'
+canary_base='gs://chromeos-releases/canary-channel/x86-alex'
+url_list="\
+${archive_base}/R18-1659.0.0-a1-b1625/image.zip
+${archive_base}/R18-1660.0.0-a1-b1626/image.zip
+${archive_base}/R20-2223.0.0-a1-b3069/image.zip
+${canary_base}/1589.0.0/ChromeOS-R18-1589.0.0-a1-b17-x86-alex.zip
+${canary_base}/1590.2.0/ChromeOS-R18-1590.2.0-a1-b18-x86-alex.zip
+${canary_base}/2223.0.0/ChromeOS-R20-2223.0.0-x86-alex.zip"
+
+mock_gsutil() {
+ if [[ $# -ne 2 || $1 != 'ls' ]]; then
+ echo 'Mock expected usage to be "ls URL"' >&2
+ return 1
+ fi
+ local pattern="$2"
+ local found
+ local url
+ while read -r url; do
+ if [[ ${url} == ${pattern} ]]; then
+ echo ${url}
+ found='yes'
+ fi
+ done <<<"${url_list}"
+ if [[ -z ${found} ]]; then
+ echo "No matches for ${pattern}" >&2
+ return 1
+ fi
+}
+
+GSUTIL=mock_gsutil
+expect -o '' \
+ cros_archive_gs_list archive x86-generic R17
+expect -o "${archive_base}/R18-1660.0.0-a1-b1626/image.zip" \
+ cros_archive_gs_list archive x86-generic R18
+expect -o "${archive_base}/R20-2223.0.0-a1-b3069/image.zip" \
+ cros_archive_gs_list archive x86-generic
+expect -o '' \
+ cros_archive_gs_list canary x86-alex R17
+expect -o "${canary_base}/1590.2.0/ChromeOS-R18-1590.2.0-a1-b18-x86-alex.zip" \
+ cros_archive_gs_list canary x86-alex R18
+expect -o "${canary_base}/2223.0.0/ChromeOS-R20-2223.0.0-x86-alex.zip" \
+ cros_archive_gs_list canary x86-alex
+
+expect -o 'R18-1660.0.0' \
+ cros_archive_gs_get_url_version archive x86-generic \
+ "${archive_base}/R18-1660.0.0-a1-b1626/image.zip"
+expect -o 'R20-2223.0.0' \
+ cros_archive_gs_get_url_version archive x86-generic \
+ "${archive_base}/R20-2223.0.0-a1-b3069/image.zip"
+
+expect -o 'R18-1590.2.0' \
+ cros_archive_gs_get_url_version canary x86-alex \
+ "${canary_base}/1590.2.0/ChromeOS-R18-1590.2.0-a1-b18-x86-alex.zip"
+expect -o 'R20-2223.0.0' \
+ cros_archive_gs_get_url_version canary x86-alex \
+ "${canary_base}/2223.0.0/ChromeOS-R20-2223.0.0-x86-alex.zip"
+
+if [[ ${total_failed} -gt 0 ]]; then
+ exit 1
+fi
