blob: b010881fd5a2730b0096e73216e58e0aba1760a6 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This library manages the interfaces to the signer for update payloads."""
from __future__ import print_function
import os
import re
import shutil
import tempfile
import time
import threading
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import gs
from chromite.lib.paygen import gslock
from chromite.lib.paygen import gspaths
# How long to sleep between polling GS to see if signer results are present.
# Signer priority value, slightly higher than the common value 50.
class SignerPayloadsClientGoogleStorage(object):
"""This class implements the Google Storage signer interface for payloads."""
def __init__(self, channel, board, version, bucket=None, unique=None,
"""This initializer identifies the build an payload that need signatures.
channel: Channel of the build whose payload is being signed.
board: Board of the build whose payload is being signed.
version: Version of the build whose payload is being signed.
bucket: Bucket used to reach the signer. [defaults 'chromeos-releases']
unique: Force known 'unique' id. Mostly for unittests.
ctx: GS Context to use for GS operations.
""" = channel
self.board = board
self.version = version
self.bucket = bucket if bucket else gspaths.ChromeosReleases.BUCKET
self._ctx = ctx if ctx is not None else gs.GSContext()
build_signing_uri = gspaths.ChromeosReleases.BuildPayloadsSigningUri(
# Uniquify the directory using our pid/thread-id. This can't collide
# with other hosts because the build is locked to our host in
# paygen_build.
if unique is None:
unique = '%d-%d' % (os.getpid(), threading.current_thread().ident)
# This is a partial URI that is extended for a lot of other URIs we use.
self.signing_base_dir = os.path.join(build_signing_uri, unique)
self.archive_uri = os.path.join(self.signing_base_dir,
def _CleanSignerFilesByKeyset(self, hashes, keyset, timeout=600):
"""Helper method that cleans up GS files associated with a single keyset.
hashes: A list of hash values to be signed by the signer in string
format. They are all expected to be 32 bytes in length.
keyset: keyset to have the hashes signed with.
timeout: Timeout for acquiring the lock on the files to clean.
gslock.LockNotAcquired if we can't get a lock on the data within timeout.
hash_names = self._CreateHashNames(len(hashes))
instructions_uri = self._CreateInstructionsURI(keyset)
request_uri = self._SignerRequestUri(instructions_uri)
signature_uris = self._CreateSignatureURIs(hash_names, keyset)
paths = [instructions_uri, request_uri]
paths += signature_uris
paths += [s + '.md5' for s in signature_uris]
end_time = time.time() + timeout
while True:
with gslock.Lock(request_uri + '.lock'):
for path in paths:
self._ctx.Remove(path, ignore_missing=True)
except gslock.LockNotAcquired:
# If we have timed out.
if time.time() > end_time:
def _CleanSignerFiles(self, hashes, keysets):
"""Helper method that cleans up all GS files associated with a signing.
Safe to call repeatedly.
hashes: A list of hash values to be signed by the signer in string
format. They are all expected to be 32 bytes in length.
keysets: list of keysets to have the hashes signed with.
May raise GSLibError if there is an extraordinary GS problem.
for keyset in keysets:
self._CleanSignerFilesByKeyset(hashes, keyset)
# After all keysets have been cleaned up, clean up the archive.
self._ctx.Remove(self.signing_base_dir, recursive=True, ignore_missing=True)
def _CreateInstructionsURI(self, keyset):
"""Construct the URI used to upload a set of instructions.
keyset: name of the keyset contained in this instruction set.
URI for the given instruction set as a string.
return os.path.join(self.signing_base_dir,
'%s.payload.signer.instructions' % keyset)
def _CreateHashNames(self, hash_count):
"""Helper method that creates file names for each hash in GS.
These names are arbitrary, and only used when working with the signer.
hash_count: How many hash names are needed?
result = []
for i in xrange(1, hash_count + 1):
result.append('%d.payload.hash' % i)
return result
def _CreateSignatureURIs(self, hash_names, keyset):
"""Helper method that creates URIs for the signature output files.
These names are the actual URIs the signer will populate with ".bin"
already included.
hash_names: The list of input_names passed to the signer.
keyset: Keyset name passed to the signer.
List of URIs expected back from the signer.
result = []
for hash_name in hash_names:
# Based on the pattern defined in _CreateInstructions.
expanded_name = '%s.%s.signed.bin' % (hash_name, keyset)
result.append(os.path.join(self.signing_base_dir, expanded_name))
return result
def _CreateArchive(self, archive_file, hashes, hash_names):
"""Take the hash strings and bundle them in the signer request format.
Take the contents of an array of strings, and put them into a specified
file in .tar.bz2 format. Each string is named with a specified name in
the tar file.
The number of hashes and number of hash_names must be equal. The
archive_file will be created or overridden as needed. It's up to
the caller to ensure it's cleaned up.
archive_file: Name of file to put the tar contents into.
hashes: List of hashes to sign, stored in strings.
hash_names: File names expected in the signer request.
tmp_dir = tempfile.mkdtemp()
# Copy hash files into tmp_dir with standard hash names.
for h, hash_name in zip(hashes, hash_names):
with open(os.path.join(tmp_dir, hash_name), 'wb') as f:
cmd = ['tar', '-cjf', archive_file] + hash_names
cmd, redirect_stdout=True, redirect_stderr=True, cwd=tmp_dir)
# Cleanup.
def _CreateInstructions(self, hash_names, keyset):
"""Create the signing instructions to send to the signer.
hash_names: The names of the hash files in the archive to sign.
keyset: Which keyset to sign the hashes with. Valid keysets are
defined on the signer. 'update_signer' is currently valid.
A string that contains the contents of the instructions to send.
pattern = """
# Auto-generated instruction file for signing payload hashes.
generate_metadata = false
keyset = %(keyset)s
channel = %(channel)s
input_files = %(input_files)s
output_names = @BASENAME@.@KEYSET@.signed
archive = metadata-disable.instructions
type = update_payload
board = %(board)s
archive = %(archive_name)s
# We reuse version for version rev because we may not know the
# correct versionrev "R24-1.2.3"
version = %(version)s
versionrev = %(version)s
# foo-channel -> foo
channel ='-channel', '')
archive_name = os.path.basename(self.archive_uri)
input_files = ' '.join(hash_names)
return pattern % {
'channel': channel,
'board': self.board,
'version': self.version,
'archive_name': archive_name,
'input_files': input_files,
'keyset': keyset,
def _SignerRequestUri(self, instructions_uri):
"""Find the URI of the empty file to create to ask the signer to sign."""
exp = r'^gs://%s/(?P<postbucket>.*)$' % self.bucket
m = re.match(exp, instructions_uri)
relative_uri ='postbucket')
return 'gs://%s/tobesigned/%d,%s' % (
relative_uri.replace('/', ','))
def _WaitForSignatures(self, signature_uris, timeout=1800):
"""Wait until all uris exist, or timeout.
signature_uris: list of uris to check for.
timeout: time in seconds to wait for all uris to be created.
True if the signatures all exist, or False.
end_time = time.time() + timeout
missing_signatures = signature_uris[:]
while missing_signatures and time.time() < end_time:
while missing_signatures and self._ctx.Exists(missing_signatures[0]):
if missing_signatures:
# If none are missing, we found them all.
return not missing_signatures
def _DownloadSignatures(self, signature_uris):
"""Download the list of URIs to in-memory strings.
signature_uris: List of URIs to download.
List of signatures in strings.
results = []
for uri in signature_uris:
with tempfile.NamedTemporaryFile(delete=False) as sig_file:
sig_file_name =
self._ctx.Copy(uri, sig_file_name)
with open(sig_file_name) as sig_file:
# Cleanup the temp file, in case it's still there.
if os.path.exists(sig_file_name):
return results
def GetHashSignatures(self, hashes, keysets=('update_signer',)):
"""Take an arbitrary list of hash files, and get them signed.
hashes: A list of hash values to be signed by the signer in string
format. They are all expected to be 32 bytes in length.
keysets: list of keysets to have the hashes signed with. The default
is almost certainly what you want. These names must match
valid keysets on the signer.
A dictionary keyed by hash with a list of signatures in string format.
The list of signatures will correspond to the list of keysets passed
hashes, keysets=['update_signer', 'update_signer-v2'] ->
{ hashes[0] : [sig_update_signer, sig_update_signer-v2], ... }
Returns None if the process failed.
Can raise a variety of GSLibError errors in extraordinary conditions.
# Hash and signature names.
hash_names = self._CreateHashNames(len(hashes))
# Create and upload the archive of hashes to sign.
with tempfile.NamedTemporaryFile() as archive_file:
self._CreateArchive(, hashes, hash_names)
self._ctx.Copy(, self.archive_uri)
# [sig_uri, ...]
all_signature_uris = []
# { hash : [sig_uri, ...], ... }
hash_signature_uris = dict([(h, []) for h in hashes])
# Upload one signing instruction file and signing request for
# each keyset.
for keyset in keysets:
instructions_uri = self._CreateInstructionsURI(keyset)
self._CreateInstructions(hash_names, keyset))
# Create signer request file with debug friendly contents.
# Remember which signatures we just requested.
uris = self._CreateSignatureURIs(hash_names, keyset)
all_signature_uris += uris
for h, sig_uri in zip(hashes, uris):
# Wait for the signer to finish all keysets.
if not self._WaitForSignatures(all_signature_uris):
logging.error('Signer request timed out.')
return None
# Download the results.
return [self._DownloadSignatures(hash_signature_uris[h]) for h in hashes]
# Clean up the signature related files from this run.
self._CleanSignerFiles(hashes, keysets)