fake_omaha: add a nebraska wrapper to handle update requests
This CL adds a new class that lets gs_archive_server handle the update
RPC requests from DUTs. This class is essentially a wrapper for
nebraska.py. This class also downloads the nebraska properties file into
the /tmp directory and passes that directory to nebraska.
BUG=chromium:1078188
TEST=Ran update_engine_client --omaha_url=<url> --follow on DUT. Output
and exact command can be found at http://gpaste/5083884435800064.
Cq-Depend: chrome-internal:3141905
Change-Id: Ia8559c698e84f15540c4814d14cabafe731e5f55
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2258234
Tested-by: Sanika Kulkarni <sanikak@chromium.org>
Reviewed-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Congbin Guo <guocb@chromium.org>
Commit-Queue: Sanika Kulkarni <sanikak@chromium.org>
Auto-Submit: Sanika Kulkarni <sanikak@chromium.org>
diff --git a/gs_cache/fake_omaha.py b/gs_cache/fake_omaha.py
index 6af5307..8660fca 100644
--- a/gs_cache/fake_omaha.py
+++ b/gs_cache/fake_omaha.py
@@ -11,7 +11,8 @@
from __future__ import division
from __future__ import print_function
-import cherrypy
+import cherrypy # pylint: disable=import-error
+import nebraska_wrapper
def get_config():
@@ -28,7 +29,11 @@
"""An application to handle fake Omaha requests."""
def POST(self, *args, **kwargs):
"""A URL handler to handle update check ping."""
+ label = '/'.join(args)
+ full_update = kwargs.pop('full_update', 'unspecified')
+ server_addr = cherrypy.request.headers.get('X-Server-Addr')
body_length = int(cherrypy.request.headers.get('Content-Length', 0))
data = cherrypy.request.rfile.read(body_length)
- return 'Fake Omaha: To be implemented\nArgs: %s\nkwargs: %s\nData: %s\n' % (
- args, kwargs, data)
+ with nebraska_wrapper.NebraskaWrapper(label, server_addr,
+ full_update) as nb:
+ return nb.HandleUpdatePing(data, **kwargs)
diff --git a/gs_cache/nebraska_wrapper.py b/gs_cache/nebraska_wrapper.py
new file mode 100644
index 0000000..c78a3cf
--- /dev/null
+++ b/gs_cache/nebraska_wrapper.py
@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A Nebraska Wrapper to handle update client requests."""
+
+from __future__ import print_function
+
+import os
+import re
+import shutil
+import tempfile
+
+import requests
+from six.moves import urllib
+
+import cherrypy # pylint: disable=import-error
+
+from nebraska import nebraska
+from chromite.lib import cros_logging as logging
+
+
+# Define module logger.
+_logger = logging.getLogger(__file__)
+
+# Define all GS Cache related constants.
+GS_CACHE_PORT = '8888'
+GS_ARCHIVE_BUCKET = 'chromeos-image-archive'
+GS_CACHE_DWLD_RPC = 'download'
+GS_CACHE_LIST_DIR_RPC = 'list_dir'
+
+
+def _log(*args, **kwargs):
+ """A wrapper function of logging.debug/info, etc."""
+ level = kwargs.pop('level', logging.DEBUG)
+ _logger.log(level, extra=cherrypy.request.headers, *args, **kwargs)
+
+
+class NebraskaWrapperError(Exception):
+ """Exception class used by this module."""
+ pass
+
+
+class NebraskaWrapper(object):
+ """Class that contains functionality that handles Chrome OS update pings."""
+
+ # Define regexes for properties file. These patterns are the same as the ones
+ # defined in chromite/lib/xbuddy/build_artifact.py. Only the '.*' in the
+ # beginning and '$' at the end is different as in this class, we need to
+ # compare the full gs URL of the file without a newline at the end to this
+ # regex pattern.
+ _FULL_PAYLOAD_PROPS_PATTERN = r'.*chromeos_.*_full_dev.*bin(\.json)$'
+ _DELTA_PAYLOAD_PROPS_PATTERN = r'.*chromeos_.*_delta_dev.*bin(\.json)$'
+
+ def __init__(self, label, server_addr, full_update):
+ """Initializes the class.
+
+ Args:
+ label: Label (string) for the update, typically in the format
+ <board>-<XXXX>/Rxx-xxxxx.x.x-<unique string>[/au_nton].
+ server_addr: IP address (string) for the server on which gs cache is
+ running.
+ full_update: Indicates whether the requested update is full or delta. The
+ string values for this argument can be 'True', 'False', or
+ 'unspecified'.
+ """
+ self._label, au_nton = self._GetLabelAndNToN(label)
+ self._gs_cache_base_url = 'http://%s:%s' % (server_addr, GS_CACHE_PORT)
+
+ full_update = full_update.lower().strip()
+ # When full_update parameter is not specified in the request, the update
+ # type is 'delta' when au_nton is True and 'full' when au_nton is False.
+ self._is_full_update = (not au_nton if full_update == 'unspecified'
+ else full_update == 'true')
+
+ self._props_dir = tempfile.mkdtemp(prefix='gsc-update')
+ self._payload_props_file = None
+
+ def __enter__(self):
+ """Called while entering context manager; does nothing."""
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ """Called while exiting context manager; cleans up temp dirs."""
+ try:
+ shutil.rmtree(self._props_dir)
+ except Exception as e:
+ _log('Something went wrong. Could not delete %s due to exception: %s',
+ self._props_dir, e, level=logging.WARNING)
+
+ @property
+ def _PayloadPropsFilename(self):
+ """Get the name of the payload properties file.
+
+ The name of the properties file is obtained from the list of files returned
+ by the list_dir RPC by matching the name of the file with the update_type
+ and file extension.
+
+ Returns:
+ Name of the payload properties file.
+
+ Raises:
+ NebraskaWrapperError if the list_dir calls returns 4xx/5xx or if the
+ correct file could not be determined.
+ """
+ if self._payload_props_file:
+ return self._payload_props_file
+
+ urlbase = self._GetListDirURL()
+ url = urllib.parse.urljoin(urlbase, self._label)
+
+ resp = requests.get(url)
+ try:
+ resp.raise_for_status()
+ except Exception as e:
+ raise NebraskaWrapperError('An error occurred while trying to complete '
+ 'the request: %s' % e)
+
+ if self._is_full_update:
+ pattern = re.compile(self._FULL_PAYLOAD_PROPS_PATTERN)
+ else:
+ pattern = re.compile(self._DELTA_PAYLOAD_PROPS_PATTERN)
+
+ # Iterate through all listed files to determine the correct payload
+ # properties file. Since the listed files will be in the format
+ # gs://<gs_bucket>/<board>/<version>/<filename>, return the filename only
+ # once a match is determined.
+ for fname in [x.strip() for x in resp.content.strip().split('\n')]:
+ if pattern.match(fname):
+ self._payload_props_file = fname.rsplit('/', 1)[-1]
+ return self._payload_props_file
+
+ raise NebraskaWrapperError(
+ 'Request to %s returned a %s but gs_archive_server was unable to '
+ 'determine the name of the properties file.' %
+ (url, resp.status_code))
+
+ def _GetLabelAndNToN(self, label):
+ """Gets the label for the request and whether the update is N-to-N.
+
+ Removes a trailing /au_nton from the label argument which determines whether
+ this specific request is an N-to-N update or not.
+
+ Args:
+ label: A string obtained from the request.
+
+ Returns:
+ A string in the format <board>-<XXXX>/Rxx-xxxxx.x.x-<unique string>.
+ A boolean that indicates whether the update is N-to-N or not.
+ """
+ # TODO(crbug.com/1102552): Remove this logic once au_nton is removed from
+ # the request.
+ if label.endswith('/au_nton'):
+ return label[:-len('/au_nton')], True
+ return label, False
+
+ def _GetDownloadURL(self):
+ """Returns the static url base that should prefix all payload responses."""
+ _log('Handling update ping as %s', self._gs_cache_base_url)
+ return self._GetURL(GS_CACHE_DWLD_RPC)
+
+ def _GetListDirURL(self):
+ """Returns the static url base that should prefix all list_dir requests."""
+ _log('Using base URL to list contents: %s', self._gs_cache_base_url)
+ return self._GetURL(GS_CACHE_LIST_DIR_RPC)
+
+ def _GetURL(self, rpc_name):
+ """Construct gs_cache URL for the given RPC.
+
+ Args:
+ rpc_name: Name of the RPC for which the URL needs to be built.
+
+ Returns:
+ Base URL to be used.
+ """
+ urlbase = urllib.parse.urljoin(self._gs_cache_base_url,
+ '%s/%s/' % (rpc_name, GS_ARCHIVE_BUCKET))
+ _log('Using static url base %s', urlbase)
+ return urlbase
+
+ def _GetPayloadPropertiesDir(self, urlbase):
+ """Download payload properties file from GS Archive
+
+ Args:
+ urlbase: Base url that should be used to form the download request.
+
+ Returns:
+ The path to the /tmp directory which stores the payload properties file
+ that nebraska will use.
+
+ Raises:
+ NebraskaWrapperError is raised if the method is unable to
+ download the file for some reason.
+ """
+ local_payload_dir = self._props_dir
+ partial_url = urllib.parse.urljoin(urlbase, '%s/' % self._label)
+ _log('Downloading %s from bucket %s.', self._PayloadPropsFilename,
+ partial_url, level=logging.INFO)
+
+ try:
+ resp = requests.get(urllib.parse.urljoin(partial_url,
+ self._PayloadPropsFilename))
+ resp.raise_for_status()
+ file_path = os.path.join(local_payload_dir, self._PayloadPropsFilename)
+ # We are not worried about multiple threads writing to the same file as
+ # we are creating a different directory for each initialization of this
+ # class anyway.
+ with open(file_path, 'w') as f:
+ f.write(resp.content)
+ except Exception as e:
+ raise NebraskaWrapperError('An error occurred while trying to complete '
+ 'the request: %s' % e)
+ _log('Path to downloaded payload properties file: %s' % file_path)
+ return local_payload_dir
+
+ def HandleUpdatePing(self, data, **kwargs):
+ """Handles an update ping from an update client.
+
+ Args:
+ data: XML blob from client.
+ kwargs: The map of query strings passed to the /update API.
+
+ Returns:
+ Update payload message for client.
+ """
+ # Get the static url base that will form that base of our update url e.g.
+ # http://<GS_CACHE_IP>:<GS_CACHE_PORT>/download/chromeos-image-archive/.
+ urlbase = self._GetDownloadURL()
+ # Change the URL's string query dictionary provided by cherrypy to a
+ # valid dictionary that has proper values for its keys. e.g. True
+ # instead of 'True'.
+ kwargs = nebraska.QueryDictToDict(kwargs)
+
+ try:
+ # Process attributes of the update check.
+ request = nebraska.Request(data)
+ if request.request_type == nebraska.Request.RequestType.EVENT:
+ _log('A non-update event notification received. Returning an ack.',
+ level=logging.INFO)
+ return nebraska.Nebraska().GetResponseToRequest(
+ request, response_props=nebraska.ResponseProperties(**kwargs))
+
+ _log('Update Check Received.')
+
+ base_url = urllib.parse.urljoin(urlbase, '%s/' % self._label)
+ _log('Responding to client to use url %s to get image', base_url,
+ level=logging.INFO)
+
+ local_payload_dir = self._GetPayloadPropertiesDir(urlbase=urlbase)
+ _log('Using %s as the update_metadata_dir for NebraskaProperties.',
+ local_payload_dir)
+
+ nebraska_props = nebraska.NebraskaProperties(
+ update_payloads_address=base_url,
+ update_metadata_dir=local_payload_dir)
+ nebraska_obj = nebraska.Nebraska(nebraska_props=nebraska_props)
+
+ return nebraska_obj.GetResponseToRequest(
+ request, response_props=nebraska.ResponseProperties(**kwargs))
+
+ except Exception as e:
+ raise NebraskaWrapperError('An error occurred while processing the '
+ 'update request: %s' % e)
