au_test_harness: update to python3
BUG=chromium:1170006
TEST=CQ passes
Change-Id: I3a269ba7315049a148daaf973922003877cd6a67
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crostestutils/+/2689376
Reviewed-by: Sean Abraham <seanabraham@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>
diff --git a/au_test_harness/au_test_test.py b/au_test_harness/au_test_test.py
index 9830f52..e5bc78a 100755
--- a/au_test_harness/au_test_test.py
+++ b/au_test_harness/au_test_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -13,7 +13,7 @@
import sys
import time
import unittest
-import mock
+from unittest import mock
from multiprocessing import Process, Value
diff --git a/au_test_harness/cros_au_test_harness.py b/au_test_harness/cros_au_test_harness.py
index 978381f..4465019 100755
--- a/au_test_harness/cros_au_test_harness.py
+++ b/au_test_harness/cros_au_test_harness.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -30,7 +30,7 @@
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import parallel
-import sudo # TODO(crbug.com/1170006): Move back to chromite.
+from chromite.lib import sudo
from chromite.lib import timeout_util
from crostestutils.au_test_harness import au_test
diff --git a/au_test_harness/cros_au_test_harness_unittest.py b/au_test_harness/cros_au_test_harness_unittest.py
index 6f09fde..833a1c5 100755
--- a/au_test_harness/cros_au_test_harness_unittest.py
+++ b/au_test_harness/cros_au_test_harness_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/au_test_harness/cros_test_proxy.py b/au_test_harness/cros_test_proxy.py
index 2a38f7a..450e653 100644
--- a/au_test_harness/cros_test_proxy.py
+++ b/au_test_harness/cros_test_proxy.py
@@ -10,7 +10,7 @@
import os
import select
import socket
-import SocketServer
+import socketserver
import threading
@@ -41,10 +41,10 @@
return data
-class CrosTestProxy(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+class CrosTestProxy(socketserver.ThreadingMixIn, socketserver.TCPServer):
"""A transparent proxy for simulating network errors"""
- class _Handler(SocketServer.BaseRequestHandler):
+ class _Handler(socketserver.BaseRequestHandler):
"""Proxy connection handler that passes data though a filter"""
def setup(self):
@@ -114,7 +114,7 @@
self.__serving = False
try:
- SocketServer.TCPServer.__init__(self,
+ socketserver.TCPServer.__init__(self,
('', port_in),
self._Handler)
except socket.error:
@@ -134,7 +134,7 @@
# ==========================
- # Override of the version of this method from SocketServer.
+ # Override of the version of this method from socketserver.
# It's duplicated, other than adding __is_started event.
# Bug chromium-os:16574
def serve_forever(self, poll_interval=0.5):
@@ -160,7 +160,7 @@
self.__is_started.clear()
self.__is_shut_down.set()
- # Duplicate override of the version of this method from SocketServer so
+ # Duplicate override of the version of this method from socketserver so
# that we can access the same __ variables as serve_forever.
# Bug chromium-os:16574
def shutdown(self):
diff --git a/au_test_harness/gce.py b/au_test_harness/gce.py
deleted file mode 100644
index 2427878..0000000
--- a/au_test_harness/gce.py
+++ /dev/null
@@ -1,716 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A convinient wrapper of the GCE python API.
-
-Public methods in class GceContext raise HttpError when the underlining call to
-Google API fails, or gce.Error on other failures.
-"""
-
-from __future__ import print_function
-
-import sys
-
-from googleapiclient.discovery import build
-from googleapiclient.errors import HttpError
-from googleapiclient.http import HttpRequest
-import httplib2
-from oauth2client.client import GoogleCredentials
-
-from chromite.lib import cros_logging as logging
-from chromite.lib import timeout_util
-
-
-class Error(Exception):
- """Base exception for this module."""
-
-
-class ResourceNotFoundError(Error):
- """Exceptions raised when requested GCE resource was not found."""
-
-
-class RetryOnServerErrorHttpRequest(HttpRequest):
- """A HttpRequest that will be retried on server errors automatically."""
-
- def __init__(self, num_retries, *args, **kwargs):
- """Constructor for RetryOnServerErrorHttpRequest."""
- self.num_retries = num_retries
- super(RetryOnServerErrorHttpRequest, self).__init__(*args, **kwargs)
-
- def execute(self, http=None, num_retries=None):
- """Excutes a RetryOnServerErrorHttpRequest.
-
- HttpRequest.execute() has the option of automatically retrying on server
- errors, i.e., 500 status codes. Call it with a non-zero value of
- |num_retries| will cause failed requests to be retried.
-
- Args:
- http: The httplib2.http to send this request through.
- num_retries: Number of retries. Class default value will be used if
- omitted.
-
- Returns:
- A deserialized object model of the response body as determined
- by the postproc. See HttpRequest.execute().
- """
- return super(RetryOnServerErrorHttpRequest, self).execute(
- http=http, num_retries=num_retries or self.num_retries)
-
-
-def _GetMetdataValue(metadata, key):
- """Finds a value corresponding to a given metadata key.
-
- Args:
- metadata: metadata object, i.e. a dict containing containing 'items'
- - a list of key-value pairs.
- key: name of the key.
-
- Returns:
- Corresponding value or None if it was not found.
- """
- for item in metadata['items']:
- if item['key'] == key:
- return item['value']
- return None
-
-
-def _UpdateMetadataValue(metadata, key, value):
- """Updates a single key-value pair in a metadata object.
-
- Args:
- metadata: metadata object, i.e. a dict containing containing 'items'
- - a list of key-value pairs.
- key: name of the key.
- value: new value for the key, or None if it should be removed.
- """
- items = metadata.setdefault('items', [])
- for item in items:
- if item['key'] == key:
- if value is None:
- items.remove(item)
- else:
- item['value'] = value
- return
-
- if value is not None:
- items.append({
- 'key': key,
- 'value': value,
- })
-
-
-class GceContext(object):
- """A convinient wrapper around the GCE Python API."""
-
- # These constants are made public so that users can customize as they need.
- DEFAULT_TIMEOUT_SEC = 5 * 60
- INSTANCE_OPERATIONS_TIMEOUT_SEC = 10 * 60
- IMAGE_OPERATIONS_TIMEOUT_SEC = 10 * 60
-
- _GCE_SCOPES = (
- 'https://www.googleapis.com/auth/compute', # CreateInstance, CreateImage
- 'https://www.googleapis.com/auth/devstorage.full_control', # CreateImage
- )
- _DEFAULT_NETWORK = 'default'
- _DEFAULT_MACHINE_TYPE = 'n1-standard-8'
-
- # Project default service account and scopes.
- _DEFAULT_SERVICE_ACCOUNT_EMAIL = 'default'
- # The list is in line with what the gcloud cli uses.
- # https://cloud.google.com/sdk/gcloud/reference/compute/instances/create
- _DEFAULT_INSTANCE_SCOPES = [
- 'https://www.googleapis.com/auth/cloud.useraccounts.readonly',
- 'https://www.googleapis.com/auth/devstorage.read_only',
- 'https://www.googleapis.com/auth/logging.write',
- ]
-
- # This is made public to allow easy customization of the retry behavior.
- RETRIES = 2
-
- def __init__(self, project, zone, credentials, thread_safe=False):
- """Initializes GceContext.
-
- Args:
- project: The GCP project to create instances in.
- zone: The default zone to create instances in.
- credentials: The credentials used to call the GCE API.
- thread_safe: Whether the client is expected to be thread safe.
- """
- self.project = project
- self.zone = zone
-
- def _BuildRequest(http, *args, **kwargs):
- """Custom request builder."""
- return self._BuildRetriableRequest(self.RETRIES, http, thread_safe,
- credentials, *args, **kwargs)
-
- self.gce_client = build('compute', 'v1', credentials=credentials,
- requestBuilder=_BuildRequest)
-
- self.region = self.GetZoneRegion(zone)
-
- @classmethod
- def ForServiceAccount(cls, project, zone, json_key_file):
- """Creates a GceContext using service account credentials.
-
- About service account:
- https://developers.google.com/api-client-library/python/auth/service-accounts
-
- Args:
- project: The GCP project to create images and instances in.
- zone: The default zone to create instances in.
- json_key_file: Path to the service account JSON key.
-
- Returns:
- GceContext.
- """
- credentials = GoogleCredentials.from_stream(json_key_file).create_scoped(
- cls._GCE_SCOPES)
- return GceContext(project, zone, credentials)
-
- @classmethod
- def ForServiceAccountThreadSafe(cls, project, zone, json_key_file):
- """Creates a thread-safe GceContext using service account credentials.
-
- About service account:
- https://developers.google.com/api-client-library/python/auth/service-accounts
-
- Args:
- project: The GCP project to create images and instances in.
- zone: The default zone to create instances in.
- json_key_file: Path to the service account JSON key.
-
- Returns:
- GceContext.
- """
- credentials = GoogleCredentials.from_stream(json_key_file).create_scoped(
- cls._GCE_SCOPES)
- return GceContext(project, zone, credentials, thread_safe=True)
-
- def CreateAddress(self, name, region=None):
- """Reserves an external IP address.
-
- Args:
- name: The name to assign to the address.
- region: Region to reserved the address in.
-
- Returns:
- The reserved address as a string.
- """
- body = {
- 'name': name,
- }
- operation = self.gce_client.addresses().insert(
- project=self.project,
- region=region or self.region,
- body=body).execute()
- self._WaitForRegionOperation(
- operation['name'], region,
- timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC)
-
- address = self.gce_client.addresses().get(
- project=self.project,
- region=region or self.region,
- address=name).execute()
-
- return address['address']
-
- def DeleteAddress(self, name, region=None):
- """Frees up an external IP address.
-
- Args:
- name: The name of the address.
- region: Region of the address.
- """
- operation = self.gce_client.addresses().delete(
- project=self.project,
- region=region or self.region,
- address=name).execute()
- self._WaitForRegionOperation(
- operation['name'], region=region or self.region,
- timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC)
-
- def GetZoneRegion(self, zone=None):
- """Resolves name of the region that a zone belongs to.
-
- Args:
- zone: The zone to resolve.
-
- Returns:
- Name of the region corresponding to the zone.
- """
- zone_resource = self.gce_client.zones().get(
- project=self.project,
- zone=zone or self.zone).execute()
- return zone_resource['region'].split('/')[-1]
-
- def CreateInstance(self, name, image, zone=None, network=None, subnet=None,
- machine_type=None, default_scopes=True,
- static_address=None, **kwargs):
- """Creates an instance with the given image and waits until it's ready.
-
- Args:
- name: Instance name.
- image: Fully spelled URL of the image, e.g., for private images,
- 'global/images/my-private-image', or for images from a
- publicly-available project,
- 'projects/debian-cloud/global/images/debian-7-wheezy-vYYYYMMDD'.
- Details:
- https://cloud.google.com/compute/docs/reference/latest/instances/insert
- zone: The zone to create the instance in. Default zone will be used if
- omitted.
- network: An existing network to create the instance in. Default network
- will be used if omitted.
- subnet: The subnet to create the instance in.
- machine_type: The machine type to use. Default machine type will be used
- if omitted.
- default_scopes: If true, the default scopes are added to the instances.
- static_address: External IP address to assign to the instance as a string.
- If None an emphemeral address will be used.
- kwargs: Other possible Instance Resource properties.
- https://cloud.google.com/compute/docs/reference/latest/instances#resource
- Note that values from kwargs will overrule properties constructed from
- positinal arguments, i.e., name, image, zone, network and
- machine_type.
-
- Returns:
- URL to the created instance.
- """
- logging.info('Creating instance "%s" with image "%s" ...', name, image)
- network = 'global/networks/%s' % network or self._DEFAULT_NETWORK
- machine_type = 'zones/%s/machineTypes/%s' % (
- zone or self.zone, machine_type or self._DEFAULT_MACHINE_TYPE)
- service_accounts = (
- {
- 'email': self._DEFAULT_SERVICE_ACCOUNT_EMAIL,
- 'scopes': self._DEFAULT_INSTANCE_SCOPES,
- },
- ) if default_scopes else ()
-
- config = {
- 'name': name,
- 'machineType': machine_type,
- 'disks': (
- {
- 'boot': True,
- 'autoDelete': True,
- 'initializeParams': {
- 'sourceImage': image,
- },
- },
- ),
- 'networkInterfaces': (
- {
- 'network': network,
- 'accessConfigs': (
- {
- 'type': 'ONE_TO_ONE_NAT',
- 'name': 'External NAT',
- },
- ),
- },
- ),
- 'serviceAccounts' : service_accounts,
- }
- config.update(**kwargs)
- if static_address is not None:
- config['networkInterfaces'][0]['accessConfigs'][0]['natIP'] = (
- static_address)
- if subnet is not None:
- region = self.GetZoneRegion(zone)
- config['networkInterfaces'][0]['subnetwork'] = (
- 'regions/%s/subnetworks/%s' % (region, subnet)
- )
- operation = self.gce_client.instances().insert(
- project=self.project,
- zone=zone or self.zone,
- body=config).execute()
- self._WaitForZoneOperation(
- operation['name'],
- timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC,
- timeout_handler=lambda: self.DeleteInstance(name))
- return operation['targetLink']
-
- def DeleteInstance(self, name, zone=None):
- """Deletes an instance with the name and waits until it's done.
-
- Args:
- name: Name of the instance to delete.
- zone: Zone where the instance is in. Default zone will be used if omitted.
- """
- logging.info('Deleting instance "%s" ...', name)
- operation = self.gce_client.instances().delete(
- project=self.project,
- zone=zone or self.zone,
- instance=name).execute()
- self._WaitForZoneOperation(
- operation['name'], timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC)
-
- def StartInstance(self, name, zone=None):
- """Starts an instance with the name and waits until it's done.
-
- Args:
- name: Name of the instance to start.
- zone: Zone where the instance is in. Default zone will be used if omitted.
- """
- logging.info('Starting instance "%s" ...', name)
- operation = self.gce_client.instances().start(
- project=self.project,
- zone=zone or self.zone,
- instance=name).execute()
- self._WaitForZoneOperation(
- operation['name'], timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC)
-
- def StopInstance(self, name, zone=None):
- """Stops an instance with the name and waits until it's done.
-
- Args:
- name: Name of the instance to stop.
- zone: Zone where the instance is in. Default zone will be used if omitted.
- """
- logging.info('Stopping instance "%s" ...', name)
- operation = self.gce_client.instances().stop(
- project=self.project,
- zone=zone or self.zone,
- instance=name).execute()
- self._WaitForZoneOperation(
- operation['name'], timeout_sec=self.INSTANCE_OPERATIONS_TIMEOUT_SEC)
-
- def CreateImage(self, name, source):
- """Creates an image with the given |source|.
-
- Args:
- name: Name of the image to be created.
- source:
- Google Cloud Storage object of the source disk, e.g.,
- 'https://storage.googleapis.com/my-gcs-bucket/test_image.tar.gz'.
-
- Returns:
- URL to the created image.
- """
- logging.info('Creating image "%s" with "source" %s ...', name, source)
- config = {
- 'name': name,
- 'rawDisk': {
- 'source': source,
- },
- }
- operation = self.gce_client.images().insert(
- project=self.project,
- body=config).execute()
- self._WaitForGlobalOperation(operation['name'],
- timeout_sec=self.IMAGE_OPERATIONS_TIMEOUT_SEC,
- timeout_handler=lambda: self.DeleteImage(name))
- return operation['targetLink']
-
- def DeleteImage(self, name):
- """Deletes an image and waits until it's deleted.
-
- Args:
- name: Name of the image to delete.
- """
- logging.info('Deleting image "%s" ...', name)
- operation = self.gce_client.images().delete(
- project=self.project,
- image=name).execute()
- self._WaitForGlobalOperation(operation['name'],
- timeout_sec=self.IMAGE_OPERATIONS_TIMEOUT_SEC)
-
- def ListInstances(self, zone=None):
- """Lists all instances.
-
- Args:
- zone: Zone where the instances are in. Default zone will be used if
- omitted.
-
- Returns:
- A list of Instance Resources if found, or an empty list otherwise.
- """
- result = self.gce_client.instances().list(project=self.project,
- zone=zone or self.zone).execute()
- return result.get('items', [])
-
- def ListImages(self):
- """Lists all images.
-
- Returns:
- A list of Image Resources if found, or an empty list otherwise.
- """
- result = self.gce_client.images().list(project=self.project).execute()
- return result.get('items', [])
-
- def GetInstance(self, instance, zone=None):
- """Gets an Instance Resource by name and zone.
-
- Args:
- instance: Name of the instance.
- zone: Zone where the instance is in. Default zone will be used if omitted.
-
- Returns:
- An Instance Resource.
-
- Raises:
- ResourceNotFoundError if instance was not found, or HttpError on other
- HTTP failures.
- """
- try:
- return self.gce_client.instances().get(project=self.project,
- zone=zone or self.zone,
- instance=instance).execute()
- except HttpError as e:
- if e.resp.status == 404:
- raise ResourceNotFoundError(
- 'Instance "%s" for project "%s" in zone "%s" was not found.' %
- (instance, self.project, zone or self.zone))
- else:
- raise
-
- def GetInstanceIP(self, instance, zone=None):
- """Gets the external IP of an instance.
-
- Args:
- instance: Name of the instance to get IP for.
- zone: Zone where the instance is in. Default zone will be used if omitted.
-
- Returns:
- External IP address of the instance.
-
- Raises:
- Error: Something went wrong when trying to get IP for the instance.
- """
- result = self.GetInstance(instance, zone)
- try:
- return result['networkInterfaces'][0]['accessConfigs'][0]['natIP']
- except (KeyError, IndexError):
- raise Error('Failed to get IP address for instance %s' % instance)
-
- def GetInstanceInternalIP(self, instance, zone=None):
- """Gets the internal IP of an instance."""
- result = self.GetInstance(instance, zone)
- try:
- return result['networkInterfaces'][0]['networkIP']
- except (KeyError, IndexError):
- raise Error('Failed to get internal IP for instance %s' % instance)
-
- def GetImage(self, image):
- """Gets an Image Resource by name.
-
- Args:
- image: Name of the image to look for.
-
- Returns:
- An Image Resource.
-
- Raises:
- ResourceNotFoundError: The requested image was not found.
- """
- try:
- return self.gce_client.images().get(project=self.project,
- image=image).execute()
- except HttpError as e:
- if e.resp.status == 404:
- raise ResourceNotFoundError('Image "%s" for project "%s" was not found.'
- % (image, self.project))
- else:
- raise
-
- def InstanceExists(self, instance, zone=None):
- """Checks if an instance exists in the current project.
-
- Args:
- instance: Name of the instance to check existence of.
- zone: Zone where the instance is in. Default zone will be used if omitted.
-
- Returns:
- True if the instance exists or False otherwise.
- """
- try:
- return self.GetInstance(instance, zone) is not None
- except ResourceNotFoundError:
- return False
-
- def ImageExists(self, image):
- """Checks if an image exists in the current project.
-
- Args:
- image: Name of the image to check existence of.
-
- Returns:
- True if the instance exists or False otherwise.
- """
- try:
- return self.GetImage(image) is not None
- except ResourceNotFoundError:
- return False
-
- def GetCommonInstanceMetadata(self, key):
- """Looks up a single project metadata value.
-
- Args:
- key: Metadata key name.
-
- Returns:
- Metadata value corresponding to the key, or None if it was not found.
- """
- projects_data = self.gce_client.projects().get(
- project=self.project).execute()
- metadata = projects_data['commonInstanceMetadata']
- return _GetMetdataValue(metadata, key)
-
- def SetCommonInstanceMetadata(self, key, value):
- """Sets a single project metadata value.
-
- Args:
- key: Metadata key to be set.
- value: New value, or None if the given key should be removed.
- """
- projects_data = self.gce_client.projects().get(
- project=self.project).execute()
- metadata = projects_data['commonInstanceMetadata']
- _UpdateMetadataValue(metadata, key, value)
- operation = self.gce_client.projects().setCommonInstanceMetadata(
- project=self.project,
- body=metadata).execute()
- self._WaitForGlobalOperation(operation['name'])
-
- def GetInstanceMetadata(self, instance, key):
- """Looks up instance's metadata value.
-
- Args:
- instance: Name of the instance.
- key: Metadata key name.
-
- Returns:
- Metadata value corresponding to the key, or None if it was not found.
- """
- instance_data = self.GetInstance(instance)
- metadata = instance_data['metadata']
- return self._GetMetdataValue(metadata, key)
-
- def SetInstanceMetadata(self, instance, key, value):
- """Sets a single instance metadata value.
-
- Args:
- instance: Name of the instance.
- key: Metadata key to be set.
- value: New value, or None if the given key should be removed.
- """
- instance_data = self.GetInstance(instance)
- metadata = instance_data['metadata']
- _UpdateMetadataValue(metadata, key, value)
- operation = self.gce_client.instances().setMetadata(
- project=self.project,
- zone=self.zone,
- instance=instance,
- body=metadata).execute()
- self._WaitForZoneOperation(operation['name'])
-
- def _WaitForZoneOperation(self, operation, zone=None, timeout_sec=None,
- timeout_handler=None):
- """Waits until a GCE ZoneOperation is finished or timed out.
-
- Args:
- operation: The GCE operation to wait for.
- zone: The zone that |operation| belongs to.
- timeout_sec: The maximum number of seconds to wait for.
- timeout_handler: A callable to be executed when timeout happens.
-
- Raises:
- Error when timeout happens or the operation fails.
- """
- get_request = self.gce_client.zoneOperations().get(
- project=self.project, zone=zone or self.zone, operation=operation)
- self._WaitForOperation(operation, get_request, timeout_sec,
- timeout_handler=timeout_handler)
-
- def _WaitForRegionOperation(self, operation, region, timeout_sec=None,
- timeout_handler=None):
- """Waits until a GCE RegionOperation is finished or timed out.
-
- Args:
- operation: The GCE operation to wait for.
- region: The region that |operation| belongs to.
- timeout_sec: The maximum number of seconds to wait for.
- timeout_handler: A callable to be executed when timeout happens.
-
- Raises:
- Error when timeout happens or the operation fails.
- """
- get_request = self.gce_client.regionOperations().get(
- project=self.project, region=region or self.region, operation=operation)
- self._WaitForOperation(operation, get_request, timeout_sec,
- timeout_handler=timeout_handler)
-
- def _WaitForGlobalOperation(self, operation, timeout_sec=None,
- timeout_handler=None):
- """Waits until a GCE GlobalOperation is finished or timed out.
-
- Args:
- operation: The GCE operation to wait for.
- timeout_sec: The maximum number of seconds to wait for.
- timeout_handler: A callable to be executed when timeout happens.
-
- Raises:
- Error when timeout happens or the operation fails.
- """
- get_request = self.gce_client.globalOperations().get(project=self.project,
- operation=operation)
- self._WaitForOperation(operation, get_request, timeout_sec=timeout_sec,
- timeout_handler=timeout_handler)
-
- def _WaitForOperation(self, operation, get_operation_request,
- timeout_sec=None, timeout_handler=None):
- """Waits until timeout or the request gets a response with a 'DONE' status.
-
- Args:
- operation: The GCE operation to wait for.
- get_operation_request:
- The HTTP request to get the operation's status.
- This request will be executed periodically until it returns a status
- 'DONE'.
- timeout_sec: The maximum number of seconds to wait for.
- timeout_handler: A callable to be executed when times out.
-
- Raises:
- Error when timeout happens or the operation fails.
- """
- def _IsDone():
- result = get_operation_request.execute()
- if result['status'] == 'DONE':
- if 'error' in result:
- raise Error(result['error'])
- return True
- return False
-
- try:
- timeout = timeout_sec or self.DEFAULT_TIMEOUT_SEC
- logging.info('Waiting up to %d seconds for operation [%s] to complete...',
- timeout, operation)
- timeout_util.WaitForReturnTrue(_IsDone, timeout, period=1)
- except timeout_util.TimeoutError:
- if timeout_handler:
- timeout_handler()
- raise Error('Timeout wating for operation [%s] to complete' % operation)
-
- def _BuildRetriableRequest(self, num_retries, http, thread_safe=False,
- credentials=None, *args, **kwargs):
- """Builds a request that will be automatically retried on server errors.
-
- Args:
- num_retries: The maximum number of times to retry until give up.
- http: An httplib2.Http object that this request will be executed through.
- thread_safe: Whether or not the request needs to be thread-safe.
- credentials: Credentials to apply to the request.
- *args: Optional positional arguments.
- **kwargs: Optional keyword arguments.
-
- Returns:
- RetryOnServerErrorHttpRequest: A request that will automatically retried
- on server errors.
- """
- if thread_safe:
- # Create a new http object for every request.
- http = credentials.authorize(httplib2.Http())
- return RetryOnServerErrorHttpRequest(num_retries, http, *args, **kwargs)
diff --git a/au_test_harness/gce_au_worker.py b/au_test_harness/gce_au_worker.py
index e63d5e4..3b054e0 100644
--- a/au_test_harness/gce_au_worker.py
+++ b/au_test_harness/gce_au_worker.py
@@ -13,13 +13,14 @@
import shutil
import tempfile
import time
-import urllib2
+import urllib.error
+import urllib.request
from multiprocessing import Process
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
-import gce # TODO(crbug.com/1170006): Move back to chromite.
+from chromite.lib import gce
from chromite.lib import gs
from chromite.lib import path_util
from crostestutils.au_test_harness import au_worker
@@ -310,7 +311,7 @@
shutil.copy(self.tarball_local, fail_directory)
if self.ssh_private_key is not None:
shutil.copy(self.ssh_private_key, fail_directory)
- except (shutil.Error, OSError, IOError) as e:
+ except OSError as e:
logging.warning('Ignoring error while copying logs: %s', e)
def _GsPathToUrl(self, gs_path):
@@ -342,7 +343,7 @@
def _OnGCE():
"""Checks if running on a GCE VM or not."""
try:
- urllib2.urlopen('http://metadata.google.internal', timeout=10)
+ urllib.request.urlopen('http://metadata.google.internal', timeout=10)
return True
- except urllib2.URLError:
+ except urllib.error.URLError:
return False
diff --git a/au_test_harness/gce_au_worker_unittest.py b/au_test_harness/gce_au_worker_unittest.py
index 379dfa4..ffe269b 100755
--- a/au_test_harness/gce_au_worker_unittest.py
+++ b/au_test_harness/gce_au_worker_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,10 +8,10 @@
from __future__ import print_function
-import mock
import os
import sys
import unittest
+from unittest import mock
import constants
sys.path.append(constants.CROS_PLATFORM_ROOT)
@@ -19,7 +19,7 @@
from chromite.lib import cros_build_lib
from chromite.lib import cros_test_lib
-import gce # TODO(crbug.com/1170006): Move back to chromite.
+from chromite.lib import gce
from chromite.lib import osutils
from chromite.lib import path_util
from crostestutils.au_test_harness.au_worker import AUWorker
diff --git a/au_test_harness/sudo.py b/au_test_harness/sudo.py
deleted file mode 100644
index 55adf06..0000000
--- a/au_test_harness/sudo.py
+++ /dev/null
@@ -1,155 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Helper methods and classes related to managing sudo."""
-
-from __future__ import print_function
-
-import errno
-import os
-import signal
-import subprocess
-import sys
-
-from chromite.lib import cros_build_lib
-from chromite.lib import cros_logging as logging
-
-
-class SudoKeepAlive(cros_build_lib.MasterPidContextManager):
- """Keep sudo auth cookie fresh.
-
- This refreshes the sudo auth cookie; this is implemented this
- way to ensure that sudo has access to both invoking tty, and
- will update the user's tty-less cookie.
- see crosbug/18393.
- """
-
- def __init__(self, ttyless_sudo=True, repeat_interval=4):
- """Run sudo with a noop, to reset the sudo timestamp.
-
- Args:
- ttyless_sudo: Whether to update the tty-less cookie.
- repeat_interval: In minutes, the frequency to run the update.
- """
- cros_build_lib.MasterPidContextManager.__init__(self)
- self._ttyless_sudo = ttyless_sudo
- self._repeat_interval = repeat_interval
- self._proc = None
- self._existing_keepalive_value = None
-
- @staticmethod
- def _IdentifyTTY():
- for source in (sys.stdin, sys.stdout, sys.stderr):
- try:
- return os.ttyname(source.fileno())
- except EnvironmentError as e:
- if e.errno not in (errno.EINVAL, errno.ENOTTY):
- raise
-
- return 'unknown'
-
- def _DaemonNeeded(self):
- """Discern which TTYs require sudo keep alive code.
-
- Returns:
- A string representing the set of ttys we need daemons for.
- This will be the empty string if no daemon is needed.
- """
- existing = os.environ.get('CROS_SUDO_KEEP_ALIVE')
- needed = set([self._IdentifyTTY()])
- if self._ttyless_sudo:
- needed.add('unknown')
- if existing is not None:
- needed -= set(existing.split(':'))
- return ':'.join(needed)
-
- def _enter(self):
- if os.getuid() == 0:
- cros_build_lib.Die('This script cannot be run as root.')
-
- start_for_tty = self._DaemonNeeded()
- if not start_for_tty:
- # Daemon is already started.
- return
-
- # Note despite the impulse to use 'sudo -v' instead of 'sudo true', the
- # builder's sudoers configuration is slightly whacked resulting in it
- # asking for password everytime. As such use 'sudo true' instead.
- cmds = ['sudo -n true 2>/dev/null',
- 'sudo -n true < /dev/null > /dev/null 2>&1']
-
- # First check to see if we're already authed. If so, then we don't
- # need to prompt the user for their password.
- for idx, cmd in enumerate(cmds):
- ret = cros_build_lib.run(
- cmd, print_cmd=False, shell=True, check=False)
-
- if ret.returncode != 0:
- tty_msg = 'Please disable tty_tickets using these instructions: %s'
- if os.path.exists('/etc/goobuntu'):
- url = 'https://goto.google.com/chromeos-sudoers'
- else:
- url = 'https://goo.gl/fz9YW'
-
- # If ttyless sudo is not strictly required for this script, don't
- # prompt for a password a second time. Instead, just complain.
- if idx > 0:
- logging.error(tty_msg, url)
- if not self._ttyless_sudo:
- break
-
- # We need to go interactive and allow sudo to ask for credentials.
- interactive_cmd = cmd.replace(' -n', '')
- cros_build_lib.run(interactive_cmd, shell=True, print_cmd=False)
-
- # Verify that sudo access is set up properly.
- try:
- cros_build_lib.run(cmd, shell=True, print_cmd=False)
- except cros_build_lib.RunCommandError:
- if idx == 0:
- raise
- cros_build_lib.Die('tty_tickets must be disabled. ' + tty_msg, url)
-
- # Anything other than a timeout results in us shutting down.
- repeat_interval = self._repeat_interval * 60
- cmd = ('while :; do read -t %i; [ $? -le 128 ] && exit; %s; done' %
- (repeat_interval, '; '.join(cmds)))
-
- def ignore_sigint():
- # We don't want our sudo process shutdown till we shut it down;
- # since it's part of the session group it however gets SIGINT.
- # Thus suppress it (which bash then inherits).
- signal.signal(signal.SIGINT, signal.SIG_IGN)
-
- # We don't use threads here.
- # pylint: disable=bad-option-value,subprocess-popen-preexec-fn
- self._proc = subprocess.Popen(['bash', '-c', cmd], shell=False,
- close_fds=True, preexec_fn=ignore_sigint,
- stdin=subprocess.PIPE)
-
- self._existing_keepalive_value = os.environ.get('CROS_SUDO_KEEP_ALIVE')
- os.environ['CROS_SUDO_KEEP_ALIVE'] = start_for_tty
-
- def _exit(self, exc_type, exc, exc_tb):
- if self._proc is None:
- return
-
- try:
- self._proc.terminate()
- self._proc.wait()
- except EnvironmentError as e:
- if e.errno != errno.ESRCH:
- raise
-
- if self._existing_keepalive_value is not None:
- os.environ['CROS_SUDO_KEEP_ALIVE'] = self._existing_keepalive_value
- else:
- os.environ.pop('CROS_SUDO_KEEP_ALIVE', None)
-
-
-def SetFileContents(path, value, cwd=None):
- """Set a given filepath contents w/ the passed in value."""
- cros_build_lib.sudo_run(['tee', path], stdout=True,
- print_cmd=False, input=value, cwd=cwd)
