autotest: Remove test_Recall and associated infrastructure
The test is unused.
BUG=chromium:356414
TEST=trybot on lumpy-release
CQ-DEPEND=CL:203107
Change-Id: Iaf06e727b4ee6fae9225d4f6a778d4f551d7874d
Reviewed-on: https://chromium-review.googlesource.com/203106
Reviewed-by: Scott James Remnant <keybuk@chromium.org>
Reviewed-by: Alex Miller <milleral@chromium.org>
Tested-by: Gaurav Shah <gauravsh@chromium.org>
Commit-Queue: Gaurav Shah <gauravsh@chromium.org>
diff --git a/client/cros/constants.py b/client/cros/constants.py
index 07c2423..f7a5068 100644
--- a/client/cros/constants.py
+++ b/client/cros/constants.py
@@ -136,9 +136,6 @@
PENDING_SHUTDOWN_PATH = '/var/lib/crash_reporter/pending_clean_shutdown'
UNCLEAN_SHUTDOWN_DETECTED_PATH = '/var/run/unclean-shutdown-detected'
-FAKE_ROOT_CA_DIR = '/etc/fake_root_ca'
-FAKE_NSSDB_DIR = FAKE_ROOT_CA_DIR + '/nssdb'
-
INTERACTIVE_XMLRPC_SERVER_PORT = 9980
INTERACTIVE_XMLRPC_SERVER_COMMAND = (
'cd /usr/local/autotest/common_lib/cros; '
diff --git a/client/cros/recall.py b/client/cros/recall.py
deleted file mode 100644
index 859510d..0000000
--- a/client/cros/recall.py
+++ /dev/null
@@ -1,215 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Context Manager for using Recall in autotest tests.
-
-Recall is a server-side system that intercepts DNS, HTTP and HTTPS
-traffic from clients for either recording or playback. This allows
-tests to be run in labs without Internet access by playing back
-pre-recorded copies of websites, allows tests to be run that use
-recorded copies of sites known to exhibit errors or many iterations
-of the same test to be run with the same data.
-
-Recall is intended to be completely transparent to the client tests,
-this context manager takes care of adjusting the client's configuration
-to redirect the traffic to the recall server (in the case it's not the
-network's default gateway already) and install a root certificate
-for HTTPS man-in-the-middling.
-
-It's instantiated as part of the control file sent from the autotest
-server when invoking the client tests.
-"""
-
-import logging, os, re, stat, subprocess, urllib2
-
-import common, constants
-from autotest_lib.client.common_lib import error
-
-
-class RecallServer(object):
- """Context manager for adjusting client configuration for Recall.
-
- Use this in a control file to wrap a client test to run against a
- server using Recall:
-
- with RecallServer(recall_server_ip):
- job.run_test(...)
-
- This is intended to be included in the control file passed from the
- autotest server to the client to be run.
- """
-
- def __init__(self, recall_server_ip):
- self._recall_server_ip = recall_server_ip
-
- def __enter__(self):
- certificate_url = ("http://%s/GetRootCertificate"
- % self._recall_server_ip)
- self._InstallRootCertificate(certificate_url)
- self._InstallDnsServer(self._recall_server_ip)
- self._InstallDefaultRoute(self._recall_server_ip)
-
- def __exit__(self, exc_type, exc_value, traceback):
- self._UninstallRootCertificate()
- self._UninstallDnsServer()
- self._UninstallDefaultRoute()
- return False
-
- def _InstallRootCertificate(self, certificate_url):
- """Download fake root cert from server and install.
-
- Mounts a tmpfs filesystem over the location that Chromium will
- look for an additional nssdb, downloads a root certificate from
- the given URL and creates an nssdb containing it.
-
- Args:
- certificate_url: URL of certificate to download.
- """
- logging.debug("Obtaining certificate from %s", certificate_url)
- f = urllib2.urlopen(certificate_url)
- try:
- certificate = f.read()
- finally:
- f.close()
-
- logging.debug("Mounting tmpfs on %s", constants.FAKE_ROOT_CA_DIR)
- cmd = ( 'mount', '-t', 'tmpfs', '-o', 'mode=0755', 'none',
- constants.FAKE_ROOT_CA_DIR )
- proc = subprocess.Popen(cmd)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to mount tmpfs")
-
- logging.debug("Creating NSS database in %s", constants.FAKE_NSSDB_DIR)
- os.mkdir(constants.FAKE_NSSDB_DIR, 0755)
-
- cmd = ( 'certutil', '-d', 'sql:' + constants.FAKE_NSSDB_DIR,
- '-N', '-f', '/dev/fd/0' )
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
- proc.communicate('\n')
- if proc.returncode != 0:
- raise error.TestError("Failed to create nssdb")
-
- logging.debug("Adding certificate to database")
- cmd = ( 'certutil', '-d', 'sql:' + constants.FAKE_NSSDB_DIR,
- '-A', '-n', "Chromium OS Test Server", '-t', 'C,,', '-a' )
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
- proc.communicate(certificate)
- if proc.returncode != 0:
- raise error.TestError("Failed to add certificate to nssdb")
-
- logging.debug("Fixing permissions")
- for filename in os.listdir(constants.FAKE_NSSDB_DIR):
- os.chmod(os.path.join(constants.FAKE_NSSDB_DIR, filename), 0644)
-
- def _UninstallRootCertificate(self):
- """Uninstall fake root cert.
-
- Unmounts the tmpfs created by _InstallRootCertificate().
- """
- logging.info("Unmounting tmpfs from %s", constants.FAKE_ROOT_CA_DIR)
- cmd = ( 'umount', '-l', constants.FAKE_ROOT_CA_DIR )
- proc = subprocess.Popen(cmd)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to umount tmpfs")
-
- def _InstallDnsServer(self, nameserver):
- """Change the system DNS server.
-
- Backs up and writes out an alternate /etc/resolv.conf pointing
- at the nameserver given.
-
- Args:
- nameserver: IP address of server to use.
- """
- resolv_conf = os.path.realpath(constants.RESOLV_CONF_FILE)
- resolv_conf_bak = resolv_conf + '.recallbak'
- resolv_dir = os.path.dirname(resolv_conf)
-
- logging.info("Changing nameserver to %s", nameserver)
- os.rename(resolv_conf, resolv_conf_bak)
- with open(resolv_conf, 'w') as resolv:
- self._resolv_dir_mode = os.stat(resolv_dir).st_mode
- os.chmod(resolv_dir, self._resolv_dir_mode & ~ (stat.S_IWUSR |
- stat.S_IWGRP |
- stat.S_IWOTH))
-
- print >>resolv, "nameserver %s" % nameserver
-
- def _UninstallDnsServer(self):
- """Restore the system DNS server.
-
- Restores the original /etc/resolv.conf from before calling
- _InstallDnsServer().
- """
- resolv_conf = os.path.realpath(constants.RESOLV_CONF_FILE)
- resolv_conf_bak = resolv_conf + '.recallbak'
- resolv_dir = os.path.dirname(resolv_conf)
-
- logging.info("Restoring original nameserver")
- os.chmod(resolv_dir, self._resolv_dir_mode)
- os.rename(resolv_conf_bak, resolv_conf)
-
- def _GetDefaultRouteGateway(self):
- """Return the gateway of the default route."""
- cmd = ( 'ip', 'route', 'show' )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- data, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestError("Failed to obtain routing table")
-
- for line in data.splitlines():
- if not line.startswith("default "):
- continue
-
- match = re.search(r'via ([^ ]*)', line)
- if not match:
- continue
-
- return match.group(1)
- else:
- raise error.TestError("Failed to obtain default route gateway")
-
- def _InstallDefaultRoute(self, gateway):
- """Change the gateway of the default route.
-
- Adjusts the system routing table so that the gateway of the
- default route points to the IP address given.
-
- Args:
- gateway: IP address of new default route gateway.
- """
- self._original_gateway = self._GetDefaultRouteGateway()
-
- logging.info("Changing default route gateway to %s", gateway)
- cmd = ( 'ip', 'route', 'change', 'default', 'via', gateway )
- proc = subprocess.Popen(cmd)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to change default route gateway")
-
- cmd = ( 'ip', 'route', 'flush', 'cache')
- proc = subprocess.Popen(cmd)
- proc.wait()
- # Discard return code, it doesn't matter so much if this fails
-
- def _UninstallDefaultRoute(self):
- """Restore the gateway of the default route.
-
- Restores the gateway of the default route to that which existed
- before calling _InstallDefaultRoute().
- """
- logging.info("Restoring original default route gateway")
- cmd = ( 'ip', 'route', 'change', 'default', 'via',
- self._original_gateway )
- proc = subprocess.Popen(cmd)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to change default route gateway")
-
- cmd = ( 'ip', 'route', 'flush', 'cache')
- proc = subprocess.Popen(cmd)
- proc.wait()
- # Discard return code, it doesn't matter so much if this fails
diff --git a/client/site_tests/test_Recall/control b/client/site_tests/test_Recall/control
deleted file mode 100644
index 81e1dfd..0000000
--- a/client/site_tests/test_Recall/control
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-AUTHOR = "Chromium OS Project"
-NAME = "test_Recall"
-PURPOSE = "Verify the Recall client-side helper"
-CRITERIA = """
-This test will succeed as long as the RecallServer class adjusts the
-default route and DNS server of the client, and installs a root
-certificate.
-"""
-TIME = "SHORT"
-TEST_CATEGORY = "General"
-TEST_CLASS = "test"
-TEST_TYPE = "client"
-
-DOC = """
-This is a test to verify the autotest infrastructure for supporting
-Recall. It's implemented as a client test and not as a unit test because
-the RecallServer class makes changes to the system it's running on, and
-we don't want that to happen to the buildbots/your desktop.
-"""
-
-job.run_test('test_Recall')
diff --git a/client/site_tests/test_Recall/test_Recall.py b/client/site_tests/test_Recall/test_Recall.py
deleted file mode 100644
index 8fd3d9b..0000000
--- a/client/site_tests/test_Recall/test_Recall.py
+++ /dev/null
@@ -1,134 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import httplib
-import os
-import re
-import stat
-import subprocess
-
-from autotest_lib.client.bin import test
-from autotest_lib.client.common_lib import error
-from autotest_lib.client.cros import constants, httpd
-from autotest_lib.client.cros.recall import RecallServer
-
-class test_Recall(test.test):
- version = 1
-
- _certificate = """\
------BEGIN CERTIFICATE-----
-MIICejCCAeOgAwIBAgIJAJAMlNbcSCFeMA0GCSqGSIb3DQEBBQUAMDMxDzANBgNV
-BAoTBkdvb2dsZTEgMB4GA1UECxMXQ2hyb21pdW0gT1MgVGVzdCBTZXJ2ZXIwHhcN
-MTExMTEwMTk0NTQ0WhcNMTExMTExMTk0NTQ0WjAzMQ8wDQYDVQQKEwZHb29nbGUx
-IDAeBgNVBAsTF0Nocm9taXVtIE9TIFRlc3QgU2VydmVyMIGfMA0GCSqGSIb3DQEB
-AQUAA4GNADCBiQKBgQCc+gTR3R/OiY+AtZCsRI3CtHr+/7q8VRuci/rJU1R58OrX
-qEPZx/rck1fpAA3rpCkfv/T7tXbmzyTVJ8cPh9scC22hM8OKppZeZSlX2hA8uocW
-iheMkuUHcP+ya4z02GXNgUdLUiWSBfyme3cdHc5+Ugp1wrAOUkLG0Ya2x01I0QID
-AQABo4GVMIGSMB0GA1UdDgQWBBQtD9gOLzc5O9aW+74nZL/VGQHRDDBjBgNVHSME
-XDBagBQtD9gOLzc5O9aW+74nZL/VGQHRDKE3pDUwMzEPMA0GA1UEChMGR29vZ2xl
-MSAwHgYDVQQLExdDaHJvbWl1bSBPUyBUZXN0IFNlcnZlcoIJAJAMlNbcSCFeMAwG
-A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEACNNJsaj/lMlmbu+tQe67GUwl
-Cy68yHMFUmY6B2jBBhQLQlCVvB1HF3Tg0YGy9+OFBx00N2ysPRNhcuE2Hwv0mM4C
-UMfRd7zhQDCEKDqqsJFOOrgu/MJKo3qkJBoreE4lmlnrIyhYpkN1TwAz4jVUCklV
-7ZKxlI+3hii+ifNCPdU=
------END CERTIFICATE-----
-"""
-
- def _respond_with_certificate(self, handler, url_args):
- self._requested_certificate = True
-
- handler.send_response(httplib.OK)
- handler.send_header('Content-Type', 'text/plain')
- handler.send_header('Content-Length', str(len(self._certificate)))
- handler.end_headers()
- handler.wfile.write(self._certificate)
- handler.wfile.flush()
-
- def initialize(self):
- # Start a local web server (for the certificate fetch)
- self._listener = httpd.HTTPListener(80, docroot=self.srcdir)
- self._listener.add_url_handler('/GetRootCertificate',
- self._respond_with_certificate)
- self._listener.run()
-
- def run_once(self):
- self._requested_certificate = False
-
- # Get the current state of things
- orig_stat = os.stat(constants.FAKE_ROOT_CA_DIR)
- orig_resolv = open(constants.RESOLV_CONF_FILE).read()
-
- cmd = ( 'ip', 'route', 'show' )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- orig_routing_table, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestFail('Could not retrieve routing table')
-
- # Run the test with the recall wrapper
- with RecallServer('127.0.0.1'):
- # Certificate much have been fetched from our http server
- if not self._requested_certificate:
- raise error.TestFail('Never requested certificate.')
-
- # tmpfs must have been mounted over the root ca directory
- test_stat = os.stat(constants.FAKE_ROOT_CA_DIR)
- if (test_stat.st_ino, test_stat.st_dev) \
- == (orig_stat.st_ino, orig_stat.st_dev):
- raise error.TestFail('Did not mount tmpfs over %s',
- constants.FAKE_ROOT_CA_DIR)
-
- # Certificate must be in the nssdb
- cmd = ( 'certutil', '-d', 'sql:' + constants.FAKE_NSSDB_DIR,
- '-L', '-a', '-n', 'Chromium OS Test Server' )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- data, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestFail('Did not find certificate in nssdb')
- if data.replace('\r', '') != self._certificate:
- raise error.TestFail('Incorrect certificate in nssdb')
-
- # DNS server must be changed to our IP
- if open('/etc/resolv.conf').read() != 'nameserver 127.0.0.1\n':
- raise error.TestFail('Nameserver not changed')
-
- # Default route must have been changed
- cmd = ( 'ip', 'route', 'show' )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- data, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestFail('Could not retrieve routing table')
- for line in data.splitlines():
- if not line.startswith("default "):
- continue
- match = re.search(r'via ([^ ]*)', line)
- if not match:
- continue
- if match.group(1) != '127.0.0.1':
- raise error.TestFail('Default route gateway not changed')
- break
- else:
- raise error.TestFail('No default route found')
-
- # Mounted tmpfs must have been unmounted again
- test_stat = os.stat(constants.FAKE_ROOT_CA_DIR)
- if (test_stat.st_ino, test_stat.st_dev) \
- != (orig_stat.st_ino, orig_stat.st_dev):
- raise error.TestFail('Did not unmount tmpfs from %s',
- constants.FAKE_ROOT_CA_DIR)
-
- # DNS Resolver must have been restored
- if open('/etc/resolv.conf').read() != orig_resolv:
- raise error.TestFail('Nameserver settings not restored')
-
- # Routing table must have been restored
- cmd = ( 'ip', 'route', 'show' )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- data, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestFail('Could not retrieve routing table')
- if data != orig_routing_table:
- raise error.TestFail('Routing table not restored')
-
- def cleanup(self):
- self._listener.stop()
diff --git a/server/cros/recall/__init__.py b/server/cros/recall/__init__.py
deleted file mode 100644
index 0fb4020..0000000
--- a/server/cros/recall/__init__.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Recall server infrastructure.
-
-Recall is a server-side system that intercepts DNS, HTTP and HTTPS
-traffic from clients for recording, manipulation and playback. This
-allows tests to be run in labs without Internet access by playing back
-pre-recorded copies of websites, allows tests to be run that use
-recorded copies of sites known to exhibit errors or many iterations
-of the same test to be run with the same data.
-
-The server infrastructure consists of DNS, HTTP and HTTPS servers and
-a Certificate Authority that is used by the HTTPS server to generate
-certificates on the fly as needed.
-
-It also consists of DNS and HTTP clients, which are built in the
-middleware pattern. Each server has an associated client that it will
-use to fetch the necessary response. Clients can be stacked to build
-functionality, for example:
-
- from autotest_lib.server.cros import recall
- c = recall.ArchivingHTTPClient(
- recall.DeterministicScriptInjector(
- recall.HTTPClient()))
-
-This would create a client that after fetching the response from the
-Internet modifies it to inject JavaScript code to wrap Math.random()
-and Date() before finally archiving it. The order is important, since
-this means the archived response is already pre-mutated.
-
-To run an autotest client test on a remote machine wrapped in a
-Recall server instance for the most common use cases you only need use
-the pre-written test_RecallServer test.
-
-If you need to create a custom type of autotest server test, you can
-subclass that test from RecallServerTest to deal with the heavy lifting
-of reconfiguring the server to redirect traffic from the client.
-
-Finally, the classes in this module on their own are useful for building
-a standalone Recall server on a preconfigured system, perhaps with a DHCP
-server that already directs clients to it.
-"""
-
-# The code is split amongst multiple files for maintainability, this is
-# not intended to be exposed to other Python code, instead they should
-# treat it as a single module.
-#
-# This is correct:
-# from autotest_lib.server.cros import recall
-# so is this:
-# import autotest_lib.server.cros.recall
-#
-# But this is NOT:
-# from autotest_lib.server.cros.recall import dns_server
-#
-# We use __all__ in each of the files so that the following only imports
-# the public API. This is a minor sprain of the coding style, but one that
-# aids usability of this package.
-from certificate_authority import *
-from dns_client import *
-from dns_server import *
-from http_client import *
-from http_server import *
-from middleware import *
diff --git a/server/cros/recall/certificate_authority.py b/server/cros/recall/certificate_authority.py
deleted file mode 100644
index 4019def..0000000
--- a/server/cros/recall/certificate_authority.py
+++ /dev/null
@@ -1,417 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Certificate Authority for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["CertificateAuthority"]
-
-import ConfigParser
-import logging
-import os
-import Queue
-import shutil
-import subprocess
-import tempfile
-import threading
-
-from cStringIO import StringIO
-
-
-class CertificateAuthority(threading.Thread):
- """Certificate Authority for Recall server.
-
- This class implements an OpenSSL Certificate Authority for use with
- Recall, in particular the HTTPS server that utilises this class to
- generate needed certificates on the fly.
-
- Certificate Authority instances are picklable, the pickle will contain
- the set of requested and completed hostnames and restoring the instance
- from the pickle will create a new CA directory layout and generate
- certificates asynchronously for them.
-
- The authority must be initialized with a subject for the root
- certificate, e.g. "/O=Google/OU=Test". The following member variables
- are available:
-
- directory: location on disk of the certificate authority.
- private_key_file: location on disk of the CA private key file.
- certificate_file: location on disk of the CA certificate file.
-
- Certificates may be requested synchronously or asynchronously, using
- ca.GetCertificateAndPrivateKey() issues a synchronous request and the
- function will return the certificate and private keys when the
- generation is complete.
-
- Alternatively ca.RequestCertificateAndPrivateKey() merely begins
- generation in the background, a later call to the former method is
- required to obtain the keys by which point they will hopefully be
- generated (if not, that call blocks until they are).
-
- A typical use for this would be a DNSClient requesting certificate
- generation so that by the time HTTPSServer gets the files, they are
- already generated and it need not block.
-
- The shutdown() method of the CA must be called to clean up.
- """
- logger = logging.getLogger("CertificateAuthority")
-
- def __init__(self, subject, default_days=365):
- super(CertificateAuthority, self).__init__()
- self._subject = subject
- self._default_days = default_days
-
- self._SetupDirectory()
- self._GenerateAuthorityCertificateAndPrivateKey(self._subject)
-
- # We use a Queue to store the set of hostnames requested and also
- # duplicate them in the _requested set so we can quickly lookup
- # whether we need to block or queue a new incoming hostname.
- #
- # The _completed hash stores the set of generated certificates and
- # private keys, when a new certificate is added to this hash the
- # _changed Event is triggered to wake up the blockers.
- #
- # All of these structures are protected by a single lock.
- self._queue = Queue.Queue()
- self._requested = set()
- self._completed = {}
- self._lock = threading.Lock()
- self._changed = threading.Event()
-
- self.logger.info("Starting")
- self.daemon = True
- self.start()
-
- def shutdown(self):
- """Shut down the server.
-
- This method must be called to clean up the CA directory and shut down
- the worker thread.
- """
- # Place None in the queue to inform the thread to shutdown, and join
- # to wait for it to do so.
- self.logger.info("Shutting down")
- self._queue.put(None)
- self._queue.join()
-
- if hasattr(self, 'directory'):
- self._CleanupDirectory()
- else:
- self.logger.info("No directory to clean up")
-
- def __getstate__(self):
- state = {}
- state['subject'] = self._subject
- state['default_days'] = self._default_days
- with self._lock:
- state['certificates'] = set()
- state['certificates'].update(self._completed.keys())
- state['certificates'].update(self._requested)
- return state
-
- def __setstate__(self, state):
- self.__init__(state['subject'], state['default_days'])
-
- for hostname in state['certificates']:
- self.GetCertificateAndPrivateKey(hostname)
-
- def run(self):
- """Worker thread method.
-
- This is called by threading.Thread, it loops continuously taking new
- requests from the queue and processing them.
-
- Call shutdown() to stop the thread and exit.
- """
- while True:
- # Queue.get() blocks forever, we place None in the queue in
- # shutdown() to break out.
- hostname = self._queue.get(block=True)
- if hostname is None:
- self._queue.task_done()
- return
-
- try:
- self._ProcessRequest(hostname)
- except:
- self.logger.exception("Certificate generation failed")
- finally:
- self._queue.task_done()
-
- def _SetupDirectory(self):
- """Setup OpenSSL CA directory.
-
- Create an OpenSSL Certificate Authority directory layout, including
- the configuration file.
- """
- self.directory = tempfile.mkdtemp(prefix='CertificateAuthority')
- self.logger.info("Creating certificate authority in %s", self.directory)
-
- config = ConfigParser.RawConfigParser()
- config.optionxform = str
-
- config.add_section('ca')
- config.set('ca', 'default_ca', 'CA_default')
-
- config.add_section('CA_default')
- config.set('CA_default', 'dir', self.directory)
-
- self._certificate_dir = os.path.join(self.directory, 'certs')
- os.mkdir(self._certificate_dir, 0700)
- config.set('CA_default', 'certs', self._certificate_dir)
-
- crl_dir = os.path.join(self.directory, 'crl')
- os.mkdir(crl_dir, 0700)
- config.set('CA_default', 'crl_dir', crl_dir)
-
- newcerts_dir = os.path.join(self.directory, 'newcerts')
- os.mkdir(newcerts_dir, 0700)
- config.set('CA_default', 'new_certs_dir', newcerts_dir)
-
- self._private_key_dir = os.path.join(self.directory, 'private')
- os.mkdir(self._private_key_dir, 0700)
-
- database_file = os.path.join(self.directory, 'index.txt')
- open(database_file, 'w').close()
- config.set('CA_default', 'database', database_file)
-
- serial_file = os.path.join(self.directory, 'serial')
- with open(serial_file, 'w') as serialfile:
- print >>serialfile, '01'
- config.set('CA_default', 'serial', serial_file)
-
- self.certificate_file = os.path.join(self._certificate_dir, 'CA.pem')
- config.set('CA_default', 'certificate', self.certificate_file)
-
- self.private_key_file = os.path.join(self._private_key_dir, 'CA.key')
- config.set('CA_default', 'private_key', self.private_key_file)
-
- config.set('CA_default', 'name_opt', 'ca_default')
- config.set('CA_default', 'cert_opt', 'ca_default')
-
- config.set('CA_default', 'default_md', 'sha1')
-
- config.set('CA_default', 'policy', 'policy_anything')
-
- config.add_section('policy_anything')
- config.set('policy_anything', 'countryName', 'optional')
- config.set('policy_anything', 'stateOrProvinceName', 'optional')
- config.set('policy_anything', 'localityName', 'optional')
- config.set('policy_anything', 'organizationName', 'optional')
- config.set('policy_anything', 'commonName', 'supplied')
- config.set('policy_anything', 'emailAddress', 'optional')
-
- self._config_file = os.path.join(self.directory, 'openssl.cnf')
- with open(self._config_file, 'w') as configfile:
- config.write(configfile)
-
- def _CleanupDirectory(self):
- """Cleanup OpenSSL directory
-
- Remove the CA directory from the disk, undoes _SetupDirectory()
- """
- self.logger.info("Removing certificate authority directory")
- shutil.rmtree(self.directory)
-
- def _GenerateAuthorityCertificateAndPrivateKey(self, subject, days=None):
- """Generate the CA certificate and private key.
-
- Args:
- subject: subject for the CA key request.
- days: days key should be valid, defaults to that passed to constructor.
-
- Returns:
- tuple of certificate and private key filenames.
- """
- self.logger.info("Generating CA certificate and key for %s", subject)
-
- cmd = ('openssl', 'req', '-x509', '-newkey', 'rsa:1024', '-nodes',
- '-subj', subject,
- '-days', '%d' % (days or self._default_days),
- '-keyout', self.private_key_file,
- '-out', self.certificate_file)
- with tempfile.TemporaryFile() as output:
- try:
- subprocess.check_call(cmd, cwd=self.directory,
- stdout=output, stderr=output)
- except subprocess.CalledProcessError:
- output.seek(0)
- self.logger.debug("openssl output:\n%s", output.read())
- raise test.TestError("Failed to generate CA certificate and key")
-
- os.chmod(self.private_key_file, 0400)
-
- return self.certificate_file, self.private_key_file
-
- def _GenerateCertificateRequestAndPrivateKey(self, basename, subject,
- days=None):
- """Generate a certificate request and private key.
-
- Args:
- basename: base for the filename (usually the hostname).
- subject: subject for the certificate request.
- days: days key should be valid, defaults to that passed to constructor.
-
- Returns:
- tuple of certificate request and private key filenames.
- """
- self.logger.info("Generate certificate request and key for %s in %s",
- subject, basename)
- private_key_file = os.path.join(self._private_key_dir, '%s.key' % basename)
- certificate_request_file = os.path.join(self.directory, '%s.csr' % basename)
-
- cmd = ('openssl', 'req', '-new', '-newkey', 'rsa:1024', '-nodes',
- '-subj', subject,
- '-days', '%d' % (days or self._default_days),
- '-keyout', private_key_file,
- '-out', certificate_request_file)
- with tempfile.TemporaryFile() as output:
- try:
- subprocess.check_call(cmd, cwd=self.directory,
- stdout=output, stderr=output)
- except subprocess.CalledProcessError:
- output.seek(0)
- self.logger.debug("openssl output:\n%s", output.read())
- raise test.TestError("Failed to generate certificate request for %s",
- subject)
-
- os.chmod(self.private_key_file, 0400)
-
- return certificate_request_file, private_key_file
-
- def _SignCertificateRequest(self, certificate_request_file, days=None):
- """Sign certificate request file.
-
- Args:
- certificate_request_file: filename of certificate request.
- days: days certificate should be valid, defaults to that passed to
- constructor.
-
- Returns:
- filename of certificate.
- """
- basename = os.path.splitext(os.path.basename(certificate_request_file))[0]
- self.logger.info("Signing certificate request for %s", basename)
-
- certificate_file = os.path.join(self._certificate_dir, '%s.pem' % basename)
-
- cmd = ('openssl', 'ca', '-config', self._config_file, '-batch',
- '-days', '%d' % (days or self._default_days),
- '-out', certificate_file,
- '-in', certificate_request_file)
- with tempfile.TemporaryFile() as output:
- try:
- subprocess.check_call(cmd, cwd=self.directory,
- stdout=output, stderr=output)
- except subprocess.CalledProcessError:
- output.seek(0)
- self.logger.debug("openssl output:\n%s", output.read())
- raise test.TestError("Failed to sign certificate for %s", basename)
-
- return certificate_file
-
- def _GenerateCertificateAndPrivateKey(self, basename, subject, days=None):
- """Generate a certificate and private key.
-
- Args:
- basename: base for the filename (usually the hostname).
- subject: subject for the certificate.
- days: days key should be valid, defaults to that passed to constructor.
-
- Returns:
- tuple of certificate and private key filenames.
- """
- certificate_request_file, private_key_file = \
- self._GenerateCertificateRequestAndPrivateKey(basename, subject,
- days=days)
- try:
- certificate_file = self._SignCertificateRequest(
- certificate_request_file, days=days)
- except:
- os.unlink(private_key_file)
- raise
- finally:
- os.unlink(certificate_request_file)
-
- return certificate_file, private_key_file
-
- def RequestCertificateAndPrivateKey(self, hostname):
- """Asynchronously request generation of certificate and private key.
-
- This method returns immediately while the certificate and private key
- are generated in the background by the worker thread, or if a
- certificate and private key already exist for the hostname given.
-
- To request the filenames, call GetCertificateAndPrivateKey()
-
- Args:
- hostname: hostname required.
- """
- with self._lock:
- if hostname in self._requested \
- or hostname in self._completed:
- return
-
- self._requested.add(hostname)
- self._queue.put(hostname)
-
- def GetCertificateAndPrivateKey(self, hostname):
- """Retrieve certificate and private key.
-
- Makes a synchronous request to obtain a certificate and private key
- for the hostname given. If they have already been generated, for
- example by a prior call to this method or to the
- RequestCertificateAndPrivateKey() method, this returns immediately.
-
- Otherwise this function will block until the worker thread has
- generated the necessary keys, which may take some time depending
- on available entropy.
-
- Args:
- hostname: hostname required.
-
- Returns:
- tuple of certificate and private key filenames.
- """
- while True:
- with self._lock:
- if hostname in self._completed:
- certificate_file, private_key_file = self._completed[hostname]
- return certificate_file, private_key_file
- elif hostname not in self._requested:
- self._requested.add(hostname)
- self._queue.put(hostname)
-
- # Wait for the _completed hash to change, and clear the event when
- # it does. All threads are woken up simultaneously for each change.
- self._changed.wait()
- self._changed.clear()
-
- def _ProcessRequest(self, hostname):
- """Process a request.
-
- Called from the worker thread to process a request, generates the
- certificate request and private key, and then signs the request,
- before placing the respective keys into the _completed hash and
- waking up all threads.
-
- Args:
- hostname: hostname requested.
- """
- self.logger.debug("Received request for %s", hostname)
- subject = "/CN=%s" % hostname
-
- (certificate_file, private_key_file) \
- = self._GenerateCertificateAndPrivateKey(hostname, subject)
-
- with self._lock:
- self._completed[hostname] = (certificate_file, private_key_file)
- self._requested.remove(hostname)
-
- self.logger.debug("Request for %s completed", hostname)
- self._changed.set()
diff --git a/server/cros/recall/dns_client.py b/server/cros/recall/dns_client.py
deleted file mode 100644
index 93d1093..0000000
--- a/server/cros/recall/dns_client.py
+++ /dev/null
@@ -1,488 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""DNS Client classes for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["DNSRequest", "DNSResponse", "DNSClient", "DNSMiddleware",
- "ArchivingDNSClient", "SymmetricDNSClient"]
-
-import logging
-import socket
-import struct
-import threading
-
-import dns.rdataclass
-import dns.rdatatype
-import dns.resolver
-
-
-class DNSMessage(object):
- """Simple DNS Message object base class.
-
- This class encapsulates the relatively complicated API of dnspython
- into an instance that holds the relevant parts of a message together
- and implements matching and hashing.
-
- DNS Message objects are picklable.
-
- Generally you instantiate either the DNSRequest or DNSResponse objects
- rather than this base class.
- """
- def __init__(self, text, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN):
- """Create a DNS Message.
-
- Args:
- text: text content of the record, e.g. IP address or hostname.
- rdtype: data type of record, use a dns.rdatatype.* constant.
- rdclass: data class of record, use a dns.rdataclass.* constant.
- """
- try:
- self.text = text.to_text()
- except AttributeError:
- self.text = text
- self.rdtype = rdtype
- self.rdclass = rdclass
-
- def __str__(self):
- return "%s %s %s" % (dns.rdataclass.to_text(self.rdclass),
- dns.rdatatype.to_text(self.rdtype),
- self.text)
-
- def __repr__(self):
- return "<%s: %s>" % (self.__class__.__name__, str(self))
-
- def __hash__(self):
- return hash(repr((self.text, self.rdtype, self.rdclass)))
-
- def __eq__(self, other):
- if other.text != self.text:
- return False
- elif other.rdtype != self.rdtype:
- return False
- elif other.rdclass != self.rdclass:
- return False
- else:
- return True
-
- def IsAddress(self):
- """Return whether this is an address message.
-
- Returns:
- True if the data class/type is IN A; for a Request that means this
- is True for a request for an IP Address (ie. a hostname lookup),
- for a Response that means this is True for a response containing
- an IP Address for a hostname.
-
- False otherwise.
- """
- return self.rdclass == dns.rdataclass.IN \
- and self.rdtype == dns.rdatatype.A
-
- def IsName(self):
- """Return whether this is a name message.
-
- Returns:
- True if the data class/type is IN PTR; for a Request that means this
- is True for a request for a hostname (ie. a reverse lookup), for a
- Response that means this is True for a response containing a
- hostname for an IP Address.
-
- False otherwise.
- """
- return self.rdclass == dns.rdataclass.IN \
- and self.rdtype == dns.rdatatype.PTR
-
-
-class DNSRequest(DNSMessage):
- """Simple DNS Request object.
-
- Encapsulates a DNS Request.
- """
- @classmethod
- def ReverseLookup(cls, address):
- """Create a DNS Request for a reverse lookup.
-
- This class method constructor creates a DNSRequest object for a
- given IP address that results in a reverse lookup.
-
- i.e. DNSRequest.ReverseLookup('192.168.1.1') results in an instance
- of <DNSRequest IN PTR 1.1.168.192.in-addr.arpa>
-
- Args:
- address: IP address in usual dotted form.
-
- Returns:
- newly created DNSRequest object.
- """
- text = '.'.join(reversed(address.split('.'))) + '.in-addr.arpa'
- return cls(text, dns.rdatatype.PTR, dns.rdataclass.IN)
-
-
-class DNSResponse(DNSMessage):
- """Simple DNS Response object.
-
- Encapsulates a DNS Response.
- """
- pass
-
-
-class DNSClient(object):
- """DNS Client for system resolver.
-
- This class implements a DNS Client that uses the system resolver to
- lookup the responses for DNSRequest objects. DNS Client objects are
- picklable.
-
- Example:
- client = DNSClient()
- request = DNSRequest('www.google.com')
- for response in client(request):
- ...
- """
- def __init__(self):
- self._resolver = dns.resolver.get_default_resolver()
-
- def __getstate__(self):
- state = self.__dict__.copy()
- del state['_resolver']
- return state
-
- def __setstate__(self):
- self.__dict__.update(state)
-
- self._resolver = dns.resolver.get_default_resolver()
-
- def __call__(self, request):
- """Lookup the request.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- DNSResponse for each reply. No responses are yielded for NXDOMAIN.
- """
- try:
- for answer in self._resolver.query(request.text,
- request.rdtype, request.rdclass):
- if answer.rdclass == dns.rdataclass.IN \
- and answer.rdtype == dns.rdatatype.A:
- text = answer.address
- elif answer.rdtype == dns.rdatatype.PTR:
- text = answer.target.to_text()
- else:
- continue
-
- yield DNSResponse(text, answer.rdtype, answer.rdclass)
- except dns.resolver.NXDOMAIN:
- return
-
-
-class DNSMiddleware(object):
- """Base class for DNS Client middleware.
-
- This class is a base class for DNSClient-compatible classes that
- accept a DNSClient argument to their constructor and surround it
- with their own processing.
-
- DNS Middleware options are picklable.
-
- When creating your subclass, if it will work without a DNS Client
- you can set the client_optional class member to True; an instance
- may raise KeyError in the case where it cannot proceed further
- without one.
- """
- client_optional = False
-
- def __init__(self, dns_client=DNSClient()):
- """Create the client middleware instance.
-
- Args:
- dns_client: DNS Client object to wrap, defaults to plain DNSClient()
- and may be None if client_optional is True for the class.
- """
- self.dns_client = dns_client
-
- @property
- def dns_client(self):
- return self._dns_client
-
- @dns_client.setter
- def dns_client(self, dns_client):
- assert dns_client is not None or self.client_optional
- self._dns_client = dns_client
-
- @dns_client.deleter
- def dns_client(self):
- assert self.client_optional
- self._dns_client = None
-
- def __call__(self, request):
- """Lookup the request.
-
- Returns an archived response if one exists, otherwise uses the next
- DNS Client in the middleware stack to resolve it, throwing KeyError
- if no further client exists.
-
- Subclasses should override this function replacing it with their
- own, there is no need to call the superclass version.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- DNSResponse for each reply. No responses are yielded for NXDOMAIN.
- """
- if not self._dns_client:
- raise KeyError
-
- responses = self._dns_client(request)
- for response in responses:
- yield response
-
-
-class ArchivingDNSClient(DNSMiddleware):
- """Archiving DNS Client middleware.
-
- This DNS Client middleware wraps a DNS Client and archives all of its
- responses. If a request is found in the archive, that is returned in
- place of using the DNS Client given to the constructor.
-
- The archive may be initialised with None as the DNS Client in which
- case KeyError will be raised if the request is not found in the archive.
-
- When using an HTTPS Server you should use SymmetricDNSClient instead.
- """
- client_optional = True
-
- logger = logging.getLogger("ArchivingDNSClient")
-
- def __init__(self, dns_client=DNSClient()):
- super(ArchivingDNSClient, self).__init__(dns_client)
-
- self._responses = {}
- self._lock = threading.Lock()
-
- def __getstate__(self):
- state = self.__dict__.copy()
- del state['_lock']
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
-
- self._lock = threading.Lock()
-
- def __call__(self, request):
- """Lookup the request.
-
- Returns an archived response if one exists, otherwise uses the next
- DNS Client in the middleware stack to resolve it, throwing KeyError
- if no further client exists.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- DNSResponse for each reply. No responses are yielded for NXDOMAIN.
- """
- try:
- with self._lock:
- for response in self._responses[request]:
- yield response
- self.logger.info("HIT %s", request)
- except KeyError:
- self.logger.info("MISS %s", request)
- if not self._dns_client:
- raise
-
- responses = self._dns_client(request)
- with self._lock:
- self._responses[request] = []
- for response in responses:
- self._responses[request].append(response)
- yield response
-
-
-class SymmetricDNSClient(DNSMiddleware):
- """Symmetric DNS Client middleware.
-
- This DNS Client middleware wraps a DNS Client and ensures that all
- responses are symmetric. I.e. a request for www.google.com will always
- return the same IP Address, eliding any multiple responses, and
- conversely no other request will return that same IP Address even if
- it could potentially resolve.
-
- The symmetricness allows the HTTPS Server to translate a destination
- IP Address to only one possible hostname for certificate generation.
-
- In the case where two hostnames resolve to only one IP Address, the
- second to be looked up is given a fake IP Address in the 10/8 range.
-
- An archive is kept of all responses and the class may be initialised
- with None as the DNS Client in which case KeyError will be raised if
- the request is not found in the archive.
- """
- client_optional = True
-
- logger = logging.getLogger("SymmetricDNSClient")
-
- def __init__(self, dns_client=DNSClient()):
- super(SymmetricDNSClient, self).__init__(dns_client)
-
- self._hostnames = {}
- self._addresses = {}
- self._reserve_addr = 0x0a000000
- self._lock = threading.Lock()
-
- def __getstate__(self):
- state = self.__dict__.copy()
- del state['_lock']
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
-
- self._lock = threading.Lock()
-
- def __call__(self, request):
- """Lookup the request.
-
- Returns an archived response if one exists, otherwise uses the next
- DNS Client in the middleware stack to resolve it, throwing KeyError
- if no further client exists.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- Single DNSResponse for the reply.
- No responses are yielded for NXDOMAIN.
- """
- if request.IsAddress():
- responses = self._LookupAddress(request)
- elif request.IsName():
- responses = self._LookupName(request)
- else:
- if not self._dns_client:
- raise KeyError
-
- self.logger.info("Fallback for request %s", request)
- responses = self._dns_client(request)
-
- for response in responses:
- yield response
-
- def _LookupAddress(self, request):
- """Lookup an address request.
-
- Translates a hostname into an IP Address, where this is the first
- lookup for either, uses the next DNS Client in the stack to obtain
- a real answer.
-
- Ideally the first IP Address from the real answer that has not already
- been returned for a different hostname is used. In the case where all
- addresses from the real reply have been returned for other hostnames,
- a fake IP address in the 10/8 range is returned.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- Single DNSResponse for the reply.
- No responses are yielded for NXDOMAIN.
- """
- hostname = request.text.rstrip('.')
-
- try:
- with self._lock:
- address = self._addresses[hostname]
- self.logger.info("HIT %s -> %s", hostname, address)
- except KeyError:
- if not self._dns_client:
- self.logger.info("MISS %s -> (no record)", hostname)
- raise
-
- responses = self._dns_client(request)
- with self._lock:
- nxdomain = True
- for response in responses:
- nxdomain = False
-
- if response.IsAddress():
- address = response.text
- if address not in self._hostnames:
- self.logger.info("MISS %s -> %s (using client)",
- hostname, address)
- break
- else:
- if nxdomain:
- address = None
- self.logger.info("MISS %s -> NXDOMAIN", hostname)
- else:
- addr = self._reserve_addr = self._reserve_addr + 1
- address = socket.inet_ntoa(struct.pack('!I', addr))
- self.logger.info("MISS %s -> %s (synthesised)", hostname, address)
-
- self._addresses[hostname] = address
- if address is not None:
- self._hostnames[address] = hostname
- if address is not None:
- yield DNSResponse(address, dns.rdatatype.A, dns.rdataclass.IN)
-
- def _LookupName(self, request):
- """Lookup a hostname request.
-
- Translates an IP Address into a hostname, where this is the first
- lookup for either, uses the next DNS Client in the stack to obtain
- a real answer.
-
- The hostname/address pair is recorded so that a forward lookup for
- the address of the hostname always returns the same result. If the
- hostname has returned a different address previously, NXDOMAIN is
- simulated for this lookup.
-
- Args:
- request: DNSRequest to lookup.
-
- Yields:
- Single DNSResponse for the reply.
- No responses are yielded for NXDOMAIN.
- """
- in_addr = '.in-addr.arpa'
- self.logger.debug('LookupName %r', request)
- assert request.text.rstrip('.').endswith(in_addr)
- address = '.'.join(
- reversed(request.text.rstrip('.')[:-len(in_addr)].split('.')))
-
- try:
- with self._lock:
- hostname = self._hostnames[address]
- self.logger.info("HIT %s -> %s", address, hostname)
- except KeyError:
- if not self._dns_client:
- self.logger.info("MISS %s -> (no record)", address)
- raise
-
- responses = self._dns_client(request)
- with self._lock:
- for response in responses:
- if response.IsName():
- hostname = response.text.rstrip('.')
- if hostname not in self._addresses:
- self.logger.info("MISS %s -> %s (using client)",
- address, hostname)
- break
- else:
- hostname = None
- self.logger.info("MISS %s -> NXDOMAIN", address)
-
- self._hostnames[address] = hostname
- if hostname is not None:
- self._addresses[hostname] = address
- if hostname is not None:
- yield DNSResponse(hostname + '.', dns.rdatatype.PTR, dns.rdataclass.IN)
diff --git a/server/cros/recall/dns_server.py b/server/cros/recall/dns_server.py
deleted file mode 100644
index 134f223..0000000
--- a/server/cros/recall/dns_server.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""DNS Server classes for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["DNSServer", "DNSRequestHandler"]
-
-import logging
-import SocketServer
-import threading
-
-import dns.flags
-import dns.message
-import dns.opcode
-import dns.rdataclass
-import dns.rdatatype
-import dns.rrset
-
-from dns_client import DNSRequest, DNSClient
-
-
-class DNSServer(SocketServer.ThreadingUDPServer,
- threading.Thread):
- """Simple multithreaded DNS Server.
-
- This class implements a multithreaded DNS Server that uses the DNS Client
- passed to the constructor to resolve requests.
-
- The shutdown() method must be called to clean up.
- """
- logger = logging.getLogger("DNSServer")
-
- def __init__(self, server_address,
- dns_client=DNSClient()):
- SocketServer.ThreadingUDPServer.__init__(self, server_address,
- DNSRequestHandler)
- self.request_queue_size = 128
-
- threading.Thread.__init__(self, target=self.serve_forever)
-
- self.dns_client = dns_client
-
- self.logger.info("Starting on %s", self.server_address)
- self.daemon = True
- self.start()
-
- def shutdown(self):
- """Shutdown the server."""
- self.logger.info("Shutting down")
- super(DNSServer, self).shutdown()
-
-
-class DNSRequestHandler(SocketServer.DatagramRequestHandler):
- """Request handler for DNS Server.
-
- Handles incoming DNS requests on behalf of DNSServer; the request
- is converted to a DNSRequest object and a response obtained from the
- DNSServer's dns_client member before being converted back to a
- dnspython object and written to the client.
- """
- logger = logging.getLogger("DNSRequestHandler")
-
- def _Error(self, query, opcode=dns.rcode.NOTIMP):
- """Generate an Error response.
-
- Generates and writes back an error response.
-
- Args:
- query: query to respond to.
- opcode: dns.rcode.* member for error type.
- """
- response = dns.message.make_response(query)
- response.flags |= dns.flags.RA | dns.flags.AA
- response.set_rcode(opcode)
- self.wfile.write(response.to_wire())
-
- def handle(self):
- """Handle the request."""
- query = dns.message.from_wire(self.rfile.read())
-
- if query.opcode() != dns.opcode.QUERY:
- self.logger.debug("Ignored unhandled DNS message type: %s",
- dns.opcode.to_text(query.opcode))
- return self._Error(query)
-
- if len(query.question) > 1:
- self.logger.debug("Ignored additional questions in DNS message")
- if query.question[0].rdclass != dns.rdataclass.IN:
- self.logger.debug("Ignored unhandled DNS query class: %s",
- dns.rdataclass.to_text(query.question[0].rdlcass))
- return self._Error(query)
- if query.question[0].rdtype != dns.rdatatype.A \
- and query.question[0].rdtype != dns.rdatatype.PTR:
- if query.question[0].rdtype != dns.rdatatype.AAAA: # IPv6 is hard
- self.logger.debug("Ignored unhandled DNS query type: %s",
- dns.rdatatype.to_text(query.question[0].rdtype))
-
- return self._Error(query)
-
- reply = dns.message.make_response(query)
- reply.flags |= dns.flags.RA | dns.flags.AA
-
- request = DNSRequest(query.question[0].name,
- query.question[0].rdtype, query.question[0].rdclass)
-
- try:
- for response in self.server.dns_client(request):
- reply.answer.append(dns.rrset.from_text(
- query.question[0].name,
- 3600,
- query.question[0].rdclass,
- query.question[0].rdtype,
- response.text))
- if not len(reply.answer):
- reply.set_rcode(dns.rcode.NXDOMAIN)
- except KeyError:
- reply.set_rcode(dns.rcode.NXDOMAIN)
-
- self.wfile.write(reply.to_wire())
diff --git a/server/cros/recall/http_client.py b/server/cros/recall/http_client.py
deleted file mode 100644
index 88b079b..0000000
--- a/server/cros/recall/http_client.py
+++ /dev/null
@@ -1,578 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""HTTP Client classes for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["HTTPConnection", "HTTPSConnection", "HTTPRequest", "HTTPResponse",
- "HTTPClient", "HTTPMiddleware", "ArchivingHTTPClient"]
-
-import httplib
-import logging
-import struct
-import threading
-import time
-import zlib
-
-
-class HTTPResponse(httplib.HTTPResponse):
- """Recordable HTTP Response.
-
- Subclasses the httplib.HTTPResponse class to add the ability to record
- the data received from the server, the delay between chunks, modify data
- recevied and pickle for later playback.
- """
-
- logger = logging.getLogger("HTTPResponse")
-
- def __init__(self, *args, **kwds):
- httplib.HTTPResponse.__init__(self, *args, **kwds)
- self._mutate_functions = []
- self._decompressor = None
- self._compressor = None
-
- def __getstate__(self):
- state = self.__dict__.copy()
- if 'fp' in state:
- del state['fp']
- del state['_mutate_functions']
- del state['_decompressor']
- del state['_compressor']
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
-
- self._mutate_functions = []
- self._decompressor = None
- self._compressor = None
-
- @property
- def headers(self):
- """List of (header, value) tuples.
-
- Unlike the superclass, this doesn't merge duplicate headers since that
- breaks cookies.
- """
- if self.msg is None:
- raise httplib.ResponseNotReady()
-
- self._headers = []
- for line in self.msg.headers:
- sep = line.index(':')
- header = line[:sep].rstrip()
- value = line[sep+1:].lstrip().rstrip('\r\n')
- self._headers.append((header, value))
-
- return self._headers
-
- def _read_status(self):
- """Read status line from server.
-
- Wrapped to set the time we received the status line, which gets converted
- to a delay by HTTPConnection.getresponse()
- """
- ret = httplib.HTTPResponse._read_status(self)
- self._status_receive_time = time.time()
- return ret
-
- def AddMutateFunction(self, function):
- """Add a function to mutate data chunks.
-
- Must be called before the chunks property is accessed and before any
- headers are sent.
-
- Performing mutation on an HTTP Request drops the Content-Length
- header from the request, requiring the server to generate it if
- necessary. It also enforces on-the-fly decompression and
- compression of the incoming chunks,
- """
- self._mutate_functions.append(function)
- # Mutate functions means the length of the content may change;
- # server must recalculate on the fly
- del self.msg['Content-Length']
-
- # It also means we have to decompress and compress again after
- content_encoding = self.getheader('Content-Encoding')
- if content_encoding in ('gzip', 'deflate'):
- self.logger.debug("Will decompresss data in order to mutate")
- if content_encoding == 'gzip':
- self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
- self._compressor = HTTPGzipCompressor()
- elif content_encoding == 'deflate':
- self._decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
- self._compressor = zlib.compressobj()
-
- def _RecordChunk(self, chunk, last=False):
- """Record and mutate chunk received from the server.
-
- This method handles the decompression and recompression of chunks.
- """
- delay = time.time() - self.start_time
- if self._decompressor:
- chunk = self._decompressor.decompress(chunk)
- for mutate_function in self._mutate_functions:
- chunk = mutate_function(chunk)
- if self._compressor:
- chunk = self._compressor.compress(chunk)
- if last:
- chunk += self._compressor.flush(zlib.Z_FULL_FLUSH)
- chunk += self._compressor.flush()
- else:
- chunk += self._compressor.flush(zlib.Z_SYNC_FLUSH)
-
- self._chunks.append((delay, chunk))
- return chunk
-
- @property
- def chunks(self):
- """Read data chunks from server.
-
- This yields each chunk, including the final empty chunk in the case of
- chunked transfer-encoding and may sleep between calls to simulate the
- delay between chunk receive times.
- """
- try:
- # TODO(keybuk): this should probably record the time of the last call
- # rather than just the delay
- last_delay = 0.0
- for delay, chunk in self._chunks:
- time.sleep(delay - last_delay)
- last_delay = delay
- yield chunk
- except AttributeError:
- self._chunks = []
-
- # (keybuk) non-chunked we just read the entire document as one chunk;
- # this means that for streaming, there's a bit of a wait the first time
- # since it has to reach the server first, but not when playing back
- if not self.chunked:
- yield self._RecordChunk(self.read(), last=True)
- return
-
- # (keybuk) this code is basically _read_chunked() from the superclass,
- # simplified to always read everything, and to yield the final empty
- # chunk since we need to send it to the client ourselves
- chunk_len = None
- while chunk_len != 0:
- line = self.fp.readline()
- i = line.find(';')
- if i >= 0:
- line = line[:i] # strip chunk-extensions
- try:
- chunk_len = int(line, 16)
- except ValueError:
- # close the connection as protocol synchronisation is
- # probably lost
- self.close()
- raise httplib.IncompleteRead(''.join(value))
-
- # (keybuk) we always want to yield the final empty chunk since we
- # need to send it to the client anyway
- yield self._RecordChunk(self._safe_read(chunk_len), last=(chunk_len==0))
- self._safe_read(2) # toss the CRLF at the end of the chunk
-
- # we read everything; close the "file"
- self.close()
-
-
-class HTTPGzipCompressor(object):
- """Wrapper around zlib.Decompress to handle HTTP gzip compression.
-
- We can't use the Python gzip module for this since it only encapsulates
- files, and we can't subclass zlib.Decompress itself because that object
- is "hidden".
-
- This wrapper only implements the compress() and flush() methods, so
- is equivalent to Python 2.4-era zlib
- """
-
- # see gzip.py _write_gzip_header() in the Python distribution
- GZIP_HEADER = (
- '\037\213' # magic header
- '\010' # compression method
- '\000' # flags (none)
- '\000\000\000\000' # packed time (use zero)
- '\002'
- '\377')
-
- def __init__(self):
- # see gzip.py in the Python distribution
- self._compressor = zlib.compressobj(
- 6, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
- self._crc = zlib.crc32('') & 0xffffffffL
- self._size = 0
- self._sent_header = False
-
- def compress(self, string):
- """Compress string, returning at least part of it.
-
- Wraps zlib.Decompress.compress() prepending the gzip header for the
- first chunk, and keeping track of the CRC and size of the compressed
- data.
-
- Args:
- string: string to be compressed;
- Returns:
- string of compressed data forming at least part of string
- """
- chunk = self._compressor.compress(string)
- if not self._sent_header:
- chunk = self.GZIP_HEADER + chunk
- self._sent_header = True
-
- self._crc = zlib.crc32(string, self._crc) & 0xffffffffL
- self._size += len(string)
- return chunk
-
- def flush(self, mode=zlib.Z_FINISH):
- """Flush all pending input.
-
- Wraps zlib.Decompress.flush(), when called without arguments or
- with zlib.Z_FINISH, appends the CRC and size of the compressed data
- as required for gzip files.
-
- Args:
- mode: see zlib.py documentation
- Returns:
- string of remaining compressed data;
- """
- chunk = self._compressor.flush(mode)
- if mode == zlib.Z_FINISH:
- chunk += struct.pack('<LL', long(self._crc), long(self._size))
- return chunk
-
-
-class HTTPConnectionMixIn: # HTTPConnection is not an object
- response_class = HTTPResponse
-
- def getresponse(self):
- """Get the response from the server.
-
- Wraps httplib.HTTPConnection to set the response's start_time variable
- used to caclulate the delay between chunks being received, and ensures
- that status_delay is the time it took to receive the status code line.
- """
- start_time = time.time()
- response = httplib.HTTPConnection.getresponse(self)
- response.start_time = start_time
- # calculate here since we can't set start_time before getresponse
- response.status_delay = response._status_receive_time - start_time
- return response
-
-class HTTPConnection(HTTPConnectionMixIn, httplib.HTTPConnection):
- pass
-
-class HTTPSConnection(HTTPConnectionMixIn, httplib.HTTPSConnection):
- pass
-
-
-class HTTPRequest(object):
- """Recordable HTTP Request.
-
- Companion request object for HTTPResponse that is picklable and hashable
- so it can be used as a key to find the appropriate HTTPResponse object;
- implements smart matching since some headers change, and some order
- changes.
-
- Because of this smart matching, one HTTPRequest object may in fact match
- several recorded ones. You should therefore use an indirect lookup where
- hashing an HTTPRequest in fact yields an array of matching requests and
- their responses, and compare the request you're looking for with those
- using MatchScore()
- """
-
- logger = logging.getLogger("HTTPRequest")
-
- unmatchable_headers = [
- 'Accept',
- 'Accept-Charset',
- 'Accept-Encoding',
- 'Accept-Language',
- 'Cache-Control',
- 'Cookie', # special matching
- 'Connection',
- 'Keep-Alive',
- 'Pragma',
- 'Referer', # special matching
- 'User-Agent',
- ]
-
- def __init__(self, host, command, path, headers=[], body=None, ssl=False):
- self.host = host
- self.command = command
- self.path = path
- self.headers = headers
- self.body = body
- self.ssl = ssl
-
- self._match_headers = self._MatchHeaders(self.headers)
-
- def __str__(self):
- return "%s %s://%s%s" % (self.command,
- self.ssl and 'https' or 'http',
- self.host, self.path)
-
- def __repr__(self):
- return "<%s: %s%r %s %r>" % (self.__class__.__name__,
- self.ssl and '(SSL) ' or '',
- self.host, self.command, self.path)
-
- def __hash__(self):
- return hash(repr((self.host, self.command, self.path, self._match_headers,
- self.body, self.ssl)))
-
- def __eq__(self, other):
- if other.host != self.host:
- return False
- elif other.command != self.command:
- return False
- elif other.ssl != self.ssl:
- return False
- elif other.path != self.path:
- return False
- elif other.body != self.body:
- return False
- elif other._match_headers != self._match_headers:
- return False
- else:
- return True
-
- def __getstate__(self):
- state = self.__dict__.copy()
- del state['_match_headers']
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
- self._match_headers = self._MatchHeaders(state['headers'])
-
- @classmethod
- def _MatchHeaders(cls, headers):
- match_headers = set()
- for header, value in headers:
- header = header.title()
- if header in cls.unmatchable_headers:
- continue
- match_headers.add((header, value))
- return match_headers
-
- @property
- def _cookies(self):
- """Values of all cookies present in the request.
-
- Returns all values from all Cookie headers, including those that provide
- multiple values.
- """
- cookies = set()
- for header, value in self.headers:
- if header.title() == 'Cookie':
- for cookie in value.split(';'):
- cookies.add(cookie.strip())
- return cookies
-
- def getheader(self, name, default=None):
- """Retrieve value of header."""
- for header, value in self.headers:
- if header.title() == name.title():
- return value
- else:
- return default
-
- def MatchScore(self, other):
- """Compare request with another and return a match score.
-
- The hash of an HTTPRequest object may match multiple other HTTPRequest
- objects, in which case you should calculate the MatchScore of the request
- you're seeking against those present and pick the highest one.
- """
- # These numbers are randomly picked and seem to work ok for now
- score = len(self._cookies.intersection(other._cookies)) * 10
- if self.getheader('referer') == other.getheader('referer'):
- score += 1000
- return score
-
-
-class HTTPClient(object):
- """Generic HTTP Client.
-
- This class implements an HTTP Client that fetches the results using
- the Python httplib library. HTTP Client objects are picklable.
-
- Example:
- client = HTTPClient()
- request = HTTPRequest('www.google.com', 'GET', '/')
- response = client(request)
- """
-
- def __call__(self, request):
- """Lookup the request.
-
- Args:
- request: HTTPRequest to lookup.
-
- Returns:
- HTTPResponse reply, which may include an error response.
- """
- if request.ssl:
- connection = HTTPSConnection(request.host)
- else:
- connection = HTTPConnection(request.host)
-
- connection.putrequest(request.command, request.path,
- skip_host=True,
- skip_accept_encoding=True)
-
- send_host = True
- send_accept_encoding = True
-
- for header, value in request.headers:
- if header.title() == 'Host':
- send_host = False
- elif header.title() == 'Accept-Encoding':
- send_accept_encoding = False
-
- connection.putheader(header, value)
-
- if send_host:
- connection.putheader('Host', request.host)
- if send_accept_encoding:
- connection.putheader('Accept-Encoding', '')
-
- connection.endheaders()
-
- if request.body is not None:
- connection.send(request.body)
-
- return connection.getresponse()
-
-
-class HTTPMiddleware(object):
- """Base class for HTTP Client middleware.
-
- This class is a base class for HTTPClient-compatible classes that
- accept a HTTPClient argument to their constructor and surround it
- with their own processing.
-
- HTTP Middleware objects are picklable.
-
- When creating your subclass, if it will work without a HTTP Client
- you can set the client_optional class member to True; an instance
- may raise KeyError in the case where it cannot proceed further
- without one.
- """
- client_optional = False
-
- def __init__(self, http_client=HTTPClient()):
- """Create the client middleware instance.
-
- Args:
- http_client: HTTP Client object to wrap, defaults to plain HTTPClient()
- and may be None if client_optional is True for the class.
- """
- self.http_client = http_client
-
- @property
- def http_client(self):
- return self._http_client
-
- @http_client.setter
- def http_client(self, http_client):
- assert http_client is not None or self.client_optional
- self._http_client = http_client
-
- @http_client.deleter
- def http_client(self):
- assert self.client_optional
- self._http_client = None
-
- def __call__(self, request):
- """Lookup the request.
-
- Subclasses should override this function replacing it with their
- own, there is no need to call the superclass version.
-
- Args:
- request: HTTPRequest to lookup.
-
- Returns:
- HTTPResponse reply, which may include an error response.
- """
- if not self._http_client:
- raise KeyError
-
- response = self._http_client(request)
- return response
-
-
-class ArchivingHTTPClient(HTTPMiddleware):
- """Archiving HTTP Client middleware.
-
- This HTTP Client middleware wraps an HTTP Client and archives all of its
- responses. If a request is found in the archive, that is returned in
- place of using the HTTP Client given to the constructor.
-
- The archive may be initialised with None as the HTTP Client in which
- case KeyError will be raised if the request is not found in the archive.
-
- Matching is done using HTTPRequest smart matching.
- """
- client_optional = True
-
- logger = logging.getLogger("ArchivingHTTPClient")
-
- def __init__(self, http_client=HTTPClient()):
- super(ArchivingHTTPClient, self).__init__(http_client)
-
- self._responses = {}
- self._lock = threading.Lock()
-
- def __getstate__(self):
- state = self.__dict__.copy()
- del state['_lock']
- return state
-
- def __setstate__(self, state):
- self.__dict__.update(state)
-
- self._lock = threading.Lock()
-
- def __call__(self, request):
- """Lookup the request.
-
- Returns an archived response if one exists, otherwise uses the next
- HTTP Client in the middleware stack to resolve it, throwing KeyError
- if no further client exists.
-
- Args:
- request: HTTPRequest to lookup.
-
- Returns:
- HTTPResponse reply, which may include an error response.
- """
- try:
- with self._lock:
- best_response, best_score = None, 0
- for original_request, response in self._responses[request]:
- score = request.MatchScore(original_request)
- if score >= best_score:
- best_response, best_score = response, score
-
- self.logger.info("HIT %s", request)
- return best_response
- except KeyError:
- self.logger.info("MISS %s", request)
- if not self._http_client:
- raise
-
- response = self._http_client(request)
- with self._lock:
- try:
- self._responses[request].append((request, response))
- except KeyError:
- self._responses[request] = [ (request, response) ]
- return response
diff --git a/server/cros/recall/http_server.py b/server/cros/recall/http_server.py
deleted file mode 100644
index a347f55..0000000
--- a/server/cros/recall/http_server.py
+++ /dev/null
@@ -1,328 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""HTTP Server classes for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["HTTPServer", "HTTPSServer", "HTTPRequestHandler"]
-
-import BaseHTTPServer
-import fnmatch
-import httplib
-import logging
-import SocketServer
-import ssl
-import tempfile
-import threading
-
-import socket_util
-from certificate_authority import CertificateAuthority
-from dns_client import DNSRequest, DNSClient
-from http_client import HTTPRequest, HTTPClient
-
-
-def _GetHostnameForAddress(dns_client, address):
- """Get the hostname for an address.
-
- Utility function that uses a DNS Client to perform a reverse lookup
- and returns the hostname without trailing periods.
- """
- for hostname in dns_client(DNSRequest.ReverseLookup(address)):
- return hostname.text.rstrip('.')
- else:
- return None
-
-
-class HTTPServer(SocketServer.ThreadingMixIn,
- BaseHTTPServer.HTTPServer,
- threading.Thread):
- """Simple multithreaded HTTP Server.
-
- This class implements a multithreaded HTTP Server that uses the HTTP Client
- passed to the constructor to resolve requests. For consistency with the
- HTTPSServer class, the constructor also accepts DNS Client and
- Certificate Authority arguments, though these will not be used.
-
- The shutdown() method must be called to clean up.
- """
- logger = logging.getLogger("HTTPServer")
-
- ssl = False
-
- def __init__(self, server_address,
- http_client=HTTPClient(),
- dns_client=DNSClient(),
- certificate_authority=None):
- BaseHTTPServer.HTTPServer.__init__(self, server_address, HTTPRequestHandler)
- self.request_queue_size = 128
-
- threading.Thread.__init__(self, target=self.serve_forever)
-
- self.http_client = http_client
- self.dns_client = dns_client
- self.certificate_authority = certificate_authority
-
- self.logger.info("Starting on %s", self.server_address)
- self.daemon = True
- self.start()
-
- def shutdown(self):
- """Shutdown the server."""
- self.logger.info("Shutting down")
- super(HTTPServer, self).shutdown()
-
-
-class HTTPSServer(SocketServer.ThreadingMixIn,
- BaseHTTPServer.HTTPServer,
- threading.Thread):
- """Multithreaded HTTPS Server.
-
- This class implements a multithreaded HTTPS Server that uses the HTTP Client
- passed to the constructor to resolve requests. The original destination
- address of incoming connections is resolved to a hostname using the passed
- DNS Client, and a certificate generated using the passed Certificate
- Authority.
-
- For best results, the DNS Client should be the SymmetricDNSClient class.
-
- The shutdown() method must be called to clean up.
- """
- logger = logging.getLogger("HTTPSServer")
-
- ssl = True
-
- def __init__(self, server_address,
- http_client=HTTPClient(),
- dns_client=DNSClient(),
- certificate_authority=None):
- BaseHTTPServer.HTTPServer.__init__(self, server_address, HTTPRequestHandler)
- self.request_queue_size = 128
-
- threading.Thread.__init__(self, target=self.serve_forever)
-
- self.http_client = http_client
- self.dns_client = dns_client
- self.certificate_authority = certificate_authority
-
- self.logger.info("Starting on %s", self.server_address)
- self.daemon = True
- self.start()
-
- def shutdown(self):
- """Shutdown the server."""
- self.logger.info("Shutting down")
- super(HTTPSServer, self).shutdown()
-
- def get_request(self):
- """Accept incoming request.
-
- Looks up the original destination of the address and resolves that to
- a hostname using the DNS Client passed to the constructor. Certificates
- and Private Keys are obtained from the class Certificate Authority,
- and each connection is individually wrapped through SSL.
- """
- (conn, address) = self.socket.accept()
- self.logger.debug("Accepted request from %s:%d", address[0], address[1])
-
- try:
- original_address, original_port \
- = socket_util.GetOriginalDestinationAddress(conn)
-
- certificate_hostname = _GetHostnameForAddress(self.dns_client,
- original_address)
- if certificate_hostname is None:
- certificate_hostname = original_address
-
- self.logger.debug("Original destination %s:%d; using certificate for %s",
- original_address, original_port, certificate_hostname)
- except (TypeError, KeyError):
- certificate_hostname = self.server_name
- self.logger.warn("Using our own certificate for this request")
-
- (certificate_file, private_key_file) = \
- self.certificate_authority.GetCertificateAndPrivateKey(
- certificate_hostname)
-
- return (ssl.wrap_socket(conn, server_side=True,
- certfile=certificate_file,
- keyfile=private_key_file),
- address)
-
-
-class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
- """Request handler for HTTP and HTTPS Servers.
-
- Handles incoming HTTP requests on behalf of HTTPServer and HTTPSServer
- (distinguished by their ssl members). The request is converted to an
- HTTPRequest object and a response obtained from the server's http_client
- member before being sent back to the client.
-
- Additionally if the incoming request is directed at the server itself,
- the first element of the path may be a function local to this class in
- which case it is run to generate the response.
- """
-
- protocol_version = 'HTTP/1.1'
-
- # Turn on buffering, we explicitly flush where we need to
- wbufsize = -1
-
- def __init__(self, request, client_address, server):
- self.logger = logging.getLogger("HTTPRequestHandler:%s:%d"
- % (client_address[0], client_address[1]))
-
- BaseHTTPServer.BaseHTTPRequestHandler.__init__(
- self, request, client_address, server)
-
- def log_request(self, code='-', size='-'):
- # we do our own request logging
- pass
-
- # reformat other log messages to our own logger
- def log_error(self, format, *args):
- self.logger.error(format, *args)
- def log_message(self, format, *args):
- self.logger.info(format, *args)
-
- # handle all methods the same way
- def do_HEAD(self):
- self._HandleRequest()
- def do_GET(self):
- self._HandleRequest()
- def do_POST(self):
- self._HandleRequest()
-
- def _RequestIsForSelf(self):
- """Check whether the request is for our server name or IP Address.
-
- Returns:
- True if request is for us, False otherwise.
- """
- server_aliases = [ self.server.server_name, self.request.getsockname()[0] ]
- try:
- sep = self.server.server_name.index('.')
- server_aliases.append(self.server.server_name[:sep])
- except ValueError:
- pass
-
- return self.host.split(':')[0] in server_aliases
-
- def _HandleRequest(self):
- """Handle the request."""
- # Lookup the hostname
- self.host = self.headers.get('host', None)
- if not self.host:
- try:
- original_address, original_port \
- = socket_util.GetOriginalDestinationAddress(self.request)
-
- hostname = _GetHostnameForAddress(self.server.dns_client,
- original_address)
- if hostname:
- self.host = '%s:%d' % (hostname, original_port)
- else:
- self.host = '%s:%d' % (original_address, original_port)
-
- self.logger.debug("Missing Host header in request, used %s", self.host)
- except TypeError:
- return self._Error("Missing Host header in request, "
- "and can't obtain original destination")
-
- # Handle requests for our own host
- if self._RequestIsForSelf():
- command = self.path[1:].split('/')
- try:
- return getattr(self, command[0])(*command[1:])
- except AttributeError:
- return self._Error("Unknown command %s" % command[0])
- except TypeError, e:
- return self._Error(str(e))
-
- content_length = int(self.headers.get('Content-Length', 0))
- if content_length:
- body = self.rfile.read(content_length)
- else:
- body = None
-
- request = HTTPRequest(self.host, self.command, self.path,
- self.headers.items(), body,
- self.server.ssl)
-
- try:
- response = self.server.http_client(request)
- except KeyError:
- return self._Error("Not found in archive", 404)
-
- if response.version == 10:
- self.protocol_version = 'HTTP/1.0'
- self.send_response(response.status, response.reason)
-
- sent_content_length = False
- for header, value in response.headers:
- self.send_header(header, value)
- if header.title() == 'Content-Length':
- sent_content_length = True
-
- # Sometimes we need to send the content-length header ourselves; in those
- # cases delay ending the headers until we receive the data from the server
- if response.chunked or sent_content_length:
- self.end_headers()
- self.wfile.flush()
- else:
- self.logger.debug("Will send Content-Length later")
-
- for chunk in response.chunks:
- if response.chunked:
- self.wfile.write('%x\r\n%s\r\n' % (len(chunk), chunk))
- else:
- if not sent_content_length:
- self.send_header('Content-Length', str(len(chunk)))
- self.end_headers()
- sent_content_length = True
- self.wfile.write(chunk)
- self.wfile.flush()
-
- # Should never happen, but let's be careful
- if not response.chunked and not sent_content_length:
- self.logger.debug("Handled empty request")
- self.send_header('Content-Length', '0')
- self.end_headers()
- self.wfile.flush()
-
- if response.version == 10:
- self.close_connection = 1
-
- def GetRootCertificate(self):
- """Generate a response with the CA's certificate.
-
- Command intended for use by clients, writes back the attached CA's
- root certificate.
- """
- with open(self.server.certificate_authority.certificate_file) \
- as cert:
- certificate = cert.read()
-
- self.send_response(httplib.OK)
- self.send_header('Content-Type', 'text/plain')
- self.send_header('Content-Length', str(len(certificate)))
- self.end_headers()
- self.wfile.write(certificate)
- self.wfile.flush()
-
- def _Error(self, message, code=httplib.INTERNAL_SERVER_ERROR):
- """Reply with an error.
-
- Generates an error reply and returns it to the client.
- """
- self.logger.warn(message)
-
- self.send_response(code)
- self.send_header('Content-Type', 'text/plain')
- self.send_header('Content-Length', str(len(message)))
- self.end_headers()
- self.wfile.write(message)
- self.wfile.flush()
diff --git a/server/cros/recall/middleware.py b/server/cros/recall/middleware.py
deleted file mode 100644
index 13341a9..0000000
--- a/server/cros/recall/middleware.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Middleware classes for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["DeterministicScriptInjector"]
-
-import logging
-import re
-
-from http_client import HTTPClient, HTTPMiddleware
-
-
-class DeterministicScriptInjector(HTTPMiddleware):
- """Mutate HTTP Requests to inject Deterministic JavaScript code.
-
- Implements HTTP Client middleware that alters text/html responses,
- inserting a <script> block to the top of the page that replaces the
- JavaScript Math.random() and Date() functions for deterministic
- behaviour.
-
- The Date() replacement uses the original request time as the returned
- value, to avoid issues where scripts loop while the date remains the
- same, the date is incremented between calls. However since scripts may
- be executed out of order, the date is only incremented every
- date_count_threshold calls.
- """
- logger = logging.getLogger("DeterministicScriptInjector")
-
- _html_re = re.compile(r'<html[^>]*>', re.IGNORECASE)
- _head_re = re.compile(r'<head[^>]*>', re.IGNORECASE)
- _body_re = re.compile(r'<body[^>]*>', re.IGNORECASE)
-
- _deterministic_script = """\
-<script>
- (function () {
- var orig_date = Date;
- var random_count = 0;
- var date_count = 0;
- var random_seed = 0.462;
- var time_seed = @@START_TIME@@;
- var random_count_threshold = 25;
- var date_count_threshold = 25;
- Math.random = function() {
- random_count++;
- if (random_count > random_count_threshold){
- random_seed += 0.1;
- random_count = 1;
- }
- return (random_seed % 1);
- };
- Date = function() {
- if (this instanceof Date) {
- date_count++;
- if (date_count > date_count_threshold){
- time_seed += 50;
- date_count = 1;
- }
- switch (arguments.length) {
- case 0: return new orig_date(time_seed);
- case 1: return new orig_date(arguments[0]);
- default: return new orig_date(arguments[0], arguments[1],
- arguments.length >= 3 ? arguments[2] : 1,
- arguments.length >= 4 ? arguments[3] : 0,
- arguments.length >= 5 ? arguments[4] : 0,
- arguments.length >= 6 ? arguments[5] : 0,
- arguments.length >= 7 ? arguments[6] : 0);
- }
- }
- return new Date().toString();
- };
- Date.__proto__ = orig_date;
- Date.prototype.constructor = Date;
- orig_date.now = function() {
- return new Date().getTime();
- };
- })();
-</script>
-"""
-
- def __call__(self, request):
- """Lookup the request.
-
- Args:
- request: HTTPRequest to lookup.
-
- Returns:
- HTTPResponse reply, which may include an error response.
- """
- response = self.http_client(request)
-
- content_type = response.getheader('Content-Type')
- if content_type and content_type.startswith('text/html'):
- self.logger.debug("Will inject script into %r for %s", response, request)
-
- start_time = int(response.start_time * 1000)
- self._inject_script = self._deterministic_script.replace(
- '@@START_TIME@@', str(start_time))
- response.AddMutateFunction(self._MutateChunk)
-
- return response
-
- def _InjectScriptAfter(self, match):
- return match.group(0) + self._inject_script
-
- def _MutateChunk(self, chunk):
- if self._inject_script:
- count = 0
- if chunk:
- chunk, count = self._head_re.subn(self._InjectScriptAfter, chunk, 1)
- if count:
- self.logger.debug("Injected script into HEAD")
- self._inject_script = None
- else:
- chunk, count = self._body_re.subn(self._InjectScriptAfter, chunk, 1)
- if count:
- self.logger.debug("Injected script into BODY")
- self._inject_script = None
-
- return chunk
diff --git a/server/cros/recall/socket_util.py b/server/cros/recall/socket_util.py
deleted file mode 100644
index 558dcac..0000000
--- a/server/cros/recall/socket_util.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Socket utilities for Recall server.
-
-This module should not be imported directly, instead the public classes
-are imported directly into the top-level recall package.
-"""
-
-__all__ = ["GetOriginalDestinationAddress"]
-
-import socket
-import struct
-
-
-SO_ORIGINAL_DST = 80
-SOCKADDR_IN = "!2xH4s8x"
-SOCKADDR_IN_LEN = 16
-
-def GetOriginalDestinationAddress(conn):
- """Get the original destination address of a connection.
-
- When connections undergo iptables redirection, the kernel stores the
- original destination address and port as a socket option. This method
- retrieves that.
-
- Args:
- conn: Socket to lookup.
-
- Returns:
- tuple of original address as string and port as integer
- or None if no original destination was found.
- """
- try:
- original_dst_addr \
- = conn.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, SOCKADDR_IN_LEN)
- original_port, original_in_addr \
- = struct.unpack(SOCKADDR_IN, original_dst_addr)
- original_addr = socket.inet_ntoa(original_in_addr)
-
- return original_addr, original_port
- except Exception, e:
- return None
diff --git a/server/cros/recall_test.py b/server/cros/recall_test.py
deleted file mode 100644
index 2d53784..0000000
--- a/server/cros/recall_test.py
+++ /dev/null
@@ -1,291 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Base class for Recall tests.
-
-Recall provides an infrastructure for proxying, recording, altering and
-playing back DNS, HTTP and HTTPS requests.
-
-This requires that an autotest server test be run, with some configuration
-changes to the server, and then the desired client test run on the client
-wrapped with the Recall context manager.
-
-The server test base class provided handles most of the heavy lifting for
-you. For an example server test, see test_RecallServer that implements the
-common case of recording and playback.
-"""
-
-import errno
-import logging
-import os
-import re
-import subprocess
-
-from autotest_lib.client.common_lib import error
-from autotest_lib.server import test, autotest
-from autotest_lib.server.cros import recall
-
-
-class RecallServerTest(test.test):
- """AutoTest test class for Recall tests.
-
- This base class handles adjusting the autotest server configuration
- to allow redirection of traffic from the remote client to it, and
- cleaning up afterwards.
-
- Subclasses should override the initialize method and setup the
- following members before calling the superclass method:
-
- certificate_authority: instance of recall.CertificateAuthority to
- use to generate or retrieve certificates.
- dns_client: instance of recall.DNSClient or compatible class to
- lookup DNS results.
- http_client: instance of recall.HTTPClient or compatible class to
- lookup HTTP results.
-
- If these are left as None, no appropriate server will be created by
- this method. The subclass may then choose to set up servers of its
- own and use InstallPortRedirect() to redirect traffic to them.
-
- Members available for use by subclasses:
-
- ANY_ADDRESS: pass to SocketServer instances to get a random port.
-
- source_address: local IP address that will likely be used to
- communicate with the remote client.
- source_interface: local interface of source_address.
-
- dns_server: recall.DNSServer created using dns_client.
- http_server: recall.HTTPServer created using http_client.
- https_server: recall.HTTPSServer created using http_client.
-
- Finally to run the client test on the remote client the subclass
- should call RunTestOnHost() in its run_once() function.
- """
- version = 1
-
- # Pass as server_address to SocketServer classes to get a random port
- ANY_ADDRESS = ('', 0)
-
- _send_redirects_sysctl_pattern = "net.ipv4.conf.%s.send_redirects"
-
- def __init__(self, job, bindir, outputdir):
- test.test.__init__(self, job, bindir, outputdir)
-
- # To be set by subclass initialize
- self.certificate_authority = None
- self.dns_client = None
- self.http_client = None
-
- # Set by our initialize
- self.dns_server = None
- self.http_server = None
- self.https_server = None
-
- self._send_redirects = None
- self._port_redirects = []
-
- def initialize(self, host):
- """Initialize the Recall server.
-
- Override in your subclass to setup the certificate_authority,
- dns_client and http_client members before calling the superclass
- method.
-
- You may also leave those as None and setup your own server
- instances if you prefer.
-
- This method sets the source_address and source_interface members
- to the local address and interface that would be used to reach
- the given host.
-
- Args:
- host: autotest host object for remote client.
- """
- if host.ip == '127.0.0.1':
- raise error.TestError("Recall server tests cannot be run against "
- "the autotest server or a VM running on it.")
-
- self._host = host
- self.source_address, self.source_interface = \
- self._GetAddressAndInterfaceForAddress(self._host.ip)
- logging.debug("Source address and interface for %s are %s, %s",
- self._host.ip,
- self.source_address, self.source_interface)
-
- # Disable ICMP redirects for the interface we use for the client
- self._send_redirects_sysctl = self._send_redirects_sysctl_pattern \
- % self.source_interface
- try:
- self._send_redirects = \
- self._GetAndSetSysctl(self._send_redirects_sysctl, '0')
- except IOError as e:
- if e.errno == errno.EACCES:
- raise error.TestError("Recall server tests must be run as root")
- else:
- raise
-
- # Setup the servers using the client classes provided
- if self.dns_client is not None:
- logging.info("Setting up DNS Server")
- self.dns_server = recall.DNSServer(self.ANY_ADDRESS,
- self.dns_client)
- self.InstallPortRedirect('udp', 53,
- self.dns_server.server_address[-1])
- self.InstallPortRedirect('tcp', 53,
- self.dns_server.server_address[-1])
- if self.http_client is not None:
- logging.info("Setting up HTTP and HTTPS Server")
- self.http_server = recall.HTTPServer(
- self.ANY_ADDRESS, self.http_client, self.dns_client,
- self.certificate_authority)
- self.InstallPortRedirect('tcp', 80,
- self.http_server.server_address[-1])
-
- if self.certificate_authority is not None:
- self.https_server = recall.HTTPSServer(
- self.ANY_ADDRESS, self.http_client, self.dns_client,
- self.certificate_authority)
- self.InstallPortRedirect('tcp', 443,
- self.https_server.server_address[-1])
-
- def cleanup(self):
- """Cleanup.
-
- If you override in your subclass, be sure to call the superclass
- method.
- """
- self._UninstallPortRedirects()
-
- if self._send_redirects is not None:
- self._GetAndSetSysctl(self._send_redirects_sysctl,
- self._send_redirects)
-
- if self.dns_server is not None:
- self.dns_server.shutdown()
- if self.http_server is not None:
- self.http_server.shutdown()
- if self.https_server is not None:
- self.https_server.shutdown()
-
- def _GetAddressAndInterfaceForAddress(self, ip_address):
- """Return the local address and interface to reach an address.
-
- Given the IP address of a remote machine, returns the local
- source address and interface that may be used to reach that
- machine.
-
- This may not be the actual return IP address the remote machine
- sees if a NAT or similar redirection is in the way, but if
- they are on the same local network, it should resolve cases of
- multiple interfaces.
-
- Args:
- ip_address: address of remote machine as string.
-
- Returns:
- tuple of local address and interface names as strings.
- """
- cmd = ( '/sbin/ip', 'route', 'get', ip_address )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- data, errdata = proc.communicate()
- if proc.returncode != 0:
- raise error.TestError("Failed to obtain local address for remote")
-
- match = re.search(r'src ([^ ]*)', data)
- if not match:
- raise error.TestError("Missing source address for remote")
- source = match.group(1)
-
- match = re.search(r'dev ([^ ]*)', data)
- if not match:
- raise error.TestError("Missing source device for remote")
- interface = match.group(1)
-
- return source, interface
-
- def _GetAndSetSysctl(self, sysctl_name, new_value=None):
- """Sets a sysctl, returning the old value.
-
- Args:
- sysctl_name: dotted-notation name of sysctl.
- new_value: new value, if None, only returns current value.
- """
- sysctl_path = os.path.join('/proc/sys', sysctl_name.replace('.', '/'))
- with open(sysctl_path, 'r+') as sysctl:
- old_value = sysctl.read()
- if new_value is not None:
- print >>sysctl, new_value
-
- return old_value
-
- def InstallPortRedirect(self, protocol, port, to_port):
- """Install a port redirection.
-
- Installs a port direction so that requests from the client for the
- given protocol and port are redirected to the local port given.
-
- Can be removed with _UninstallPortRedirects().
-
- Args:
- protocol: protocol to redirect ('tcp' or 'udp').
- port: integer port to redirect, or tuple of range to redirect.
- to_port: integer port to redirect to on current machine.
- """
- try:
- port_spec = ':'.join(str(p) for p in port)
- except TypeError:
- port_spec = str(port)
-
- logging.debug("Installing port redirection for %s %s -> %d",
- protocol, port_spec, to_port)
- cmd = ( '/sbin/iptables', '-t', 'nat', '-A', 'PREROUTING',
- '-p', protocol, '-s', self._host.ip,
- '--dport', port_spec, '-j', 'REDIRECT',
- '--to-ports', '%d' % to_port )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to install port redirection")
-
- self._port_redirects.append((protocol, port_spec, to_port))
-
- def _UninstallPortRedirects(self):
- """Uninstall port redirections.
-
- Removes all port redirects installed with _InstallPortRedirect().
- """
- for protocol, port_spec, to_port in self._port_redirects:
- cmd = ( '/sbin/iptables', '-t', 'nat', '-D', 'PREROUTING',
- '-p', protocol, '-s', self._host.ip,
- '--dport', port_spec, '-j', 'REDIRECT',
- '--to-ports', '%d' % to_port )
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- proc.wait()
- if proc.returncode != 0:
- raise error.TestError("Failed to uninstall port redirection")
-
- def RunTestOnHost(self, test, host, **args):
- """Run client test on host using this Recall server.
-
- Args:
- test: name of test to run.
- host: host object to run test on.
-
- Additional keyword arguments are passed as arguments to the test
- being run.
- """
- # (keybuk) don't hurt me, I copied this code from autotest itself
- opts = ["%s=%s" % (o[0], repr(o[1])) for o in args.items()]
- cmd = ", ".join([repr(test)] + opts)
- control = """\
-from autotest_lib.client.cros.recall import RecallServer
-with RecallServer(%r):
- job.run_test(%s)
-""" % (self.source_address, cmd)
-
- logging.debug("Running control file %s", control)
-
- client_autotest = autotest.Autotest(host)
- return client_autotest.run(control)
diff --git a/server/site_tests/test_RecallServer/control b/server/site_tests/test_RecallServer/control
deleted file mode 100644
index ef68223..0000000
--- a/server/site_tests/test_RecallServer/control
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-AUTHOR = "Chromium OS Project"
-NAME = "test_RecallServer"
-PURPOSE = "Run a client test using a Recall server"
-CRITERIA = """
-This test will succeed if the client test specified succeeds.
-"""
-TIME = "SHORT"
-TEST_CATEGORY = "General"
-TEST_CLASS = "test"
-TEST_TYPE = "server"
-
-DOC = """
-This test creates a Recall server instance, and runs a specified client
-test on the client machine intercepting DNS, HTTP and HTTPS traffic for
-recording, manipulation and later playback.
-
-Sample usage:
-
- # Run desktopui_UrlFetch on the client, record results which can
- # be found as <RESULTS DIR>/pickle
- ./run_remote_tests.sh --remote=<DEVICE IP> test_RecallServer \
- -a 'desktopui_UrlFetch'
-
- # Run desktopui_UrlFetch on the client, only returning results
- # found in the given pickle file created from a previous recording
- ./run_remote_tests.sh --remote=<DEVICE IP> test_RecallServer \
- -a 'desktopui_UrlFetch /path/to/pickle'
-
- # Run desktopui_UrlFetch on the client 100 times, the first time
- # will be recorded, the subsequent 99 will use the playback only
- ./run_remote_tests.sh --remote=<DEVICE IP> test_RecallServer \
- -a 'desktopui_UrlFetch num_iterations=100'
-
- # Run desktopui_UrlFetch on the client using a simple proxy server
- # that doesn't record (mostly for infrastructure testing)
- ./run_remote_tests.sh --remote=<DEVICE IP> test_RecallServer \
- -a 'desktopui_UrlFetch proxy_only=1'
-
-Infrastructure restrictions mean that this test must always be run as
-root (or with sudo), and may not be run against the 127.0.0.1 address.
-"""
-
-dict_args = {}
-if args:
- if args and '=' not in args[0]:
- dict_args['test'] = args.pop(0)
- if args and '=' not in args[0]:
- dict_args['pickle_file'] = args.pop(0)
- dict_args.update(x.split('=') for x in args)
-
-def run(machine):
- host = hosts.create_host(machine)
- job.run_test("test_RecallServer", host=host, **dict_args)
-
-parallel_simple(run, machines)
diff --git a/server/site_tests/test_RecallServer/test_RecallServer.py b/server/site_tests/test_RecallServer/test_RecallServer.py
deleted file mode 100644
index 0a7df11..0000000
--- a/server/site_tests/test_RecallServer/test_RecallServer.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import cPickle as pickle
-import logging
-import os
-
-from autotest_lib.client.common_lib import error
-from autotest_lib.server import test, autotest
-from autotest_lib.server.cros import recall_test, recall
-
-class test_RecallServer(recall_test.RecallServerTest):
- version = 1
-
- _certificate_authority_subject = "/O=Google/OU=Chromium OS Test Server"
-
- def initialize(self, host, pickle_file=None, proxy_only=False):
- if pickle_file is not None:
- logging.info("Restoring from pickle %s", pickle_file)
- self.certificate_authority, self.dns_client, self.http_client \
- = pickle.load(open(pickle_file))
- else:
- logging.info("Setting up recall server")
- self.certificate_authority = recall.CertificateAuthority(
- subject=self._certificate_authority_subject,
- default_days=1)
-
- if proxy_only:
- self.dns_client = recall.DNSClient()
- self.http_client = recall.HTTPClient()
- else:
- self.dns_client = recall.SymmetricDNSClient()
- self.http_client = recall.ArchivingHTTPClient(
- recall.DeterministicScriptInjector())
-
- recall_test.RecallServerTest.initialize(self, host)
-
- def run_once(self, host, test, proxy_only=False, num_iterations=1, **args):
- logging.info("Running test %s on remote client %s", test, host.ip)
- self.RunTestOnHost(test, host, **args)
-
- # Remove the second-level client so further iterations, including
- # later runs with the pickle we write out, don't proxy
- del self.dns_client.dns_client
- del self.http_client.http_client
- logging.info("Recording/proxying disabled")
-
- if not proxy_only:
- dump_file = os.path.join(self.resultsdir, 'pickle')
- logging.debug("Saving results to %s", dump_file)
- pickle.dump((self.certificate_authority,
- self.dns_client, self.http_client),
- open(dump_file, 'w'))
-
- # Repeat test for subsequent iterations now proxying is disabled
- if isinstance(num_iterations, str):
- num_iterations = int(num_iterations)
- if num_iterations > 1:
- logging.info("Running %d more iterations", num_iterations - 1)
- self.RunTestOnHost(test, host, iterations=(num_iterations - 1),
- **args)
