nebraska: Add README and clean up help output.
Add a README and clean up help output.
BUG=chromium:892450
TEST=N/A
Change-Id: Ie2cf0e474a432d1ceda42e7edc9462c0ad0d4415
Reviewed-on: https://chromium-review.googlesource.com/1337519
Commit-Ready: Colin Howes <chowes@google.com>
Tested-by: Colin Howes <chowes@google.com>
Reviewed-by: Amin Hassani <ahassani@chromium.org>
diff --git a/nebraska/README.md b/nebraska/README.md
new file mode 100644
index 0000000..f3f1e2d
--- /dev/null
+++ b/nebraska/README.md
@@ -0,0 +1,82 @@
+# Nebraska
+
+Nebraska is a mock Omaha server built to facilitate testing of new DLCs
+(DownLoadable Content) as well as update and install mechanisms in Chrome OS. It
+is designed to be a lightweight, simple, and dependency-free standalone server
+that can run on a host machine or DUT. Its single purpose is to respond
+to update and install requests with Omaha-like responses based on some provided
+metadata. All other related functions (i.e. generating payloads, transferring
+files to a DUT, and serving the payloads themselves) must be handled
+elsewhere.
+
+## System Requirements
+
+The entire server is implemented in `nebraska.py` and Nebraska does not depend
+on anything outside of the Python standard libraries, so this file on its own
+should be sufficient to run the server on any device with Python 2.7.10 or
+newer.
+
+WARNING: External dependencies should never be added to Nebraska!
+
+## Payload Metadata
+
+Nebraska handles requests based on the contents of install and update payload
+directories. These directories should be populated with JSON files describing
+the payloads you wish to make available. On receiving a request, Nebraska
+searches the install or update payload directory as appropriate and formulates
+a response based on metadata describing a matching payload if it can find one.
+Nebraska expects at least one of these directories to be specified on startup.
+
+### Metadata Format
+
+The information in these files should be output by `paygen` when generating
+a payload, and should have the following key-value pairs. See `sample.json` for
+an example.
+
+```
+appid: The appid of the app provided by the payload.
+name: Payload file name.
+version: App version.
+is_delta: True iff the payload is a delta update.
+src_version: The source version if the payload is a delta update.
+size: Total size of the payload.
+metadata_sig: Metadata signature.
+metadata_size: Metadata size.
+hash_sha256: SHA256 of the payload.
+```
+
+## Payload URLs
+
+Nebraska does not serve the payload itself. It only provides payload metadata
+formatted in an Omaha-like response along with a URL where the payload can be
+found. The base of this URL can be given at startup.
+
+The URL can be the URL of some other server that serves payloads, or can be a
+file URL that points to a directory on the DUT containing update and install
+payloads. When responding to a request, Nebraska constructs the URL it uses in
+its response by appending "update" or "install" onto the end of the given base
+URL depending on whether the request is for an update or install. The complete
+URL for a payload can be constructed by concatenating the URL given in by
+Nebraska with the name of the payload, which is also given in the response from
+Nebraska. Giving a file URL in place of server URL is possible due to the fact
+that update engine relies on `libcurl` to handle the payload transfer, which is
+able to handle local files in the same way it handles remote URLs.
+
+## Known Issues
+
+### Lazy Version Checking
+
+We only check the first two version components when doing version comparisons.
+This is partially due to the way the final version component is constructed in
+development builds vs. production builds.
+
+### Detecting Update/Install
+
+In order to signal that a DLC should be installed rather than updated, update
+engine includes a request for the platform app without a request for an update.
+This allows Omaha to differentiate between an update operation where Chrome OS
+and all DLCs will be updated to a strictly greater version, and an install
+operation where one or more DLCs of the same version as the platform app will be
+installed. We currently decide whether a request is for an update or install
+based on whether exactly one other app (which we assume to be the platform app)
+the request list does not have an associated update request.
diff --git a/nebraska/appindex_unittest.py b/nebraska/appindex_unittest.py
index 49db56d..e0a3015 100755
--- a/nebraska/appindex_unittest.py
+++ b/nebraska/appindex_unittest.py
@@ -15,8 +15,8 @@
from unittest_common import AppDataGenerator
_NEBRASKA_PORT = 11235
-_SOURCE_DIR = "test_source_dir"
-_TARGET_DIR = "test_target_dir"
+_INSTALL_DIR = "test_install_dir"
+_UPDATE_DIR = "test_update_dir"
# pylint: disable=protected-access
@@ -124,10 +124,10 @@
with mock.patch('nebraska.os.listdir') as listdir_mock:
with mock.patch('nebraska.open') as open_mock:
listdir_mock.return_value = []
- app_index = nebraska.AppIndex(_SOURCE_DIR)
+ app_index = nebraska.AppIndex(_INSTALL_DIR)
app_index.Scan()
self.assertFalse(app_index._index)
- listdir_mock.assert_called_once_with(_SOURCE_DIR)
+ listdir_mock.assert_called_once_with(_INSTALL_DIR)
open_mock.assert_not_called()
def testScanNoJson(self):
@@ -135,10 +135,10 @@
with mock.patch('nebraska.os.listdir') as listdir_mock:
with mock.patch('nebraska.open') as open_mock:
listdir_mock.return_value = ["foo.bin", "bar.bin", "json"]
- app_index = nebraska.AppIndex(_SOURCE_DIR)
+ app_index = nebraska.AppIndex(_INSTALL_DIR)
app_index.Scan()
self.assertFalse(app_index._index)
- listdir_mock.assert_called_once_with(_SOURCE_DIR)
+ listdir_mock.assert_called_once_with(_INSTALL_DIR)
open_mock.assert_not_called()
def testScanMultiple(self):
@@ -162,9 +162,9 @@
mock.mock_open(read_data=JSONStrings.app_foobar).return_value
]
- app_index = nebraska.AppIndex(_SOURCE_DIR)
+ app_index = nebraska.AppIndex(_INSTALL_DIR)
app_index.Scan()
- listdir_mock.assert_called_once_with(_SOURCE_DIR)
+ listdir_mock.assert_called_once_with(_INSTALL_DIR)
self.assertTrue(set(app_index._index.keys()) ==
set(['foo', 'bar', 'foobar']))
self.assertTrue(len(app_index._index['foo']) == 2)
@@ -193,7 +193,7 @@
]
with self.assertRaises(IOError):
- app_index = nebraska.AppIndex(_SOURCE_DIR)
+ app_index = nebraska.AppIndex(_INSTALL_DIR)
app_index.Scan()
def testScanInvalidApp(self):
@@ -218,7 +218,7 @@
]
with self.assertRaises(KeyError):
- app_index = nebraska.AppIndex(_SOURCE_DIR)
+ app_index = nebraska.AppIndex(_INSTALL_DIR)
app_index.Scan()
diff --git a/nebraska/nebraska.py b/nebraska/nebraska.py
index 24ed526..b20770f 100755
--- a/nebraska/nebraska.py
+++ b/nebraska/nebraska.py
@@ -273,18 +273,18 @@
format based on the format of an XML response template.
"""
- def __init__(self, request, target_index, source_index, payload_addr):
+ def __init__(self, request, update_index, install_index, payload_addr):
"""Initialize a reponse from a list of matching apps.
Args:
request: Request instance describing client requests.
- target_index: Index of update payloads.
- source_index: Index of install payloads.
+ update_index: Index of update payloads.
+ install_index: Index of install payloads.
payload_addr: Address of payload server.
"""
self._request = request
- self._target_index = target_index
- self._source_index = source_index
+ self._update_index = update_index
+ self._install_index = install_index
self._payload_addr = payload_addr
curr = datetime.now()
@@ -314,8 +314,8 @@
logging.debug("Request for appid %s", str(app_request))
response_xml.append(self.AppResponse(
app_request,
- self._target_index,
- self._source_index,
+ self._update_index,
+ self._install_index,
self._payload_addr).Compile())
except Exception as err:
@@ -333,17 +333,17 @@
responses to pings and events as appropriate.
"""
- def __init__(self, app_request, target_index, source_index, payload_addr):
+ def __init__(self, app_request, update_index, install_index, payload_addr):
"""Initialize an AppResponse.
Attributes:
app_request: AppRequest representing a client request.
- target_index: Index of update payloads.
- source_index: Index of install payloads.
+ update_index: Index of update payloads.
+ install_index: Index of install payloads.
payload_addr: Address serving payloads.
"""
- _SOURCE_PATH = "/source/"
- _TARGET_PATH = "/target/"
+ _INSTALL_PATH = "/install/"
+ _UPDATE_PATH = "/update/"
self._app_request = app_request
self._app_data = None
@@ -352,19 +352,19 @@
if self._app_request.request_type == \
self._app_request.RequestType.INSTALL:
- self._app_data = source_index.Find(self._app_request)
- self._payload_url = payload_addr + _SOURCE_PATH
+ self._app_data = install_index.Find(self._app_request)
+ self._payload_url = payload_addr + _INSTALL_PATH
self._err_not_found = self._app_data is None
elif self._app_request.request_type == \
self._app_request.RequestType.UPDATE:
- self._app_data = target_index.Find(self._app_request)
- self._payload_url = payload_addr + _TARGET_PATH
+ self._app_data = update_index.Find(self._app_request)
+ self._payload_url = payload_addr + _UPDATE_PATH
# This differentiates between apps that are not in the index and apps
# that are available, but do not have an update available. Omaha treats
# the former as an error, whereas the latter case should result in a
# response containing a "noupdate" tag.
self._err_not_found = self._app_data is None and \
- not target_index.Contains(app_request)
+ not update_index.Contains(app_request)
if self._app_data:
logging.debug("Found matching payload: %s", str(self._app_data))
@@ -640,8 +640,8 @@
try:
response = Response(
request,
- self.server.owner.target_index,
- self.server.owner.source_index,
+ self.server.owner.update_index,
+ self.server.owner.install_index,
self.server.owner.payload_addr)
response_str = response.GetXMLString()
except Exception as err:
@@ -660,36 +660,36 @@
"""A simple Omaha server instance.
A simple mock of an Omaha server. Responds to XML-formatted update/install
- requests based on the contents of metadata files in target and source
+ requests based on the contents of metadata files in update and install
directories, respectively. These metadata files are used to configure
responses to Omaha requests from Update Engine and describe update and install
payloads provided by another server.
"""
- def __init__(self, payload_addr, target_dir, source_dir=None, port=0):
+ def __init__(self, payload_addr, update_dir, install_dir, port=0):
"""Initializes a server instance.
Args:
payload_addr: Address and port of the payload server.
- target_dir: Directory to index for information about target payloads.
- source_dir: Directory to index for information about source payloads.
+ update_dir: Directory to index for information about update payloads.
+ install_dir: Directory to index for information about install payloads.
port: Port the server should run on, 0 if the OS should assign a port.
Attributes:
- target_index: Index of metadata files in the target directory.
- source_index: Index of metadata files in the source directory.
+ update_index: Index of metadata files in the update directory.
+ install_index: Index of metadata files in the install directory.
"""
self._port = port
self._httpd = None
self._server_thread = None
self.payload_addr = payload_addr.strip('/')
- self.source_index = AppIndex(source_dir)
- self.target_index = AppIndex(target_dir)
+ self.update_index = AppIndex(update_dir)
+ self.install_index = AppIndex(install_dir)
def Start(self):
"""Starts a mock Omaha HTTP server."""
- self.target_index.Scan()
- self.source_index.Scan()
+ self.update_index.Scan()
+ self.install_index.Scan()
self._httpd = HTTPServer(('', self.Port()), NebraskaHandler)
self._port = self._httpd.server_port
@@ -718,18 +718,17 @@
"""
parser = argparse.ArgumentParser(description=__doc__)
- optional_args = parser.add_argument_group('Optional Arguments')
- optional_args.add_argument('--target-dir', metavar='DIR', default=None,
- help='Directory containing payloads for updates.',
- required=False)
- optional_args.add_argument('--source-dir', metavar='DIR', default=None,
- help='Directory containing payloads for '
- 'installation.', required=False)
- optional_args.add_argument('--port', metavar='PORT', type=int, default=0,
- help='Port to run the server on', required=False)
- optional_args.add_argument('--payload-addr', metavar='ADDRESS',
- help='Address and port of the payload server',
- default="http://127.0.0.1:8080")
+ parser.add_argument('--update-payloads', metavar='DIR', default=None,
+ help='Directory containing payloads for updates.',
+ required=False)
+ parser.add_argument('--install-payloads', metavar='DIR', default=None,
+ help='Directory containing payloads for installation.',
+ required=False)
+ parser.add_argument('--port', metavar='PORT', type=int, default=0,
+ help='Port to run the server on', required=False)
+ parser.add_argument('--payload-addr', metavar='URL',
+ help='Base payload URL.',
+ default="http://127.0.0.1:8080")
return parser.parse_args(argv[1:])
@@ -738,13 +737,13 @@
logging.basicConfig(level=logging.DEBUG)
opts = ParseArguments(argv)
- if not opts.target_dir and not opts.source_dir:
+ if not opts.update_payloads and not opts.install_payloads:
logging.error("Need to specify at least one payload directory.")
return os.EX_USAGE
nebraska = NebraskaServer(payload_addr=opts.payload_addr,
- target_dir=opts.target_dir,
- source_dir=opts.source_dir,
+ update_dir=opts.update_payloads,
+ install_dir=opts.install_payloads,
port=opts.port)
nebraska.Start()
diff --git a/nebraska/nebraska_unittest.py b/nebraska/nebraska_unittest.py
index ee0a4d7..93f02d8 100755
--- a/nebraska/nebraska_unittest.py
+++ b/nebraska/nebraska_unittest.py
@@ -15,8 +15,8 @@
from unittest_common import NebraskaHandler, NebraskaGenerator
_NEBRASKA_PORT = 11235
-_SOURCE_DIR = "test_source_dir"
-_TARGET_DIR = "test_target_dir"
+_INSTALL_DIR = "test_install_dir"
+_UPDATE_DIR = "test_update_dir"
_PAYLOAD_ADDR = "111.222.212:2357"
@@ -71,12 +71,12 @@
def testStart(self):
"""Tests Start."""
server = nebraska.NebraskaServer(
- _SOURCE_DIR, _TARGET_DIR, _PAYLOAD_ADDR, _NEBRASKA_PORT)
+ _INSTALL_DIR, _UPDATE_DIR, _PAYLOAD_ADDR, _NEBRASKA_PORT)
with mock.patch('nebraska.HTTPServer') as server_mock:
with mock.patch('nebraska.threading.Thread') as thread_mock:
- server.source_index = mock.MagicMock()
- server.target_index = mock.MagicMock()
+ server.install_index = mock.MagicMock()
+ server.update_index = mock.MagicMock()
server.Start()
@@ -88,13 +88,13 @@
mock.call(target=server._httpd.serve_forever),
mock.call().start()))
- server.source_index.Scan.assert_called_once()
- server.target_index.Scan.assert_called_once()
+ server.install_index.Scan.assert_called_once()
+ server.update_index.Scan.assert_called_once()
def testStop(self):
"""Tests Stop."""
nebraska_server = NebraskaGenerator(
- _PAYLOAD_ADDR, _TARGET_DIR, _SOURCE_DIR, _NEBRASKA_PORT)
+ _PAYLOAD_ADDR, _UPDATE_DIR, _INSTALL_DIR, _NEBRASKA_PORT)
# pylint: disable=protected-access
nebraska_server._httpd = mock.MagicMock(name="_httpd")
diff --git a/nebraska/response_unittest.py b/nebraska/response_unittest.py
index 09250a0..07c2b8a 100755
--- a/nebraska/response_unittest.py
+++ b/nebraska/response_unittest.py
@@ -140,13 +140,13 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
- target_index.Find.side_effect = app_list
+ update_index.Find.side_effect = app_list
response = nebraska.Response(
- request, target_index, source_index, payload_addr)
+ request, update_index, install_index, payload_addr)
response = response.GetXMLString()
@@ -177,19 +177,19 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- target_index.Find.return_value = match
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ update_index.Find.return_value = match
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertFalse(response._err_not_found)
self.assertTrue(response._app_data is match)
- target_index.Find.assert_called_once_with(app_request)
- source_index.Find.assert_not_called()
+ update_index.Find.assert_called_once_with(app_request)
+ install_index.Find.assert_not_called()
def testAppResponseInstall(self):
"""Tests AppResponse generation for install request with match."""
@@ -206,19 +206,19 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- source_index.Find.return_value = match
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ install_index.Find.return_value = match
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertFalse(response._err_not_found)
self.assertTrue(response._app_data is match)
- source_index.Find.assert_called_once_with(app_request)
- target_index.Find.assert_not_called()
+ install_index.Find.assert_called_once_with(app_request)
+ update_index.Find.assert_not_called()
def testAppResponseNoMatch(self):
"""Tests AppResponse generation for update request with an unknown appid."""
@@ -230,20 +230,20 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- target_index.Find.return_value = None
- target_index.Contains.return_value = False
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ update_index.Find.return_value = None
+ update_index.Contains.return_value = False
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertTrue(response._err_not_found)
self.assertTrue(response._app_data is None)
- target_index.Find.assert_called_once_with(app_request)
- source_index.Find.assert_not_called()
+ update_index.Find.assert_called_once_with(app_request)
+ install_index.Find.assert_not_called()
def testAppResponseNoUpdate(self):
"""Tests AppResponse generation for update request with no new versions."""
@@ -255,20 +255,20 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- target_index.Find.return_value = None
- target_index.Contains.return_value = True
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ update_index.Find.return_value = None
+ update_index.Contains.return_value = True
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertFalse(response._err_not_found)
self.assertTrue(response._app_data is None)
- target_index.Find.assert_called_once_with(app_request)
- source_index.Find.assert_not_called()
+ update_index.Find.assert_called_once_with(app_request)
+ install_index.Find.assert_not_called()
def testAppResponsePing(self):
"""Tests AppResponse generation for no-op with a ping request."""
@@ -280,20 +280,20 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- target_index.Find.return_value = None
- target_index.Contains.return_value = True
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ update_index.Find.return_value = None
+ update_index.Contains.return_value = True
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertFalse(response._err_not_found)
self.assertTrue(response._app_data is None)
- target_index.Find.assert_not_called()
- source_index.Find.assert_not_called()
+ update_index.Find.assert_not_called()
+ install_index.Find.assert_not_called()
def testAppResponseEvent(self):
"""Tests AppResponse generation for requests with events."""
@@ -306,20 +306,20 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- target_index.Find.return_value = None
- target_index.Contains.return_value = True
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ update_index.Find.return_value = None
+ update_index.Contains.return_value = True
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
self.assertTrue(response._app_request == app_request)
self.assertFalse(response._err_not_found)
self.assertTrue(response._app_data is None)
- target_index.Find.assert_not_called()
- source_index.Find.assert_not_called()
+ update_index.Find.assert_not_called()
+ install_index.Find.assert_not_called()
def testCompileSuccess(self):
"""Tests successful compilation of an AppData instance."""
@@ -336,12 +336,12 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- source_index.Find.return_value = match
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ install_index.Find.return_value = match
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
compiled_response = response.Compile()
@@ -358,7 +358,7 @@
self.assertTrue(action_tag is not None)
self.assertTrue(compiled_response.attrib['appid'] == match.appid)
- self.assertTrue(url_tag.attrib['codebase'] == payload_addr + "/source/")
+ self.assertTrue(url_tag.attrib['codebase'] == payload_addr + "/install/")
self.assertTrue(manifest_tag.attrib['version'] == match.version)
self.assertTrue(package_tag.attrib['hash_sha256'] == match.sha256_hash)
self.assertTrue(package_tag.attrib['fp'] == "1.%s" % match.sha256_hash)
@@ -385,11 +385,11 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
response.Compile()
@@ -416,12 +416,12 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- source_index.Find.return_value = match
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ install_index.Find.return_value = match
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
response.Compile()
@@ -446,12 +446,12 @@
payload_addr = "www.google.com"
- target_index = mock.MagicMock()
- source_index = mock.MagicMock()
- source_index.Find.return_value = match
+ update_index = mock.MagicMock()
+ install_index = mock.MagicMock()
+ install_index.Find.return_value = match
response = nebraska.Response.AppResponse(
- app_request, target_index, source_index, payload_addr)
+ app_request, update_index, install_index, payload_addr)
response.Compile()
