paygen: Deprecate release.conf
This CL removes the usage of crostools.config and
crostools.query from paygen_build_lib. No data
used during payload generation is collected from
the release.conf file.
All accesses of release.conf have been replaced
with querying the data published by goldeneye
in GCS.
BUG=chromium:525271
TEST=Unittests.
Reviewed-on: https://chromium-review.googlesource.com/300684
Commit-Ready: Matthew Sartori <msartori@chromium.org>
Tested-by: Matthew Sartori <msartori@chromium.org>
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Reviewed-by: Christine Lee <leecy@chromium.org>
(cherry picked from commit 503f1b2e00ce2871367d5cde5537d091578cad1c)
Change-Id: I7d03ff492d64cb0975d394c998c25a3ac6737bd1
Reviewed-on: https://chromium-review.googlesource.com/303216
Tested-by: Matthew Sartori <msartori@chromium.org>
Reviewed-by: Dharani Govindan <dharani@chromium.org>
diff --git a/lib/paygen/paygen_build_lib.py b/lib/paygen/paygen_build_lib.py
index 4b95713..e83f1cd 100644
--- a/lib/paygen/paygen_build_lib.py
+++ b/lib/paygen/paygen_build_lib.py
@@ -13,7 +13,6 @@
from __future__ import print_function
-import ConfigParser
import json
import os
import shutil
@@ -50,17 +49,12 @@
# will fail. We quietly ignore the failure, but leave bombs around that will
# explode if people try to really use this library.
try:
- from crostools.config import config
- from crostools.omaha import query
-
# pylint: disable=F0401
from site_utils.autoupdate.lib import test_params
from site_utils.autoupdate.lib import test_control
# pylint: enable=F0401
except ImportError:
- config = None
- query = None
test_params = None
test_control = None
@@ -71,8 +65,10 @@
# Used to format timestamps on archived paygen.log file names in GS.
PAYGEN_LOG_TIMESTAMP_FORMAT = '%Y%m%d-%H%M%S-UTC'
-# Used to lookup all FSIs for all boards.
+# Board and device information published by goldeneye.
+BOARDS_URI = 'gs://chromeos-build-release-console/boards.json'
FSI_URI = 'gs://chromeos-build-release-console/fsis.json'
+OMAHA_URI = 'gs://chromeos-build-release-console/omaha_status.json'
class Error(Exception):
@@ -117,7 +113,7 @@
class BoardNotConfigured(EarlyExit):
- """The board does not exist in the crostools release config."""
+ """The board does not exist in the published goldeneye records."""
RESULT = 26
@@ -552,28 +548,26 @@
List of gspaths.Build instances for each build so discovered. The list
may be empty.
"""
- # TODO(dgarrett): Switch to JSON mechanism in _DiscoverAllFsiBuilds
- # after it's in production, and after we clear the change with the TPMs.
- # At that time, check and ignore FSIs without the is_delta_supported flag.
- # TODO(pprabhu): Can't switch to _DiscoverAllFsiBuilds till the HACK there
- # is removed.
+ results = []
# FSI versions are only defined for the stable-channel.
if self._build.channel != 'stable-channel':
- return []
+ return results
- try:
- fsi_versions = config.GetListValue(self._build.board, 'fsi_images')
- except ConfigParser.NoOptionError:
- # fsi_images is an optional field.
- return []
+ contents = json.loads(gslib.Cat(FSI_URI))
- results = []
- for version in fsi_versions:
- results.append(gspaths.Build(version=version,
- board=self._build.board,
- channel=self._build.channel,
- bucket=self._build.bucket))
+ for fsi in contents.get('fsis', []):
+ fsi_active = fsi['board']['is_active']
+ fsi_board = fsi['board']['public_codename']
+ fsi_version = fsi['chrome_os_version']
+ fsi_support_delta = fsi['is_delta_supported']
+
+ if fsi_active and fsi_support_delta and fsi_board == self._build.board:
+ results.append(gspaths.Build(version=fsi_version,
+ board=fsi_board,
+ channel=self._build.channel,
+ bucket=self._build.bucket))
+
return results
def _DiscoverAllFsiBuilds(self):
@@ -590,20 +584,14 @@
may be empty.
"""
results = []
- # XXX:HACK -- FSI builds for this board is known to brick the DUTs in the
- # lab. As a workaround, we're dropping test coverage for this board
- # temporarily (crbug.com/460174).
- # TODO(pprabhu) Remove hack once we have a real solution (crbug.com/462320).
- if self._build.board == 'peach-pit':
- return results
-
contents = json.loads(gslib.Cat(FSI_URI))
for fsi in contents.get('fsis', []):
fsi_board = fsi['board']['public_codename']
fsi_version = fsi['chrome_os_version']
+ fsi_lab_stable = fsi['is_lab_stable']
- if fsi_board == self._build.board:
+ if fsi_lab_stable and fsi_board == self._build.board:
results.append(fsi_version)
return results
@@ -620,16 +608,21 @@
know about the currently published version, this always contain zero or
one entries.
"""
- self._previous_version = query.FindLatestPublished(self._build.channel,
- self._build.board)
+ results = []
- if self._previous_version:
- return [gspaths.Build(gspaths.Build(version=self._previous_version,
- board=self._build.board,
- channel=self._build.channel,
- bucket=self._build.bucket))]
+ contents = json.loads(gslib.Cat(OMAHA_URI))
+ for nmo in contents.get('omaha_data', []):
+ nmo_board = nmo['board']['public_codename']
+ nmo_channel = nmo['channel']
+ nmo_version = nmo['chrome_os_version']
- return []
+ if nmo_board == self._build.board and nmo_channel == self._build.channel:
+ results.append(gspaths.Build(gspaths.Build(version=nmo_version,
+ board=self._build.board,
+ channel=self._build.channel,
+ bucket=self._build.bucket)))
+
+ return results
def _DiscoverRequiredFullPayloads(self, images):
"""Find the Payload objects for the images from the current build.
@@ -1392,8 +1385,11 @@
BoardNotConfigured if the board is unknown.
"""
# Right now, we just validate that the board exists.
- if board not in config.GetCompleteBoardSet():
- raise BoardNotConfigured(board)
+ boards = json.loads(gslib.Cat(BOARDS_URI))
+ for b in boards.get('boards', []):
+ if b['public_codename'] == board:
+ return
+ raise BoardNotConfigured(board)
def CreatePayloads(build, work_dir, site_config, dry_run=False,
diff --git a/lib/paygen/paygen_build_lib_unittest.py b/lib/paygen/paygen_build_lib_unittest.py
index e5b7be4..5b0ddd7 100644
--- a/lib/paygen/paygen_build_lib_unittest.py
+++ b/lib/paygen/paygen_build_lib_unittest.py
@@ -317,17 +317,30 @@
False,
paygen_build_lib.ImageMissing)
- @unittest.skipIf(not paygen_build_lib.config, 'Internal crostools required.')
def testDiscoverActiveFsiBuilds(self):
- """Using test release.conf values, test _DiscoverActiveFsiBuilds."""
+ """Using test goldeneye values, test _DiscoverActiveFsiBuilds."""
+ # Set up mock goldeneye fsi information.
+ mock_return_fsi = paygen_build_lib.json.dumps(
+ {'fsis':
+ [{u'is_delta_supported': True, u'chrome_os_version': u'2465.105.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': True, u'chrome_version': u'31.0.1650.61'},
+ {u'is_delta_supported': True, u'chrome_os_version': u'2467.109.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': False, u'chrome_version': u'31.0.1650.61'},
+ {u'is_delta_supported': False, u'chrome_os_version': u'2913.331.0',
+ u'board':
+ {u'public_codename': u'valid-board', u'is_active': True},
+ u'is_lab_stable': True, u'chrome_version': u'31.0.1650.61'}]
+ }
+ )
- test_config = """
-[valid-board]
-fsi_images: 2913.331.0,2465.105.0
-
-[no-fsi-board]
-"""
- paygen_build_lib.config.LoadTestConfig(test_config)
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.FSI_URI).AndReturn(mock_return_fsi)
+ gslib.Cat(paygen_build_lib.FSI_URI).AndReturn(mock_return_fsi)
+ self.mox.ReplayAll()
# Test a board with FSI values on stable-channel.
paygen = paygen_build_lib._PaygenBuild(
@@ -343,7 +356,7 @@
version='2465.105.0'),
gspaths.Build(board='valid-board',
channel='stable-channel',
- version='2913.331.0')])
+ version='2467.109.0')])
# Test a board without FSI values on stable-channel.
paygen = paygen_build_lib._PaygenBuild(
@@ -363,10 +376,7 @@
self.assertEqual(paygen._DiscoverActiveFsiBuilds(), [])
- paygen_build_lib.config.LoadGlobalConfig()
-
@cros_test_lib.NetworkTest()
- @unittest.skipIf(not paygen_build_lib.config, 'Internal crostools required.')
def testDiscoverAllFsiBuilds(self):
"""Using test release.conf values, test _DiscoverActiveFsiBuilds."""
paygen = paygen_build_lib._PaygenBuild(
@@ -379,19 +389,56 @@
self.assertEqual(paygen._DiscoverAllFsiBuilds(),
['0.12.433.257', '0.14.811.132', '1412.205.0'])
- @unittest.skipIf(not paygen_build_lib.query, 'Internal crostools required.')
def testDiscoverNmoBuild(self):
"""Test _DiscoverNmoBuild (N minus One)."""
paygen = self._GetPaygenBuildInstance()
- self.mox.StubOutWithMock(paygen_build_lib.query, 'FindLatestPublished')
+ # Set up mock goldeneye omaha status information.
+ mock_return_foo = paygen_build_lib.json.dumps(
+ {'omaha_data':
+ [{u'is_mp_keyset': True, u'chrome_version': u'47.0.2514.0',
+ u'keyset': u'foo-mp',
+ u'board':
+ {u'public_codename': u'foo-board', u'is_active': True},
+ u'chrome_os_version': u'7478.0.0', u'channel': u'foo-channel',
+ u'payloads':
+ [{u'max_fraction': False,
+ u'name': u'foo-channel/foo/7478.0.0/payloads/'
+ u'chromeos_7475.0.0-7478.0.0_foo_foo'
+ u'-channel_delta_mp.bin-877f148a914c1cdbe2'
+ u'42aa4247a1d135.signed', u'fraction': 1.0},
+ {u'max_fraction': False,
+ u'name': u'foo-channel/foo/7478.0.0/payloads/'
+ u'chromeos_7478.0.0_foo_foo-channel_'
+ u'full_mp.bin-fddc0ae18c9845325c13704ee00b'
+ u'd0a4.signed', u'fraction': 1.0}]}]
+ }
+ )
- # Set up the test replay script.
- paygen_build_lib.query.FindLatestPublished(
- 'foo-channel', 'foo-board').AndReturn('1.0.0')
+ mock_return_not_foo = paygen_build_lib.json.dumps(
+ {'omaha_data':
+ [{u'is_mp_keyset': True, u'chrome_version': u'47.0.2514.0',
+ u'keyset': u'notfoo-mp',
+ u'board':
+ {u'public_codename': u'notfoo-board', u'is_active': True},
+ u'chrome_os_version': u'7478.0.0', u'channel': u'notfoo-channel',
+ u'payloads':
+ [{u'max_fraction': False,
+ u'name': u'notfoo-channel/notfoo/7478.0.0/payloads/'
+ u'chromeos_7475.0.0-7478.0.0_notfoo_notfoo'
+ u'-channel_delta_mp.bin-877f148a914c1cdbe2'
+ u'42aa4247a1d135.signed', u'fraction': 1.0},
+ {u'max_fraction': False,
+ u'name': u'notfoo-channel/notfoo/7478.0.0/payloads/'
+ u'chromeos_7478.0.0_notfoo_notfoo-channel_'
+ u'full_mp.bin-fddc0ae18c9845325c13704ee00b'
+ u'd0a4.signed', u'fraction': 1.0}]}]
+ }
+ )
- paygen_build_lib.query.FindLatestPublished(
- 'foo-channel', 'foo-board').AndReturn(None)
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.OMAHA_URI).AndReturn(mock_return_foo)
+ gslib.Cat(paygen_build_lib.OMAHA_URI).AndReturn(mock_return_not_foo)
# Run the test verification.
self.mox.ReplayAll()
@@ -400,7 +447,7 @@
[gspaths.Build(bucket='crt',
channel='foo-channel',
board='foo-board',
- version='1.0.0')])
+ version='7478.0.0')])
self.assertEqual(paygen._DiscoverNmoBuild(), [])
@@ -1244,8 +1291,6 @@
self.assertTrue(result.startswith(
os.path.join(self.tempdir, 'paygen_build-control_files')))
- @unittest.skipIf(not paygen_build_lib.config,
- 'Internal crostools repository needed.')
@unittest.skipIf(not paygen_build_lib.test_control,
'Autotest repository needed.')
def testEmitControlFile(self):
@@ -1527,11 +1572,25 @@
def testValidateBoardConfig(self):
"""Test ValidateBoardConfig."""
+ mock_return = paygen_build_lib.json.dumps(
+ {'boards':
+ [{u'omaha_config_name': u'autoupdate-ascii-memento.config',
+ u'public_codename': u'x86-mario',
+ u'hwid_match': u'IEC MARIO FISH 2330|IEC MARIO FISH 2330 DEV|'
+ u'IEC MARIO PONY 6101|IEC MARIO PONY DVT 8784|'
+ u'IEC MARIO PONY EVT 3495|IEC MARIO PONY TEST 6101',
+ u'is_active': True,
+ u'app_id': u'{87efface-864d-49a5-9bb3-4b050a7c227a}',
+ u'config_name': u'cr48', u'is_test_blacklist': False,
+ u'omaha_ping_hwid_match': u'IEC MARIO FISH 2330 DEV',
+ u'is_in_canary_release': True}]
+ }
+ )
- # If we are running on an external builder, we can't see the config.
- # Without the config, we can't validate.
- if not paygen_build_lib.config:
- return
+ self.mox.StubOutWithMock(gslib, 'Cat')
+ gslib.Cat(paygen_build_lib.BOARDS_URI).AndReturn(mock_return)
+ gslib.Cat(paygen_build_lib.BOARDS_URI).AndReturn(mock_return)
+ self.mox.ReplayAll()
# Test a known board works.
paygen_build_lib.ValidateBoardConfig('x86-mario')
