scripts/cros_build_packages.py - mirrors/cros/chromiumos/chromite - Git at Google

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 import optparse
 import os
 import multiprocessing
 import tempfile
 import sys


 def main(argv):
   """Build packages according to options specified on command-line."""

   if os.getuid() != 0:
     cros_build_lib.Die("superuser access required")

   scripts_dir = os.path.abspath(__file__ + "/../../..")
   builder = PackageBuilder(scripts_dir)
   options, _ = builder.ParseArgs(argv)

   # Calculate packages to install.
   # TODO(davidjames): Grab these from a spec file.
   packages = ["chromeos-base/chromeos"]
   if options.withdev:
     packages.append("chromeos-base/chromeos-dev")
   if options.withfactory:
     packages.append("chromeos-base/chromeos-factoryinstall")
   if options.withtest:
     packages.append("chromeos-base/chromeos-test")

   if options.usetarball:
     builder.ExtractTarball(options, packages)
   else:
     builder.BuildTarball(options, packages)


 def _Apply(args):
   """Call the function specified in args[0], with arguments in args[1:]."""
   return apply(args[0], args[1:])


 def _GetLatestPrebuiltPrefix(board):
   """Get the latest prebuilt prefix for the specified board.

   Args:
     board: The board you want prebuilts for.
   Returns:
     Latest prebuilt prefix.
   """
   # TODO(davidjames): Also append profile names here.
   prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
   tmpfile = tempfile.NamedTemporaryFile()
   _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
   tmpfile.seek(0)
   latest = tmpfile.read().strip()
   tmpfile.close()
   return "%s/%s" % (prefix, latest)


 def _GetPrebuiltDownloadCommands(prefix):
   """Return a list of commands for grabbing packages.

   There must be a file called "packages/Packages" that contains the list of
   packages. The specified list of commands will fill the packages directory
   with the bzipped packages from the specified prefix.

   Args:
     prefix: Url prefix to download packages from.
   Returns:
     List of commands for grabbing packages.
   """

   cmds = []
   for line in file("packages/Packages"):
     if line.startswith("CPV: "):
       pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
       path = "%s/%s.tbz2" % (pkgpath, pkgname)
       url = "%s/%s" % (prefix, path)
       dirname = "packages/%s" % pkgpath
       fullpath = "packages/%s" % path
       if not os.path.exists(dirname):
         os.makedirs(dirname)
       if not os.path.exists(fullpath):
         cmds.append("curl -s %s -o %s" % (url, fullpath))
   return cmds


 def _Run(cmd, retries=0):
   """Run the specified command.

   If the command fails, and the retries have been exhausted, the program exits
   with an appropriate error message.

   Args:
     cmd: The command to run.
     retries: If exit code is non-zero, retry this many times.
   """
   try:
     result = cros_build_lib.RunCommandWithRetries(retries, cmd, shell=True)
   except cros_build_lib.RunCommandException:
     cros_build_lib.Die("Command failed, exiting: %s" % cmd)
   cros_build_lib.Info("Command succeeded: %s" % cmd)


 def _RunManyParallel(cmds, retries=0):
   """Run list of provided commands in parallel.

   To work around a bug in the multiprocessing module, we use map_async instead
   of the usual map function. See http://bugs.python.org/issue9205

   Args:
     cmds: List of commands to run.
     retries: Number of retries per command.
   """
   # TODO(davidjames): Move this to common library.
   pool = multiprocessing.Pool()
   args = []
   for cmd in cmds:
     args.append((_Run, cmd, retries))
   result = pool.map_async(_Apply, args, chunksize=1)
   while True:
     try:
       result.get(60*60)
       break
     except multiprocessing.TimeoutError:
       pass


 class PackageBuilder(object):
   """A class for building and extracting tarballs of Chromium OS packages."""

   def __init__(self, scripts_dir):
     self.scripts_dir = scripts_dir

   def BuildTarball(self, options, packages):
     """Build a tarball with the specified packages.

     Args:
       options: Options object, as output by ParseArgs.
       packages: List of packages to build.
     """

     board = options.board

     # Run setup_board. TODO(davidjames): Integrate the logic used in
     # setup_board into chromite.
     _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))

     # Create complete build directory
     _Run(self._EmergeBoardCmd(options, packages))

     # Archive build directory as tarballs
     os.chdir("/build/%s" % board)
     cmds = [
         "tar -c --wildcards --exclude='usr/lib/debug/*' "
         "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board,
         "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board
     ]

     # Run list of commands.
     _RunManyParallel(cmds)

   def ExtractTarball(self, options, packages):
     """Extract the latest build tarball, then update the specified packages.

     Args:
       options: Options object, as output by ParseArgs.
       packages: List of packages to update.
     """

     board = options.board
     prefix = _GetLatestPrebuiltPrefix(board)

     # If the user doesn't have emerge-${BOARD} setup yet, we need to run
     # setup_board. TODO(davidjames): Integrate the logic used in setup_board
     # into chromite.
     if not os.path.exists("/usr/local/bin/emerge-%s" % board):
       _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))

     # Delete old build directory. This process might take a while, so do it in
     # the background.
     cmds = []
     if os.path.exists("/build/%s" % board):
       tempdir = tempfile.mkdtemp()
       _Run("mv /build/%s %s" % (board, tempdir))
       cmds.append("rm -rf %s" % tempdir)

     # Create empty build directory, and chdir into it.
     os.makedirs("/build/%s/packages" % board)
     os.chdir("/build/%s" % board)

     # Download and expand build tarball.
     build_url = "%s/%s-build.tgz" % (prefix, board)
     cmds.append("curl -s %s | tar -xz" % build_url)

     # Download and expand debug tarball (if requested).
     if options.debug:
       debug_url = "%s/%s-debug.tgz" % (prefix, board)
       cmds.append("curl -s %s | tar -xz" % debug_url)

     # Download prebuilt packages.
     _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
     cmds.extend(_GetPrebuiltDownloadCommands(prefix))

     # Run list of commands, with three retries per command, in case the network
     # is flaky.
     _RunManyParallel(cmds, retries=3)

     # Emerge remaining packages.
     _Run(self._EmergeBoardCmd(options, packages))

   def ParseArgs(self, argv):
     """Parse arguments from the command line using optparse."""

     # TODO(davidjames): We should use spec files for this.
     default_board = self._GetDefaultBoard()
     parser = optparse.OptionParser()
     parser.add_option("--board", dest="board", default=default_board,
                       help="The board to build packages for.")
     parser.add_option("--debug", action="store_true", dest="debug",
                       default=False, help="Include debug symbols.")
     parser.add_option("--nowithdev", action="store_false", dest="withdev",
                       default=True,
                       help="Don't build useful developer friendly utilities.")
     parser.add_option("--nowithtest", action="store_false", dest="withtest",
                       default=True, help="Build packages required for testing.")
     parser.add_option("--nowithfactory", action="store_false",
                       dest="withfactory", default=True,
                       help="Build factory installer")
     parser.add_option("--nousepkg", action="store_false",
                       dest="usepkg", default=True,
                       help="Don't use binary packages.")
     parser.add_option("--nousetarball", action="store_false",
                       dest="usetarball", default=True,
                       help="Don't use tarball.")
     parser.add_option("--nofast", action="store_false", dest="fast",
                       default=True,
                       help="Don't merge packages in parallel.")

     return parser.parse_args(argv)

   def _EmergeBoardCmd(self, options, packages):
     """Calculate board emerge command."""
     board = options.board
     scripts_dir = self.scripts_dir
     emerge_board = "emerge-%s" % board
     if options.fast:
       emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
     usepkg = ""
     if options.usepkg:
       usepkg = "g"
     return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))

   def _GetDefaultBoard(self):
     """Get the default board configured by the user."""

     default_board_file = "%s/.default_board" % self.scripts_dir
     default_board = None
     if os.path.exists(default_board_file):
       default_board = file(default_board_file).read().strip()
     return default_board
	#!/usr/bin/python
	# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	import optparse
	import os
	import multiprocessing
	import tempfile
	import sys


	def main(argv):
	"""Build packages according to options specified on command-line."""

	if os.getuid() != 0:
	cros_build_lib.Die("superuser access required")

	scripts_dir = os.path.abspath(__file__ + "/../../..")
	builder = PackageBuilder(scripts_dir)
	options, _ = builder.ParseArgs(argv)

	# Calculate packages to install.
	# TODO(davidjames): Grab these from a spec file.
	packages = ["chromeos-base/chromeos"]
	if options.withdev:
	packages.append("chromeos-base/chromeos-dev")
	if options.withfactory:
	packages.append("chromeos-base/chromeos-factoryinstall")
	if options.withtest:
	packages.append("chromeos-base/chromeos-test")

	if options.usetarball:
	builder.ExtractTarball(options, packages)
	else:
	builder.BuildTarball(options, packages)


	def _Apply(args):
	"""Call the function specified in args[0], with arguments in args[1:]."""
	return apply(args[0], args[1:])


	def _GetLatestPrebuiltPrefix(board):
	"""Get the latest prebuilt prefix for the specified board.

	Args:
	board: The board you want prebuilts for.
	Returns:
	Latest prebuilt prefix.
	"""
	# TODO(davidjames): Also append profile names here.
	prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
	tmpfile = tempfile.NamedTemporaryFile()
	_Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
	tmpfile.seek(0)
	latest = tmpfile.read().strip()
	tmpfile.close()
	return "%s/%s" % (prefix, latest)


	def _GetPrebuiltDownloadCommands(prefix):
	"""Return a list of commands for grabbing packages.

	There must be a file called "packages/Packages" that contains the list of
	packages. The specified list of commands will fill the packages directory
	with the bzipped packages from the specified prefix.

	Args:
	prefix: Url prefix to download packages from.
	Returns:
	List of commands for grabbing packages.
	"""

	cmds = []
	for line in file("packages/Packages"):
	if line.startswith("CPV: "):
	pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
	path = "%s/%s.tbz2" % (pkgpath, pkgname)
	url = "%s/%s" % (prefix, path)
	dirname = "packages/%s" % pkgpath
	fullpath = "packages/%s" % path
	if not os.path.exists(dirname):
	os.makedirs(dirname)
	if not os.path.exists(fullpath):
	cmds.append("curl -s %s -o %s" % (url, fullpath))
	return cmds


	def _Run(cmd, retries=0):
	"""Run the specified command.

	If the command fails, and the retries have been exhausted, the program exits
	with an appropriate error message.

	Args:
	cmd: The command to run.
	retries: If exit code is non-zero, retry this many times.
	"""
	try:
	result = cros_build_lib.RunCommandWithRetries(retries, cmd, shell=True)
	except cros_build_lib.RunCommandException:
	cros_build_lib.Die("Command failed, exiting: %s" % cmd)
	cros_build_lib.Info("Command succeeded: %s" % cmd)


	def _RunManyParallel(cmds, retries=0):
	"""Run list of provided commands in parallel.

	To work around a bug in the multiprocessing module, we use map_async instead
	of the usual map function. See http://bugs.python.org/issue9205

	Args:
	cmds: List of commands to run.
	retries: Number of retries per command.
	"""
	# TODO(davidjames): Move this to common library.
	pool = multiprocessing.Pool()
	args = []
	for cmd in cmds:
	args.append((_Run, cmd, retries))
	result = pool.map_async(_Apply, args, chunksize=1)
	while True:
	try:
	result.get(60*60)
	break
	except multiprocessing.TimeoutError:
	pass


	class PackageBuilder(object):
	"""A class for building and extracting tarballs of Chromium OS packages."""

	def __init__(self, scripts_dir):
	self.scripts_dir = scripts_dir

	def BuildTarball(self, options, packages):
	"""Build a tarball with the specified packages.

	Args:
	options: Options object, as output by ParseArgs.
	packages: List of packages to build.
	"""

	board = options.board

	# Run setup_board. TODO(davidjames): Integrate the logic used in
	# setup_board into chromite.
	_Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))

	# Create complete build directory
	_Run(self._EmergeBoardCmd(options, packages))

	# Archive build directory as tarballs
	os.chdir("/build/%s" % board)
	cmds = [
	"tar -c --wildcards --exclude='usr/lib/debug/*' "
	"--exclude='packages/' \| pigz -c > packages/%s-build.tgz" % board,
	"tar -c usr/lib/debug/* \| pigz -c > packages/%s-debug.tgz" % board
	]

	# Run list of commands.
	_RunManyParallel(cmds)

	def ExtractTarball(self, options, packages):
	"""Extract the latest build tarball, then update the specified packages.

	Args:
	options: Options object, as output by ParseArgs.
	packages: List of packages to update.
	"""

	board = options.board
	prefix = _GetLatestPrebuiltPrefix(board)

	# If the user doesn't have emerge-${BOARD} setup yet, we need to run
	# setup_board. TODO(davidjames): Integrate the logic used in setup_board
	# into chromite.
	if not os.path.exists("/usr/local/bin/emerge-%s" % board):
	_Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))

	# Delete old build directory. This process might take a while, so do it in
	# the background.
	cmds = []
	if os.path.exists("/build/%s" % board):
	tempdir = tempfile.mkdtemp()
	_Run("mv /build/%s %s" % (board, tempdir))
	cmds.append("rm -rf %s" % tempdir)

	# Create empty build directory, and chdir into it.
	os.makedirs("/build/%s/packages" % board)
	os.chdir("/build/%s" % board)

	# Download and expand build tarball.
	build_url = "%s/%s-build.tgz" % (prefix, board)
	cmds.append("curl -s %s \| tar -xz" % build_url)

	# Download and expand debug tarball (if requested).
	if options.debug:
	debug_url = "%s/%s-debug.tgz" % (prefix, board)
	cmds.append("curl -s %s \| tar -xz" % debug_url)

	# Download prebuilt packages.
	_Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
	cmds.extend(_GetPrebuiltDownloadCommands(prefix))

	# Run list of commands, with three retries per command, in case the network
	# is flaky.
	_RunManyParallel(cmds, retries=3)

	# Emerge remaining packages.
	_Run(self._EmergeBoardCmd(options, packages))

	def ParseArgs(self, argv):
	"""Parse arguments from the command line using optparse."""

	# TODO(davidjames): We should use spec files for this.
	default_board = self._GetDefaultBoard()
	parser = optparse.OptionParser()
	parser.add_option("--board", dest="board", default=default_board,
	help="The board to build packages for.")
	parser.add_option("--debug", action="store_true", dest="debug",
	default=False, help="Include debug symbols.")
	parser.add_option("--nowithdev", action="store_false", dest="withdev",
	default=True,
	help="Don't build useful developer friendly utilities.")
	parser.add_option("--nowithtest", action="store_false", dest="withtest",
	default=True, help="Build packages required for testing.")
	parser.add_option("--nowithfactory", action="store_false",
	dest="withfactory", default=True,
	help="Build factory installer")
	parser.add_option("--nousepkg", action="store_false",
	dest="usepkg", default=True,
	help="Don't use binary packages.")
	parser.add_option("--nousetarball", action="store_false",
	dest="usetarball", default=True,
	help="Don't use tarball.")
	parser.add_option("--nofast", action="store_false", dest="fast",
	default=True,
	help="Don't merge packages in parallel.")

	return parser.parse_args(argv)

	def _EmergeBoardCmd(self, options, packages):
	"""Calculate board emerge command."""
	board = options.board
	scripts_dir = self.scripts_dir
	emerge_board = "emerge-%s" % board
	if options.fast:
	emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
	usepkg = ""
	if options.usepkg:
	usepkg = "g"
	return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))

	def _GetDefaultBoard(self):
	"""Get the default board configured by the user."""

	default_board_file = "%s/.default_board" % self.scripts_dir
	default_board = None
	if os.path.exists(default_board_file):
	default_board = file(default_board_file).read().strip()
	return default_board