blob: 81a2b3a7c39e8bca10df615f533f4dcd498db0b0 [file] [log] [blame]
#!/usr/bin/python
"""Script to bootstrap the chroot using new toolchain.
This script allows you to build/install a customized version of gcc/binutils,
either by specifying branch or a local directory.
This script must be executed outside chroot.
Below is some typical usage -
## Build gcc located at /local/gcc/dir and do a bootstrap using the new compiler
## for the chromeos root. The script tries to find a valid chromeos tree all
## the way up from your current working directory.
./build_tool.py --gcc_dir=/loca/gcc/dir --bootstrap
## Build binutils, using remote branch "mobile_toolchain_v17" and do a bootstrap
## using the new binutils for the chromeos root. The script tries to find a
## valid chromeos tree all the way up from your current working directory.
./build_tool.py --binutils_branch=cros/mobile_toolchain_v17 \
--chromeos_root=/chromeos/dir --bootstrap
## Same as above except only do it for board daisy - no bootstrapping involved.
./build_tool.py --binutils_branch=cros/mobile_toolchain_v16 \
--chromeos_root=/chromeos/dir --board=daisy
"""
__author__ = 'shenhan@google.com (Han Shen)'
import optparse
import os
import re
import sys
import repo_to_repo
from utils import command_executer
from utils import logger
from utils import misc
REPO_PATH_PATTERN = 'src/third_party/{0}'
TEMP_BRANCH_NAME = 'internal_testing_branch_no_use'
CHROMIUMOS_OVERLAY_PATH = 'src/third_party/chromiumos-overlay'
EBUILD_PATH_PATTERN = 'src/third_party/chromiumos-overlay/sys-devel/{0}'
class Bootstrapper(object):
"""Class that handles bootstrap process.
"""
def __init__(self, chromeos_root, gcc_branch=None, gcc_dir=None,
binutils_branch=None, binutils_dir=None,
board=None, setup_tool_ebuild_file_only=False):
self._chromeos_root = chromeos_root
self._gcc_branch = gcc_branch
self._gcc_branch_tree = None
self._gcc_dir = gcc_dir
self._gcc_ebuild_file = None
self._gcc_ebuild_file_name = None
self._binutils_branch = binutils_branch
self._binutils_branch_tree = None
self._binutils_dir = binutils_dir
self._binutils_ebuild_file = None
self._binutils_ebuild_file_name = None
self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only
self._ce = command_executer.GetCommandExecuter()
self._logger = logger.GetLogger()
self._board = board
def IsTreeSame(self, t1, t2):
diff = 'diff -qr -x .git -x .svn "{0}" "{1}"'.format(t1, t2)
if self._ce.RunCommand(diff, print_to_console=False) == 0:
self._logger.LogOutput('"{0}" and "{1}" are the same."'.format(t1, t2))
return True
self._logger.LogWarning('"{0}" and "{1}" are different."'.format(t1, t2))
return False
def SubmitToLocalBranch(self):
"""Copy source code to the chromium source tree and submit it locally."""
if self._gcc_dir:
if not self.SubmitToolToLocalBranch(
tool_name='gcc', tool_dir=self._gcc_dir):
return False
self._gcc_branch = TEMP_BRANCH_NAME
if self._binutils_dir:
if not self.SubmitToolToLocalBranch(
tool_name='binutils', tool_dir=self._binutils_dir):
return False
self._binutils_branch = TEMP_BRANCH_NAME
return True
def SubmitToolToLocalBranch(self, tool_name, tool_dir):
"""Copy the source code to local chromium source tree.
Args:
tool_name: either 'gcc' or 'binutils'
tool_dir: the tool source dir to be used
Returns:
True if all succeeded False otherwise.
"""
# The next few steps creates an internal branch to sync with the tool dir
# user provided.
chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
# 0. Test to see if git tree is free of local changes.
if not misc.IsGitTreeClean(chrome_tool_dir):
self._logger.LogError(
'Git repository "{0}" not clean, aborted.'.format(chrome_tool_dir))
return False
# 1. Checkout/create a (new) branch for testing.
command = 'cd "{0}" && git checkout -B {1}'.format(
chrome_tool_dir, TEMP_BRANCH_NAME)
ret = self._ce.RunCommand(command)
if ret:
self._logger.LogError('Failed to create a temp branch for test, aborted.')
return False
if self.IsTreeSame(tool_dir, chrome_tool_dir):
self._logger.LogOutput(
'"{0}" and "{1}" are the same, sync skipped.'.format(
tool_dir, chrome_tool_dir))
return True
# 2. Sync sources from user provided tool dir to chromiumos tool git.
local_tool_repo = repo_to_repo.FileRepo(tool_dir)
chrome_tool_repo = repo_to_repo.GitRepo(chrome_tool_dir, TEMP_BRANCH_NAME)
chrome_tool_repo._root_dir = chrome_tool_dir
# Delete all stuff before start mapping.
self._ce.RunCommand('cd {0} && rm -rf *'.format(chrome_tool_dir))
local_tool_repo.MapSources(chrome_tool_repo.GetRoot())
# 3. Ensure after sync tree is the same.
if self.IsTreeSame(tool_dir, chrome_tool_dir):
self._logger.LogOutput('Sync successfully done.')
else:
self._logger.LogError('Sync not successful, aborted.')
return False
# 4. Commit all changes.
ret = chrome_tool_repo.CommitLocally(
'Synced with tool source tree at - "{0}".'.format(tool_dir))
if ret:
self._logger.LogError('Commit to local branch "{0}" failed, aborted.'.
format(TEMP_BRANCH_NAME))
return False
return True
def CheckoutBranch(self):
"""Checkout working branch for the tools.
Returns:
True: if operation succeeds.
"""
if self._gcc_branch:
rv = self.CheckoutToolBranch('gcc', self._gcc_branch)
if rv:
self._gcc_branch_tree = rv
else:
return False
if self._binutils_branch:
rv = self.CheckoutToolBranch('binutils', self._binutils_branch)
if rv:
self._binutils_branch_tree = rv
else:
return False
return True
def CheckoutToolBranch(self, tool_name, tool_branch):
"""Checkout the tool branch for a certain tool.
Args:
tool_name: either 'gcc' or 'binutils'
tool_branch: tool branch to use
Returns:
True: if operation succeeds. Otherwise False.
"""
chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
command = 'cd "{0}" && git checkout {1}'.format(
chrome_tool_dir, tool_branch)
if not self._ce.RunCommand(command, print_to_console=True):
# Get 'TREE' value of this commit
command = ('cd "{0}" && git cat-file -p {1} '
'| grep -E "^tree [a-f0-9]+$" '
'| cut -d" " -f2').format(chrome_tool_dir, tool_branch)
ret, stdout, _ = self._ce.RunCommand(
command, return_output=True, print_to_console=False)
# Pipe operation always has a zero return value. So need to check if
# stdout is valid.
if not ret and stdout and re.match(
'[0-9a-h]{40}', stdout.strip(), re.IGNORECASE):
tool_branch_tree = stdout.strip()
self._logger.LogOutput('Find tree for {0} branch "{1}" - "{2}"'.format(
tool_name, tool_branch, tool_branch_tree))
return tool_branch_tree
self._logger.LogError(('Failed to checkout "{0}" or failed to '
'get tree value, aborted.').format(tool_branch))
return None
def FindEbuildFile(self):
"""Find the ebuild files for the tools.
Returns:
True: if operation succeeds.
"""
if self._gcc_branch:
(rv, ef, efn) = self.FindToolEbuildFile('gcc')
if rv:
self._gcc_ebuild_file = ef
self._gcc_ebuild_file_name = efn
else:
return False
if self._binutils_branch:
(rv, ef, efn) = self.FindToolEbuildFile('binutils')
if rv:
self._binutils_ebuild_file = ef
self._binutils_ebuild_file_name = efn
else:
return False
return True
def FindToolEbuildFile(self, tool_name):
"""Find ebuild file for a specific tool.
Args:
tool_name: either "gcc" or "binutils".
Returns:
A triplet that consisits of whether operation succeeds or not,
tool ebuild file full path and tool ebuild file name.
"""
# To get the active gcc ebuild file, we need a workable chroot first.
if not os.path.exists(
os.path.join(self._chromeos_root, 'chroot')) and self._ce.RunCommand(
'cd "{0}" && cros_sdk --create'.format(self._chromeos_root)):
self._logger.LogError(('Failed to install a initial chroot, aborted.\n'
'If previous bootstrap failed, do a '
'"cros_sdk --delete" to remove '
'in-complete chroot.'))
return (False, None, None)
rv, stdout, _ = self._ce.ChrootRunCommand(
self._chromeos_root, 'equery w sys-devel/{0}'.format(tool_name),
return_output=True, print_to_console=True)
if rv:
self._logger.LogError(
('Failed to execute inside chroot '
'"equery w sys-devel/{0}", aborted.').format(tool_name))
return (False, None, None)
m = re.match(r'^.*/({0}/(.*\.ebuild))$'.format(
EBUILD_PATH_PATTERN.format(tool_name)), stdout)
if not m:
self._logger.LogError(
('Failed to find {0} ebuild file, aborted. '
'If previous bootstrap failed, do a "cros_sdk --delete" to remove '
'in-complete chroot.').format(tool_name))
return (False, None, None)
tool_ebuild_file = os.path.join(self._chromeos_root, m.group(1))
tool_ebuild_file_name = m.group(2)
return (True, tool_ebuild_file, tool_ebuild_file_name)
def InplaceModifyEbuildFile(self):
"""Modify the ebuild file.
Returns:
True if operation succeeds.
"""
# Note we shall not use remote branch name (eg. "cros/gcc.gnu.org/...") in
# CROS_WORKON_COMMIT, we have to use GITHASH. So we call GitGetCommitHash on
# tool_branch.
if self._gcc_branch:
tool_branch_githash = misc.GitGetCommitHash(
self.GetChromeOsToolDir('gcc'), self._gcc_branch)
if not tool_branch_githash:
return False
if not self.InplaceModifyToolEbuildFile(
tool_branch_githash, self._gcc_branch_tree, self._gcc_ebuild_file):
return False
if self._binutils_branch:
tool_branch_githash = misc.GitGetCommitHash(
self.GetChromeOsToolDir('binutils'), self._binutils_branch)
if not self.InplaceModifyToolEbuildFile(
tool_branch_githash, self._binutils_branch_tree,
self._binutils_ebuild_file):
return False
return True
@staticmethod
def ResetToolEbuildFile(chromeos_root, tool_name):
"""Reset tool ebuild file to clean state.
Args:
chromeos_root: chromeos source tree
tool_name: either "gcc" or "binutils"
Returns:
True if operation succeds.
"""
rv = misc.GetGitChangesAsList(
os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH),
path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)),
staged=False)
if rv:
cmd = 'cd {0} && git checkout --'.format(os.path.join(
chromeos_root, CHROMIUMOS_OVERLAY_PATH))
for g in rv:
cmd += ' ' + g
rv = command_executer.GetCommandExecuter().RunCommand(cmd)
if rv:
logger.GetLogger().LogWarning(
'Failed to reset the ebuild file. Please refer to log above.')
return False
else:
logger.GetLogger().LogWarning(
'Note - did not find any modified {0} ebuild file.'.format(tool_name))
# Fall through
return True
def GetChromeOsToolDir(self, tool_name):
"""Return the chromeos git dir for a specific tool.
Args:
tool_name: either 'gcc' or 'binutils'.
Returns:
Absolute git path for the tool.
"""
return os.path.join(
self._chromeos_root, REPO_PATH_PATTERN.format(tool_name))
def InplaceModifyToolEbuildFile(
self, tool_branch_githash, tool_branch_tree, tool_ebuild_file):
"""Using sed to fill properly values into the ebuild file.
Args:
tool_branch_githash: githash for tool_branch
tool_branch_tree: treeish for the tool branch
tool_ebuild_file: tool ebuild file
Returns:
True: if operation succeeded.
"""
command = ('sed -i '
'-e \'/^CROS_WORKON_COMMIT=".*"/i'
' # The following line is modified by script.\' '
'-e \'s!^CROS_WORKON_COMMIT=".*"$!CROS_WORKON_COMMIT="{0}"!\' '
'-e \'/^CROS_WORKON_TREE=".*"/i'
' # The following line is modified by script.\' '
'-e \'s!^CROS_WORKON_TREE=".*"$!CROS_WORKON_TREE="{1}"!\' '
'{2}').format(tool_branch_githash,
tool_branch_tree,
tool_ebuild_file)
rv = self._ce.RunCommand(command)
if rv:
self._logger.LogError(
'Failed to modify commit and tree value for "{0}"", aborted.'.format(
tool_ebuild_file))
return False
# Warn that the ebuild file has been modified.
self._logger.LogWarning(
('Ebuild file "{0}" is modified, to revert the file - \n'
'bootstrap_compiler.py --chromeos_root={1} '
'--reset_tool_ebuild_file').format(
tool_ebuild_file, self._chromeos_root))
return True
def DoBuildForBoard(self):
"""Build tool for a specific board.
Returns:
True if operation succeeds.
"""
if self._gcc_branch:
if not self.DoBuildToolForBoard('gcc'):
return False
if self._binutils_branch:
if not self.DoBuildToolForBoard('binutils'):
return False
return True
def DoBuildToolForBoard(self, tool_name):
"""Build a specific tool for a specific board.
Args:
tool_name: either "gcc" or "binutils"
Returns:
True if operation succeeds.
"""
boards_to_build = self._board.split(',')
failed = []
for board in boards_to_build:
if board == 'host':
command = 'sudo emerge sys-devel/{0}'.format(tool_name)
else:
target = misc.GetCtargetFromBoard(board, self._chromeos_root)
if not target:
self._logger.LogError(
'Unsupported board "{0}", skip.'.format(board))
failed.append(board)
continue
command = 'sudo emerge cross-{0}/{1}'.format(target, tool_name)
rv = self._ce.ChrootRunCommand(self._chromeos_root, command,
return_output=False, print_to_console=True)
if rv:
self._logger.LogError(
'Build "{0}" failed for "{1}", aborted.'.format(tool_name, board))
failed.append(board)
else:
self._logger.LogOutput(
'Successfully built "{0}" for board "{1}".'.format(tool_name, board))
if failed:
self._logger.LogError(
'Failed to build {0} for the following board(s): "{1}"'.format(
tool_name, ' '.join(failed)))
return False
# All boards build successfully
return True
def DoBootstrapping(self):
"""Do bootstrapping the chroot.
Returns:
True if operation succeeds.
"""
logfile = os.path.join(self._chromeos_root, 'bootstrap.log')
command = 'cd "{0}" && cros_sdk --delete --bootstrap |& tee "{1}"'.format(
self._chromeos_root, logfile)
rv = self._ce.RunCommand(command, return_output=False,
print_to_console=True)
if rv:
self._logger.LogError('Bootstrapping failed, log file - "{0}"\n'.format(
logfile))
return False
## Workaround for - crbug/331713.
## We do not test for success, failure is not important at this step.
self._ce.ChrootRunCommand(
self._chromeos_root, 'sudo emerge dev-util/pkgconfig',
return_output=False, print_to_console=True)
self._logger.LogOutput('Bootstrap succeeded.')
return True
def Do(self):
"""Entrance of the class.
Returns:
True if everything is ok.
"""
if (self.SubmitToLocalBranch() and
self.CheckoutBranch() and
self.FindEbuildFile() and
self.InplaceModifyEbuildFile()):
if self._setup_tool_ebuild_file_only:
# Everything is done, we are good.
ret = True
else:
if self._board:
ret = self.DoBuildForBoard()
else:
# This implies '--bootstrap'.
ret = self.DoBootstrapping()
else:
ret = False
return ret
def Main(argv):
parser = optparse.OptionParser()
parser.add_option('-c', '--chromeos_root', dest='chromeos_root',
help=('Optional. ChromeOs root dir. '
'When not specified, chromeos root will be deduced '
'from current working directory.'))
parser.add_option('--gcc_branch', dest='gcc_branch',
help=('The branch to test against. '
'This branch must be a local branch '
'inside "src/third_party/gcc". '
'Notice, this must not be used with "--gcc_dir".'))
parser.add_option('--binutils_branch', dest='binutils_branch',
help=('The branch to test against binutils. '
'This branch must be a local branch '
'inside "src/third_party/binutils". '
'Notice, this must not be used with '
'"--binutils_dir".'))
parser.add_option('-g', '--gcc_dir', dest='gcc_dir',
help=('Use a local gcc tree to do bootstrapping. '
'Notice, this must not be used with "--gcc_branch".'))
parser.add_option('--binutils_dir', dest='binutils_dir',
help=('Use a local binutils tree to do bootstrapping. '
'Notice, this must not be used with '
'"--binutils_branch".'))
parser.add_option('--fixperm', dest='fixperm',
default=False, action='store_true',
help=('Fix the (notorious) permission error '
'while trying to bootstrap the chroot. '
'Note this takes an extra 10-15 minutes '
'and is only needed once per chromiumos tree.'))
parser.add_option('--setup_tool_ebuild_file_only',
dest='setup_tool_ebuild_file_only',
default=False, action='store_true',
help=('Setup gcc and/or binutils ebuild file '
'to pick up the branch (--gcc/binutils_branch) or '
'use gcc and/or binutils source (--gcc/binutils_dir) '
'and exit. Keep chroot as is. This should not be '
'used with --gcc/binutils_dir/branch options.'))
parser.add_option('--reset_tool_ebuild_file', dest='reset_tool_ebuild_file',
default=False, action='store_true',
help=('Reset the modification that is done by this script.'
'Note, when this script is running, it will modify '
'the active gcc/binutils ebuild file. Use this '
'option to reset (what this script has done) '
'and exit. This should not be used with -- '
'gcc/binutils_dir/branch options.'))
parser.add_option('--board', dest='board', default=None,
help=('Only build toolchain for specific board(s). '
'Use "host" to build for host. '
'Use "," to seperate multiple boards. '
'This does not perform a chroot bootstrap.'))
parser.add_option('--bootstrap', dest='bootstrap',
default=False, action='store_true',
help=('Performs a chroot bootstrap. '
'Note, this will *destroy* your current chroot.'))
options = parser.parse_args(argv)[0]
# Trying to deduce chromeos root from current directory.
if not options.chromeos_root:
logger.GetLogger().LogOutput('Trying to deduce chromeos root ...')
wdir = os.getcwd()
while wdir and wdir != '/':
if misc.IsChromeOsTree(wdir):
logger.GetLogger().LogOutput('Find chromeos_root: {}'.format(wdir))
options.chromeos_root = wdir
break
wdir = os.path.dirname(wdir)
if not options.chromeos_root:
parser.error('Missing or failing to deduce mandatory option "--chromeos".')
return 1
options.chromeos_root = os.path.abspath(
os.path.expanduser(options.chromeos_root))
if not os.path.isdir(options.chromeos_root):
logger.GetLogger().LogError(
'"{0}" does not exist.'.format(options.chromeos_root))
return 1
if options.fixperm:
# Fix perm error before continuing.
cmd = (r'sudo find "{0}" \( -name ".cache" -type d -prune \) -o '
r'\( -name "chroot" -type d -prune \) -o '
r'\( -type f -exec chmod a+r {{}} \; \) -o '
r'\( -type d -exec chmod a+rx {{}} \; \)').format(
options.chromeos_root)
logger.GetLogger().LogOutput(
'Fixing perm issues for chromeos root, this might take some time.')
command_executer.GetCommandExecuter().RunCommand(cmd)
if options.reset_tool_ebuild_file:
if (options.gcc_dir or options.gcc_branch or
options.binutils_dir or options.binutils_branch):
logger.GetLogger().LogWarning(
'Ignoring any "--gcc/binutils_dir" and/or "--gcc/binutils_branch".')
if options.setup_tool_ebuild_file_only:
logger.GetLogger().LogError(
('Conflict options "--reset_tool_ebuild_file" '
'and "--setup_tool_ebuild_file_only".'))
return 1
rv = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'gcc')
rv1 = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'binutils')
return 0 if (rv and rv1) else 1
if options.gcc_dir:
options.gcc_dir = os.path.abspath(os.path.expanduser(options.gcc_dir))
if not os.path.isdir(options.gcc_dir):
logger.GetLogger().LogError(
'"{0}" does not exist.'.format(options.gcc_dir))
return 1
if options.gcc_branch and options.gcc_dir:
parser.error('Only one of "--gcc_dir" and "--gcc_branch" can be specified.')
return 1
if options.binutils_dir:
options.binutils_dir = os.path.abspath(
os.path.expanduser(options.binutils_dir))
if not os.path.isdir(options.binutils_dir):
logger.GetLogger().LogError(
'"{0}" does not exist.'.format(options.binutils_dir))
return 1
if options.binutils_branch and options.binutils_dir:
parser.error('Only one of "--binutils_dir" and '
'"--binutils_branch" can be specified.')
return 1
if (not (options.binutils_branch or options.binutils_dir or
options.gcc_branch or options.gcc_dir)):
parser.error(('At least one of "--gcc_dir", "--gcc_branch", '
'"--binutils_dir" and "--binutils_branch" must '
'be specified.'))
return 1
if not options.board and not options.bootstrap:
parser.error('You must specify either "--board" or "--bootstrap".')
return 1
if options.board and options.bootstrap:
parser.error('You must specify only one of "--board" and "--bootstrap".')
return 1
if Bootstrapper(
options.chromeos_root,
gcc_branch=options.gcc_branch, gcc_dir=options.gcc_dir,
binutils_branch=options.binutils_branch,
binutils_dir=options.binutils_dir,
board=options.board,
setup_tool_ebuild_file_only=options.setup_tool_ebuild_file_only).Do():
return 0
return 1
if __name__ == '__main__':
retval = Main(sys.argv)
sys.exit(retval)