compute/setup_bot.py

# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Should be run on a GCE instance to set up the build environment."""

from __future__ import print_function

import getpass
import os
import shutil

from chromite.compute import compute_configs
from chromite.compute import bot_constants
from chromite.lib import cros_build_lib
from chromite.lib import osutils


# Make the script more readable.
RunCommand = cros_build_lib.RunCommand
SudoRunCommand = cros_build_lib.SudoRunCommand


BOT_CREDS_PATH = bot_constants.BOT_CREDS_TMP_PATH
# Most credentials are stored in the home directory.
HOME_DIR = osutils.ExpandPath('~')


def SetupPrerequisites():
  """Installs packages required for Chrome OS build."""
  SudoRunCommand(['apt-get', 'update'])
  SudoRunCommand(['apt-get', '-y', '--force-yes', 'upgrade'])
  # Chrome OS pre-requisite packages.
  packages = ['git', 'curl', 'pbzip2', 'gawk', 'gitk', 'subversion']
  # Required for CIDB.
  packages += ['python-sqlalchemy', 'python-mysqldb']
  # Required for payload generation outside of the chroot.
  packages += ['python-protobuf']
  # Required to install python packages only available via pip.
  packages += ['python-pip']

  # Packages to monitor system performance and usage.
  packages += ['sysstat']

  SudoRunCommand(['apt-get', '-y', 'install'] + packages)
  SetupPipPrerequisites()


def SetupPipPrerequisites():
  """Installs python packages via pip.

  This assumes that pip itself is installed already.
  """
  # dict of package to version. Provide version None if you don't care about the
  # version installed.
  packages = {'python-statsd': '1.7.0', 'google-api-python-client': '1.4.0'}

  for package, version in packages.iteritems():
    install_atom = package
    if version is not None:
      install_atom += ('==' + version)
    SudoRunCommand(['pip', 'install', install_atom])


def InstallChromeDependencies():
  """Installs packages required to build Chromium."""
  # The install-build-deps.sh relies on some packages that are not in
  # the base image. Install them first before invoking the script.
  SudoRunCommand(['apt-get', '-y', 'install',
                  'gcc-arm-linux-gnueabihf',
                  'g++-4.8-multilib-arm-linux-gnueabihf',
                  'gcc-4.8-multilib-arm-linux-gnueabihf',
                  'realpath'])

  with osutils.TempDir(prefix='tmp-chrome-deps') as tempdir:
    RunCommand(['git', 'clone', bot_constants.CHROMIUM_BUILD_URL], cwd=tempdir)
    RunCommand([os.path.join(tempdir, 'build', 'install-build-deps.sh'),
                '--syms', '--no-prompt'])


def SetMountCount():
  """Sets mount count to a large number."""
  for drive in compute_configs.DRIVES:
    SudoRunCommand(['tune2fs', '-c', '150', os.path.join('dev', drive)],
                   redirect_stdout=True)


def _SetupSVN():
  """Sets up the chromium svn username/password."""
  # Create a ~/.subversion directory.
  RunCommand(['svn', 'ls', 'http://src.chromium.org/svn'], redirect_stdout=True)
  # Change the setting to store the svn password.
  sed_str = ('s/# store-plaintext-passwords = '
             'no/store-plaintext-passwords = yes/g')
  RunCommand(['sed', '-i', '-e', sed_str,
              osutils.ExpandPath(os.path.join('~', '.subversion', 'servers'))])

  password_path = osutils.ExpandPath(
      os.path.join(BOT_CREDS_PATH, bot_constants.SVN_PASSWORD_FILE))
  password = osutils.ReadFile(password_path).strip()
  # `svn ls` each repository to store the password in ~/.subversion.
  for svn_host in bot_constants.CHROMIUM_SVN_HOSTS:
    for svn_repo in bot_constants.CHROMIUM_SVN_REPOS:
      RunCommand(['svn', 'ls', '--username', bot_constants.BUILDBOT_SVN_USER,
                  '--password', password, 'svn://%s/%s' % (svn_host, svn_repo)],
                 redirect_stdout=True)

def _SetupGoB():
  """Sets up GoB credentials."""
  RunCommand(['git', 'config', '--global', 'user.email',
              bot_constants.GIT_USER_EMAIL])
  RunCommand(['git', 'config', '--global', 'user.name',
              bot_constants.GIT_USER_NAME])

  RunCommand(['git', 'clone', bot_constants.GCOMPUTE_TOOLS_URL],
             cwd=HOME_DIR, redirect_stdout=True)

  # Run git-cookie-authdaemon at boot time by adding it to
  # /etc/rc.local
  rc_local_path = os.path.join(os.path.sep, 'etc', 'rc.local')
  daemon_path = os.path.join(HOME_DIR, 'gcompute-tools',
                             'git-cookie-authdaemon')
  daemon_cmd = ['su', bot_constants.BUILDBOT_USER, '-c', daemon_path]
  content = osutils.ReadFile(rc_local_path).replace('exit 0', '')
  content += (' '.join(daemon_cmd) + '\n')
  content += 'exit 0\n'

  with osutils.TempDir() as tempdir:
    tmp_file = os.path.join(tempdir, 'rc.local')
    osutils.WriteFile(tmp_file, content)
    os.chmod(tmp_file, 755)
    SudoRunCommand(['mv', tmp_file, rc_local_path])
  # Also run the daemon now so that subsequent setup steps get credentials.
  # NB: It's important to redirect all pipes because the daemonize code here is
  # broken, it leaves open fds behind, causing ssh to hang.
  SudoRunCommand(daemon_cmd,
                 mute_output=True, combine_stdout_stderr=True)


def _SetupCIDB():
  """Copies cidb credentials."""
  shutil.copytree(os.path.join(BOT_CREDS_PATH, bot_constants.CIDB_CREDS_DIR),
                  os.path.join(HOME_DIR, bot_constants.CIDB_CREDS_DIR))


def _SetupTreeStatus():
  """Copies credentials for updating tree status."""
  shutil.copy(
      os.path.join(BOT_CREDS_PATH, bot_constants.TREE_STATUS_PASSWORD_FILE),
      HOME_DIR)


def _SetupGmail():
  """Copies credentials for accessing gmail API."""
  shutil.copy(
      os.path.join(BOT_CREDS_PATH, bot_constants.GMAIL_CREDENTIALS_FILE),
      HOME_DIR)


def SetupCredentials():
  """Sets up various credentials."""
  _SetupSVN()
  _SetupGoB()
  _SetupCIDB()
  _SetupTreeStatus()
  _SetupGmail()


def SetupBuildbotEnvironment():
  """Sets up the buildbot environment."""

  # Append host entries to /etc/hosts. This includes the buildbot
  # master IP address.
  host_entries = RunCommand(
      ['cat', os.path.join(BOT_CREDS_PATH, bot_constants.HOST_ENTRIES)],
      capture_output=True).output
  SudoRunCommand(['tee', '-a', '/etc/hosts'], input=host_entries)

  # Create the buildbot directory.
  SudoRunCommand(['mkdir', '-p', bot_constants.BUILDBOT_DIR])
  SudoRunCommand(['chown', '-R', '%s:%s' % (bot_constants.BUILDBOT_USER,
                                            bot_constants.BUILDBOT_USER),
                  bot_constants.BUILDBOT_DIR])

  with osutils.TempDir() as tempdir:
    # Download depot tools to a temp directory to bootstrap. `gclient
    # sync` will create depot_tools in BUILDBOT_DIR later.
    tmp_depot_tools_path = os.path.join(tempdir, 'depot_tools')
    RunCommand(['git', 'clone', bot_constants.DEPOT_TOOLS_URL],
               cwd=tempdir, redirect_stdout=True)
    # `gclient` relies on depot_tools in $PATH, pass the extra
    # envinornment variable.
    path_env = '%s:%s' % (os.getenv('PATH'), tmp_depot_tools_path)
    RunCommand(['gclient', 'config', bot_constants.BUILDBOT_GIT_REPO],
               cwd=bot_constants.BUILDBOT_DIR, extra_env={'PATH': path_env})
    RunCommand(['gclient', 'sync', '--jobs', '5'],
               cwd=bot_constants.BUILDBOT_DIR,
               redirect_stdout=True, extra_env={'PATH': path_env})

  # Set up buildbot password.
  config_dir = os.path.join(bot_constants.BUILDBOT_DIR, 'build', 'site_config')
  shutil.copy(
      os.path.join(BOT_CREDS_PATH, bot_constants.BUILDBOT_PASSWORD_FILE),
      config_dir)

  # Update the environment variable.
  depot_tools_path = os.path.join(bot_constants.BUILDBOT_DIR, 'depot_tools')
  RunCommand(['bash', '-c', r'echo export PATH=\$PATH:%s >> ~/.bashrc'
              % depot_tools_path])

  # Start buildbot slave at startup.
  crontab_content = ''
  result = RunCommand(
      ['crontab', '-l'], capture_output=True, error_code_ok=True)
  crontab_content = result.output if result.returncode == 0 else ''
  crontab_content += ('SHELL=/bin/bash\nUSER=chrome-bot\n'
                      '@reboot cd /b/build/slave && make start\n')
  RunCommand(['crontab', '-'], input=crontab_content)


def TuneSystemSettings():
  """Tune the system settings for our build environment."""
  # Increase the user-level file descriptor limits.
  entries = ('*       soft    nofile  65536\n'
             '*       hard    nofile  65536\n')
  SudoRunCommand(['tee', '-a', '/etc/security/limits.conf'], input=entries)


def main(_argv):
  assert getpass.getuser() == bot_constants.BUILDBOT_USER, (
      'This script should be run by %s instead of %s!' % (
          bot_constants.BUILDBOT_USER, getpass.getuser()))
  SetupPrerequisites()
  InstallChromeDependencies()
  SetupCredentials()
  SetupBuildbotEnvironment()
  TuneSystemSettings()