#!/usr/bin/python
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
A script to generate and store the manifests for the canary builders to use.
"""

import fnmatch
import logging
import logging.handlers
import os
import re
import shutil
import tempfile

import chromite.lib.cros_build_lib as cros_lib

logging_format = '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s'
date_format = '%Y/%m/%d %H:%M:%S'
logging.basicConfig(level=logging.INFO, format=logging_format,
                    datefmt=date_format)

# Pattern for matching build name format. E.g, 12.3.4.5,1.0.25.3
VER_PATTERN = '(\d+).(\d+).(\d+).(\d+)'

class VersionUpdateException(Exception):
  """Exception gets thrown for failing to update the version file"""
  pass


class InvalidOptionException(Exception):
  """Exception gets thrown for invalid options"""
  pass


class GitCommandException(Exception):
  """Exception gets thrown for a git command that fails to execute."""
  pass

class SrcCheckOutException(Exception):
  """Exception gets thrown for failure to sync sources"""
  pass

class StatusUpdateException(Exception):
  """Exception gets thrown for failure to update the status"""
  pass

class GenerateBuildSpecException(Exception):
  """Exception gets thrown for failure to Generate a buildspec for the build"""
  pass


class GitMirror(object):
  """ A Class to deal with the source tree/mirror setup and sync repos
    Args:
      mirror_dir: location for setting up git mirror
      repo_url: gitserver URL to fetch manifests from
      manifest_file: manifest_file to use for syncing
  """
  def __init__(self, mirror_dir, repo_url, manifest_file='full.xml'):
    self.mirror_dir = mirror_dir
    self.repo_url = repo_url
    self.manifest_file = manifest_file
    self.InitMirrorRepos()

  def InitMirrorRepos(self):
    """Initialize the git mirroring"""
    if not os.path.exists(self.mirror_dir):
      os.makedirs(self.mirror_dir)
      logging.debug('git mirror does not exist. Creating it for the first time')

    if not os.path.exists(os.path.join(self.mirror_dir, '.repo')):
      cros_lib.RunCommand(['repo', 'init', '-u', self.repo_url, '-m',
                           self.manifest_file, '--mirror'], cwd=self.mirror_dir,
                           input='\n\ny\n')

  def SyncMirror(self):
    """Sync/update the git mirror location"""
    cros_lib.RunCommand(['repo', 'sync'], cwd=self.mirror_dir)


class VersionInfo(object):
  """Class to encapsualte the chrome os version info
    You can instantiate this class in two ways.
    1)using a version file, specifically chromeos_version.sh,
    which contains the version information.
    2) just passing in the 4 version components (major, minor, sp and patch)
    Args:
      ver_maj: major version
      ver_min: minor version
      ver_sp:  sp version
      ver_patch: patch version
      version_file: version file location.
  """
  def __init__(self, ver_maj=0, ver_min=0, ver_sp=0, ver_patch=0,
               incr_type=None, version_file=None):
    self.ver_maj = ver_maj
    self.ver_min = ver_min
    self.ver_sp = ver_sp
    self.ver_patch = ver_patch
    self.incr_type = incr_type
    if version_file:
      self.version_file = version_file
      self.load()

  def FindValue(self, key, line):
    """Given the key find the value from the line, if it finds key = value
    Args:
      key: key to look for
      line: string to search
    returns:
       None: on a non match
       value: for a matching key
    """
    regex = '.*(%s)\s*=\s*(\d+)$' % key

    match = re.match(regex, line)
    if match:
      return match.group(2)
    return None

  def load(self):
    """Read the version file and set the version components"""
    with open(self.version_file, 'r') as version_fh:
      for line in version_fh:
        if not line.strip():
          continue

        match = self.FindValue('CHROMEOS_VERSION_MAJOR', line)
        if match:
          self.ver_maj = match
          logging.debug('Set the major version to:%s', self.ver_maj)
          continue

        match = self.FindValue('CHROMEOS_VERSION_MINOR', line)
        if match:
          self.ver_min = match
          logging.debug('Set the minor version to:%s', self.ver_min)
          continue

        match = self.FindValue('CHROMEOS_VERSION_BRANCH', line)
        if match:
          self.ver_sp = match
          logging.debug('Set the sp version to:%s', self.ver_sp)
          continue

        match = self.FindValue('CHROMEOS_VERSION_PATCH', line)
        if match:
          self.ver_patch = match
          logging.debug('Set the patch version to:%s', self.ver_patch)
          continue
    logging.debug(self.VersionString())

  def IncrementVersion(self):
    """Updates the version file by incrementing the patch component"""
    if not self.version_file:
      raise VersionUpdateException('Cannot call IncrementVersion without '
                                   'an associated version_file')
    if not self.incr_type:
      raise VersionUpdateException('Need to specify the part of the version to'
                                   'increment')

    if self.incr_type == 'branch':
      self.ver_sp = str(int(self.ver_sp) + 1)
      self.ver_patch = '0'
    if self.incr_type == 'patch':
      self.ver_patch = str(int(self.ver_patch) + 1)
    temp_file = tempfile.mkstemp(suffix='mvp', prefix='tmp', dir=None,
                                 text=True)[1]
    with open(self.version_file, 'r') as source_version_fh:
      with open(temp_file, 'w') as temp_fh:
        for line in source_version_fh:
          old_patch = self.FindValue('CHROMEOS_VERSION_PATCH', line)
          if old_patch:
            temp_fh.write(line.replace(old_patch, self.ver_patch, 1))
            continue

          old_sp = self.FindValue('CHROMEOS_VERSION_BRANCH', line)
          if old_sp:
            temp_fh.write(line.replace(old_sp, self.ver_sp, 1))
            continue

          temp_fh.write(line)
        temp_fh.close()
      source_version_fh.close()
    shutil.copyfile(temp_file, self.version_file)
    os.unlink(temp_file)
    return self.VersionString()

  def VersionString(self):
    """returns the version string"""
    return '%s.%s.%s.%s' % (self.ver_maj, self.ver_min, self.ver_sp,
                            self.ver_patch)
  def DirPrefix(self):
    """returns the sub directory suffix in manifest-versions"""
    return '%s.%s' % (self.ver_maj, self.ver_min)

  def BuildPrefix(self):
    """returns the build prefix to match the buildspecs in  manifest-versions"""
    if self.incr_type == 'patch':
      return '%s.%s.%s' % (self.ver_maj, self.ver_min, self.ver_sp)

    if self.incr_type == 'branch':
      return '%s.%s' % (self.ver_maj, self.ver_min)

    return None


class SpecsInfo(object):
  """Class that contains information about the buildspecs
    Args:
      working_dir: location of the buildspecs location
      build_name: name of the build that we will be dealing with
      dir_prefix: prefix of the version for sub-directory location
      build_prefix: prefix of the build version for build specs matching
  """
  def __init__(self, working_dir, ver_manifests, build_name, dir_prefix,
               build_prefix):
    self.working_dir = working_dir
    self.ver_manifests = ver_manifests
    self.build_name = build_name
    self.dir_prefix = dir_prefix
    self.build_prefix = build_prefix
    self.root_dir = os.path.join(self.working_dir, self.ver_manifests)
    self.all_specs_dir = os.path.join(self.root_dir, 'buildspecs',
                                      self.dir_prefix)
    self.pass_dir = os.path.join(self.root_dir, 'build-name', self.build_name,
                                 'pass', self.dir_prefix)
    self.fail_dir = os.path.join(self.root_dir, 'build-name', self.build_name,
                                 'fail', self.dir_prefix)
    self.inflight_dir = os.path.join(self.root_dir, 'build-name',
                                     self.build_name, 'inflight',
                                     self.dir_prefix)
    logging.debug('Build Specs Dir = %s', self.all_specs_dir)


class BuildSpecs(object):
  """ Class to manage buildspecs
    Args:
      working_dir:location where the ver_manifests repo is synced to
      ver_manifests: Name of the ver_manifests repo. Default: manifest-versions
      build_name: Name of the build that we are interested in. E.g., x86-alex
      dir_prefix: prefix of the version numbers for sub directory look up
      build_prefix: prefix of the build version for build specs matching
      incr_type: part of the version to increment. 'patch or branch'
  """
  def __init__(self, working_dir, ver_manifests, build_name, dir_prefix,
               build_prefix, incr_type):
    self.specs_info = SpecsInfo(working_dir, ver_manifests, build_name,
                                dir_prefix, build_prefix)
    self.all = self.GetMatchingSpecs(self.specs_info.all_specs_dir, incr_type)
    self.latest = '0.0.0.0'
    self.incr_type = incr_type

    if self.all:
      self.latest = self.all[-1]
    self.all = set(self.GetMatchingSpecs(self.specs_info.all_specs_dir,
                                         self.incr_type))
    self.passed = self.GetMatchingSpecs(self.specs_info.pass_dir,
                                        self.incr_type)
    self.failed = self.GetMatchingSpecs(self.specs_info.fail_dir,
                                        self.incr_type)
    self.inflight = self.GetMatchingSpecs(self.specs_info.inflight_dir,
                                          self.incr_type)
    self.processed = sorted((self.passed + self.failed + self.inflight),
                            key=lambda s: map(int, s.split('.')))
    self.last_processed = '0.0.0.0'
    if self.processed:
      self.last_processed = self.processed[-1]
    logging.debug('Last processed build for %s is %s' %
                  (build_name, self.last_processed))

    difference = list(self.all.difference(set(self.processed)))

    for build in difference[:]:
      build1 = map(int, build.split("."))
      build2 = map(int, self.last_processed.split("."))

      if cmp(build1 , build2) == 1:
        logging.debug('Still need to build %s' % build)
      else:
        logging.debug('Ignoring build %s less than %s' %
                      (build, self.last_processed))
        difference.remove(build)

    self.unprocessed = sorted(difference, key=lambda s: map(int, s.split('.')))

  def GetMatchingSpecs(self, specs_info, incr_type):
    """Returns the sorted list of buildspecs that match '*.xml'
      Args:
        specs_info: SpecsInfo object for the buildspecs location information.
    """
    matched_manifests = []
    if os.path.exists(specs_info):
      all_manifests = os.listdir(specs_info)
      match_string = self.specs_info.build_prefix + '.*.xml'

      if incr_type == 'branch':
         match_string = self.specs_info.build_prefix + '.*.0.xml'

      matched_manifests = fnmatch.filter(
          all_manifests, match_string)
      matched_manifests = [os.path.splitext(m)[0] for m in matched_manifests]

    return matched_manifests

  def FindNextBuild(self, latest=False):
    """"Find the next build to build from unprocessed buildspecs
      Args:
        latest: True of False. (Returns the last known build on latest =True
      Returns:
        Returns the next build to be built or None
    """
    if not self.unprocessed:
      return None

    if latest:
      return self.unprocessed[-1]

    return self.unprocessed[0]

  def ExistsBuildSpec(self, build_number):
    """"Find out if the build_number exists in existing buildspecs
      Args:
        build_number: build number to check for existence of build spec
      Returns:
        True if the buildspec exists, False if it doesn't
    """
    return build_number in self.all


class BuildSpecsManager(object):
  """A Class to manage buildspecs and their states
    Args:
      working_dir: location of the buildspecs repo
      build_name: build_name e.g., x86-mario, x86-mario-factory
      dir_prefix: version prefix to match for the sub directories
      build_prefix: prefix of the build version for build specs matching
  """

  def __init__(self, working_dir, ver_manifests, build_name, dir_prefix,
               build_prefix):
    self.specs_info = SpecsInfo(working_dir, ver_manifests, build_name,
                                dir_prefix, build_prefix)

  def SetUpSymLinks(self, src_file, dest_file, remove_file=None):
    """Setting up symlinks for various statuses in the git repo
      Args:
        status: status type to set to, (one of inflight, pass, fail)
        src_file: source for the symlink
        dest_file: destination for the symlink
        remove_file: symlink that needs to be deleted for clearing the old state
    """
    dest_dir = os.path.dirname(dest_file)
    if os.path.lexists(dest_file):
      os.unlink(dest_file)

    if not os.path.exists(dest_dir):
      os.makedirs(dest_dir)
    rel_src_file = os.path.relpath(src_file, dest_dir)
    logging.debug('Linking %s to %s', rel_src_file, dest_file)
    os.symlink(rel_src_file, dest_file)

    if remove_file and os.path.lexists(remove_file):
      logging.debug('REMOVE: Removing  %s', remove_file)
      os.unlink(remove_file)


  def SetInFlight(self, version):
    """Marks a buildspec as inflight by creating a symlink in 'inflight' dir
      Args:
        version: version of the buildspec to mark as inflight
    """
    dest_file = '%s.xml' % os.path.join(self.specs_info.inflight_dir, version)
    src_file = '%s.xml' % os.path.join(self.specs_info.all_specs_dir, version)
    logging.debug('Setting build in flight  %s: %s', src_file, dest_file)
    self.SetUpSymLinks(src_file, dest_file)

  def SetFailed(self, version):
    """Marks a buildspec as failed by creating a symlink in 'fail' dir
      Args:
        version: version of the buildspec to mark as failed
    """
    dest_file = '%s.xml' % os.path.join(self.specs_info.fail_dir, version)
    src_file = '%s.xml' % os.path.join(self.specs_info.all_specs_dir, version)
    remove_file = '%s.xml' % os.path.join(self.specs_info.inflight_dir, version)
    logging.debug('Setting build to failed  %s: %s', src_file, dest_file)
    self.SetUpSymLinks(src_file, dest_file, remove_file)

  def SetPassed(self, version):
    """Marks a buildspec as failed by creating a symlink in 'pass' dir
      Args:
        version: version of the buildspec to mark as passed
    """
    dest_file = '%s.xml' % os.path.join(self.specs_info.pass_dir, version)
    src_file = '%s.xml' % os.path.join(self.specs_info.all_specs_dir, version)
    remove_file = '%s.xml' % os.path.join(self.specs_info.inflight_dir, version)
    logging.debug('Setting build to passed  %s: %s', src_file, dest_file)
    self.SetUpSymLinks(src_file, dest_file, remove_file)


def CloneGitRepo(working_dir, repo_url):
  """"Clone Given git repo
    Args:
      repo_url: git repo to clione
      repo_dir: location where it should be cloned to
  """
  if not os.path.exists(working_dir):
    os.makedirs(working_dir)
  cros_lib.RunCommand(['git', 'clone', repo_url], cwd=working_dir)

def PushChanges(git_repo, message, use_repo=False, dry_run=True):
  """Do the final commit into the git repo
      Args:
        git_repo: git repo to push
        message: Commit message
        use_repo: use repo tool for pushing changes. Default: False
        dry_run: If true, don't actually push changes to the server
    raises: GitCommandException
  """
  branch = 'temp_auto_checkin_branch'
  try:
    if use_repo:
      cros_lib.RunCommand(['repo', 'start', branch, '.'], cwd=git_repo)
      cros_lib.RunCommand(['repo', 'sync', '.'], cwd=git_repo)
      cros_lib.RunCommand(['git', 'config', 'push.default', 'tracking'],
                          cwd=git_repo)
    else:
      cros_lib.RunCommand(['git', 'pull', '--force'], cwd=git_repo)
    cros_lib.RunCommand(['git', 'add', '-A'], cwd=git_repo)
    cros_lib.RunCommand(['git', 'commit', '-am', message], cwd=git_repo)

    push_cmd = ['git', 'push', '--verbose']
    if dry_run: push_cmd.append('--dry-run')
    cros_lib.RunCommand(push_cmd, cwd=git_repo)
  except cros_lib.RunCommandError, e:
    err_msg = 'Failed to commit to %s' % e.message
    logging.error(err_msg)
    git_status = cros_lib.RunCommand(['git', 'status'], cwd=git_repo)
    logging.error('Current repo %s status: %s', git_repo, git_status)
    CleanGitChanges(git_repo)
    raise GitCommandException(err_msg)
  finally:
    if use_repo:
      cros_lib.RunCommand(['repo', 'abandon', branch], cwd=git_repo)

def CleanGitChanges(git_repo):
  """"Clean git repo chanages
    Args:
      git_repo: location of the git repo to clean
    raises: GitCommandException: when fails to clean
  """
  try:
    cros_lib.RunCommand(['git', 'clean', '-d', '-f'], cwd=git_repo)
    cros_lib.RunCommand(['git', 'reset', '--hard', 'HEAD'], cwd=git_repo)
  except cros_lib.RunCommandError, e:
    err_msg = 'Failed to clean git repo %s' % e.message
    logging.error(err_msg)
    raise GitCommandException(err_msg)


def SetLogFileHandler(logfile):
  """This sets the logging handler to a file.
    define a Handler which writes INFO messages or higher to the sys.stderr
    Add the log message handler to the logger
    Args:
      logfile: name of the logfile to open
  """
  logfile_handler = logging.handlers.RotatingFileHandler(logfile, backupCount=5)
  logfile_handler.setLevel(logging.DEBUG)
  logfile_handler.setFormatter(logging.Formatter(logging_format))
  logging.getLogger().addHandler(logfile_handler)


def CheckOutSources(cros_source, source_dir, branch,
                    manifest_file='full.xml', retries=0):
  """"Fetches the sources from the git server using repo tool
    Args:
      cros_source: GitMirror Object
      source_dir: Location to sync the sources to
      branch: branch to use for the manifest.xml for repo tool
      manifest_file: manifest file to use for the file definition
      retries: Number of times to try to fetch sources
  """
  while True:
    if not os.path.exists(source_dir):
      os.makedirs(source_dir)
    try:
      cros_source.SyncMirror()
      cros_lib.RunCommand(['repo', 'init', '-u', cros_source.repo_url,
                           '-b', branch, '-m',
                           manifest_file, '--reference',
                           cros_source.mirror_dir], cwd=source_dir,
                           input='\n\ny\n')
      cros_lib.RunCommand(['repo', 'sync'], cwd=source_dir)
      break
    except cros_lib.RunCommandError, e:
      if retries < 1:
        err_msg = 'Failed to sync sources %s' % e.message
        logging.error(err_msg)
        raise SrcCheckOutException(err_msg)
      logging.info('Failed to sync sources %s', e.message)
      logging.info('But will try  %d times', retries)
      CleanUP(source_dir)
      retries = retries - 1

def ExportManifest(source_dir, output_file):
  """Exports manifests file
      Args:
        source_dir: repo root
        output_file: output_file to out put the manifest to
  """
  cros_lib.RunCommand(['repo', 'manifest', '-r', '-o', output_file],
                      cwd=source_dir)


def SetupExistingBuildSpec(build_specs_manager, next_build, dry_run):
  build_specs_manager.SetInFlight(next_build)
  commit_message = 'Automatic: Start %s %s' % (
      build_specs_manager.specs_info.build_name, next_build)
  PushChanges(build_specs_manager.specs_info.root_dir,
              commit_message,
              dry_run=dry_run)


def GenerateNewBuildSpec(source_dir, branch, latest_build, build_specs_root,
                         build_specs_manager, cros_source, version_info,
                         increment_version,
                         dry_run):
  """Generates a new buildspec for the builders to consume
    Checks to see, if there are new changes that need to be built from the last
    time another buildspec was created. Updates the version number in version
    number file. Generates the manifest as the next buildspecs. pushes it to git
    and returns the next build spec number
    If there are no new changes returns None
  Args:
    source_dir: location of the source tree root
    branch: branch to sync the sources to
    latest_build: latest known build to match the latest sources against
    build_specs_root: location of the build specs root
    build_specs_manager: BuildSpecsManager object
    cros_source: GitMirror object
    version_info: VersionInfo Object
    increment_version: True or False for incrementing the chromeos_version.sh
    dry_run: if True don't push changes externally
  Returns:
    next build number: on new changes or
    None: on no new changes
  """
  temp_manifest_file = tempfile.mkstemp(suffix='mvp', text=True)[1]

  ExportManifest(source_dir, temp_manifest_file)

  latest_spec_file = '%s.xml' % os.path.join(
      build_specs_manager.specs_info.all_specs_dir, latest_build)

  logging.debug('Calling DiffManifests with %s, %s', latest_spec_file,
                temp_manifest_file)
  if not (latest_build == '0.0.0.0' or
          DiffManifests(temp_manifest_file, latest_spec_file)):
    return None

  next_version = version_info.VersionString()

  if increment_version:
    next_version = version_info.IncrementVersion()
    logging.debug('Incremented version number to  %s', next_version)
    message = 'Automatic: Updating the new version number %s' % next_version
    PushChanges(os.path.dirname(version_info.version_file), message,
                use_repo=True, dry_run=dry_run)
    CheckOutSources(cros_source, source_dir, branch)

  next_spec_file = '%s.xml' % os.path.join(
      build_specs_manager.specs_info.all_specs_dir, next_version)
  if not os.path.exists(os.path.dirname(next_spec_file)):
    os.makedirs(os.path.dirname(next_spec_file))
  ExportManifest(source_dir, next_spec_file)
  message = 'Automatic: Buildspec %s. Created by %s' % (
      next_version, build_specs_manager.specs_info.build_name)
  logging.debug('Created New Build Spec %s', message)

  build_specs_manager.SetInFlight(next_version)
  PushChanges(build_specs_root, message, dry_run=dry_run)
  return next_version


def DiffManifests(manifest1, manifest2):
  """Diff two manifest files. excluding the revisions to manifest-verisons.git
    Args:
      manifest1: First manifest file to compare
      manifest2: Second manifest file to compare
    Returns:
      True: If the manifests are different
      False: If the manifests are same
  """
  black_list = ['manifest-versions']
  blacklist_pattern = re.compile(r'|'.join(black_list))
  with open(manifest1, 'r') as manifest1_fh:
    with open(manifest2, 'r') as manifest2_fh:
      for (line1, line2) in zip(manifest1_fh, manifest2_fh):
        if blacklist_pattern.search(line1):
          logging.debug('%s ignored %s', line1, line2)
          continue

        if line1 != line2:
          return True
      return False


def SetStatus(build_version, status, working_dir, ver_manifests, build_name,
              dry_run):
  """Set the status of a particular build by moving the build_spec files around
      Args:
        build_version: version of the build that we should set the status for
        status: status to set. One of 'pass|fail'
        working_dir: working dir where our version-manifests repos are synced
        ver_manifests: name of the version manifests repo
        build_name: Name of the build that we are settting the status for
        dry_run: if True don't push changes externally
  """
  logging.info('Setting Status for %s as %s', build_version, status)
  match = re.search(VER_PATTERN, build_version)
  logging.debug('Matched %s - %s - %s - %s', match.group(1), match.group(2),
                match.group(3), match.group(4))
  version_info = VersionInfo(ver_maj=match.group(1), ver_min=match.group(2),
                             ver_sp=match.group(3), ver_patch=match.group(4))
  logging.debug('Using Version: %s', version_info.VersionString())
  logging.debug('Using Directory Prefix: %s', version_info.DirPrefix())
  logging.debug('Using Build Prefix: %s', version_info.BuildPrefix())

  build_specs_manager = BuildSpecsManager(working_dir, ver_manifests,
                                          build_name, version_info.DirPrefix(),
                                          version_info.BuildPrefix())
  if status == 'pass':
    build_specs_manager.SetPassed(build_version)
  if status == 'fail':
    build_specs_manager.SetFailed(build_version)

  commit_message = 'Automatic checkin: status = %s build_version %s for %s' % (
      status, build_version, build_name)
  PushChanges(build_specs_manager.specs_info.root_dir, commit_message,
              dry_run=dry_run)


def RemoveDirs(dir_name):
  """Remove directories recursively, if they exist"""
  if os.path.exists(dir_name):
    shutil.rmtree(dir_name)

def CleanUP(source_dir=None, working_dir=None, git_mirror_dir=None):
  """Clean  up of all the directories that are created by this script
    Args:
      source_dir: directory where the sources are. Default: None
      working_dir: directory where manifest-versions is. Default: None
      git_mirror_dir: directory where git mirror is setup. Default: None
  """
  if source_dir:
    logging.debug('Cleaning source dir = %s', source_dir)
    RemoveDirs(source_dir)

  if working_dir:
    logging.debug('Cleaning working dir = %s', working_dir)
    RemoveDirs(working_dir)

  if git_mirror_dir:
    logging.debug('Cleaning git mirror dir = %s', git_mirror_dir)
    RemoveDirs(git_mirror_dir)

def GenerateWorkload(tmp_dir, source_repo, manifest_repo,
                     branch, version_file,
                     build_name, incr_type, latest=False,
                     dry_run=False,
                     retries=0):
  """Function to see, if there are any new builds that need to be built.
    Args:
      tmp_dir: Temporary working directory
      source_repo: git URL to repo with source code
      manifest_repo: git URL to repo with manifests/status values
      branch: Which branch to build ('master' or mainline)
      version_file: location of chromeos_version.sh
      build_name: name of the build_name
      incr_type: part of the version number to increment 'patch or branch'
      latest: Whether we need to handout the latest build. Default: False
      dry_run: if True don't push changes externally
      retries: Number of retries for updating the status
    Returns:
      next_build: a string of the next build number for the builder to consume
                  or None in case of no need to build.
    Raises:
      GenerateBuildSpecException in case of failure to generate a buildspec
  """

  ver_manifests=os.path.basename(manifest_repo)

  git_mirror_dir = os.path.join(tmp_dir, 'mirror')
  source_dir = os.path.join(tmp_dir, 'source')
  working_dir = os.path.join(tmp_dir, 'working')

  while True:
    try:
      cros_source = GitMirror(git_mirror_dir, source_repo)
      CheckOutSources(cros_source, source_dir, branch)

      # Let's be conservative and remove the version_manifests directory.
      RemoveDirs(working_dir)
      CloneGitRepo(working_dir, manifest_repo)

      version_file = os.path.join(source_dir, version_file)
      logging.debug('Using VERSION _FILE = %s', version_file)
      version_info = VersionInfo(version_file=version_file, incr_type=incr_type)
      build_specs = BuildSpecs(working_dir, ver_manifests, build_name,
                               version_info.DirPrefix(),
                               version_info.BuildPrefix(),
                               version_info.incr_type)
      current_build = version_info.VersionString()
      logging.debug('Current build in %s: %s', version_file, current_build)

      build_specs_manager = BuildSpecsManager(
        working_dir, ver_manifests, build_name, version_info.DirPrefix(),
        version_info.BuildPrefix())

      next_build = build_specs.FindNextBuild(latest)
      if next_build:
        logging.debug('Using an existing build spec: %s', next_build)
        SetupExistingBuildSpec(build_specs_manager, next_build, dry_run)
        return next_build

      # If the build spec doesn't exist for the current version in chromeos_
      # version.sh. we should just create the build_spec for that version,
      # instead of incrementing chromeos_version.sh

      increment_version = build_specs.ExistsBuildSpec(current_build)
      return GenerateNewBuildSpec(source_dir, branch, build_specs.latest,
                                  build_specs.specs_info.root_dir,
                                  build_specs_manager, cros_source,
                                  version_info, increment_version,
                                  dry_run)

    except (cros_lib.RunCommandError, GitCommandException), e:
      if retries < 1:
        err_msg = 'Failed to generate buildspec. error: %s' % e
        logging.error(err_msg)
        raise GenerateBuildSpecException(err_msg)

      logging.info('Failed to generate a buildspec for consumption: %s',
                   e.message)
      logging.info('But will try  %d times', retries)
      CleanUP(source_dir, working_dir)
      retries = retries - 1

def UpdateStatus(tmp_dir, manifest_repo, build_name,
                 build_version, success, dry_run=False, retries=0):
  """Updates the status of the build
    Args:
      tmp_dir: Temporary working directory, best shared with GenerateWorkload
      manifest_repo: git URL to repo with manifests/status values
      build_name: name of the build_name
      build_version: value returned by GenerateWorkload for this build
      success: True for success, False for failure
      dry_run: if True don't push changes externally
      retries: Number of retries for updating the status
    Raises:
      GenerateBuildSpecExtension in case of failure to generate a buildspec
  """

  working_dir = os.path.join(tmp_dir, 'working')
  ver_manifests=os.path.basename(manifest_repo)
  ver_manifests_dir = os.path.join(working_dir, ver_manifests)

  if success:
    status = 'pass'
  else:
    status = 'fail'

  while True:
    try:
      if not os.path.exists(ver_manifests_dir):
        CloneGitRepo(manifest_repo, working_dir)
      cros_lib.RunCommand(['git', 'checkout', 'master'], cwd=ver_manifests_dir)
      logging.debug('Updating the status info for %s to %s', build_name,
                    status)
      SetStatus(build_version, status, working_dir, ver_manifests,
                build_name, dry_run)
      logging.debug('Updated the status info for %s to %s', build_name,
                    status)
      break
    except (GitCommandException, cros_lib.RunCommandError), e:
      if retries <= 0:
        logging.error('Failed to update the status for %s to %s', build_name,
                      status)
        err_msg = 'with the following error: %s' % e.message
        logging.error(err_msg)
        raise StatusUpdateException(err_msg)

      logging.info('Failed to update the status for %s to %s', build_name,
                   status)
      logging.info('But will try  %d times', retries)
      RemoveDirs(ver_manifests_dir)
      retries = retries - 1