#!/usr/bin/python
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to download ChromeOS images or test folders from google storage and
# output download location or directly copy (image) to a usb stick.
# In order to make usb sticks, file needs to be run from a place such that
# ./REL_SRC_DIR reaches chromiumos/src, or the src_dir input must be
# specified.
# Downloads image via gsutil to chromiumos/src/REL_DL_DIR.

"""Download and output or run image_to_usb command."""

import argparse
from multiprocessing import Manager
from multiprocessing import Process
import os
import shutil
import subprocess

# Relative path from this file to the chromiumos/src folder
REL_SRC_DIR = '../../../'
# Path to default download directory relative to chromiumos/src folder,
# i.e. REL_SRC_DIR+REL_DIR_DIR is a direct path from here to the directory
REL_DL_DIR = 'build/crosdl/'

# Conversions from common simplified/misspelled names of boards
PLATFORM_CONVERT = {'spring': 'daisy-spring', 'alex': 'x86-alex',
                    'alex-he': 'x86-alex-he', 'mario': 'x86-mario',
                    'zgb': 'x86-zgb', 'zgb-he': 'x86-zgb-he',
                    'pit': 'peach-pit', 'pi': 'peach-pi',
                    'snow': 'daisy', 'lucas': 'daisy', 'big': 'nyan-big'}

# Download types
RECOVERY = 0
TEST = 1
AUTOTEST = 2
FACTORY = 3


def _GenerateGstorageLink(c, p, b):
  """Generate Google storage link given channel, platform, and build."""
  return 'gs://chromeos-releases/%s-channel/%s/%s/' % (c, p, b)


def _GenerateFolderName(download_type, b, p, c, mp):
  """Generate a folder name unique to the download."""
  if download_type == TEST:
    type_string = 'Test'
  elif download_type == AUTOTEST:
    type_string = 'Autotest'
  elif download_type == FACTORY:
    type_string = 'Factory'
  else:
    type_string = 'Recovery'
  return '%s_%s_%s_%s%s' % (p, b, c, type_string, mp)


def main():
  """Download and output or run image_to_usb command."""
  parser = argparse.ArgumentParser(
      description=('Download a testing resource: recovery image, test image, '
                   'autotest folder, or factory\nbundle.  Optionally make usb '
                   'sticks from these downloaded images.\n\n'
                   'e.g. ./crosdl.py -c dev -b 4996.0.0 -p '
                   'link daisy --tostick /dev/sdc /dev/sda.\n\nDefault '
                   'download location is src/%s.' % REL_DL_DIR),
      formatter_class=argparse.RawDescriptionHelpFormatter)
  group_type = parser.add_mutually_exclusive_group()
  group_type.add_argument('-r', '--recovery', dest='recovery',
                          action='store_true', help='Recovery image (default).')
  group_type.add_argument('-t', '--test', dest='test', action='store_true',
                          help='Test image.')
  group_type.add_argument('-a', '--autotest', dest='autotest',
                          action='store_true', help='Autotest folder.')
  group_type.add_argument('-f', '--factory', dest='factory',
                          action='store_true', help='Factory bundle.')
  parser.add_argument('-c', '--channel', dest='channel', default='dev',
                      choices=['canary', 'dev', 'beta', 'stable'],
                      help='Channel (dev default).')
  parser.add_argument('-b', '--build', dest='build',
                      help='Build number, e.g. 4996.0.0.')
  parser.add_argument('-p', '--platform', dest='board', nargs='+',
                      help='Platform(s) to download, e.g. link daisy.')
  parser.add_argument('--premp', action='store_true',
                      help='PreMP image instead of MP.')
  parser.add_argument('--force', dest='force', action='store_true',
                      help=('Force new download of builds, even if files '
                            'already exist.'))
  group_usb = parser.add_mutually_exclusive_group()
  group_usb.add_argument('--to_stick', dest='to_stick', nargs='*',
                         help=('Copy to usb stick after download in a one-'
                               'to-one way.  Can either specify drive(s) '
                               '(e.g. /dev/sdc) at which to install or leave '
                               'blank for interactive dialog later.  For '
                               'multiple boards, list out all drives (e.g. '
                               '/dev/sdc /dev/sda /dev/sdd).  Script will '
                               'match this list directly to the list of input '
                               'boards (in -p).'))
  group_usb.add_argument('--one_to_multiple_sticks', dest='to_many', nargs='+',
                         help=('Copy one image to the listed multiple usb '
                               'sticks after download (e.g. /dev/sdc /dev/sdd)'
                               '.  Must specify only one board at a time.'))
  parser.add_argument('--folder', dest='folder',
                      help=('Specify a new download folder (default is src/'
                            '%s).' % REL_DL_DIR))
  parser.add_argument('--src_dir', dest='src_dir',
                      help=('Specify path to chromium src/ folder.  For running'
                            'this script from other locations.'))
  parser.add_argument('--delete_files', dest='delete', action='store_true',
                      help=('Delete any files downloaded from this command'
                            'when finished with copying to usb stick.  '
                            'Applicable only when using --to_stick or '
                            '--one_to_multiple_sticks arguments.'))
  parser.add_argument('--clear_folder', dest='clear', action='store_true',
                      help=('Delete all the sub-folders in the download '
                            'location (useful if you have filled up your '
                            'harddrive).'))
  arguments = parser.parse_args()

  # Make sure the default or user defined src folder actually points to src/.
  src_dir = arguments.src_dir if arguments.src_dir else REL_SRC_DIR
  if os.path.basename(os.path.abspath(src_dir)) != 'src':
    print 'Could not find src/ directory.  Try passing the --src_dir input.'
    return

  # Set download folder as user defined or default.
  user_folder = arguments.folder
  if user_folder:
    download_folder = user_folder
  else:
    download_folder = os.path.join(src_dir, REL_DL_DIR)

  # Delete download folder contents if clearfolder flag present.
  if arguments.clear:
    if os.path.exists(download_folder):
      print 'Deleting sub-folder contents of %s.' % download_folder
      for item in os.listdir(download_folder):
        item_path = os.path.join(download_folder, item)
        if os.path.isdir(item_path):
          shutil.rmtree(item_path)
    else:
      print 'Download folder %s did not exist.  Exiting.' % download_folder
    return

  # Require board and platform arguments if not clearing downloads.
  if not (arguments.board and arguments.build):
    print ('Must provide build number and platform(s).  See crosdl.py -h for '
           'usage description.')
    return

  # Require --deletefiles flag to be used only with --tostick flag.
  if arguments.delete and (arguments.to_stick or arguments.to_many):
    print 'Will delete all newly downloaded files once finished.'
  elif arguments.delete:
    print ('This command will download and immediately delete all files.  '
           'You probably meant to use the --tostick flag as well.')
    return

  # Deal with board name(s).
  dupe_boards = []
  boards = arguments.board
  for i in xrange(len(boards)):
    boards[i] = boards[i].lower().replace('_', '-')
    if boards[i] in PLATFORM_CONVERT:
      boards[i] = PLATFORM_CONVERT[boards[i]]
    # Disallow duplicates.
    if boards[i] in dupe_boards:
      print '%s is listed twice in the boards list!' % boards[i]
      return 1
    dupe_boards.append(boards[i])

  # Set is_test based on input flags.
  if arguments.test:
    print 'Downloading test image(s).'
    download_type = TEST
    is_image = True
  elif arguments.autotest:
    print 'Downloading autotest folder(s).'
    download_type = AUTOTEST
    is_image = False
  elif arguments.factory:
    print 'Downloading factory folder(s).'
    download_type = FACTORY
    is_image = False
  else: # RECOVERY
    print 'Downloading recovery image(s).'
    download_type = RECOVERY
    is_image = True

  # String to identify premp/mp images.
  mp_str = ''
  if download_type == RECOVERY:
    mp_str = '_premp' if arguments.premp else '_mp'

  # Disallow 'to many' option for more than one board
  installing_many = arguments.to_many
  if len(boards) > 1 and installing_many:
    print ('Please only specify one board if using --one_to_multiple_'
           'sticks option.  See help menu.')
    return 1

  # If installing multiple images, must provide drive names.
  installing_one = type(arguments.to_stick) == list
  if installing_one and (len(boards) > 1 or len(arguments.to_stick) > 1):
    if not arguments.to_stick:
      print ('Error: To make sticks for multiple boards, please provide drive '
             'names (e.g. /dev/sdc /dev/sdd).  See -h for help.')
      return 1
    if len(arguments.to_stick) != len(boards):
      print ('Error: Was given %d boards but %d usb drive locations.'
             % (len(boards), len(arguments.to_stick)))
      return 1
    for drive in arguments.to_stick:
      if not os.path.exists(drive):
        print '%s does not exist!' % drive

  # Disallow 'to usb' options if not an image
  if not is_image and (installing_one or installing_many):
    print ('Can only copy to usb if downloading an image.  See help menu.')
    return 1

  # Request sudo permissions if installing later.
  if installing_one or installing_many:
    subprocess.call(['sudo', '-v'])

  # Subroutine to download a file for a single board.
  channel = arguments.channel
  build = arguments.build
  def _DownloadBoard(board, output_str, dl_error, dl_folder):
    """Download the file for a single board."""
    # Assume error happened unless changed below.
    dl_error[board] = True

    # See if file already exists locally.
    folder_name = _GenerateFolderName(download_type=download_type, p=board,
                                      b=build, c=channel, mp=mp_str)
    folder_path = os.path.join(download_folder, folder_name)
    dl_folder[board] = folder_path
    if download_type == TEST:
      target_name = 'chromiumos_test_image.bin'
    elif download_type == AUTOTEST:
      target_name = ''
    elif download_type == FACTORY:
      target_name = ''
    else: # RECOVERY
      target_name = 'recovery_image.bin'
    target_path = os.path.join(folder_path, target_name)

    # Skip for already present files, else download new file.
    if os.path.exists(target_path) and not arguments.force:
      print '%s: Found file locally.  Skipping download.' % board
    else:
      # Make folder if needed.
      if not os.path.exists(folder_path):
        subprocess.call(['mkdir', '-p', folder_path])

      # Generate search terms.
      folder = _GenerateGstorageLink(c=channel, p=board, b=build)
      if download_type == TEST:
        file_search = '%s*test*.tar.xz' % folder
      elif download_type == AUTOTEST:
        file_search = '%s*hwqual*.tar.bz2' % folder
      elif download_type == FACTORY:
        file_search = '%s*factory*.zip' % folder
      else: # RECOVERY
        file_search = '%s*recovery*%s*%s*.bin' % (folder, channel, mp_str)
        file_search_2 = ('%schromeos-signing*/*recovery*%s*%s*.bin'
                         % (folder, channel, mp_str))

      # Output error if no files found
      def _no_file_error(message):
        """Actions to take if file is not found."""
        print '%s: %s' % (board, message)
        output_str[board] = '%s: Could not find file.' % board

      # Look for folder while file belongs.
      try:
        possible_files = subprocess.check_output(['gsutil', 'ls', folder])
      except subprocess.CalledProcessError:
        _no_file_error('Could not find folder %s where this file is supposed '
                       'to be.  Please check input values.' % folder)
        return 1

      # Look for file in folder.
      try:
        possible_files = subprocess.check_output(['gsutil', 'ls', file_search])
      except subprocess.CalledProcessError:
        if download_type == RECOVERY:
          try:
            possible_files = subprocess.check_output(['gsutil', 'ls',
                                                      file_search_2])
          except subprocess.CalledProcessError:
            _no_file_error('Could not find file but found folder.')
            return 1
        else:
          _no_file_error('Could not find file but found folder.')
          return 1

      # Locate exact file_name.
      possible_files = possible_files.splitlines()
      if len(possible_files) != 1:
        _no_file_error('Found %d possible files, not 1.' % len(possible_files))
        return 1
      gsfile_path = possible_files[0]
      file_name = os.path.basename(gsfile_path)

      # Download file to local machine.
      try:
        subprocess.call(['gsutil', 'cp', gsfile_path, folder_path])
      except subprocess.CalledProcessError:
        print ('gsutil error.  Try running this command outside of '
               'chroot?')
        output_str[board] = '%s: Could not run gsutil command.' % board
        return 1

      # Untar/rename files as needed.
      file_path = os.path.join(folder_path, file_name)
      if download_type == TEST:
        subprocess.call(['tar', '-xf', file_path, '-C', folder_path])
        os.remove(file_path)
      elif download_type == AUTOTEST:
        print '%s: running tar -xf command' % board
        subprocess.call(['tar', '-xf', file_path, '-C', folder_path])
        target_name = 'autotest'
        target_path = os.path.join(folder_path, file_name[:-len('.tar.bz2')],
                                   target_name)
        os.remove(file_path)
      elif download_type == FACTORY:
        print 'trying to unzip %s to %s' % (file_path, folder_path)
        subprocess.call(['unzip', '-q', file_path, '-d', folder_path])
        os.remove(file_path)
      else: # RECOVERY
        os.rename(file_path, os.path.join(folder_path, target_name))

    # Report successful download and return path to downloaded thing
    output_str[board] = target_path
    dl_error[board] = False
    print '%s: DONE' % board

  # For each board, download file.
  manager = Manager()
  output_str = manager.dict()
  dl_error = manager.dict()
  dl_folder = manager.dict()
  jobs = []
  for board in boards:
    # Run download in separate process.
    proc = Process(target=_DownloadBoard, args=(board, output_str, dl_error,
                                                dl_folder,))
    jobs.append(proc)
    proc.start()

  # Wait for all downloads to finish.
  for job in jobs:
    job.join()

  # Print output or run image_to_usb command.
  errors = ''
  if installing_one or installing_many:
    jobs = []
    for i in xrange(len(boards)):
      board = boards[i]
      # If board downloaded without errors, install.  Else, skip.
      if not dl_error[board]:
        path_to_script = os.path.join(src_dir, 'scripts', 'image_to_usb.sh')
        cmd = ('/usr/bin/sudo /bin/sh %s --from=%s'
               % (path_to_script, output_str[board]))
        # Copy this (i-th) board to i-th drive
        if installing_one:
          # If drive argument was provided, use it.  Else, leave it out.
          if arguments.to_stick:
            usb_drive = arguments.to_stick[i]
            cmd = '%s --to=%s -y' % (cmd, usb_drive)
            print 'Copying %s to %s.' % (board, usb_drive)
          proc = subprocess.Popen(cmd.split(' '))
          jobs.append(proc)
        # Copy this (only) board to all drives
        else:
          for usb_drive in arguments.to_many:
            cmd_per_usb = '%s --to=%s -y' % (cmd, usb_drive)
            print 'Copying %s to %s.' % (board, usb_drive)
            proc = subprocess.Popen(cmd_per_usb.split(' '))
            jobs.append(proc)
      else:
        errors += '%s\n' % output_str[board]
    # Wait for all copies to finish.
    for job in jobs:
      job.wait()
    if arguments.delete:
      print 'Deleting all files created for %s.' % board
      shutil.rmtree(dl_folder[board])
  else:
    print '\nDownloaded File(s):'
    for board in boards:
      if not dl_error[board]:
        print output_str[board]
      else:
        errors += '%s\n' % output_str[board]

  # Summarize errors, if any.
  print '\nScript complete.'
  print errors

if __name__ == '__main__':
  main()