#!/usr/bin/python

# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module to deal with result cache."""

import getpass
import glob
import hashlib
import os
import pickle
import re
import tempfile
import json
import sys

from utils import command_executer
from utils import misc

from image_checksummer import ImageChecksummer

SCRATCH_BASE = "/home/%s/cros_scratch"
SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
RESULTS_FILE = "results.txt"
MACHINE_FILE = "machine.txt"
AUTOTEST_TARBALL = "autotest.tbz2"
PERF_RESULTS_FILE = "perf-results.txt"
TELEMETRY_RESULT_DEFAULTS_FILE = "default-telemetry-results.json"

class Result(object):
  """ This class manages what exactly is stored inside the cache without knowing
  what the key of the cache is. For runs with perf, it stores perf.data,
  perf.report, etc. The key generation is handled by the ResultsCache class.
  """

  def __init__(self, logger, label):
    self._chromeos_root = label.chromeos_root
    self._logger = logger
    self._ce = command_executer.GetCommandExecuter(self._logger)
    self._temp_dir = None
    self.label = label
    self.results_dir = None
    self.perf_data_files = []
    self.perf_report_files = []

  def _CopyFilesTo(self, dest_dir, files_to_copy):
    file_index = 0
    for file_to_copy in files_to_copy:
      if not os.path.isdir(dest_dir):
        command = "mkdir -p %s" % dest_dir
        self._ce.RunCommand(command)
      dest_file = os.path.join(dest_dir,
                               ("%s.%s" % (os.path.basename(file_to_copy),
                                           file_index)))
      ret = self._ce.CopyFiles(file_to_copy,
                               dest_file,
                               recursive=False)
      if ret:
        raise Exception("Could not copy results file: %s" % file_to_copy)

  def CopyResultsTo(self, dest_dir):
    self._CopyFilesTo(dest_dir, self.perf_data_files)
    self._CopyFilesTo(dest_dir, self.perf_report_files)

  def _GetNewKeyvals(self, keyvals_dict):
    # Initialize 'units' dictionary.
    units_dict = {}
    for k in keyvals_dict:
      units_dict[k] = ""
    results_files = self._GetDataMeasurementsFiles()
    for f in results_files:
      # Make sure we can find the results file
      if os.path.exists(f):
        data_filename = f
      else:
        # Otherwise get the base filename and create the correct
        # path for it.
        f_dir, f_base = misc.GetRoot(f)
        data_filename = os.path.join(self._chromeos_root, "/tmp",
                                     self._temp_dir, f_base)
      if os.path.exists(data_filename):
        with open(data_filename, "r") as data_file:
          lines = data_file.readlines()
          for line in lines:
            tmp_dict = json.loads(line)
            key = tmp_dict["graph"] + "__" + tmp_dict["description"]
            keyvals_dict[key] = tmp_dict["value"]
            units_dict[key] = tmp_dict["units"]

    return keyvals_dict, units_dict


  def _GetTelemetryResultsKeyvals(self, keyvals_dict, units_dict):
    """
    keyvals_dict is the dictionary of key-value pairs that is used for
    generating Crosperf reports.

    Telemetry tests return many values (fields) that are not of
    interest, so we have created a json file that indicates, for each
    Telemetry benchmark, what the default return fields of interest
    are.

    units_dict is a dictionary of the units for the return values in
    keyvals_dict.  After looking for the keys in the keyvals_dict in
    the json file of "interesting" default return fields, we append
    the units to the name of the field, to make the report easier to
    understand.  We don't append the units to the results name earlier,
    because the units are not part of the field names in the json file.

    This function reads that file into a dictionary, and finds the
    entry for the current benchmark (if it exists).  The entry
    contains a list of return fields to use in the report.  For each
    field in the default list, we look for the field in the input
    keyvals_dict, and if we find it we copy the entry into our results
    dictionary. We then return the results dictionary, which gets used
    for actually generating the report.
    """


    # Check to see if telemetry_Crosperf succeeded; if not, there's no point
    # in going further...

    succeeded = False
    if "telemetry_Crosperf" in keyvals_dict:
      if keyvals_dict["telemetry_Crosperf"] == "PASS":
        succeeded = True

    if not succeeded:
      return keyvals_dict

    # Find the Crosperf directory, and look there for the telemetry
    # results defaults file, if it exists.
    results_dict = {}
    dirname, basename = misc.GetRoot(sys.argv[0])
    fullname = os.path.join(dirname, TELEMETRY_RESULT_DEFAULTS_FILE)
    if os.path.exists (fullname):
      # Slurp the file into a dictionary.  The keys in the dictionary are
      # the benchmark names.  The value for a key is a list containing the
      # names of all the result fields that should be returned in a 'default'
      # report.
      result_defaults = json.load(open(fullname))
      # Check to see if the current benchmark test actually has an entry in
      # the dictionary.
      if self.test_name and self.test_name in result_defaults:
        result_list = result_defaults[self.test_name]
        # We have the default results list.  Make sure it's not empty...
        if len(result_list) > 0:
          # ...look for each default result in the dictionary of actual
          # result fields returned. If found, add the field and its value
          # to our final results dictionary.
          for r in result_list:
            if r in keyvals_dict:
              val = keyvals_dict[r]
              units = units_dict[r]
              # Add the units to the key name, for the report.
              newkey = r + " (" + units + ")"
              results_dict[newkey] = val
    if len(results_dict) == 0:
      # We did not find/create any new entries.  Therefore use the keyvals_dict
      # that was passed in, but update the entry names to have the units.
      for k in keyvals_dict:
        val = keyvals_dict[k]
        units = units_dict[k]
        newkey = k + " (" + units + ")"
        results_dict[newkey] = val
    keyvals_dict = results_dict
    return keyvals_dict

  def _GetKeyvals(self, show_all):
    results_in_chroot = os.path.join(self._chromeos_root,
                                     "chroot", "tmp")
    if not self._temp_dir:
      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
      self._ce.RunCommand(command)

    command = ("python generate_test_report --no-color --csv %s" %
               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
                                            command,
                                            return_output=True)
    keyvals_dict = {}
    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
                                                 self._temp_dir)
    for line in out.splitlines():
      tokens = re.split("=|,", line)
      key = tokens[-2]
      if key.startswith(tmp_dir_in_chroot):
        key = key[len(tmp_dir_in_chroot) + 1:]
      value = tokens[-1]
      keyvals_dict[key] = value

    # Check to see if there is a perf_measurements file and get the
    # data from it if so.
    keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict)
    if not show_all and self.suite == "telemetry_Crosperf":
      # We're running telemetry tests and the user did not ask to
      # see all the results, so get the default results, to be used
      # for generating the report.
      keyvals_dict = self._GetTelemetryResultsKeyvals(keyvals_dict,
                                                      units_dict)
    return keyvals_dict

  def _GetResultsDir(self):
    mo = re.search(r"Results placed in (\S+)", self.out)
    if mo:
      result = mo.group(1)
      return result
    raise Exception("Could not find results directory.")

  def _FindFilesInResultsDir(self, find_args):
    if not self.results_dir:
      return None

    command = "find %s %s" % (self.results_dir,
                              find_args)
    ret, out, _ = self._ce.RunCommand(command, return_output=True)
    if ret:
      raise Exception("Could not run find command!")
    return out

  def _GetPerfDataFiles(self):
    return self._FindFilesInResultsDir("-name perf.data").splitlines()

  def _GetPerfReportFiles(self):
    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()

  def _GetDataMeasurementsFiles(self):
    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()

  def _GeneratePerfReportFiles(self):
    perf_report_files = []
    for perf_data_file in self.perf_data_files:
      # Generate a perf.report and store it side-by-side with the perf.data
      # file.
      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
                                                       perf_data_file)
      perf_report_file = "%s.report" % perf_data_file
      if os.path.exists(perf_report_file):
        raise Exception("Perf report file already exists: %s" %
                        perf_report_file)
      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
                                                         perf_report_file)
      perf_path = os.path.join (self._chromeos_root,
                                "chroot",
                                "usr/bin/perf")

      perf_file = "/usr/sbin/perf"
      if os.path.exists(perf_path):
        perf_file = "/usr/bin/perf"

      command = ("%s report "
                 "-n "
                 "--symfs /build/%s "
                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
                 "--kallsyms /build/%s/boot/System.map-* "
                 "-i %s --stdio "
                 "> %s" %
                 (perf_file,
                  self._board,
                  self._board,
                  self._board,
                  chroot_perf_data_file,
                  chroot_perf_report_file))
      self._ce.ChrootRunCommand(self._chromeos_root,
                                command)

      # Add a keyval to the dictionary for the events captured.
      perf_report_files.append(
          misc.GetOutsideChrootPath(self._chromeos_root,
                                    chroot_perf_report_file))
    return perf_report_files

  def _GatherPerfResults(self):
    report_id = 0
    for perf_report_file in self.perf_report_files:
      with open(perf_report_file, "r") as f:
        report_contents = f.read()
        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
          num_events = group[0]
          event_name = group[1]
          key = "perf_%s_%s" % (report_id, event_name)
          value = str(misc.UnitToNumber(num_events))
          self.keyvals[key] = value

  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
    self._board = self.label.board
    self.out = out
    self.err = err
    self.retval = retval
    self.test_name = test
    self.suite = suite
    self.chroot_results_dir = self._GetResultsDir()
    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
                                                 self.chroot_results_dir)
    self.perf_data_files = self._GetPerfDataFiles()
    # Include all perf.report data in table.
    self.perf_report_files = self._GeneratePerfReportFiles()
    # TODO(asharif): Do something similar with perf stat.

    # Grab keyvals from the directory.
    self._ProcessResults(show_all)

  def _ProcessResults(self, show_all):
    # Note that this function doesn't know anything about whether there is a
    # cache hit or miss. It should process results agnostic of the cache hit
    # state.
    self.keyvals = self._GetKeyvals(show_all)
    self.keyvals["retval"] = self.retval
    # Generate report from all perf.data files.
    # Now parse all perf report files and include them in keyvals.
    self._GatherPerfResults()

  def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite):
    self.test_name = test
    self.suite = suite
    # Read in everything from the cache directory.
    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
      self.out = pickle.load(f)
      self.err = pickle.load(f)
      self.retval = pickle.load(f)

    # Untar the tarball to a temporary directory
    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
                                                       "chroot", "tmp"))

    command = ("cd %s && tar xf %s" %
               (self._temp_dir,
                os.path.join(cache_dir, AUTOTEST_TARBALL)))
    ret = self._ce.RunCommand(command)
    if ret:
      raise Exception("Could not untar cached tarball")
    self.results_dir = self._temp_dir
    self.perf_data_files = self._GetPerfDataFiles()
    self.perf_report_files = self._GetPerfReportFiles()
    self._ProcessResults(show_all)

  def CleanUp(self, rm_chroot_tmp):
    if rm_chroot_tmp and self.results_dir:
      command = "rm -rf %s" % self.results_dir
      self._ce.RunCommand(command)
    if self._temp_dir:
      command = "rm -rf %s" % self._temp_dir
      self._ce.RunCommand(command)

  def StoreToCacheDir(self, cache_dir, machine_manager):
    # Create the dir if it doesn't exist.
    temp_dir = tempfile.mkdtemp()

    # Store to the temp directory.
    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
      pickle.dump(self.out, f)
      pickle.dump(self.err, f)
      pickle.dump(self.retval, f)

    if self.results_dir:
      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
      command = ("cd %s && "
                 "tar "
                 "--exclude=var/spool "
                 "--exclude=var/log "
                 "-cjf %s ." % (self.results_dir, tarball))
      ret = self._ce.RunCommand(command)
      if ret:
        raise Exception("Couldn't store autotest output directory.")
    # Store machine info.
    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
    # this function.
    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
      f.write(machine_manager.machine_checksum_string[self.label.name])

    if os.path.exists(cache_dir):
      command = "rm -rf {0}".format(cache_dir)
      self._ce.RunCommand(command)

    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
    command += "chmod g+x {0} && ".format(temp_dir)
    command += "mv {0} {1}".format(temp_dir, cache_dir)
    ret = self._ce.RunCommand(command)
    if ret:
      command = "rm -rf {0}".format(temp_dir)
      self._ce.RunCommand(command)
      raise Exception("Could not move dir %s to dir %s" %
                      (temp_dir, cache_dir))

  @classmethod
  def CreateFromRun(cls, logger, label, out, err, retval, show_all, test,
                    suite="pyauto"):
    if suite == "telemetry":
      result = TelemetryResult(logger, label)
    else:
      result = cls(logger, label)
    result._PopulateFromRun(out, err, retval, show_all, test, suite)
    return result

  @classmethod
  def CreateFromCacheHit(cls, logger, label, cache_dir, show_all, test,
                         suite="pyauto"):
    if suite == "telemetry":
      result = TelemetryResult(logger, label)
    else:
      result = cls(logger, label)
    try:
      result._PopulateFromCacheDir(cache_dir, show_all, test, suite)

    except Exception as e:
      logger.LogError("Exception while using cache: %s" % e)
      return None
    return result


class TelemetryResult(Result):

  def __init__(self, logger, label):
    super(TelemetryResult, self).__init__(logger, label)

  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
    self.out = out
    self.err = err
    self.retval = retval

    self._ProcessResults()

  def _ProcessResults(self):
    # The output is:
    # url,average_commit_time (ms),...
    # www.google.com,33.4,21.2,...
    # We need to convert to this format:
    # {"www.google.com:average_commit_time (ms)": "33.4",
    #  "www.google.com:...": "21.2"}
    # Added note:  Occasionally the output comes back
    # with "JSON.stringify(window.automation.GetResults())" on
    # the first line, and then the rest of the output as
    # described above.

    lines = self.out.splitlines()
    self.keyvals = {}

    if lines:
      if lines[0].startswith("JSON.stringify"):
        lines = lines[1:]

    if not lines:
      return
    labels = lines[0].split(",")
    for line in lines[1:]:
      fields = line.split(",")
      if len(fields) != len(labels):
        continue
      for i in range(1, len(labels)):
        key = "%s %s" % (fields[0], labels[i])
        value = fields[i]
        self.keyvals[key] = value
    self.keyvals["retval"] = self.retval

  def _PopulateFromCacheDir(self, cache_dir):
    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
      self.out = pickle.load(f)
      self.err = pickle.load(f)
      self.retval = pickle.load(f)
    self._ProcessResults()


class CacheConditions(object):
  # Cache hit only if the result file exists.
  CACHE_FILE_EXISTS = 0

  # Cache hit if the checksum of cpuinfo and totalmem of
  # the cached result and the new run match.
  MACHINES_MATCH = 1

  # Cache hit if the image checksum of the cached result and the new run match.
  CHECKSUMS_MATCH = 2

  # Cache hit only if the cached result was successful
  RUN_SUCCEEDED = 3

  # Never a cache hit.
  FALSE = 4

  # Cache hit if the image path matches the cached image path.
  IMAGE_PATH_MATCH = 5

  # Cache hit if the uuid of hard disk mataches the cached one

  SAME_MACHINE_MATCH = 6


class ResultsCache(object):

  """ This class manages the key of the cached runs without worrying about what
  is exactly stored (value). The value generation is handled by the Results
  class.
  """
  CACHE_VERSION = 6

  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
           test_args, profiler_args, machine_manager, board, cache_conditions,
           logger_to_use, label, share_users, suite, show_all_results):
    self.chromeos_image = chromeos_image
    self.chromeos_root = chromeos_root
    self.test_name = test_name
    self.iteration = iteration
    self.test_args = test_args
    self.profiler_args = profiler_args
    self.board = board
    self.cache_conditions = cache_conditions
    self.machine_manager = machine_manager
    self._logger = logger_to_use
    self._ce = command_executer.GetCommandExecuter(self._logger)
    self.label = label
    self.share_users = share_users
    self.suite = suite
    self.show_all = show_all_results

  def _GetCacheDirForRead(self):
    matching_dirs = []
    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
      matching_dirs += glob.glob(glob_path)

    if matching_dirs:
      # Cache file found.
      return matching_dirs[0]
    else:
      return None

  def _GetCacheDirForWrite(self):
    return self._FormCacheDir(self._GetCacheKeyList(False))[0]

  def _FormCacheDir(self, list_of_strings):
    cache_key = " ".join(list_of_strings)
    cache_dir = misc.GetFilenameFromString(cache_key)
    if self.label.cache_dir:
      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
      cache_path = [os.path.join(cache_home, cache_dir)]
    else:
      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]

    for i in [x.strip() for x in self.share_users.split(",")]:
      path = SCRATCH_BASE % i
      cache_path.append(os.path.join(path, cache_dir))

    return cache_path

  def _GetCacheKeyList(self, read):
    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
      machine_checksum = "*"
    else:
      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
      checksum = "*"
    elif self.label.image_type == "trybot":
      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
    elif self.label.image_type == "official":
      checksum = "*"
    else:
      checksum = ImageChecksummer().Checksum(self.label)

    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
      image_path_checksum = "*"
    else:
      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()

    machine_id_checksum = ""
    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
      machine_id_checksum = "*"
    else:
      for machine in self.machine_manager.GetMachines(self.label):
        if machine.name == self.label.remote[0]:
          machine_id_checksum = machine.machine_id_checksum
          break

    temp_test_args = "%s %s" % (self.test_args, self.profiler_args)
    test_args_checksum = hashlib.md5(
        "".join(temp_test_args)).hexdigest()
    return (image_path_checksum,
            self.test_name, str(self.iteration),
            test_args_checksum,
            checksum,
            machine_checksum,
            machine_id_checksum,
            str(self.CACHE_VERSION))

  def ReadResult(self):
    if CacheConditions.FALSE in self.cache_conditions:
      cache_dir = self._GetCacheDirForWrite()
      command = "rm -rf {0}".format(cache_dir)
      self._ce.RunCommand(command)
      return None
    cache_dir = self._GetCacheDirForRead()

    if not cache_dir:
      return None

    if not os.path.isdir(cache_dir):
      return None

    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
    result = Result.CreateFromCacheHit(self._logger,
                                       self.label,
                                       cache_dir,
                                       self.show_all,
                                       self.test_name,
                                       self.suite)
    if not result:
      return None

    if (result.retval == 0 or
        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
      return result

    return None

  def StoreResult(self, result):
    cache_dir = self._GetCacheDirForWrite()
    result.StoreToCacheDir(cache_dir, self.machine_manager)


class MockResultsCache(ResultsCache):
  def Init(self, *args):
    pass

  def ReadResult(self):
    return None

  def StoreResult(self, result):
    pass


class MockResult(Result):
  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
    self.out = out
    self.err = err
    self.retval = retval