site_utils/gs_offloader.py

#!/usr/bin/python
#
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

__help__ = """Script to archive old Autotest results to Google Storage.

Uses gsutil to archive files to the configured Google Storage bucket. Upon
successful copy, the local results directory is deleted.
"""

__author__ = 'dalecurtis@google.com (Dale Curtis)'

import os
import re
import shutil
import subprocess
import sys
import time


# Google Storage bucket URI to store results in.
GS_URI = 'gs://chromeos-autotest-results'

# Maximum age in days; all older results will be archived.
MAX_AGE_DAYS = 7

# Nice setting for process, the higher the number the lower the priority.
NICENESS = 10

# Location of Autotest results on disk.
RESULTS_DIR = '/usr/local/autotest/results'


def main():
  max_age = MAX_AGE_DAYS
  if len(sys.argv) > 1:
    if sys.argv[1].lower() in ('--help', '-h', '-?'):
      print __help__
      print 'Defaults:'
      print '  Destination: ' + GS_URI
      print '  Max Log Age: %s days' % max_age
      print '  Results path: ' + RESULTS_DIR
      print '\nUsage:'
      print '  ./gs_offloader.py [max log age in days]\n'
      sys.exit(0)
    else:
      max_age = int(sys.argv[1])

  max_age = time.time() - 60 * 60 * 24 * max_age

  # Nice our process (carried to subprocesses) so we don't kill the system.
  os.nice(NICENESS)

  # os.listdir returns relative paths, so change to where we need to be to avoid
  # an os.path.join on each loop.
  os.chdir(RESULTS_DIR)

  # Only pick up directories of the form <job #>-<job user>.
  job_matcher = re.compile('^\d+-\w+')

  # Iterate over all directories in RESULTS_DIR.
  for d in os.listdir('.'):
    if (job_matcher.match(d) and os.path.isdir(d)
        and os.stat(d).st_mtime < max_age):
      # Update command list.
      cmd_list = ['gsutil', '-m', 'cp', '-eR', '-a', 'project-private', d,
                  GS_URI]

      # Save stdout and stderr in case of failure.
      process = subprocess.Popen(
          cmd_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
      output = process.communicate()[0]

      if process.returncode == 0:
        # Everything copied okay, so remove the local directory.
        shutil.rmtree(d)
      else:
        # Copy failed, dump logs and continue to the next directory.
        print output


if __name__ == '__main__':
  main()