site_utils/admin/clean_staged_images.py

#!/usr/bin/python
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Clean Staged Images.

This script is responsible for removing older builds from the Chrome OS
devserver as well as old cros-version: labels from the Autotest DB.

There are two different options to remove builds, one is for "regular builds"
which is builds that happen around 4 times a day, and the other option is for
"Paladin builds" which are builds that happen as part of the Commit Queue and
are far more frequent. Since Paladin builds are only used for the BVT HWTest
Step we can be more agressive when it comes to pruning these builds.
"""

from distutils import version
import optparse
import glob
import logging
import os
import re
import sys
import shutil
import time

import common
from autotest_lib.frontend.afe import rpc_client_lib
from autotest_lib.cli.rpc import AFE_RPC_PATH
from autotest_lib.client.common_lib import global_config
from autotest_lib.server.cros.dynamic_suite.constants import VERSION_PREFIX

# This filename must be kept in sync with devserver's downloader.py
_TIMESTAMP_FILENAME = 'staged.timestamp'
_HOURS_TO_SECONDS = 60*60
_EXEMPTED_DIRECTORIES = [ 'servo-images' ]


def validate_and_parse_build_milestone(build_name):
    """Parse the build name and ensure it is a proper build name.

    Example build name: R21-4555.2.3 or R21-2368.0.0-rc30

    @param build_name The name of the build.
    @returns the milestone of the build if it is valid.

    >>> validate_and_parse_build_milestone('R21-4555.2.3')
    'R21'
    >>> validate_and_parse_build_milestone('R21-2368.0.0-rc30')
    'R21'
    """
    pattern = '(R\d+)-(\d+\.\d+\.\d+)'
    match = re.search(pattern, build_name)
    if match:
        return match.group(1)


def sort_builds(build_list):
    """Sort a list of builds of format R21-121.0.0 or R21-121.0.0-rc3.

    @param build_list: The list of builds to sort.
    @returns a list of builds sorted in reverse order.
    """
    return sorted(build_list, key=lambda s: version.LooseVersion(s),
                  reverse=True)


def file_is_too_old(build_path, max_age_hours):
    """Test to see if the build at |build_path| is older than |max_age_hours|.

    @param build_path The path to the build (ie. 'build_dir/R21-2035.0.0')
    @param max_age_hours The maximum allowed age of a build in hours.
    @return True if the build is older than |max_age|, False otherwise.
    """
    cutoff = time.time() - max_age_hours * _HOURS_TO_SECONDS
    timestamp_path = os.path.join(build_path, _TIMESTAMP_FILENAME)
    if os.path.exists(timestamp_path):
        age = os.stat(timestamp_path).st_mtime
        if age < cutoff:
            return True
    else:
        # If the timestamp doesn't exist, then make one so we can wipe the
        # build on a later run.
        with open(timestamp_path, 'a'):
            os.utime(timestamp_path, None)
    return False


def prune_builds(build_dir, max_age_hours):
    """Remove any builds older than |max_age_hours| hours old.

    Prune a directory down to only having builds last used sooner than
    |max_age_hours| seconds ago. This will prune down all milestones to only
    having fresh builds.

    @param build_dir: The build dir to prune builds in.
    @param max_age_hours: The max age in hours of builds to keep around.
    """
    logging.debug('Pruning %s down to builds older than %d hours ago.',
                  build_dir, max_age_hours)
    build_dict = {}
    # Create a dict that is of the format:
    # build_dict[milestone][build_name] = build_path
    # e.g. {'R21' : {'R21-2056.0.0': 'build_dir/R21-2056.0.0',
    #                'R21-2035.0.0': 'build_dir/R21-2035.0.0'}
    for entry in glob.glob(build_dir + '/*'):
        build_name = os.path.basename(entry)
        milestone = validate_and_parse_build_milestone(build_name)
        if not milestone:
            logging.debug('Skipping %s', build_name)
            continue
        build_dict.setdefault(milestone, {})[os.path.basename(entry)] = entry

    for milestone in build_dict:
        for (entry, path) in build_dict[milestone].items():
            if file_is_too_old(path, max_age_hours):
                logging.debug('Deleting %s', path)
                shutil.rmtree(path)


def delete_labels_of_unstaged_builds(build_dir):
    """Delete labels of builds that are no longer staged on the dev server.

    @param build_dir: The build dir to compare known labels against.
    """
    autotest_server = global_config.global_config.get_config_value('SERVER',
                                                                   'hostname')
    web_server = 'http://%s/%s' % (autotest_server, AFE_RPC_PATH)
    rpc_interface = rpc_client_lib.get_proxy(web_server)
    builder_name = os.path.basename(build_dir.rstrip('/')).split('/')[-1]
    vers = '%s%s/' % (VERSION_PREFIX, builder_name)
    all_labels = rpc_interface.get_labels(name__startswith=vers)
    # Build a list of cros-version:build_name/CROS_VERSION.
    # e.g. cros-version:x86-mario-release/R20-2268.41.0
    build_labels_to_keep = [vers + os.path.basename(entry) for entry in
                    glob.glob(build_dir + '/*')]
    for label in all_labels:
        if label['name'] not in build_labels_to_keep:
            logging.debug('Removing label %s', label['name'])
            rpc_interface.delete_label(label['id'])


def prune_builds_and_labels(builds_dir, keep_duration, keep_paladin_duration):
    """Prune the build dirs and also delete old labels.

    @param builds_dir: The builds dir where all builds are staged.
      on the chromeos-devserver this is ~chromeos-test/images/
    @param keep_duration: How old of regular builds to keep around.
    @param keep_paladin_duration: How old of Paladin builds to keep around.
    """
    if not os.path.exists(builds_dir):
        logging.error('Builds dir %s does not exist', builds_dir)
        return

    for build_dir in glob.glob(builds_dir + '/*'):
        if os.path.basename(build_dir) in _EXEMPTED_DIRECTORIES:
            logging.debug('Skipping %s', build_dir)
            continue
        logging.debug('Processing %s', build_dir)
        if build_dir.endswith('-paladin'):
            keep = keep_paladin_duration
        else:
            keep = keep_duration

        prune_builds(build_dir, keep)
        # TODO(scottz): Skip label pruning for now.
        # Refer to bug crosbug.com/33247
        # delete_labels_of_unstaged_builds(build_dir)

def main():
    """Main routine."""
    usage = 'usage: %prog [options] images_dir'
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-a', '--max-age', default=24, type=int,
                      help='Number of hours to keep normal builds: %default')
    parser.add_option('-p', '--max-paladin-age', default=24, type=int,
                      help='Number of hours to keep paladin builds: %default')
    parser.add_option('-v', '--verbose',
                      dest='verbose', action='store_true', default=False,
                      help='Run in verbose mode')
    options, args = parser.parse_args()
    if len(args) != 1:
      parser.print_usage()
      sys.exit(1)

    if options.verbose:
        logging.getLogger().setLevel(logging.DEBUG)
    else:
        logging.getLogger().setLevel(logging.INFO)

    prune_builds_and_labels(args[0], options.max_age,
                            options.max_paladin_age)


if __name__ == '__main__':
    main()