| #!/usr/bin/python2 -u |
| |
| """ |
| Utility to upload or remove the packages from the packages repository. |
| """ |
| |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| import logging, optparse, os, shutil, sys, tempfile |
| import common |
| from autotest_lib.client.common_lib import utils as client_utils |
| from autotest_lib.client.common_lib import global_config, error |
| from autotest_lib.client.common_lib import packages |
| from autotest_lib.server import utils as server_utils |
| |
| c = global_config.global_config |
| logging.basicConfig(level=logging.DEBUG) |
| |
| ACTION_REMOVE = 'remove' |
| ACTION_UPLOAD = 'upload' |
| ACTION_TAR_ONLY = 'tar_only' |
| |
| def get_exclude_string(client_dir): |
| ''' |
| Get the exclude string for the tar command to exclude specific |
| subdirectories inside client_dir. |
| For profilers we need to exclude everything except the __init__.py |
| file so that the profilers can be imported. |
| ''' |
| exclude_string = ('--exclude="deps/*" --exclude="tests/*" --exclude=.git ' |
| '--exclude="site_tests/*" --exclude="**.pyc"') |
| |
| # Get the profilers directory |
| prof_dir = os.path.join(client_dir, 'profilers') |
| |
| # Include the __init__.py file for the profilers and exclude all its |
| # subdirectories |
| for f in os.listdir(prof_dir): |
| if os.path.isdir(os.path.join(prof_dir, f)): |
| exclude_string += ' --exclude="profilers/%s"' % f |
| |
| # The '.' here is needed to zip the files in the current |
| # directory. We use '-C' for tar to change to the required |
| # directory i.e. src_dir and then zip up the files in that |
| # directory(which is '.') excluding the ones in the exclude_dirs |
| exclude_string += " ." |
| |
| # TODO(milleral): This is sad and ugly. http://crbug.com/258161 |
| # Surprisingly, |exclude_string| actually means argument list, and |
| # we'd like to package up the current global_config.ini also, so let's |
| # just tack it on here. |
| # Also note that this only works because tar prevents us from un-tarring |
| # files into parent directories. |
| exclude_string += " ../global_config.ini" |
| |
| return exclude_string |
| |
| |
| def parse_args(): |
| parser = optparse.OptionParser() |
| parser.add_option("-d", "--dependency", help="package the dependency" |
| " from client/deps directory and upload to the repo", |
| dest="dep") |
| parser.add_option("-p", "--profiler", help="package the profiler " |
| "from client/profilers directory and upload to the repo", |
| dest="prof") |
| parser.add_option("-t", "--test", help="package the test from client/tests" |
| " or client/site_tests and upload to the repo.", |
| dest="test") |
| parser.add_option("-c", "--client", help="package the client " |
| "directory alone without the tests, deps and profilers", |
| dest="client", action="store_true", default=False) |
| parser.add_option("-f", "--file", help="simply uploads the specified" |
| "file on to the repo", dest="file") |
| parser.add_option("-r", "--repository", help="the URL of the packages" |
| "repository location to upload the packages to.", |
| dest="repo", default=None) |
| parser.add_option("-o", "--output_dir", help="the output directory" |
| "to place tarballs and md5sum files in.", |
| dest="output_dir", default=None) |
| parser.add_option("-a", "--action", help="the action to perform", |
| dest="action", choices=(ACTION_UPLOAD, ACTION_REMOVE, |
| ACTION_TAR_ONLY), default=None) |
| parser.add_option("--all", help="Upload all the files locally " |
| "to all the repos specified in global_config.ini. " |
| "(includes the client, tests, deps and profilers)", |
| dest="all", action="store_true", default=False) |
| |
| options, args = parser.parse_args() |
| return options, args |
| |
| def get_build_dir(name, dest_dir, pkg_type): |
| """Method to generate the build directory where the tarball and checksum |
| is stored. The following package types are handled: test, dep, profiler. |
| Package type 'client' is not handled. |
| """ |
| if pkg_type == 'client': |
| # NOTE: The "tar_only" action for pkg_type "client" has no use |
| # case yet. No known invocations of packager.py with |
| # --action=tar_only send in clients in the command line. Please |
| # confirm the behaviour is expected before this type is enabled for |
| # "tar_only" actions. |
| print('Tar action not supported for pkg_type= %s, name = %s' % |
| pkg_type, name) |
| return None |
| # For all packages, the work-dir should have 'client' appended to it. |
| base_build_dir = os.path.join(dest_dir, 'client') |
| if pkg_type == 'test': |
| build_dir = os.path.join(get_test_dir(name, base_build_dir), name) |
| else: |
| # For profiler and dep, we append 's', and then append the name. |
| # TODO(pmalani): Make this less fiddly? |
| build_dir = os.path.join(base_build_dir, pkg_type + 's', name) |
| return build_dir |
| |
| def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, |
| action, dest_dir=None): |
| """Method to upload or remove package depending on the flag passed to it. |
| |
| If tar_only is set to True, this routine is solely used to generate a |
| tarball and compute the md5sum from that tarball. |
| If the tar_only flag is True, then the remove flag is ignored. |
| """ |
| exclude_string = ' .' |
| names = [p.strip() for p in pkg_names.split(',')] |
| for name in names: |
| print("process_packages: Processing %s ... " % name) |
| if pkg_type == 'client': |
| pkg_dir = src_dir |
| exclude_string = get_exclude_string(pkg_dir) |
| elif pkg_type == 'test': |
| # if the package is a test then look whether it is in client/tests |
| # or client/site_tests |
| pkg_dir = os.path.join(get_test_dir(name, src_dir), name) |
| else: |
| # for the profilers and deps |
| pkg_dir = os.path.join(src_dir, name) |
| |
| pkg_name = pkgmgr.get_tarball_name(name, pkg_type) |
| |
| exclude_string_tar = (( |
| ' --exclude="**%s" --exclude="**%s.checksum" ' % |
| (pkg_name, pkg_name)) + exclude_string) |
| if action == ACTION_TAR_ONLY: |
| # We don't want any pre-existing tarballs and checksums to |
| # be repackaged, so we should purge these. |
| build_dir = get_build_dir(name, dest_dir, pkg_type) |
| try: |
| packages.check_diskspace(build_dir) |
| except error.RepoDiskFullError as e: |
| msg = ("Work_dir directory for packages %s does not have " |
| "enough space available: %s" % (build_dir, e)) |
| raise error.RepoDiskFullError(msg) |
| tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, |
| build_dir, exclude_string_tar) |
| |
| # Create the md5 hash too. |
| md5sum = pkgmgr.compute_checksum(tarball_path) |
| md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum') |
| with open(md5sum_filepath, "w") as f: |
| f.write(md5sum) |
| |
| elif action == ACTION_UPLOAD: |
| # Tar the source and upload |
| temp_dir = tempfile.mkdtemp() |
| try: |
| try: |
| packages.check_diskspace(temp_dir) |
| except error.RepoDiskFullError as e: |
| msg = ("Temporary directory for packages %s does not have " |
| "enough space available: %s" % (temp_dir, e)) |
| raise error.RepoDiskFullError(msg) |
| |
| # Check if tarball already exists. If it does, then don't |
| # create a tarball again. |
| tarball_path = os.path.join(pkg_dir, pkg_name); |
| if os.path.exists(tarball_path): |
| print("process_packages: Tarball %s already exists" % |
| tarball_path) |
| else: |
| tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, |
| temp_dir, |
| exclude_string_tar) |
| # Compare the checksum with what packages.checksum has. If they |
| # match then we don't need to perform the upload. |
| if not pkgmgr.compare_checksum(tarball_path): |
| pkgmgr.upload_pkg(tarball_path, update_checksum=True) |
| else: |
| logging.warning('Checksum not changed for %s, not copied ' |
| 'in packages/ directory.', tarball_path) |
| finally: |
| # remove the temporary directory |
| shutil.rmtree(temp_dir) |
| elif action == ACTION_REMOVE: |
| pkgmgr.remove_pkg(pkg_name, remove_checksum=True) |
| print("Done.") |
| |
| |
| def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir): |
| """Tar all packages up and return a list of each tar created""" |
| tarballs = [] |
| exclude_string = ' .' |
| names = [p.strip() for p in pkg_names.split(',')] |
| for name in names: |
| print("tar_packages: Processing %s ... " % name) |
| if pkg_type == 'client': |
| pkg_dir = src_dir |
| exclude_string = get_exclude_string(pkg_dir) |
| elif pkg_type == 'test': |
| # if the package is a test then look whether it is in client/tests |
| # or client/site_tests |
| pkg_dir = os.path.join(get_test_dir(name, src_dir), name) |
| else: |
| # for the profilers and deps |
| pkg_dir = os.path.join(src_dir, name) |
| |
| pkg_name = pkgmgr.get_tarball_name(name, pkg_type) |
| |
| # We don't want any pre-existing tarballs and checksums to |
| # be repackaged, so we should purge these. |
| exclude_string_tar = (( |
| ' --exclude="**%s" --exclude="**%s.checksum" ' % |
| (pkg_name, pkg_name)) + exclude_string) |
| # Check if tarball already exists. If it does, don't duplicate |
| # the effort. |
| tarball_path = os.path.join(pkg_dir, pkg_name) |
| if os.path.exists(tarball_path): |
| print("tar_packages: Tarball %s already exists" % tarball_path) |
| else: |
| tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, |
| temp_dir, exclude_string_tar) |
| tarballs.append(tarball_path) |
| return tarballs |
| |
| |
| def process_all_packages(pkgmgr, client_dir, action): |
| """Process a full upload of packages as a directory upload.""" |
| dep_dir = os.path.join(client_dir, "deps") |
| prof_dir = os.path.join(client_dir, "profilers") |
| # Directory where all are kept |
| temp_dir = tempfile.mkdtemp() |
| try: |
| packages.check_diskspace(temp_dir) |
| except error.RepoDiskFullError as e: |
| print("Temp destination for packages is full %s, aborting upload: %s" |
| % (temp_dir, e)) |
| os.rmdir(temp_dir) |
| sys.exit(1) |
| |
| # process tests |
| tests_list = get_subdir_list('tests', client_dir) |
| tests = ','.join(tests_list) |
| |
| # process site_tests |
| site_tests_list = get_subdir_list('site_tests', client_dir) |
| site_tests = ','.join(site_tests_list) |
| |
| # process deps |
| deps_list = get_subdir_list('deps', client_dir) |
| deps = ','.join(deps_list) |
| |
| # process profilers |
| profilers_list = get_subdir_list('profilers', client_dir) |
| profilers = ','.join(profilers_list) |
| |
| # Update md5sum |
| if action == ACTION_UPLOAD: |
| all_packages = [] |
| all_packages.extend(tar_packages(pkgmgr, 'profiler', profilers, |
| prof_dir, temp_dir)) |
| all_packages.extend(tar_packages(pkgmgr, 'dep', deps, dep_dir, |
| temp_dir)) |
| all_packages.extend(tar_packages(pkgmgr, 'test', site_tests, |
| client_dir, temp_dir)) |
| all_packages.extend(tar_packages(pkgmgr, 'test', tests, client_dir, |
| temp_dir)) |
| all_packages.extend(tar_packages(pkgmgr, 'client', 'autotest', |
| client_dir, temp_dir)) |
| for package in all_packages: |
| pkgmgr.upload_pkg(package, update_checksum=True) |
| client_utils.run('rm -rf ' + temp_dir) |
| elif action == ACTION_REMOVE: |
| process_packages(pkgmgr, 'test', tests, client_dir, action=action) |
| process_packages(pkgmgr, 'test', site_tests, client_dir, action=action) |
| process_packages(pkgmgr, 'client', 'autotest', client_dir, |
| action=action) |
| process_packages(pkgmgr, 'dep', deps, dep_dir, action=action) |
| process_packages(pkgmgr, 'profiler', profilers, prof_dir, |
| action=action) |
| |
| |
| # Get the list of sub directories present in a directory |
| def get_subdir_list(name, client_dir): |
| dir_name = os.path.join(client_dir, name) |
| return [f for f in |
| os.listdir(dir_name) |
| if os.path.isdir(os.path.join(dir_name, f)) ] |
| |
| |
| # Look whether the test is present in client/tests and client/site_tests dirs |
| def get_test_dir(name, client_dir): |
| names_test = os.listdir(os.path.join(client_dir, 'tests')) |
| names_site_test = os.listdir(os.path.join(client_dir, 'site_tests')) |
| if name in names_test: |
| src_dir = os.path.join(client_dir, 'tests') |
| elif name in names_site_test: |
| src_dir = os.path.join(client_dir, 'site_tests') |
| else: |
| print("Test %s not found" % name) |
| sys.exit(0) |
| return src_dir |
| |
| |
| def main(): |
| # get options and args |
| options, args = parse_args() |
| |
| server_dir = server_utils.get_server_dir() |
| autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) |
| |
| # extract the pkg locations from global config |
| repo_urls = c.get_config_value('PACKAGES', 'fetch_location', |
| type=list, default=[]) |
| upload_paths = c.get_config_value('PACKAGES', 'upload_location', |
| type=list, default=[]) |
| |
| if options.repo: |
| upload_paths.append(options.repo) |
| |
| # Having no upload paths basically means you're not using packaging. |
| if not upload_paths: |
| print("No upload locations found. Please set upload_location under" |
| " PACKAGES in the global_config.ini or provide a location using" |
| " the --repository option.") |
| return |
| |
| client_dir = os.path.join(autotest_dir, "client") |
| |
| # Bail out if the client directory does not exist |
| if not os.path.exists(client_dir): |
| sys.exit(0) |
| |
| dep_dir = os.path.join(client_dir, "deps") |
| prof_dir = os.path.join(client_dir, "profilers") |
| |
| # Due to the delayed uprev-ing of certain ebuilds, we need to support |
| # both the legacy command line and the new one. |
| # So, if the new "action" option isn't specified, try looking for the |
| # old style remove/upload argument |
| if options.action is None: |
| if len(args) == 0 or args[0] not in ['upload', 'remove']: |
| print("Either 'upload' or 'remove' needs to be specified " |
| "for the package") |
| sys.exit(0) |
| cur_action = args[0] |
| else: |
| cur_action = options.action |
| |
| if cur_action == ACTION_TAR_ONLY and options.output_dir is None: |
| print("An output dir has to be specified") |
| sys.exit(0) |
| |
| pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, |
| upload_paths=upload_paths, |
| run_function_dargs={'timeout':600}) |
| |
| if options.all: |
| process_all_packages(pkgmgr, client_dir, action=cur_action) |
| |
| if options.client: |
| process_packages(pkgmgr, 'client', 'autotest', client_dir, |
| action=cur_action) |
| |
| if options.dep: |
| process_packages(pkgmgr, 'dep', options.dep, dep_dir, |
| action=cur_action, dest_dir=options.output_dir) |
| |
| if options.test: |
| process_packages(pkgmgr, 'test', options.test, client_dir, |
| action=cur_action, dest_dir=options.output_dir) |
| |
| if options.prof: |
| process_packages(pkgmgr, 'profiler', options.prof, prof_dir, |
| action=cur_action, dest_dir=options.output_dir) |
| |
| if options.file: |
| if cur_action == ACTION_REMOVE: |
| pkgmgr.remove_pkg(options.file, remove_checksum=True) |
| elif cur_action == ACTION_UPLOAD: |
| pkgmgr.upload_pkg(options.file, update_checksum=True) |
| |
| |
| if __name__ == "__main__": |
| main() |
