lib/portage/_emirrordist/main.py

# Copyright 2013-2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

import argparse
import logging
import sys

import portage
from portage import os
from portage.util import normalize_path, writemsg_level, _recursive_file_list
from portage.util._async.run_main_scheduler import run_main_scheduler
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.util._eventloop.global_event_loop import global_event_loop
from .Config import Config
from .MirrorDistTask import MirrorDistTask

if sys.hexversion >= 0x3000000:
	# pylint: disable=W0622
	long = int

seconds_per_day = 24 * 60 * 60

common_options = (
	{
		"longopt"  : "--dry-run",
		"help"     : "perform a trial run with no changes made (usually combined "
			"with --verbose)",
		"action"   : "store_true"
	},
	{
		"longopt"  : "--verbose",
		"shortopt" : "-v",
		"help"     : "display extra information on stderr "
			"(multiple occurences increase verbosity)",
		"action"   : "count",
		"default"  : 0,
	},
	{
		"longopt"  : "--ignore-default-opts",
		"help"     : "do not use the EMIRRORDIST_DEFAULT_OPTS environment variable",
		"action"   : "store_true"
	},
	{
		"longopt"  : "--distfiles",
		"help"     : "distfiles directory to use (required)",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--jobs",
		"shortopt" : "-j",
		"help"     : "number of concurrent jobs to run",
		"type"     : int
	},
	{
		"longopt"  : "--load-average",
		"shortopt" : "-l",
		"help"     : "load average limit for spawning of new concurrent jobs",
		"metavar"  : "LOAD",
		"type"     : float
	},
	{
		"longopt"  : "--tries",
		"help"     : "maximum number of tries per file, 0 means unlimited (default is 10)",
		"default"  : 10,
		"type"     : int
	},
	{
		"longopt"  : "--repo",
		"help"     : "name of repo to operate on"
	},
	{
		"longopt"  : "--config-root",
		"help"     : "location of portage config files",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--repositories-configuration",
		"help"     : "override configuration of repositories (in format of repos.conf)"
	},
	{
		"longopt"  : "--strict-manifests",
		"help"     : "manually override \"strict\" FEATURES setting",
		"choices"  : ("y", "n"),
		"metavar"  : "<y|n>",
	},
	{
		"longopt"  : "--failure-log",
		"help"     : "log file for fetch failures, with tab-delimited "
			"output, for reporting purposes",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--success-log",
		"help"     : "log file for fetch successes, with tab-delimited "
			"output, for reporting purposes",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--scheduled-deletion-log",
		"help"     : "log file for scheduled deletions, with tab-delimited "
			"output, for reporting purposes",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--delete",
		"help"     : "enable deletion of unused distfiles",
		"action"   : "store_true"
	},
	{
		"longopt"  : "--deletion-db",
		"help"     : "database file used to track lifetime of files "
			"scheduled for delayed deletion",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--deletion-delay",
		"help"     : "delay time for deletion, measured in seconds",
		"metavar"  : "SECONDS"
	},
	{
		"longopt"  : "--temp-dir",
		"help"     : "temporary directory for downloads",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--mirror-overrides",
		"help"     : "file holding a list of mirror overrides",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--mirror-skip",
		"help"     : "comma delimited list of mirror targets to skip "
			"when fetching"
	},
	{
		"longopt"  : "--restrict-mirror-exemptions",
		"help"     : "comma delimited list of mirror targets for which to "
			"ignore RESTRICT=\"mirror\""
	},
	{
		"longopt"  : "--verify-existing-digest",
		"help"     : "use digest as a verification of whether existing "
			"distfiles are valid",
		"action"   : "store_true"
	},
	{
		"longopt"  : "--distfiles-local",
		"help"     : "distfiles-local directory to use",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--distfiles-db",
		"help"     : "database file used to track which ebuilds a "
			"distfile belongs to",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--recycle-dir",
		"help"     : "directory for extended retention of files that "
			"are removed from distdir with the --delete option",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--recycle-db",
		"help"     : "database file used to track lifetime of files "
			"in recycle dir",
		"metavar"  : "FILE"
	},
	{
		"longopt"  : "--recycle-deletion-delay",
		"help"     : "delay time for deletion of unused files from "
			"recycle dir, measured in seconds (defaults to "
			"the equivalent of 60 days)",
		"default"  : 60 * seconds_per_day,
		"metavar"  : "SECONDS",
		"type"     : int
	},
	{
		"longopt"  : "--fetch-log-dir",
		"help"     : "directory for individual fetch logs",
		"metavar"  : "DIR"
	},
	{
		"longopt"  : "--whitelist-from",
		"help"     : "specifies a file containing a list of files to "
			"whitelist, one per line, # prefixed lines ignored",
		"action"   : "append",
		"metavar"  : "FILE"
	},
)

def parse_args(args):
	description = "emirrordist - a fetch tool for mirroring " \
		"of package distfiles"
	usage = "emirrordist [options] <action>"
	parser = argparse.ArgumentParser(description=description, usage=usage)

	actions = parser.add_argument_group('Actions')
	actions.add_argument("--version",
		action="store_true",
		help="display portage version and exit")
	actions.add_argument("--mirror",
		action="store_true",
		help="mirror distfiles for the selected repository")

	common = parser.add_argument_group('Common options')
	for opt_info in common_options:
		opt_pargs = [opt_info["longopt"]]
		if opt_info.get("shortopt"):
			opt_pargs.append(opt_info["shortopt"])
		opt_kwargs = {"help" : opt_info["help"]}
		for k in ("action", "choices", "default", "metavar", "type"):
			if k in opt_info:
				opt_kwargs[k] = opt_info[k]
		common.add_argument(*opt_pargs, **opt_kwargs)

	options, args = parser.parse_known_args(args)

	return (parser, options, args)

def emirrordist_main(args):

	# The calling environment is ignored, so the program is
	# completely controlled by commandline arguments.
	env = {}

	if not sys.stdout.isatty():
		portage.output.nocolor()
		env['NOCOLOR'] = 'true'

	parser, options, args = parse_args(args)

	if options.version:
		sys.stdout.write("Portage %s\n" % portage.VERSION)
		return os.EX_OK

	config_root = options.config_root

	if options.repositories_configuration is not None:
		env['PORTAGE_REPOSITORIES'] = options.repositories_configuration

	settings = portage.config(config_root=config_root,
		local_config=False, env=env)

	default_opts = None
	if not options.ignore_default_opts:
		default_opts = settings.get('EMIRRORDIST_DEFAULT_OPTS', '').split()

	if default_opts:
		parser, options, args = parse_args(default_opts + args)

		settings = portage.config(config_root=config_root,
			local_config=False, env=env)

	if options.repo is None:
		if len(settings.repositories.prepos) == 2:
			for repo in settings.repositories:
				if repo.name != "DEFAULT":
					options.repo = repo.name
					break

		if options.repo is None:
			parser.error("--repo option is required")

	repo_path = settings.repositories.treemap.get(options.repo)
	if repo_path is None:
		parser.error("Unable to locate repository named '%s'" % (options.repo,))

	if options.jobs is not None:
		options.jobs = int(options.jobs)

	if options.load_average is not None:
		options.load_average = float(options.load_average)

	if options.failure_log is not None:
		options.failure_log = normalize_path(
			os.path.abspath(options.failure_log))

		parent_dir = os.path.dirname(options.failure_log)
		if not (os.path.isdir(parent_dir) and
			os.access(parent_dir, os.W_OK|os.X_OK)):
			parser.error(("--failure-log '%s' parent is not a "
				"writable directory") % options.failure_log)

	if options.success_log is not None:
		options.success_log = normalize_path(
			os.path.abspath(options.success_log))

		parent_dir = os.path.dirname(options.success_log)
		if not (os.path.isdir(parent_dir) and
			os.access(parent_dir, os.W_OK|os.X_OK)):
			parser.error(("--success-log '%s' parent is not a "
				"writable directory") % options.success_log)

	if options.scheduled_deletion_log is not None:
		options.scheduled_deletion_log = normalize_path(
			os.path.abspath(options.scheduled_deletion_log))

		parent_dir = os.path.dirname(options.scheduled_deletion_log)
		if not (os.path.isdir(parent_dir) and
			os.access(parent_dir, os.W_OK|os.X_OK)):
			parser.error(("--scheduled-deletion-log '%s' parent is not a "
				"writable directory") % options.scheduled_deletion_log)

		if options.deletion_db is None:
			parser.error("--scheduled-deletion-log requires --deletion-db")

	if options.deletion_delay is not None:
		options.deletion_delay = long(options.deletion_delay)
		if options.deletion_db is None:
			parser.error("--deletion-delay requires --deletion-db")

	if options.deletion_db is not None:
		if options.deletion_delay is None:
			parser.error("--deletion-db requires --deletion-delay")
		options.deletion_db = normalize_path(
			os.path.abspath(options.deletion_db))

	if options.temp_dir is not None:
		options.temp_dir = normalize_path(
			os.path.abspath(options.temp_dir))

		if not (os.path.isdir(options.temp_dir) and
			os.access(options.temp_dir, os.W_OK|os.X_OK)):
			parser.error(("--temp-dir '%s' is not a "
				"writable directory") % options.temp_dir)

	if options.distfiles is not None:
		options.distfiles = normalize_path(
			os.path.abspath(options.distfiles))

		if not (os.path.isdir(options.distfiles) and
			os.access(options.distfiles, os.W_OK|os.X_OK)):
			parser.error(("--distfiles '%s' is not a "
				"writable directory") % options.distfiles)
	else:
		parser.error("missing required --distfiles parameter")

	if options.mirror_overrides is not None:
		options.mirror_overrides = normalize_path(
			os.path.abspath(options.mirror_overrides))

		if not (os.access(options.mirror_overrides, os.R_OK) and
			os.path.isfile(options.mirror_overrides)):
			parser.error(
				"--mirror-overrides-file '%s' is not a readable file" %
				options.mirror_overrides)

	if options.distfiles_local is not None:
		options.distfiles_local = normalize_path(
			os.path.abspath(options.distfiles_local))

		if not (os.path.isdir(options.distfiles_local) and
			os.access(options.distfiles_local, os.W_OK|os.X_OK)):
			parser.error(("--distfiles-local '%s' is not a "
				"writable directory") % options.distfiles_local)

	if options.distfiles_db is not None:
		options.distfiles_db = normalize_path(
			os.path.abspath(options.distfiles_db))

	if options.tries is not None:
		options.tries = int(options.tries)

	if options.recycle_dir is not None:
		options.recycle_dir = normalize_path(
			os.path.abspath(options.recycle_dir))
		if not (os.path.isdir(options.recycle_dir) and
			os.access(options.recycle_dir, os.W_OK|os.X_OK)):
			parser.error(("--recycle-dir '%s' is not a "
				"writable directory") % options.recycle_dir)

	if options.recycle_db is not None:
		if options.recycle_dir is None:
			parser.error("--recycle-db requires "
				"--recycle-dir to be specified")
		options.recycle_db = normalize_path(
			os.path.abspath(options.recycle_db))

	if options.recycle_deletion_delay is not None:
		options.recycle_deletion_delay = \
			long(options.recycle_deletion_delay)

	if options.fetch_log_dir is not None:
		options.fetch_log_dir = normalize_path(
			os.path.abspath(options.fetch_log_dir))

		if not (os.path.isdir(options.fetch_log_dir) and
			os.access(options.fetch_log_dir, os.W_OK|os.X_OK)):
			parser.error(("--fetch-log-dir '%s' is not a "
				"writable directory") % options.fetch_log_dir)

	if options.whitelist_from:
		normalized_paths = []
		for x in options.whitelist_from:
			path = normalize_path(os.path.abspath(x))
			if not os.access(path, os.R_OK):
				parser.error("--whitelist-from '%s' is not readable" % x)
			if os.path.isfile(path):
				normalized_paths.append(path)
			elif os.path.isdir(path):
				for file in _recursive_file_list(path):
					if not os.access(file, os.R_OK):
						parser.error("--whitelist-from '%s' directory contains not readable file '%s'" % (x, file))
					normalized_paths.append(file)
			else:
				parser.error("--whitelist-from '%s' is not a regular file or a directory" % x)
		options.whitelist_from = normalized_paths

	if options.strict_manifests is not None:
		if options.strict_manifests == "y":
			settings.features.add("strict")
		else:
			settings.features.discard("strict")

	settings.lock()

	portdb = portage.portdbapi(mysettings=settings)

	# Limit ebuilds to the specified repo.
	portdb.porttrees = [repo_path]

	portage.util.initialize_logger()

	if options.verbose > 0:
		l = logging.getLogger()
		l.setLevel(l.getEffectiveLevel() - 10 * options.verbose)

	with Config(options, portdb,
		SchedulerInterface(global_event_loop())) as config:

		if not options.mirror:
			parser.error('No action specified')

		returncode = os.EX_OK

		if options.mirror:
			signum = run_main_scheduler(MirrorDistTask(config))
			if signum is not None:
				sys.exit(128 + signum)

	return returncode