blob: 1c9989a72269bc46509d19b7a5e93af77228b1ab [file] [log] [blame]
# -*- coding:utf-8 -*-
from __future__ import print_function, unicode_literals
import errno
import io
import logging
import platform
import re
import shutil
import signal
import sys
import tempfile
import time
from itertools import chain
try:
from urllib.parse import parse_qs, urlsplit, urlunsplit
except ImportError:
from urlparse import parse_qs, urlsplit, urlunsplit
from _emerge.UserQuery import UserQuery
from repoman._portage import portage
from portage import os
from portage import _encodings
from portage import _unicode_encode
from portage.output import (
bold, create_color_func, green, red)
from portage.package.ebuild.digestgen import digestgen
from portage.util import writemsg_level
from repoman.copyrights import update_copyright
from repoman.gpg import gpgsign, need_signature
from repoman import utilities
from repoman.modules.vcs.vcs import vcs_files_to_cps
from repoman import VERSION
bad = create_color_func("BAD")
class Actions(object):
'''Handles post check result output and performs
the various vcs activities for committing the results'''
def __init__(self, repo_settings, options, scanner, vcs_settings):
self.repo_settings = repo_settings
self.options = options
self.scanner = scanner
self.vcs_settings = vcs_settings
self.repoman_settings = repo_settings.repoman_settings
self.suggest = {
'ignore_masked': False,
'include_dev': False,
}
if scanner.have['pmasked'] and not (options.without_mask or options.ignore_masked):
self.suggest['ignore_masked'] = True
if scanner.have['dev_keywords'] and not options.include_dev:
self.suggest['include_dev'] = True
def inform(self, can_force, result):
'''Inform the user of all the problems found'''
if ((self.suggest['ignore_masked'] or self.suggest['include_dev'])
and not self.options.quiet):
self._suggest()
if self.options.mode != 'commit':
self._non_commit(result)
return False
else:
self._fail(result, can_force)
if self.options.pretend:
utilities.repoman_sez(
"\"So, you want to play it safe. Good call.\"\n")
return True
def perform(self, qa_output):
myautoadd = self._vcs_autoadd()
self._vcs_deleted()
changes = self.get_vcs_changed()
mynew, mychanged, myremoved, no_expansion, expansion = changes
# Manifests need to be regenerated after all other commits, so don't commit
# them now even if they have changed.
mymanifests = set()
myupdates = set()
for f in mychanged + mynew:
if "Manifest" == os.path.basename(f):
mymanifests.add(f)
else:
myupdates.add(f)
myupdates.difference_update(myremoved)
myupdates = list(myupdates)
mymanifests = list(mymanifests)
myheaders = []
commitmessage = self.options.commitmsg
if self.options.commitmsgfile:
try:
f = io.open(
_unicode_encode(
self.options.commitmsgfile,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='replace')
commitmessage = f.read()
f.close()
del f
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
portage.writemsg(
"!!! File Not Found:"
" --commitmsgfile='%s'\n" % self.options.commitmsgfile)
else:
raise
if commitmessage[:9].lower() in ("cat/pkg: ",):
commitmessage = self.msg_prefix() + commitmessage[9:]
if commitmessage is not None and commitmessage.strip():
res, expl = self.verify_commit_message(commitmessage)
if not res:
print(bad("RepoMan does not like your commit message:"))
print(expl)
if self.options.force:
print('(but proceeding due to --force)')
else:
sys.exit(1)
else:
commitmessage = None
msg_qa_output = qa_output
initial_message = None
while True:
commitmessage = self.get_new_commit_message(
msg_qa_output, commitmessage)
res, expl = self.verify_commit_message(commitmessage)
if res:
break
else:
full_expl = '''Issues with the commit message were found. Please fix it or remove
the whole commit message to abort.
''' + expl
msg_qa_output = (
[' %s\n' % x for x in full_expl.splitlines()]
+ qa_output)
commitmessage = commitmessage.rstrip()
# Update copyright for new and changed files
year = time.strftime('%Y', time.gmtime())
updated_copyright = []
for fn in chain(mynew, mychanged):
if fn.endswith('.diff') or fn.endswith('.patch'):
continue
if update_copyright(fn, year, pretend=self.options.pretend):
updated_copyright.append(fn)
if updated_copyright and not (
self.options.pretend or self.repo_settings.repo_config.thin_manifest):
for cp in sorted(self._vcs_files_to_cps(iter(updated_copyright))):
self._manifest_gen(cp)
myupdates, broken_changelog_manifests = self.changelogs(
myupdates, mymanifests, myremoved, mychanged, myautoadd,
mynew, commitmessage)
lines = commitmessage.splitlines()
lastline = lines[-1]
if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None:
commitmessage += '\n'
commit_footer = self.get_commit_footer()
commitmessage += commit_footer
print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
if not self.vcs_settings.needs_keyword_expansion:
# With some VCS types there's never any keyword expansion, so
# there's no need to regenerate manifests and all files will be
# committed in one big commit at the end.
logging.debug("VCS type doesn't need keyword expansion")
print()
elif not self.repo_settings.repo_config.thin_manifest:
logging.debug("perform: Calling thick_manifest()")
self.vcs_settings.changes.thick_manifest(myupdates, myheaders,
no_expansion, expansion)
logging.info("myupdates: %s", myupdates)
logging.info("myheaders: %s", myheaders)
uq = UserQuery(self.options)
if self.options.ask and uq.query('Commit changes?', True) != 'Yes':
print("* aborting commit.")
sys.exit(128 + signal.SIGINT)
# Handle the case where committed files have keywords which
# will change and need a priming commit before the Manifest
# can be committed.
if (myupdates or myremoved) and myheaders:
self.priming_commit(myupdates, myremoved, commitmessage)
# When files are removed and re-added, the cvs server will put /Attic/
# inside the $Header path. This code detects the problem and corrects it
# so that the Manifest will generate correctly. See bug #169500.
# Use binary mode in order to avoid potential character encoding issues.
self.vcs_settings.changes.clear_attic(myheaders)
if self.scanner.repolevel == 1:
utilities.repoman_sez(
"\"You're rather crazy... "
"doing the entire repository.\"\n")
self.vcs_settings.changes.digest_regen(myupdates, myremoved, mymanifests,
self.scanner, broken_changelog_manifests)
if self.repo_settings.sign_manifests:
self.sign_manifest(myupdates, myremoved, mymanifests)
self.vcs_settings.changes.update_index(mymanifests, myupdates)
self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage)
if self.options.quiet:
return
print()
if self.vcs_settings.vcs:
print("Commit complete.")
else:
print(
"repoman was too scared"
" by not seeing any familiar version control file"
" that he forgot to commit anything")
utilities.repoman_sez(
"\"If everyone were like you, I'd be out of business!\"\n")
return
def _vcs_files_to_cps(self, vcs_file_iter):
"""
Iterate over the given modified file paths returned from the vcs,
and return a frozenset containing category/pn strings for each
modified package.
@param vcs_file_iter: file paths from vcs module
@type iter
@rtype: frozenset
@return: category/pn strings for each package.
"""
return vcs_files_to_cps(
vcs_file_iter,
self.repo_settings.repodir,
self.scanner.repolevel,
self.scanner.reposplit,
self.scanner.categories)
def _manifest_gen(self, cp):
"""
Generate manifest for a cp.
@param cp: category/pn string
@type str
@rtype: bool
@return: True if successful, False otherwise
"""
self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, cp)
return bool(digestgen(
mysettings=self.repoman_settings,
myportdb=self.repo_settings.portdb))
def _suggest(self):
print()
if self.suggest['ignore_masked']:
print(bold(
"Note: use --without-mask to check "
"KEYWORDS on dependencies of masked packages"))
if self.suggest['include_dev']:
print(bold(
"Note: use --include-dev (-d) to check "
"dependencies for 'dev' profiles"))
print()
def _non_commit(self, result):
if result['full']:
print(bold("Note: type \"repoman full\" for a complete listing."))
if result['warn'] and not result['fail']:
if self.options.quiet:
print(bold("Non-Fatal QA errors found"))
else:
utilities.repoman_sez(
"\"You're only giving me a partial QA payment?\n"
" I'll take it this time, but I'm not happy.\""
)
elif not result['fail']:
if self.options.quiet:
print("No QA issues found")
else:
utilities.repoman_sez(
"\"If everyone were like you, I'd be out of business!\"")
elif result['fail']:
print(bad("Please fix these important QA issues first."))
if not self.options.quiet:
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
sys.exit(1)
def _fail(self, result, can_force):
if result['fail'] and can_force and self.options.force and not self.options.pretend:
utilities.repoman_sez(
" \"You want to commit even with these QA issues?\n"
" I'll take it this time, but I'm not happy.\"\n")
elif result['fail']:
if self.options.force and not can_force:
print(bad(
"The --force option has been disabled"
" due to extraordinary issues."))
print(bad("Please fix these important QA issues first."))
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
sys.exit(1)
def _vcs_autoadd(self):
myunadded = self.vcs_settings.changes.unadded
myautoadd = []
if myunadded:
for x in range(len(myunadded) - 1, -1, -1):
xs = myunadded[x].split("/")
if self.repo_settings.repo_config.find_invalid_path_char(myunadded[x]) != -1:
# The Manifest excludes this file,
# so it's safe to ignore.
del myunadded[x]
elif xs[-1] == "files":
print("!!! files dir is not added! Please correct this.")
sys.exit(-1)
elif xs[-1] == "Manifest":
# It's a manifest... auto add
myautoadd += [myunadded[x]]
del myunadded[x]
if myunadded:
print(red(
"!!! The following files are in your local tree"
" but are not added to the master"))
print(red(
"!!! tree. Please remove them from the local tree"
" or add them to the master tree."))
for x in myunadded:
print(" ", x)
print()
print()
sys.exit(1)
return myautoadd
def _vcs_deleted(self):
if self.vcs_settings.changes.has_deleted:
print(red(
"!!! The following files are removed manually"
" from your local tree but are not"))
print(red(
"!!! removed from the repository."
" Please remove them, using \"%s remove [FILES]\"."
% self.vcs_settings.vcs))
for x in self.vcs_settings.changes.deleted:
print(" ", x)
print()
print()
sys.exit(1)
def get_vcs_changed(self):
'''Holding function which calls the approriate VCS module for the data'''
changes = self.vcs_settings.changes
# re-run the scan to pick up a newly modified Manifest file
logging.debug("RE-scanning for changes...")
changes.scan()
if not changes.has_changes:
utilities.repoman_sez(
"\"Doing nothing is not always good for QA.\"")
print()
print("(Didn't find any changed files...)")
print()
sys.exit(1)
return (changes.new, changes.changed, changes.removed,
changes.no_expansion, changes.expansion)
https_bugtrackers = frozenset([
'bitbucket.org',
'bugs.gentoo.org',
'github.com',
'gitlab.com',
])
def get_commit_footer(self):
portage_version = getattr(portage, "VERSION", None)
gpg_key = self.repoman_settings.get("PORTAGE_GPG_KEY", "")
dco_sob = self.repoman_settings.get("DCO_SIGNED_OFF_BY", "")
report_options = []
if self.options.force:
report_options.append("--force")
if self.options.ignore_arches:
report_options.append("--ignore-arches")
if self.scanner.include_arches is not None:
report_options.append(
"--include-arches=\"%s\"" %
" ".join(sorted(self.scanner.include_arches)))
if portage_version is None:
sys.stderr.write("Failed to insert portage version in message!\n")
sys.stderr.flush()
portage_version = "Unknown"
# Common part of commit footer
commit_footer = ""
for tag, bug in chain(
(('Bug', x) for x in self.options.bug),
(('Closes', x) for x in self.options.closes)):
# case 1: pure number NNNNNN
if bug.isdigit():
bug = 'https://bugs.gentoo.org/%s' % (bug, )
else:
purl = urlsplit(bug)
qs = parse_qs(purl.query)
# case 2: long Gentoo bugzilla URL to shorten
if (purl.netloc == 'bugs.gentoo.org' and
purl.path == '/show_bug.cgi' and
tuple(qs.keys()) == ('id',)):
bug = urlunsplit(('https', purl.netloc,
qs['id'][-1], '', purl.fragment))
# case 3: bug tracker w/ http -> https
elif (purl.scheme == 'http' and
purl.netloc in self.https_bugtrackers):
bug = urlunsplit(('https',) + purl[1:])
commit_footer += "\n%s: %s" % (tag, bug)
# Use new footer only for git (see bug #438364).
if self.vcs_settings.vcs in ["git"]:
commit_footer += "\nPackage-Manager: Portage-%s, Repoman-%s" % (
portage.VERSION, VERSION)
if report_options:
commit_footer += "\nRepoMan-Options: " + " ".join(report_options)
if self.repo_settings.sign_manifests:
commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
else:
unameout = platform.system() + " "
if platform.system() in ["Darwin", "SunOS"]:
unameout += platform.processor()
else:
unameout += platform.machine()
commit_footer += "\n(Portage version: %s/%s/%s" % \
(portage_version, self.vcs_settings.vcs, unameout)
if report_options:
commit_footer += ", RepoMan options: " + " ".join(report_options)
if self.repo_settings.sign_manifests:
commit_footer += ", signed Manifest commit with key %s" % \
(gpg_key, )
else:
commit_footer += ", unsigned Manifest commit"
commit_footer += ")"
if dco_sob:
commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
return commit_footer
def changelogs(self, myupdates, mymanifests, myremoved, mychanged, myautoadd,
mynew, changelog_msg):
broken_changelog_manifests = []
if self.options.echangelog in ('y', 'force'):
logging.info("checking for unmodified ChangeLog files")
committer_name = utilities.get_committer_name(env=self.repoman_settings)
for x in sorted(vcs_files_to_cps(
chain(myupdates, mymanifests, myremoved),
self.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)):
catdir, pkgdir = x.split("/")
checkdir = self.repo_settings.repodir + "/" + x
checkdir_relative = ""
if self.scanner.repolevel < 3:
checkdir_relative = os.path.join(pkgdir, checkdir_relative)
if self.scanner.repolevel < 2:
checkdir_relative = os.path.join(catdir, checkdir_relative)
checkdir_relative = os.path.join(".", checkdir_relative)
changelog_path = os.path.join(checkdir_relative, "ChangeLog")
changelog_modified = changelog_path in self.scanner.changed.changelogs
if changelog_modified and self.options.echangelog != 'force':
continue
# get changes for this package
cdrlen = len(checkdir_relative)
check_relative = lambda e: e.startswith(checkdir_relative)
split_relative = lambda e: e[cdrlen:]
clnew = list(map(split_relative, filter(check_relative, mynew)))
clremoved = list(map(split_relative, filter(check_relative, myremoved)))
clchanged = list(map(split_relative, filter(check_relative, mychanged)))
# Skip ChangeLog generation if only the Manifest was modified,
# as discussed in bug #398009.
nontrivial_cl_files = set()
nontrivial_cl_files.update(clnew, clremoved, clchanged)
nontrivial_cl_files.difference_update(['Manifest'])
if not nontrivial_cl_files and self.options.echangelog != 'force':
continue
new_changelog = utilities.UpdateChangeLog(
checkdir_relative, committer_name, changelog_msg,
os.path.join(self.repo_settings.repodir, 'skel.ChangeLog'),
catdir, pkgdir,
new=clnew, removed=clremoved, changed=clchanged,
pretend=self.options.pretend)
if new_changelog is None:
writemsg_level(
"!!! Updating the ChangeLog failed\n",
level=logging.ERROR, noiselevel=-1)
sys.exit(1)
# if the ChangeLog was just created, add it to vcs
if new_changelog:
myautoadd.append(changelog_path)
# myautoadd is appended to myupdates below
else:
myupdates.append(changelog_path)
if self.options.ask and not self.options.pretend:
# regenerate Manifest for modified ChangeLog (bug #420735)
self.repoman_settings["O"] = checkdir
digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb)
else:
broken_changelog_manifests.append(x)
if myautoadd:
print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
self.vcs_settings.changes.add_items(myautoadd)
myupdates += myautoadd
return myupdates, broken_changelog_manifests
def add_manifest(self, mymanifests, myheaders, myupdates, myremoved,
commitmessage):
myfiles = mymanifests[:]
# If there are no header (SVN/CVS keywords) changes in
# the files, this Manifest commit must include the
# other (yet uncommitted) files.
if not myheaders:
myfiles += myupdates
myfiles += myremoved
myfiles.sort()
commitmessagedir = tempfile.mkdtemp(".repoman.msg")
commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG")
with open(commitmessagefile, "wb") as mymsg:
mymsg.write(_unicode_encode(commitmessage))
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
shutil.rmtree(commitmessagedir)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
def priming_commit(self, myupdates, myremoved, commitmessage):
myfiles = myupdates + myremoved
commitmessagedir = tempfile.mkdtemp(".repoman.msg")
commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG")
with open(commitmessagefile, "wb") as mymsg:
mymsg.write(_unicode_encode(commitmessage))
separator = '-' * 78
print()
print(green("Using commit message:"))
print(green(separator))
print(commitmessage)
print(green(separator))
print()
# Having a leading ./ prefix on file paths can trigger a bug in
# the cvs server when committing files to multiple directories,
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
shutil.rmtree(commitmessagedir)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
def sign_manifest(self, myupdates, myremoved, mymanifests):
try:
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved, mymanifests),
self.scanner.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)):
self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x)
manifest_path = os.path.join(self.repoman_settings["O"], "Manifest")
if not need_signature(manifest_path):
continue
gpgsign(manifest_path, self.repoman_settings, self.options)
except portage.exception.PortageException as e:
portage.writemsg("!!! %s\n" % str(e))
portage.writemsg("!!! Disabled FEATURES='sign'\n")
self.repo_settings.sign_manifests = False
def msg_prefix(self):
prefix = ""
if self.scanner.repolevel > 1:
prefix = "/".join(self.scanner.reposplit[1:]) + ": "
return prefix
def get_new_commit_message(self, qa_output, old_message=None):
msg_prefix = old_message or self.msg_prefix()
try:
editor = os.environ.get("EDITOR")
if editor and utilities.editor_is_executable(editor):
commitmessage = utilities.get_commit_message_with_editor(
editor, message=qa_output, prefix=msg_prefix)
else:
print("EDITOR is unset or invalid. Please set EDITOR to your preferred editor.")
print(bad("* no EDITOR found for commit message, aborting commit."))
sys.exit(1)
except KeyboardInterrupt:
logging.fatal("Interrupted; exiting...")
sys.exit(1)
if (not commitmessage or not commitmessage.strip()
or commitmessage.strip() == msg_prefix):
print("* no commit message? aborting commit.")
sys.exit(1)
return commitmessage
@staticmethod
def verify_commit_message(msg):
"""
Check whether the message roughly complies with GLEP66. Returns
(True, None) if it does, (False, <explanation>) if it does not.
"""
problems = []
paras = msg.strip().split('\n\n')
summary = paras.pop(0)
if ':' not in summary:
problems.append('summary line must start with a logical unit name, e.g. "cat/pkg:"')
if '\n' in summary.strip():
problems.append('commit message must start with a *single* line of summary, followed by empty line')
# accept 69 overall or unit+50, in case of very long package names
elif len(summary.strip()) > 69 and len(summary.split(':', 1)[-1]) > 50:
problems.append('summary line is too long (max 69 characters)')
multiple_footers = False
gentoo_bug_used = False
bug_closes_without_url = False
body_too_long = False
footer_re = re.compile(r'^[\w-]+:')
found_footer = False
for p in paras:
lines = p.splitlines()
# if all lines look like footer material, we assume it's footer
# else, we assume it's body text
if all(footer_re.match(l) for l in lines if l.strip()):
# multiple footer-like blocks?
if found_footer:
multiple_footers = True
found_footer = True
for l in lines:
if l.startswith('Gentoo-Bug'):
gentoo_bug_used = True
elif l.startswith('Bug:') or l.startswith('Closes:'):
if 'http://' not in l and 'https://' not in l:
bug_closes_without_url = True
else:
for l in lines:
# we recommend wrapping at 72 but accept up to 80;
# do not complain if we have single word (usually
# it means very long URL)
if len(l.strip()) > 80 and len(l.split()) > 1:
body_too_long = True
if multiple_footers:
problems.append('multiple footer-style blocks found, please combine them into one')
if gentoo_bug_used:
problems.append('please replace Gentoo-Bug with GLEP 66-compliant Bug/Closes')
if bug_closes_without_url:
problems.append('Bug/Closes footers require full URL')
if body_too_long:
problems.append('body lines should be wrapped at 72 (max 80) characters')
if problems:
return (False, '\n'.join('- %s' % x for x in problems))
return (True, None)