repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py

# -*- coding:utf-8 -*-

'''Package Metadata Checks operations'''

import sys
import re

from itertools import chain
from collections import Counter

try:
	from lxml import etree
	from lxml.etree import ParserError
except (SystemExit, KeyboardInterrupt):
	raise
except (ImportError, SystemError, RuntimeError, Exception):
	# broken or missing xml support
	# https://bugs.python.org/issue14988
	msg = ["Please emerge dev-python/lxml in order to use repoman."]
	from portage.output import EOutput
	out = EOutput()
	for line in msg:
		out.eerror(line)
	sys.exit(1)

# import our initialized portage instance
from repoman._portage import portage
from repoman.metadata import metadata_dtd_uri
from repoman.modules.scan.scanbase import ScanBase

from portage.exception import InvalidAtom
from portage import os
from portage.dep import Atom
from portage.xml.metadata import parse_metadata_use

from .use_flags import USEFlagChecks

metadata_xml_encoding = 'UTF-8'
metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \
	% (metadata_xml_encoding,)
metadata_doctype_name = 'pkgmetadata'


class PkgMetadata(ScanBase, USEFlagChecks):
	'''Package metadata.xml checks'''

	def __init__(self, **kwargs):
		'''PkgMetadata init function

		@param repo_settings: settings instance
		@param qatracker: QATracker instance
		@param options: argparse options instance
		@param metadata_xsd: path of metadata.xsd
		'''
		super(PkgMetadata, self).__init__(**kwargs)
		repo_settings = kwargs.get('repo_settings')
		self.qatracker = kwargs.get('qatracker')
		self.options = kwargs.get('options')
		self.metadata_xsd = kwargs.get('metadata_xsd')
		self.globalUseFlags = kwargs.get('uselist')
		self.repoman_settings = repo_settings.repoman_settings
		self.musedict = {}
		self.muselist = set()

	def check(self, **kwargs):
		'''Performs the checks on the metadata.xml for the package
		@param xpkg: the pacakge being checked
		@param checkdir: string, directory path
		@param checkdirlist: list of checkdir's
		@param repolevel: integer
		@returns: boolean
		'''
		xpkg = kwargs.get('xpkg')
		checkdir = kwargs.get('checkdir')
		checkdirlist = kwargs.get('checkdirlist').get()

		self.musedict = {}
		if self.options.mode in ['manifest']:
			self.muselist = frozenset(self.musedict)
			return False

		# metadata.xml file check
		if "metadata.xml" not in checkdirlist:
			self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
			self.muselist = frozenset(self.musedict)
			return False

		# metadata.xml parse check
		metadata_bad = False

		# read metadata.xml into memory
		try:
			_metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml'))
		except (ParserError, SyntaxError, EnvironmentError) as e:
			metadata_bad = True
			self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e))
			del e
			self.muselist = frozenset(self.musedict)
			return False

		indentation_chars = Counter()
		for l in etree.tostring(_metadata_xml).splitlines():
			indentation_chars.update(re.match(b"\s*", l).group(0))
		if len(indentation_chars) > 1:
			self.qatracker.add_error("metadata.warning", "%s/metadata.xml: %s" %
				(xpkg, "inconsistent use of tabs and spaces in indentation")
			)

		xml_encoding = _metadata_xml.docinfo.encoding
		if xml_encoding.upper() != metadata_xml_encoding:
			self.qatracker.add_error(
				"metadata.bad", "%s/metadata.xml: "
				"xml declaration encoding should be '%s', not '%s'" %
				(xpkg, metadata_xml_encoding, xml_encoding))

		if not _metadata_xml.docinfo.doctype:
			metadata_bad = True
			self.qatracker.add_error(
				"metadata.bad",
				"%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing"))
		else:
			doctype_system = _metadata_xml.docinfo.system_url
			if doctype_system.replace('http://', 'https://') != metadata_dtd_uri:
				if doctype_system is None:
					system_problem = "but it is undefined"
				else:
					system_problem = "not '%s'" % doctype_system
				self.qatracker.add_error(
					"metadata.bad", "%s/metadata.xml: "
					"DOCTYPE: SYSTEM should refer to '%s', %s" %
					(xpkg, metadata_dtd_uri, system_problem))
			doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1]
			if doctype_name != metadata_doctype_name:
				self.qatracker.add_error(
					"metadata.bad", "%s/metadata.xml: "
					"DOCTYPE: name should be '%s', not '%s'" %
					(xpkg, metadata_doctype_name, doctype_name))

		# load USE flags from metadata.xml
		self.musedict = parse_metadata_use(_metadata_xml)
		for atom in chain(*self.musedict.values()):
			if atom is None:
				continue
			try:
				atom = Atom(atom)
			except InvalidAtom as e:
				self.qatracker.add_error(
					"metadata.bad",
					"%s/metadata.xml: Invalid atom: %s" % (xpkg, e))
			else:
				if atom.cp != xpkg:
					self.qatracker.add_error(
						"metadata.bad",
						"%s/metadata.xml: Atom contains "
						"unexpected cat/pn: %s" % (xpkg, atom))

		# Only carry out if in package directory or check forced
		if not metadata_bad:
			validator = etree.XMLSchema(file=self.metadata_xsd)
			if not validator.validate(_metadata_xml):
				self._add_validate_errors(xpkg, validator.error_log)
		self.muselist = frozenset(self.musedict)
		return False

	def check_unused(self, **kwargs):
		'''Reports on any unused metadata.xml use descriptions

		@param xpkg: the pacakge being checked
		@param used_useflags: use flag list
		@param validity_future: Future instance
		'''
		xpkg = kwargs.get('xpkg')
		valid_state = kwargs.get('validity_future').get()
		# check if there are unused local USE-descriptions in metadata.xml
		# (unless there are any invalids, to avoid noise)
		if valid_state:
			for myflag in self.muselist.difference(self.usedUseFlags):
				self.qatracker.add_error(
					"metadata.bad",
					"%s/metadata.xml: unused local USE-description: '%s'"
					% (xpkg, myflag))
		return False

	def _add_validate_errors(self, xpkg, log):
		listed = set()
		for error in log:
			msg_prefix = error.message.split(":",1)[0]
			info = "%s %s" % (error.line, msg_prefix)
			if info not in listed:
				listed.add(info)
				self.qatracker.add_error(
					"metadata.bad",
					"%s/metadata.xml: line: %s, %s"
					% (xpkg, error.line, error.message))

	@property
	def runInPkgs(self):
		'''Package level scans'''
		return (True, [self.check])

	@property
	def runInEbuilds(self):
		return (True, [self.check_useflags])

	@property
	def runInFinal(self):
		'''Final scans at the package level'''
		return (True, [self.check_unused])