lib/portage/dbapi/IndexedPortdb.py

# Copyright 2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

import errno
import io
import functools
import operator
import os

import portage
from portage import _encodings
from portage.dep import Atom
from portage.exception import FileNotFound
from portage.cache.index.IndexStreamIterator import IndexStreamIterator
from portage.cache.index.pkg_desc_index import \
	pkg_desc_index_line_read, pkg_desc_index_node
from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy
from portage.versions import _pkg_str

class IndexedPortdb(object):
	"""
	A portdbapi interface that uses a package description index to
	improve performance. If the description index is missing for a
	particular repository, then all metadata for that repository is
	obtained using the normal pordbapi.aux_get method.

	For performance reasons, the match method only supports package
	name and version constraints. For the same reason, the xmatch
	method is not implemented.
	"""

	# Match returns unordered results.
	match_unordered = True

	_copy_attrs = ('cpv_exists', 'findname', 'getFetchMap',
		'_aux_cache_keys', '_cpv_sort_ascending',
		'_have_root_eclass_dir')

	def __init__(self, portdb):

		self._portdb = portdb

		for k in self._copy_attrs:
			setattr(self, k, getattr(portdb, k))

		self._desc_cache = None
		self._cp_map = None
		self._unindexed_cp_map = None

	def _init_index(self):

		cp_map = {}
		desc_cache = {}
		self._desc_cache = desc_cache
		self._cp_map = cp_map
		index_missing = []

		streams = []
		for repo_path in self._portdb.porttrees:
			outside_repo = os.path.join(self._portdb.depcachedir,
				repo_path.lstrip(os.sep))
			filenames = []
			for parent_dir in (repo_path, outside_repo):
				filenames.append(os.path.join(parent_dir,
					"metadata", "pkg_desc_index"))

			repo_name = self._portdb.getRepositoryName(repo_path)

			try:
				f = None
				for filename in filenames:
					try:
						f = io.open(filename,
							encoding=_encodings["repo.content"])
					except IOError as e:
						if e.errno not in (errno.ENOENT, errno.ESTALE):
							raise
					else:
						break

				if f is None:
					raise FileNotFound(filename)

				streams.append(iter(IndexStreamIterator(f,
					functools.partial(pkg_desc_index_line_read,
					repo = repo_name))))
			except FileNotFound:
				index_missing.append(repo_path)

		if index_missing:
			self._unindexed_cp_map = {}

			class _NonIndexedStream(object):
				def __iter__(self_):
					for cp in self._portdb.cp_all(
						trees = index_missing):
						# Don't call cp_list yet, since it's a waste
						# if the package name does not match the current
						# search.
						self._unindexed_cp_map[cp] = index_missing
						yield pkg_desc_index_node(cp, (), None)

			streams.append(iter(_NonIndexedStream()))

		if streams:
			if len(streams) == 1:
				cp_group_iter = ([node] for node in streams[0])
			else:
				cp_group_iter = MultiIterGroupBy(streams,
					key = operator.attrgetter("cp"))

			for cp_group in cp_group_iter:

				new_cp = None
				cp_list = cp_map.get(cp_group[0].cp)
				if cp_list is None:
					new_cp = cp_group[0].cp
					cp_list = []
					cp_map[cp_group[0].cp] = cp_list

				for entry in cp_group:
					cp_list.extend(entry.cpv_list)
					if entry.desc is not None:
						for cpv in entry.cpv_list:
							desc_cache[cpv] = entry.desc

				if new_cp is not None:
					yield cp_group[0].cp

	def cp_all(self, sort=True):
		"""
		Returns an ordered iterator instead of a list, so that search
		results can be displayed incrementally.
		"""
		if self._cp_map is None:
			return self._init_index()
		return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)

	def match(self, atom):
		"""
		For performance reasons, only package name and version
		constraints are supported, and the returned list is
		unordered.
		"""
		if not isinstance(atom, Atom):
			atom = Atom(atom)
		cp_list = self._cp_map.get(atom.cp)
		if cp_list is None:
			return []

		if self._unindexed_cp_map is not None:
			try:
				unindexed = self._unindexed_cp_map.pop(atom.cp)
			except KeyError:
				pass
			else:
				cp_list.extend(self._portdb.cp_list(atom.cp,
					mytree=unindexed))

		if atom == atom.cp:
			return cp_list[:]
		else:
			return portage.match_from_list(atom, cp_list)

	def aux_get(self, cpv, attrs, myrepo=None):
		if len(attrs) == 1 and attrs[0] == "DESCRIPTION":
			try:
				return [self._desc_cache[cpv]]
			except KeyError:
				pass
		return self._portdb.aux_get(cpv, attrs)