blob: 4a039e63ee23796d59d75a17c2019262d51ac9cc [file] [log] [blame]
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Adapted from portage/ -- Portage binary-package helper functions
# Copyright 2003-2004 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import operator
import os
import tempfile
import time
import urllib2
class PackageIndex(object):
"""A parser for the Portage Packages index file.
The Portage Packages index file serves to keep track of what packages are
included in a tree. It contains the following sections:
1) The header. The header tracks general key/value pairs that don't apply
to any specific package. E.g., it tracks the base URL of the packages
file, and the number of packages included in the file. The header is
terminated by a blank line.
2) The body. The body is a list of packages. Each package contains a list
of key/value pairs. Packages are either terminated by a blank line or
by the end of the file. Every package has a CPV entry, which serves as
a unique identifier for the package.
def __init__(self):
# The header tracks general key/value pairs that don't apply to any
# specific package. E.g., it tracks the base URL of the packages.
self.header = {}
# A list of packages (stored as a list of dictionaries).
self.packages = []
# Whether or not the PackageIndex has been modified since the last time it
# was written.
self.modified = False
def _PopulateDuplicateDB(self, db):
"""Populate db with SHA1 -> URL mapping for packages.
db: Dictionary to populate with SHA1 -> URL mapping for packages.
uri = self.header['URI']
for pkg in self.packages:
cpv, sha1 = pkg['CPV'], pkg.get('SHA1')
if sha1:
path = pkg.get('PATH', cpv + '.tbz2')
db[sha1] = '%s/%s' % (uri.rstrip('/'), path)
def _ReadPkgIndex(self, pkgfile):
"""Read a list of key/value pairs from the Packages file into a dictionary.
Both header entries and package entries are lists of key/value pairs, so
they can both be read by this function. Entries can be terminated by empty
lines or by the end of the file.
This function will read lines from the specified file until it encounters
the a blank line or the end of the file.
Keys and values in the Packages file are separated by a colon and a space.
Keys may contain capital letters, numbers, and underscores, but may not
contain colons. Values may contain any character except a newline. In
particular, it is normal for values to contain colons.
Lines that have content, and do not contain a valid key/value pair, are
ignored. This is for compatibility with the Portage package parser, and
to allow for future extensions to the Packages file format.
All entries must contain at least one key/value pair. If the end of the
fils is reached, an empty dictionary is returned.
pkgfile: A python file object.
Returns the dictionary of key-value pairs that was read from the file.
d = {}
for line in pkgfile:
line = line.rstrip('\n')
if not line:
assert d, 'Packages entry must contain at least one key/value pair'
line = line.split(': ', 1)
if len(line) == 2:
k, v = line
d[k] = v
return d
def _WritePkgIndex(self, pkgfile, entry):
"""Write header entry or package entry to packages file.
The keys and values will be separated by a colon and a space. The entry
will be terminated by a blank line.
pkgfile: A python file object.
entry: A dictionary of the key/value pairs to write.
lines = ['%s: %s' % (k, v) for k, v in sorted(entry.items()) if v]
pkgfile.write('%s\n\n' % '\n'.join(lines))
def _ReadHeader(self, pkgfile):
"""Read header of packages file.
pkgfile: A python file object.
assert not self.header, 'Should only read header once.'
self.header = self._ReadPkgIndex(pkgfile)
def _ReadBody(self, pkgfile):
"""Read body of packages file.
Before calling this function, you must first read the header (using
pkgfile: A python file object.
assert self.header, 'Should read header first.'
assert not self.packages, 'Should only read body once.'
# Read all of the sections in the body by looping until we reach the end
# of the file.
while True:
d = self._ReadPkgIndex(pkgfile)
if not d:
if 'CPV' in d:
def Read(self, pkgfile):
"""Read the entire packages file.
pkgfile: A python file object.
def RemoveFilteredPackages(self, filter_fn):
"""Remove packages which match filter_fn.
filter_fn: A function which operates on packages. If it returns True,
the package should be removed.
filtered = [p for p in self.packages if not filter_fn(p)]
if filtered != self.packages:
self.modified = True
self.packages = filtered
def ResolveDuplicateUploads(self, pkgindexes):
"""Point packages at files that have already been uploaded.
For each package in our index, check if there is an existing package that
has already been uploaded to the same base URI. If so, point that package
at the existing file, so that we don't have to upload the file.
pkgindexes: A list of PackageIndex objects containing info about packages
that have already been uploaded.
A list of the packages that still need to be uploaded.
db = {}
for pkgindex in pkgindexes:
uploads = []
base_uri = self.header['URI']
for pkg in self.packages:
sha1 = pkg.get('SHA1')
uri = db.get(sha1)
if sha1 and uri and uri.startswith(base_uri):
pkg['PATH'] = uri[len(base_uri):].lstrip('/')
return uploads
def SetUploadLocation(self, base_uri, path_prefix):
"""Set upload location to base_uri + path_prefix.
base_uri: Base URI for all packages in the file. We set
self.header['URI'] to this value, so all packages must live under
this directory.
path_prefix: Path prefix to use for all current packages in the file.
This will be added to the beginning of the path for every package.
self.header['URI'] = base_uri
for pkg in self.packages:
path = pkg['CPV'] + '.tbz2'
pkg['PATH'] = '%s/%s' % (path_prefix.rstrip('/'), path)
def Write(self, pkgfile):
"""Write a packages file to disk.
If 'modified' flag is set, the TIMESTAMP and PACKAGES fields in the header
will be updated before writing to disk.
pkgfile: A python file object.
if self.modified:
self.header['TIMESTAMP'] = str(long(time.time()))
self.header['PACKAGES'] = str(len(self.packages))
self.modified = False
self._WritePkgIndex(pkgfile, self.header)
for metadata in sorted(self.packages, key=operator.itemgetter('CPV')):
self._WritePkgIndex(pkgfile, metadata)
def WriteToNamedTemporaryFile(self):
"""Write pkgindex to a temporary file.
pkgindex: The PackageIndex object.
A temporary file containing the packages from pkgindex.
f = tempfile.NamedTemporaryFile()
return f
def _RetryUrlOpen(url, tries=3):
"""Open the specified url, retrying if we run into temporary errors.
We retry for both network errors and 5xx Server Errors. We do not retry
for HTTP errors with a non-5xx code.
url: The specified url.
tries: The number of times to try.
The result of urllib2.urlopen(url).
for i in range(tries):
return urllib2.urlopen(url)
except urllib2.HTTPError as e:
if i + 1 >= tries or e.code < 500:
print 'Cannot GET %s: %s' % (url, str(e))
except urllib2.URLError as e:
if i + 1 >= tries:
print 'Cannot GET %s: %s' % (url, str(e))
print 'Sleeping for 10 seconds before retrying...'
def GrabRemotePackageIndex(binhost_url):
"""Grab the latest binary package database from the specified URL.
binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
A PackageIndex object, if the Packages file can be retrieved. If the
server returns status code 404, None is returned.
url = '%s/Packages' % binhost_url.rstrip('/')
f = _RetryUrlOpen(url)
except urllib2.HTTPError as e:
if e.code == 404:
return None
pkgindex = PackageIndex()
pkgindex.header.setdefault('URI', binhost_url)
return pkgindex
def GrabLocalPackageIndex(package_path):
"""Read a local packages file from disk into a PackageIndex() object.
package_path: Directory containing Packages file.
A PackageIndex object.
packages_file = file(os.path.join(package_path, 'Packages'))
pkgindex = PackageIndex()
return pkgindex