| #!/usr/bin/env python3 |
| # pylint: disable=line-too-long, subprocess-run-check, unused-argument |
| # Copyright 2021 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Convert Cargo metadata to ebuilds using cros-rust. |
| |
| Used to add new Rust projects and building up ebuilds for the dependency tree. |
| """ |
| |
| import argparse |
| from collections import defaultdict |
| from datetime import datetime |
| import json |
| import logging |
| import os |
| from pprint import pprint |
| import re |
| import shutil |
| import subprocess |
| import sys |
| |
| SCRIPT_NAME = 'cargo2ebuild.py' |
| AUTOGEN_NOTICE = '\n# This file was automatically generated by {}'.format( |
| SCRIPT_NAME) |
| |
| CRATE_DL_URI = 'https://crates.io/api/v1/crates/{PN}/{PV}/download' |
| |
| # Required parameters |
| # copyright_year: Current year for copyright assignment. |
| # description: Description of the crates. |
| # homepage: Homepage of the crates. |
| # license: Ebuild compatible license string. |
| # dependencies: Ebuild compatible dependency string. |
| # autogen_notice: Autogenerated notification string. |
| EBUILD_TEMPLATE = ( |
| """# Copyright {copyright_year} The Chromium OS Authors. All rights reserved. |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| EAPI="7" |
| |
| CROS_RUST_REMOVE_DEV_DEPS=1{remove_target_var} |
| |
| inherit cros-rust |
| |
| DESCRIPTION='{description}' |
| HOMEPAGE='{homepage}' |
| SRC_URI="https://crates.io/api/v1/crates/${{PN}}/${{PV}}/download -> ${{P}}.crate" |
| |
| LICENSE="{license}" |
| SLOT="${{PV}}/${{PR}}" |
| KEYWORDS="*" |
| |
| {dependencies}{autogen_notice} |
| """) |
| |
| # Required parameters: |
| # copyright_year: Current year for copyright assignment. |
| # crate_features: Features to add to this empty crate. |
| # autogen_notice: Autogenerated notification string. |
| EMPTY_CRATE = ( |
| """# Copyright {copyright_year} The Chromium OS Authors. All rights reserved. |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| EAPI="7" |
| |
| CROS_RUST_EMPTY_CRATE=1 |
| {crate_features} |
| inherit cros-rust |
| |
| DESCRIPTION="Empty crate" |
| HOMEPAGE="" |
| |
| LICENSE="BSD-Google" |
| SLOT="${{PV}}/${{PR}}" |
| KEYWORDS="*" |
| {autogen_notice} |
| """) |
| |
| LICENSES = { |
| 'Apache-2.0': 'Apache-2.0', |
| 'MIT': 'MIT', |
| 'BSD-2-Clause': 'BSD-2', |
| 'BSD-3-Clause': 'BSD', |
| '0BSD': '0BSD', |
| 'ISC': 'ISC', |
| } |
| |
| VERSION_RE = ( |
| '^(?P<dep>([\\^~=]|>=))?' # Dependency type: ^, ~, =, >= |
| '(?P<major>[0-9]+|[*])' # Major version (can be *) |
| '(.(?P<minor>[0-9]+|[*]))?' # Minor version |
| '(.(?P<patch>[0-9]+|[*]))?' # Patch version |
| '([+\\-].*)?$' # Any semver values beyond patch version |
| ) |
| |
| EBUILD_RE = ( |
| '^(?P<name>[-a-zA-Z0-9_]+)?' # Ebuild name |
| '-(?P<version>[0-9]+(.[0-9]+)?(.[0-9]+)?)' |
| '([-_].*)?\\.ebuild$' # Any semver values beyond patch version |
| ) |
| |
| FEATURES_RE = ( |
| '^CROS_RUST_EMPTY_CRATE_FEATURES=\\(' |
| '(?P<features>([^)]|$)*)' # Features array |
| '\\)' |
| ) |
| |
| DEP_GRAPH_ROOT = '*root*' |
| |
| |
| class VersionParseError(Exception): |
| """Error that is returned when parsing a version fails.""" |
| |
| |
| class VersionRange: |
| """Represents a range of Rust crate versions.""" |
| @staticmethod |
| def from_str(value): |
| """Generate a VersionRange from a crate requirement string.""" |
| if value == '*': |
| return VersionRange(v_min=(0, 0, 0)) |
| |
| v_min = None |
| v_min_inclusive = True |
| v_max = None |
| v_max_inclusive = False |
| |
| # Handle a pair of constraints. |
| parts = value.split(', ') |
| if len(parts) == 2: |
| for part in parts: |
| if part.startswith('>='): |
| dep, major, minor, patch = version_to_tuple(part[2:], missing=0) |
| v_min = (major, minor, patch) |
| elif part.startswith('<='): |
| v_max_inclusive = True |
| dep, major, minor, patch = version_to_tuple(part[2:], missing=0) |
| v_max = (major, minor, patch) |
| elif part.startswith('>'): |
| v_min_inclusive = False |
| dep, major, minor, patch = version_to_tuple(part[1:], missing=0) |
| v_min = (major, minor, patch) |
| elif part.startswith('<'): |
| dep, major, minor, patch = version_to_tuple(part[1:], missing=0) |
| v_max = (major, minor, patch) |
| if v_max and v_min and v_max < v_min: |
| raise VersionParseError |
| # We are not going to worry about more than two constraints until we see it. |
| elif len(parts) != 1: |
| raise VersionParseError( |
| 'Version constraint has more than two parts: {}'.format(parts)) |
| # Handle the typical case of a single constraint operator. |
| else: |
| dep, major, minor, patch = version_to_tuple(value, missing=-1) |
| v_min = (major, minor, patch) |
| |
| if dep == '=': |
| v_max = v_min |
| v_max_inclusive = True |
| elif dep == '~': |
| major = max(v_min[0], 0) |
| minor = max(v_min[1], 0) |
| patch = max(v_min[2], 0) |
| if v_min[0] < 0: |
| raise VersionParseError( |
| 'The ~ constraint operator requires a major version: {}'.format(value)) |
| if v_min[1] < 0 and v_min[2] < 0: |
| v_max = (major + 1, 0, 0) |
| else: |
| v_max = (major, minor + 1, 0) |
| v_min = (major, minor, patch) |
| elif dep == '>=': |
| major = max(v_min[0], 0) |
| minor = max(v_min[1], 0) |
| patch = max(v_min[2], 0) |
| v_min = (major, minor, patch) |
| v_min_inclusive = True |
| v_max = None |
| elif dep and dep != '^': |
| raise VersionParseError('Unrecognized operator: "{}"'.format(dep)) |
| else: |
| major = max(v_min[0], 0) |
| minor = max(v_min[1], 0) |
| patch = max(v_min[2], 0) |
| |
| v_min = (major, minor, patch) |
| if major > 0: |
| v_max = (major + 1, 0, 0) |
| elif minor > 0: |
| v_max = (0, minor + 1, 0) |
| else: |
| v_max_inclusive = True |
| v_max = v_min |
| |
| return VersionRange(v_min=v_min, v_min_inclusive=v_min_inclusive, v_max=v_max, |
| v_max_inclusive=v_max_inclusive) |
| |
| def __init__(self, v_min=None, v_min_inclusive=True, v_max=None, v_max_inclusive=False): |
| self.min = v_min |
| self.min_inclusive = v_min_inclusive |
| self.max = v_max |
| self.max_inclusive = v_max_inclusive |
| |
| def contains(self, version): |
| """Returns `True` if version is inside the range; `False` otherwise. |
| |
| Args: |
| version: Can be either a tuple of integers (major, minor, patch) or a string that will be converted to a |
| tuple of integers. Trailing characters will be removed from the version (e.g. "+openssl-1.0.1"). |
| """ |
| if isinstance(version, str): |
| try: |
| filtered_version = re.sub(r'([0-9]+\.[0-9]+\.[0-9]+)[^0-9].*$', r'\1', version) |
| if filtered_version != version: |
| logging.warning('Filtered version "%s" to "%s"', version, filtered_version) |
| version = tuple([int(a) for a in filtered_version.split('.')]) |
| except Exception as e: |
| print(version) |
| raise e |
| |
| if self.min: |
| if self.min_inclusive: |
| if self.min > version: |
| return False |
| elif self.min >= version: |
| return False |
| if self.max: |
| if self.max_inclusive: |
| if self.max < version: |
| return False |
| elif self.max <= version: |
| return False |
| return True |
| |
| def to_ebuild_str(self, category_and_name): |
| """Return an ebuild DEPEND entry equivalent to the range. |
| |
| Args: |
| category_and_name: A string of the form 'dev-rust/crate-name' where 'dev-rust' is the category and |
| 'crate-name' is the name of the ebuild. |
| |
| Returns: |
| A string that can be used in an ebuild's DEPEND field. |
| """ |
| if not self.min or self.min == (0, 0, 0) and not self.max: |
| return '{}:='.format(category_and_name) |
| |
| min_bound = '>=' if self.min_inclusive else '>' |
| if self.max_inclusive: |
| max_bound = '<=' |
| suffix = '' |
| else: |
| max_bound = '<' |
| # The _alpha suffix is needed because pre-release versions are treated as less than the actual release |
| # by portage so 3.0.0_beta2 is treated as less than 3.0.0. Consequently, 3.0.0_beta2 may satisfy the |
| # requirement >=2.0.1:= <3.0.0 for portage but not meet the requirement for cargo. Adding the _alpha |
| # suffix covers this edge case. |
| suffix = '_alpha' |
| if not self.max: |
| return '{}{}-{}.{}.{}:='.format(min_bound, category_and_name, self.min[0], self.min[1], |
| self.min[2]) |
| |
| for x in range(len(self.min)): |
| if self.min[x] != self.max[x]: |
| break |
| else: |
| # We want to allow revisions other than -r0, so use `~` instead of `=`. |
| return '~{}-{}.{}.{}:='.format(category_and_name, self.min[0], self.min[1], |
| self.min[2]) |
| |
| if self.min_inclusive and not self.max_inclusive and self.min[x] + 1 == self.max[x]: |
| if x == 0 and self.min[1] == 0 and self.min[2] == 0: |
| return '={}-{}*:='.format(category_and_name, self.min[0]) |
| if x == 1 and self.min[2] == 0: |
| return '={}-{}.{}*:='.format(category_and_name, self.min[0], self.min[1]) |
| |
| # The slot operator needs to be on the upper bound (b/199734226) otherwise the depgraph operation pulls in the |
| # wrong slot for the lower bound that may violate the upper bound. This issue shows up in the DEPEND string for |
| # binary packages. |
| return '{0}{1}-{2}.{3}.{4} {5}{1}-{6}.{7}.{8}{9}:='.format( |
| min_bound, category_and_name, self.min[0], self.min[1], self.min[2], |
| max_bound, self.max[0], self.max[1], self.max[2], suffix) |
| |
| |
| class DepGraphNode: |
| """A node in the dep-graph for a specific version of a package used by DepGraph.""" |
| def __init__(self, package): |
| self.package = package |
| self.dependencies = [] |
| |
| def __repr__(self): |
| return '{}'.format([(a['name'], a['req']) for a in self.dependencies]) |
| |
| def add_dependency(self, dep): |
| """Adds a dependency to the list.""" |
| self.dependencies.append(dep) |
| |
| |
| class DepGraph: |
| """A model of the dependency relationships between crates. |
| |
| The functionality this provides over using the Cargo metadata directly is: |
| * Filtering based on cros-rust.eclass features. |
| * Tracking of system ebuilds for detecting edge cases such as: |
| * Already fulfilled dependencies. |
| * Optional crates that would replace or supersede non-empty ebuilds. |
| * Empty crates that are missing definitions for required features. |
| """ |
| def __init__(self): |
| # name -> version -> DepGraphNode |
| self.graph = defaultdict(dict) |
| |
| # name -> version -> None or [features] |
| self.system_crates = defaultdict(dict) |
| |
| # The crate that will be initially checked when flattening the depgraph into a list of required and optional |
| # crates. |
| self.root = None |
| |
| def add_package(self, package): |
| """Add a crate to the dependency graph from a parsed metadata.json generated by cargo. |
| |
| Args: |
| package: A parsed package from the Cargo metadata. |
| |
| Returns: |
| filter_target: True if CROS_RUST_REMOVE_TARGET_CFG=1 should be set in the ebuild. |
| deps: A list of dependencies from the Cargo metadata JSON that still apply after filtering. |
| """ |
| dependencies = package['dependencies'] |
| |
| # Check if there are target specific dependencies and if so if they can be handled by |
| # CROS_RUST_REMOVE_TARGET_CFG. This requires that there not be complicated cfg blocks since the cros-rust eclass |
| # only applies simplistic filtering. |
| has_filterable_target = False |
| has_unfilterable_target = False |
| for dep in dependencies: |
| # Skip all dev dependencies. We will still handle normal and build deps. |
| if dep.get('kind', None) == 'dev': |
| continue |
| |
| target = dep.get('target', None) |
| if target is None: |
| continue |
| |
| if target_applies_to_chromeos(target): |
| continue |
| |
| if not target_is_filterable(target): |
| logging.debug('target "%s" is unfilterable.', target) |
| has_unfilterable_target = True |
| else: |
| has_filterable_target = True |
| if has_unfilterable_target: |
| logging.warning('"%s-%s" might benefit from better target specific dependency filtering', |
| package['name'], package['version']) |
| filter_target = has_filterable_target and not has_unfilterable_target |
| |
| graph_node = DepGraphNode(package) |
| self.graph[package['name']][package['version']] = graph_node |
| |
| # Check each dependency and add it if it is relevant to Chrome OS. |
| deps = [] |
| for dep in dependencies: |
| # Skip all dev dependencies. We will still handle normal and build deps. |
| if dep.get('kind', None) == 'dev': |
| continue |
| |
| # Skip filterable configuration dependant dependencies. |
| target = dep.get('target', None) |
| if filter_target and target is not None and not target_applies_to_chromeos(target): |
| continue |
| |
| graph_node.add_dependency(dep) |
| deps.append(dep) |
| return filter_target, deps |
| |
| def find_system_crates(self, target_dir, names=None): |
| """Take note of crates already present in the target directory. |
| |
| Args: |
| target_dir: The overlay directory where the ebuilds exist. |
| names: A list of package names to check. If unspecified all the crates present in the DepGraph are checked. |
| """ |
| if names is None: |
| names = self.graph |
| |
| for name in names: |
| # Skip if we have already done this crate. |
| if name in self.system_crates: |
| continue |
| available = self.system_crates[name] |
| |
| # Check if an empty package already has a non-empty ebuild. |
| ebuild_target_dir = get_ebuild_dir(name, target_dir) |
| |
| # Find all ebuilds in ebuild_target dir and if they have |
| # CROS_RUST_EMPTY_CRATE in them, check if features are set. |
| for root, _, files in os.walk(ebuild_target_dir): |
| files.sort() |
| for efile in files: |
| if not efile.endswith('.ebuild'): |
| continue |
| efile_path = os.path.join(root, efile) |
| |
| m = re.match(EBUILD_RE, efile) |
| if not m or m.group('name') != name: |
| logging.warning("Found misplaced ebuild: '%s'.", efile_path) |
| continue |
| |
| version = m.group('version') |
| |
| with open(efile_path, 'r') as ebuild: |
| contents = ebuild.read() |
| if 'CROS_RUST_EMPTY_CRATE' not in contents: |
| features = None |
| else: |
| m = re.search(FEATURES_RE, contents, re.MULTILINE) |
| if m: |
| sanitized = re.sub('[^a-zA-Z0-9_ \t\n-]+', '', m.group('features')) |
| features = set(a.strip() for a in sanitized.split() if a) |
| else: |
| features = set() |
| logging.debug("Found ebuild: '%s'.", efile_path[len(target_dir):]) |
| available[version] = features |
| |
| def set_root(self, package): |
| """Set the root of the dependency graph used when flattening the graph into a list of dependencies.""" |
| self.root = package |
| |
| def resolve_version_range(self, name, version_range): |
| """Find a dependency graph entry that fits the constraints. |
| |
| Optional dependencies often are not found, so the minimum version in version_range is returned in those cases. |
| |
| Args: |
| name: crate name |
| version_range: A VersionRange used to match against crate versions. |
| |
| Returns: |
| version: Returns the version of the match or the minimum matching version if no match is found. |
| node: None if no match is found otherwise returns the DepGraphNode of the match. |
| """ |
| for version, node in self.graph[name].items(): |
| if version_range.contains(version): |
| return version, node |
| if not version_range.min: |
| return '0.0.0', None |
| return '.'.join(map(str, version_range.min)), None |
| |
| def check_dependencies(self, args, package, features, optional, features_only=False): |
| """Check the dependencies of a specified package with the specified features enabled. |
| |
| Args: |
| args: The command line flags set for cargo2ebuild. |
| package: A parsed package from the Cargo metadata. |
| features: A set containing the enabled features for this package. This will be expanded by the function to |
| include any additional features implied by the original set of features (e.g. if default is set, all the |
| default features will be added). |
| optional: A dictionary of sets to which any new optional dependencies are added. It has the format: |
| (package_name, package_version): set() |
| where the set contains the required features. |
| features_only: If True, this function only checks the optional dependencies for ones that are enabled if |
| the features specified by `features` are enabled. |
| |
| Returns: |
| A list of (package_name, package_version) that are required by the specified package with the specified |
| features enabled. |
| """ |
| name = package['name'] |
| version = package['version'] |
| node = self.graph[name][version] |
| |
| enabled_optional_deps = get_feature_dependencies(package, features) |
| |
| new_required = [] |
| for dep in node.dependencies: |
| dep_name = dep['name'] |
| if features_only and dep_name not in enabled_optional_deps: |
| continue |
| req = VersionRange.from_str(dep['req']) |
| dep_version, dep_node = self.resolve_version_range(dep_name, req) |
| identifier = (dep_name, dep_version) |
| |
| dep_features = set(dep['features']) |
| if dep_node and dep.get('uses_default_features', False): |
| dep_features.add('default') |
| |
| is_required = not dep.get('optional', False) or dep_name in enabled_optional_deps |
| if dep_name in enabled_optional_deps: |
| if not features_only: |
| logging.info('Optional dependency required by feature: "%s-%s"', dep_name, dep_version) |
| dep_features.update(enabled_optional_deps[dep_name]) |
| |
| found = False |
| sys_version = None |
| repl_versions = [] |
| if dep_name not in self.system_crates: |
| self.find_system_crates(args.target_dir, names=[dep_name]) |
| for sys_version, sys_features in self.system_crates[dep_name].items(): |
| sys_crate_is_empty = sys_features is not None |
| missing_features = dep_features.difference(sys_features) if sys_crate_is_empty else set() |
| if not args.overwrite_existing_ebuilds and sys_version == dep_version: |
| if sys_crate_is_empty: |
| if is_required: |
| logging.error('Empty crate "%s-%s" should be replaced with full version.', |
| dep_name, dep_version) |
| elif missing_features: |
| logging.error('Empty crate "%s-%s" has missing features: %s.', |
| dep_name, dep_version, missing_features) |
| found = True |
| break |
| if not sys_crate_is_empty and req.contains(sys_version): |
| if not args.no_substitute or (not is_required and dep_node is None): |
| found = True |
| break |
| if VersionRange.from_str('^{}'.format(sys_version)).contains(dep_version): |
| if sys_crate_is_empty: |
| dep_features.update(sys_features) |
| else: |
| repl_versions.append(sys_version) |
| if found and not features_only: |
| logging.debug('Using system crate: "%s-%s".', dep_name, sys_version) |
| continue |
| |
| if is_required: |
| if dep_node is None: |
| logging.error('Required crate "%s" "%s" not in dep-graph. It will be omitted.', |
| dep_name, dep['req']) |
| else: |
| logging.debug('New crate required: "%s-%s".', dep_name, dep_version) |
| new_required.append((dep_node.package, dep_features)) |
| else: |
| # If the empty crate would replace a non-empty version of the same crate, treat it as required. |
| if repl_versions: |
| if dep_node is not None: |
| logging.info('Empty crate for "%s-%s" would replace non-empty versions %s. ' |
| 'Upgrading to non-empty.', dep_name, dep_version, repl_versions) |
| new_required.append((dep_node.package, dep_features)) |
| else: |
| logging.error('Required crate "%s-%s" not in metadata. Fix: add it to Cargo.toml', |
| dep_name, dep_version) |
| else: |
| logging.debug('New optional crate required: "%s-%s".', dep_name, dep_version) |
| # Include all features supported by the crate. |
| if dep_node is not None: |
| dep_features = dep_node.package['features'].keys() |
| optional[identifier].update(dep_features if features else ()) |
| return new_required |
| |
| def flatten(self, args): |
| """Flatten the dependency graph into a list of required and optional crates. |
| |
| Returns: |
| Two dictionaries that map tuples of package names and versions to sets of features: |
| (name, version) -> set(features) |
| The first dictionary contains the required crates with their required features, while the second contains |
| the optional crates, versions, and their features. |
| """ |
| # (name, version) -> set(features) |
| required = defaultdict(set) |
| optional = defaultdict(set) |
| remaining = [(self.root, {'default'})] |
| while remaining: |
| to_check, features = remaining.pop(0) |
| name = to_check['name'] |
| version = to_check['version'] |
| |
| identifier = (name, version) |
| if identifier in required: |
| features_to_enable = features.difference(required[identifier]) |
| if not features_to_enable: |
| continue |
| logging.debug('Extending dependencies for "%s-%s" to include features: "%s"', |
| name, version, features_to_enable) |
| remaining.extend(self.check_dependencies(args, to_check, features_to_enable, optional, |
| features_only=True)) |
| else: |
| logging.debug('Resolving dependencies for "%s-%s".', name, version) |
| remaining.extend(self.check_dependencies(args, to_check, features, optional)) |
| required[identifier].update(features) |
| |
| return dict(required), {a: b for a, b in optional.items() if a not in required} |
| |
| |
| def target_applies_to_chromeos(target): |
| """Return true if the target would be accepted by the cros-rust eclass.""" |
| return ( |
| target.startswith('cfg(unix') or |
| target.startswith('cfg(linux') or |
| target.startswith('cfg(not(windows)') or |
| '-linux-gnu' in target |
| ) |
| |
| |
| def target_is_filterable(target): |
| """Checks to see if the cros-rust eclass would probably handle this target properly. |
| |
| Returns: |
| False if the Cargo.toml target configuration block would not be accepted by the cros-rust eclass, but is probably needed. |
| """ |
| if target_applies_to_chromeos(target): |
| return True |
| if 'unix' in target or 'linux' in target or 'not(' in target: |
| return False |
| return True |
| |
| |
| def get_feature_dependencies(package, features): |
| """Expand the features set to include implied features and return the enabled optional dependencies. |
| |
| Args: |
| package: A parsed package from the Cargo metadata. |
| features: A set() of features that is expanded to include any additional dependencies implied by the original |
| set of features. |
| |
| Returns: |
| A dictionary that maps crate names to the required features for the specific crate. |
| Note: that no version is supplied; it must be obtained by finding the named dependency in the package's |
| optional dependency requirements. |
| """ |
| feature_list = list(features) |
| enabled_deps = defaultdict(set) |
| lookup = package.get('features', {}) |
| for feature in feature_list: |
| if feature not in lookup: |
| if feature != 'default': |
| logging.error('Requested feature "%s" not listed in crate "%s".', |
| feature, package['name']) |
| continue |
| for dep in lookup[feature]: |
| # Check if it is a feature instead of a dependency. |
| if dep in lookup: |
| if dep not in features: |
| feature_list.append(dep) |
| features.add(dep) |
| continue |
| parts = dep.split('/') |
| |
| # This creates an empty set if there is not one already. |
| entry = enabled_deps[parts[0]] |
| if len(parts) > 1: |
| entry.add(parts[1]) |
| return dict(enabled_deps) |
| |
| |
| def prepare_staging(args): |
| """Prepare staging directory.""" |
| sdir = args.staging_dir |
| dirs = [ |
| os.path.join(sdir, 'ebuild', 'dev-rust'), |
| os.path.join(sdir, 'crates') |
| ] |
| for d in dirs: |
| os.makedirs(d, exist_ok=True) |
| |
| |
| def load_metadata(manifest_path): |
| """Run cargo metadata and get metadata for build.""" |
| cwd = os.path.dirname(manifest_path) |
| cmd = [ |
| 'cargo', 'metadata', '--format-version', '1', '--manifest-path', |
| manifest_path |
| ] |
| output = subprocess.check_output(cmd, cwd=cwd) |
| |
| return json.loads(output) |
| |
| |
| def get_crate_path(package, staging_dir): |
| """Get path to crate in staging directory.""" |
| return os.path.join( |
| staging_dir, 'crates', '{}-{}.crate'.format(package['name'], |
| package['version'])) |
| |
| |
| def get_clean_crate_name(package): |
| """Clean up crate name to {name}-{major}.{minor}.{patch}.""" |
| return '{}-{}.crate'.format(package['name'], |
| get_clean_package_version(package['version'])) |
| |
| |
| def version_to_tuple(version, missing=-1): |
| """Extract dependency type and semver from a given version string.""" |
| def version_to_int(num): |
| if not num or num == '*': |
| return missing |
| return int(num) |
| |
| m = re.match(VERSION_RE, version) |
| if not m: |
| raise VersionParseError( |
| 'Invalid SemVer: {}'.format(version)) |
| |
| dep = m.group('dep') |
| major = m.group('major') |
| minor = m.group('minor') |
| patch = m.group('patch') |
| |
| has_star = any([x == '*' for x in [major, minor, patch]]) |
| |
| major = version_to_int(major) |
| minor = version_to_int(minor) |
| patch = version_to_int(patch) |
| |
| if has_star: |
| dep = '~' |
| elif not dep: |
| dep = '^' |
| |
| return dep, major, minor, patch |
| |
| |
| def get_clean_package_version(version): |
| """Get package version in the format {major}.{minor}.{patch}.""" |
| (_, major, minor, patch) = version_to_tuple(version, missing=0) |
| return '{}.{}.{}'.format(major, minor, patch) |
| |
| |
| def get_ebuild_dir(name, staging_dir): |
| """Get the directory that contains specific ebuilds.""" |
| return os.path.join(staging_dir, 'dev-rust', name) |
| |
| |
| def get_ebuild_path(name, version, staging_dir, make_dir=False): |
| """Get path to ebuild in given directory.""" |
| ebuild_dir = get_ebuild_dir(name, staging_dir) |
| |
| ebuild_path = os.path.join( |
| ebuild_dir, |
| '{}-{}.ebuild'.format(name, get_clean_package_version(version))) |
| |
| if make_dir: |
| os.makedirs(ebuild_dir, exist_ok=True) |
| |
| return ebuild_path |
| |
| |
| def download_package(package, staging_dir): |
| """Download the crate from crates.io.""" |
| dl_uri = CRATE_DL_URI.format(PN=package['name'], PV=package['version']) |
| crate_path = get_crate_path(package, staging_dir) |
| |
| # Already downloaded previously |
| if os.path.isfile(crate_path): |
| return |
| |
| ret = subprocess.run( |
| ['curl', '-L', dl_uri, '-o', crate_path], |
| stdout=subprocess.DEVNULL, |
| stderr=subprocess.DEVNULL).returncode |
| |
| if ret: |
| logging.error('Failed to download "%s": %s', dl_uri, ret) |
| |
| |
| def get_description(package): |
| """Get a description of the crate from metadata.""" |
| if package.get('description', None): |
| desc = re.sub("[`']", '"', package['description']) |
| return desc.strip() |
| |
| return '' |
| |
| |
| def get_homepage(package): |
| """Get the homepage of the crate from metadata or use crates.io.""" |
| if package.get('homepage', None): |
| return package['homepage'] |
| |
| return 'https://crates.io/crates/{}'.format(package['name']) |
| |
| |
| def convert_license(cargo_license, package): |
| """Convert licenses from cargo to a format usable in ebuilds.""" |
| cargo_license = '' if not cargo_license else cargo_license |
| has_or = ' OR ' in cargo_license |
| delim = ' OR ' if has_or else '/' |
| |
| found = [a.strip() for a in cargo_license.split(delim) if a] |
| licenses_or = [] |
| for f in found: |
| if f in LICENSES: |
| licenses_or.append(LICENSES[f]) |
| |
| if not licenses_or: |
| logging.error('"%s" is missing an appropriate license: "%s"', package['name'], cargo_license) |
| return "$(die 'Please replace with appropriate license')" |
| |
| if len(licenses_or) > 1: |
| lstr = '|| ( {} )'.format(' '.join(licenses_or)) |
| else: |
| lstr = licenses_or[0] |
| |
| return lstr |
| |
| |
| def convert_dependencies(dependencies, filter_target=False): |
| """Convert crate dependencies to ebuild dependencies.""" |
| deps = [] |
| for dep in dependencies: |
| # Convert version requirement to ebuild DEPEND. |
| try: |
| # Convert requirement to version tuple |
| bounds = ( |
| VersionRange.from_str(dep['req']) |
| .to_ebuild_str('dev-rust/{}'.format(dep['name'])) |
| ) |
| deps.append('\t{}'.format(bounds)) |
| except VersionParseError: |
| logging.error('Failed to parse dep version for "%s": %s', |
| dep['name'], dep['req'], exc_info=True) |
| # Rarely dependencies look something like ">=0.6, <0.8" |
| deps.append("\t$(die 'Please replace with proper DEPEND: {} = {}')".format( |
| dep['name'], dep['req'])) |
| |
| remove_target_var = '\nCROS_RUST_REMOVE_TARGET_CFG=1' if filter_target else '' |
| if not deps: |
| return '', remove_target_var |
| |
| # Add DEPEND= into template with all dependencies |
| # RDEPEND="${DEPEND}" required for race in cros-rust |
| fmtstring = 'DEPEND="\n{}\n"\nRDEPEND="${{DEPEND}}"\n' |
| return fmtstring.format('\n'.join(deps)), remove_target_var |
| |
| |
| def package_ebuild(package, ebuild_dir, crate_dependencies, filter_target): |
| """Create ebuild from metadata and write to ebuild directory.""" |
| logging.debug('Processing "%s-%s"', package['name'], package['version']) |
| ebuild_path = get_ebuild_path(package['name'], package['version'], ebuild_dir, make_dir=True) |
| |
| autogen_notice = AUTOGEN_NOTICE |
| |
| # Check if version matches clean version or modify the autogen notice |
| if package['version'] != get_clean_package_version(package['version']): |
| autogen_notice = '\n'.join([ |
| autogen_notice, |
| '# ${{PV}} was changed from the original {}'.format( |
| package['version']) |
| ]) |
| |
| (ebuild_dependencies, remove_target_var) = convert_dependencies(crate_dependencies, filter_target) |
| template_info = { |
| 'copyright_year': datetime.now().year, |
| 'remove_target_var': remove_target_var, |
| 'description': get_description(package), |
| 'homepage': get_homepage(package), |
| 'license': convert_license(package['license'], package), |
| 'dependencies': ebuild_dependencies, |
| 'autogen_notice': autogen_notice, |
| } |
| |
| with open(ebuild_path, 'w') as ebuild: |
| ebuild.write(EBUILD_TEMPLATE.format(**template_info)) |
| |
| |
| def upload_gsutil(package, staging_dir, no_upload=False): |
| """Upload crate to distfiles.""" |
| if no_upload: |
| return |
| |
| crate_path = get_crate_path(package, staging_dir) |
| crate_name = get_clean_crate_name(package) |
| ret = subprocess.run([ |
| 'gsutil', 'cp', '-a', 'public-read', crate_path, |
| 'gs://chromeos-localmirror/distfiles/{}'.format(crate_name) |
| ]).returncode |
| |
| if ret: |
| logging.error('Failed to upload "%s" to chromeos-localmirror: %s', crate_name, ret) |
| |
| |
| def update_ebuild(package, args): |
| """Update ebuild with generated one and generate MANIFEST.""" |
| staging_dir = args.staging_dir |
| ebuild_dir = os.path.join(staging_dir, 'ebuild') |
| target_dir = args.target_dir |
| ebuild_src = get_ebuild_path(package['name'], package['version'], ebuild_dir) |
| ebuild_dest = get_ebuild_path(package['name'], package['version'], target_dir, make_dir=True) |
| |
| # TODO make this aware of the revision numbers of existing versions |
| # when doing a replacement. |
| |
| # Do not overwrite existing ebuilds unless explicitly asked to. |
| if args.overwrite_existing_ebuilds or not os.path.exists(ebuild_dest): |
| shutil.copy(ebuild_src, ebuild_dest) |
| upload_gsutil(package, staging_dir, no_upload=args.no_upload) |
| else: |
| logging.info('ebuild %s already exists, skipping.', ebuild_dest) |
| |
| # Generate manifest w/ ebuild digest |
| ret = subprocess.run(['ebuild', ebuild_dest, 'digest']).returncode |
| if ret: |
| logging.error('ebuild %s digest failed: %s', ebuild_dest, ret) |
| |
| |
| def process_package(package, staging_dir, dep_graph): |
| """Process each package listed in the metadata. |
| |
| This includes following: |
| * Setting the package as the root of the dep graph if it is included by source (rather than from crates.io). |
| * Adding the package to the the dep_graph. |
| * Downloading the crate file, so it can be potentially uploaded to localmirror. |
| * Generating an ebuild for the crate in the staging directory. |
| """ |
| ebuild_dir = os.path.join(staging_dir, 'ebuild') |
| |
| # Add the user submitted package to the set of required packages. |
| if package.get('source', None) is None: |
| dep_graph.set_root(package) |
| filter_target, crate_dependencies = dep_graph.add_package(package) |
| |
| download_package(package, staging_dir) |
| package_ebuild(package, ebuild_dir, crate_dependencies, filter_target) |
| |
| |
| def process_empty_package(name, version, features, args): |
| """Process packages that should generate empty ebuilds.""" |
| staging_dir = args.staging_dir |
| ebuild_dir = os.path.join(staging_dir, 'ebuild') |
| target_dir = args.target_dir |
| |
| ebuild_src = get_ebuild_path(name, version, ebuild_dir, make_dir=True) |
| ebuild_dest = get_ebuild_path(name, version, target_dir, make_dir=True) |
| |
| crate_features = '' |
| if features: |
| crate_features = 'CROS_RUST_EMPTY_CRATE_FEATURES=( {} )'.format( |
| ' '.join(['"{}"'.format(x) for x in features])) |
| |
| template_info = { |
| 'copyright_year': datetime.now().year, |
| 'crate_features': crate_features, |
| 'autogen_notice': AUTOGEN_NOTICE, |
| } |
| |
| logging.debug('Writing empty crate: %s', ebuild_src) |
| with open(ebuild_src, 'w') as ebuild: |
| ebuild.write(EMPTY_CRATE.format(**template_info)) |
| |
| # Do not overwrite existing ebuilds unless explicitly asked to. |
| if not args.overwrite_existing_ebuilds and os.path.exists(ebuild_dest): |
| logging.info('ebuild %s already exists, skipping.', ebuild_dest) |
| return |
| |
| if not args.dry_run and name not in args.skip: |
| shutil.copy(ebuild_src, ebuild_dest) |
| |
| |
| def main(argv): |
| """Convert dependencies from Cargo.toml into ebuilds.""" |
| args = parse_args(argv) |
| |
| logging_kwargs = {'stream': sys.stderr, 'format': '%(levelname)s: %(message)s'} |
| if args.verbose > 1: |
| logging_kwargs['level'] = logging.DEBUG |
| elif args.verbose > 0: |
| logging_kwargs['level'] = logging.INFO |
| else: |
| logging_kwargs['level'] = logging.WARNING |
| logging.basicConfig(**logging_kwargs) |
| |
| prepare_staging(args) |
| |
| dep_graph = DepGraph() |
| metadata = load_metadata(args.manifest_path) |
| for p in metadata['packages']: |
| process_package(p, args.staging_dir, dep_graph) |
| |
| dep_graph.find_system_crates(args.target_dir) |
| |
| required_packages, optional_packages = dep_graph.flatten(args) |
| if args.verbose > 2: |
| print('Dependency graph:') |
| pprint(dict(dep_graph.graph)) |
| print('System versions:') |
| pprint(dict(dep_graph.system_crates)) |
| print('Required versions:') |
| pprint(required_packages) |
| print('Optional versions:') |
| pprint(optional_packages) |
| |
| for (name, version), features in optional_packages.items(): |
| process_empty_package(name, version, features, args) |
| |
| if not args.dry_run: |
| for p in metadata['packages']: |
| if p['name'] in args.skip: |
| continue |
| identifier = (p['name'], p['version']) |
| if identifier in required_packages: |
| update_ebuild(p, args) |
| display_dir = args.target_dir |
| else: |
| display_dir = '/'.join((args.staging_dir, 'ebuild')) |
| print('Generated ebuilds can be found in: {}'.format(display_dir)) |
| |
| |
| def parse_args(argv): |
| """Parse the command-line arguments.""" |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument( |
| '-s', |
| '--staging-dir', |
| default='/tmp/cargo2ebuild-staging', |
| help='Staging directory for temporary and generated files') |
| |
| parser.add_argument( |
| '-d', |
| '--dry-run', |
| action='store_true', |
| help='Generate dependency tree but do not upload anything') |
| |
| parser.add_argument('-t', |
| '--target-dir', |
| default='/mnt/host/source/src/third_party/chromiumos-overlay', |
| help='Path to chromiumos-overlay') |
| |
| parser.add_argument('-k', |
| '--skip', |
| action='append', |
| help='Skip these packages when updating ebuilds') |
| parser.add_argument('-n', |
| '--no-upload', |
| action='store_true', |
| help='Skip uploading crates to distfiles') |
| parser.add_argument('-X', |
| '--no-substitute', |
| action='store_true', |
| help='Do not substitute system versions for required crates') |
| parser.add_argument('-x', |
| '--overwrite-existing-ebuilds', |
| action='store_true', |
| help='If an ebuild already exists, overwrite it.') |
| parser.add_argument('-v', |
| '--verbose', |
| action='count', |
| default=0, |
| help='Enable verbose logging') |
| |
| parser.add_argument('manifest_path', |
| nargs='?', |
| default='./Cargo.toml', |
| help='Cargo.toml used to generate ebuilds.') |
| |
| args = parser.parse_args(argv) |
| |
| # Require target directory if not dry run |
| if not args.target_dir and not args.dry_run: |
| raise Exception('Target directory must be set unless dry-run is True.') |
| |
| if not args.skip: |
| args.skip = [] |
| |
| return args |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
