From 17498d5bea1c6261e354a767f2553e411d4d7237 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 8 Dec 2024 14:51:50 -0800 Subject: [PATCH 1/2] Add `--elide-unused-requires-dist` lock option. This cuts down lock file size without changing any observable result of using the lock file. It should cut down on lock subset times since there is both less to parse and less dependencies to rule out after the parse, but this effect is unmeasured at present. --- CHANGES.md | 10 ++ package/pex-scie.lock | 40 +---- pex/cli/commands/lock.py | 52 +++++- pex/resolve/lock_downloader.py | 7 +- pex/resolve/locked_resolve.py | 1 + pex/resolve/lockfile/create.py | 9 +- pex/resolve/lockfile/json_codec.py | 11 +- pex/resolve/lockfile/model.py | 30 +++- pex/resolve/lockfile/requires_dist.py | 162 +++++++++++++++++ pex/resolve/lockfile/updater.py | 54 ++++-- pex/version.py | 2 +- scripts/gen-scie-platform.py | 67 +++---- tests/integration/cli/commands/test_export.py | 1 + .../test_lock_elide_unused_requires_dist.py | 96 ++++++++++ tests/resolve/lockfile/test_requires_dist.py | 170 ++++++++++++++++++ 15 files changed, 614 insertions(+), 98 deletions(-) create mode 100644 pex/resolve/lockfile/requires_dist.py create mode 100644 tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py create mode 100644 tests/resolve/lockfile/test_requires_dist.py diff --git a/CHANGES.md b/CHANGES.md index 3b375e9e6..2e0352d37 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Release Notes +## 2.25.0 + +This release adds support for +`pex3 lock {create,sync} --elide-unused-requires-dist`. This new lock +option causes any dependencies of a locked requirement that can never +be activated to be elided from the lock file. This leads to no material +difference in lock file use, but it does cut down on the lock file size. + +* Add `--elide-unused-requires-dist` lock option. (#2613) + ## 2.24.3 This release fixes a long-standing bug in resolve checking. Previously, diff --git a/package/pex-scie.lock b/package/pex-scie.lock index 62b50791d..a93d31c10 100644 --- a/package/pex-scie.lock +++ b/package/pex-scie.lock @@ -4,6 +4,7 @@ "allow_wheels": true, "build_isolation": true, "constraints": [], + "elide_unused_requires_dist": true, "excluded": [], "locked_resolves": [ { @@ -17,13 +18,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -45,13 +40,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -73,13 +62,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -101,13 +84,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -123,8 +100,8 @@ "only_wheels": [], "overridden": [], "path_mappings": {}, - "pex_version": "2.17.0", - "pip_version": "24.2", + "pex_version": "2.24.3", + "pip_version": "24.3.1", "prefer_older_binary": false, "requirements": [ "psutil>=5.3" @@ -134,5 +111,6 @@ "style": "strict", "target_systems": [], "transitive": true, - "use_pep517": null + "use_pep517": null, + "use_system_time": false } diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 318da3a6e..942de3eac 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -76,6 +76,8 @@ from typing import IO, Dict, Iterable, List, Mapping, Optional, Set, Text, Tuple, Union import attr # vendor:skip + + from pex.resolve.lockfile.updater import Update else: from pex.third_party import attr @@ -526,6 +528,20 @@ def add_create_lock_options(cls, create_parser): ) ), ) + create_parser.add_argument( + "--elide-unused-requires-dist", + "--no-elide-unused-requires-dist", + dest="elide_unused_requires_dist", + type=bool, + default=False, + action=HandleBoolAction, + help=( + "When creating the lock, elide dependencies from the 'requires_dists' lists that " + "can never be active due to markers. This does not change the reachable content of " + "the lock, but it does cut down on lock file size. This currently only elides " + "extras deps that are never activated, but may trim more in the future." + ), + ) cls._add_lock_options(create_parser) cls._add_resolve_options(create_parser) cls.add_json_options(create_parser, entity="lock", include_switch=False) @@ -899,6 +915,7 @@ def _create(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -907,7 +924,10 @@ def _create(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) targets = try_( self._resolve_targets( @@ -1242,7 +1262,7 @@ def _process_lock_update( dry_run = self.options.dry_run path_mappings = self._get_path_mappings() output = sys.stdout if dry_run is DryRunStyle.DISPLAY else sys.stderr - updates = [] # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]] + updates = [] # type: List[Update] warnings = [] # type: List[str] for resolve_update in lock_update.resolves: platform = resolve_update.updated_resolve.target_platform @@ -1317,7 +1337,7 @@ def _process_lock_update( ) if update_req: requirements_by_project_name[project_name] = update_req - else: + elif isinstance(update, ArtifactsUpdate): message_lines = [ " {lead_in} {project_name} {version} artifacts:".format( lead_in="Would update" if dry_run else "Updated", @@ -1351,8 +1371,25 @@ def _process_lock_update( ) for artifact in update.removed ) - print("\n".join(message_lines), file=output) + else: + message_lines = [ + " {lead_in} {project_name} {version} requirements:".format( + lead_in="Would update" if dry_run else "Updated", + project_name=project_name, + version=update.version, + ) + ] + if update.added: + message_lines.extend( + " + {added}".format(added=req) for req in update.added + ) + if update.removed: + message_lines.extend( + " - {removed}".format(removed=req) for req in update.removed + ) + print("\n".join(message_lines), file=output) + if fingerprint_updates: warnings.append( "Detected fingerprint changes in the following locked {projects} for lock " @@ -1547,6 +1584,7 @@ def _sync(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -1555,7 +1593,10 @@ def _sync(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) lock_file_path = self.options.lock if os.path.exists(lock_file_path): @@ -1566,6 +1607,7 @@ def _sync(self): style=lock_configuration.style, requires_python=SortedTuple(lock_configuration.requires_python), target_systems=SortedTuple(lock_configuration.target_systems), + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, pip_version=pip_configuration.version, resolver_version=pip_configuration.resolver_version, allow_prereleases=pip_configuration.allow_prereleases, diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index aa83d01ac..a89885eb7 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -25,7 +25,6 @@ DownloadableArtifact, FileArtifact, LocalProjectArtifact, - LockConfiguration, VCSArtifact, ) from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager @@ -233,11 +232,7 @@ def create( file_lock_style=file_lock_style, downloader=ArtifactDownloader( resolver=resolver, - lock_configuration=LockConfiguration( - style=lock.style, - requires_python=lock.requires_python, - target_systems=lock.target_systems, - ), + lock_configuration=lock.lock_configuration(), target=target, package_index_configuration=PackageIndexConfiguration.create( pip_version=pip_version, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 684a73f0c..a57edd272 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -87,6 +87,7 @@ class LockConfiguration(object): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib(default=()) # type: Tuple[str, ...] target_systems = attr.ib(default=()) # type: Tuple[TargetSystem.Value, ...] + elide_unused_requires_dist = attr.ib(default=False) # type: bool @requires_python.validator @target_systems.validator diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 5d153121c..220092665 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -341,9 +341,11 @@ def lock(self, downloaded): package_index_configuration=self.package_index_configuration, max_parallel_jobs=self.max_parallel_jobs, ), - platform_tag=None - if self.lock_configuration.style == LockStyle.UNIVERSAL - else target.platform.tag, + platform_tag=( + None + if self.lock_configuration.style == LockStyle.UNIVERSAL + else target.platform.tag + ), ) for target, resolved_requirements in resolved_requirements_by_target.items() ) @@ -456,6 +458,7 @@ def create( excluded=dependency_configuration.excluded, overridden=dependency_configuration.all_overrides(), locked_resolves=locked_resolves, + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, ) if lock_configuration.style is LockStyle.UNIVERSAL and ( diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py index b8283ce79..ddfe4bde6 100644 --- a/pex/resolve/lockfile/json_codec.py +++ b/pex/resolve/lockfile/json_codec.py @@ -209,6 +209,8 @@ def parse_version_specifier( for index, target_system in enumerate(get("target_systems", list, optional=True) or ()) ] + elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False + only_wheels = [ parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index)) for index, project_name in enumerate(get("only_wheels", list, optional=True) or ()) @@ -231,7 +233,7 @@ def parse_version_specifier( for index, constraint in enumerate(get("constraints", list)) ] - use_system_time = get("use_system_time", bool, optional=True) + use_system_time = get("use_system_time", bool, optional=True) or False excluded = [ parse_requirement(req, path=".excluded[{index}]".format(index=index)) @@ -337,6 +339,7 @@ def assemble_tag( style=get_enum_value(LockStyle, "style"), requires_python=get("requires_python", list), target_systems=target_systems, + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=get_enum_value( PipVersion, "pip_version", @@ -354,10 +357,7 @@ def assemble_tag( prefer_older_binary=get("prefer_older_binary", bool), use_pep517=get("use_pep517", bool, optional=True), build_isolation=get("build_isolation", bool), - # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and - # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In - # those old locks there was no "use_system_time" field. - use_system_time=use_system_time if use_system_time is not None else True, + use_system_time=use_system_time, ), transitive=get("transitive", bool), excluded=excluded, @@ -391,6 +391,7 @@ def as_json_data( "style": str(lockfile.style), "requires_python": list(lockfile.requires_python), "target_systems": [str(target_system) for target_system in lockfile.target_systems], + "elide_unused_requires_dist": lockfile.elide_unused_requires_dist, "pip_version": str(lockfile.pip_version), "resolver_version": str(lockfile.resolver_version), "requirements": [ diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index bb309580e..36293fa34 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -11,7 +11,14 @@ from pex.pep_503 import ProjectName from pex.pip.version import PipVersion, PipVersionValue from pex.requirements import LocalProjectRequirement -from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem +from pex.resolve.locked_resolve import ( + LocalProjectArtifact, + LockConfiguration, + LockedResolve, + LockStyle, + TargetSystem, +) +from pex.resolve.lockfile import requires_dist from pex.resolve.resolved_requirement import Pin from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion from pex.sorted_tuple import SortedTuple @@ -47,6 +54,7 @@ def create( source=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver_version=None, # type: Optional[ResolverVersion.Value] + elide_unused_requires_dist=False, # type: bool ): # type: (...) -> Lockfile @@ -94,6 +102,7 @@ def extract_requirement(req): style=style, requires_python=SortedTuple(requires_python), target_systems=SortedTuple(target_systems), + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=pip_ver, resolver_version=resolver_version or ResolverVersion.default(pip_ver), requirements=SortedTuple(resolve_requirements, key=str), @@ -110,7 +119,14 @@ def extract_requirement(req): transitive=transitive, excluded=SortedTuple(excluded), overridden=SortedTuple(overridden), - locked_resolves=SortedTuple(locked_resolves), + locked_resolves=SortedTuple( + ( + requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve) + if elide_unused_requires_dist + else locked_resolve + ) + for locked_resolve in locked_resolves + ), local_project_requirement_mapping=requirement_by_local_project_directory, source=source, ) @@ -119,6 +135,7 @@ def extract_requirement(req): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib() # type: SortedTuple[str] target_systems = attr.ib() # type: SortedTuple[TargetSystem.Value] + elide_unused_requires_dist = attr.ib() # type: bool pip_version = attr.ib() # type: PipVersionValue resolver_version = attr.ib() # type: ResolverVersion.Value requirements = attr.ib() # type: SortedTuple[Requirement] @@ -139,6 +156,15 @@ def extract_requirement(req): local_project_requirement_mapping = attr.ib(eq=False) # type: Mapping[str, Requirement] source = attr.ib(default=None, eq=False) # type: Optional[str] + def lock_configuration(self): + # type: () -> LockConfiguration + return LockConfiguration( + style=self.style, + requires_python=self.requires_python, + target_systems=self.target_systems, + elide_unused_requires_dist=self.elide_unused_requires_dist, + ) + def build_configuration(self): # type: () -> BuildConfiguration return BuildConfiguration.create( diff --git a/pex/resolve/lockfile/requires_dist.py b/pex/resolve/lockfile/requires_dist.py new file mode 100644 index 000000000..410f7177c --- /dev/null +++ b/pex/resolve/lockfile/requires_dist.py @@ -0,0 +1,162 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import operator +from collections import defaultdict, deque + +from pex.dist_metadata import Requirement +from pex.exceptions import production_assert +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement, LockedResolve +from pex.sorted_tuple import SortedTuple +from pex.third_party.packaging.markers import Marker, Variable +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Callable, DefaultDict, Dict, Iterable, List, Optional, Tuple, Union + + import attr # vendor:skip + + EvalExtra = Callable[[ProjectName], bool] +else: + from pex.third_party import attr + + +_OPERATORS = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +class _Op(object): + def __init__(self, lhs): + self.lhs = lhs # type: EvalExtra + self.rhs = None # type: Optional[EvalExtra] + + +class _And(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) and cast("EvalExtra", self.rhs)(extra) + + +class _Or(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) or cast("EvalExtra", self.rhs)(extra) + + +def _parse_extra_item( + stack, # type: List[EvalExtra] + item, # type: Union[str, List, Tuple] + marker, # type: Marker +): + # type: (...) -> None + + if item == "and": + stack.append(_And(stack.pop())) + elif item == "or": + stack.append(_Or(stack.pop())) + elif isinstance(item, list): + for element in item: + _parse_extra_item(stack, element, marker) + elif isinstance(item, tuple): + lhs, op, rhs = item + if isinstance(lhs, Variable) and "extra" == str(lhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(rhs))) + elif isinstance(rhs, Variable) and "extra" == str(rhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(lhs))) + else: + # Any other condition could potentially be true. + check = lambda _: True + if stack: + production_assert(isinstance(stack[-1], _Op)) + cast(_Op, stack[-1]).rhs = check + else: + stack.append(check) + else: + raise ValueError("Marker is invalid: {marker}".format(marker=marker)) + + +def _parse_extra_check(marker): + # type: (Marker) -> EvalExtra + checks = [] # type: List[EvalExtra] + for item in marker._markers: + _parse_extra_item(checks, item, marker) + production_assert(len(checks) == 1) + return checks[0] + + +_EXTRA_CHECKS = {} # type: Dict[str, EvalExtra] + + +def _parse_marker_for_extra_check(marker): + # type: (Marker) -> EvalExtra + maker_str = str(marker) + eval_extra = _EXTRA_CHECKS.get(maker_str) + if not eval_extra: + eval_extra = _parse_extra_check(marker) + _EXTRA_CHECKS[maker_str] = eval_extra + return eval_extra + + +def _evaluate_for_extras( + marker, # type: Optional[Marker] + extras, # type: Iterable[str] +): + # type: (...) -> bool + if not marker: + return True + eval_extra = _parse_marker_for_extra_check(marker) + return any(eval_extra(ProjectName(extra)) for extra in (extras or [""])) + + +def remove_unused_requires_dist( + resolve_requirements, # type: Iterable[Requirement] + locked_resolve, # type: LockedResolve +): + # type: (...) -> LockedResolve + + locked_req_by_project_name = { + locked_req.pin.project_name: locked_req for locked_req in locked_resolve.locked_requirements + } + requires_dist_by_locked_req = defaultdict( + OrderedSet + ) # type: DefaultDict[LockedRequirement, OrderedSet[Requirement]] + seen = set() + requirements = deque(resolve_requirements) + while requirements: + requirement = requirements.popleft() + if requirement in seen: + continue + + seen.add(requirement) + locked_req = locked_req_by_project_name[requirement.project_name] + for dep in locked_req.requires_dists: + if _evaluate_for_extras(dep.marker, requirement.extras): + requires_dist_by_locked_req[locked_req].add(dep) + requirements.append(dep) + + return attr.evolve( + locked_resolve, + locked_requirements=SortedTuple( + attr.evolve( + locked_requirement, + requires_dists=SortedTuple( + requires_dist_by_locked_req[locked_requirement], key=str + ), + ) + for locked_requirement in locked_resolve.locked_requirements + ), + ) diff --git a/pex/resolve/lockfile/updater.py b/pex/resolve/lockfile/updater.py index d66224352..4658777e7 100644 --- a/pex/resolve/lockfile/updater.py +++ b/pex/resolve/lockfile/updater.py @@ -189,12 +189,39 @@ def calculate_updates( removed = attr.ib() # type: Tuple[Artifact, ...] +@attr.s(frozen=True) +class RequirementsUpdate(object): + @classmethod + def calculate( + cls, + version, # type: Version + original, # type: Tuple[Requirement, ...] + updated, # type: Tuple[Requirement, ...] + ): + # type: (...) -> RequirementsUpdate + added = [] # type: List[Requirement] + removed = [] # type: List[Requirement] + for req in updated: + if req not in original: + added.append(req) + for req in original: + if req not in updated: + removed.append(req) + return cls(version=version, added=tuple(added), removed=tuple(removed)) + + version = attr.ib() # type: Version + added = attr.ib() # type: Tuple[Requirement, ...] + removed = attr.ib() # type: Tuple[Requirement, ...] + + +if TYPE_CHECKING: + Update = Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate, RequirementsUpdate] + + @attr.s(frozen=True) class ResolveUpdate(object): updated_resolve = attr.ib() # type: LockedResolve - updates = attr.ib( - factory=dict - ) # type: Mapping[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]] + updates = attr.ib(factory=dict) # type: Mapping[ProjectName, Optional[Update]] @attr.s(frozen=True) @@ -481,9 +508,7 @@ def update_resolve( updated_resolve = updated_lock_file.locked_resolves[0] updated_requirements = updated_lock_file.requirements - updates = ( - OrderedDict() - ) # type: OrderedDict[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]] + updates = OrderedDict() # type: OrderedDict[ProjectName, Optional[Update]] if self.deletes or self.dependency_configuration.excluded: reduced_requirements = [ @@ -533,6 +558,8 @@ def update_resolve( updated_pin = updated_requirement.pin original_artifacts = tuple(locked_requirement.iter_artifacts()) updated_artifacts = tuple(updated_requirement.iter_artifacts()) + original_requirements = locked_requirement.requires_dists + updated_requirements = updated_requirement.requires_dists # N.B.: We use a custom key for artifact equality comparison since `Artifact` # contains a `verified` attribute that can both vary based on Pex's current @@ -613,6 +640,12 @@ def artifacts_differ(): original=original_artifacts, updated=updated_artifacts, ) + elif original_requirements != updated_requirements: + updates[project_name] = RequirementsUpdate.calculate( + version=original_pin.version, + original=original_requirements, + updated=updated_requirements, + ) elif project_name in self.update_constraints_by_project_name: updates[project_name] = None @@ -661,11 +694,6 @@ def create( ): # type: (...) -> LockUpdater - lock_configuration = LockConfiguration( - style=lock_file.style, - requires_python=lock_file.requires_python, - target_systems=lock_file.target_systems, - ) pip_configuration = PipConfiguration( version=lock_file.pip_version, resolver_version=lock_file.resolver_version, @@ -680,7 +708,7 @@ def create( ) return cls( lock_file=lock_file, - lock_configuration=lock_configuration, + lock_configuration=lock_file.lock_configuration(), pip_configuration=pip_configuration, dependency_configuration=dependency_configuration, ) @@ -767,7 +795,7 @@ def _perform_update( ) # type: OrderedDict[Optional[tags.Tag], LockedResolve] resolve_updates_by_platform_tag = ( {} - ) # type: Dict[Optional[tags.Tag], Mapping[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]]] + ) # type: Dict[Optional[tags.Tag], Mapping[ProjectName, Optional[Update]]] # TODO(John Sirois): Consider parallelizing this. The underlying Jobs are down a few layers; # so this will likely require using multiprocessing. diff --git a/pex/version.py b/pex/version.py index d5e8f4a47..0e97cd0b2 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.24.3" +__version__ = "2.25.0" diff --git a/scripts/gen-scie-platform.py b/scripts/gen-scie-platform.py index 70b00a771..cec3866bd 100644 --- a/scripts/gen-scie-platform.py +++ b/scripts/gen-scie-platform.py @@ -195,6 +195,7 @@ def create_lock( ), "--pip-version", "latest", + "--elide-unused-requires-dist", "--indent", "2", "--lock", @@ -285,6 +286,7 @@ def main(out: IO[str]) -> str | int | None: parser.add_argument("--all", action="store_true") parser.add_argument("-f", "--force", action="store_true") parser.add_argument("--lock-file", type=Path, default=PACKAGE_DIR / "pex-scie.lock") + parser.add_argument("-L", "--only-sync-lock", action="store_true") parser.add_argument("-v", "--verbose", action="store_true") try: options = parser.parse_args() @@ -298,39 +300,40 @@ def main(out: IO[str]) -> str | int | None: logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING) generated_files: list[Path] = [] - if options.all: - try: - generated_files.extend( - ensure_all_complete_platforms( - dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + if not options.only_sync_lock: + if options.all: + try: + generated_files.extend( + ensure_all_complete_platforms( + dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + ) ) - ) - except ( - GitHubError, - github.GithubException, - github.BadAttributeException, - httpx.HTTPError, - ) as e: - return str(e) - - try: - create_lock( - lock_file=options.lock_file, - complete_platforms=generated_files, - scie_config=scie_config, - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(options.lock_file) - else: - complete_platform_file = options.dest_dir / f"{plat}.json" - try: - create_complete_platform( - complete_platform_file=complete_platform_file, scie_config=scie_config - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(complete_platform_file) + except ( + GitHubError, + github.GithubException, + github.BadAttributeException, + httpx.HTTPError, + ) as e: + return str(e) + else: + complete_platform_file = options.dest_dir / f"{plat}.json" + try: + create_complete_platform( + complete_platform_file=complete_platform_file, scie_config=scie_config + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(complete_platform_file) + + try: + create_lock( + lock_file=options.lock_file, + complete_platforms=tuple(options.dest_dir.glob("*.json")), + scie_config=scie_config, + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(options.lock_file) for file in generated_files: print(str(file), file=out) diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index 3d616f639..0a5d4423b 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -45,6 +45,7 @@ style=LockStyle.UNIVERSAL, requires_python=SortedTuple(), target_systems=SortedTuple(), + elide_unused_requires_dist=False, pip_version=PipVersion.DEFAULT, resolver_version=ResolverVersion.PIP_2020, requirements=SortedTuple([Requirement.parse("ansicolors")]), diff --git a/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py new file mode 100644 index 000000000..8453d319e --- /dev/null +++ b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py @@ -0,0 +1,96 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from typing import Dict + +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement +from pex.resolve.lockfile import json_codec +from pex.resolve.lockfile.model import Lockfile +from testing.cli import run_pex3 +from testing.pytest.tmp import Tempdir + + +def index_locked_reqs(lockfile): + # type: (Lockfile) -> Dict[ProjectName, LockedRequirement] + return { + locked_req.pin.project_name: locked_req + for locked_resolve in lockfile.locked_resolves + for locked_req in locked_resolve.locked_requirements + } + + +def test_lock_elide_unused_requires_dist(tmpdir): + # type: (Tempdir) -> None + + lock = tmpdir.join("lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--indent", + "2", + "-o", + lock, + ).assert_success() + lockfile = json_codec.load(lock) + + elided_lock = tmpdir.join("elided_lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--elide-unused-requires-dist", + "--indent", + "2", + "-o", + elided_lock, + ).assert_success() + elided_lockfile = json_codec.load(elided_lock) + + assert lockfile != elided_lockfile + + locked_reqs = index_locked_reqs(lockfile) + requests = locked_reqs[ProjectName("requests")] + + elided_locked_reqs = index_locked_reqs(elided_lockfile) + elided_requests = elided_locked_reqs[ProjectName("requests")] + + assert requests != elided_requests + + assert requests.pin == elided_requests.pin + assert list(requests.iter_artifacts()) == list(elided_requests.iter_artifacts()) + assert requests.requires_python == elided_requests.requires_python + + assert requests.requires_dists != elided_requests.requires_dists + assert len(elided_requests.requires_dists) < len(requests.requires_dists) + elided_deps = set(requests.requires_dists) - set(elided_requests.requires_dists) + assert len(elided_deps) > 0 + assert not any( + elided_dep.project_name in elided_locked_reqs for elided_dep in elided_deps + ), "No dependencies that require extra activation should have been locked." + + run_pex3( + "lock", + "sync", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--elide-unused-requires-dist", + "--indent", + "2", + "--lock", + lock, + ).assert_success() + assert elided_lockfile == json_codec.load(lock) diff --git a/tests/resolve/lockfile/test_requires_dist.py b/tests/resolve/lockfile/test_requires_dist.py new file mode 100644 index 000000000..05bd88a48 --- /dev/null +++ b/tests/resolve/lockfile/test_requires_dist.py @@ -0,0 +1,170 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.dist_metadata import Requirement +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import Artifact, LockedRequirement, LockedResolve +from pex.resolve.lockfile import requires_dist +from pex.resolve.resolved_requirement import Fingerprint, Pin +from pex.sorted_tuple import SortedTuple + +req = Requirement.parse + + +def locked_req( + project_name, # type: str + version, # type: str + *requirements # type: str +): + # type: (...) -> LockedRequirement + return LockedRequirement.create( + pin=Pin(project_name=ProjectName(project_name), version=Version(version)), + artifact=Artifact.from_url( + "https://artifact.store/{project_name}-{version}-py2.py3-none-any.whl".format( + project_name=project_name, version=version + ), + fingerprint=Fingerprint(algorithm="md5", hash="abcd0123"), + ), + requires_dists=map(req, requirements), + ) + + +def locked_resolve(*locked_requirements): + # type: (*LockedRequirement) -> LockedResolve + return LockedResolve(locked_requirements=SortedTuple(locked_requirements)) + + +def test_remove_unused_requires_dist_noop(): + # type: () -> None + + locked_resolve_with_no_extras = locked_resolve( + locked_req("foo", "1.0", "bar", "baz"), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + ) + assert locked_resolve_with_no_extras == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], locked_resolve=locked_resolve_with_no_extras + ) + + +def test_remove_unused_requires_dist_simple(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_extras(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ), + ), ( + "The python_version marker clause might evaluate to true, which should be enough to retain " + "the baz dep even though the 'tests' extra is never activated." + ) + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' and python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ), "The 'tests' extra is never active; so the baz dep should never be reached." + + +def test_remove_unused_requires_dist_complex_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "baz; python_version == '3.11.*' and (extra == 'admin' or extra == 'docs')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) From 40d51f4c83eb7183da14667d772f49553e9325c2 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 9 Dec 2024 11:49:08 -0800 Subject: [PATCH 2/2] Fix bad lock test data. The urllib3 Requires-Dist metadata was missed in #1585. --- tests/integration/cli/commands/test_lock.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index ef80ff262..5112a64bd 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -494,7 +494,15 @@ def test_create_universal_platform_check(tmpdir): } ], "project_name": "urllib3", - "requires_dists": [], + "requires_dists": [ + "brotlipy>=0.6.0; extra == \\"brotli\\"", + "pyOpenSSL>=0.14; extra == \\"secure\\"", + "cryptography>=1.3.4; extra == \\"secure\\"", + "idna>=2.0.0; extra == \\"secure\\"", + "certifi; extra == \\"secure\\"", + "ipaddress; python_version == \\"2.7\\" and extra == \\"secure\\"", + "PySocks!=1.5.7,<2.0,>=1.5.6; extra == \\"socks\\"" + ], "requires_python": null, "version": "1.25.11" }