diff --git a/CHANGES.md b/CHANGES.md index 3b375e9e6..2e0352d37 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Release Notes +## 2.25.0 + +This release adds support for +`pex3 lock {create,sync} --elide-unused-requires-dist`. This new lock +option causes any dependencies of a locked requirement that can never +be activated to be elided from the lock file. This leads to no material +difference in lock file use, but it does cut down on the lock file size. + +* Add `--elide-unused-requires-dist` lock option. (#2613) + ## 2.24.3 This release fixes a long-standing bug in resolve checking. Previously, diff --git a/package/pex-scie.lock b/package/pex-scie.lock index 62b50791d..a93d31c10 100644 --- a/package/pex-scie.lock +++ b/package/pex-scie.lock @@ -4,6 +4,7 @@ "allow_wheels": true, "build_isolation": true, "constraints": [], + "elide_unused_requires_dist": true, "excluded": [], "locked_resolves": [ { @@ -17,13 +18,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -45,13 +40,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -73,13 +62,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -101,13 +84,7 @@ } ], "project_name": "psutil", - "requires_dists": [ - "enum34; python_version <= \"3.4\" and extra == \"test\"", - "ipaddress; python_version < \"3.0\" and extra == \"test\"", - "mock; python_version < \"3.0\" and extra == \"test\"", - "pywin32; sys_platform == \"win32\" and extra == \"test\"", - "wmi; sys_platform == \"win32\" and extra == \"test\"" - ], + "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7", "version": "6.0.0" } @@ -123,8 +100,8 @@ "only_wheels": [], "overridden": [], "path_mappings": {}, - "pex_version": "2.17.0", - "pip_version": "24.2", + "pex_version": "2.24.3", + "pip_version": "24.3.1", "prefer_older_binary": false, "requirements": [ "psutil>=5.3" @@ -134,5 +111,6 @@ "style": "strict", "target_systems": [], "transitive": true, - "use_pep517": null + "use_pep517": null, + "use_system_time": false } diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 318da3a6e..942de3eac 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -76,6 +76,8 @@ from typing import IO, Dict, Iterable, List, Mapping, Optional, Set, Text, Tuple, Union import attr # vendor:skip + + from pex.resolve.lockfile.updater import Update else: from pex.third_party import attr @@ -526,6 +528,20 @@ def add_create_lock_options(cls, create_parser): ) ), ) + create_parser.add_argument( + "--elide-unused-requires-dist", + "--no-elide-unused-requires-dist", + dest="elide_unused_requires_dist", + type=bool, + default=False, + action=HandleBoolAction, + help=( + "When creating the lock, elide dependencies from the 'requires_dists' lists that " + "can never be active due to markers. This does not change the reachable content of " + "the lock, but it does cut down on lock file size. This currently only elides " + "extras deps that are never activated, but may trim more in the future." + ), + ) cls._add_lock_options(create_parser) cls._add_resolve_options(create_parser) cls.add_json_options(create_parser, entity="lock", include_switch=False) @@ -899,6 +915,7 @@ def _create(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -907,7 +924,10 @@ def _create(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) targets = try_( self._resolve_targets( @@ -1242,7 +1262,7 @@ def _process_lock_update( dry_run = self.options.dry_run path_mappings = self._get_path_mappings() output = sys.stdout if dry_run is DryRunStyle.DISPLAY else sys.stderr - updates = [] # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]] + updates = [] # type: List[Update] warnings = [] # type: List[str] for resolve_update in lock_update.resolves: platform = resolve_update.updated_resolve.target_platform @@ -1317,7 +1337,7 @@ def _process_lock_update( ) if update_req: requirements_by_project_name[project_name] = update_req - else: + elif isinstance(update, ArtifactsUpdate): message_lines = [ " {lead_in} {project_name} {version} artifacts:".format( lead_in="Would update" if dry_run else "Updated", @@ -1351,8 +1371,25 @@ def _process_lock_update( ) for artifact in update.removed ) - print("\n".join(message_lines), file=output) + else: + message_lines = [ + " {lead_in} {project_name} {version} requirements:".format( + lead_in="Would update" if dry_run else "Updated", + project_name=project_name, + version=update.version, + ) + ] + if update.added: + message_lines.extend( + " + {added}".format(added=req) for req in update.added + ) + if update.removed: + message_lines.extend( + " - {removed}".format(removed=req) for req in update.removed + ) + print("\n".join(message_lines), file=output) + if fingerprint_updates: warnings.append( "Detected fingerprint changes in the following locked {projects} for lock " @@ -1547,6 +1584,7 @@ def _sync(self): for interpreter_constraint in target_configuration.interpreter_constraints ), target_systems=tuple(self.options.target_systems), + elide_unused_requires_dist=self.options.elide_unused_requires_dist, ) elif self.options.target_systems: return Error( @@ -1555,7 +1593,10 @@ def _sync(self): ) ) else: - lock_configuration = LockConfiguration(style=self.options.style) + lock_configuration = LockConfiguration( + style=self.options.style, + elide_unused_requires_dist=self.options.elide_unused_requires_dist, + ) lock_file_path = self.options.lock if os.path.exists(lock_file_path): @@ -1566,6 +1607,7 @@ def _sync(self): style=lock_configuration.style, requires_python=SortedTuple(lock_configuration.requires_python), target_systems=SortedTuple(lock_configuration.target_systems), + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, pip_version=pip_configuration.version, resolver_version=pip_configuration.resolver_version, allow_prereleases=pip_configuration.allow_prereleases, diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index aa83d01ac..a89885eb7 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -25,7 +25,6 @@ DownloadableArtifact, FileArtifact, LocalProjectArtifact, - LockConfiguration, VCSArtifact, ) from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager @@ -233,11 +232,7 @@ def create( file_lock_style=file_lock_style, downloader=ArtifactDownloader( resolver=resolver, - lock_configuration=LockConfiguration( - style=lock.style, - requires_python=lock.requires_python, - target_systems=lock.target_systems, - ), + lock_configuration=lock.lock_configuration(), target=target, package_index_configuration=PackageIndexConfiguration.create( pip_version=pip_version, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 684a73f0c..a57edd272 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -87,6 +87,7 @@ class LockConfiguration(object): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib(default=()) # type: Tuple[str, ...] target_systems = attr.ib(default=()) # type: Tuple[TargetSystem.Value, ...] + elide_unused_requires_dist = attr.ib(default=False) # type: bool @requires_python.validator @target_systems.validator diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 5d153121c..220092665 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -341,9 +341,11 @@ def lock(self, downloaded): package_index_configuration=self.package_index_configuration, max_parallel_jobs=self.max_parallel_jobs, ), - platform_tag=None - if self.lock_configuration.style == LockStyle.UNIVERSAL - else target.platform.tag, + platform_tag=( + None + if self.lock_configuration.style == LockStyle.UNIVERSAL + else target.platform.tag + ), ) for target, resolved_requirements in resolved_requirements_by_target.items() ) @@ -456,6 +458,7 @@ def create( excluded=dependency_configuration.excluded, overridden=dependency_configuration.all_overrides(), locked_resolves=locked_resolves, + elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist, ) if lock_configuration.style is LockStyle.UNIVERSAL and ( diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py index b8283ce79..ddfe4bde6 100644 --- a/pex/resolve/lockfile/json_codec.py +++ b/pex/resolve/lockfile/json_codec.py @@ -209,6 +209,8 @@ def parse_version_specifier( for index, target_system in enumerate(get("target_systems", list, optional=True) or ()) ] + elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False + only_wheels = [ parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index)) for index, project_name in enumerate(get("only_wheels", list, optional=True) or ()) @@ -231,7 +233,7 @@ def parse_version_specifier( for index, constraint in enumerate(get("constraints", list)) ] - use_system_time = get("use_system_time", bool, optional=True) + use_system_time = get("use_system_time", bool, optional=True) or False excluded = [ parse_requirement(req, path=".excluded[{index}]".format(index=index)) @@ -337,6 +339,7 @@ def assemble_tag( style=get_enum_value(LockStyle, "style"), requires_python=get("requires_python", list), target_systems=target_systems, + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=get_enum_value( PipVersion, "pip_version", @@ -354,10 +357,7 @@ def assemble_tag( prefer_older_binary=get("prefer_older_binary", bool), use_pep517=get("use_pep517", bool, optional=True), build_isolation=get("build_isolation", bool), - # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and - # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In - # those old locks there was no "use_system_time" field. - use_system_time=use_system_time if use_system_time is not None else True, + use_system_time=use_system_time, ), transitive=get("transitive", bool), excluded=excluded, @@ -391,6 +391,7 @@ def as_json_data( "style": str(lockfile.style), "requires_python": list(lockfile.requires_python), "target_systems": [str(target_system) for target_system in lockfile.target_systems], + "elide_unused_requires_dist": lockfile.elide_unused_requires_dist, "pip_version": str(lockfile.pip_version), "resolver_version": str(lockfile.resolver_version), "requirements": [ diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index bb309580e..36293fa34 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -11,7 +11,14 @@ from pex.pep_503 import ProjectName from pex.pip.version import PipVersion, PipVersionValue from pex.requirements import LocalProjectRequirement -from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem +from pex.resolve.locked_resolve import ( + LocalProjectArtifact, + LockConfiguration, + LockedResolve, + LockStyle, + TargetSystem, +) +from pex.resolve.lockfile import requires_dist from pex.resolve.resolved_requirement import Pin from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion from pex.sorted_tuple import SortedTuple @@ -47,6 +54,7 @@ def create( source=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver_version=None, # type: Optional[ResolverVersion.Value] + elide_unused_requires_dist=False, # type: bool ): # type: (...) -> Lockfile @@ -94,6 +102,7 @@ def extract_requirement(req): style=style, requires_python=SortedTuple(requires_python), target_systems=SortedTuple(target_systems), + elide_unused_requires_dist=elide_unused_requires_dist, pip_version=pip_ver, resolver_version=resolver_version or ResolverVersion.default(pip_ver), requirements=SortedTuple(resolve_requirements, key=str), @@ -110,7 +119,14 @@ def extract_requirement(req): transitive=transitive, excluded=SortedTuple(excluded), overridden=SortedTuple(overridden), - locked_resolves=SortedTuple(locked_resolves), + locked_resolves=SortedTuple( + ( + requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve) + if elide_unused_requires_dist + else locked_resolve + ) + for locked_resolve in locked_resolves + ), local_project_requirement_mapping=requirement_by_local_project_directory, source=source, ) @@ -119,6 +135,7 @@ def extract_requirement(req): style = attr.ib() # type: LockStyle.Value requires_python = attr.ib() # type: SortedTuple[str] target_systems = attr.ib() # type: SortedTuple[TargetSystem.Value] + elide_unused_requires_dist = attr.ib() # type: bool pip_version = attr.ib() # type: PipVersionValue resolver_version = attr.ib() # type: ResolverVersion.Value requirements = attr.ib() # type: SortedTuple[Requirement] @@ -139,6 +156,15 @@ def extract_requirement(req): local_project_requirement_mapping = attr.ib(eq=False) # type: Mapping[str, Requirement] source = attr.ib(default=None, eq=False) # type: Optional[str] + def lock_configuration(self): + # type: () -> LockConfiguration + return LockConfiguration( + style=self.style, + requires_python=self.requires_python, + target_systems=self.target_systems, + elide_unused_requires_dist=self.elide_unused_requires_dist, + ) + def build_configuration(self): # type: () -> BuildConfiguration return BuildConfiguration.create( diff --git a/pex/resolve/lockfile/requires_dist.py b/pex/resolve/lockfile/requires_dist.py new file mode 100644 index 000000000..410f7177c --- /dev/null +++ b/pex/resolve/lockfile/requires_dist.py @@ -0,0 +1,162 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import operator +from collections import defaultdict, deque + +from pex.dist_metadata import Requirement +from pex.exceptions import production_assert +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement, LockedResolve +from pex.sorted_tuple import SortedTuple +from pex.third_party.packaging.markers import Marker, Variable +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Callable, DefaultDict, Dict, Iterable, List, Optional, Tuple, Union + + import attr # vendor:skip + + EvalExtra = Callable[[ProjectName], bool] +else: + from pex.third_party import attr + + +_OPERATORS = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +class _Op(object): + def __init__(self, lhs): + self.lhs = lhs # type: EvalExtra + self.rhs = None # type: Optional[EvalExtra] + + +class _And(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) and cast("EvalExtra", self.rhs)(extra) + + +class _Or(_Op): + def __call__(self, extra): + # type: (ProjectName) -> bool + production_assert(self.rhs is not None) + return self.lhs(extra) or cast("EvalExtra", self.rhs)(extra) + + +def _parse_extra_item( + stack, # type: List[EvalExtra] + item, # type: Union[str, List, Tuple] + marker, # type: Marker +): + # type: (...) -> None + + if item == "and": + stack.append(_And(stack.pop())) + elif item == "or": + stack.append(_Or(stack.pop())) + elif isinstance(item, list): + for element in item: + _parse_extra_item(stack, element, marker) + elif isinstance(item, tuple): + lhs, op, rhs = item + if isinstance(lhs, Variable) and "extra" == str(lhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(rhs))) + elif isinstance(rhs, Variable) and "extra" == str(rhs): + check = lambda extra: _OPERATORS[str(op)](extra, ProjectName(str(lhs))) + else: + # Any other condition could potentially be true. + check = lambda _: True + if stack: + production_assert(isinstance(stack[-1], _Op)) + cast(_Op, stack[-1]).rhs = check + else: + stack.append(check) + else: + raise ValueError("Marker is invalid: {marker}".format(marker=marker)) + + +def _parse_extra_check(marker): + # type: (Marker) -> EvalExtra + checks = [] # type: List[EvalExtra] + for item in marker._markers: + _parse_extra_item(checks, item, marker) + production_assert(len(checks) == 1) + return checks[0] + + +_EXTRA_CHECKS = {} # type: Dict[str, EvalExtra] + + +def _parse_marker_for_extra_check(marker): + # type: (Marker) -> EvalExtra + maker_str = str(marker) + eval_extra = _EXTRA_CHECKS.get(maker_str) + if not eval_extra: + eval_extra = _parse_extra_check(marker) + _EXTRA_CHECKS[maker_str] = eval_extra + return eval_extra + + +def _evaluate_for_extras( + marker, # type: Optional[Marker] + extras, # type: Iterable[str] +): + # type: (...) -> bool + if not marker: + return True + eval_extra = _parse_marker_for_extra_check(marker) + return any(eval_extra(ProjectName(extra)) for extra in (extras or [""])) + + +def remove_unused_requires_dist( + resolve_requirements, # type: Iterable[Requirement] + locked_resolve, # type: LockedResolve +): + # type: (...) -> LockedResolve + + locked_req_by_project_name = { + locked_req.pin.project_name: locked_req for locked_req in locked_resolve.locked_requirements + } + requires_dist_by_locked_req = defaultdict( + OrderedSet + ) # type: DefaultDict[LockedRequirement, OrderedSet[Requirement]] + seen = set() + requirements = deque(resolve_requirements) + while requirements: + requirement = requirements.popleft() + if requirement in seen: + continue + + seen.add(requirement) + locked_req = locked_req_by_project_name[requirement.project_name] + for dep in locked_req.requires_dists: + if _evaluate_for_extras(dep.marker, requirement.extras): + requires_dist_by_locked_req[locked_req].add(dep) + requirements.append(dep) + + return attr.evolve( + locked_resolve, + locked_requirements=SortedTuple( + attr.evolve( + locked_requirement, + requires_dists=SortedTuple( + requires_dist_by_locked_req[locked_requirement], key=str + ), + ) + for locked_requirement in locked_resolve.locked_requirements + ), + ) diff --git a/pex/resolve/lockfile/updater.py b/pex/resolve/lockfile/updater.py index d66224352..4658777e7 100644 --- a/pex/resolve/lockfile/updater.py +++ b/pex/resolve/lockfile/updater.py @@ -189,12 +189,39 @@ def calculate_updates( removed = attr.ib() # type: Tuple[Artifact, ...] +@attr.s(frozen=True) +class RequirementsUpdate(object): + @classmethod + def calculate( + cls, + version, # type: Version + original, # type: Tuple[Requirement, ...] + updated, # type: Tuple[Requirement, ...] + ): + # type: (...) -> RequirementsUpdate + added = [] # type: List[Requirement] + removed = [] # type: List[Requirement] + for req in updated: + if req not in original: + added.append(req) + for req in original: + if req not in updated: + removed.append(req) + return cls(version=version, added=tuple(added), removed=tuple(removed)) + + version = attr.ib() # type: Version + added = attr.ib() # type: Tuple[Requirement, ...] + removed = attr.ib() # type: Tuple[Requirement, ...] + + +if TYPE_CHECKING: + Update = Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate, RequirementsUpdate] + + @attr.s(frozen=True) class ResolveUpdate(object): updated_resolve = attr.ib() # type: LockedResolve - updates = attr.ib( - factory=dict - ) # type: Mapping[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]] + updates = attr.ib(factory=dict) # type: Mapping[ProjectName, Optional[Update]] @attr.s(frozen=True) @@ -481,9 +508,7 @@ def update_resolve( updated_resolve = updated_lock_file.locked_resolves[0] updated_requirements = updated_lock_file.requirements - updates = ( - OrderedDict() - ) # type: OrderedDict[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]] + updates = OrderedDict() # type: OrderedDict[ProjectName, Optional[Update]] if self.deletes or self.dependency_configuration.excluded: reduced_requirements = [ @@ -533,6 +558,8 @@ def update_resolve( updated_pin = updated_requirement.pin original_artifacts = tuple(locked_requirement.iter_artifacts()) updated_artifacts = tuple(updated_requirement.iter_artifacts()) + original_requirements = locked_requirement.requires_dists + updated_requirements = updated_requirement.requires_dists # N.B.: We use a custom key for artifact equality comparison since `Artifact` # contains a `verified` attribute that can both vary based on Pex's current @@ -613,6 +640,12 @@ def artifacts_differ(): original=original_artifacts, updated=updated_artifacts, ) + elif original_requirements != updated_requirements: + updates[project_name] = RequirementsUpdate.calculate( + version=original_pin.version, + original=original_requirements, + updated=updated_requirements, + ) elif project_name in self.update_constraints_by_project_name: updates[project_name] = None @@ -661,11 +694,6 @@ def create( ): # type: (...) -> LockUpdater - lock_configuration = LockConfiguration( - style=lock_file.style, - requires_python=lock_file.requires_python, - target_systems=lock_file.target_systems, - ) pip_configuration = PipConfiguration( version=lock_file.pip_version, resolver_version=lock_file.resolver_version, @@ -680,7 +708,7 @@ def create( ) return cls( lock_file=lock_file, - lock_configuration=lock_configuration, + lock_configuration=lock_file.lock_configuration(), pip_configuration=pip_configuration, dependency_configuration=dependency_configuration, ) @@ -767,7 +795,7 @@ def _perform_update( ) # type: OrderedDict[Optional[tags.Tag], LockedResolve] resolve_updates_by_platform_tag = ( {} - ) # type: Dict[Optional[tags.Tag], Mapping[ProjectName, Optional[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]]] + ) # type: Dict[Optional[tags.Tag], Mapping[ProjectName, Optional[Update]]] # TODO(John Sirois): Consider parallelizing this. The underlying Jobs are down a few layers; # so this will likely require using multiprocessing. diff --git a/pex/version.py b/pex/version.py index d5e8f4a47..0e97cd0b2 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.24.3" +__version__ = "2.25.0" diff --git a/scripts/gen-scie-platform.py b/scripts/gen-scie-platform.py index 70b00a771..cec3866bd 100644 --- a/scripts/gen-scie-platform.py +++ b/scripts/gen-scie-platform.py @@ -195,6 +195,7 @@ def create_lock( ), "--pip-version", "latest", + "--elide-unused-requires-dist", "--indent", "2", "--lock", @@ -285,6 +286,7 @@ def main(out: IO[str]) -> str | int | None: parser.add_argument("--all", action="store_true") parser.add_argument("-f", "--force", action="store_true") parser.add_argument("--lock-file", type=Path, default=PACKAGE_DIR / "pex-scie.lock") + parser.add_argument("-L", "--only-sync-lock", action="store_true") parser.add_argument("-v", "--verbose", action="store_true") try: options = parser.parse_args() @@ -298,39 +300,40 @@ def main(out: IO[str]) -> str | int | None: logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING) generated_files: list[Path] = [] - if options.all: - try: - generated_files.extend( - ensure_all_complete_platforms( - dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + if not options.only_sync_lock: + if options.all: + try: + generated_files.extend( + ensure_all_complete_platforms( + dest_dir=options.dest_dir, scie_config=scie_config, force=options.force + ) ) - ) - except ( - GitHubError, - github.GithubException, - github.BadAttributeException, - httpx.HTTPError, - ) as e: - return str(e) - - try: - create_lock( - lock_file=options.lock_file, - complete_platforms=generated_files, - scie_config=scie_config, - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(options.lock_file) - else: - complete_platform_file = options.dest_dir / f"{plat}.json" - try: - create_complete_platform( - complete_platform_file=complete_platform_file, scie_config=scie_config - ) - except subprocess.CalledProcessError as e: - return str(e) - generated_files.append(complete_platform_file) + except ( + GitHubError, + github.GithubException, + github.BadAttributeException, + httpx.HTTPError, + ) as e: + return str(e) + else: + complete_platform_file = options.dest_dir / f"{plat}.json" + try: + create_complete_platform( + complete_platform_file=complete_platform_file, scie_config=scie_config + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(complete_platform_file) + + try: + create_lock( + lock_file=options.lock_file, + complete_platforms=tuple(options.dest_dir.glob("*.json")), + scie_config=scie_config, + ) + except subprocess.CalledProcessError as e: + return str(e) + generated_files.append(options.lock_file) for file in generated_files: print(str(file), file=out) diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index 3d616f639..0a5d4423b 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -45,6 +45,7 @@ style=LockStyle.UNIVERSAL, requires_python=SortedTuple(), target_systems=SortedTuple(), + elide_unused_requires_dist=False, pip_version=PipVersion.DEFAULT, resolver_version=ResolverVersion.PIP_2020, requirements=SortedTuple([Requirement.parse("ansicolors")]), diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index ef80ff262..5112a64bd 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -494,7 +494,15 @@ def test_create_universal_platform_check(tmpdir): } ], "project_name": "urllib3", - "requires_dists": [], + "requires_dists": [ + "brotlipy>=0.6.0; extra == \\"brotli\\"", + "pyOpenSSL>=0.14; extra == \\"secure\\"", + "cryptography>=1.3.4; extra == \\"secure\\"", + "idna>=2.0.0; extra == \\"secure\\"", + "certifi; extra == \\"secure\\"", + "ipaddress; python_version == \\"2.7\\" and extra == \\"secure\\"", + "PySocks!=1.5.7,<2.0,>=1.5.6; extra == \\"socks\\"" + ], "requires_python": null, "version": "1.25.11" } diff --git a/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py new file mode 100644 index 000000000..8453d319e --- /dev/null +++ b/tests/integration/cli/commands/test_lock_elide_unused_requires_dist.py @@ -0,0 +1,96 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from typing import Dict + +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import LockedRequirement +from pex.resolve.lockfile import json_codec +from pex.resolve.lockfile.model import Lockfile +from testing.cli import run_pex3 +from testing.pytest.tmp import Tempdir + + +def index_locked_reqs(lockfile): + # type: (Lockfile) -> Dict[ProjectName, LockedRequirement] + return { + locked_req.pin.project_name: locked_req + for locked_resolve in lockfile.locked_resolves + for locked_req in locked_resolve.locked_requirements + } + + +def test_lock_elide_unused_requires_dist(tmpdir): + # type: (Tempdir) -> None + + lock = tmpdir.join("lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--indent", + "2", + "-o", + lock, + ).assert_success() + lockfile = json_codec.load(lock) + + elided_lock = tmpdir.join("elided_lock.json") + run_pex3( + "lock", + "create", + "requests==2.31.0", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--elide-unused-requires-dist", + "--indent", + "2", + "-o", + elided_lock, + ).assert_success() + elided_lockfile = json_codec.load(elided_lock) + + assert lockfile != elided_lockfile + + locked_reqs = index_locked_reqs(lockfile) + requests = locked_reqs[ProjectName("requests")] + + elided_locked_reqs = index_locked_reqs(elided_lockfile) + elided_requests = elided_locked_reqs[ProjectName("requests")] + + assert requests != elided_requests + + assert requests.pin == elided_requests.pin + assert list(requests.iter_artifacts()) == list(elided_requests.iter_artifacts()) + assert requests.requires_python == elided_requests.requires_python + + assert requests.requires_dists != elided_requests.requires_dists + assert len(elided_requests.requires_dists) < len(requests.requires_dists) + elided_deps = set(requests.requires_dists) - set(elided_requests.requires_dists) + assert len(elided_deps) > 0 + assert not any( + elided_dep.project_name in elided_locked_reqs for elided_dep in elided_deps + ), "No dependencies that require extra activation should have been locked." + + run_pex3( + "lock", + "sync", + "--style", + "universal", + "--interpreter-constraint", + ">=3.7,<3.14", + "--elide-unused-requires-dist", + "--indent", + "2", + "--lock", + lock, + ).assert_success() + assert elided_lockfile == json_codec.load(lock) diff --git a/tests/resolve/lockfile/test_requires_dist.py b/tests/resolve/lockfile/test_requires_dist.py new file mode 100644 index 000000000..05bd88a48 --- /dev/null +++ b/tests/resolve/lockfile/test_requires_dist.py @@ -0,0 +1,170 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.dist_metadata import Requirement +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.resolve.locked_resolve import Artifact, LockedRequirement, LockedResolve +from pex.resolve.lockfile import requires_dist +from pex.resolve.resolved_requirement import Fingerprint, Pin +from pex.sorted_tuple import SortedTuple + +req = Requirement.parse + + +def locked_req( + project_name, # type: str + version, # type: str + *requirements # type: str +): + # type: (...) -> LockedRequirement + return LockedRequirement.create( + pin=Pin(project_name=ProjectName(project_name), version=Version(version)), + artifact=Artifact.from_url( + "https://artifact.store/{project_name}-{version}-py2.py3-none-any.whl".format( + project_name=project_name, version=version + ), + fingerprint=Fingerprint(algorithm="md5", hash="abcd0123"), + ), + requires_dists=map(req, requirements), + ) + + +def locked_resolve(*locked_requirements): + # type: (*LockedRequirement) -> LockedResolve + return LockedResolve(locked_requirements=SortedTuple(locked_requirements)) + + +def test_remove_unused_requires_dist_noop(): + # type: () -> None + + locked_resolve_with_no_extras = locked_resolve( + locked_req("foo", "1.0", "bar", "baz"), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + ) + assert locked_resolve_with_no_extras == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], locked_resolve=locked_resolve_with_no_extras + ) + + +def test_remove_unused_requires_dist_simple(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_extras(): + # type: () -> None + + assert locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req("foo", "1.0", "bar; extra == 'extra1'", "baz; extra == 'tests'", "spam"), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ) + + +def test_remove_unused_requires_dist_mixed_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' or python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("baz", "1.0"), + locked_req("spam", "1.0"), + ), + ), ( + "The python_version marker clause might evaluate to true, which should be enough to retain " + "the baz dep even though the 'tests' extra is never activated." + ) + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo[extra1]")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; extra == 'extra1'", + "baz; extra == 'tests' and python_version > '3.11'", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + ), "The 'tests' extra is never active; so the baz dep should never be reached." + + +def test_remove_unused_requires_dist_complex_markers(): + # type: () -> None + + assert locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ) == requires_dist.remove_unused_requires_dist( + resolve_requirements=[req("foo")], + locked_resolve=locked_resolve( + locked_req( + "foo", + "1.0", + "bar; python_version < '3' and (extra == 'docs' or python_version >= '3')", + "baz; python_version == '3.11.*' and (extra == 'admin' or extra == 'docs')", + "spam", + ), + locked_req("bar", "1.0"), + locked_req("spam", "1.0"), + ), + )