Add --elide-unused-requires-dist lock option.

CHANGES.md

@@ -1,5 +1,15 @@
 # Release Notes
 
+## 2.25.0
+
+This release adds support for
+`pex3 lock {create,sync} --elide-unused-requires-dist`. This new lock
+option causes any dependencies of a locked requirement that can never
+be activated to be elided from the lock file. This leads to no material
+difference in lock file use, but it does cut down on the lock file size.
+
+* Add `--elide-unused-requires-dist` lock option. (#2613)
+
 ## 2.24.3
 
 This release fixes a long-standing bug in resolve checking. Previously,

package/pex-scie.lock

@@ -4,6 +4,7 @@
   "allow_wheels": true,
   "build_isolation": true,
   "constraints": [],
+  "elide_unused_requires_dist": true,
   "excluded": [],
   "locked_resolves": [
     {
@@ -17,13 +18,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -45,13 +40,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -73,13 +62,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -101,13 +84,7 @@
             }
           ],
           "project_name": "psutil",
-          "requires_dists": [
-            "enum34; python_version <= \"3.4\" and extra == \"test\"",
-            "ipaddress; python_version < \"3.0\" and extra == \"test\"",
-            "mock; python_version < \"3.0\" and extra == \"test\"",
-            "pywin32; sys_platform == \"win32\" and extra == \"test\"",
-            "wmi; sys_platform == \"win32\" and extra == \"test\""
-          ],
+          "requires_dists": [],
           "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
           "version": "6.0.0"
         }
@@ -123,8 +100,8 @@
   "only_wheels": [],
   "overridden": [],
   "path_mappings": {},
-  "pex_version": "2.17.0",
-  "pip_version": "24.2",
+  "pex_version": "2.24.3",
+  "pip_version": "24.3.1",
   "prefer_older_binary": false,
   "requirements": [
     "psutil>=5.3"
@@ -134,5 +111,6 @@
   "style": "strict",
   "target_systems": [],
   "transitive": true,
-  "use_pep517": null
+  "use_pep517": null,
+  "use_system_time": false
 }

pex/cli/commands/lock.py

@@ -76,6 +76,8 @@
     from typing import IO, Dict, Iterable, List, Mapping, Optional, Set, Text, Tuple, Union
 
     import attr  # vendor:skip
+
+    from pex.resolve.lockfile.updater import Update
 else:
     from pex.third_party import attr
 
@@ -526,6 +528,20 @@ def add_create_lock_options(cls, create_parser):
                 )
             ),
         )
+        create_parser.add_argument(
+            "--elide-unused-requires-dist",
+            "--no-elide-unused-requires-dist",
+            dest="elide_unused_requires_dist",
+            type=bool,
+            default=False,
+            action=HandleBoolAction,
+            help=(
+                "When creating the lock, elide dependencies from the 'requires_dists' lists that "
+                "can never be active due to markers. This does not change the reachable content of "
+                "the lock, but it does cut down on lock file size. This currently only elides "
+                "extras deps that are never activated, but may trim more in the future."
+            ),
+        )
         cls._add_lock_options(create_parser)
         cls._add_resolve_options(create_parser)
         cls.add_json_options(create_parser, entity="lock", include_switch=False)
@@ -899,6 +915,7 @@ def _create(self):
                     for interpreter_constraint in target_configuration.interpreter_constraints
                 ),
                 target_systems=tuple(self.options.target_systems),
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
             )
         elif self.options.target_systems:
             return Error(
@@ -907,7 +924,10 @@ def _create(self):
                 )
             )
         else:
-            lock_configuration = LockConfiguration(style=self.options.style)
+            lock_configuration = LockConfiguration(
+                style=self.options.style,
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+            )
 
         targets = try_(
             self._resolve_targets(
@@ -1242,7 +1262,7 @@ def _process_lock_update(
         dry_run = self.options.dry_run
         path_mappings = self._get_path_mappings()
         output = sys.stdout if dry_run is DryRunStyle.DISPLAY else sys.stderr
-        updates = []  # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]
+        updates = []  # type: List[Update]
         warnings = []  # type: List[str]
         for resolve_update in lock_update.resolves:
             platform = resolve_update.updated_resolve.target_platform
@@ -1317,7 +1337,7 @@ def _process_lock_update(
                         )
                         if update_req:
                             requirements_by_project_name[project_name] = update_req
-                else:
+                elif isinstance(update, ArtifactsUpdate):
                     message_lines = [
                         "  {lead_in} {project_name} {version} artifacts:".format(
                             lead_in="Would update" if dry_run else "Updated",
@@ -1351,8 +1371,25 @@ def _process_lock_update(
                             )
                             for artifact in update.removed
                         )
-
                     print("\n".join(message_lines), file=output)
+                else:
+                    message_lines = [
+                        "  {lead_in} {project_name} {version} requirements:".format(
+                            lead_in="Would update" if dry_run else "Updated",
+                            project_name=project_name,
+                            version=update.version,
+                        )
+                    ]
+                    if update.added:
+                        message_lines.extend(
+                            "    + {added}".format(added=req) for req in update.added
+                        )
+                    if update.removed:
+                        message_lines.extend(
+                            "    - {removed}".format(removed=req) for req in update.removed
+                        )
+                    print("\n".join(message_lines), file=output)
+
             if fingerprint_updates:
                 warnings.append(
                     "Detected fingerprint changes in the following locked {projects} for lock "
@@ -1547,6 +1584,7 @@ def _sync(self):
                     for interpreter_constraint in target_configuration.interpreter_constraints
                 ),
                 target_systems=tuple(self.options.target_systems),
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
             )
         elif self.options.target_systems:
             return Error(
@@ -1555,7 +1593,10 @@ def _sync(self):
                 )
             )
         else:
-            lock_configuration = LockConfiguration(style=self.options.style)
+            lock_configuration = LockConfiguration(
+                style=self.options.style,
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+            )
 
         lock_file_path = self.options.lock
         if os.path.exists(lock_file_path):
@@ -1566,6 +1607,7 @@ def _sync(self):
                 style=lock_configuration.style,
                 requires_python=SortedTuple(lock_configuration.requires_python),
                 target_systems=SortedTuple(lock_configuration.target_systems),
+                elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
                 pip_version=pip_configuration.version,
                 resolver_version=pip_configuration.resolver_version,
                 allow_prereleases=pip_configuration.allow_prereleases,

pex/resolve/lock_downloader.py

@@ -25,7 +25,6 @@
     DownloadableArtifact,
     FileArtifact,
     LocalProjectArtifact,
-    LockConfiguration,
     VCSArtifact,
 )
 from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager
@@ -233,11 +232,7 @@ def create(
                 file_lock_style=file_lock_style,
                 downloader=ArtifactDownloader(
                     resolver=resolver,
-                    lock_configuration=LockConfiguration(
-                        style=lock.style,
-                        requires_python=lock.requires_python,
-                        target_systems=lock.target_systems,
-                    ),
+                    lock_configuration=lock.lock_configuration(),
                     target=target,
                     package_index_configuration=PackageIndexConfiguration.create(
                         pip_version=pip_version,

pex/resolve/locked_resolve.py

@@ -87,6 +87,7 @@ class LockConfiguration(object):
     style = attr.ib()  # type: LockStyle.Value
     requires_python = attr.ib(default=())  # type: Tuple[str, ...]
     target_systems = attr.ib(default=())  # type: Tuple[TargetSystem.Value, ...]
+    elide_unused_requires_dist = attr.ib(default=False)  # type: bool
 
     @requires_python.validator
     @target_systems.validator

pex/resolve/lockfile/create.py

@@ -341,9 +341,11 @@ def lock(self, downloaded):
                     package_index_configuration=self.package_index_configuration,
                     max_parallel_jobs=self.max_parallel_jobs,
                 ),
-                platform_tag=None
-                if self.lock_configuration.style == LockStyle.UNIVERSAL
-                else target.platform.tag,
+                platform_tag=(
+                    None
+                    if self.lock_configuration.style == LockStyle.UNIVERSAL
+                    else target.platform.tag
+                ),
             )
             for target, resolved_requirements in resolved_requirements_by_target.items()
         )
@@ -456,6 +458,7 @@ def create(
         excluded=dependency_configuration.excluded,
         overridden=dependency_configuration.all_overrides(),
         locked_resolves=locked_resolves,
+        elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
     )
 
     if lock_configuration.style is LockStyle.UNIVERSAL and (

pex/resolve/lockfile/json_codec.py

@@ -209,6 +209,8 @@ def parse_version_specifier(
         for index, target_system in enumerate(get("target_systems", list, optional=True) or ())
     ]
 
+    elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False
+
     only_wheels = [
         parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index))
         for index, project_name in enumerate(get("only_wheels", list, optional=True) or ())
@@ -231,7 +233,7 @@ def parse_version_specifier(
         for index, constraint in enumerate(get("constraints", list))
     ]
 
-    use_system_time = get("use_system_time", bool, optional=True)
+    use_system_time = get("use_system_time", bool, optional=True) or False
 
     excluded = [
         parse_requirement(req, path=".excluded[{index}]".format(index=index))
@@ -337,6 +339,7 @@ def assemble_tag(
         style=get_enum_value(LockStyle, "style"),
         requires_python=get("requires_python", list),
         target_systems=target_systems,
+        elide_unused_requires_dist=elide_unused_requires_dist,
         pip_version=get_enum_value(
             PipVersion,
             "pip_version",
@@ -354,10 +357,7 @@ def assemble_tag(
             prefer_older_binary=get("prefer_older_binary", bool),
             use_pep517=get("use_pep517", bool, optional=True),
             build_isolation=get("build_isolation", bool),
-            # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and
-            # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In
-            # those old locks there was no "use_system_time" field.
-            use_system_time=use_system_time if use_system_time is not None else True,
+            use_system_time=use_system_time,
         ),
         transitive=get("transitive", bool),
         excluded=excluded,
@@ -391,6 +391,7 @@ def as_json_data(
         "style": str(lockfile.style),
         "requires_python": list(lockfile.requires_python),
         "target_systems": [str(target_system) for target_system in lockfile.target_systems],
+        "elide_unused_requires_dist": lockfile.elide_unused_requires_dist,
         "pip_version": str(lockfile.pip_version),
         "resolver_version": str(lockfile.resolver_version),
         "requirements": [

pex/resolve/lockfile/model.py

@@ -11,7 +11,14 @@
 from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.requirements import LocalProjectRequirement
-from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem
+from pex.resolve.locked_resolve import (
+    LocalProjectArtifact,
+    LockConfiguration,
+    LockedResolve,
+    LockStyle,
+    TargetSystem,
+)
+from pex.resolve.lockfile import requires_dist
 from pex.resolve.resolved_requirement import Pin
 from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
 from pex.sorted_tuple import SortedTuple
@@ -47,6 +54,7 @@ def create(
         source=None,  # type: Optional[str]
         pip_version=None,  # type: Optional[PipVersionValue]
         resolver_version=None,  # type: Optional[ResolverVersion.Value]
+        elide_unused_requires_dist=False,  # type: bool
     ):
         # type: (...) -> Lockfile
 
@@ -94,6 +102,7 @@ def extract_requirement(req):
             style=style,
             requires_python=SortedTuple(requires_python),
             target_systems=SortedTuple(target_systems),
+            elide_unused_requires_dist=elide_unused_requires_dist,
             pip_version=pip_ver,
             resolver_version=resolver_version or ResolverVersion.default(pip_ver),
             requirements=SortedTuple(resolve_requirements, key=str),
@@ -110,7 +119,14 @@ def extract_requirement(req):
             transitive=transitive,
             excluded=SortedTuple(excluded),
             overridden=SortedTuple(overridden),
-            locked_resolves=SortedTuple(locked_resolves),
+            locked_resolves=SortedTuple(
+                (
+                    requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve)
+                    if elide_unused_requires_dist
+                    else locked_resolve
+                )
+                for locked_resolve in locked_resolves
+            ),
             local_project_requirement_mapping=requirement_by_local_project_directory,
             source=source,
         )
@@ -119,6 +135,7 @@ def extract_requirement(req):
     style = attr.ib()  # type: LockStyle.Value
     requires_python = attr.ib()  # type: SortedTuple[str]
     target_systems = attr.ib()  # type: SortedTuple[TargetSystem.Value]
+    elide_unused_requires_dist = attr.ib()  # type: bool
     pip_version = attr.ib()  # type: PipVersionValue
     resolver_version = attr.ib()  # type: ResolverVersion.Value
     requirements = attr.ib()  # type: SortedTuple[Requirement]
@@ -139,6 +156,15 @@ def extract_requirement(req):
     local_project_requirement_mapping = attr.ib(eq=False)  # type: Mapping[str, Requirement]
     source = attr.ib(default=None, eq=False)  # type: Optional[str]
 
+    def lock_configuration(self):
+        # type: () -> LockConfiguration
+        return LockConfiguration(
+            style=self.style,
+            requires_python=self.requires_python,
+            target_systems=self.target_systems,
+            elide_unused_requires_dist=self.elide_unused_requires_dist,
+        )
+
     def build_configuration(self):
         # type: () -> BuildConfiguration
         return BuildConfiguration.create(