Support --sh-boot for --layout {loose,packed}. (#2645)

CHANGES.md

@@ -1,5 +1,15 @@
 # Release Notes
 
+## 2.30.0
+
+This release brings `--sh-boot` support to PEXes with
+`--layout {loose,packed}`. Previously, the `--sh-boot` option only took
+effect for traditional PEX zip files. Now all PEX output and runtime
+schemes, in any combination, can benefit from the reduced boot latency
+`--sh-boot` brings on all runs of a PEX after the first.
+
+* Support `--sh-boot` for `--layout {loose,packed}`. (#2645)
+
 ## 2.29.0
 
 This release brings 1st class support for newer Pip's

pex/bin/pex.py

@@ -1349,6 +1349,7 @@ def do_main(
                     pex_name=pex_file,
                     targets=targets,
                     python_shebang=options.python_shebang,
+                    layout=options.layout,
                 )
 
         pex_builder.build(

pex/cache/dirs.py

@@ -6,10 +6,11 @@
 import glob
 import os
 
-from pex.common import is_exe, safe_rmtree
+from pex.common import safe_rmtree
 from pex.compatibility import commonpath
 from pex.enum import Enum
 from pex.exceptions import production_assert
+from pex.executables import is_exe
 from pex.orderedset import OrderedSet
 from pex.typing import TYPE_CHECKING, cast
 from pex.variables import ENV, Variables

pex/cli/commands/lock.py

@@ -14,7 +14,7 @@
 from pex.argparse import HandleBoolAction
 from pex.cli.command import BuildTimeCommand
 from pex.commands.command import JsonMixin, OutputMixin
-from pex.common import is_exe, pluralize, safe_delete, safe_open
+from pex.common import pluralize, safe_delete, safe_open
 from pex.compatibility import commonpath, shlex_quote
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dist_metadata import (
@@ -26,6 +26,7 @@
 )
 from pex.enum import Enum
 from pex.exceptions import production_assert
+from pex.executables import is_exe
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
 from pex.pep_376 import InstalledWheel, Record

pex/cli/commands/venv.py

@@ -11,9 +11,10 @@
 from pex import pex_warnings
 from pex.cli.command import BuildTimeCommand
 from pex.commands.command import JsonMixin, OutputMixin
-from pex.common import DETERMINISTIC_DATETIME, CopyMode, is_script, open_zip, pluralize
+from pex.common import DETERMINISTIC_DATETIME, CopyMode, open_zip, pluralize
 from pex.dist_metadata import Distribution
 from pex.enum import Enum
+from pex.executables import is_script
 from pex.executor import Executor
 from pex.pex import PEX
 from pex.pex_info import PexInfo

pex/common.py

@@ -23,6 +23,7 @@
 from zipfile import ZipFile, ZipInfo
 
 from pex.enum import Enum
+from pex.executables import chmod_plus_x
 from pex.typing import TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
@@ -446,72 +447,6 @@ def safe_sleep(seconds):
             current_time = time.time()
 
 
-def chmod_plus_x(path):
-    # type: (Text) -> None
-    """Equivalent of unix `chmod a+x path`"""
-    path_mode = os.stat(path).st_mode
-    path_mode &= int("777", 8)
-    if path_mode & stat.S_IRUSR:
-        path_mode |= stat.S_IXUSR
-    if path_mode & stat.S_IRGRP:
-        path_mode |= stat.S_IXGRP
-    if path_mode & stat.S_IROTH:
-        path_mode |= stat.S_IXOTH
-    os.chmod(path, path_mode)
-
-
-def chmod_plus_w(path):
-    # type: (str) -> None
-    """Equivalent of unix `chmod +w path`"""
-    path_mode = os.stat(path).st_mode
-    path_mode &= int("777", 8)
-    path_mode |= stat.S_IWRITE
-    os.chmod(path, path_mode)
-
-
-def is_exe(path):
-    # type: (str) -> bool
-    """Determines if the given path is a file executable by the current user.
-
-    :param path: The path to check.
-    :return: `True if the given path is a file executable by the current user.
-    """
-    return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
-
-
-def is_script(
-    path,  # type: str
-    pattern=None,  # type: Optional[str]
-    check_executable=True,  # type: bool
-):
-    # type: (...) -> bool
-    """Determines if the given path is a script.
-
-    A script is a file that starts with a shebang (#!...) line.
-
-    :param path: The path to check.
-    :param pattern: An optional pattern to match against the shebang (excluding the leading #!).
-    :param check_executable: Check that the script is executable by the current user.
-    :return: True if the given path is a script.
-    """
-    if check_executable and not is_exe(path):
-        return False
-    with open(path, "rb") as fp:
-        if b"#!" != fp.read(2):
-            return False
-        if not pattern:
-            return True
-        return bool(re.match(pattern, fp.readline().decode("utf-8")))
-
-
-def is_python_script(
-    path,  # type: str
-    check_executable=True,  # type: bool
-):
-    # type: (...) -> bool
-    return is_script(path, pattern=r"(?i)^.*(?:python|pypy)", check_executable=check_executable)
-
-
 def can_write_dir(path):
     # type: (str) -> bool
     """Determines if the directory at path can be written to by the current process.

pex/executables.py

@@ -0,0 +1,147 @@
+# Copyright 2025 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os
+import re
+import stat
+from textwrap import dedent
+
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import BinaryIO, Callable, Optional, Text, Tuple
+
+
+def chmod_plus_x(path):
+    # type: (Text) -> None
+    """Equivalent of unix `chmod a+x path`"""
+    path_mode = os.stat(path).st_mode
+    path_mode &= int("777", 8)
+    if path_mode & stat.S_IRUSR:
+        path_mode |= stat.S_IXUSR
+    if path_mode & stat.S_IRGRP:
+        path_mode |= stat.S_IXGRP
+    if path_mode & stat.S_IROTH:
+        path_mode |= stat.S_IXOTH
+    os.chmod(path, path_mode)
+
+
+def is_exe(path):
+    # type: (Text) -> bool
+    """Determines if the given path is a file executable by the current user.
+
+    :param path: The path to check.
+    :return: True if the given path is a file executable by the current user.
+    """
+    return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
+
+
+def is_script(
+    path,  # type: Text
+    pattern=None,  # type: Optional[bytes]
+    check_executable=True,  # type: bool
+    extra_check=None,  # type: Optional[Callable[[bytes, BinaryIO], bool]]
+):
+    # type: (...) -> bool
+    """Determines if the given path is a script.
+
+    A script is a file that starts with a shebang (#!...) line.
+
+    :param path: The path to check.
+    :param pattern: Optional pattern to match against the shebang line (excluding the leading #!
+                    and trailing \n).
+    :param check_executable: Check that the script is executable by the current user.
+    :param extra_check: Optional callable accepting the shebang line (excluding the leading #! and
+                        trailing \n) and a file opened for binary read pointing just after that
+                        line.
+    :return: True if the given path is a script.
+    """
+    if check_executable and not is_exe(path):
+        return False
+    with open(path, "rb") as fp:
+        if b"#!" != fp.read(2):
+            return False
+        if not pattern:
+            return True
+        shebang_suffix = fp.readline().rstrip()
+        if bool(re.match(pattern, shebang_suffix)):
+            return True
+        if extra_check:
+            return extra_check(shebang_suffix, fp)
+        return False
+
+
+def create_sh_python_redirector_shebang(sh_script_content):
+    # type: (str) -> Tuple[str, str]
+    """Create a shebang block for a Python file that uses /bin/sh to find an appropriate Python.
+
+    The script should be POSIX compliant sh and terminate on all execution paths with an
+    explicit exit or exec.
+
+    The returned shebang block will include the leading `#!` but will not include a trailing new
+    line character.
+
+    :param sh_script_content: A POSIX compliant sh script that always explicitly terminates.
+    :return: A shebang line and trailing block of text that can be combined for use as a shebang
+             header for a Python file.
+    """
+    # This trick relies on /bin/sh being ubiquitous and the concordance of:
+    #
+    # 1. Python: Has triple quoted strings plus allowance for free-floating string values in
+    #    python files.
+    # 2. sh: Any number of pairs of `'` evaluating away when followed immediately by a
+    #    command string (`''command` -> `command`).
+    # 3. sh: The `:` noop command which accepts and discards arbitrary args.
+    #    See: https://pubs.opengroup.org/onlinepubs/009604599/utilities/colon.html
+    # 4. sh: Lazy parsing allowing for invalid sh content immediately following an exit or exec
+    #        line.
+    #
+    # The end result is a file that is both a valid sh script with a short shebang and a valid
+    # Python program.
+    return "#!/bin/sh", (
+        dedent(
+            """\
+            '''': pshprs
+            {sh_script_content}
+            '''
+            """
+        )
+        .format(sh_script_content=sh_script_content.rstrip())
+        .strip()
+    )
+
+
+def is_python_script(
+    path,  # type: Text
+    check_executable=True,  # type: bool
+):
+    # type: (...) -> bool
+    return is_script(
+        path,
+        pattern=br"""(?ix)
+        # The aim is to admit the common shebang forms:
+        # + python
+        # + /usr/bin/env (<args>)? <python bin name> (<args>)?
+        # + /absolute/path/to/<python bin name> (<args>)?
+        # The 1st corresponds to the special placeholder shebang #!python specified here:
+        # + https://peps.python.org/pep-0427
+        # + https://packaging.python.org/specifications/binary-distribution-format
+        (?:^|.*\W)
+
+        # Python executable names Pex supports (see PythonIdentity).
+        (?:
+              python
+            | pypy
+        )
+
+        # Optional Python version
+        (?:\d+(?:\.\d+)*)?
+
+        # Support a shebang with an argument to the interpreter at the end.
+        (?:\s[^\s]|$)
+        """,
+        check_executable=check_executable,
+        extra_check=lambda shebang, fp: shebang == b"/bin/sh" and fp.read(13) == b"'''': pshprs\n",
+    )

pex/finders.py

@@ -6,14 +6,15 @@
 import ast
 import os
 
-from pex.common import is_python_script, open_zip, safe_mkdtemp
+from pex.common import open_zip, safe_mkdtemp
 from pex.dist_metadata import (
     CallableEntryPoint,
     Distribution,
     DistributionType,
     ModuleEntryPoint,
     NamedEntryPoint,
 )
+from pex.executables import is_python_script
 from pex.pep_376 import InstalledWheel
 from pex.pep_503 import ProjectName
 from pex.typing import TYPE_CHECKING, cast

pex/interpreter.py

@@ -17,7 +17,8 @@
 
 from pex import third_party
 from pex.cache.dirs import InterpreterDir
-from pex.common import is_exe, safe_mkdtemp, safe_rmtree
+from pex.common import safe_mkdtemp, safe_rmtree
+from pex.executables import is_exe
 from pex.executor import Executor
 from pex.jobs import Job, Retain, SpawnedJob, execute_parallel
 from pex.orderedset import OrderedSet

pex/layout.py

@@ -11,8 +11,9 @@
 from pex.atomic_directory import atomic_directory
 from pex.cache import access as cache_access
 from pex.cache.dirs import BootstrapDir, InstalledWheelDir, UserCodeDir
-from pex.common import ZipFileEx, is_script, open_zip, safe_copy, safe_mkdir, safe_mkdtemp
+from pex.common import ZipFileEx, open_zip, safe_copy, safe_mkdir, safe_mkdtemp
 from pex.enum import Enum
+from pex.executables import is_script
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV, unzip_dir
@@ -559,7 +560,9 @@ def extract_code(self, dest_dir):
             if root == self._path:
                 dirs[:] = [d for d in dirs if d not in ("__pex__", DEPS_DIR)]
                 files[:] = [
-                    f for f in files if f not in ("__main__.py", PEX_INFO_PATH, BOOTSTRAP_DIR)
+                    f
+                    for f in files
+                    if f not in ("__main__.py", "pex", PEX_INFO_PATH, BOOTSTRAP_DIR)
                 ]
             for d in dirs:
                 safe_mkdir(os.path.join(dest_dir, rel_root, d))
@@ -635,7 +638,7 @@ def extract_code(self, dest_dir):
             rel_root = os.path.relpath(root, self._path)
             if root == self._path:
                 dirs[:] = [d for d in dirs if d not in ("__pex__", DEPS_DIR, BOOTSTRAP_DIR)]
-                files[:] = [f for f in files if f not in ("__main__.py", PEX_INFO_PATH)]
+                files[:] = [f for f in files if f not in ("__main__.py", "pex", PEX_INFO_PATH)]
             for d in dirs:
                 safe_mkdir(os.path.join(dest_dir, rel_root, d))
             for f in files:

pex/pep_427.py

@@ -14,10 +14,11 @@
 from textwrap import dedent
 
 from pex import pex_warnings
-from pex.common import chmod_plus_x, is_pyc_file, iter_copytree, open_zip, safe_open, touch
+from pex.common import is_pyc_file, iter_copytree, open_zip, safe_open, touch
 from pex.compatibility import commonpath, get_stdout_bytes_buffer
 from pex.dist_metadata import CallableEntryPoint, Distribution, ProjectNameAndVersion
 from pex.enum import Enum
+from pex.executables import chmod_plus_x
 from pex.interpreter import PythonInterpreter
 from pex.pep_376 import InstalledFile, InstalledWheel, Record
 from pex.pep_503 import ProjectName

pex/pex.py

@@ -542,6 +542,7 @@ def execute(self):
 
         This function makes assumptions that it is the last function called by the interpreter.
         """
+        pex_file = self._vars.PEX
         if self._vars.PEX_TOOLS:
             if not self._pex_info.includes_tools:
                 die(
@@ -551,11 +552,10 @@ def execute(self):
 
             from pex.tools import main as tools
 
-            sys.exit(tools.main(pex=PEX(sys.argv[0])))
+            sys.exit(tools.main(pex=PEX(pex_file or sys.argv[0])))
 
         self.activate()
 
-        pex_file = self._vars.PEX
         if pex_file:
             try:
                 from setproctitle import setproctitle  # type: ignore[import]