diff --git a/CHANGES.md b/CHANGES.md index c38392edc..6faa6a834 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Release Notes +## 2.30.0 + +This release brings `--sh-boot` support to PEXes with +`--layout {loose,packed}`. Previously, the `--sh-boot` option only took +effect for traditional PEX zip files. Now all PEX output and runtime +schemes, in any combination, can benefit from the reduced boot latency +`--sh-boot` brings on all runs of a PEX after the first. + +* Support `--sh-boot` for `--layout {loose,packed}`. (#2645) + ## 2.29.0 This release brings 1st class support for newer Pip's diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 8b3ace8d1..58e57532d 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -1349,6 +1349,7 @@ def do_main( pex_name=pex_file, targets=targets, python_shebang=options.python_shebang, + layout=options.layout, ) pex_builder.build( diff --git a/pex/cache/dirs.py b/pex/cache/dirs.py index bbfd043d4..d10753544 100644 --- a/pex/cache/dirs.py +++ b/pex/cache/dirs.py @@ -6,10 +6,11 @@ import glob import os -from pex.common import is_exe, safe_rmtree +from pex.common import safe_rmtree from pex.compatibility import commonpath from pex.enum import Enum from pex.exceptions import production_assert +from pex.executables import is_exe from pex.orderedset import OrderedSet from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV, Variables diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index cb0970a36..d67360d3a 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -14,7 +14,7 @@ from pex.argparse import HandleBoolAction from pex.cli.command import BuildTimeCommand from pex.commands.command import JsonMixin, OutputMixin -from pex.common import is_exe, pluralize, safe_delete, safe_open +from pex.common import pluralize, safe_delete, safe_open from pex.compatibility import commonpath, shlex_quote from pex.dependency_configuration import DependencyConfiguration from pex.dist_metadata import ( @@ -26,6 +26,7 @@ ) from pex.enum import Enum from pex.exceptions import production_assert +from pex.executables import is_exe from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel, Record diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index cb95fc31e..9cc670599 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -11,9 +11,10 @@ from pex import pex_warnings from pex.cli.command import BuildTimeCommand from pex.commands.command import JsonMixin, OutputMixin -from pex.common import DETERMINISTIC_DATETIME, CopyMode, is_script, open_zip, pluralize +from pex.common import DETERMINISTIC_DATETIME, CopyMode, open_zip, pluralize from pex.dist_metadata import Distribution from pex.enum import Enum +from pex.executables import is_script from pex.executor import Executor from pex.pex import PEX from pex.pex_info import PexInfo diff --git a/pex/common.py b/pex/common.py index 056ce6fbf..0c239ecfa 100644 --- a/pex/common.py +++ b/pex/common.py @@ -23,6 +23,7 @@ from zipfile import ZipFile, ZipInfo from pex.enum import Enum +from pex.executables import chmod_plus_x from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -446,72 +447,6 @@ def safe_sleep(seconds): current_time = time.time() -def chmod_plus_x(path): - # type: (Text) -> None - """Equivalent of unix `chmod a+x path`""" - path_mode = os.stat(path).st_mode - path_mode &= int("777", 8) - if path_mode & stat.S_IRUSR: - path_mode |= stat.S_IXUSR - if path_mode & stat.S_IRGRP: - path_mode |= stat.S_IXGRP - if path_mode & stat.S_IROTH: - path_mode |= stat.S_IXOTH - os.chmod(path, path_mode) - - -def chmod_plus_w(path): - # type: (str) -> None - """Equivalent of unix `chmod +w path`""" - path_mode = os.stat(path).st_mode - path_mode &= int("777", 8) - path_mode |= stat.S_IWRITE - os.chmod(path, path_mode) - - -def is_exe(path): - # type: (str) -> bool - """Determines if the given path is a file executable by the current user. - - :param path: The path to check. - :return: `True if the given path is a file executable by the current user. - """ - return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK) - - -def is_script( - path, # type: str - pattern=None, # type: Optional[str] - check_executable=True, # type: bool -): - # type: (...) -> bool - """Determines if the given path is a script. - - A script is a file that starts with a shebang (#!...) line. - - :param path: The path to check. - :param pattern: An optional pattern to match against the shebang (excluding the leading #!). - :param check_executable: Check that the script is executable by the current user. - :return: True if the given path is a script. - """ - if check_executable and not is_exe(path): - return False - with open(path, "rb") as fp: - if b"#!" != fp.read(2): - return False - if not pattern: - return True - return bool(re.match(pattern, fp.readline().decode("utf-8"))) - - -def is_python_script( - path, # type: str - check_executable=True, # type: bool -): - # type: (...) -> bool - return is_script(path, pattern=r"(?i)^.*(?:python|pypy)", check_executable=check_executable) - - def can_write_dir(path): # type: (str) -> bool """Determines if the directory at path can be written to by the current process. diff --git a/pex/executables.py b/pex/executables.py new file mode 100644 index 000000000..34a28172f --- /dev/null +++ b/pex/executables.py @@ -0,0 +1,147 @@ +# Copyright 2025 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +import re +import stat +from textwrap import dedent + +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import BinaryIO, Callable, Optional, Text, Tuple + + +def chmod_plus_x(path): + # type: (Text) -> None + """Equivalent of unix `chmod a+x path`""" + path_mode = os.stat(path).st_mode + path_mode &= int("777", 8) + if path_mode & stat.S_IRUSR: + path_mode |= stat.S_IXUSR + if path_mode & stat.S_IRGRP: + path_mode |= stat.S_IXGRP + if path_mode & stat.S_IROTH: + path_mode |= stat.S_IXOTH + os.chmod(path, path_mode) + + +def is_exe(path): + # type: (Text) -> bool + """Determines if the given path is a file executable by the current user. + + :param path: The path to check. + :return: True if the given path is a file executable by the current user. + """ + return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK) + + +def is_script( + path, # type: Text + pattern=None, # type: Optional[bytes] + check_executable=True, # type: bool + extra_check=None, # type: Optional[Callable[[bytes, BinaryIO], bool]] +): + # type: (...) -> bool + """Determines if the given path is a script. + + A script is a file that starts with a shebang (#!...) line. + + :param path: The path to check. + :param pattern: Optional pattern to match against the shebang line (excluding the leading #! + and trailing \n). + :param check_executable: Check that the script is executable by the current user. + :param extra_check: Optional callable accepting the shebang line (excluding the leading #! and + trailing \n) and a file opened for binary read pointing just after that + line. + :return: True if the given path is a script. + """ + if check_executable and not is_exe(path): + return False + with open(path, "rb") as fp: + if b"#!" != fp.read(2): + return False + if not pattern: + return True + shebang_suffix = fp.readline().rstrip() + if bool(re.match(pattern, shebang_suffix)): + return True + if extra_check: + return extra_check(shebang_suffix, fp) + return False + + +def create_sh_python_redirector_shebang(sh_script_content): + # type: (str) -> Tuple[str, str] + """Create a shebang block for a Python file that uses /bin/sh to find an appropriate Python. + + The script should be POSIX compliant sh and terminate on all execution paths with an + explicit exit or exec. + + The returned shebang block will include the leading `#!` but will not include a trailing new + line character. + + :param sh_script_content: A POSIX compliant sh script that always explicitly terminates. + :return: A shebang line and trailing block of text that can be combined for use as a shebang + header for a Python file. + """ + # This trick relies on /bin/sh being ubiquitous and the concordance of: + # + # 1. Python: Has triple quoted strings plus allowance for free-floating string values in + # python files. + # 2. sh: Any number of pairs of `'` evaluating away when followed immediately by a + # command string (`''command` -> `command`). + # 3. sh: The `:` noop command which accepts and discards arbitrary args. + # See: https://pubs.opengroup.org/onlinepubs/009604599/utilities/colon.html + # 4. sh: Lazy parsing allowing for invalid sh content immediately following an exit or exec + # line. + # + # The end result is a file that is both a valid sh script with a short shebang and a valid + # Python program. + return "#!/bin/sh", ( + dedent( + """\ + '''': pshprs + {sh_script_content} + ''' + """ + ) + .format(sh_script_content=sh_script_content.rstrip()) + .strip() + ) + + +def is_python_script( + path, # type: Text + check_executable=True, # type: bool +): + # type: (...) -> bool + return is_script( + path, + pattern=br"""(?ix) + # The aim is to admit the common shebang forms: + # + python + # + /usr/bin/env ()? ()? + # + /absolute/path/to/ ()? + # The 1st corresponds to the special placeholder shebang #!python specified here: + # + https://peps.python.org/pep-0427 + # + https://packaging.python.org/specifications/binary-distribution-format + (?:^|.*\W) + + # Python executable names Pex supports (see PythonIdentity). + (?: + python + | pypy + ) + + # Optional Python version + (?:\d+(?:\.\d+)*)? + + # Support a shebang with an argument to the interpreter at the end. + (?:\s[^\s]|$) + """, + check_executable=check_executable, + extra_check=lambda shebang, fp: shebang == b"/bin/sh" and fp.read(13) == b"'''': pshprs\n", + ) diff --git a/pex/finders.py b/pex/finders.py index 747e325ab..fb80c5227 100644 --- a/pex/finders.py +++ b/pex/finders.py @@ -6,7 +6,7 @@ import ast import os -from pex.common import is_python_script, open_zip, safe_mkdtemp +from pex.common import open_zip, safe_mkdtemp from pex.dist_metadata import ( CallableEntryPoint, Distribution, @@ -14,6 +14,7 @@ ModuleEntryPoint, NamedEntryPoint, ) +from pex.executables import is_python_script from pex.pep_376 import InstalledWheel from pex.pep_503 import ProjectName from pex.typing import TYPE_CHECKING, cast diff --git a/pex/interpreter.py b/pex/interpreter.py index 761d38619..7323fbc3d 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -17,7 +17,8 @@ from pex import third_party from pex.cache.dirs import InterpreterDir -from pex.common import is_exe, safe_mkdtemp, safe_rmtree +from pex.common import safe_mkdtemp, safe_rmtree +from pex.executables import is_exe from pex.executor import Executor from pex.jobs import Job, Retain, SpawnedJob, execute_parallel from pex.orderedset import OrderedSet diff --git a/pex/layout.py b/pex/layout.py index e38425bb9..5f5699fef 100644 --- a/pex/layout.py +++ b/pex/layout.py @@ -11,8 +11,9 @@ from pex.atomic_directory import atomic_directory from pex.cache import access as cache_access from pex.cache.dirs import BootstrapDir, InstalledWheelDir, UserCodeDir -from pex.common import ZipFileEx, is_script, open_zip, safe_copy, safe_mkdir, safe_mkdtemp +from pex.common import ZipFileEx, open_zip, safe_copy, safe_mkdir, safe_mkdtemp from pex.enum import Enum +from pex.executables import is_script from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.variables import ENV, unzip_dir @@ -559,7 +560,9 @@ def extract_code(self, dest_dir): if root == self._path: dirs[:] = [d for d in dirs if d not in ("__pex__", DEPS_DIR)] files[:] = [ - f for f in files if f not in ("__main__.py", PEX_INFO_PATH, BOOTSTRAP_DIR) + f + for f in files + if f not in ("__main__.py", "pex", PEX_INFO_PATH, BOOTSTRAP_DIR) ] for d in dirs: safe_mkdir(os.path.join(dest_dir, rel_root, d)) @@ -635,7 +638,7 @@ def extract_code(self, dest_dir): rel_root = os.path.relpath(root, self._path) if root == self._path: dirs[:] = [d for d in dirs if d not in ("__pex__", DEPS_DIR, BOOTSTRAP_DIR)] - files[:] = [f for f in files if f not in ("__main__.py", PEX_INFO_PATH)] + files[:] = [f for f in files if f not in ("__main__.py", "pex", PEX_INFO_PATH)] for d in dirs: safe_mkdir(os.path.join(dest_dir, rel_root, d)) for f in files: diff --git a/pex/pep_427.py b/pex/pep_427.py index 046725c4e..291598476 100644 --- a/pex/pep_427.py +++ b/pex/pep_427.py @@ -14,10 +14,11 @@ from textwrap import dedent from pex import pex_warnings -from pex.common import chmod_plus_x, is_pyc_file, iter_copytree, open_zip, safe_open, touch +from pex.common import is_pyc_file, iter_copytree, open_zip, safe_open, touch from pex.compatibility import commonpath, get_stdout_bytes_buffer from pex.dist_metadata import CallableEntryPoint, Distribution, ProjectNameAndVersion from pex.enum import Enum +from pex.executables import chmod_plus_x from pex.interpreter import PythonInterpreter from pex.pep_376 import InstalledFile, InstalledWheel, Record from pex.pep_503 import ProjectName diff --git a/pex/pex.py b/pex/pex.py index 4da401d4d..2ce7c478a 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -542,6 +542,7 @@ def execute(self): This function makes assumptions that it is the last function called by the interpreter. """ + pex_file = self._vars.PEX if self._vars.PEX_TOOLS: if not self._pex_info.includes_tools: die( @@ -551,11 +552,10 @@ def execute(self): from pex.tools import main as tools - sys.exit(tools.main(pex=PEX(sys.argv[0]))) + sys.exit(tools.main(pex=PEX(pex_file or sys.argv[0]))) self.activate() - pex_file = self._vars.PEX if pex_file: try: from setproctitle import setproctitle # type: ignore[import] diff --git a/pex/pex_boot.py b/pex/pex_boot.py index a936a79fe..fa7d5755e 100644 --- a/pex/pex_boot.py +++ b/pex/pex_boot.py @@ -115,7 +115,7 @@ def __maybe_run_venv__( ): # type: (...) -> Optional[str] - from pex.common import is_exe + from pex.executables import is_exe from pex.tracer import TRACER from pex.variables import venv_dir @@ -180,7 +180,13 @@ def boot( break installed_from = os.environ.pop(__INSTALLED_FROM__, None) - sys.argv[0] = installed_from or sys.argv[0] + if installed_from: + if os.path.isfile(installed_from): + sys.argv[0] = installed_from + else: + pex_exe = os.path.join(installed_from, "pex") + if os.path.isfile(pex_exe): + sys.argv[0] = pex_exe sys.path[0] = os.path.abspath(sys.path[0]) sys.path.insert(0, os.path.abspath(os.path.join(entry_point, bootstrap_dir))) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 82bca4fd8..a3a205afd 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -17,7 +17,6 @@ from pex.common import ( Chroot, CopyMode, - chmod_plus_x, deterministic_walk, is_pyc_file, is_pyc_temporary_file, @@ -31,6 +30,7 @@ from pex.compiler import Compiler from pex.dist_metadata import Distribution, DistributionType, MetadataError from pex.enum import Enum +from pex.executables import chmod_plus_x, create_sh_python_redirector_shebang from pex.finders import get_entry_point_from_console_script, get_script_from_distributions from pex.interpreter import PythonInterpreter from pex.layout import Layout @@ -660,6 +660,19 @@ def build( compress=compress, bytecode_compile=bytecode_compile, ) + if layout in (Layout.LOOSE, Layout.PACKED): + pex_script = os.path.join(tmp_pex, "pex") + if self._header: + main_py = os.path.join(tmp_pex, "__main__.py") + with open(pex_script, "w") as script_fp: + print(self._shebang, file=script_fp) + print(self._header, file=script_fp) + with open(main_py) as main_fp: + main_fp.readline() # Throw away shebang line. + shutil.copyfileobj(main_fp, script_fp) + chmod_plus_x(pex_script) + os.rename(pex_script, main_py) + os.symlink("__main__.py", pex_script) if os.path.isdir(path): shutil.rmtree(path, True) @@ -673,19 +686,26 @@ def set_sh_boot_script( pex_name, # type: str targets, # type: Targets python_shebang, # type: Optional[str] + layout=Layout.ZIPAPP, # type: Layout.Value ): if not self._frozen: raise Exception("Generating a sh_boot script requires the pex to be frozen.") - self.set_shebang("/bin/sh") script = create_sh_boot_script( pex_name=pex_name, pex_info=self._pex_info, targets=targets, interpreter=self.interpreter, python_shebang=python_shebang, + layout=layout, ) - self.set_header(script) + if layout is Layout.ZIPAPP: + self.set_shebang("/bin/sh") + self.set_header(script) + else: + shebang, header = create_sh_python_redirector_shebang(script) + self.set_shebang(shebang) + self.set_header(header) def _build_packedapp( self, diff --git a/pex/pyenv.py b/pex/pyenv.py index ba36ab59a..38988fe00 100644 --- a/pex/pyenv.py +++ b/pex/pyenv.py @@ -7,7 +7,7 @@ import re import subprocess -from pex.common import is_exe +from pex.executables import is_exe from pex.tracer import TRACER from pex.typing import TYPE_CHECKING diff --git a/pex/repl/pex_repl.py b/pex/repl/pex_repl.py index 0d4d7aa5f..042d7e9ce 100644 --- a/pex/repl/pex_repl.py +++ b/pex/repl/pex_repl.py @@ -11,9 +11,10 @@ from zipfile import is_zipfile from pex.cli_util import prog_path -from pex.common import is_exe, pluralize +from pex.common import pluralize from pex.compatibility import commonpath from pex.dist_metadata import Distribution +from pex.executables import is_exe from pex.layout import Layout from pex.pex_info import PexInfo from pex.repl import custom diff --git a/pex/scie/science.py b/pex/scie/science.py index e42d275c9..af04e5f16 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -13,10 +13,11 @@ from pex import toml from pex.atomic_directory import atomic_directory from pex.cache.dirs import CacheDir, UnzipDir -from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open +from pex.common import pluralize, safe_mkdtemp, safe_open from pex.compatibility import shlex_quote from pex.dist_metadata import NamedEntryPoint, parse_entry_point from pex.exceptions import production_assert +from pex.executables import chmod_plus_x, is_exe from pex.fetcher import URLFetcher from pex.hashing import Sha256 from pex.layout import Layout diff --git a/pex/sh_boot.py b/pex/sh_boot.py index f932da7dd..98b01a51d 100644 --- a/pex/sh_boot.py +++ b/pex/sh_boot.py @@ -13,6 +13,7 @@ from pex.dist_metadata import Distribution from pex.interpreter import PythonInterpreter, calculate_binary_name from pex.interpreter_constraints import InterpreterConstraints, iter_compatible_versions +from pex.layout import Layout from pex.orderedset import OrderedSet from pex.pep_440 import Version from pex.pex_info import PexInfo @@ -123,6 +124,7 @@ def create_sh_boot_script( targets, # type: Targets interpreter, # type: PythonInterpreter python_shebang=None, # type: Optional[str] + layout=Layout.ZIPAPP, # type: Layout.Value ): # type: (...) -> str """Creates the body of a POSIX `sh` compatible script that executes a PEX ZIPAPP appended to it. @@ -199,7 +201,7 @@ def create_sh_boot_script( # We're a --venv execution mode PEX installed under the PEX_ROOT and the venv # interpreter to use is embedded in the shebang of our venv pex script; so just # execute that script directly. - export PEX="$0" + export PEX="{pex}" exec "${{INSTALLED_PEX}}/bin/python" ${{VENV_PYTHON_ARGS}} "${{INSTALLED_PEX}}" \\ "$@" fi @@ -227,7 +229,7 @@ def create_sh_boot_script( # We're a --zipapp execution mode PEX installed under the PEX_ROOT with a # __main__.py in our top-level directory; so execute Python against that # directory. - export __PEX_EXE__="$0" + export __PEX_EXE__="{pex}" exec "${{python_exe}}" ${{PYTHON_ARGS}} "${{INSTALLED_PEX}}" "$@" else # The slow path: this PEX zipapp is not installed yet. Run the PEX zipapp so it @@ -261,4 +263,5 @@ def create_sh_boot_script( venv_python_args=" ".join( shlex_quote(venv_python_arg) for venv_python_arg in venv_python_args ), + pex="$0" if layout is Layout.ZIPAPP else '$(dirname "$0")', ) diff --git a/pex/venv/installer.py b/pex/venv/installer.py index cd78f72cf..f30e56fd6 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -11,10 +11,11 @@ from pex import layout, pex_warnings, repl from pex.cache import access as cache_access -from pex.common import CopyMode, chmod_plus_x, iter_copytree, pluralize +from pex.common import CopyMode, iter_copytree, pluralize from pex.compatibility import is_valid_python_identifier from pex.dist_metadata import Distribution from pex.environment import PEXEnvironment +from pex.executables import chmod_plus_x from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel from pex.pep_440 import Version @@ -535,6 +536,7 @@ def mount(cls, pex): "__main__.py", "__pex__", "__pycache__", + "pex", cache_access.LAST_ACCESS_FILE, layout.BOOTSTRAP_DIR, layout.DEPS_DIR, diff --git a/pex/venv/venv_pex.py b/pex/venv/venv_pex.py index daed27e8e..613494eb6 100644 --- a/pex/venv/venv_pex.py +++ b/pex/venv/venv_pex.py @@ -5,11 +5,11 @@ import os import sys -from types import CodeType TYPE_CHECKING = False if TYPE_CHECKING: + from types import CodeType from typing import Any, Dict, Iterable, List, Optional, Tuple _PY2_EXEC_FUNCTION = """ @@ -101,7 +101,12 @@ def maybe_log(*message): pex_file = os.environ.get("PEX", None) if pex_file: pex_file_path = os.path.realpath(pex_file) - sys.argv[0] = pex_file_path + if os.path.isfile(pex_file_path): + sys.argv[0] = pex_file_path + else: + pex_exe = os.path.join(pex_file_path, "pex") + if os.path.isfile(pex_exe): + sys.argv[0] = pex_exe os.environ["PEX"] = pex_file_path try: from setproctitle import setproctitle # type: ignore[import] diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index 413df644d..de7d1f7da 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -15,10 +15,11 @@ from textwrap import dedent from pex.atomic_directory import AtomicDirectory, atomic_directory -from pex.common import is_exe, safe_mkdir, safe_open +from pex.common import safe_mkdir, safe_open from pex.compatibility import commonpath, get_stdout_bytes_buffer from pex.dist_metadata import Distribution, find_distributions from pex.enum import Enum +from pex.executables import is_exe from pex.executor import Executor from pex.fetcher import URLFetcher from pex.interpreter import ( diff --git a/pex/version.py b/pex/version.py index 39be95acf..2842169b6 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.29.0" +__version__ = "2.30.0" diff --git a/testing/__init__.py b/testing/__init__.py index fb0538b2b..f7e3b1efe 100644 --- a/testing/__init__.py +++ b/testing/__init__.py @@ -423,7 +423,7 @@ def create_pex_command( # type: (...) -> List[str] cmd = [python or sys.executable, "-mpex"] if not quiet: - cmd.append("-vvvvv") + cmd.append("-v") if args: cmd.extend(args) return cmd diff --git a/testing/docker.py b/testing/docker.py index 1b401ad7e..e9aaeacf8 100644 --- a/testing/docker.py +++ b/testing/docker.py @@ -9,7 +9,8 @@ import pytest -from pex.common import chmod_plus_x, is_exe, safe_mkdtemp +from pex.common import safe_mkdtemp +from pex.executables import chmod_plus_x, is_exe from pex.typing import TYPE_CHECKING from testing import pex_project_dir diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index e15ac1d1a..fda8c7503 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -16,7 +16,8 @@ import pytest from pex.cache.dirs import CacheDir -from pex.common import chmod_plus_x, is_exe, safe_open +from pex.common import safe_open +from pex.executables import chmod_plus_x, is_exe from pex.fetcher import URLFetcher from pex.layout import Layout from pex.orderedset import OrderedSet diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 164763655..62819e2b3 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -19,17 +19,10 @@ from pex import targets from pex.cache.dirs import CacheDir, InterpreterDir -from pex.common import ( - environment_as, - is_exe, - safe_mkdir, - safe_open, - safe_rmtree, - temporary_dir, - touch, -) +from pex.common import environment_as, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch from pex.compatibility import WINDOWS, commonpath from pex.dist_metadata import Distribution, Requirement, is_wheel +from pex.executables import is_exe from pex.fetcher import URLFetcher from pex.interpreter import PythonInterpreter from pex.layout import Layout diff --git a/tests/integration/test_issue_2017.py b/tests/integration/test_issue_2017.py index 67d2fc2bf..2c8d69b31 100644 --- a/tests/integration/test_issue_2017.py +++ b/tests/integration/test_issue_2017.py @@ -12,8 +12,9 @@ import pytest from pex.atomic_directory import atomic_directory -from pex.common import is_exe, safe_open +from pex.common import safe_open from pex.compatibility import urlparse +from pex.executables import is_exe from pex.fetcher import URLFetcher from pex.pip.version import PipVersion, PipVersionValue from pex.typing import TYPE_CHECKING diff --git a/tests/integration/test_issue_2413.py b/tests/integration/test_issue_2413.py index 9c3129055..01e9fbb80 100644 --- a/tests/integration/test_issue_2413.py +++ b/tests/integration/test_issue_2413.py @@ -10,7 +10,8 @@ import pytest -from pex.common import is_exe, safe_open +from pex.common import safe_open +from pex.executables import is_exe from pex.pep_503 import ProjectName from pex.typing import TYPE_CHECKING from pex.venv.virtualenv import InstallationChoice, Virtualenv diff --git a/tests/integration/test_pep_427.py b/tests/integration/test_pep_427.py index b984717a0..9e5383597 100644 --- a/tests/integration/test_pep_427.py +++ b/tests/integration/test_pep_427.py @@ -7,7 +7,8 @@ from glob import glob from textwrap import dedent -from pex.common import is_exe, safe_open +from pex.common import safe_open +from pex.executables import is_exe from pex.pep_427 import install_wheel_interpreter from pex.pip.installation import get_pip from pex.resolve.configured_resolver import ConfiguredResolver diff --git a/tests/integration/test_sh_boot.py b/tests/integration/test_sh_boot.py index be04d9055..8929115a7 100644 --- a/tests/integration/test_sh_boot.py +++ b/tests/integration/test_sh_boot.py @@ -11,6 +11,7 @@ import pytest from pex.common import safe_open +from pex.layout import Layout from pex.typing import TYPE_CHECKING from testing import all_pythons, make_env, run_pex_command from testing.pytest.tmp import Tempdir @@ -174,23 +175,33 @@ def test_python_shebang_respected(tmpdir): assert output.startswith(version), output -EXECUTION_MODE_ARGS_PERMUTATIONS = [ - pytest.param([], id="ZIPAPP"), - pytest.param(["--venv"], id="VENV"), - pytest.param(["--sh-boot"], id="ZIPAPP (--sh-boot)"), - pytest.param(["--venv", "--sh-boot"], id="VENV (--sh-boot)"), -] +execution_mode = pytest.mark.parametrize( + "execution_mode_args", + [ + pytest.param([], id="ZIPAPP"), + pytest.param(["--venv"], id="VENV"), + pytest.param(["--sh-boot"], id="ZIPAPP (--sh-boot)"), + pytest.param(["--venv", "--sh-boot"], id="VENV (--sh-boot)"), + ], +) +layouts = pytest.mark.parametrize( + "layout", [pytest.param(layout, id=layout.value) for layout in Layout.values()] +) -@pytest.mark.parametrize("execution_mode_args", EXECUTION_MODE_ARGS_PERMUTATIONS) +@execution_mode +@layouts def test_issue_1782( tmpdir, # type: Tempdir pex_project_dir, # type: str execution_mode_args, # type: List[str] + layout, # type: Layout.Value ): # type: (...) -> None pex = os.path.realpath(tmpdir.join("pex.sh")) + pex_exe = pex if layout is Layout.ZIPAPP else os.path.join(pex, "pex") + pex_root = os.path.realpath(tmpdir.join("pex_root")) python = "python{major}.{minor}".format(major=sys.version_info[0], minor=sys.version_info[1]) run_pex_command( @@ -206,18 +217,20 @@ def test_issue_1782( pex, "--python-shebang", "/usr/bin/env {python}".format(python=python), + "--layout", + str(layout), ] + execution_mode_args ).assert_success() if sys.version_info[:2] >= (3, 14) and "--venv" not in execution_mode_args: - argv0 = r"python(?:3(?:\.\d{{2,}})?)? {pex}".format(pex=re.escape(pex)) + argv0 = r"python(?:3(?:\.\d{{2,}})?)? {pex}".format(pex=re.escape(pex_exe)) else: - argv0 = re.escape(os.path.basename(pex)) + argv0 = re.escape(os.path.basename(pex_exe)) usage_line_re = re.compile(r"^usage: {argv0}".format(argv0=argv0)) help_line1 = ( subprocess.check_output( - args=[pex, "-h"], env=make_env(COLUMNS=max(80, len(usage_line_re.pattern) + 10)) + args=[pex_exe, "-h"], env=make_env(COLUMNS=max(80, len(usage_line_re.pattern) + 10)) ) .decode("utf-8") .splitlines()[0] @@ -231,21 +244,26 @@ def test_issue_1782( assert ( pex == subprocess.check_output( - args=[pex, "-c", "import os; print(os.environ['PEX'])"], env=make_env(PEX_INTERPRETER=1) + args=[pex_exe, "-c", "import os; print(os.environ['PEX'])"], + env=make_env(PEX_INTERPRETER=1), ) .decode("utf-8") .strip() ) -@pytest.mark.parametrize("execution_mode_args", EXECUTION_MODE_ARGS_PERMUTATIONS) +@execution_mode +@layouts def test_argv0( tmpdir, # type: Any execution_mode_args, # type: List[str] + layout, # type: Layout.Value ): # type: (...) -> None pex = os.path.realpath(os.path.join(str(tmpdir), "pex.sh")) + pex_exe = pex if layout is Layout.ZIPAPP else os.path.join(pex, "pex") + src = os.path.join(str(tmpdir), "src") with safe_open(os.path.join(src, "app.py"), "w") as fp: fp.write( @@ -267,12 +285,14 @@ def main(): ) run_pex_command( - args=["-D", src, "-e", "app:main", "-o", pex] + execution_mode_args + args=["-D", src, "-e", "app:main", "-o", pex, "--layout", str(layout)] + execution_mode_args ).assert_success() - assert {"PEX": pex, "argv0": pex} == json.loads(subprocess.check_output(args=[pex])) + assert {"PEX": pex, "argv0": pex_exe} == json.loads(subprocess.check_output(args=[pex_exe])) - run_pex_command(args=["-D", src, "-m", "app", "-o", pex] + execution_mode_args).assert_success() - data = json.loads(subprocess.check_output(args=[pex])) + run_pex_command( + args=["-D", src, "-m", "app", "-o", pex, "--layout", str(layout)] + execution_mode_args + ).assert_success() + data = json.loads(subprocess.check_output(args=[pex_exe])) assert pex == data.pop("PEX") assert "app.py" == os.path.basename(data.pop("argv0")), ( "When executing modules we expect runpy.run_module to `alter_sys` in order to support " diff --git a/tests/integration/test_shebang_length_limit.py b/tests/integration/test_shebang_length_limit.py index 40ed1eea1..e701f93ec 100644 --- a/tests/integration/test_shebang_length_limit.py +++ b/tests/integration/test_shebang_length_limit.py @@ -15,7 +15,8 @@ from pex.atomic_directory import atomic_directory from pex.cache.dirs import CacheDir -from pex.common import chmod_plus_x, safe_open, touch +from pex.common import safe_open, touch +from pex.executables import chmod_plus_x from pex.typing import TYPE_CHECKING from testing import IS_PYPY, make_project, run_pex_command from testing.cli import run_pex3 diff --git a/tests/pip/test_log_analyzer.py b/tests/pip/test_log_analyzer.py index afaae43a1..09810295e 100644 --- a/tests/pip/test_log_analyzer.py +++ b/tests/pip/test_log_analyzer.py @@ -9,7 +9,7 @@ import pytest -from pex.common import chmod_plus_x +from pex.executables import chmod_plus_x from pex.jobs import Job from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob from pex.typing import TYPE_CHECKING diff --git a/tests/test_common.py b/tests/test_common.py index e1fabeb25..d3c4eb191 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -11,15 +11,13 @@ Chroot, ZipFileEx, can_write_dir, - chmod_plus_x, deterministic_walk, - is_exe, - is_script, open_zip, safe_open, temporary_dir, touch, ) +from pex.executables import chmod_plus_x from pex.typing import TYPE_CHECKING from testing import NonDeterministicWalk @@ -29,7 +27,7 @@ import mock # type: ignore[no-redef,import] if TYPE_CHECKING: - from typing import Any, Iterator, Tuple + from typing import Iterator, Tuple def extract_perms(path): @@ -308,54 +306,3 @@ def test_safe_open_relative(temporary_working_dir): abs_path = os.path.join(temporary_working_dir, rel_path) with open(abs_path) as fp: assert "contents" == fp.read() - - -def test_is_exe(tmpdir): - # type: (Any) -> None - all_exe = os.path.join(str(tmpdir), "all_exe") - touch(all_exe) - chmod_plus_x(all_exe) - assert is_exe(all_exe) - - other_exe = os.path.join(str(tmpdir), "other_exe") - touch(other_exe) - os.chmod(other_exe, 0o665) - assert not is_exe(other_exe) - - not_exe = os.path.join(str(tmpdir), "not_exe") - touch(not_exe) - assert not is_exe(not_exe) - - exe_dir = os.path.join(str(tmpdir), "exe_dir") - os.mkdir(exe_dir) - chmod_plus_x(exe_dir) - assert not is_exe(exe_dir) - - -def test_is_script(tmpdir): - # type: (Any) -> None - exe = os.path.join(str(tmpdir), "exe") - - touch(exe) - assert not is_exe(exe) - assert not is_script(exe) - - chmod_plus_x(exe) - assert is_exe(exe) - assert not is_script(exe) - - with open(exe, "wb") as fp: - fp.write(bytearray([0xCA, 0xFE, 0xBA, 0xBE])) - assert not is_script(fp.name) - - with open(exe, "wb") as fp: - fp.write(b"#!/mystery\n") - fp.write(bytearray([0xCA, 0xFE, 0xBA, 0xBE])) - assert is_script(exe) - assert is_script(exe, pattern=r"^/mystery") - assert not is_script(exe, pattern=r"^python") - - os.chmod(exe, 0o665) - assert is_script(exe, check_executable=False) - assert not is_script(exe) - assert not is_exe(exe) diff --git a/tests/test_executables.py b/tests/test_executables.py new file mode 100644 index 000000000..4e3e0eccf --- /dev/null +++ b/tests/test_executables.py @@ -0,0 +1,115 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os + +from pex.common import touch +from pex.executables import chmod_plus_x, is_exe, is_python_script, is_script +from testing.pytest.tmp import Tempdir + + +def test_is_exe(tmpdir): + # type: (Tempdir) -> None + + all_exe = tmpdir.join("all_exe") + touch(all_exe) + chmod_plus_x(all_exe) + assert is_exe(all_exe) + + other_exe = tmpdir.join("other_exe") + touch(other_exe) + os.chmod(other_exe, 0o665) + assert not is_exe(other_exe) + + not_exe = tmpdir.join("not_exe") + touch(not_exe) + assert not is_exe(not_exe) + + exe_dir = tmpdir.join("exe_dir") + os.mkdir(exe_dir) + chmod_plus_x(exe_dir) + assert not is_exe(exe_dir) + + +def test_is_script(tmpdir): + # type: (Tempdir) -> None + + exe = tmpdir.join("exe") + + touch(exe) + assert not is_exe(exe) + assert not is_script(exe, pattern=None, check_executable=True) + + chmod_plus_x(exe) + assert is_exe(exe) + assert not is_script(exe, pattern=None, check_executable=True) + + with open(exe, "wb") as fp: + fp.write(bytearray([0xCA, 0xFE, 0xBA, 0xBE])) + assert not is_script(fp.name, pattern=None, check_executable=True) + + with open(exe, "wb") as fp: + fp.write(b"#!/mystery\n") + fp.write(bytearray([0xCA, 0xFE, 0xBA, 0xBE])) + assert is_script(exe, pattern=None, check_executable=True) + assert is_script(exe, pattern=br"^/mystery", check_executable=True) + assert not is_script(exe, pattern=br"^python", check_executable=True) + + os.chmod(exe, 0o665) + assert is_script(exe, pattern=None, check_executable=False) + assert not is_script(exe, pattern=None, check_executable=True) + assert not is_exe(exe) + + +def test_is_python_script(tmpdir): + # type: (Tempdir) -> None + + exe = tmpdir.join("exe") + + touch(exe) + assert not is_python_script(exe, check_executable=False) + assert not is_python_script(exe, check_executable=True) + + def write_shebang(shebang): + # type: (str) -> None + with open(exe, "w") as fp: + fp.write(shebang) + + write_shebang("#!python") + assert is_python_script(exe, check_executable=False) + assert not is_python_script(exe, check_executable=True) + + chmod_plus_x(exe) + assert is_python_script(exe, check_executable=True) + + write_shebang("#!/usr/bin/python") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/python3") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/python3.13") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/python -sE") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env python") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env python2.7") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env python -sE") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env -S python") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env -S python3") + assert is_python_script(exe) + + write_shebang("#!/usr/bin/env -S python -sE") + assert is_python_script(exe) diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index de95e0661..e76f709ac 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -14,7 +14,8 @@ import pytest from pex.cache.dirs import InterpreterDir -from pex.common import chmod_plus_x, environment_as, safe_mkdir, safe_mkdtemp, temporary_dir, touch +from pex.common import environment_as, safe_mkdir, safe_mkdtemp, temporary_dir, touch +from pex.executables import chmod_plus_x from pex.executor import Executor from pex.interpreter import PythonInterpreter, create_shebang from pex.jobs import Job