From 464735f1f188fd2dd6eaac12d2985e851e4658bd Mon Sep 17 00:00:00 2001 From: Aohan Dang Date: Tue, 8 Oct 2024 10:14:57 -0400 Subject: [PATCH 01/11] Fix issue with absolute path with Python 3.13 on Windows With Python 3.13 on Windows, `os.path.isabs()` no longer returns `True` for a path that starts with a slash. Thus, when the argument to `_make_relative()` is an absolute path, the return value starts with a slash on Python 3.13 and does not start with a slash on older Python versions. This causes the extension module build directory to be calculated incorrectly with Python 3.13 on Windows. Fix this by ensuring that the return value does not start with a slash. --- distutils/ccompiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 5e73e56d02..fdbb1ca795 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -989,7 +989,8 @@ def _make_relative(base): # Chop off the drive no_drive = os.path.splitdrive(base)[1] # If abs, chop off leading / - return no_drive[os.path.isabs(no_drive) :] + is_abs = os.path.isabs(no_drive) or sys.platform == 'win32' and (no_drive.startswith('/') or no_drive.startswith('\\')) + return no_drive[is_abs:] def shared_object_filename(self, basename, strip_dir=False, output_dir=''): assert output_dir is not None From edfd6d2159374575cdcc16834712243ed366c64f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 26 Dec 2024 21:18:30 -0500 Subject: [PATCH 02/11] Collapse startswith operations Co-authored-by: Avasam --- distutils/ccompiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index fdbb1ca795..6979d160eb 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -989,7 +989,7 @@ def _make_relative(base): # Chop off the drive no_drive = os.path.splitdrive(base)[1] # If abs, chop off leading / - is_abs = os.path.isabs(no_drive) or sys.platform == 'win32' and (no_drive.startswith('/') or no_drive.startswith('\\')) + is_abs = os.path.isabs(no_drive) or sys.platform == 'win32' and no_drive.startswith(('/', "\\")) return no_drive[is_abs:] def shared_object_filename(self, basename, strip_dir=False, output_dir=''): From bba52647d2e90368e7662d5573bb9a63a9184318 Mon Sep 17 00:00:00 2001 From: Sam James Date: Sat, 28 Dec 2024 16:45:50 +0000 Subject: [PATCH 03/11] Don't duplicate CFLAGS Followup to af7fcbb0d56ae14753db53acd8792eddb4d8f814. I accidentally left that in when trying two approaches. Reported at https://github.com/pypa/distutils/pull/322#discussion_r1898349462. --- distutils/sysconfig.py | 1 - distutils/tests/test_sysconfig.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/distutils/sysconfig.py b/distutils/sysconfig.py index 358d1079dc..ef3def83eb 100644 --- a/distutils/sysconfig.py +++ b/distutils/sysconfig.py @@ -341,7 +341,6 @@ def customize_compiler(compiler): ldshared = _add_flags(ldshared, 'LD') ldcxxshared = _add_flags(ldcxxshared, 'LD') cflags = os.environ.get('CFLAGS', cflags) - cflags = _add_flags(cflags, 'C') ldshared = _add_flags(ldshared, 'C') cxxflags = os.environ.get('CXXFLAGS', cxxflags) ldcxxshared = _add_flags(ldcxxshared, 'CXX') diff --git a/distutils/tests/test_sysconfig.py b/distutils/tests/test_sysconfig.py index 867e7dcb39..43d77c23fa 100644 --- a/distutils/tests/test_sysconfig.py +++ b/distutils/tests/test_sysconfig.py @@ -130,11 +130,9 @@ def test_customize_compiler(self): comp = self.customize_compiler() assert comp.exes['archiver'] == 'env_ar --env-arflags' assert comp.exes['preprocessor'] == 'env_cpp --env-cppflags' - assert ( - comp.exes['compiler'] == 'env_cc --env-cflags --env-cflags --env-cppflags' - ) + assert comp.exes['compiler'] == 'env_cc --env-cflags --env-cppflags' assert comp.exes['compiler_so'] == ( - 'env_cc --env-cflags --env-cflags --env-cppflags --sc-ccshared' + 'env_cc --env-cflags --env-cppflags --sc-ccshared' ) assert ( comp.exes['compiler_cxx'] From d13d5a7d9210114447aae0ba1a814ae6af8aeffe Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 04:39:08 -0500 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20?= =?UTF-8?q?(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- distutils/ccompiler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 6979d160eb..fbf1f7a4cf 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -989,7 +989,11 @@ def _make_relative(base): # Chop off the drive no_drive = os.path.splitdrive(base)[1] # If abs, chop off leading / - is_abs = os.path.isabs(no_drive) or sys.platform == 'win32' and no_drive.startswith(('/', "\\")) + is_abs = ( + os.path.isabs(no_drive) + or sys.platform == 'win32' + and no_drive.startswith(('/', "\\")) + ) return no_drive[is_abs:] def shared_object_filename(self, basename, strip_dir=False, output_dir=''): From 1400152e10663f51109869b9d387c50820a95767 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 04:56:38 -0500 Subject: [PATCH 05/11] Extract a classmethod _make_out_path_exts suitable for isolated testing. --- distutils/ccompiler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index fbf1f7a4cf..6e303c34e6 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -969,10 +969,16 @@ def out_extensions(self): return dict.fromkeys(self.src_extensions, self.obj_extension) def _make_out_path(self, output_dir, strip_dir, src_name): + return self._make_out_path_exts( + output_dir, strip_dir, src_name, self.out_extensions + ) + + @classmethod + def _make_out_path_exts(cls, output_dir, strip_dir, src_name, extensions): base, ext = os.path.splitext(src_name) - base = self._make_relative(base) + base = cls._make_relative(base) try: - new_ext = self.out_extensions[ext] + new_ext = extensions[ext] except LookupError: raise UnknownFileError(f"unknown file type '{ext}' (from '{src_name}')") if strip_dir: From b16c1de88e33bea41ec5c74abdb225e9f8704a3a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 05:12:54 -0500 Subject: [PATCH 06/11] Re-write _make_relative to rely on pathlib for cross-platform and cross-python compatibility. --- distutils/ccompiler.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 6e303c34e6..9b637f8c98 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -4,6 +4,7 @@ for the Distutils compiler abstraction model.""" import os +import pathlib import re import sys import types @@ -976,31 +977,20 @@ def _make_out_path(self, output_dir, strip_dir, src_name): @classmethod def _make_out_path_exts(cls, output_dir, strip_dir, src_name, extensions): base, ext = os.path.splitext(src_name) + base = pathlib.PurePath(base) + # Ensure base is relative to honor output_dir (python/cpython#37775). base = cls._make_relative(base) try: new_ext = extensions[ext] except LookupError: raise UnknownFileError(f"unknown file type '{ext}' (from '{src_name}')") if strip_dir: - base = os.path.basename(base) - return os.path.join(output_dir, base + new_ext) + base = base.name + return os.path.join(output_dir, base.with_suffix(new_ext)) @staticmethod - def _make_relative(base): - """ - In order to ensure that a filename always honors the - indicated output_dir, make sure it's relative. - Ref python/cpython#37775. - """ - # Chop off the drive - no_drive = os.path.splitdrive(base)[1] - # If abs, chop off leading / - is_abs = ( - os.path.isabs(no_drive) - or sys.platform == 'win32' - and no_drive.startswith(('/', "\\")) - ) - return no_drive[is_abs:] + def _make_relative(base: pathlib.Path): + return base.relative_to(base.anchor) def shared_object_filename(self, basename, strip_dir=False, output_dir=''): assert output_dir is not None From 57152cf9766590e205346752aa632842b5e8a741 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 05:19:35 -0500 Subject: [PATCH 07/11] Move suppress_path_mangle to test_ccompiler to limit the scope. --- conftest.py | 2 +- distutils/tests/test_ccompiler.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 3b9444f78c..578b7aac59 100644 --- a/conftest.py +++ b/conftest.py @@ -92,7 +92,7 @@ def monkeysession(request): mpatch.undo() -@pytest.fixture(autouse=True, scope="session") +@pytest.fixture(scope="module") def suppress_path_mangle(monkeysession): """ Disable the path mangling in CCompiler. Workaround for #169. diff --git a/distutils/tests/test_ccompiler.py b/distutils/tests/test_ccompiler.py index d23b907cad..7ebfed56be 100644 --- a/distutils/tests/test_ccompiler.py +++ b/distutils/tests/test_ccompiler.py @@ -7,6 +7,8 @@ import pytest +pytestmark = pytest.mark.usefixtures('suppress_path_mangle') + def _make_strs(paths): """ From ede1af29d210311f5f033f7226da58a19f197183 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 05:23:51 -0500 Subject: [PATCH 08/11] Add tests and fix failure identified in the tests. --- distutils/ccompiler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 9b637f8c98..7b5baaf9ae 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -976,6 +976,13 @@ def _make_out_path(self, output_dir, strip_dir, src_name): @classmethod def _make_out_path_exts(cls, output_dir, strip_dir, src_name, extensions): + r""" + >>> exts = {'.c': '.o'} + >>> CCompiler._make_out_path_exts('.', False, '/foo/bar.c', exts).replace('\\', '/') + './foo/bar.o' + >>> CCompiler._make_out_path_exts('.', True, '/foo/bar.c', exts).replace('\\', '/') + './bar.o' + """ base, ext = os.path.splitext(src_name) base = pathlib.PurePath(base) # Ensure base is relative to honor output_dir (python/cpython#37775). @@ -985,7 +992,7 @@ def _make_out_path_exts(cls, output_dir, strip_dir, src_name, extensions): except LookupError: raise UnknownFileError(f"unknown file type '{ext}' (from '{src_name}')") if strip_dir: - base = base.name + base = pathlib.PurePath(base.name) return os.path.join(output_dir, base.with_suffix(new_ext)) @staticmethod From 36ce8b329524088cfa53b9a4bffcce3a8d233539 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 4 Jan 2025 05:38:01 -0500 Subject: [PATCH 09/11] Refactor for simplicity. --- distutils/ccompiler.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/distutils/ccompiler.py b/distutils/ccompiler.py index 7b5baaf9ae..714f13d8d3 100644 --- a/distutils/ccompiler.py +++ b/distutils/ccompiler.py @@ -983,14 +983,13 @@ def _make_out_path_exts(cls, output_dir, strip_dir, src_name, extensions): >>> CCompiler._make_out_path_exts('.', True, '/foo/bar.c', exts).replace('\\', '/') './bar.o' """ - base, ext = os.path.splitext(src_name) - base = pathlib.PurePath(base) + src = pathlib.PurePath(src_name) # Ensure base is relative to honor output_dir (python/cpython#37775). - base = cls._make_relative(base) + base = cls._make_relative(src) try: - new_ext = extensions[ext] + new_ext = extensions[src.suffix] except LookupError: - raise UnknownFileError(f"unknown file type '{ext}' (from '{src_name}')") + raise UnknownFileError(f"unknown file type '{src.suffix}' (from '{src}')") if strip_dir: base = pathlib.PurePath(base.name) return os.path.join(output_dir, base.with_suffix(new_ext)) From 5ed9d93e77aa3e2c70d8cea1bfeb15549932169f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 5 Jan 2025 10:25:29 -0500 Subject: [PATCH 10/11] Add news fragment. --- newsfragments/4790.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4790.feature.rst diff --git a/newsfragments/4790.feature.rst b/newsfragments/4790.feature.rst new file mode 100644 index 0000000000..c667e36258 --- /dev/null +++ b/newsfragments/4790.feature.rst @@ -0,0 +1 @@ +Synced with pypa/distutils@ff11eed0c including bugfix for duplicate CFLAGS, adaption to support Python 3.13 is_abs in the C compiler (#4669), and setting of ``Py_GIL_DISABLED=1`` for free threaded Python on Windows (#4662). From c384f184d20e8232a1ce73f88d151b9808b66949 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 5 Jan 2025 10:37:08 -0500 Subject: [PATCH 11/11] Py_GIL_Disabled was handled previously. --- newsfragments/4790.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/4790.feature.rst b/newsfragments/4790.feature.rst index c667e36258..21139708b3 100644 --- a/newsfragments/4790.feature.rst +++ b/newsfragments/4790.feature.rst @@ -1 +1 @@ -Synced with pypa/distutils@ff11eed0c including bugfix for duplicate CFLAGS, adaption to support Python 3.13 is_abs in the C compiler (#4669), and setting of ``Py_GIL_DISABLED=1`` for free threaded Python on Windows (#4662). +Synced with pypa/distutils@ff11eed0c including bugfix for duplicate CFLAGS and adaption to support Python 3.13 is_abs in the C compiler (#4669).