From 007c33000ba1072ff41022ef2d8b2526571c09d0 Mon Sep 17 00:00:00 2001 From: memento Date: Thu, 25 Jan 2024 15:37:07 -0600 Subject: [PATCH] (dbg) got lost, refactored a bit, this is probably going to fails badly :'(? --- ...{build_shared.py => build_from_cmdline.py} | 0 setup.py | 322 +++--------------- setup_build_extension.py | 73 ++++ setup_build_secp256k1_with_make.py | 125 +++++++ setup_support.py | 52 +++ 5 files changed, 291 insertions(+), 281 deletions(-) rename _cffi_build/{build_shared.py => build_from_cmdline.py} (100%) create mode 100644 setup_build_extension.py create mode 100644 setup_build_secp256k1_with_make.py diff --git a/_cffi_build/build_shared.py b/_cffi_build/build_from_cmdline.py similarity index 100% rename from _cffi_build/build_shared.py rename to _cffi_build/build_from_cmdline.py diff --git a/setup.py b/setup.py index 682add5f0..d28c96dfa 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,28 @@ -import errno -import os import os.path -import pathlib import platform import shutil import subprocess -import tarfile -from io import BytesIO import sys from setuptools import Distribution as _Distribution, setup, find_packages, __version__ as setuptools_version from setuptools._distutils import log from setuptools._distutils.errors import DistutilsError -from setuptools.command.build_clib import build_clib as _build_clib -from setuptools.command.build_ext import build_ext as _build_ext from setuptools.extension import Extension from setuptools.command.develop import develop as _develop from setuptools.command.dist_info import dist_info as _dist_info from setuptools.command.egg_info import egg_info as _egg_info from setuptools.command.sdist import sdist as _sdist +from setup_build_extension import BuildCFFIForSharedLib, BuildCFFISetuptools, BuildCFFIForStaticLib +from setup_build_secp256k1_with_make import BuildClibWithMake + try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel except ImportError: _bdist_wheel = None sys.path.append(os.path.abspath(os.path.dirname(__file__))) -from setup_support import absolute, build_flags, detect_dll, has_system_lib # noqa: E402 +from setup_support import detect_dll, has_system_lib, download_library # noqa: E402 BUILDING_FOR_WINDOWS = detect_dll() @@ -48,45 +44,6 @@ ) -def download_library(command, libdir=LIB_NAME, force=False): - if command.dry_run: - return - - if force: - shutil.rmtree(libdir, ignore_errors=True) - - if os.path.exists(os.path.join(libdir, 'autogen.sh')): - # Library already downloaded - return - - # Ensure the path exists - os.makedirs(libdir, exist_ok=True) - - # _download will use shutil.move, thus remove the directory - os.rmdir(libdir) - - command.announce(f'Downloading {LIB_NAME} source code', level=log.INFO) - from requests.exceptions import RequestException - try: - _download_library(libdir) - except RequestException as e: - raise SystemExit(f'Unable to download {LIB_NAME} library: {e!s}', ) from e - - -def _download_library(libdir): - import requests - r = requests.get(LIB_TARBALL_URL, stream=True, timeout=10) - status_code = r.status_code - if status_code != 200: - raise SystemExit(f'Unable to download {LIB_NAME} library: HTTP-Status: {status_code}') - content = BytesIO(r.raw.read()) - content.seek(0) - with tarfile.open(fileobj=content) as tf: - dirname = tf.getnames()[0].partition('/')[0] - tf.extractall() - shutil.move(dirname, libdir) - - class egg_info(_egg_info): def run(self): # Ensure library has been downloaded (sdist might have been skipped) @@ -120,139 +77,10 @@ def run(self): download_library(self) _bdist_wheel.run(self) - else: bdist_wheel = None -class build_clib(_build_clib): - def initialize_options(self): - _build_clib.initialize_options(self) - self.build_flags = None - - def finalize_options(self): - _build_clib.finalize_options(self) - if self.build_flags is None: - self.build_flags = {'include_dirs': [], 'library_dirs': [], 'define': []} - - def get_source_files(self): - # Ensure library has been downloaded (sdist might have been skipped) - if not has_system_lib(): - download_library(self) - - return [ - filename - for root, _, filenames in os.walk(absolute('libsecp256k1')) - for filename in filenames - ] - - def build_libraries(self, libraries): - raise Exception('build_libraries') - - def check_library_list(self, libraries): - raise Exception('check_library_list') - - def get_library_names(self): - return build_flags(LIB_NAME, 'l', os.path.join(os.path.abspath(self.build_clib), 'lib', 'pkgconfig')) - - def run(self): - cwd = pathlib.Path().absolute() - - log.info('SECP256K1 build options:') - if has_system_lib(): - log.info('Using system library') - return - - # build_temp = os.path.abspath(self.build_temp) - build_external_library = os.path.join(cwd, 'build_external_library') - built_lib_dir = os.path.join(build_external_library, LIB_NAME) - installed_lib_dir = os.path.abspath(self.build_clib) - - try: - os.makedirs(build_external_library) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - download_library(self, libdir=built_lib_dir) - - autoreconf = 'autoreconf -if --warnings=all' - bash = shutil.which('bash') - subprocess.check_call([bash, '-c', autoreconf], cwd=built_lib_dir) # noqa S603 - - for filename in [ - os.path.join(built_lib_dir, 'configure'), - os.path.join(built_lib_dir, 'build-aux', 'compile'), - os.path.join(built_lib_dir, 'build-aux', 'config.guess'), - os.path.join(built_lib_dir, 'build-aux', 'config.sub'), - os.path.join(built_lib_dir, 'build-aux', 'depcomp'), - os.path.join(built_lib_dir, 'build-aux', 'install-sh'), - os.path.join(built_lib_dir, 'build-aux', 'missing'), - os.path.join(built_lib_dir, 'build-aux', 'test-driver'), - ]: - try: - os.chmod(filename, 0o700) - except OSError as e: - # some of these files might not exist depending on autoconf version - if e.errno != errno.ENOENT: - # If the error isn't 'No such file or directory' something - # else is wrong and we want to know about it - raise - - cmd = [ - 'configure', - '--disable-shared', - '--enable-static', - '--disable-dependency-tracking', - '--with-pic', - '--enable-module-extrakeys', - '--enable-module-recovery', - '--enable-module-schnorrsig', - '--prefix', - installed_lib_dir.replace('\\', '/'), - '--enable-experimental', - '--enable-module-ecdh', - '--enable-benchmark=no', - '--enable-tests=no', - '--enable-exhaustive-tests=no', - ] - if 'COINCURVE_CROSS_HOST' in os.environ: - cmd.append(f"--host={os.environ['COINCURVE_CROSS_HOST']}") - - log.debug(f"Running configure: {' '.join(cmd)}") - # Prepend the working directory to the PATH - os.environ['PATH'] = built_lib_dir + os.pathsep + os.environ['PATH'] - subprocess.check_call([bash, '-c', ' '.join(cmd)], cwd=built_lib_dir) # noqa S603 - - subprocess.check_call([MAKE], cwd=built_lib_dir) # noqa S603 - subprocess.check_call([MAKE, 'install'], cwd=built_lib_dir) # noqa S603 - - self.build_flags['include_dirs'].extend(build_flags(LIB_NAME, - 'I', - os.path.join(installed_lib_dir, 'lib', 'pkgconfig'))) - self.build_flags['library_dirs'].extend(build_flags(LIB_NAME, - 'L', - os.path.join(installed_lib_dir, 'lib', 'pkgconfig'))) - if not has_system_lib(): - self.build_flags['define'].append(('CFFI_ENABLE_RECOVERY', None)) - self.announce('build_clib Done', level=log.INFO) - - -class build_ext(_build_ext): - def run(self): - if self.distribution.has_c_libraries(): - _build_clib = self.get_finalized_command('build_clib') - self.include_dirs.append(os.path.join(_build_clib.build_clib, 'include')) - self.include_dirs.extend(_build_clib.build_flags['include_dirs']) - - self.library_dirs.insert(0, os.path.join(_build_clib.build_clib, 'lib')) - self.library_dirs.extend(_build_clib.build_flags['library_dirs']) - - self.define = _build_clib.build_flags['define'] - - return _build_ext.run(self) - - class develop(_develop): def run(self): if not has_system_lib(): @@ -263,56 +91,46 @@ def run(self): _develop.run(self) -package_data = {'coincurve': ['py.typed']} +class Distribution(_Distribution): + def has_c_libraries(self): + return not has_system_lib() -class BuildCFFIForSharedLib(_build_ext): - def build_extensions(self): - build_script = os.path.join('_cffi_build', 'build_shared.py') - c_file = self.extensions[0].sources[0] - subprocess.run([sys.executable, build_script, c_file, '0'], shell=False, check=True) # noqa S603 - super().build_extensions() +pkgconfig = shutil.which('pkg-config') +if pkgconfig is None: + raise DistutilsError('pkg-config is required') +package_data = {'coincurve': ['py.typed']} -log.info(f'Sys Libs: {has_system_lib()}') -if has_system_lib(): +extension = Extension( + name='coincurve._libsecp256k1', + sources=[os.path.join('coincurve', '_libsecp256k1.c')], + # ABI?: py_limited_api=True, +) - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() +# Cases to consider: +# . Building for any OS, use system libsecp256k1 +# . Building for Windows Native, build secp256k1 locally +# . Building for Windows with cross-compile, build secp256k1 locally +# . Building for other OS, build secp256k1 locally - # --- SECP256K1 package definitions --- - secp256k1_package = 'libsecp256k1' +# Building for any OS, use system libsecp256k1 - extension = Extension( - name='coincurve._libsecp256k1', - sources=[os.path.join('coincurve', '_libsecp256k1.c')], - # ABI?: py_limited_api=True, - ) +if has_system_lib(): + log.info('Using system library') extension.extra_compile_args = [ - subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 + subprocess.check_output([pkgconfig, '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 ] extension.extra_link_args = [ - subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 + subprocess.check_output([pkgconfig, '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 + subprocess.check_output([pkgconfig, '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 ] - if os.name == 'nt' or sys.platform == 'win32': - # Apparently, the linker on Windows interprets -lxxx as xxx.lib, not libxxx.lib - for i, v in enumerate(extension.__dict__.get('extra_link_args')): - extension.__dict__['extra_link_args'][i] = v.replace('-L', '/LIBPATH:') - - if v.startswith('-l'): - v = v.replace('-l', 'lib') - extension.__dict__['extra_link_args'][i] = f'{v}.lib' - setup_kwargs = dict( - setup_requires=['cffi>=1.3.0', 'requests'], ext_modules=[extension], cmdclass={ - 'build_clib': build_clib, - 'build_ext': build_ext, + 'build_ext': BuildCFFIForStaticLib, 'develop': develop, 'egg_info': egg_info, 'sdist': sdist, @@ -321,83 +139,25 @@ def has_c_libraries(self): ) else: - log.info(f'Windows: {BUILDING_FOR_WINDOWS}') - if BUILDING_FOR_WINDOWS: - - class Distribution(_Distribution): - def is_pure(self): - return False - + setup_kwargs = dict( + ext_modules=[extension], + cmdclass={ + 'build_clib': BuildClibWithMake, + 'build_ext': BuildCFFIForStaticLib, + 'develop': develop, + 'egg_info': egg_info, + 'sdist': sdist, + 'bdist_wheel': bdist_wheel, + }, + ) + if BUILDING_FOR_WINDOWS: package_data['coincurve'].append('libsecp256k1.dll') - log.info(f'Building for Windows: {os.name}:{sys.platform}') - if os.name == 'nt' or sys.platform == 'win32': - # Native build on Windows - log.info('Building for Windows') - extension = Extension( - name='coincurve._libsecp256k1', - sources=[os.path.join('coincurve', '_libsecp256k1.c')], - # ABI?: py_limited_api=True, - ) - - extension.extra_compile_args = [ - subprocess.check_output(['pkg-config', '--cflags-only-I', 'libsecp256k1']).strip().decode('utf-8') # noqa S603 - ] - extension.extra_link_args = [ - subprocess.check_output(['pkg-config', '--libs-only-L', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - subprocess.check_output(['pkg-config', '--libs-only-l', 'libsecp256k1']).strip().decode('utf-8'), # noqa S603 - ] - - log.info(f'links: {extension.extra_link_args}') - # Apparently, the linker on Windows interprets -lxxx as xxx.lib, not libxxx.lib - for i, v in enumerate(extension.__dict__.get('extra_link_args')): - extension.__dict__['extra_link_args'][i] = v.replace('-L', '/LIBPATH:') - - # We may not need this. SECP256K1 builds with gcc -> libsecp256k1.a - # if v.startswith('-l'): - # v = v.replace('-l', 'lib') - # extension.__dict__['extra_link_args'][i] = f'{v}.lib' - - setup_kwargs = dict( - ext_modules=[extension], - cmdclass={ - 'build_clib': build_clib, - 'build_ext': BuildCFFIForSharedLib, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) - else: - setup_kwargs = {} - - else: - - class Distribution(_Distribution): - def has_c_libraries(self): - return not has_system_lib() - - - log.info(f'Building for what: {BUILDING_FOR_WINDOWS}:{os.name}:{sys.platform}') - setup_kwargs = dict( - setup_requires=['cffi>=1.3.0', 'requests'], - ext_package='coincurve', - cffi_modules=['_cffi_build/build.py:ffi'], - cmdclass={ - 'build_clib': build_clib, - 'build_ext': build_ext, - 'develop': develop, - 'egg_info': egg_info, - 'sdist': sdist, - 'bdist_wheel': bdist_wheel, - }, - ) - setup( name='coincurve', version='19.0.0', - install_requires=['asn1crypto', 'cffi>=1.3.0'], + requires=['asn1crypto', 'cffi(>=1.3.0)'], packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', LIB_NAME, 'tests')), package_data=package_data, diff --git a/setup_build_extension.py b/setup_build_extension.py new file mode 100644 index 000000000..35e5eb658 --- /dev/null +++ b/setup_build_extension.py @@ -0,0 +1,73 @@ +import os +import subprocess +import sys +from typing import TYPE_CHECKING, cast + +from setuptools._distutils import log +from setuptools.command.build_ext import build_ext as _build_ext + +if TYPE_CHECKING: + from setup_build_secp256k1_with_make import BuildClibWithMake + + +def _update_extensions_for_msvc(extension, compiler): + if compiler == 'gcc': + return + + path_to_lib = '' + for i, v in enumerate(extension.__dict__.get('extra_link_args')): + # Replace -L with /LIBPATH: for MSVC + if v.startswith('-L'): + path_to_lib = v[2:] + extension.__dict__['extra_link_args'][i] = f'/LIBPATH:{path_to_lib}' + + # Replace -l with the library filename for MSVC + if v.startswith('-l'): + v = v.replace('-l', 'lib') + + if os.path.isfile(path_to_lib + v + '.lib'): + extension.__dict__['extra_link_args'][i] = f'{v}.lib' + + if os.path.isfile(path_to_lib + v + '.a'): + extension.__dict__['extra_link_args'][i] = f'{v}.a' + + +class BuildCFFIForSharedLib(_build_ext): + def build_extensions(self): + log.info(f'Cmdline CFFI Shared for: {os.name}:{sys.platform}:{(self.compiler.compiler[0])}') + build_script = os.path.join('_cffi_build', 'build_from_cmdline.py') + + _update_extensions_for_msvc(self.extensions[0], self.compiler.compiler[0]) + c_file = self.extensions[0].sources[0] + + subprocess.run([sys.executable, build_script, c_file, '0'], shell=False, check=True) # noqa S603 + super().build_extensions() + + +class BuildCFFIForStaticLib(_build_ext): + def build_extensions(self): + log.info(f'Cmdline CFFI Static for: {os.name}:{sys.platform}:{(self.compiler.compiler[0])}') + build_script = os.path.join('_cffi_build', 'build.py') + + _update_extensions_for_msvc(self.extensions[0], self.compiler.compiler[0]) + c_file = self.extensions[0].sources[0] + + subprocess.run([sys.executable, build_script, c_file, '1'], shell=False, check=True) # noqa S603 + super().build_extensions() + + +class BuildCFFISetuptools(_build_ext): + def run(self): + log.info(f'Setuptools CFFI static for: {os.name}:{sys.platform}:{(self.compiler.compiler[0])}') + if self.distribution.has_c_libraries(): + log.info(' Locally built C-lib') + _build_clib: BuildClibWithMake = cast(self.get_finalized_command('build_clib'), BuildClibWithMake) + self.include_dirs.append(os.path.join(_build_clib.build_clib, 'include')) + self.include_dirs.extend(_build_clib.build_flags['include_dirs']) + + self.library_dirs.insert(0, os.path.join(_build_clib.build_clib, 'lib')) + self.library_dirs.extend(_build_clib.build_flags['library_dirs']) + + self.define = _build_clib.build_flags['define'] + + return _build_ext.run(self) diff --git a/setup_build_secp256k1_with_make.py b/setup_build_secp256k1_with_make.py new file mode 100644 index 000000000..d330de30c --- /dev/null +++ b/setup_build_secp256k1_with_make.py @@ -0,0 +1,125 @@ +import errno +import os +import pathlib +import shutil + +from setuptools._distutils import log +from setuptools.command.build_clib import build_clib as _build_clib + +from setup_support import absolute, build_flags, download_library, has_system_lib + + +class BuildClibWithMake(_build_clib): + def __init__(self, dist): + super().__init__(dist) + self.build_flags = None + + def initialize_options(self: _build_clib): + _build_clib.initialize_options(self) + self.build_flags = None + + def finalize_options(self: _build_clib): + _build_clib.finalize_options(self) + if self.build_flags is None: + self.build_flags = {'include_dirs': [], 'library_dirs': [], 'define': []} + + def get_source_files(self): + from setup import LIB_NAME + # Ensure library has been downloaded (sdist might have been skipped) + if not has_system_lib(): + download_library(self) + + return [f for root, _, fns in os.walk(absolute(LIB_NAME)) for f in fns] + + def build_libraries(self, libraries): + raise NotImplementedError('build_libraries') + + def check_library_list(self, libraries): + raise NotImplementedError('check_library_list') + + def get_library_names(self): + from setup import LIB_NAME + return build_flags(LIB_NAME, 'l', os.path.join(os.path.abspath(self.build_clib), 'lib', 'pkgconfig')) + + def run(self): + from setup import LIB_NAME + cwd = pathlib.Path().absolute() + + log.info('SECP256K1 build options:') + if has_system_lib(): + log.info('Using system library') + return + + # build_temp = os.path.abspath(self.build_temp) + build_external_library = os.path.join(cwd, 'build_external_library') + built_lib_dir = os.path.join(build_external_library, LIB_NAME) + installed_lib_dir = os.path.abspath(self.build_clib) + + try: + os.makedirs(build_external_library) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + download_library(self, libdir=built_lib_dir) + + autoreconf = 'autoreconf -if --warnings=all' + bash = shutil.which('bash') + subprocess.check_call([bash, '-c', autoreconf], cwd=built_lib_dir) # noqa S603 + + for filename in [ + os.path.join(built_lib_dir, 'configure'), + os.path.join(built_lib_dir, 'build-aux', 'compile'), + os.path.join(built_lib_dir, 'build-aux', 'config.guess'), + os.path.join(built_lib_dir, 'build-aux', 'config.sub'), + os.path.join(built_lib_dir, 'build-aux', 'depcomp'), + os.path.join(built_lib_dir, 'build-aux', 'install-sh'), + os.path.join(built_lib_dir, 'build-aux', 'missing'), + os.path.join(built_lib_dir, 'build-aux', 'test-driver'), + ]: + try: + os.chmod(filename, 0o700) + except OSError as e: + # some of these files might not exist depending on autoconf version + if e.errno != errno.ENOENT: + # If the error isn't 'No such file or directory' something + # else is wrong, and we want to know about it + raise + + cmd = [ + 'configure', + '--disable-shared', + '--enable-static', + '--disable-dependency-tracking', + '--with-pic', + '--enable-module-extrakeys', + '--enable-module-recovery', + '--enable-module-schnorrsig', + '--prefix', + installed_lib_dir.replace('\\', '/'), + '--enable-experimental', + '--enable-module-ecdh', + '--enable-benchmark=no', + '--enable-tests=no', + '--enable-exhaustive-tests=no', + ] + if 'COINCURVE_CROSS_HOST' in os.environ: + cmd.append(f"--host={os.environ['COINCURVE_CROSS_HOST']}") + + log.debug(f"Running configure: {' '.join(cmd)}") + # Prepend the working directory to the PATH + os.environ['PATH'] = built_lib_dir + os.pathsep + os.environ['PATH'] + subprocess.check_call([bash, '-c', ' '.join(cmd)], cwd=built_lib_dir) # noqa S603 + + subprocess.check_call([MAKE], cwd=built_lib_dir) # noqa S603 + subprocess.check_call([MAKE, 'install'], cwd=built_lib_dir) # noqa S603 + + self.build_flags['include_dirs'].extend( + build_flags(LIB_NAME, 'I', os.path.join(installed_lib_dir, 'lib', 'pkgconfig')) + ) + self.build_flags['library_dirs'].extend( + build_flags(LIB_NAME, 'L', os.path.join(installed_lib_dir, 'lib', 'pkgconfig')) + ) + if not has_system_lib(): + self.build_flags['define'].append(('CFFI_ENABLE_RECOVERY', None)) + self.announce('build_clib Done', level=log.INFO) diff --git a/setup_support.py b/setup_support.py index b487abac6..0a98290c3 100644 --- a/setup_support.py +++ b/setup_support.py @@ -2,7 +2,9 @@ import os import shutil import subprocess +import tarfile from contextlib import contextmanager, suppress +from io import BytesIO from tempfile import mkdtemp from setuptools._distutils import log @@ -104,3 +106,53 @@ def detect_dll(): if fn.endswith('.dll'): return True return False + + +def download_library(command, libdir=None, force=False): + if libdir is None: + from setup import LIB_NAME, LIB_TARBALL_URL + + libdir = LIB_NAME + + if command.dry_run: + return + + if force: + shutil.rmtree(libdir, ignore_errors=True) + + if os.path.exists(os.path.join(libdir, 'autogen.sh')): + # Library already downloaded + return + + # Ensure the path exists + os.makedirs(libdir, exist_ok=True) + + # _download will use shutil.move, thus remove the directory + os.rmdir(libdir) + + command.announce(f'Downloading {libdir} source code', level=log.INFO) + + from requests.exceptions import RequestException + + try: + _download_library(libdir) + except RequestException as e: + raise SystemExit( + f'Unable to download {libdir} library: {e!s}', + ) from e + + +def _download_library(libdir): + import requests + from setup import LIB_TARBALL_URL + + r = requests.get(LIB_TARBALL_URL, stream=True, timeout=10) + status_code = r.status_code + if status_code != 200: + raise SystemExit(f'Unable to download {libdir} library: HTTP-Status: {status_code}') + content = BytesIO(r.raw.read()) + content.seek(0) + with tarfile.open(fileobj=content) as tf: + dirname = tf.getnames()[0].partition('/')[0] + tf.extractall() + shutil.move(dirname, libdir)