Build python wheels via scikit-build-core

This converts the setuptools configuration for building python wheels to use scikit-build-core, which has better support for CMake. There is no more setup.py; the configuration is entirely in `pyproject.toml` and the compile/link is done exclusively via cmake. The build/publish policy is: * A PR that changes any of the python wheel source/configuration (src/wrappers/python/* or .github/workflows/python-wheels.yml) triggers a build as a check. * PRs that change other library source do *not* trigger a build of the python wheels. Note that the primary CI workflow does build and test the bindings, although only for a single python version on a single arch for Linux/macOS/Windows. The wheel building validates multiple python versions and architectures, but involves signifant computation/time. Currently, the python wheels are a thin wrapper about basic read/write functions that don't add significant additional functionality to the library. Any potential problem will almost certainly get caught by the primary CI. * A tag of the form `v3.[0-9]+.[0-9]+-rc*` (e.g. `v3.2.4-rc`) triggers a full build of the wheels followed by a publish to `test.pypi.org`. This validates release candidates. * Publishing a release triggers a full build of the wheels followed by a publish to `pypi.org`. Signed-off-by: Cary Phillips <[email protected]>
cary-ilm · Feb 8, 2024 · 8f9cb39 · 8f9cb39
1 parent 8743203
commit 8f9cb39
Show file tree

Hide file tree

Showing 17 changed files with 482 additions and 208 deletions.
diff --git a/.github/workflows/python-wheels-publish-test.yml b/.github/workflows/python-wheels-publish-test.yml
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) Contributors to the OpenEXR Project.
+
+name: Publish python distribution 📦 to TestPyPI
+
+on:
+
+  # Publish python wheels to test.pypi when a release candidate is tagged, 
+  # e.g. v3.4.5-rc, v3.4.5-rc6, etc.
+
+  push:
+    tags:
+      - v3.[0-9]+.[0-9]+-rc*
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Python Wheels - ${{ matrix.os }} 
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/openexr
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Python 
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Build wheel
+        uses: pypa/[email protected]
+        with:
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_LINUX: x86_64 
+          CIBW_ARCHS_MACOS: x86_64 arm64 universal2
+          # Skip python 3.6 since scikit-build-core requires 3.7+
+          # Skip 32-bit wheels builds on Windows
+          # Also skip the PyPy builds, since they fail the unit tests
+          CIBW_SKIP: cp36-* *-win32 *_i686 pp* 
+          CIBW_TEST_SKIP: "*-macosx_universal2:arm64"
+          CIBW_ENVIRONMENT: OPENEXR_RELEASE_CANDIDATE_TAG="${{ github.ref_name }}"
+
+      - name: Upload artifact
+        uses: actions/[email protected]
+        with:
+          name: wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/openexr
+
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download Linux artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-ubuntu-latest
+        path: dist
+    - name: Download macOS artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-macos-latest
+        path: dist
+    - name: Download Windows artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-windows-latest
+        path: dist
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
diff --git a/.github/workflows/python-wheels-publish.yml b/.github/workflows/python-wheels-publish.yml
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) Contributors to the OpenEXR Project.
+
+name: Publish python distribution 📦 to PyPI
+
+on:
+  # Publish wheels to pypi on release
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Python Wheels - ${{ matrix.os }} 
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/openexr
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Python 
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Build wheel
+        uses: pypa/[email protected]
+        with:
+          output-dir: wheelhouse
+        env:
+          CIBW_BUILD: cp312-*
+          CIBW_ARCHS_LINUX: x86_64 
+          CIBW_ARCHS_MACOS: x86_64 arm64 universal2
+          # Skip python 3.6 since scikit-build-core requires 3.7+
+          # Skip 32-bit wheels builds on Windows
+          # Also skip the PyPy builds, since they fail the unit tests
+          CIBW_SKIP: cp36-* *-win32 *_i686 pp* 
+          CIBW_TEST_SKIP: "*arm64"
+
+      - name: Upload artifact
+        uses: actions/[email protected]
+        with:
+          name: wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
+  publish-to-pypi:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/openexr
+
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download Linux artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-ubuntu-latest
+        path: dist
+    - name: Download macOS artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-macos-latest
+        path: dist
+    - name: Download Windows artifacts
+      uses: actions/[email protected]
+      with:
+        name: wheels-windows-latest
+        path: dist
+    - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml
@@ -1,91 +1,67 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) Contributors to the OpenEXR Project.
-#
 
-name: Python wheels
+name: Python Wheels
 
 on:
+
+  # Run on all changes (PR and push) to the python binding
+  # source/configuration files, except on the release branches, which
+  # have their own workflow, which also publish to pypi/test.pypi.
+  # Note that changes to the core libraries will *not*
+  # trigger building the wheels. However, the main ci workflow does
+  # build and test the bindings (for a single python version on a
+  # single arch)
+
   push:
     branches-ignore:
-      - RB-2.*
-    tags-ignore:
-      - v1.*
-      - v2.*
+      - RB-*
     paths:
-      - '**'
-      - '!**.md'
-      - '!website/**'
-      - 'website/src/**'
-      - '!bazel/**'
+      - 'src/wrappers/python/**'
+      - 'pyproject.toml'
+      - '.github/workflows/python-wheels.yml'
   pull_request:
     branches-ignore:
-      - RB-2.*
-    tags-ignore:
-      - v1.*
-      - v2.*
+      - RB-*
     paths:
-      - '**'
-      - '!**.md'
-      - '!website/**'
-      - 'website/src/**'
-      - '!bazel/**'
+      - 'src/wrappers/python/**'
+      - 'pyproject.toml'
+      - '.github/workflows/python-wheels.yml'
 
 permissions:
   contents: read
 
 jobs:
   build_wheels:
-    name: Build Python wheels
+    name: Python Wheels - ${{ matrix.os }} 
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, windows-latest, macOS-latest]
-    env:
-      # On macOS we build both x86 and arm to support Intel and Apple Silicon.
-      CIBW_ARCHS_MACOS: x86_64 arm64 
-      # Skip 32-bit wheels builds on Windows.
-      # Also skip the PyPy builds, since they fail the unittests
-      CIBW_SKIP: "*-win32 *_i686 pp*"
-      # The CI platform is Intel based so we are doing cross compilation
-      # for arm64. It is not currently possible to test arm64 when cross
-      # compiling. 
-      CIBW_TEST_SKIP: "*_arm64"
-      CIBW_BEFORE_BUILD: >
-        echo "Installing OpenEXR..." &&
-        cd openexr.build &&
-        cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../openexr.install -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_PREFIX_PATH=../openexr.install -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_TESTING=OFF -DOPENEXR_INSTALL_EXAMPLES=OFF -DOPENEXR_BUILD_TOOLS=OFF -DBUILD_SHARED_LIBS=OFF -DOPENEXR_FORCE_INTERNAL_DEFLATE=ON -DOPENEXR_FORCE_INTERNAL_IMATH=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON ../ &&
-        cmake --build ./ --config Release --clean-first &&
-        cmake --install ./ --config Release &&
-        cd ..
-      CIBW_TEST_REQUIRES: pytest
-      CIBW_TEST_COMMAND: pytest {project}/src/wrappers/python/tests/
+        os: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
-      - uses: actions/checkout@v3
-
-      # Used to host cibuildwheel
-      - uses: actions/setup-python@v4
-        with:
-            python-version: '3.x'
 
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.16.2
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - name: Create setup.py
-        run: |
-            mv ${{github.workspace}}/src/wrappers/python/setup.py ${{github.workspace}}/setup.py
-            mv ${{github.workspace}}/src/wrappers/python/Imath.py ${{github.workspace}}/Imath.py
-            mv ${{github.workspace}}/src/wrappers/python/OpenEXR.cpp ${{github.workspace}}/OpenEXR.cpp
-
-      - name: Create folders
-        run: |
-            mkdir -p ${{github.workspace}}/openexr.build
-            mkdir -p ${{github.workspace}}/openexr.install
+      - name: Install Python 
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
 
-      - name: Build wheels
-        run: python -m cibuildwheel --output-dir wheelhouse
+      - name: Build wheel
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS_MACOS: x86_64 arm64 universal2
+          # Skip python 3.6 since scikit-build-core requires 3.7+
+          # Skip 32-bit wheels builds on Windows
+          # Also skip the PyPy builds, since they fail the unit tests
+          CIBW_SKIP: cp36-* *-win32 *_i686 pp* 
+          CIBW_TEST_SKIP: "*-macosx*arm64"
 
-      - uses: actions/upload-artifact@v3
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
         with:
-          name: "Python wheels"
-          path: ./wheelhouse/*.whl
+          name: wheel-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
diff --git a/.github/workflows/website_workflow.yml b/.github/workflows/website_workflow.yml
@@ -14,13 +14,17 @@ on:
     branches:-ignore:
       - RB-2.*
       - RB-3.*
+    tags-ignore:
+      - v3.[0-9]+.[0-9]+-rc*
     paths:
       - 'website/**'
 
   pull_request:
     branches:-ignore:
       - RB-2.*
       - RB-3.*
+    tags-ignore:
+      - v3.[0-9]+.[0-9]+-rc*
     paths:
       - 'website/**'
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -46,12 +46,6 @@ set(OPENEXR_VERSION_API "${OpenEXR_VERSION_MAJOR}_${OpenEXR_VERSION_MINOR}")
 set(OPENEXR_LIB_SOVERSION 31)
 set(OPENEXR_LIB_VERSION "${OPENEXR_LIB_SOVERSION}.${OPENEXR_VERSION}") # e.g. "31.3.2.0"
 
-option(OPENEXR_INSTALL "Install OpenEXR libraries" ON)
-option(OPENEXR_INSTALL_TOOLS "Install OpenEXR tools" ON)
-if(OPENEXR_INSTALL_TOOLS AND NOT OPENEXR_INSTALL)
-  message(SEND_ERROR "OPENEXR_INSTALL_TOOLS requires OPENEXR_INSTALL")
-endif()
-
 include(cmake/LibraryDefine.cmake)
 include(cmake/OpenEXRSetup.cmake)
 add_subdirectory(cmake)
@@ -73,8 +67,13 @@ if(BUILD_TESTING AND NOT OPENEXR_IS_SUBPROJECT)
 endif()
 
 # Include these two modules without enable/disable options
-add_subdirectory(src/lib)
-add_subdirectory(src/bin)
+if (OPENEXR_BUILD_LIBS)
+  add_subdirectory(src/lib)
+endif()
+
+if(OPENEXR_BUILD_TOOLS AND OPENEXR_BUILD_LIBS)
+  add_subdirectory(src/bin)
+endif()
 
 # Tell CMake where to find the OpenEXRConfig.cmake file. Makes it possible to call 
 # find_package(OpenEXR) in downstream projects
@@ -83,8 +82,7 @@ set(OpenEXR_DIR "${CMAKE_CURRENT_BINARY_DIR}/cmake" CACHE PATH "" FORCE)
 # Can be empty since we already defined the targets in add_subdirectory
 file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/cmake/OpenEXRTargets.cmake" "# Dummy file")
 
-option(OPENEXR_INSTALL_EXAMPLES "Install OpenEXR examples" ON)
-if(OPENEXR_INSTALL_EXAMPLES)
+if(OPENEXR_BUILD_EXAMPLES AND OPENEXR_BUILD_LIBS)
   add_subdirectory( src/examples )
 endif()
 
@@ -112,7 +110,7 @@ endif()
 #set(CTEST_DROP_SITE_CDASH TRUE)
 include(CTest)
 
-if(BUILD_TESTING AND NOT OPENEXR_IS_SUBPROJECT)
+if(BUILD_TESTING AND OPENEXR_BUILD_LIBS AND NOT OPENEXR_IS_SUBPROJECT)
   add_subdirectory(src/test)
 endif()
 
@@ -132,12 +130,11 @@ if (BUILD_WEBSITE AND NOT OPENEXR_IS_SUBPROJECT)
   add_subdirectory(website)
 endif()
 
-if (NOT OPENEXR_IS_SUBPROJECT)
+if (OPENEXR_BUILD_LIBS AND NOT OPENEXR_IS_SUBPROJECT)
   # Even if not building the website, still make sure the website example code compiles.
   add_subdirectory(website/src)
 endif()
 
-option(OPENEXR_BUILD_PYTHON "Set ON to build python bindings")
-if (OPENEXR_BUILD_PYTHON AND NOT OPENEXR_IS_SUBPROJECT)
+if (OPENEXR_BUILD_PYTHON AND OPENEXR_BUILD_LIBS AND NOT OPENEXR_IS_SUBPROJECT)
   add_subdirectory(src/wrappers/python)
 endif()