From 4b5c09ad2759ca0c20b99422f4a3fdfd727eef8a Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Wed, 22 Jan 2025 09:48:43 +0100 Subject: [PATCH] tools/docker: rework python wheel generation * use of muslinux_1_2 instead of alpine * use of manylinux_2_28 * Fix auditwheel repair --- tools/docker/Makefile | 99 ++++--- tools/docker/python/amd64/alpine.Dockerfile | 52 ---- .../docker/python/amd64/manylinux.Dockerfile | 20 +- .../docker/python/amd64/musllinux.Dockerfile | 46 +++ .../{arm64v8 => arm64}/manylinux.Dockerfile | 20 +- .../docker/python/arm64/musllinux.Dockerfile | 46 +++ tools/docker/python/arm64v8/alpine.Dockerfile | 49 ---- tools/docker/python/build-manylinux.sh | 24 +- tools/docker/python/build-musllinux.sh | 271 ++++++++++++++++++ 9 files changed, 463 insertions(+), 164 deletions(-) delete mode 100644 tools/docker/python/amd64/alpine.Dockerfile create mode 100644 tools/docker/python/amd64/musllinux.Dockerfile rename tools/docker/python/{arm64v8 => arm64}/manylinux.Dockerfile (81%) create mode 100644 tools/docker/python/arm64/musllinux.Dockerfile delete mode 100644 tools/docker/python/arm64v8/alpine.Dockerfile create mode 100755 tools/docker/python/build-musllinux.sh diff --git a/tools/docker/Makefile b/tools/docker/Makefile index 9385b1b33d..d0803291e6 100644 --- a/tools/docker/Makefile +++ b/tools/docker/Makefile @@ -33,25 +33,25 @@ help: @echo -e "\t${BOLD}test_archives${RESET}: Test each OR-Tools archives for all ${BOLD}${RESET} and ${BOLD}${RESET}." @echo @echo -e "${BOLD}PYTHON TARGETS${RESET}" - @echo -e "\t${BOLD}python${RESET}: Build alpine and manylinux2014 python 'ortools' wheel packages (3.8+)." + @echo -e "\t${BOLD}python${RESET}: Build musllinux and manylinux python 'ortools' wheel packages (3.8+)." @echo -e "\t${BOLD}python_${RESET}: Build all python 'ortools' wheel packages (3.8+) for a specific platform." @echo -e "\t${BOLD}python__${RESET}: Build all python 'ortools' wheel packages (3.8+) for a specific platform." @echo -e "\t${BOLD}python__${RESET}: Build python 'ortools' wheel packages (3.8+) for a specific target." @echo -e "\t${BOLD}save_python_${RESET}: Save python 'ortools' image." - @echo -e "\t${BOLD}clean_python_${RESET}: Clean manylinux2014 and alpine python 'ortools' wheel packages." + @echo -e "\t${BOLD}clean_python_${RESET}: Clean manylinux and musllinux python 'ortools' wheel packages." @echo -e "\t${BOLD}sh_python_${RESET}: Run a container using the python 'ortools' image." @echo @echo -e "\t${BOLD}${RESET}:" @echo -e "\t\t${BOLD}amd64${RESET}" - @echo -e "\t\t${BOLD}arm64v8${RESET}" + @echo -e "\t\t${BOLD}arm64${RESET}" @echo @echo -e "\t${BOLD}${RESET}:" @echo -e "\t\t${BOLD}_${RESET}" @echo -e "\t\t${BOLD}_manylinux_cp${RESET}" @echo @echo -e "\t${BOLD}${RESET}:" - @echo -e "\t\t${BOLD}alpine${RESET} (latest)" - @echo -e "\t\t${BOLD}manylinux${RESET} (manylinux2014)" + @echo -e "\t\t${BOLD}musllinux${RESET} (musllinux_1_2)" + @echo -e "\t\t${BOLD}manylinux${RESET} (manylinux_2_28)" @echo @echo -e "\t${BOLD}${RESET}:" @echo -e "\t\t${BOLD}38${RESET} Python3.8" @@ -68,7 +68,7 @@ help: @echo -e "\t\t${BOLD}test${RESET}" @echo -e "\t\t${BOLD}export${RESET}" @echo -e "\te.g. 'make python_amd64_manylinux_cp39_export'" - @echo -e "\te.g. 'make python_arm64v8_alpine_export'" + @echo -e "\te.g. 'make python_arm64_musllinux_export'" @echo @echo @echo -e "${BOLD}ARCHIVE TARGETS${RESET}" @@ -187,8 +187,8 @@ test_delivery: test_archives # $* stem # $< first prerequist # $@ target name -PYTHON_PLATFORMS := amd64 arm64v8 -PYTHON_DISTROS := manylinux alpine +PYTHON_PLATFORMS := amd64 arm64 +PYTHON_DISTROS := manylinux musllinux PYTHON_STAGES := env devel build test export: @@ -230,9 +230,8 @@ python_$1_manylinux_cp$2_$3: python/$1/manylinux.Dockerfile export/python/manyli .PHONY: save_python_$1_manylinux_cp$2_$3 save_python_$1_manylinux_cp$2_$3: cache/python/docker_$1_manylinux_cp$2_$3.tar -cache/python/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3 +cache/python/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3 | cache/python @rm -f $$@ - mkdir -p cache/python/ docker save ${IMAGE}:$$< -o $$@ .PHONY: clean_python_$1_manylinux_cp$2_$3 @@ -266,7 +265,7 @@ python_$1_manylinux_cp$2_export: python_$1_manylinux_cp$2_build endef $(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,amd64,${version}))) -$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64v8,${version}))) +$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64,${version}))) # Merge define manylinux_merge = @@ -281,41 +280,45 @@ clean_python_$1_manylinux_$2: $(addprefix clean_python_$1_manylinux_cp, $(addsuf endef $(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,amd64,${stage}))) -$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64v8,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64,${stage}))) -## ALPINE ## -export/python/alpine: | export/python +## MUSLLINUX ## +export/python/musllinux: | export/python -mkdir -p $@ -define alpine_loop = -#$$(info alpine_loop: PLATFORM:'$1' STAGE:'$2') +export/python/musllinux/build-musllinux.sh: python/build-musllinux.sh | export/python/musllinux + cp $< $@ + +define musllinux_inner = +#$$(info musllinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3') -.PHONY: python_$1_alpine_$2 -python_$1_alpine_$2: python/$1/alpine.Dockerfile | export/python/alpine +.PHONY: python_$1_musllinux_cp$2_$3 +python_$1_musllinux_cp$2_$3: python/$1/musllinux.Dockerfile | export/python/musllinux/build-musllinux.sh @docker image rm -f ${IMAGE}:$$@ 2>/dev/null - ${DOCKER_BUILD_CMD} \ + ${DOCKER_BUILDX_CMD} --platform linux/$1 \ --tag ${IMAGE}:$$@ \ --build-arg GIT_BRANCH=${OR_TOOLS_BRANCH} \ --build-arg GIT_SHA1=${OR_TOOLS_SHA1} \ --build-arg OR_TOOLS_PATCH=${OR_TOOLS_PATCH} \ - --target=$2 \ + --build-arg PYTHON_VERSION=$2 \ + --target=$3 \ -f $$< \ - export/python/alpine + export/python/musllinux -.PHONY: save_python_$1_alpine_$2 -save_python_$1_alpine_$2: cache/python/docker_$1_alpine_$2.tar -cache/python/docker_$1_alpine_$2.tar: python_$1_alpine_$2 | cache/python +.PHONY: save_python_$1_musllinux_cp$2_$3 +save_python_$1_musllinux_cp$2_$3: cache/python/docker_$1_musllinux_cp$2_$3.tar +cache/python/docker_$1_musllinux_cp$2_$3.tar: python_$1_musllinux_cp$2_$3 | cache/python @rm -f $$@ docker save ${IMAGE}:$$< -o $$@ -.PHONY: clean_python_$1_alpine_$2 -clean_python_$1_alpine_$2: python/$1/alpine.Dockerfile | export/python/alpine - docker image rm -f ${IMAGE}:python_$1_alpine_$2 2>/dev/null - rm -f cache/python/docker_$1_alpine_$2.tar +.PHONY: clean_python_$1_musllinux_cp$2_$3 +clean_python_$1_musllinux_cp$2_$3: python/$1/musllinux.Dockerfile | export/python/musllinux/build-musllinux.sh + docker image rm -f ${IMAGE}:python_$1_musllinux_cp$2_$3 2>/dev/null + rm -f cache/python/docker_$1_musllinux_cp$2_$3.tar # Debug purpose -.PHONY: sh_python_$1_alpine_$2 -sh_python_$1_alpine_$2: python_$1_alpine_$2 +.PHONY: sh_python_$1_musllinux_cp$2_$3 +sh_python_$1_musllinux_cp$2_$3: python_$1_musllinux_cp$2_$3 ${DOCKER_RUN_CMD} \ -v `pwd`/export:/export \ -it \ @@ -323,18 +326,38 @@ sh_python_$1_alpine_$2: python_$1_alpine_$2 ${IMAGE}:$$< endef -$(foreach stage,${PYTHON_STAGES},$(eval $(call alpine_loop,amd64,${stage}))) -$(foreach stage,${PYTHON_STAGES},$(eval $(call alpine_loop,arm64v8,${stage}))) +define musllinux_outer = +#$$(info musllinux_outer: PLATFORM: '$1' VERSION: '$2') -alpine_export_targets = $(addprefix python_, $(addsuffix _alpine_export, ${PYTHON_PLATFORMS})) -.PHONY: ${alpine_export_targets} -${alpine_export_targets}: python_%_alpine_export: python_%_alpine_build | export +$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call musllinux_inner,$1,$2,$${stage}))) + +.PHONY: python_$1_musllinux_cp$2_export +python_$1_musllinux_cp$2_export: python_$1_musllinux_cp$2_build ${DOCKER_RUN_CMD} \ -v `pwd`/export:/export \ -it \ - --name ortools_$< \ - ${IMAGE}:$< \ + --name ortools_$$< \ + ${IMAGE}:$$< \ "cp build*/python/dist/*.whl /export/python" +endef + +$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,amd64,${version}))) +$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,arm64,${version}))) + +# Merge +define musllinux_merge = +#$$(info musllinux_merge: PLATFORM:'$1' STAGE:'$2') + +.PHONY: python_$1_musllinux_$2 +python_$1_musllinux_$2: $(addprefix python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: save_python_$1_musllinux_$2 +save_python_$1_musllinux_$2: $(addprefix save_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: clean_python_$1_musllinux_$2 +clean_python_$1_musllinux_$2: $(addprefix clean_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +endef + +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,amd64,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,arm64,${stage}))) ## MERGE DISTRO ## define python_distro_merge = @@ -349,7 +372,7 @@ clean_python_$1_$2: $(addprefix clean_python_$1_, $(addsuffix _$2, ${PYTHON_DIST endef $(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,amd64,${stage}))) -$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64v8,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64,${stage}))) ## MERGE PLATFORM ## define clean_python_platform = diff --git a/tools/docker/python/amd64/alpine.Dockerfile b/tools/docker/python/amd64/alpine.Dockerfile deleted file mode 100644 index b1158dbb3a..0000000000 --- a/tools/docker/python/amd64/alpine.Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Create a virtual environment with all tools installed -# ref: https://hub.docker.com/_/alpine -FROM alpine:edge AS env - -# Install system build dependencies -ENV PATH=/usr/local/bin:$PATH -RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools -ENTRYPOINT ["/bin/sh", "-c"] -CMD ["/bin/sh"] - -# SWIG -RUN apk add --no-cache swig - -# Python -RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ - py3-numpy py3-pandas py3-matplotlib -RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ -&& python3 -m pip install absl-py mypy mypy-protobuf - -################ -## OR-TOOLS ## -################ -FROM env AS devel -ENV GIT_URL https://github.com/google/or-tools - -ARG GIT_BRANCH -ENV GIT_BRANCH ${GIT_BRANCH:-main} -ARG GIT_SHA1 -ENV GIT_SHA1 ${GIT_SHA1:-unknown} - -# Download sources -# use GIT_SHA1 to modify the command -# i.e. avoid docker reusing the cache when new commit is pushed -WORKDIR /root -RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ -&& cd /project \ -&& git reset --hard "${GIT_SHA1}" -WORKDIR /project - -# Build project -FROM devel AS build -RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DVENV_USE_SYSTEM_SITE_PACKAGES=ON \ - -DBUILD_CXX_SAMPLES=OFF -DBUILD_CXX_EXAMPLES=OFF -RUN cmake --build build -v -j8 -# Rename wheel package ortools-version+musl-.... -RUN cp build/python/dist/ortools-*.whl . -RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}" -RUN rm build/python/dist/ortools-*.whl -RUN mv *.whl build/python/dist/ - -FROM build AS test -RUN cmake --build build --target test diff --git a/tools/docker/python/amd64/manylinux.Dockerfile b/tools/docker/python/amd64/manylinux.Dockerfile index 2e52264e43..1933738a40 100644 --- a/tools/docker/python/amd64/manylinux.Dockerfile +++ b/tools/docker/python/amd64/manylinux.Dockerfile @@ -1,16 +1,17 @@ -FROM quay.io/pypa/manylinux2014_x86_64:latest AS env -# note: CMake 3.30.5 and SWIG 4.2.1 are already installed +FROM quay.io/pypa/manylinux_2_28_x86_64:latest AS env +# note: Almalinux:8 based image with +# CMake 3.31.2 and SWIG 4.3.0 already installed -RUN yum -y update \ -&& yum -y install \ +RUN dnf -y update \ +&& dnf -y install \ curl wget \ git patch \ which pkgconfig autoconf libtool \ make gcc-c++ \ redhat-lsb openssl-devel pcre2-devel \ zlib-devel unzip zip \ -&& yum clean all \ -&& rm -rf /var/cache/yum +&& dnf clean all \ +&& rm -rf /var/cache/dnf ENTRYPOINT ["/usr/bin/bash", "-c"] CMD ["/usr/bin/bash"] @@ -36,13 +37,14 @@ RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ && git reset --hard "${GIT_SHA1}" WORKDIR /project +# Copy build script and setup env +ENV PLATFORM x86_64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION ${PYTHON_VERSION:-3} COPY build-manylinux.sh . RUN chmod a+x "build-manylinux.sh" FROM devel AS build -ENV PLATFORM x86_64 -ARG PYTHON_VERSION -ENV PYTHON_VERSION ${PYTHON_VERSION:-3} RUN ./build-manylinux.sh build FROM build as test diff --git a/tools/docker/python/amd64/musllinux.Dockerfile b/tools/docker/python/amd64/musllinux.Dockerfile new file mode 100644 index 0000000000..4c2f64fdbf --- /dev/null +++ b/tools/docker/python/amd64/musllinux.Dockerfile @@ -0,0 +1,46 @@ +FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS env +# CMake 3.31.2 and SWIG 4.3.0 already installed + +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["/bin/sh"] + +## Python +#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ +# py3-numpy py3-pandas py3-matplotlib +#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ +#&& python3 -m pip install absl-py mypy mypy-protobuf + +################ +## OR-TOOLS ## +################ +FROM env AS devel +ENV GIT_URL https://github.com/google/or-tools + +ARG GIT_BRANCH +ENV GIT_BRANCH ${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1 ${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM x86_64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION ${PYTHON_VERSION:-3} +COPY build-musllinux.sh . +RUN chmod a+x "build-musllinux.sh" + +FROM devel AS build +RUN ./build-musllinux.sh build + +FROM build as test +RUN ./build-musllinux.sh test diff --git a/tools/docker/python/arm64v8/manylinux.Dockerfile b/tools/docker/python/arm64/manylinux.Dockerfile similarity index 81% rename from tools/docker/python/arm64v8/manylinux.Dockerfile rename to tools/docker/python/arm64/manylinux.Dockerfile index 6071834b4c..116fe68de2 100644 --- a/tools/docker/python/arm64v8/manylinux.Dockerfile +++ b/tools/docker/python/arm64/manylinux.Dockerfile @@ -1,18 +1,19 @@ # To build it on x86_64 please read # https://github.com/multiarch/qemu-user-static#getting-started -FROM quay.io/pypa/manylinux2014_aarch64:latest AS env -# note: CMake 3.30.5 and SWIG 4.2.1 are already installed +FROM quay.io/pypa/manylinux_2_28_aarch64:latest AS env +# note: Almalinux:8 based image with +# CMake 3.31.2 and SWIG 4.3.0 already installed -RUN yum -y update \ -&& yum -y install \ +RUN dnf -y update \ +&& dnf -y install \ curl wget \ git patch \ which pkgconfig autoconf libtool \ make gcc-c++ \ redhat-lsb openssl-devel pcre2-devel \ zlib-devel unzip zip \ -&& yum clean all \ -&& rm -rf /var/cache/yum +&& dnf clean all \ +&& rm -rf /var/cache/dnf ENTRYPOINT ["/usr/bin/bash", "-c"] CMD ["/usr/bin/bash"] @@ -38,13 +39,14 @@ RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ && git reset --hard "${GIT_SHA1}" WORKDIR /project +# Copy build script and setup env +ENV PLATFORM aarch64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION ${PYTHON_VERSION:-3} COPY build-manylinux.sh . RUN chmod a+x "build-manylinux.sh" FROM devel AS build -ENV PLATFORM aarch64 -ARG PYTHON_VERSION -ENV PYTHON_VERSION ${PYTHON_VERSION:-3} RUN ./build-manylinux.sh build FROM build as test diff --git a/tools/docker/python/arm64/musllinux.Dockerfile b/tools/docker/python/arm64/musllinux.Dockerfile new file mode 100644 index 0000000000..719dc6417e --- /dev/null +++ b/tools/docker/python/arm64/musllinux.Dockerfile @@ -0,0 +1,46 @@ +FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS env +# CMake 3.31.2 and SWIG 4.3.0 already installed + +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["/bin/sh"] + +## Python +#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ +# py3-numpy py3-pandas py3-matplotlib +#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ +#&& python3 -m pip install absl-py mypy mypy-protobuf + +################ +## OR-TOOLS ## +################ +FROM env AS devel +ENV GIT_URL https://github.com/google/or-tools + +ARG GIT_BRANCH +ENV GIT_BRANCH ${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1 ${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM aarch64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION ${PYTHON_VERSION:-3} +COPY build-musllinux.sh . +RUN chmod a+x "build-musllinux.sh" + +FROM devel AS build +RUN ./build-musllinux.sh build + +FROM build as test +RUN ./build-musllinux.sh test diff --git a/tools/docker/python/arm64v8/alpine.Dockerfile b/tools/docker/python/arm64v8/alpine.Dockerfile deleted file mode 100644 index 4a8dbfc96a..0000000000 --- a/tools/docker/python/arm64v8/alpine.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Create a virtual environment with all tools installed -# ref: https://hub.docker.com/_/alpine -FROM arm64v8/alpine:edge AS env - -# Install system build dependencies -ENV PATH=/usr/local/bin:$PATH -RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools -ENTRYPOINT ["/bin/sh", "-c"] -CMD ["/bin/sh"] - -# SWIG -RUN apk add --no-cache swig - -# Python -RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ - py3-numpy py3-pandas py3-matplotlib -RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ -&& python3 -m pip install absl-py mypy mypy-protobuf - -################ -## OR-TOOLS ## -################ -FROM env AS devel -ENV GIT_URL https://github.com/google/or-tools - -ARG GIT_BRANCH -ENV GIT_BRANCH ${GIT_BRANCH:-main} -ARG GIT_SHA1 -ENV GIT_SHA1 ${GIT_SHA1:-unknown} - -# Download sources -# use GIT_SHA1 to modify the command -# i.e. avoid docker reusing the cache when new commit is pushed -RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \ -&& cd /project \ -&& git reset --hard "${GIT_SHA1}" -WORKDIR /project - -FROM devel AS build -RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -RUN cmake --build build -v -j8 -# Rename wheel package ortools-version+musl-.... -RUN cp build/python/dist/ortools-*.whl . -RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}" -RUN rm build/python/dist/ortools-*.whl -RUN mv *.whl build/python/dist/ - -FROM build AS test -RUN cmake --build build --target test diff --git a/tools/docker/python/build-manylinux.sh b/tools/docker/python/build-manylinux.sh index 8a8f17a50f..1b8cbf6654 100755 --- a/tools/docker/python/build-manylinux.sh +++ b/tools/docker/python/build-manylinux.sh @@ -107,6 +107,7 @@ function build_wheel() { function check_wheel() { assert_defined BUILD_DIR + assert_defined VENV_DIR # Check the wheel artifact # Arguments: # $1 the python root directory @@ -115,9 +116,14 @@ function check_wheel() { exit 1 fi + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U auditwheel + # Check mypy files declare -a MYPY_FILES=( "ortools/algorithms/python/knapsack_solver.pyi" + "ortools/algorithms/python/set_cover.pyi" "ortools/constraint_solver/pywrapcp.pyi" "ortools/graph/python/linear_sum_assignment.pyi" "ortools/graph/python/max_flow.pyi" @@ -138,16 +144,20 @@ function check_wheel() { done # Check all generated wheel packages + ROOT_DIR=$(pwd) pushd "${BUILD_DIR}/python/dist" for FILE in *.whl; do # if no files found do nothing [[ -e "$FILE" ]] || continue - auditwheel show "$FILE" || true - auditwheel -v repair --plat "manylinux2014_$PLATFORM" "$FILE" -w "$export_root" - #auditwheel -v repair --plat manylinux2014_x86_64 "$FILE" -w "$export_root" - #auditwheel -v repair --plat manylinux2014_aarch64 "$FILE" -w "$export_root" + python -m auditwheel show "$FILE" || true + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${ROOT_DIR}/${BUILD_DIR}/lib64" python -m auditwheel repair --plat "manylinux_2_28_$PLATFORM" "$FILE" -w . + #python -m auditwheel -v repair --plat manylinux_2_28_x86_64 "$FILE" -w . + #python -m auditwheel -v repair --plat manylinux_2_28_aarch64 "$FILE" -w . done popd + + # Restore environment + deactivate } function test_wheel() { @@ -186,9 +196,9 @@ function test_wheel() { "ortools/linear_solver/samples/simple_lp_program.py" "ortools/linear_solver/samples/simple_mip_program.py" "ortools/sat/samples/simple_sat_program.py" - "ortools/routing/samples/tsp.py" - "ortools/routing/samples/vrp.py" - "ortools/routing/samples/cvrptw_break.py" + "ortools/constraint_solver/samples/tsp.py" + "ortools/constraint_solver/samples/vrp.py" + "ortools/constraint_solver/samples/cvrptw_break.py" ) # Run all the specified test scripts using the current environment. diff --git a/tools/docker/python/build-musllinux.sh b/tools/docker/python/build-musllinux.sh new file mode 100755 index 0000000000..e4688d8baf --- /dev/null +++ b/tools/docker/python/build-musllinux.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +# Copyright 2010-2025 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build all the wheel artifacts for the platforms supported by musllinux_1_2 and +# export them to the specified location. +set -exo pipefail + +function assert_defined(){ + if [[ -z "${!1}" ]]; then + >&2 echo "Variable '${1}' must be defined" + exit 1 + fi +} + +function usage() { + local -r NAME=$(basename "$0") + echo -e "$NAME - Build using a cross toolchain. + +SYNOPSIS +\t$NAME [-h|--help] [build|test|all] + +DESCRIPTION +\tBuild wheel artifacts. + +\tYou MUST define the following variables before running this script: +\t* PLATFORM: x86_64 aarch64 +\t* PYTHON_VERSION: 3 38 39 310 311 312 313 +note: PYTHON_VERSION=3 will generate for all pythons which could take time... + +OPTIONS +\t-h --help: show this help text +\tbuild: build the project using each python (note: remove previous build dir) +\ttest: install each wheel in a venv then test it (note: don't build !) +\tall: build + test (default) + +EXAMPLES +* Using export +export PLATFORM=x86_64 +export PYTHON_VERSION=39 +$0 build + +* One-liner: +PLATFORM=x86_64 PYTHON_VERSION=39 $0 build" +} + +function contains_element() { + # Look for the presence of an element in an array. Echoes '0' if found, + # '1' otherwise. + # Arguments: + # $1 the element to be searched + # $2 the array to search into + local e match="$1" + shift + for e; do + [[ "$e" == "$match" ]] && return 0 + done + return 1 +} + +function build_wheel() { + assert_defined BUILD_DIR + assert_defined VENV_DIR + # Build the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + # this is needed so protoc can call the correct python executable + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${VENV_DIR}" + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U pip setuptools wheel absl-py # absl-py is needed by make test_python + pip install -U mypy mypy-protobuf # need to generate protobuf mypy files + + echo "current dir: $(pwd)" + + if [[ ! -e "CMakeLists.txt" ]] || [[ ! -d "cmake" ]]; then + >&2 echo "Can't find project's CMakeLists.txt or cmake" + exit 2 + fi + cmake -S. -B"${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$1" \ + -DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_EXAMPLES=OFF #--debug-find + cmake --build "${BUILD_DIR}" -v -j4 + + # Restore environment + deactivate +} + +function check_wheel() { + assert_defined BUILD_DIR + # Check the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Check mypy files + declare -a MYPY_FILES=( + "ortools/algorithms/python/knapsack_solver.pyi" + "ortools/algorithms/python/set_cover.pyi" + "ortools/constraint_solver/pywrapcp.pyi" + "ortools/graph/python/linear_sum_assignment.pyi" + "ortools/graph/python/max_flow.pyi" + "ortools/graph/python/min_cost_flow.pyi" + "ortools/init/python/init.pyi" + "ortools/linear_solver/python/model_builder_helper.pyi" + "ortools/linear_solver/pywraplp.pyi" + "ortools/pdlp/python/pdlp.pyi" + "ortools/sat/python/cp_model_helper.pyi" + "ortools/scheduling/python/rcpsp.pyi" + "ortools/util/python/sorted_interval_list.pyi" + ) + for FILE in "${MYPY_FILES[@]}"; do + if [[ ! -f "${BUILD_DIR}/python/${FILE}" ]]; then + echo "error: ${FILE} missing in the python project" + exit 1 + fi + done + + # Check all generated wheel packages + pushd "${BUILD_DIR}/python/dist" + for FILE in *.whl; do + # if no files found do nothing + [[ -e "$FILE" ]] || continue + auditwheel show "$FILE" || true + auditwheel -v repair --plat "musllinux_1_2_$PLATFORM" "$FILE" -w "$export_root" + #auditwheel -v repair --plat musllinux_1_2_x86_64 "$FILE" -w "$export_root" + #auditwheel -v repair --plat musllinux_1_2_aarch64 "$FILE" -w "$export_root" + done + popd +} + +function test_wheel() { + assert_defined BUILD_DIR + assert_defined TEST_DIR + # Test the wheel artifacts + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${TEST_DIR}" + + # shellcheck source=/dev/null + source "${TEST_DIR}/bin/activate" + pip install -U pip setuptools wheel + + # Install the wheel artifact + #pwd + local -r WHEEL_FILE=$(find "${BUILD_DIR}"/python/dist/*.whl | head -1) + echo "WHEEL file: ${WHEEL_FILE}" + pip install --no-cache-dir "$WHEEL_FILE" + pip show ortools + + # Python scripts to be used as tests for the installed wheel. This list of files + # has been taken from the 'test_python' make target. + declare -a TESTS=( + "ortools/algorithms/samples/simple_knapsack_program.py" + "ortools/graph/samples/simple_max_flow_program.py" + "ortools/graph/samples/simple_min_cost_flow_program.py" + "ortools/linear_solver/samples/simple_lp_program.py" + "ortools/linear_solver/samples/simple_mip_program.py" + "ortools/sat/samples/simple_sat_program.py" + "ortools/constraint_solver/samples/tsp.py" + "ortools/constraint_solver/samples/vrp.py" + "ortools/constraint_solver/samples/cvrptw_break.py" + ) + + # Run all the specified test scripts using the current environment. + local -r ROOT_DIR=$(pwd) + pushd "$(mktemp -d)" # ensure we are not importing something from $PWD + python --version + for TEST in "${TESTS[@]}"; do + python "${ROOT_DIR}/${TEST}" + done + popd + + # Restore environment + deactivate +} + +function build() { + # For each python platform provided by musllinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + VENV_DIR="env_${PYTAG}" + build_wheel "$PYROOT" + check_wheel "$PYROOT" + done +} + +function tests() { + # For each python platform provided by musllinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + TEST_DIR="test_${PYTAG}" + test_wheel "$PYROOT" + done +} + +# Main +function main() { + case ${1} in + -h | --help) + usage; exit ;; + esac + + assert_defined PLATFORM + assert_defined PYTHON_VERSION + + # Setup + declare -a SKIPS=( "cp36-cp36m" "cp37-cp37m" ) + + case ${1} in + build) + build ;; + test) + tests ;; + *) + build + tests ;; + esac +} + +main "${1:-all}"