From 3afda7e798220ce1f0764a53a3490b6614855c1c Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Mon, 16 Dec 2024 15:11:37 +0000 Subject: [PATCH] ci: Organise shared helper scripts Move shared helper scripts used by Docker builds under docker/scripts. --- .../dist-aarch64-linux/Dockerfile | 11 ++-- .../dist-aarch64-linux/build-clang.sh | 46 ----------------- .../dist-aarch64-linux/build-gcc.sh | 51 ------------------- .../host-aarch64/dist-aarch64-linux/shared.sh | 16 ------ .../host-x86_64/dist-i686-linux/Dockerfile | 8 +-- .../dist-powerpc64le-linux/Dockerfile | 2 +- .../dist-powerpc64le-linux/shared.sh | 16 ------ .../host-x86_64/dist-x86_64-linux/Dockerfile | 9 ++-- .../dist-x86_64-linux/build-zstd.sh | 29 ----------- .../host-x86_64/dist-x86_64-linux/shared.sh | 16 ------ .../host-x86_64/x86_64-gnu-llvm-18/Dockerfile | 4 +- .../host-x86_64/x86_64-gnu-llvm-19/Dockerfile | 4 +- .../host-x86_64/x86_64-gnu-tools/Dockerfile | 4 +- .../build-clang.sh | 4 +- .../build-gcc.sh | 10 ++-- .../build-gccjit.sh | 0 .../build-zstd.sh | 0 17 files changed, 31 insertions(+), 199 deletions(-) delete mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh delete mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh delete mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh delete mode 100644 src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh delete mode 100755 src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh delete mode 100644 src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-clang.sh (95%) rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-gcc.sh (87%) rename src/ci/docker/{host-x86_64/dist-x86_64-linux => scripts}/build-gccjit.sh (100%) rename src/ci/docker/{host-aarch64/dist-aarch64-linux => scripts}/build-zstd.sh (100%) diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile index c7778dbf6bbff..4f4caa5fa50cf 100644 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -43,10 +43,10 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp RUN mkdir /home/user -COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM -COPY host-aarch64/dist-aarch64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ ENV CC=gcc CXX=g++ @@ -56,12 +56,13 @@ COPY scripts/cmake.sh /tmp/ RUN ./cmake.sh # Build LLVM+Clang -COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=AArch64 RUN ./build-clang.sh ENV CC=clang CXX=clang++ # Build zstd to enable `llvm.libzstd`. -COPY host-aarch64/dist-aarch64-linux/build-zstd.sh /tmp/ +COPY scripts/build-zstd.sh /tmp/ RUN ./build-zstd.sh COPY scripts/sccache.sh /scripts/ @@ -82,7 +83,7 @@ ENV RUST_CONFIGURE_ARGS \ --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ --set llvm.link-shared=true \ - --set llvm.thin-lto=false \ + --set llvm.thin-lto=true \ --set llvm.libzstd=true \ --set llvm.ninja=false \ --set rust.debug-assertions=false \ diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh deleted file mode 100755 index fb5f4d8c4f795..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -source shared.sh - -# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh -LLVM=llvmorg-19.1.5 - -mkdir llvm-project -cd llvm-project - -curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ - tar xzf - --strip-components=1 - -mkdir clang-build -cd clang-build - -# For whatever reason the default set of include paths for clang is different -# than that of gcc. As a result we need to manually include our sysroot's -# include path, /rustroot/include, to clang's default include path. -INC="/rustroot/include:/usr/include" - -# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) -# but sanitizers aren't currently building. Since we don't need those, just -# disable them. BOLT is used for optimizing LLVM. -hide_output \ - cmake ../llvm \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/rustroot \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_BUILD_MEMPROF=OFF \ - -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ - -DLLVM_TARGETS_TO_BUILD=AArch64 \ - -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ - -DC_INCLUDE_DIRS="$INC" - -hide_output make -j$(nproc) -hide_output make install - -cd ../.. -rm -rf llvm-project diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh deleted file mode 100755 index ad33b21b9b5aa..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -ex - -source shared.sh - -# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. -GCC=9.5.0 - -curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - -cd gcc-$GCC - -# FIXME(#49246): Remove the `sed` below. -# -# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this -# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue: -# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection -# timed out" error, and even when the download completed, the file is usually corrupted. This causes -# nothing to be landed that day. -# -# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability -# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third -# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the -# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server -# instead here. -# -# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to -# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or -# the expression will need to be updated. That new URL is available at: -# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 -# -sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites - -./contrib/download_prerequisites -mkdir ../gcc-build -cd ../gcc-build - -# '-fno-reorder-blocks-and-partition' is required to -# enable BOLT optimization of the C++ standard library, -# which is included in librustc_driver.so -hide_output ../gcc-$GCC/configure \ - --prefix=/rustroot \ - --enable-languages=c,c++ \ - --disable-gnu-unique-object \ - --enable-cxx-flags='-fno-reorder-blocks-and-partition' -hide_output make -j$(nproc) -hide_output make install -ln -s gcc /rustroot/bin/cc - -cd .. -rm -rf gcc-build -rm -rf gcc-$GCC diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile index 414bcc52484c9..7cf1c80dfbc65 100644 --- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile @@ -46,10 +46,11 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp RUN mkdir /home/user -COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM nowadays -COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ +ENV GCC_BUILD_TARGET=i686 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ COPY scripts/cmake.sh /tmp/ @@ -57,7 +58,8 @@ RUN ./cmake.sh # Now build LLVM+Clang, afterwards configuring further compilations to use the # clang/clang++ compilers. -COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=X86 RUN ./build-clang.sh ENV CC=clang CXX=clang++ diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile index 9ef391892497f..9d3be51d037d1 100644 --- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile @@ -18,7 +18,7 @@ RUN /scripts/crosstool-ng-build.sh WORKDIR /build RUN apt-get install -y --no-install-recommends rpm2cpio cpio -COPY host-x86_64/dist-powerpc64le-linux/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/ +COPY scripts/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/ RUN ./build-powerpc64le-toolchain.sh COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index e857f38e68a85..c13c340871c9a 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -46,10 +46,10 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig WORKDIR /tmp RUN mkdir /home/user -COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/ +COPY scripts/shared.sh /tmp/ # Need at least GCC 5.1 to compile LLVM nowadays -COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/ +COPY scripts/build-gcc.sh /tmp/ RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ # LLVM 17 needs cmake 3.20 or higher. @@ -58,12 +58,13 @@ RUN ./cmake.sh # Now build LLVM+Clang, afterwards configuring further compilations to use the # clang/clang++ compilers. -COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ +COPY scripts/build-clang.sh /tmp/ +ENV LLVM_BUILD_TARGETS=X86 RUN ./build-clang.sh ENV CC=clang CXX=clang++ # Build zstd to enable `llvm.libzstd`. -COPY host-x86_64/dist-x86_64-linux/build-zstd.sh /tmp/ +COPY scripts/build-zstd.sh /tmp/ RUN ./build-zstd.sh COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh deleted file mode 100755 index a3d37ccc31120..0000000000000 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -ex - -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/zstd_build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/zstd_build.log - trap - ERR - kill $PING_LOOP_PID - rm /tmp/zstd_build.log - set -x -} - -ZSTD=1.5.6 -curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf - - -cd zstd-$ZSTD -CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 -hide_output make install - -cd .. -rm -rf zstd-$ZSTD diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh deleted file mode 100644 index dc86dddd464f2..0000000000000 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile index 42df58517caf4..1cfa227344344 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile @@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.randomize-layout=true \ --set rust.thin-lto-import-instr-limit=10 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile index f2aadbe87cfec..b33cb6478afa2 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile @@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.randomize-layout=true \ --set rust.thin-lto-import-instr-limit=10 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 2a09cd54b139a..ab749b3fdd5a6 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -89,8 +89,8 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu # assertions enabled! Therefore, we cannot force download CI rustc. #ENV FORCE_CI_RUSTC 1 -COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ -COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ +COPY scripts/shared.sh /scripts/ +COPY scripts/build-gccjit.sh /scripts/ RUN /scripts/build-gccjit.sh /scripts diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/scripts/build-clang.sh similarity index 95% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh rename to src/ci/docker/scripts/build-clang.sh index 2e08c87f278c0..47bfcfbecab38 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/scripts/build-clang.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -ex +set -exu source shared.sh @@ -34,7 +34,7 @@ hide_output \ -DCOMPILER_RT_BUILD_XRAY=OFF \ -DCOMPILER_RT_BUILD_MEMPROF=OFF \ -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ - -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_TARGETS_TO_BUILD=$LLVM_BUILD_TARGETS \ -DLLVM_INCLUDE_BENCHMARKS=OFF \ -DLLVM_INCLUDE_TESTS=OFF \ -DLLVM_INCLUDE_EXAMPLES=OFF \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/scripts/build-gcc.sh similarity index 87% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh rename to src/ci/docker/scripts/build-gcc.sh index e939a5d7eac4d..78a038215e4b9 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/scripts/build-gcc.sh @@ -50,7 +50,9 @@ cd .. rm -rf gcc-build rm -rf gcc-$GCC -# FIXME: clang doesn't find 32-bit libraries in /rustroot/lib, -# but it does look all the way under /rustroot/lib/[...]/32, -# so we can link stuff there to help it out. -ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/ +if [[ $GCC_BUILD_TARGET == "i686" ]]; then + # FIXME: clang doesn't find 32-bit libraries in /rustroot/lib, + # but it does look all the way under /rustroot/lib/[...]/32, + # so we can link stuff there to help it out. + ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/ +fi diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh similarity index 100% rename from src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh rename to src/ci/docker/scripts/build-gccjit.sh diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh similarity index 100% rename from src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh rename to src/ci/docker/scripts/build-zstd.sh