ci: merge docker images (solana-labs#35078)

* ci: add new ci docker image

* ci: use the new docker image

* fix shellcheck

* add readme for the new docker image

* remove old docker images

* remove unused check in docs/build.sh

* use the new image in net.sh

.buildkite/scripts/build-bench.sh

@@ -22,5 +22,5 @@ EOF
 
 # shellcheck disable=SC2016
 group "bench" \
-  "$(build_steps "bench-part-1" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part1.sh")" \
-  "$(build_steps "bench-part-2" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part2.sh")"
+  "$(build_steps "bench-part-1" "ci/docker-run-default-image.sh ci/bench/part1.sh")" \
+  "$(build_steps "bench-part-2" "ci/docker-run-default-image.sh ci/bench/part2.sh")"

.buildkite/scripts/build-stable.sh

@@ -12,7 +12,7 @@ partitions=$(
   cat <<EOF
 {
   "name": "partitions",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-partition.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-partition.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent",
   "parallelism": 2,
@@ -25,7 +25,7 @@ local_cluster_partitions=$(
   cat <<EOF
 {
   "name": "local-cluster",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-local-cluster-partially.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-local-cluster-partially.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent",
   "parallelism": 5,
@@ -38,7 +38,7 @@ localnet=$(
   cat <<EOF
 {
   "name": "localnet",
-  "command": ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/stable/run-localnet.sh",
+  "command": "ci/docker-run-default-image.sh ci/stable/run-localnet.sh",
   "timeout_in_minutes": 30,
   "agent": "$agent"
 }

ci/buildkite-pipeline.sh

@@ -140,9 +140,9 @@ wait_step() {
 }
 
 all_test_steps() {
-  command_step checks1 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 check
-  command_step checks2 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-dev-context-only-utils.sh check-bins" 15 check
-  command_step checks3 ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-dev-context-only-utils.sh check-all-targets" 15 check
+  command_step checks1 "ci/docker-run-default-image.sh ci/test-checks.sh" 20 check
+  command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins" 15 check
+  command_step checks3 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-all-targets" 15 check
   wait_step
 
   # Full test suite
@@ -156,7 +156,7 @@ all_test_steps() {
              ^ci/rust-version.sh \
              ^ci/test-docs.sh \
       ; then
-    command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15
+    command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15
   else
     annotate --style info --context test-docs \
       "Docs skipped as no .rs files were modified"
@@ -182,7 +182,7 @@ all_test_steps() {
              cargo-test-sbf$ \
       ; then
     cat >> "$output_file" <<"EOF"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh"
+  - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh"
     name: "stable-sbf"
     timeout_in_minutes: 35
     artifact_paths: "sbf-dumps.tar.bz2"
@@ -226,7 +226,7 @@ EOF
              ^ci/test-stable.sh \
              ^sdk/ \
       ; then
-    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+    command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20
   else
     annotate --style info \
       "wasm skipped as no relevant files were modified"
@@ -258,7 +258,7 @@ EOF
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80
+    command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"
@@ -296,7 +296,7 @@ pull_or_push_steps() {
 
     if [ -z "$diff_other_than_version_bump" ]; then
       echo "Diff only contains version bump."
-      command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+      command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20
       exit 0
     fi
   fi

ci/buildkite-secondary.yml

@@ -3,7 +3,7 @@
 # Pull requests to not run these steps.
 steps:
   - name: "cargo audit"
-    command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/do-audit.sh"
+    command: "ci/docker-run-default-image.sh ci/do-audit.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 10

ci/buildkite-solana-private.sh

@@ -134,7 +134,7 @@ wait_step() {
 }
 
 all_test_steps() {
-  command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+  command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20
   wait_step
 
   # Full test suite
@@ -146,7 +146,7 @@ all_test_steps() {
              ^ci/rust-version.sh \
              ^ci/test-docs.sh \
       ; then
-    command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15
+    command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15
   else
     annotate --style info --context test-docs \
       "Docs skipped as no .rs files were modified"
@@ -168,7 +168,7 @@ all_test_steps() {
              ^sdk/ \
       ; then
     cat >> "$output_file" <<"EOF"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh"
+  - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh"
     name: "stable-sbf"
     timeout_in_minutes: 35
     artifact_paths: "sbf-dumps.tar.bz2"
@@ -208,7 +208,7 @@ EOF
              ^ci/test-stable.sh \
              ^sdk/ \
       ; then
-    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+    command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20
   else
     annotate --style info \
       "wasm skipped as no relevant files were modified"
@@ -238,7 +238,7 @@ EOF
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80
+    command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"

ci/dependabot-pr.sh

@@ -11,7 +11,7 @@ fi
 
 source ci/rust-version.sh stable
 
-ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh
+ci/docker-run-default-image.sh ci/dependabot-updater.sh
 
 if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then
   echo --- ok

ci/docker-run-default-image.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1091
+source "$here/rust-version.sh"
+
+"$here/docker-run.sh" "${ci_docker_image:?}" "$@"

ci/docker-rust/Dockerfile → ci/docker/Dockerfile

@@ -1,10 +1,12 @@
 FROM ubuntu:20.04
 
 ARG \
-  RUST_VERSION=1.75.0 \
+  RUST_VERSION= \
+  RUST_NIGHTLY_VERSION= \
   GOLANG_VERSION=1.21.3 \
   NODE_MAJOR=18 \
-  SCCACHE_VERSION=v0.5.4
+  SCCACHE_VERSION=v0.5.4 \
+  GRCOV_VERSION=v0.8.18
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -21,7 +23,10 @@ ENV \
   CARGO_HOME=/usr/local/cargo \
   PATH="$PATH:/usr/local/cargo/bin"
 
-RUN apt-get update && \
+RUN \
+  if [ -z "$RUST_VERSION" ]; then echo "ERROR: The RUST_VERSION argument is required!" && exit 1; fi && \
+  if [ -z "$RUST_NIGHTLY_VERSION" ]; then echo "ERROR: The RUST_NIGHTLY_VERSION argument is required!" && exit 1; fi && \
+  apt-get update && \
   apt-get install --no-install-recommends -y \
   # basic
   tzdata \
@@ -65,6 +70,9 @@ RUN apt-get update && \
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --no-modify-path --profile minimal --default-toolchain $RUST_VERSION -y && \
   rustup component add rustfmt && \
   rustup component add clippy && \
+  rustup install $RUST_NIGHTLY_VERSION && \
+  rustup component add clippy --toolchain=$RUST_NIGHTLY_VERSION && \
+  rustup component add rustfmt --toolchain=$RUST_NIGHTLY_VERSION && \
   rustup target add wasm32-unknown-unknown && \
   cargo install cargo-audit && \
   cargo install cargo-hack && \
@@ -74,6 +82,9 @@ RUN apt-get update && \
   cargo install svgbob_cli && \
   cargo install wasm-pack && \
   cargo install rustfilt && \
+  rustup show && \
+  rustc --version && \
+  cargo --version && \
   chmod -R a+w $CARGO_HOME $RUSTUP_HOME && \
   rm -rf $CARGO_HOME/registry && \
   # sccache
@@ -101,5 +112,14 @@ RUN apt-get update && \
   chmod -R a+w /.config && \
   mkdir /.npm && \
   chmod -R a+w /.npm && \
+  # grcov
+  curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \
+  tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \
+  mv ./grcov $CARGO_HOME/bin && \
+  rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \
+  # codecov
+  curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+  chmod +x codecov && \
+  mv codecov /usr/bin && \
   # clean lists
   rm -rf /var/lib/apt/lists/*

ci/docker/README.md

@@ -0,0 +1,11 @@
+Docker image containing rust, rust nightly and some preinstalled packages used in CI
+
+This image is manually maintained:
+
+#### CLI
+
+1. Edit
+   1. `ci/rust-version.sh` for rust and rust nightly version
+   2. `ci/docker/Dockerfile` for other packages
+2. Ensure you're a member of the [Solana Docker Hub Organization](https://hub.docker.com/u/solanalabs/) and already `docker login`
+3. Run `ci/docker/build.sh` to build/publish the new image

ci/docker/build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1091
+source "$here/../rust-version.sh"
+
+platform=()
+if [[ $(uname -m) = arm64 ]]; then
+  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
+  platform+=(--platform linux/amd64)
+fi
+
+echo "build image: ${ci_docker_image:?}"
+docker build "${platform[@]}" \
+  -f "$here/Dockerfile" \
+  --build-arg "RUST_VERSION=${rust_stable:?}" \
+  --build-arg "RUST_NIGHTLY_VERSION=${rust_nightly:?}" \
+  -t "$ci_docker_image" .
+
+docker push "$ci_docker_image"

ci/publish-crate.sh

@@ -72,7 +72,7 @@ for Cargo_toml in $Cargo_tomls; do
       echo "Attempt ${i} of ${numRetries}"
       # The rocksdb package does not build with the stock rust docker image so use
       # the solana rust docker image
-      if ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"; then
+      if ci/docker-run-default-image.sh bash -exc "cd $crate; $cargoCommand"; then
         break
       fi