Improve integration tests

- Upgraded version of CSI driver registrar
- Dump logs from all components on failure
- Deploy SPIRE infrastructure before workloads
- Update to later version of Kind
- Add test support for K8s 1.26
- Align Kind node hashes with K8s version under test
- Align kubelet version with K8s version
- Dropped some unneeded config from the agent config

Signed-off-by: Andrew Harding <[email protected]>

.github/workflows/pr_build.yaml

@@ -62,10 +62,11 @@ jobs:
         # Kubernetes, but can go back farther as long as we don't need heroics
         # to pull it off (i.e. kubectl version juggling).
         k8s-version:
-          - v1.25.3
-          - v1.24.7
-          - v1.23.13
-          - v1.22.15
+          - v1.26.3
+          - v1.25.8
+          - v1.24.12
+          - v1.23.17
+          - v1.22.17
           - v1.21.14
 
     steps:
@@ -83,7 +84,7 @@ jobs:
           tar xvf images.tar.gz
           make load-images
       - name: Run integration tests
-        run: K8S_VERSION=${{ matrix.k8s-version }} KUBECTL_VERSION=${{ matrix.k8s-version }} test/run.sh
+        run: K8S_VERSION=${{ matrix.k8s-version }} test/run.sh
 
   success:
     runs-on: ubuntu-22.04

.github/workflows/release_build.yaml

@@ -63,10 +63,11 @@ jobs:
         # Kubernetes, but can go back farther as long as we don't need heroics
         # to pull it off (i.e. kubectl version juggling).
         k8s-version:
-          - v1.25.3
-          - v1.24.7
-          - v1.23.13
-          - v1.22.15
+          - v1.26.3
+          - v1.25.8
+          - v1.24.12
+          - v1.23.17
+          - v1.22.17
           - v1.21.14
     steps:
       - name: Checkout
@@ -83,7 +84,7 @@ jobs:
           tar xvf images.tar.gz
           make load-images
       - name: Run integration tests
-        run: K8S_VERSION=${{ matrix.k8s-version }} KUBECTL_VERSION=${{ matrix.k8s-version }} test/run.sh
+        run: K8S_VERSION=${{ matrix.k8s-version }} test/run.sh
 
   publish:
     runs-on: ubuntu-latest

test/config/kustomization.yaml → test/config/spire/kustomization.yaml

@@ -3,5 +3,3 @@ resources:
 - spiffe-csi-driver.yaml
 - spire-server.yaml
 - spire-agent.yaml
-- spiffe-csi-test-workload-1.yaml
-- spiffe-csi-test-workload-2.yaml

test/config/spiffe-csi-driver.yaml → test/config/spire/spiffe-csi-driver.yaml

@@ -69,7 +69,7 @@ spec:
         # of all the little details required to register a CSI driver with
         # the kubelet.
         - name: node-driver-registrar
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0
           imagePullPolicy: IfNotPresent
           args: [
             "-csi-address", "/spiffe-csi/csi.sock",

test/config/spire-agent.yaml → test/config/spire/spire-agent.yaml

@@ -88,8 +88,6 @@ spec:
   selector:
     matchLabels:
       app: spire-agent
-  updateStrategy:
-    type: RollingUpdate
   template:
     metadata:
       namespace: spire-system

test/config/spire-server.yaml → test/config/spire/spire-server.yaml

@@ -105,12 +105,6 @@ data:
       trust_domain = "test"
       data_dir = "/run/spire/server/data"
       log_level = "DEBUG"
-      federation {
-        bundle_endpoint {
-          address = "0.0.0.0"
-          port = 8443
-        }
-      }
     }
 
     plugins {

test/config/workloads/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- test-workload-1.yaml
+- test-workload-2.yaml

test/run.sh

@@ -4,18 +4,38 @@ set -e -o pipefail
 
 DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
+
 # Versions under test
-KIND_VERSION=${KIND_VERSION:-v0.17.0}
-KUBECTL_VERSION=${KUBECTL_VERSION:-v1.25.3}
-
-# e.g. K8S_VERSION=v1.22.4
-# https://hub.docker.com/r/kindest/node/tags
-if test -n "$K8S_VERSION"; then
-    KIND_NODE="--image=kindest/node:$K8S_VERSION"
-else
-    # Just use the default.
-    KIND_NODE=""
-fi
+K8S_VERSION=${K8S_VERSION:-v1.26.3}
+
+# Determine which Kind node to use for the K8s version under test. The node
+# hashes are tightly coupled to the Kind version used and they must be updated
+# together.
+KIND_VERSION=v0.18.0
+case "$K8S_VERSION" in
+    v1.26.3)
+        KIND_NODE="sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f"
+        ;;
+    v1.25.8)
+        KIND_NODE="sha256:00d3f5314cc35327706776e95b2f8e504198ce59ac545d0200a89e69fce10b7f"
+        ;;
+    v1.24.12)
+        KIND_NODE="sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16"
+        ;;
+    v1.23.17)
+        KIND_NODE="sha256:e5fd1d9cd7a9a50939f9c005684df5a6d145e8d695e78463637b79464292e66c"
+        ;;
+    v1.22.17)
+        KIND_NODE="sha256:c8a828709a53c25cbdc0790c8afe12f25538617c7be879083248981945c38693"
+        ;;
+    v1.21.14)
+        KIND_NODE="sha256:27ef72ea623ee879a25fe6f9982690a3e370c68286f4356bf643467c552a3888"
+        ;;
+    *)
+        echo "no kind node available for Kind $KIND_VERSION and Kubernetes $K8S_VERSION" 1>&2
+        exit 1
+        ;;
+esac
 
 # Export the Kind cluster name so we don't have to specify it on every kind
 # invocation
@@ -49,18 +69,52 @@ case "${ARCH}" in
         ;;
 esac
 
-
+SUCCESS=
 cleanup() {
-    delete-cluster
-    rm -rf "${TMPDIR}"
+    if [ -z "$SUCCESS" ]; then
+        echo "================================================="
+        echo "LOGS: SPIRE Server"
+        echo "================================================="
+        "${KUBECTL}" logs -nspire-system deployment/spire-server --all-containers=true || true
+
+        echo "================================================="
+        echo "LOGS: SPIRE Agent"
+        echo "================================================="
+        "${KUBECTL}" logs -nspire-system daemonset/spire-agent --all-containers=true || true
+
+        echo "================================================="
+        echo "LOGS: SPIFFE CSI Driver"
+        echo "================================================="
+        "${KUBECTL}" logs -nspire-system daemonset/spiffe-csi-driver --all-containers=true || true
+
+        echo "================================================="
+        echo "LOGS: Test Workload 1"
+        echo "================================================="
+        "${KUBECTL}" logs deployment/test-workload-1 --all-containers=true || true
+
+        echo "================================================="
+        echo "LOGS: Test Workload 2"
+        echo "================================================="
+        "${KUBECTL}" logs deployment/test-workload-2 --all-containers=true || true
+
+    fi
+
+    [ -n "$SKIPCLEANUP" ] || delete-cluster
+    [ -n "$SKIPCLEANUP" ] || rm -rf "${TMPDIR}"
+
+    if [ -z "$SUCCESS" ]; then
+        echo "================================================="
+        echo "!!! FAILED !!!!"
+        echo "================================================="
+    fi
 }
 
 build-workload() {
     (cd "${DIR}"/workload; docker build -t spiffe-csi-test-workload:test .)
 }
 
 download-kubectl() {
-    local _url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl"
+    local _url="https://dl.k8s.io/release/${K8S_VERSION}/bin/${OS}/${ARCH}/kubectl"
     echo "Downloading ${_url}..."
     curl -s -Lo "${KUBECTL}" "${_url}"
     chmod +x "${KUBECTL}"
@@ -75,7 +129,7 @@ download-kind() {
 
 create-cluster() {
     echo "Creating cluster..."
-    "${KIND}" create cluster $KIND_NODE --config "${DIR}/config/cluster.yaml"
+    "${KIND}" create cluster --image=kindest/node@$KIND_NODE --config "${DIR}/config/cluster.yaml"
     echo "Cluster created."
     "${KUBECTL}" version
 }
@@ -98,19 +152,25 @@ load-images() {
     echo "Images loaded."
 }
 
-apply-yaml() {
-    "${KUBECTL}" apply -k "${DIR}"/config
+deploy-spire() {
+    "${KUBECTL}" apply -k "${DIR}"/config/spire
     echo "Waiting for SPIRE server rollout..."
     "${KUBECTL}" rollout status -w --timeout=1m -nspire-system deployment/spire-server
     echo "Waiting for SPIRE agent rollout..."
     "${KUBECTL}" rollout status -w --timeout=1m -nspire-system daemonset/spire-agent
+    echo "Waiting for SPIFFE CSI Driver rollout..."
+    "${KUBECTL}" rollout status -w --timeout=1m -nspire-system daemonset/spiffe-csi-driver
+}
+
+deploy-workloads() {
+    "${KUBECTL}" apply -k "${DIR}"/config/workloads
     echo "Waiting for test workload 1 rollout..."
     "${KUBECTL}" rollout status -w --timeout=1m deployment/test-workload-1
     echo "Waiting for test workload 2 rollout..."
     "${KUBECTL}" rollout status -w --timeout=1m deployment/test-workload-2
 }
 
-register-workload() {
+register-workloads() {
     "${KUBECTL}" exec \
         -nspire-system \
         deployment/spire-server -- \
@@ -155,9 +215,10 @@ download-kind
 download-kubectl
 create-cluster
 load-images
-apply-yaml
-register-workload
+deploy-spire
+register-workloads
+deploy-workloads
 check-workload-status "test-workload-1"
 check-workload-status "test-workload-2"
-"${KUBECTL}" logs -nspire-system daemonset/spiffe-csi-driver -c spiffe-csi-driver
+SUCCESS=1
 echo "Done."