end-to-end gpu driver testing enhancement

Signed-off-by: shiva kumar <[email protected]>
NVIDIA · Aug 17, 2024 · 84f80c9 · 84f80c9
1 parent 600b7bf
commit 84f80c9
Show file tree

Hide file tree

Showing 10 changed files with 351 additions and 44 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: CI
+name: End-to-end tests
 
 on:
   workflow_run:
@@ -21,15 +21,10 @@ on:
       - completed
     branches:
       - main
-
+ 
 jobs:
   e2e-tests-nvidiadriver:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        driver:
-          - 535.183.06
-          - 550.90.07
 
     steps:
     - name: Check out code
@@ -41,7 +36,6 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
-        AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }}
       with:
         aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -59,11 +53,32 @@ jobs:
         echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
         echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
         echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
+        DRIVER_VERSIONS=$(grep '^DRIVER_VERSIONS ?=' versions.mk | awk -F' ?= ' '{print $2}')
+        echo "DRIVER_VERSIONS=$DRIVER_VERSIONS" >> $GITHUB_ENV
         
     - name: Validate gpu driver
       env:
         TEST_CASE: "./tests/cases/nvidia-driver.sh"
       run: |
         sudo chmod 644 ${{ github.workspace }}/.cache/key
         echo "${{ secrets.AWS_SSH_KEY }}" > ${private_key} && chmod 400 ${private_key}
-        ./tests/ci-run-e2e.sh ${TEST_CASE} ${COMMIT_SHORT_SHA}-${{ matrix.driver }}
+        rc=0
+        for driver_version in ${DRIVER_VERSIONS}; do
+          echo "Running e2e for DRIVER_VERSION=$driver_version"
+          ./tests/ci-run-e2e.sh ${TEST_CASE} ${COMMIT_SHORT_SHA}-${driver_version} || status=$?
+          if [ $status -ne 0 ]; then
+            echo "e2e validation failed for driver version $driver_version with status $status"
+            rc=$status
+          fi
+        done
+        source ./tests/scripts/.definitions.sh
+        ./tests/scripts/pull.sh ${LOG_DIR} logs
+        exit $rc
+
+    - name: Archive test logs
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: nvidiadriver-e2e-test-logs
+        path: ./logs/
+        retention-days: 15  
diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml
@@ -34,6 +34,7 @@ jobs:
     strategy:
       matrix:
         driver:
+          - 470.256.02
           - 535.183.06
           - 550.90.07
         dist:
@@ -55,6 +56,10 @@ jobs:
           - ispr: true
             dist: ubuntu20.04
             driver: 550.90.07
+          - driver: 470.256.02
+            dist: ubuntu22.04
+          - driver: 470.256.02
+            dist: rhel9
       fail-fast: false
     steps:
       - uses: actions/checkout@v4

diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
@@ -19,3 +19,8 @@ CASES_DIR="$( cd "${TEST_DIR}/cases" && pwd )"
 : ${HELM_NVIDIA_REPO:="https://helm.ngc.nvidia.com/nvidia"}
 
 : ${TARGET_DRIVER_VERSION:="550.90.07"}
+
+: ${DAEMON_POD_STATUS_TIME_OUT:="15m"}
+: ${POD_STATUS_TIME_OUT:="2m"}
+
+: ${LOG_DIR:="/tmp/logs"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
@@ -2,35 +2,20 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	local current_time=0
-	while :; do
-		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+	local pod_status_time_out=$2
+
+	echo "Checking $pod_label pod"
+
+	kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
 
-		echo "Checking $pod_label pod readiness"
-		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+	echo "Checking $pod_label pod readiness"
 
-		if [ "${is_pod_ready}" = "True" ]; then
-			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
-			if [ "${is_pod_terminating}" != "" ]; then
-				echo "pod $pod_label is in terminating state..."
-			else
-				echo "Pod $pod_label is ready"
-				break;
-			fi
-		fi
-
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
-			echo "timeout reached"
-			exit 1;
-		fi
-
-		# Echo useful information on stdout
+	if kubectl wait -n ${TEST_NAMESPACE} --for=condition=Ready pod -l app=$pod_label --timeout ${pod_status_time_out}; then
+		return 0	
+	else
+		# print status of pod
 		kubectl get pods -n ${TEST_NAMESPACE}
+	fi
 
-		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
-		sleep 5
-	done
+	return 1
 }
diff --git a/tests/scripts/end-to-end-nvidia-driver.sh b/tests/scripts/end-to-end-nvidia-driver.sh
@@ -7,8 +7,12 @@ echo ""
 echo ""
 echo "--------------Installing the GPU Operator--------------"
 
-# Install the operator with usePrecompiled mode set to true
 ${SCRIPT_DIR}/install-operator.sh
 
 "${SCRIPT_DIR}"/verify-operator.sh
+
+echo "--------------Verification completed for GPU Operator, uninstalling the operator--------------"
+
+${SCRIPT_DIR}/uninstall-operator.sh ${TEST_NAMESPACE} "gpu-operator"
+
 echo "--------------Verification completed for GPU Operator--------------"