test(NNS): run upgrade_canisters_with_golden_nns_state daily

.bazelrc

@@ -77,7 +77,7 @@ build --build_tag_filters="-system_test,-upload,-fuzz_test"
 test --test_tag_filters="-system_test,-post_master,-fuzz_test"
 test:alltests --test_tag_filters=""
 test:paritytests --test_tag_filters="-system_test"
-test:ci --test_tag_filters="-post_master,-system_test_hourly,-system_test_nightly,-system_test_nightly_nns,-system_test_staging,-system_test_hotfix,-fuzz_test"
+test:ci --test_tag_filters="-post_master,-system_test_hourly,-system_test_nightly,-system_test_nightly_nns,-system_test_staging,-system_test_hotfix,-fuzz_test,-nns_tests_nightly"
 
 test --test_output=errors
 test --test_env=RUST_BACKTRACE=full

.github/actions/bazel-test-all/action.yaml

@@ -24,6 +24,8 @@ inputs:
     default: "false"
   HONEYCOMB_API_TOKEN:
     required: false
+  SSH_PRIVATE_KEY:
+    required: false
 
 runs:
   using: "composite"
@@ -39,6 +41,15 @@ runs:
             sudo chown -RL 1001:1001 /cache
           fi
 
+          if [ -n "$SSH_PRIVATE_KEY" ]; then
+            # The following adds the SSH private key to the ssh-agent such that CI can SSH into the backup pod.
+            test -z "${SSH_AUTH_SOCK:-}" && { eval "$(ssh-agent -s)"; ssh-add - <<< "${SSH_PRIVATE_KEY}"; }
+            rm -rf ~/.ssh
+            mkdir -p ~/.ssh
+            chmod 0700 ~/.ssh
+            echo -e "Host *\nUser github-runner\n" > ~/.ssh/config
+          fi
+
           ${GITHUB_WORKSPACE}/gitlab-ci/src/bazel-ci/main.sh
           BAZEL_EXIT_CODE="$?"
 
@@ -55,3 +66,4 @@ runs:
           BAZEL_STARTUP_ARGS: ${{ inputs.BAZEL_STARTUP_ARGS }}
           RUN_ON_DIFF_ONLY: ${{ inputs.RUN_ON_DIFF_ONLY }}
           HONEYCOMB_API_TOKEN: ${{ inputs.HONEYCOMB_API_TOKEN }}
+          SSH_PRIVATE_KEY: ${{ inputs.SSH_PRIVATE_KEY }}

.github/workflows/schedule-daily.yml

@@ -17,6 +17,10 @@ env:
   CI_PROJECT_DIR: ${{ github.workspace }}
   CI_MERGE_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
   ROOT_PIPELINE_ID: ${{ github.run_id }}
+  DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
+  DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
+  BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
+  BUILDEVENT_DATASET: "github-ci-dfinity"
 
 jobs:
   # TODO: there was some issue with this job so commented out for now:
@@ -100,3 +104,29 @@ jobs:
           BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
           ZH2_DLL01_CSV_SECRETS: "${{ secrets.ZH2_DLL01_CSV_SECRETS }}"
           ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}"
+
+  nns-tests-nightly:
+    name: NNS Tests Nightly
+    runs-on:
+      group: dind-large
+    container:
+      image: ghcr.io/dfinity/ic-build@sha256:5bd0f059ad6e21966e9c644516b6ecd52d5ed44f1b18a76b91b59740a9d639a2
+    timeout-minutes: 20
+    if: ${{ vars.RUN_CI == 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Before script
+        id: before-script
+        shell: bash
+        run: ./gitlab-ci/src/ci-scripts/before-script.sh
+      - name: Run NNS Tests Nightly
+        id: bazel-test-all
+        uses: ./.github/actions/bazel-test-all/
+        with:
+          BAZEL_COMMAND: "test"
+          BAZEL_TARGETS: "//rs/nns/..."
+          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
+          BAZEL_EXTRA_ARGS: "--keep_going --verbose_failures --test_tag_filters=nns_tests_nightly --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all"
+          HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }}
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

rs/nns/integration_tests/BUILD.bazel

@@ -391,7 +391,8 @@ rust_ic_test(
     env = DEV_ENV,
     proc_macro_deps = MACRO_DEPENDENCIES + MACRO_DEV_DEPENDENCIES,
     tags = [
-        "manual",  # CI should not be downloading nns_state.tar.zst.
+        "nns_tests_nightly",  # Run this test in the nns-tests-nightly GitHub Action job.
+        "no-sandbox",  # such that the test can access the file $SSH_AUTH_SOCK.
         "requires-network",  # Because mainnet state is downloaded (and used).
     ],
     deps = DEPENDENCIES + DEV_DEPENDENCIES,

rs/nns/integration_tests/src/upgrade_canisters_with_golden_nns_state.rs

@@ -16,7 +16,6 @@ use ic_nns_test_utils::{
     },
 };
 use ic_nns_test_utils_golden_nns_state::new_state_machine_with_golden_nns_state_or_panic;
-use std::str::FromStr;
 use std::{
     env,
     fmt::{Debug, Formatter},