chore(system-tests-k8s): misc changes (#3541)

* remove docker login
* run system tests 4 times during the night time
* also run longer system tests on manual dispatch
* set concurrency

.github/workflows/system-tests-k8s.yml

@@ -8,7 +8,10 @@ name: System Tests K8s
 
 on:
   schedule:
-    - cron: "0 3 * * *"
+    - cron: "0 1 * * *" # Run at 1 AM
+    - cron: "0 3 * * *" # Run at 3 AM
+    - cron: "0 5 * * *" # Run at 5 AM
+    - cron: "0 7 * * *" # Run at 7 AM
   pull_request:
     paths:
       - '.github/workflows/system-tests-k8s.yml'
@@ -23,6 +26,10 @@ on:
         required: false
         default: '10'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
 env:
   TARGETS: |
     ${{ github.event_name == 'schedule' && '//...' ||
@@ -108,20 +115,13 @@ jobs:
         --privileged --cgroupns host
         -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
     timeout-minutes: 150
+    # always run after 'system-tests-k8s' job only on manual dispatch and schedule
+    if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
     needs: [system-tests-k8s]
-    # always run after 'system-tests-k8s' job but on schedule event only
-    if: ${{ always() && github.event_name == 'schedule' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Login to Dockerhub
-        shell: bash
-        run: ./ci/scripts/docker-login.sh
-        env:
-          DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
-          DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
-
       - name: Set KUBECONFIG
         shell: bash
         run: |
@@ -158,7 +158,7 @@ jobs:
           BAZEL_COMMAND: "test"
           BAZEL_TARGETS: "${{ env.TARGETS }}"
           BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
-          BAZEL_EXTRA_ARGS: "--local_test_jobs=${{ env.JOBS }} --test_tag_filters=k8s --k8s"
+          BAZEL_EXTRA_ARGS: "--local_test_jobs=${{ env.JOBS }} --test_tag_filters=k8s --k8s --flaky_test_attempts=3"
           BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
 
       - name: Upload bazel-bep