Merge branch 'master' into maciej-icrc-v4

dfinity · Dec 11, 2024 · 8a3bf31 · 8a3bf31
2 parents 98f81db + b006ae9
commit 8a3bf31
Show file tree

Hide file tree

Showing 426 changed files with 12,828 additions and 100,862 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -97,7 +97,7 @@ go_deps.bzl               @dfinity/idx
 /rs/bitcoin/mock/                                       @dfinity/cross-chain-team
 /rs/bitcoin/client/                                     @dfinity/networking
 /rs/bitcoin/consensus/                                  @dfinity/execution @dfinity/consensus
-/rs/bitcoin/kyt/                                        @dfinity/cross-chain-team
+/rs/bitcoin/checker/                                    @dfinity/cross-chain-team
 /rs/bitcoin/service/                                    @dfinity/networking
 /rs/bitcoin/replica_types/                              @dfinity/execution
 /rs/bitcoin/validation                                  @dfinity/networking @dfinity/execution
@@ -209,6 +209,7 @@ go_deps.bzl               @dfinity/idx
 /rs/rust_canisters/response_payload_test/               @dfinity/execution
 /rs/rust_canisters/stable_structures/                   @dfinity/execution
 /rs/rust_canisters/stable_memory_integrity              @dfinity/execution
+/rs/rust_canisters/statesync_test                       @dfinity/ic-message-routing-owners
 /rs/rust_canisters/canister_creator                     @dfinity/execution
 /rs/rust_canisters/load_simulator                       @dfinity/execution
 /rs/rust_canisters/xnet_test/                           @dfinity/ic-message-routing-owners
@@ -235,28 +236,25 @@ go_deps.bzl               @dfinity/idx
 /rs/test_utilities/state/                               @dfinity/execution @dfinity/ic-message-routing-owners
 /rs/test_utilities/types/src/batch/                     @dfinity/consensus
 /rs/tests/                                              @dfinity/idx
-/rs/tests/research                                      @dfinity/research @dfinity/idx
-/rs/tests/dashboards/IC/execution-metrics.json          @dfinity/execution @dfinity/idx
-/rs/tests/dashboards/IC/bitcoin.json                    @dfinity/execution @dfinity/idx
-/rs/tests/driver/src/driver/simulate_network.rs         @dfinity/networking
-/rs/tests/boundary_nodes/                               @dfinity/boundary-node @dfinity/idx
-/rs/tests/ckbtc/                                        @dfinity/cross-chain-team @dfinity/idx
-/rs/tests/consensus/                                    @dfinity/consensus @dfinity/idx
-/rs/tests/cross_chain/                                  @dfinity/cross-chain-team @dfinity/idx
-/rs/tests/crypto/                                       @dfinity/crypto-team @dfinity/idx
-/rs/tests/dre/                                          @dfinity/dre @dfinity/idx
-/rs/tests/execution/                                    @dfinity/execution @dfinity/idx
-/rs/tests/financial_integrations/                       @dfinity/finint @dfinity/idx
-/rs/tests/message_routing/                              @dfinity/ic-message-routing-owners @dfinity/idx
-/rs/tests/networking/                                   @dfinity/networking @dfinity/idx
-/rs/tests/nns/                                          @dfinity/nns-team @dfinity/idx
-/rs/tests/node/                                         @dfinity/node @dfinity/idx
-/rs/tests/query_stats/                                  @dfinity/execution @dfinity/consensus @dfinity/idx
-/rs/tests/sdk/                                          @dfinity/sdk @dfinity/idx
-/rs/tests/src/ledger_tests/                             @dfinity/finint  @dfinity/idx
-/rs/tests/src/nns_tests/                                @dfinity/nns-team @dfinity/idx
-/rs/tests/src/rosetta_test.rs                           @dfinity/finint @dfinity/idx
-/rs/tests/src/rosetta_tests/                            @dfinity/finint @dfinity/idx
+/rs/tests/research                                      @dfinity/research
+/rs/tests/driver/src/driver/simulate_network.rs         @dfinity/networking @dfinity/idx
+/rs/tests/boundary_nodes/                               @dfinity/boundary-node
+/rs/tests/ckbtc/                                        @dfinity/cross-chain-team
+/rs/tests/consensus/                                    @dfinity/consensus
+/rs/tests/cross_chain/                                  @dfinity/cross-chain-team
+/rs/tests/crypto/                                       @dfinity/crypto-team
+/rs/tests/dre/                                          @dfinity/dre
+/rs/tests/execution/                                    @dfinity/execution
+/rs/tests/financial_integrations/                       @dfinity/finint
+/rs/tests/message_routing/                              @dfinity/ic-message-routing-owners
+/rs/tests/networking/                                   @dfinity/networking
+/rs/tests/nns/                                          @dfinity/nns-team
+/rs/tests/node/                                         @dfinity/node
+/rs/tests/query_stats/                                  @dfinity/execution @dfinity/consensus
+/rs/tests/sdk/                                          @dfinity/sdk
+/rs/tests/src/ledger_tests/                             @dfinity/finint
+/rs/tests/src/rosetta_test.rs                           @dfinity/finint
+/rs/tests/src/rosetta_tests/                            @dfinity/finint
 /rs/tests/k8s/                                          @dfinity/idx @dfinity/node
 /rs/tla_instrumentation/                                @dfinity/research @dfinity/formal-models
 /rs/tools/                                              @dfinity/ic-interface-owners

diff --git a/.github/workflows-source/ci-main.yml b/.github/workflows-source/ci-main.yml
@@ -97,6 +97,12 @@ jobs:
     runs-on:
       group: zh1
       labels: dind-large
+    env:
+      AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
+      # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
+      OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
+      CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
+      RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
     steps:
       - <<: *checkout
       - <<: *docker-login
@@ -129,7 +135,7 @@ jobs:
           TEST_TAG_FILTERS=$(IFS=,; echo "${EXCLUDED_TEST_TAGS[*]/#/-}")
           # Determine BAZEL_EXTRA_ARGS based on event type or branch name
           BAZEL_EXTRA_ARGS="--test_tag_filters=$TEST_TAG_FILTERS"
-          if [[ "${{ github.event_name }}" == 'merge_group' ]]; then
+          if [[ "$CI_EVENT_NAME" == 'merge_group' ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate --flaky_test_attempts=3"
           elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate"
@@ -138,17 +144,9 @@ jobs:
           fi
           # Export BAZEL_EXTRA_ARGS to environment
           echo "BAZEL_EXTRA_ARGS=$BAZEL_EXTRA_ARGS" >> $GITHUB_ENV
-        env:
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
       - name: Run Bazel Test All
         id: bazel-test-all
         uses: ./.github/actions/bazel-test-all/
-        env:
-          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
-          # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
-          OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
-          CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
         with:
           BAZEL_COMMAND: "test"
           BAZEL_TARGETS: "//..."

diff --git a/.github/workflows-source/schedule-daily.yml b/.github/workflows-source/schedule-daily.yml
@@ -163,6 +163,14 @@ jobs:
           BAZEL_EXTRA_ARGS: "--test_tag_filters=system_test_benchmark --//bazel:enable_upload_perf_systest_results=True --keep_going --jobs 1"
           BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
       - <<: *bazel-bep
+      - name: Post Slack Notification
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: failure()
+        with:
+          channel-id: eng-crypto-alerts
+          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
 
   dependency-scan-nightly:
     name: Dependency Scan Nightly

diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml
@@ -37,6 +37,12 @@ jobs:
     runs-on:
       group: zh1
       labels: dind-large
+    env:
+      AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
+      # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
+      OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
+      CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
+      RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -77,7 +83,7 @@ jobs:
           TEST_TAG_FILTERS=$(IFS=,; echo "${EXCLUDED_TEST_TAGS[*]/#/-}")
           # Determine BAZEL_EXTRA_ARGS based on event type or branch name
           BAZEL_EXTRA_ARGS="--test_tag_filters=$TEST_TAG_FILTERS"
-          if [[ "${{ github.event_name }}" == 'merge_group' ]]; then
+          if [[ "$CI_EVENT_NAME" == 'merge_group' ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate --flaky_test_attempts=3"
           elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate"
@@ -86,17 +92,9 @@ jobs:
           fi
           # Export BAZEL_EXTRA_ARGS to environment
           echo "BAZEL_EXTRA_ARGS=$BAZEL_EXTRA_ARGS" >> $GITHUB_ENV
-        env:
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
       - name: Run Bazel Test All
         id: bazel-test-all
         uses: ./.github/actions/bazel-test-all/
-        env:
-          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
-          # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
-          OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
-          CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
         with:
           BAZEL_COMMAND: "test"
           BAZEL_TARGETS: "//..."

diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml
@@ -198,6 +198,14 @@ jobs:
           path: |
             bazel-bep.pb
             profile.json
+      - name: Post Slack Notification
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: failure()
+        with:
+          channel-id: eng-crypto-alerts
+          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
   dependency-scan-nightly:
     name: Dependency Scan Nightly
     runs-on:

diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml
@@ -6,13 +6,11 @@ on:
   workflow_dispatch:
 
 env:
-  BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   CI_COMMIT_SHA: ${{ github.sha }}
   CI_JOB_NAME: ${{ github.job }}
   CI_PROJECT_DIR: ${{ github.workspace }}
   CI_RUN_ID: ${{ github.run_id }}
-  BUILDEVENT_DATASET: "github-ci-dfinity"
 
 jobs:
 
@@ -46,6 +44,7 @@ jobs:
           BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
           BAZEL_COMMAND: "run"
           BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
+          CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
           RUST_BACKTRACE: "full"
           TARGETS: ${{ matrix.targets }}
diff --git a/.github/workflows/slack-workflow-run.yml b/.github/workflows/slack-workflow-run.yml
@@ -55,6 +55,11 @@ jobs:
             POST_TO_SLACK="true"
           fi
 
+          # Non-IDX alerts
+          if [[ "$TRIGGERING_WORKFLOW_NAME" == "Schedule Rust Benchmarks" ]]; then
+            CHANNEL="eng-crypto-alerts"
+          fi
+
           echo "channel=${CHANNEL}" >> $GITHUB_OUTPUT
           echo "message=${FULL_MESSAGE}" >> $GITHUB_OUTPUT
           echo "post_to_slack=${POST_TO_SLACK}" >> $GITHUB_OUTPUT

diff --git a/.github/workflows/system-tests-k8s.yml b/.github/workflows/system-tests-k8s.yml
@@ -7,11 +7,12 @@ name: System Tests K8s
 #   --field jobs="32"
 
 on:
-  schedule:
-    - cron: "0 3 * * *"
-  pull_request:
-    paths:
-      - '.github/workflows/system-tests-k8s.yml'
+  # XXX: temporarily disabling until we have kubevirt 1.4.0
+  # schedule:
+  #   - cron: "0 3 * * *"
+  # pull_request:
+  #   paths:
+  #     - '.github/workflows/system-tests-k8s.yml'
   workflow_dispatch:
     inputs:
       targets:

diff --git a/.github/workflows/update-mainnet-revisions.yaml b/.github/workflows/update-mainnet-revisions.yaml
@@ -0,0 +1,32 @@
+name: Update IC versions file
+
+on:
+  schedule:
+    - cron: "10 * * * *"
+  workflow_dispatch:
+
+jobs:
+  update-ic-versions-file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          token: ${{ steps.app-token.outputs.token }}
+
+      - name: Update IC versions file
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -eEuxo pipefail
+
+          time python ci/src/mainnet_revisions/mainnet_revisions.py
+        shell: bash
diff --git a/.gitignore b/.gitignore
@@ -86,8 +86,6 @@ dev-root-ca.crt
 cpp/**/infogetty
 cpp/**/infogetty.o
 cpp/**/network_info.o
-cpp/**/prestorecon
-cpp/**/prestorecon.o
 
 # Various build artifacts
 artifacts