Merge branch 'master' into marko-remove-docker-login

.bazelignore

@@ -0,0 +1 @@
+target

.bazelversion

@@ -1 +1 @@
-7.3.1
+7.4.1

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec",
+  "image": "ghcr.io/dfinity/ic-build@sha256:4fd13b47285e783c3a6f35aadd9559d097c0de162a1cf221ead66ab1598d5d45",
   "remoteUser": "ubuntu",
   "privileged": true,
   "runArgs": [
@@ -14,7 +14,11 @@
   ],
   "workspaceMount": "source=${localWorkspaceFolder},target=/ic,type=bind",
   "workspaceFolder": "/ic",
-  "initializeCommand": "mkdir -p ~/.aws ~/.ssh ~/.cache ~/.local/share/fish && touch ~/.zsh_history ~/.bash_history",
+  "initializeCommand": "mkdir -p ~/.aws ~/.ssh ~/.cache/cargo ~/.local/share/fish && touch ~/.zsh_history ~/.bash_history",
+  "containerEnv": {
+    "CARGO_TARGET_DIR": "/home/ubuntu/.cache/cargo",
+    "USER": "${localEnv:USER}"
+  },
   "mounts": [
     {
       "source": "${localEnv:HOME}/.cache",
@@ -54,8 +58,17 @@
         "vadimcn.vscode-lldb",
         "bazelbuild.vscode-bazel",
         "stackbuild.bazel-stack-vscode",
-        "bungcip.better-toml"
-      ]
+        "tamasfe.even-better-toml"
+      ],
+      "settings": {
+        // Exclude bazel output directories from search
+        "search.exclude": {
+          "bazel-bin/": true,
+          "bazel-ic/": true,
+          "bazel-out/": true,
+          "targets/": true
+        }
+      }
     }
   }
-}
+}

.github/CI_README.md

@@ -17,3 +17,14 @@ Due to some limitations of GitHub Actions CI, we need to generate our own CI yam
 1. Add your new workflow to `workflow-source`. Include any anchors you would like to use under the block `anchors`. If you name it something else, it will break.
 1. Push your changes to GitHub which will trigger CI. This will automatically run a custom script (`generate-ci.py`) which will generate the full yaml file from your anchors, as well as delete the `anchors` block, as this will not work for github actions. This new yaml file will automatically be placed in the `workflows` directory.
 1. Check that this new workflow file is correct.
+
+## Using GitHub Apps in CI
+Our CI contains several automated steps to either generate commits or PRs, usually for linting or version updates. Previously we used Personal Access Tokens generated by a service account to create a commit or PR with the correct permissions but we have now moved to GitHub apps. These have several benefits:
+- they generate a token which is only valid while the workflow is running, enhancing security and removing the need for regular rotation
+- they can be managed via the organization and do not require a separate service account
+- they can be installed on mutliple repos, however they only have access to the repo they are installed in
+
+## Automated PRs by Bots
+To increase security and prevent a bot from accidentally changing files that it shouldn't, we are introducing a new config file stored in `.github/repo_policies/bot_approved_files.json` in all public repos where any files changed by a bot must be listed. If not, the workflow will fail and the PR can't be merged.
+
+Workflow defined in: https://github.com/dfinity/public-workflows/blob/main/.github/workflows/repo_policies.yml

.github/actions/bazel-test-all/action.yaml

@@ -30,11 +30,6 @@ runs:
         run: |
           set +e # manual error handling to ensure we can run some post-build commands
 
-          # temporarily set permissions again until we can figure out issue
-          if [ -e /cache ]; then
-            sudo chown -RL 1001:1001 /cache
-          fi
-
           if [ -n "$SSH_PRIVATE_KEY" ]; then
             # The following adds the SSH private key to the ssh-agent such that CI can SSH into the backup pod.
             test -z "${SSH_AUTH_SOCK:-}" && { eval "$(ssh-agent -s)"; ssh-add - <<< "${SSH_PRIVATE_KEY}"; }
@@ -58,15 +53,20 @@ runs:
               rm "$exportout"
           fi
 
+          # output node name to gihub step summary
+          [ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
+
           exit "$BAZEL_EXIT_CODE"
         env:
           BAZEL_COMMAND: ${{ inputs.BAZEL_COMMAND }}
           BAZEL_TARGETS: ${{ inputs.BAZEL_TARGETS }}
           BAZEL_CI_CONFIG: ${{ inputs.BAZEL_CI_CONFIG }}
           BAZEL_EXTRA_ARGS: ${{ inputs.BAZEL_EXTRA_ARGS }}
           BAZEL_STARTUP_ARGS: ${{ inputs.BAZEL_STARTUP_ARGS }}
-          CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
-          MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           BUILDEVENT_APIKEY: ${{ inputs.BUILDEVENT_APIKEY }}
+          CI_EVENT_NAME: ${{ github.event_name }}
+          CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
+          MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           SSH_PRIVATE_KEY: ${{ inputs.SSH_PRIVATE_KEY }}

.github/repo_policies/BOT_APPROVED_FILES

@@ -0,0 +1,8 @@
+# List of approved files that can be changed by a bot via an automated PR
+# This is to increase security and prevent accidentally updating files that shouldn't be changed by a bot
+
+ic-os/boundary-guestos/context/docker-base.*
+ic-os/guestos/context/docker-base.*
+ic-os/hostos/context/docker-base.*
+ic-os/setupos/context/docker-base.*
+mainnet-*-revisions.json

.github/workflows-source/ci-main.yml

@@ -23,8 +23,6 @@ concurrency:
 env:
   CI_COMMIT_SHA: ${{ github.sha }}
   CI_JOB_NAME: ${{ github.job }}
-  CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-  CI_PIPELINE_SOURCE: ${{ github.event_name }}
   CI_PROJECT_DIR: ${{ github.workspace }}
   CI_EVENT_NAME: ${{ github.event_name }}
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
@@ -34,7 +32,7 @@ env:
 
 anchors:
   image: &image
-    image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
+    image: ghcr.io/dfinity/ic-build@sha256:4fd13b47285e783c3a6f35aadd9559d097c0de162a1cf221ead66ab1598d5d45
   dind-large-setup: &dind-large-setup
     runs-on:
       labels: dind-large
@@ -98,6 +96,12 @@ jobs:
     runs-on:
       group: zh1
       labels: dind-large
+    env:
+      AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
+      # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
+      OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
+      CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
+      RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
     steps:
       - <<: *checkout
       - <<: *before-script
@@ -130,7 +134,7 @@ jobs:
           TEST_TAG_FILTERS=$(IFS=,; echo "${EXCLUDED_TEST_TAGS[*]/#/-}")
           # Determine BAZEL_EXTRA_ARGS based on event type or branch name
           BAZEL_EXTRA_ARGS="--test_tag_filters=$TEST_TAG_FILTERS"
-          if [[ "${{ github.event_name }}" == 'merge_group' ]]; then
+          if [[ "$CI_EVENT_NAME" == 'merge_group' ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate --flaky_test_attempts=3"
           elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
               BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate"
@@ -139,17 +143,9 @@ jobs:
           fi
           # Export BAZEL_EXTRA_ARGS to environment
           echo "BAZEL_EXTRA_ARGS=$BAZEL_EXTRA_ARGS" >> $GITHUB_ENV
-        env:
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
       - name: Run Bazel Test All
         id: bazel-test-all
         uses: ./.github/actions/bazel-test-all/
-        env:
-          AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
-          # Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
-          OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
-          CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
-          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
         with:
           BAZEL_COMMAND: "test"
           BAZEL_TARGETS: "//..."
@@ -163,6 +159,7 @@ jobs:
   bazel-build-all-config-check:
     <<: *dind-large-setup
     name: Bazel Build All Config Check
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI_BUILD_CHECK') }}
     steps:
       - <<: *checkout
       - <<: *before-script
@@ -181,7 +178,11 @@ jobs:
     runs-on:
       labels: macOS
     # Run on protected branches, but only on public repo
-    if: ${{ github.ref_protected && github.repository == 'dfinity/ic' }}
+    # Allow running if CI_MACOS_INTEL label is used
+    if: |
+      (github.ref_protected && github.repository == 'dfinity/ic') ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'CI_MACOS_INTEL'))
+
     steps:
       - <<: *checkout
       - name: Set PATH
@@ -212,7 +213,6 @@ jobs:
     <<: *dind-large-setup
     steps:
       - <<: *checkout
-      - <<: *before-script
       - name: Run Bazel Build Fuzzers
         id: bazel-build-fuzzers
         uses:  ./.github/actions/bazel-test-all/
@@ -228,7 +228,6 @@ jobs:
     <<: *dind-large-setup
     steps:
       - <<: *checkout
-      - <<: *before-script
       - name: Run Bazel Build Fuzzers AFL
         id: bazel-build-fuzzers-afl
         uses:  ./.github/actions/bazel-test-all/
@@ -251,7 +250,8 @@ jobs:
         run: |
           set -xeuo pipefail
           export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
-          pip3 install --ignore-installed -r requirements.txt
+          # Ignore externally-managed-environment pip error, install packages system-wide.
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
           cd ci/src
           pytest -m "not fails_on_merge_train" -v -o junit_family=xunit1 \
             --junitxml=../../test_report.xml --cov=. --cov-report=term \
@@ -264,6 +264,10 @@ jobs:
   build-ic:
     name: Build IC
     <<: *dind-large-setup
+    # keep options from dind-large-setup but run on dind-small-setup
+    runs-on:
+      group: ch1
+      labels: dind-small
     if: ${{ github.event_name != 'merge_group' }}
     steps:
       - <<: *checkout
@@ -273,6 +277,7 @@ jobs:
         shell: bash
         run: |
           set -eExuo pipefail
+          [ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
           REPO_NAME="${GITHUB_REPOSITORY##*/}"
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
           mkdir -p "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}/artifacts"
@@ -348,7 +353,9 @@ jobs:
 
   cargo-clippy-linux:
     name: Cargo Clippy Linux
-    <<: *dind-large-setup
+    <<: *dind-small-setup
+    runs-on:
+      group: ch1
     steps:
       - <<: *checkout
       - name: Filter Rust Files [*.{rs,toml,lock}]
@@ -379,7 +386,9 @@ jobs:
 
   cargo-build-release-linux:
     name: Cargo Build Release Linux
-    <<: *dind-large-setup
+    <<: *dind-small-setup
+    runs-on:
+      group: ch1
     steps:
       - <<: *checkout
       - name: Filter Rust Files [*.{rs,toml,lock}]
@@ -407,21 +416,3 @@ jobs:
           set -eExuo pipefail
           buildevents cmd "$CI_RUN_ID" "$CI_JOB_NAME" build-command -- \
               cargo build --release --locked
-
-  # CI job is also executed in Schedule Hourly
-  bazel-test-coverage:
-    name: Bazel Test Coverage
-    <<: *dind-large-setup
-    if: |
-      github.event_name == 'pull_request' &&
-      contains(github.event.pull_request.labels.*.name, 'CI_COVERAGE')
-    steps:
-      - <<: *checkout
-      - <<: *before-script
-      - name: Run Bazel Test Coverage
-        shell: bash
-        run: |
-          ./ci/scripts/bazel-coverage.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows-source/ci-pr-only.yml

@@ -2,6 +2,7 @@ name: CI PR Only
 # Jobs that run on PRs, but no other pipelines
 
 on:
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -10,14 +11,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 env:
-  CI_PIPELINE_SOURCE: ${{ github.event_name }}
   CI_PROJECT_DIR: ${{ github.workspace }}
   MERGE_BRANCH: ${{ github.event.pull_request.base.ref }}
   ORG: ${{ github.repository_owner }}
 
 anchors:
   image: &image
-    image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
+    image: ghcr.io/dfinity/ic-build@sha256:4fd13b47285e783c3a6f35aadd9559d097c0de162a1cf221ead66ab1598d5d45
   dind-small-setup: &dind-small-setup
     timeout-minutes: 30
     runs-on:
@@ -34,23 +34,19 @@ anchors:
       <<: *image
       options: >-
         -e NODE_NAME
-  before-script: &before-script
-    name: Before script
-    id: before-script
-    shell: bash
-    run: |
-      [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
   checkout: &checkout
     name: Checkout
     uses: actions/checkout@v4
+  skip-merge-group: &skip-merge-group
+    if: ${{ github.event_name != 'merge_group' }}
 
 jobs:
   bazel-build-fuzzers-archives:
     name: Bazel Build Fuzzers Archives
     <<: *dind-large-setup
+    <<: *skip-merge-group
     steps:
       - <<: *checkout
-      - <<: *before-script
       - name: Filter Relevant Files
         uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
         id: filter
@@ -83,13 +79,21 @@ jobs:
   lock-generate:
     name: Lock Generate
     <<: *dind-small-setup
+    <<: *skip-merge-group
+    env:
+      CI_EVENT_NAME: ${{ github.event_name }}
     steps:
+      - name: Create GitHub App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }}
+          private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }}
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
-          token: ${{ secrets.IDX_PUSH_TO_PR }}
-      - <<: *before-script
+          token: ${{ steps.app-token.outputs.token }}
       - name: Filter Relevant Files
         uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
         id: filter
@@ -112,6 +116,7 @@ jobs:
   dependencies-check:
     name: Dependency Scan for PR
     <<: *dind-small-setup
+    <<: *skip-merge-group
     timeout-minutes: 60
     permissions:
       contents: read
@@ -130,7 +135,6 @@ jobs:
       - <<: *checkout
         with:
           fetch-depth: 256
-      - <<: *before-script
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -139,7 +143,8 @@ jobs:
         id: setup-python-deps
         shell: bash
         run: |
-          pip3 install --ignore-installed -r requirements.txt
+          # Ignore externally-managed-environment pip error, install packages system-wide.
+          PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --ignore-installed -r requirements.txt
       - name: Dependency Scan for Pull Request
         id: dependencies-check
         shell: bash
@@ -148,3 +153,20 @@ jobs:
           export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
           cd ci/src/dependencies/
           $SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_merge_job.py
+
+  # CI job is also executed in Schedule Hourly
+  bazel-test-coverage:
+    name: Bazel Test Coverage
+    <<: *dind-large-setup
+    <<: *skip-merge-group
+    if: contains(github.event.pull_request.labels.*.name, 'CI_COVERAGE')
+    steps:
+      - <<: *checkout
+      - name: Run Bazel Test Coverage
+        shell: bash
+        run: |
+          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
+          ./ci/scripts/bazel-coverage.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}