Merge branch 'main' into robertomonteromiguel/prune_ghcr

DataDog · May 24, 2024 · c8d6ec9 · c8d6ec9
2 parents 5ed95d5 + 80e5bb8
commit c8d6ec9
Show file tree

Hide file tree

Showing 195 changed files with 3,319 additions and 2,200 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -76,23 +76,57 @@ jobs:
     name: Get impacted libraries
     uses: ./.github/workflows/compute-impacted-libraries.yml
 
+  get_dev_artifacts:
+    needs:
+    - impacted_libraries
+    strategy:
+      matrix:
+        library: ${{ fromJson(needs.impacted_libraries.outputs.impacted_libraries) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get library artifact
+        run: ./utils/scripts/load-binary.sh ${{ matrix.library }}
+      - name: Get agent artifact
+        run: ./utils/scripts/load-binary.sh agent
+      # ### appsec-event-rules is now a private repo. The GH_TOKEN provided can't read private repos.
+      # ### skipping this, waiting for a proper solution
+      # - name: Load WAF rules
+      #   if: matrix.version == 'dev'
+      #   run: ./utils/scripts/load-binary.sh waf_rule_set
+      #   env:
+      #     GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries_dev_${{ matrix.library }}
+          path: binaries/
+
   system_tests:
     name: System Tests
     needs:
     - lint
     - test_the_test
     - scenarios
     - impacted_libraries
+    - get_dev_artifacts
     strategy:
       matrix:
         library: ${{ fromJson(needs.impacted_libraries.outputs.impacted_libraries) }}
+        version:
+        - prod
+        - dev
       fail-fast: false
     uses: ./.github/workflows/system-tests.yml
     secrets: inherit
     with:
       library: ${{ matrix.library }}
       scenarios: ${{ needs.scenarios.outputs.scenarios }}
       scenarios_groups: ${{ needs.scenarios.outputs.scenarios_groups }}
+      binaries_artifact: ${{ matrix.version == 'dev' && format('binaries_dev_{0}', matrix.library) || '' }}
+      ci_environment: ${{ matrix.version }}
       build_python_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') }}
       build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
@@ -104,6 +138,7 @@ jobs:
     - lint
     - test_the_test
     - impacted_libraries
+    - get_dev_artifacts  # non official set-up, this needs put this job in last
     if: contains(needs.impacted_libraries.outputs.impacted_libraries, 'ruby')
     uses: ./.github/workflows/run-docker-mode.yml
     secrets: inherit

diff --git a/.github/workflows/run-docker-mode.yml b/.github/workflows/run-docker-mode.yml
@@ -87,15 +87,15 @@ jobs:
             path: dd-trace-rb
         app:
           - rack
-          - sinatra14
-          - sinatra20
+          # - sinatra14
+          # - sinatra20
           - sinatra21
-          - rails50
-          - rails51
-          - rails52
-          - rails60
-          - rails61
-          - rails70
+          # - rails50
+          # - rails51
+          # - rails52
+          # - rails60
+          # - rails61
+          # - rails70
           - rails71
     runs-on: ubuntu-latest
     name: Build (${{ matrix.app }})
@@ -164,79 +164,79 @@ jobs:
           - ruby
         app:
           - rack
-          - sinatra14
-          - sinatra20
+          # - sinatra14
+          # - sinatra20
           - sinatra21
-          - rails50
-          - rails51
-          - rails52
-          - rails60
-          - rails61
-          - rails70
+          # - rails50
+          # - rails51
+          # - rails52
+          # - rails60
+          # - rails61
+          # - rails70
           - rails71
         scenario:
           - DEFAULT
-          - APPSEC_DISABLED
-          - APPSEC_BLOCKING_FULL_DENYLIST
-          - APPSEC_REQUEST_BLOCKING
+          # - APPSEC_DISABLED
+          # - APPSEC_BLOCKING_FULL_DENYLIST
+          # - APPSEC_REQUEST_BLOCKING
         include:
           - library: ruby
             app: rack
             scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
-          - library: ruby
-            app: rack
-            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
-          - library: ruby
-            app: rack
-            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
-          - library: ruby
-            app: rack
-            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
-          - library: ruby
-            app: rack
-            scenario: APPSEC_CUSTOM_RULES
-          - library: ruby
-            app: rack
-            scenario: APPSEC_MISSING_RULES
-          - library: ruby
-            app: rack
-            scenario: APPSEC_CORRUPTED_RULES
-          - library: ruby
-            app: rack
-            scenario: APPSEC_LOW_WAF_TIMEOUT
-          - library: ruby
-            app: rack
-            scenario: APPSEC_CUSTOM_OBFUSCATION
-          - library: ruby
-            app: rack
-            scenario: APPSEC_RATE_LIMITER
-          - library: ruby
-            app: rails70
-            scenario: APPSEC_AUTO_EVENTS_EXTENDED
-          - library: ruby
-            app: rails70
-            scenario: APPSEC_API_SECURITY
-          - library: ruby
-            app: rack
-            scenario: APPSEC_RULES_MONITORING_WITH_ERRORS
-          - library: ruby
-            app: rack
-            scenario: SAMPLING
-          - library: ruby
-            app: rack
-            scenario: PROFILING
-          - library: ruby
-            app: rack
-            scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
-          - library: ruby
-            app: rack
-            scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED
-          - library: ruby
-            app: rack
-            scenario: TELEMETRY_LOG_GENERATION_DISABLED
-          - library: ruby
-            app: rack
-            scenario: TELEMETRY_METRIC_GENERATION_DISABLED
+          # - library: ruby
+          #   app: rack
+          #   scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+          # - library: ruby
+          #   app: rack
+          #   scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
+          # - library: ruby
+          #   app: rack
+          #   scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_CUSTOM_RULES
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_MISSING_RULES
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_CORRUPTED_RULES
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_LOW_WAF_TIMEOUT
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_CUSTOM_OBFUSCATION
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_RATE_LIMITER
+          # - library: ruby
+          #   app: rails70
+          #   scenario: APPSEC_AUTO_EVENTS_EXTENDED
+          # - library: ruby
+          #   app: rails70
+          #   scenario: APPSEC_API_SECURITY
+          # - library: ruby
+          #   app: rack
+          #   scenario: APPSEC_RULES_MONITORING_WITH_ERRORS
+          # - library: ruby
+          #   app: rack
+          #   scenario: SAMPLING
+          # - library: ruby
+          #   app: rack
+          #   scenario: PROFILING
+          # - library: ruby
+          #   app: rack
+          #   scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
+          # - library: ruby
+          #   app: rack
+          #   scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED
+          # - library: ruby
+          #   app: rack
+          #   scenario: TELEMETRY_LOG_GENERATION_DISABLED
+          # - library: ruby
+          #   app: rack
+          #   scenario: TELEMETRY_METRIC_GENERATION_DISABLED
     runs-on: ubuntu-latest
     needs:
       - build-harness
@@ -285,15 +285,15 @@ jobs:
           - ruby
         app:
           - rack
-          - sinatra14
-          - sinatra20
+          # - sinatra14
+          # - sinatra20
           - sinatra21
-          - rails50
-          - rails51
-          - rails52
-          - rails60
-          - rails61
-          - rails70
+          # - rails50
+          # - rails51
+          # - rails52
+          # - rails60
+          # - rails61
+          # - rails70
           - rails71
     runs-on: ubuntu-latest
     needs: