From 1f3931dc10ccd7a537624db49bd162cea511b2f4 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Fri, 21 Jun 2024 10:16:46 +0300 Subject: [PATCH 1/3] performance tests 675bb6344beb71c0e2232216f323cb10d9ea5149 --- .changes/v0.5.27.md | 3 + .github/actions/setup-yc/action.yml | 25 +++ .github/workflows/test.yml | 83 ++++++-- .github/workflows/yc.yml | 127 ++++++++++++ .mapping.json | 45 ++++ CHANGELOG.md | 4 + README.md | 11 + cli/cli.go | 2 +- performance-test/automation/_agent_create.sh | 190 +++++++++++++++++ performance-test/automation/_agent_delete.sh | 115 +++++++++++ .../automation/_compose_test_create_args.sh | 180 ++++++++++++++++ performance-test/automation/_functions.sh | 155 ++++++++++++++ performance-test/automation/_test_check.sh | 117 +++++++++++ performance-test/automation/_test_run.sh | 194 ++++++++++++++++++ performance-test/automation/_variables.sh | 169 +++++++++++++++ performance-test/automation/agent.sh | 120 +++++++++++ .../automation/default_check_report.sh | 35 ++++ .../automation/default_check_summary.sh | 14 ++ performance-test/automation/s3_upload.sh | 28 +++ performance-test/automation/test.sh | 138 +++++++++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 60 ++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 60 ++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 60 ++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 60 ++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 61 ++++++ .../meta.json | 13 ++ .../payload.json | 1 + .../test-config.yaml | 60 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 62 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 61 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 62 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 61 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 62 ++++++ .../meta.json | 15 ++ .../test-config.yaml | 61 ++++++ 50 files changed, 2644 insertions(+), 15 deletions(-) create mode 100644 .changes/v0.5.27.md create mode 100644 .github/actions/setup-yc/action.yml create mode 100644 .github/workflows/yc.yml create mode 100644 performance-test/automation/_agent_create.sh create mode 100644 performance-test/automation/_agent_delete.sh create mode 100644 performance-test/automation/_compose_test_create_args.sh create mode 100644 performance-test/automation/_functions.sh create mode 100644 performance-test/automation/_test_check.sh create mode 100644 performance-test/automation/_test_run.sh create mode 100644 performance-test/automation/_variables.sh create mode 100644 performance-test/automation/agent.sh create mode 100644 performance-test/automation/default_check_report.sh create mode 100644 performance-test/automation/default_check_summary.sh create mode 100644 performance-test/automation/s3_upload.sh create mode 100644 performance-test/automation/test.sh create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json create mode 100644 performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml create mode 100644 performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json create mode 100644 performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml diff --git a/.changes/v0.5.27.md b/.changes/v0.5.27.md new file mode 100644 index 000000000..48d73e55c --- /dev/null +++ b/.changes/v0.5.27.md @@ -0,0 +1,3 @@ +## v0.5.27 - 2024-06-18 +### Added +* Performance test on release diff --git a/.github/actions/setup-yc/action.yml b/.github/actions/setup-yc/action.yml new file mode 100644 index 000000000..f273e0bf0 --- /dev/null +++ b/.github/actions/setup-yc/action.yml @@ -0,0 +1,25 @@ +name: 'Setup YC tools' +description: 'Setup and configure required tools' +runs: + using: "composite" + steps: + - name: Check secret is set + shell: bash + run: | + if [[ -z "$YC_LT_AUTHORIZED_KEY_JSON" ]]; then echo "YC_LT_AUTHORIZED_KEY_JSON is empty" && exit 1; else echo "YC_LT_AUTHORIZED_KEY_JSON is set"; fi + - name: install utilities + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y curl jq + sudo curl -f -s -LO https://storage.yandexcloud.net/yandexcloud-yc/install.sh + sudo bash install.sh -i /usr/local/yandex-cloud -n + sudo ln -sf /usr/local/yandex-cloud/bin/yc /usr/local/bin/yc + - name: configure yc cli + shell: bash + run: | + echo "$YC_LT_AUTHORIZED_KEY_JSON" > key.json + yc config profile create sa-profile + yc config set service-account-key ./key.json + yc config set format json + yc config set folder-id "$YC_LT_FOLDER_ID" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 320827441..a52b12b35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,15 @@ on: - master - dev +env: + YC_LT_FOLDER_ID: b1gacohsvc2kc4d76tu5 + YC_LT_AUTHORIZED_KEY_JSON: ${{ secrets.YC_LOADTESTING_CI_AUTHORIZED_KEY_JSON }} + YC_LT_TEST_AGENT_FILTER: "name = 'agent-pandora-perf-medium'" + YC_LT_TEST_EXTRA_DESCRIPTION: "GitHub Actions workflow - ${{github.run_id}}" + YC_LT_SKIP_TEST_CHECK: "1" + YC_LT_DATA_BUCKET: ${{ secrets.YC_LT_DATA_BUCKET }} + YC_LT_OUTPUT_DIR: ${{github.workspace}}/performance-test/output + jobs: run-unit-tests: name: Unit Tests @@ -19,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu, macOS] + os: [ubuntu] env: OS: ${{ matrix.os }}-latest GO: ${{ matrix.go-version }} @@ -28,18 +37,64 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.21.x - cache: true - - - name: Test - run: go test -race -coverprofile unit.txt -covermode atomic ./... + - name: Parse to Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_ID: 161082234 + run: | + cat < imbalance.txt + ## Performance tests + - pandora-perf-grpc-300inst-sleep0ms: 12344 + - pandora-perf-grpc-3000inst-sleep0ms: 15555 + EOF + + sed G imbalance.txt > imbalance.md - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 + - name: Update release + id: update_release + uses: tubone24/update_release@v1.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: v0.5.27.alpha13 with: - file: ./unit.txt - flags: unit,${{ matrix.os }},go-${{ matrix.go-version }} - name: unit + body_path: ./imbalance.md + is_append_body: true + +# build-and-upload: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/setup-yc +# - name: Install Go +# uses: actions/setup-go@v3 +# with: +# go-version: 1.21.x +# cache: true +# - name: Test +# run: go test -race -covermode atomic ./... +# - name: Build +# run: | +# export GOOS=linux +# export CGO_ENABLED=0 +# go build -o pandora_perf_2 +# - name: Upload +# run: | +# source performance-test/automation/_functions.sh && source performance-test/automation/_variables.sh; yc_s3_upload ./pandora_perf_2 pandora-perf ${YC_LT_DATA_BUCKET} +# +# test-pandora-perf: +# needs: [ build-and-upload ] +# runs-on: ubuntu-latest +# concurrency: { group: loadtesting } +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/setup-yc +# - name: Run Test HTTP-300inst-sleep0ms +# run: | +# stripped_tag="${{ github.event.release.tag_name }}" +# STRIPPED_TAG=${stripped_tag:1} +# YC_LT_VERBOSE=2 YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms +# - name: Upload Artifacts GRPC 300inst-sleep0ms +# uses: actions/upload-artifact@v4 +# with: +# name: pandora-perf-grpc-300inst-sleep0ms +# path: ${{ env.YC_LT_OUTPUT_DIR }} diff --git a/.github/workflows/yc.yml b/.github/workflows/yc.yml new file mode 100644 index 000000000..dce4b5518 --- /dev/null +++ b/.github/workflows/yc.yml @@ -0,0 +1,127 @@ +name: Performance tests + +on: + release: + types: [ created ] +env: + YC_LT_FOLDER_ID: b1gacohsvc2kc4d76tu5 + YC_LT_AUTHORIZED_KEY_JSON: ${{ secrets.YC_LOADTESTING_CI_AUTHORIZED_KEY_JSON }} + YC_LT_TEST_AGENT_FILTER: "name = 'agent-pandora-perf-medium'" + YC_LT_TEST_EXTRA_DESCRIPTION: "GitHub Actions workflow - ${{github.run_id}}" + YC_LT_SKIP_TEST_CHECK: "1" + YC_LT_DATA_BUCKET: ${{ secrets.YC_LT_DATA_BUCKET }} + YC_LT_OUTPUT_DIR: ${{github.workspace}}/performance-test/output + +jobs: + build-and-upload: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-yc + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.21.x + cache: true + - name: Test + run: go test -race -covermode atomic ./... + - name: Build + run: | + export GOOS=linux + export CGO_ENABLED=0 + go build -o pandora_perf_2 + - name: Upload + run: | + source performance-test/automation/_functions.sh && source performance-test/automation/_variables.sh; yc_s3_upload ./pandora_perf_2 pandora-perf ${YC_LT_DATA_BUCKET} + + test-pandora-perf: + needs: [ build-and-upload ] + runs-on: ubuntu-latest + concurrency: { group: loadtesting } + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-yc + - name: Run Test HTTP-300inst-sleep0ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false + - name: Run Test HTTP-2000inst-sleep0ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false + - name: Run Test HTTP-2000inst-sleep50ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false + - name: Run Test HTTP-300inst-sleep0ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true + - name: Run Test HTTP-2000inst-sleep0ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true + - name: Run Test HTTP-2000inst-sleep50ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true + - name: Run Test GRPC-2000inst-sleep50ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false + - name: Run Test GRPC-2000inst-sleep50ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false + - name: Run Test GRPC-300inst-sleep0ms-overflow-false + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false + - name: Run Test GRPC-2000inst-sleep50ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true + - name: Run Test GRPC-2000inst-sleep50ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true + - name: Run Test GRPC-300inst-sleep0ms-overflow-true + run: | + stripped_tag="${{ github.event.release.tag_name }}" + STRIPPED_TAG=${stripped_tag:1} + YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true + - name: Upload Artifacts GRPC 300inst-sleep0ms + uses: actions/upload-artifact@v4 + with: + name: pandora-perf-grpc-300inst-sleep0ms + path: ${{ env.YC_LT_OUTPUT_DIR }} + + - name: Parse to Release 2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_ID: 161082234 + run: | + cat < imbalance.md + + ## Performance tests + + EOF + find $YC_LT_OUTPUT_DIR -name 'summary.json' -exec jq -r '"- \(.details.name): **\(.summary.imbalance_point.rps // "0" | tonumber)**"' {} \; | sort >> imbalance.md + + - name: Update release + id: update_release + uses: tubone24/update_release@v1.3.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + body_path: ./imbalance.md + is_append_body: true diff --git a/.mapping.json b/.mapping.json index 123e1e713..2b9242b50 100644 --- a/.mapping.json +++ b/.mapping.json @@ -24,9 +24,12 @@ ".changes/v0.5.24.md":"load/projects/pandora/.changes/v0.5.24.md", ".changes/v0.5.25.md":"load/projects/pandora/.changes/v0.5.25.md", ".changes/v0.5.26.md":"load/projects/pandora/.changes/v0.5.26.md", + ".changes/v0.5.27.md":"load/projects/pandora/.changes/v0.5.27.md", ".changie.yaml":"load/projects/pandora/.changie.yaml", + ".github/actions/setup-yc/action.yml":"load/projects/pandora/.github/actions/setup-yc/action.yml", ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml", ".github/workflows/test.yml":"load/projects/pandora/.github/workflows/test.yml", + ".github/workflows/yc.yml":"load/projects/pandora/.github/workflows/yc.yml", ".gitignore":"load/projects/pandora/.gitignore", ".goxc.json":"load/projects/pandora/.goxc.json", ".travis.yml":"load/projects/pandora/.travis.yml", @@ -379,6 +382,48 @@ "lib/zaputil/stack_extract_core.go":"load/projects/pandora/lib/zaputil/stack_extract_core.go", "lib/zaputil/stack_extract_core_test.go":"load/projects/pandora/lib/zaputil/stack_extract_core_test.go", "main.go":"load/projects/pandora/main.go", + "performance-test/automation/_agent_create.sh":"load/projects/pandora/performance-test/automation/_agent_create.sh", + "performance-test/automation/_agent_delete.sh":"load/projects/pandora/performance-test/automation/_agent_delete.sh", + "performance-test/automation/_compose_test_create_args.sh":"load/projects/pandora/performance-test/automation/_compose_test_create_args.sh", + "performance-test/automation/_functions.sh":"load/projects/pandora/performance-test/automation/_functions.sh", + "performance-test/automation/_test_check.sh":"load/projects/pandora/performance-test/automation/_test_check.sh", + "performance-test/automation/_test_run.sh":"load/projects/pandora/performance-test/automation/_test_run.sh", + "performance-test/automation/_variables.sh":"load/projects/pandora/performance-test/automation/_variables.sh", + "performance-test/automation/agent.sh":"load/projects/pandora/performance-test/automation/agent.sh", + "performance-test/automation/default_check_report.sh":"load/projects/pandora/performance-test/automation/default_check_report.sh", + "performance-test/automation/default_check_summary.sh":"load/projects/pandora/performance-test/automation/default_check_summary.sh", + "performance-test/automation/s3_upload.sh":"load/projects/pandora/performance-test/automation/s3_upload.sh", + "performance-test/automation/test.sh":"load/projects/pandora/performance-test/automation/test.sh", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json", + "performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json", + "performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json", + "performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml", + "performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json", + "performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml":"load/projects/pandora/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml", "script/checkfmt.sh":"load/projects/pandora/script/checkfmt.sh", "script/coverage.sh":"load/projects/pandora/script/coverage.sh", "tests/acceptance/common.go":"load/projects/pandora/tests/acceptance/common.go", diff --git a/CHANGELOG.md b/CHANGELOG.md index ecf26ae85..1b960e620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## v0.5.27 - 2024-06-18 +### Added +* Performance test on release + ## v0.5.26 - 2024-05-21 ### Added * scenario config local block in yaml diff --git a/README.md b/README.md index 5d26d2cfa..097dc9dc6 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,17 @@ pandora myconfig.yaml Or use Pandora with [Yandex.Tank](https://yandextank.readthedocs.io/en/latest/core_and_modules.html#pandora) and [Overload](https://overload.yandex.net). +## Performance tests + +These tests are run when a release is created from a tag. See [.github/workflows/yc.yml](.github/workflows/yc.yml) + +The scripts used to run are [performance-test/automation](performance-test/automation) + +And the tests are in the following directory - [performance-test/test-config](performance-test/test-config). +Where one directory is one test. +Each test has a autostop point configured. +When all tests pass, all their autostop points will be written in the release description. + ## Changelog Install https://github.com/miniscruff/changie diff --git a/cli/cli.go b/cli/cli.go index 809972adf..2572fae43 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap/zapcore" ) -const Version = "0.5.26" +const Version = "0.5.27" const defaultConfigFile = "load" const stdinConfigSelector = "-" diff --git a/performance-test/automation/_agent_create.sh b/performance-test/automation/_agent_create.sh new file mode 100644 index 000000000..dc5aa769b --- /dev/null +++ b/performance-test/automation/_agent_create.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Retrieve arguments from command line # +# ---------------------------------------------------------------------------- # + +_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + echo "Usage: $(basename "$0") [ARG]..." + echo "" + echo "Create an agent and wait until it is READY_FOR_TEST" + echo "" + echo "Provided arguments are passed to 'yc loadtesting agent create [ARG]...' as is." + echo "If missing, some argument values are defaulted YC_LT_* environment variables." + exit 0 + ;; + --service-account-id) + VAR_AGENT_SA_ID=$2 + shift + shift + ;; + --name) + _AGENT_NAME=$2 + shift + shift + ;; + --description) + VAR_AGENT_DESCRIPTION=$2 + shift + shift + ;; + --labels) + VAR_AGENT_LABELS=$2 + shift + shift + ;; + --zone) + VAR_AGENT_ZONE=$2 + shift + shift + ;; + --cores) + VAR_AGENT_CORES=$2 + shift + shift + ;; + --memory) + VAR_AGENT_MEMORY=$2 + shift + shift + ;; + --network-interface) + VAR_AGENT_SUBNET_ID= + VAR_AGENT_SECURITY_GROUP_IDS= + _ARGS+=(--network-interface "$2") + shift + shift + ;; + --) + shift + ;; + *) + _ARGS+=("$1") + shift + ;; + esac +done + +assert_installed yc jq +assert_not_empty YC_LT_AGENT_SA_ID +assert_not_empty YC_LT_AGENT_SUBNET_ID +assert_not_empty YC_LT_AGENT_SECURITY_GROUP_IDS + +if [[ -z "$_AGENT_NAME" ]]; then + _AGENT_NAME="$VAR_AGENT_NAME_PREFIX$(rand_str)" +fi + +# ---------------------------------------------------------------------------- # +# Assert variables # +# ---------------------------------------------------------------------------- # + +if [[ -z "${VAR_FOLDER_ID:-$(yc_ config get folder-id)}" ]]; then + _log "Folder ID must be specified either via YC_LT_FOLDER_ID or via CLI profile." + exit 1 +fi + +# ---------------------------------------------------------------------------- # +# Compose command line options # +# ---------------------------------------------------------------------------- # + +if [[ -n $_AGENT_NAME ]]; then + _ARGS+=(--name "$_AGENT_NAME") +fi +if [[ -n $VAR_AGENT_SA_ID ]]; then + _ARGS+=(--service-account-id "$VAR_AGENT_SA_ID") +fi +if [[ -n $VAR_AGENT_DESCRIPTION ]]; then + _ARGS+=(--description "$VAR_AGENT_DESCRIPTION") +fi +if [[ -n $VAR_AGENT_LABELS ]]; then + _ARGS+=(--labels "$VAR_AGENT_LABELS") +fi +if [[ -n $VAR_AGENT_ZONE ]]; then + _ARGS+=(--zone "$VAR_AGENT_ZONE") +fi +if [[ -n $VAR_AGENT_CORES ]]; then + _ARGS+=(--cores "$VAR_AGENT_CORES") +fi +if [[ -n $VAR_AGENT_MEMORY ]]; then + _ARGS+=(--memory "$VAR_AGENT_MEMORY") +fi +if [[ -n ${VAR_AGENT_SUBNET_ID} || -n ${VAR_AGENT_SECURITY_GROUP_IDS} ]]; then + _ARGS+=(--network-interface) + _ARGS+=("subnet-id=$VAR_AGENT_SUBNET_ID,security-group-ids=$VAR_AGENT_SECURITY_GROUP_IDS") +fi + +# ---------------------------------------------------------------------------- # +# Create an agent # +# ---------------------------------------------------------------------------- # + +_log_stage "[$_AGENT_NAME]" +_log_push_stage "[CREATE]" + +_log "Creating..." + +if ! _agent=$(yc_lt agent create "${_ARGS[@]}"); then + _log "Failed to create an agent. $_agent" + exit 1 +fi + +_agent_id=$(echo "$_agent" | jq -r '.id') +_log "Agent created. id=$_agent_id" + +# ---------------------------------------------------------------------------- # +# Wait until agent is READY_FOR_TEST # +# ---------------------------------------------------------------------------- # + +_log_stage "[WAIT]" +_log "Waiting for agent to be ready..." + +_TICK="5" +_TIMEOUT="600" + +_ts_start=$(date +%s) +_ts_timeout=$((_ts_start + _TIMEOUT)) +while [ "$(date +%s)" -lt $_ts_timeout ]; do + _elapsed=$(($(date +%s) - _ts_start)) + + if ! _status=$(yc_lt agent get "$_agent_id" | jq -r '.status'); then + _log "Failed to get agent status" + sleep "$_TICK" + continue + fi + + if [[ "$_status" == "READY_FOR_TEST" ]]; then + _log_stage "[READY]" + _logv 1 "Wow! Just ${_elapsed}s!" + _log "READY_FOR_TEST" + + echo "$_agent_id" + exit 0 + fi + + if ((_elapsed % (_TICK * 6) == 0)); then + _logv 1 "${_elapsed}s passed. Status is $_status. Waiting..." + fi + + _logv 2 "$_status. Next check in ${_TICK}s" + sleep "$_TICK" +done + +echo "$_agent_id" + +_log_stage "[WAIT_FAILED]" +_log "STATUS=$_status. Timeout of ${_TIMEOUT}s exceeded" +_log "Agent is not ready and likely cant be used in tests!" +exit 1 diff --git a/performance-test/automation/_agent_delete.sh b/performance-test/automation/_agent_delete.sh new file mode 100644 index 000000000..0c5fa7cfb --- /dev/null +++ b/performance-test/automation/_agent_delete.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Retrieve arguments from command line # +# ---------------------------------------------------------------------------- # + +_NAME_SUBSTRING="$VAR_AGENT_NAME_PREFIX" +_LABELS="$VAR_AGENT_LABELS" +_IDS=() +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + echo "Usage: $(basename "$0") [--name-substring] [--labels LABELS] [ID1 [...IDN]]" + echo "" + echo "Delete agents matching given parameters." + echo " --name-substring NAME_SUBSTRING - delete agents with name containing this substring [default env YC_LT_AGENT_NAME_PREFIX]" + echo " --labels KEY1=VAL1[,KEYN=VALN] - delete agents with these labels [default env YC_LT_AGENT_LABELS]" + echo " ID1 [...IDN] - agent's id must be one of the specified" + exit 0 + ;; + --name-substring) + _NAME_SUBSTRING=$2 + shift + shift + ;; + --labels) + _LABELS=$2 + shift + shift + ;; + *) + _IDS+=("$1") + shift + ;; + esac +done + +assert_installed yc jq curl + +if [[ -z $_NAME_SUBSTRING && -z $_LABELS && -z ${_IDS[*]} ]]; then + _log "Cannot pick an agent to delete. At least one of arguments must be specified." + exit 1 +fi + +# ---------------------------------------------------------------------------- # +# Compose filter # +# ---------------------------------------------------------------------------- # + +_filters=() +if [[ -n $_NAME_SUBSTRING ]]; then + _filters+=("name contains \"$_NAME_SUBSTRING\"") +fi +if [[ -n $_LABELS ]]; then + IFS=',' read -ra _labels_arr <<<"$_LABELS" + for _kv in "${_labels_arr[@]}"; do + IFS='=' read -r _key _value <<<"$_kv" + _filters+=("labels.$_key = \"$_value\"") + done +fi +if [[ -n ${_IDS[*]} ]]; then + _joined=$(IFS=','; echo "${_IDS[*]}") + _filter+=("id in ($_joined)") +fi + +_filter_str='' +for _f in "${_filters[@]}"; do + if [[ -n $_filter_str ]]; then + _filter_str="$_filter_str and $_f" + else + _filter_str="$_f" + fi +done + +if [[ -z $_filter_str ]]; then + _log "Error! Filter is empty" + exit 1 +fi + +_log "Filter: $_filter_str" + +# ---------------------------------------------------------------------------- # +# Determine which agents should be deleted # +# ---------------------------------------------------------------------------- # + +_log "Determining which agents to be deleted..." + + +_delete_ids=() +IFS=' ' read -ra _delete_ids < \ + <(yc_lt agent list --filter "$_filter_str" | jq -r '[.[].id] | join(" ")') + +if [[ ${#_delete_ids} -eq 0 ]]; then + _log "No agents were found for given filter" + exit 0 +fi + +_log "Agents to be deleted: ${_delete_ids[*]}" + +# ---------------------------------------------------------------------------- # +# Delete agents # +# ---------------------------------------------------------------------------- # + +_log "Deleting agents..." +yc_lt agent delete "${_delete_ids[@]}" diff --git a/performance-test/automation/_compose_test_create_args.sh b/performance-test/automation/_compose_test_create_args.sh new file mode 100644 index 000000000..822461e12 --- /dev/null +++ b/performance-test/automation/_compose_test_create_args.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +_metafile='' +_config_ids=() +_extra_description='' +_extra_labels='' +_extra_agent_filter='' +_extra_data_fnames=() +_extra_data_fspecs=() +while [[ $# -gt 0 ]]; do + case "$1" in + -m | --meta) + [[ -n $2 ]] + _metafile=$2 + shift + shift + ;; + -c | --config-id) + [[ -n $2 ]] + _config_ids+=("$2") + shift + shift + ;; + -d | --extra-test-data) + [[ -n $2 && -n $3 && -n $4 ]] + _extra_data_fnames+=("$2") + _extra_data_fspecs+=("name=$2,s3file=$3,s3bucket=$4") + shift + shift + shift + shift + ;; + --extra-labels) + [[ -n $2 ]] + [[ -z $_extra_labels ]] + _extra_labels=$2 + shift + shift + ;; + --extra-agent-filter) + [[ -n $2 ]] + [[ -z $_extra_agent_filter ]] + _extra_agent_filter=$2 + shift + shift + ;; + --extra-description) + [[ -n $2 ]] + [[ -z $_extra_description ]] + _extra_description=$2 + shift + shift + ;; + --help | -h | *) + echo "Usage: $(basename "$0") [-m META_JSON_FILE] [-c CONFIG_ID]... [-d FILE_LOCAL_NAME FILE_BUCKET_NAME BUCKET_NAME]... [... OPTIONS]" + echo "" + echo "Compose arguments for 'yc loadtesting test create'." + echo " -m|--meta META_JSON_FILE - path to a json file with test description" + echo " -c|--config-id CONFIG_ID - ID of a test configuration file (may be defined multiple times)" + echo " -d|--extra-test-data FILE_NAME FILE_NAME_IN_BUCKET BUCKET - extra test data (may be defined multiple times)" + echo " --extra-labels KEY1=VAL1[,KEYN=VALN] - extra labels" + echo " --extra-agent-filter AGENT_FILTER - extra agent filter" + echo " --extra-description DESCRIPTION - extra description" + exit 0 + ;; + esac +done + +_multi_factor="1" +_name=$(readlink -f "$_metafile" | xargs dirname | xargs basename) +_description='' +_labels='' +_agent_filter='' +_data_fnames=() +_data_fspecs=() + +# ---------------------------------------------------------------------------- # +# read from metafile # +# ---------------------------------------------------------------------------- # + +if [[ -f "$_metafile" ]]; then + function read_meta { + jq -re "$@" < "$_metafile" + return 0 + } + + # shellcheck disable=SC2016 + _multi_factor=$(read_meta --arg d "$_multi_factor" ' + .multi // $d + | tostring + ') + # shellcheck disable=SC2016 + _name=$(read_meta --arg d "$_name" ' + .name // $d + | tostring + ') + _description=$(read_meta ' + .description // "" + | tostring + ') + _labels=$(read_meta ' + .labels // {} + | to_entries + | map("\(.key)=\(.value)") + | join(",") + ') + _agent_filter=$(read_meta ' + .agent_labels // {} + | to_entries + | map("labels.\(.key)=\"\(.value)\"") + | join(" and ") + ') + IFS=$'\n' read -d '' -ra _data_fnames < <(read_meta ' + .external_data // [] + | [.[] | select(.name? and .s3file? and .s3bucket?)] + | map("\(.name)") + | join("\n") + ') || true + IFS=$'\n' read -d '' -ra _data_fspecs < <(read_meta ' + .external_data // [] + | [.[] | select(.name? and .s3file? and .s3bucket?)] + | map("name=\(.name),s3file=\(.s3file),s3bucket=\(.s3bucket)") + | join("\n") + ') || true +fi + +# ---------------------------------------------------------------------------- # +# add extras # +# ---------------------------------------------------------------------------- # + +if [[ -n ${_extra_description} ]]; then + if [[ -n ${_description} ]]; then + _description="$_description\n\n$_extra_description" + else + _description="$_extra_description" + fi +fi + +if [[ -n ${_extra_labels} ]]; then + if [[ -n ${_labels} ]]; then + _labels="$_labels,$_extra_labels" + else + _labels="$_extra_labels" + fi +fi + +if [[ -n ${_extra_agent_filter} ]]; then + if [[ -n ${_agent_filter} ]]; then + _agent_filter="$_agent_filter and $_extra_agent_filter" + else + _agent_filter="$_extra_agent_filter" + fi +fi + +_data_fnames+=("${_extra_data_fnames[@]}") +_data_fspecs+=("${_extra_data_fspecs[@]}") + +# ---------------------------------------------------------------------------- # +# compose args # +# ---------------------------------------------------------------------------- # + +ARGS=() +ARGS+=(--name "$_name") +ARGS+=(--description "$_description") +ARGS+=(--labels "$_labels") +for _ in $(seq 1 "$_multi_factor"); do + for _config_id in "${_config_ids[@]}"; do + _cfg="id=$_config_id,agent-by-filter=$_agent_filter" + for _fname in "${_data_fnames[@]}"; do + _cfg="$_cfg,test-data=$_fname" + done + + ARGS+=(--configuration "$_cfg") + done +done +for _fspec in "${_data_fspecs[@]}"; do + ARGS+=(--test-data "$_fspec") +done + +(IFS=$'\t'; echo "${ARGS[*]}") diff --git a/performance-test/automation/_functions.sh b/performance-test/automation/_functions.sh new file mode 100644 index 000000000..854f4a54d --- /dev/null +++ b/performance-test/automation/_functions.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +if [[ -v _LOG_STAGE_STR ]]; then + export _LOG_STAGE=() + IFS=$'\n' read -d '' -ra _LOG_STAGE <<< "$_LOG_STAGE_STR" || true +else + export _LOG_STAGE_STR='' + export _LOG_STAGE=() +fi + +function _log_push_stage { + _LOG_STAGE+=("$1") + _LOG_STAGE_STR=$(IFS=$'\n'; echo "${_LOG_STAGE[*]}") +} + +function _log_pop_stage { + if [[ ${#_LOG_STAGE[@]} -gt 0 ]]; then + _N=${#_LOG_STAGE[@]} + _LOG_STAGE=("${_LOG_STAGE[@]::${_N}-1}") + fi + _LOG_STAGE_STR=$(IFS=$'\n'; echo "${_LOG_STAGE[*]}") +} + +function _log_stage { + _log_pop_stage + _log_push_stage "$1" +} + +function _log { + if [[ "$1" == '-f' ]]; then + shift + echo >&2 "${_LOG_STAGE[*]}" ":" && cat >&2 "$@" + else + echo >&2 "${_LOG_STAGE[*]}" ":" "$@" + fi +} + +function _logv { + if [[ $VAR_VERBOSE -ge "$1" ]]; then + shift + _log "$@" + fi +} + +function assert_installed { + for _cmd in "$@"; do + if ! command -v "$_cmd" 1>/dev/null 2>&1; then + _log "ERROR!!! Assertion failed: $_cmd is not installed" + exit 1 + fi + done + return 0 +} + +function assert_not_empty { + if [[ -z "${!1}" ]]; then + _log "ERROR!!! Assertion failed: variable $1 is empty or not defined" + exit 1 + fi + return 0 +} + +function rand_str { + ( + set +o pipefail + LC_ALL=C tr -d -c '0-9a-f' /dev/null \ + <"$file" + + return $? +} + +function yc_s3_delete { + local -r bucket_path=$1 + local -r bucket=${2:-"$VAR_DATA_BUCKET"} + + assert_not_empty bucket + assert_not_empty bucket_path + + local -r token=${VAR_TOKEN:-$(yc_get_token)} + local -r auth_h="X-YaCloud-SubjectToken: $token" + curl -L -H "$auth_h" -X DELETE "$VAR_OBJECT_STORAGE_URL/$bucket/$bucket_path" \ + 2>/dev/null + + return $? +} + +function yc_test_url { + local -r test_id=$1 + local -r folder_id=${VAR_FOLDER_ID:-$(yc_ config get folder-id)} + echo "$VAR_WEB_CONSOLE_URL/folders/$folder_id/load-testing/tests/$test_id" +} + +function check_json_val { + local -r description=${1:-"$2 $3"} + local -r filter=$2 + local -r condition=$3 + local -r file=${4:-$_CHECK_JSON_FILE} + echo "- $description" + echo "-- $(jq -r "$filter" "$file" 2>/dev/null) $condition" + if jq -re "($filter) $condition" "$file" >/dev/null ; then + echo "-- OK" + return 0 + else + echo "-- filter: $filter" + echo "-- FAIL" + return 1 + fi +} diff --git a/performance-test/automation/_test_check.sh b/performance-test/automation/_test_check.sh new file mode 100644 index 000000000..1acfac0fc --- /dev/null +++ b/performance-test/automation/_test_check.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Arguments and constants # +# ---------------------------------------------------------------------------- # + +while [[ $# -gt 0 ]]; do + case "$1" in + --dir) + _TEST_DIR=$2 + shift + shift + ;; + --id) + _TEST_ID=$2 + shift + shift + ;; + --output | -o) + _OUTPUT_DIR=$2 + shift + shift + ;; + --help | -h | *) + echo "Usage: $(basename "$0") --id TEST_ID [--dir SCRIPT_DIR] [-o OUTPUT_DIR]" + echo "" + echo "Obtain test results and check them with two check scripts in passed SCRIPT_DIR directory:" + # shellcheck disable=SC2016 + echo '- SCRIPT_DIR/check_summary.sh $(yc --format json loadtesting test get TEST_ID)' + # shellcheck disable=SC2016 + echo '- SCRIPT_DIR/check_report.sh $(yc --format json loadtesting test get-report-table TEST_ID)' + echo "" + echo "If corresponding checks are not found in SCRIPT_DIR, the default checks will be run instead." + exit 0 + ;; + esac +done + +assert_installed yc jq curl +assert_not_empty _TEST_DIR +assert_not_empty _TEST_ID + +_OUTPUT_DIR=${_OUTPUT_DIR:-"./check-$_TEST_DIR"} +mkdir -p "$_OUTPUT_DIR" + +set +e +export -f run_script +export -f check_json_val + +rc=0 + +# ------------------------------- Check summary ------------------------------ # +# -------------- (yc --format json loadtesting test get TEST_ID) ------------- # + +if ! yc_lt test get "$_TEST_ID" >"$_OUTPUT_DIR/summary.json"; then + echo "ERROR!!! failed to download test summary" + exit 1 +fi + +export _DEFAULT_CHECK="$_SCRIPT_DIR/default_check_summary.sh" +if [[ -f "$_TEST_DIR/check_summary.sh" ]]; then + echo "Running: $_TEST_DIR/check_summary.sh $_OUTPUT_DIR/summary.json" + if ! /usr/bin/env bash "$_TEST_DIR/check_summary.sh" "$_OUTPUT_DIR/summary.json"; then + rc=1 + fi + +elif [[ -f "$_DEFAULT_CHECK" ]]; then + echo "Running: $_DEFAULT_CHECK $_OUTPUT_DIR/summary.json" + if ! /usr/bin/env bash "$_DEFAULT_CHECK" "$_OUTPUT_DIR/summary.json"; then + rc=1 + fi + +else + echo "ERROR: check_summary.sh script not found" + rc=1 +fi + +# ------------------------------- Check report ------------------------------- # +# ------- (yc --format json loadtesting test get-report-table TEST_ID) ------- # + +if ! yc_lt test get-report-table "$_TEST_ID" >"$_OUTPUT_DIR/report.json"; then + echo "ERROR!!! failed to download test report" + exit 1 +fi + +export _DEFAULT_CHECK="$_SCRIPT_DIR/default_check_report.sh" +if [[ -f "$_TEST_DIR/check_report.sh" ]]; then + echo "Running: $_TEST_DIR/check_report.sh $_OUTPUT_DIR/report.json" + if ! /usr/bin/env bash "$_TEST_DIR/check_report.sh" "$_OUTPUT_DIR/report.json"; then + rc=1 + fi + +elif [[ -f "$_DEFAULT_CHECK" ]]; then + echo "Running: $_DEFAULT_CHECK $_OUTPUT_DIR/report.json" + if ! /usr/bin/env bash "$_DEFAULT_CHECK" "$_OUTPUT_DIR/report.json"; then + rc=1 + fi + +else + echo "ERROR: check_summary.sh script not found" + rc=1 +fi + +# ----------------------------------- exit ----------------------------------- # + +exit $rc diff --git a/performance-test/automation/_test_run.sh b/performance-test/automation/_test_run.sh new file mode 100644 index 000000000..106897bfd --- /dev/null +++ b/performance-test/automation/_test_run.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Arguments and constants # +# ---------------------------------------------------------------------------- # + +while [[ $# -gt 0 ]]; do + case "$1" in + --help | -h) + echo "Usage: $(basename "$0") TEST_DIR" + echo "" + echo "Run test with configurations defined in TEST_DIR/$VAR_TEST_CONFIG_MASK" + echo "Additional test parameters may be defined in TEST_DIR/meta.json" + exit 0 + ;; + *) + _TEST_DIR=$1 + break + ;; + esac +done + +assert_installed yc jq curl +assert_not_empty _TEST_DIR + +_TEMP_BUCKET_DIR="test-runs/$(rand_str)" +declare -r _TEMP_BUCKET_DIR + +# ---------------------------------------------------------------------------- # +# sanity check, before anything is created # +# ---------------------------------------------------------------------------- # + +_logv 1 "## Sanity check..." +run_script "$_SCRIPT_DIR/_compose_test_create_args.sh" \ + --meta "$_TEST_DIR/meta.json" \ + -c 12345 \ + -c 54321 \ + -d local1 inbucket1 bucket1 \ + -d local2 inbucket2 bucket2 \ + >/dev/null + +# ---------------------------------------------------------------------------- # +# prepare test configuration files # +# ---------------------------------------------------------------------------- # + +_logv 1 "## Prepare test configurations" +_config_ids=() + +# ------------------------- list configuration files ------------------------- # + +_config_files=() +while IFS= read -d '' -r _file; do _config_files+=("$_file"); done < \ + <(find "$_TEST_DIR" -type f -name "$VAR_TEST_CONFIG_MASK" -maxdepth 1 -print0) + +if [[ ${#_config_files[@]} -eq 0 ]]; then + _log "ERROR!!! No config files found in $_TEST_DIR. Config file mask: $VAR_TEST_CONFIG_MASK" + exit 1 +fi + +_logv 1 "Found test configuration files: ${_config_files[*]}" + +# ------------------------ upload configuration files ------------------------ # + +_logv 1 "Uploading configurations..." +for _file in "${_config_files[@]}"; do + _args=() + + # substitute YC_LT_TARGET in config file if the variable is set + if [[ -n $YC_LT_TARGET ]]; then + _config_content=$(cat "$_file") + # shellcheck disable=SC2016 + _config_content=${_config_content/'${YC_LT_TARGET}'/"$YC_LT_TARGET"} + _args+=(--yaml-string "$_config_content") + else + _args+=(--from-yaml-file "$_file") + fi + + _config_id=$(yc_lt test-config create "${_args[@]}" | jq -r '.id') + _logv 1 "- created test configuration $_config_id from $_file" + + _config_ids+=("$_config_id") +done + +# ---------------------------------------------------------------------------- # +# prepare local data files # +# ---------------------------------------------------------------------------- # + +_logv 1 "## Prepare local data files" +_local_data_fnames=() + +function cleanup_temp_data_files { + _log "Cleaning up data files..." + for _fname in "${_local_data_fnames[@]}"; do + _temp_s3_file="$_TEMP_BUCKET_DIR/$_fname" + if ! yc_s3_delete "$_temp_s3_file" "$VAR_DATA_BUCKET" >/dev/null; then + _log "- failed to delete $_temp_s3_file" + fi + done +} +trap cleanup_temp_data_files EXIT + +# --------------------------- list local data files -------------------------- # + +function is_data_file { + _non_data_files=("${_config_files[@]}") + _non_data_files+=("$_TEST_DIR/meta.json") + _non_data_files+=("$_TEST_DIR/check_summary.sh") + _non_data_files+=("$_TEST_DIR/check_report.sh") + for _ndf in "${_non_data_files[@]}"; do + if [[ $1 == "$_ndf" ]]; then + return 1 + fi + done + return 0 +} + +_local_data_files=() +while IFS= read -d '' -r _file; do + if is_data_file "$_file"; then + _local_data_files+=("$_file") + fi +done < <(find "$_TEST_DIR" -type f -print0) + +_logv 1 "Found local data files: ${_local_data_files[*]}" + +# --------------------- upload local data files to bucket -------------------- # + +if [[ ${#_local_data_files[@]} -gt 0 && -n $VAR_DATA_BUCKET ]]; then + _logv 1 "Uploading local data files... (should be deleted after test)" + _logv 1 "upload params: bucket=$VAR_DATA_BUCKET; common-prefix=$_TEMP_BUCKET_DIR/" + + for _file in "${_local_data_files[@]}"; do + _fname=${_file#"$_TEST_DIR/"} + _temp_s3_file="$_TEMP_BUCKET_DIR/$_fname" + if ! yc_s3_upload "$_file" "$_temp_s3_file" "$VAR_DATA_BUCKET" >/dev/null; then + _log "- failed to upload $_temp_s3_file" + continue + fi + + _logv 1 "- uploaded local data file $_fname" + _local_data_fnames+=("$_fname") + done + +elif [[ ${#_local_data_files[@]} -gt 0 ]]; then + _logv 1 "Upload failed: YC_LT_DATA_BUCKET is not specified." +fi + +# ---------------------------------------------------------------------------- # +# Determine test parameters # +# ---------------------------------------------------------------------------- # + +_logv 1 "## Compose command arguments" + +_composer_args=() +_composer_args+=(--meta "$_TEST_DIR/meta.json") +_composer_args+=(--extra-agent-filter "${VAR_TEST_AGENT_FILTER:-}") +_composer_args+=(--extra-labels "${VAR_TEST_EXTRA_LABELS:-}") +_composer_args+=(--extra-description "${VAR_TEST_EXTRA_DESCRIPTION:-}") +for _id in "${_config_ids[@]}"; do + _composer_args+=(-c "$_id") +done +for _fname in "${_local_data_fnames[@]}"; do + _composer_args+=(-d "$_fname" "$_TEMP_BUCKET_DIR/$_fname" "$VAR_DATA_BUCKET") +done + +RUN_ARGS=() +IFS=$'\t' read -d '' -ra RUN_ARGS < \ + <(run_script "$_SCRIPT_DIR/_compose_test_create_args.sh" "${_composer_args[@]}") \ + || true + +_logv 1 "Test run arguments: ${RUN_ARGS[*]}" + +# ---------------------------------------------------------------------------- # +# Run the test # +# ---------------------------------------------------------------------------- # + +_logv 1 "## Starting test..." +_test_id=$(yc_lt test create "${RUN_ARGS[@]}" | jq -r '.id') + +_logv 1 "Started. Test url: $(yc_test_url "$_test_id")/test-report" + +_logv 1 "## Waiting for test to finish..." +yc_lt test wait --idle-timeout 60s "$_test_id" diff --git a/performance-test/automation/_variables.sh b/performance-test/automation/_variables.sh new file mode 100644 index 000000000..e2996df7a --- /dev/null +++ b/performance-test/automation/_variables.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +if [[ -n ${__YC_LT_VARS_DEFINED} ]]; then + return 0 +fi + +export __YC_LT_VARS_DEFINED=1 + +function _def_var { + local -r _name=$1 + + local -r _env="YC_LT_${_name}" + + local -r _var_default="__DEFAULT_${_name}" + local -r _var="VAR_${_name}" + + local -r _val_default=$2 + local -r _val=${!_env-"${_val_default}"} + + export "$_var_default=$_val_default" + export "$_var=$_val" + + local -r _gap=$(printf -- '.%.0s' {1..80}) + local -r _info_actual="$_var: \"${!_var:0:60}\"" + local -r _info_env="$_env: \"${!_env:0:60}\"" + _logv 2 "${_info_actual} ${_gap:${#_info_actual}} env ${_info_env}" +} + +# ---------------------------------------------------------------------------- # +# General # +# ---------------------------------------------------------------------------- # + +# env: YC_LT_VERBOSE +# format: 0, 1, 2 +# Scripts verbosity level +_def_var VERBOSE "0" + +# env: YC_LT_OUTPUT_DIR +# format: path to local directory +_def_var OUTPUT_DIR "$PWD/.loadtesting" + +# env: YC_LT_CLI_PROFILE +# format: string +# YC CLI profile which will be used to create agents and tests. +# An ID of a folder is taken from profile, or, if specified, from YC_LT_FOLDER_ID. +_def_var CLI_PROFILE "" + +# env: YC_LT_FOLDER_ID +# format: string, cloud-id +# ID of a cloud folder where tests are performed. +_def_var FOLDER_ID "" + +# env: YC_LT_DATA_BUCKET +# format: string, bucket-name +# Name of an object storage bucket used as a storage for test data. If needed, +# local files are automatically uploaded to the bucket. +_def_var DATA_BUCKET "" + +# env: YC_LT_CLI_INTERACTIVE +# format: 0 or 1 +# Defines whether interactive CLI input is allowed. +_def_var CLI_INTERACTIVE "1" + +# env: YC_LT_TOKEN +# format: string, token +# A token. Normally, no need to specify explicitly +_def_var TOKEN "" + +# ---------------------------------------------------------------------------- # +# Agent # +# ---------------------------------------------------------------------------- # + +# env: YC_LT_AGENTS_CNT +# format: positive number +# Number of agents which should be created when 'agent.sh create' is called. +_def_var AGENTS_CNT "1" + +# env: YC_LT_AGENT_SA_ID +# format: string, cloud-id +# ID of a service account with which agent VM will be created. +_def_var AGENT_SA_ID "" + +# ---------------------------- Agent: VM settings ---------------------------- # + +# env: YC_LT_AGENT_ZONE +# format: identifier of an availability zone +# Agent's VM will be created in the specified zone +_def_var AGENT_ZONE "ru-central1-b" + +# env: YC_LT_AGENT_SUBNET_ID +# format: string, cloud-id +# ID of a subnet in which agent's VM will be created +_def_var AGENT_SUBNET_ID "" + +# env: YC_LT_AGENT_SECURITY_GROUP_IDS +# format: list of cloud-id in format [id[,id[,...]]] +# IDS of security groups assigned to created agent. +_def_var AGENT_SECURITY_GROUP_IDS "" + +# env: YC_LT_AGENT_CORES +# format: positive number > 1 +# A number of CPU cores with which agent VM will be created. +_def_var AGENT_CORES "2" + +# env: YC_LT_AGENT_MEMORY +# format: positive number + scale specifier +# Amount of RAM with which agent VM will be created. +_def_var AGENT_MEMORY "2G" + +# ----------------- Agent: service settings and customization ---------------- # + +# env: YC_LT_AGENT_LABELS +# format: list of key-value pairs in format [key=value[,key=value[,...]]] +# Labels of an agent created by 'agent.sh create' +_def_var AGENT_LABELS "ci=true,author=$USER" + +# env: YC_LT_AGENT_NAME_PREFIX +# format: string +# Name (or prefix) of an agent created by 'agent.sh create' +_def_var AGENT_NAME_PREFIX "ci-lt-agent" + +# env: YC_LT_AGENT_DESCRIPTION +# format: string +# Description of an agent created by 'agent.sh create' +_def_var AGENT_DESCRIPTION "Created via script by $USER" + +# ---------------------------------------------------------------------------- # +# Test # +# ---------------------------------------------------------------------------- # + +# env: YC_LT_SKIP_TEST_CHECK +# format: 0 or 1 +# Specifies whether checks should be performed after a test has finished. +_def_var SKIP_TEST_CHECK "0" + +# env: YC_LT_TEST_AGENT_FILTER +# format: filter string +# Filter expression by which agents will be selected to execute a test. +# The expression will be ANDed to the ones specified in meta.json +# Example: +# - agents containing 'onetime-agent-' in name: 'name contains "onetime-agent-"' +# - agents with labels ci=true and author=foobar: 'labels.ci = "true" and labels.author = "foobar"' +_def_var TEST_AGENT_FILTER "labels.ci=true and labels.author=$USER" + +# env: YC_LT_TEST_EXTRA_LABELS +# format: list of key-value pairs in format [key=value[,key=value[,...]]] +# Additional (to the ones specified in meta.json) labels with which tests will be created. +_def_var TEST_EXTRA_LABELS "ci=true" + +# env: YC_LT_TEST_EXTRA_DESCRIPTION +# format: string +# Additional (to the one specified in meta.json) description which which tests will be creatd. +_def_var TEST_EXTRA_DESCRIPTION "" + +# ---------------------------------------------------------------------------- # +# Constants customization # +# ---------------------------------------------------------------------------- # + +# env: YC_LT_TEST_CONFIG_MASK +# format: GLOB mask +_def_var TEST_CONFIG_MASK "test-config*.yaml" + +# env: YC_LT_OBJECT_STORAGE_URL +# format: url +_def_var OBJECT_STORAGE_URL "https://storage.yandexcloud.net" + +# env: YC_LT_WEB_CONSOLE_URL +# format: url +_def_var WEB_CONSOLE_URL "https://console.yandex.cloud" diff --git a/performance-test/automation/agent.sh b/performance-test/automation/agent.sh new file mode 100644 index 000000000..d353de40d --- /dev/null +++ b/performance-test/automation/agent.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Retrieve arguments from command line # +# ---------------------------------------------------------------------------- # + +_CMD='' + +while [[ $# -gt 0 ]]; do + case "$1" in + create) + _CMD='create' + shift + break + ;; + delete) + _CMD='delete' + shift + break + ;; + -h | --help | *) + echo "Usage: $(basename "$0") subcommand [ARG]..." + echo "" + echo "Subcommands:" + echo " $(basename "$0") create [--count N] [ARG]..." + echo " create specified number of agents" + echo " $(basename "$0") delete [ARG]..." + echo " delete agents" + exit 0 + ;; + esac +done + +if [[ -z "${VAR_FOLDER_ID:-$(yc_ config get folder-id)}" ]]; then + _log "Folder ID must be specified either via YC_LT_FOLDER_ID or via CLI profile." + exit 1 +fi + +if [[ "$_CMD" == 'create' ]]; then + _CNT=$VAR_AGENTS_CNT + while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + echo "Usage: $(basename "$0") create [--count N] [ARG]..." + echo "" + echo "Call agent creation subroutine N times and wait until all agents are READY_FOR_TEST" + echo "" + echo "Subroutine help:" + run_script "$_SCRIPT_DIR/_agent_create.sh" --help + exit 0 + ;; + --count) + _CNT="$2" + shift + shift + break + ;; + --) + shift + break + ;; + *) + break + ;; + esac + done + + _log "Compute Agents create request. Number of agents: $_CNT" + _pids=() + for _i in $(seq 1 "$_CNT"); do + _log_stage "[$_i]" + run_script "$_SCRIPT_DIR/_agent_create.sh" "$@" & + _pids+=("$!") + done + + _rc=0 + for _pid in "${_pids[@]}"; do + wait "$_pid" + _rc=$((_rc | $?)) + done + + exit ${_rc} + +elif [[ "$_CMD" == 'delete' ]]; then + while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + echo "Usage: $(basename "$0") delete [ARG]..." + echo "" + echo "Call agent deletion subroutine" + echo "" + echo "Subroutine help:" + run_script "$_SCRIPT_DIR/_agent_delete.sh" --help + exit 0 + ;; + --) + shift + break + ;; + *) + break + ;; + esac + done + + _log "Compute Agents delete request." + run_script "$_SCRIPT_DIR/_agent_delete.sh" "$@" + exit $? +fi diff --git a/performance-test/automation/default_check_report.sh b/performance-test/automation/default_check_report.sh new file mode 100644 index 000000000..b3a5cd381 --- /dev/null +++ b/performance-test/automation/default_check_report.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +_CHECK_JSON_FILE="$1" + +rc=0 + +check_json_val \ + 'report status is READY' \ + '.status' \ + '== "READY"' + +rc=$((rc | $?)) + +check_json_val \ + 'has successfully sent requests' \ + '.overall.net_codes."0" // "-1" | tonumber' \ + '> 0' + +rc=$((rc | $?)) + +check_json_val \ + 'has non-zero response time requests' \ + '.overall.quantiles.q100 // "-1" | tonumber' \ + '> 0' + +rc=$((rc | $?)) + +check_json_val \ + '50th response time percentile is less than 10s' \ + '.overall.quantiles.q50 // "-1" | tonumber' \ + '< 10000' + +rc=$((rc | $?)) + +exit $rc diff --git a/performance-test/automation/default_check_summary.sh b/performance-test/automation/default_check_summary.sh new file mode 100644 index 000000000..47af45c11 --- /dev/null +++ b/performance-test/automation/default_check_summary.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +_CHECK_JSON_FILE="$1" + +rc=0 + +check_json_val \ + 'test status doesnt indicate an error' \ + '.summary.status' \ + '| IN("DONE", "AUTOSTOPPED")' + +rc=$((rc | $?)) + +exit $rc diff --git a/performance-test/automation/s3_upload.sh b/performance-test/automation/s3_upload.sh new file mode 100644 index 000000000..b15c01945 --- /dev/null +++ b/performance-test/automation/s3_upload.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +# ---------------------------------------------------------------------------- # +# Retrieve arguments from command line # +# ---------------------------------------------------------------------------- # + +_ARGS=() + + +_file="${1}" +_temp_s3_file="${2}" +var_data_bucket="${3}" + +if ! yc_s3_upload "$_file" "$_temp_s3_file" "$var_data_bucket" >/dev/null; then + _log "- failed to upload $_temp_s3_file" + exit 1 +fi diff --git a/performance-test/automation/test.sh b/performance-test/automation/test.sh new file mode 100644 index 000000000..a26184e0f --- /dev/null +++ b/performance-test/automation/test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +set -e + +# shellcheck disable=SC2155 +export _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +# shellcheck source=_functions.sh +source "$_SCRIPT_DIR/_functions.sh" + +# shellcheck source=_variables.sh +source "$_SCRIPT_DIR/_variables.sh" + +_DIRS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --help | -h) + echo "Usage: $(basename "$0") TEST_DIR1 [TEST_DIR2]..." + echo "" + echo "Sequentially run and check results of tests defined in directories passed as arguments," + echo "print summary." + echo "" + echo "Specifically, for each provided argument:" + echo "1. call '_test_run.sh TEST_DIR' to run the test (see '_test_run.sh --help')" + echo "2. call '_test_check.sh --id TEST_ID --dir TEST_DIR' to check the results (see '_test_check.sh --help')" + exit 0 + ;; + *) + _DIRS+=("$1") + shift + ;; + esac +done + +assert_installed yc jq curl + +_logv 1 "YC CLI provile: ${VAR_CLI_PROFILE:-"current aka <$(yc_ config profile list | grep ' ACTIVE')>"}" +_logv 1 "" + +_log -f < >(tee /dev/stderr))") + _tests_failed=$((_tests_failed + 1)) + continue + fi + + _test_id= + _log_stage "[RUN]" + _log "Running..." + if _test=$(run_script "$_SCRIPT_DIR/_test_run.sh" "$_test_dir"); then + _test_id=$(jq -r '.id' <<< "$_test") + _logv 1 "ID=$_test_id" + _logv 1 "STATUS=$(jq -r '.summary.status' <<< "$_test")" + _log "FINISHED: $(yc_test_url "$_test_id")/test-report)" + else + _msg="FAILED: error; test=$_test" + _tests_failure_reports+=("$(_log "$_msg" 2> >(tee /dev/stderr))") + _tests_failed=$((_tests_failed + 1)) + continue + fi + + _out_dir="$VAR_OUTPUT_DIR/$_test_id" + mkdir -p "$_out_dir" + yc_lt test get "$_test_id" >"$_out_dir/summary.json" + yc_lt test get-report-table "$_test_id" >"$_out_dir/report.json" + + _log_stage "[CHECK]" + if [[ "${VAR_SKIP_TEST_CHECK:-0}" == 0 ]]; then + _resfile="$_out_dir/check_result.txt" + + _log "Performing checks..." + if run_script "$_SCRIPT_DIR/_test_check.sh" --id "$_test_id" --dir "$_test_dir" -o "$_out_dir" >"$_resfile"; then + _logv 1 -f <"$_resfile" + _log "ALL CHECKS PASSED" + else + _log -f <"$_resfile" + _msg="FAILED: checks did not pass. Result in $_resfile" + _tests_failure_reports+=("$(_log "$_msg" 2> >(tee /dev/stderr))") + _tests_failed=$((_tests_failed + 1)) + fi + else + _log "skipped due to YC_LT_SKIP_TEST_CHECK" + fi + + _log "" +done + +_log_pop_stage +_log_pop_stage +_log_stage "" + +_summary_header="[ OK - $((_tests_total - _tests_failed)) | FAILED - $_tests_failed ]" +_log "==================== $_summary_header ====================" +_log "" +if ((_tests_failed != 0)); then + _log "$_tests_failed out of $_tests_total tests have failed:" + for _msg in "${_tests_failure_reports[@]}"; do + _log "$_msg" + done +fi +_log "" +_log "==================== $_summary_header ====================" + +echo "$_tests_failed" +exit "$_tests_failed" diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json new file mode 100644 index 000000000..b7adb63f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json new file mode 100644 index 000000000..264b58e16 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"0ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..2048936a1 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-false/test-config.yaml @@ -0,0 +1,60 @@ +autostop: + enabled: true + autostop: + - quantile (50,85ms,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-2000inst-sleep0ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 20000 + duration: 70s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json new file mode 100644 index 000000000..2a217146e --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": true, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json new file mode 100644 index 000000000..264b58e16 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"0ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..8e0f664a4 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep0ms-overflow-true/test-config.yaml @@ -0,0 +1,60 @@ +autostop: + enabled: true + autostop: + - net(777, 500, 7s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-2000inst-sleep0ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 20000 + duration: 70s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json new file mode 100644 index 000000000..b7adb63f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json new file mode 100644 index 000000000..23380dd19 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"50ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..f5f0e488e --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-false/test-config.yaml @@ -0,0 +1,60 @@ +autostop: + enabled: true + autostop: + - quantile (50,120ms,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-2000inst-sleep50ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 20000 + duration: 70s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json new file mode 100644 index 000000000..2a217146e --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": true, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json new file mode 100644 index 000000000..23380dd19 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"50ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..48d783c05 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-2000inst-sleep50ms-overflow-true/test-config.yaml @@ -0,0 +1,60 @@ +autostop: + enabled: true + autostop: + - net(777, 500, 7s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-2000inst-sleep50ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 20000 + duration: 70s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json new file mode 100644 index 000000000..b7adb63f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json new file mode 100644 index 000000000..264b58e16 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"0ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..12c7f2b61 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-false/test-config.yaml @@ -0,0 +1,61 @@ +autostop: + enabled: true + autostop: + - quantile (50,11ms,12s) + - instances(300,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-300inst-sleep0ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 300 + rps: + - type: line + from: 1 + to: 30000 + duration: 80s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json new file mode 100644 index 000000000..2a217146e --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/meta.json @@ -0,0 +1,13 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "grpc", + "discard_overflow": true, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json new file mode 100644 index 000000000..264b58e16 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/payload.json @@ -0,0 +1 @@ +{"tag": "hello", "call": "target.TargetService.Hello", "payload": {"name": "sweet tester","sleep":"0ms"}} diff --git a/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..afb50ba22 --- /dev/null +++ b/performance-test/test-config/pandora-perf-grpc-300inst-sleep0ms-overflow-true/test-config.yaml @@ -0,0 +1,60 @@ +autostop: + enabled: true + autostop: + - net(777, 1100, 7s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-grpc-300inst-sleep0ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: gRPC + gun: + type: grpc + target: 127.0.0.1:8091 + tls: false + ammo: + type: grpc/json + file: ./payload.json + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 300 + rps: + - type: line + from: 1 + to: 30000 + duration: 80s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json new file mode 100644 index 000000000..309d983ca --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": false, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..fe0a4ce5d --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-false/test-config.yaml @@ -0,0 +1,62 @@ +autostop: + enabled: true + autostop: + - quantile (50,16ms,10s) + - instances(2000,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-2000inst-sleep0ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=0ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 40000 + duration: 60s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json new file mode 100644 index 000000000..9cc4104f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": true, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..80c3462cb --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep0ms-overflow-true/test-config.yaml @@ -0,0 +1,61 @@ +autostop: + enabled: true + autostop: + - net(777, 2000, 5s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-2000inst-sleep0ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=0ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 40000 + duration: 60s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json new file mode 100644 index 000000000..309d983ca --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": false, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..b7aeba596 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-false/test-config.yaml @@ -0,0 +1,62 @@ +autostop: + enabled: true + autostop: + - quantile (50,60ms,10s) + - instances(2000,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-2000inst-sleep50ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=50ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 40000 + duration: 60s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json new file mode 100644 index 000000000..9cc4104f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": true, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..d51fcdc89 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-2000inst-sleep50ms-overflow-true/test-config.yaml @@ -0,0 +1,61 @@ +autostop: + enabled: true + autostop: + - net(777, 2100, 5s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-2000inst-sleep50ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=50ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 2000 + rps: + - type: line + from: 1 + to: 40000 + duration: 60s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json new file mode 100644 index 000000000..309d983ca --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": false, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml new file mode 100644 index 000000000..149bcd7c7 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-false/test-config.yaml @@ -0,0 +1,62 @@ +autostop: + enabled: true + autostop: + - quantile (50,4ms,10s) + - instances(300,10s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-300inst-sleep0ms-overflow-false + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=0ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 300 + rps: + - type: line + from: 1 + to: 50000 + duration: 70s + discard_overflow: false + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} diff --git a/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json new file mode 100644 index 000000000..9cc4104f8 --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/meta.json @@ -0,0 +1,15 @@ +{ + "labels": { + "type": "pandora-perf", + "generator": "pandora", + "target": "local-mock", + "pandora_generator": "http", + "provider": "uri-inconfig", + "discard_overflow": true, + "previder_preloaded": false, + "shared_client": 0 + }, + "external_data": [ + {"name": "pandora-perf", "s3bucket": "pandora-perf", "s3file": "pandora-perf"} + ] +} \ No newline at end of file diff --git a/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml new file mode 100644 index 000000000..dd9d5b76b --- /dev/null +++ b/performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms-overflow-true/test-config.yaml @@ -0,0 +1,61 @@ +autostop: + enabled: true + autostop: + - net(777, 6100, 5s) + package: yandextank.plugins.Autostop + report_file: autostop_report.txt +uploader: + enabled: true + package: yandextank.plugins.DataUploader + job_name: pandora-perf-http-uri-300inst-sleep0ms-overflow-true + job_dsc: '' + ver: '' + api_address: loadtesting.api.cloud.yandex.net:443 +pandora: + enabled: true + package: yandextank.plugins.Pandora + pandora_cmd: ./pandora-perf + config_content: + pools: + - id: HTTP + gun: + type: http + target: localhost:8092 + ssl: false + ammo: + type: uri + uris: + - /hello?sleep=0ms hellotag + result: + type: phout + destination: ./phout.log + startup: + type: once + times: 300 + rps: + - type: line + from: 1 + to: 50000 + duration: 60s + discard_overflow: true + log: + level: error + monitoring: + expvar: + enabled: true + port: 1234 +telegraf: + enabled: true + package: yandextank.plugins.Telegraf + config: + hosts: + localhost: null + metrics: + cpu: null + mem: null + diskio: null + net: null + netstat: null + system: null + kernel: null +core: {} From c89292ca47ecb3c70343343059725c9842234f5a Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Tue, 25 Jun 2024 12:35:21 +0300 Subject: [PATCH 2/3] discard overflow sliding window discard overflow sliding window 119a0330b912b4023932f3612b25cb36c8792fed --- .changes/v0.5.28.md | 3 + .github/workflows/test.yml | 83 ++++--------------- .mapping.json | 2 +- CHANGELOG.md | 4 + cli/cli.go | 2 +- core/coreutil/waiter.go | 23 ++--- docs/eng/best_practices/discard-overflow.md | 3 +- docs/rus/best_practices.md | 3 - docs/rus/best_practices/discard-overflow.md | 3 +- performance-test/automation/_agent_create.sh | 0 performance-test/automation/_agent_delete.sh | 0 .../automation/_compose_test_create_args.sh | 0 performance-test/automation/_functions.sh | 0 performance-test/automation/_test_check.sh | 0 performance-test/automation/_test_run.sh | 0 performance-test/automation/_variables.sh | 0 performance-test/automation/agent.sh | 0 .../automation/default_check_report.sh | 0 .../automation/default_check_summary.sh | 0 performance-test/automation/s3_upload.sh | 0 performance-test/automation/test.sh | 0 21 files changed, 40 insertions(+), 86 deletions(-) create mode 100644 .changes/v0.5.28.md delete mode 100644 docs/rus/best_practices.md mode change 100644 => 100755 performance-test/automation/_agent_create.sh mode change 100644 => 100755 performance-test/automation/_agent_delete.sh mode change 100644 => 100755 performance-test/automation/_compose_test_create_args.sh mode change 100644 => 100755 performance-test/automation/_functions.sh mode change 100644 => 100755 performance-test/automation/_test_check.sh mode change 100644 => 100755 performance-test/automation/_test_run.sh mode change 100644 => 100755 performance-test/automation/_variables.sh mode change 100644 => 100755 performance-test/automation/agent.sh mode change 100644 => 100755 performance-test/automation/default_check_report.sh mode change 100644 => 100755 performance-test/automation/default_check_summary.sh mode change 100644 => 100755 performance-test/automation/s3_upload.sh mode change 100644 => 100755 performance-test/automation/test.sh diff --git a/.changes/v0.5.28.md b/.changes/v0.5.28.md new file mode 100644 index 000000000..c41a54b85 --- /dev/null +++ b/.changes/v0.5.28.md @@ -0,0 +1,3 @@ +## v0.5.28 - 2024-06-24 +### Changed +* `discard_overflow` logic. Waiter wait 2 seconds sliding window before skip payload diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a52b12b35..320827441 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,15 +10,6 @@ on: - master - dev -env: - YC_LT_FOLDER_ID: b1gacohsvc2kc4d76tu5 - YC_LT_AUTHORIZED_KEY_JSON: ${{ secrets.YC_LOADTESTING_CI_AUTHORIZED_KEY_JSON }} - YC_LT_TEST_AGENT_FILTER: "name = 'agent-pandora-perf-medium'" - YC_LT_TEST_EXTRA_DESCRIPTION: "GitHub Actions workflow - ${{github.run_id}}" - YC_LT_SKIP_TEST_CHECK: "1" - YC_LT_DATA_BUCKET: ${{ secrets.YC_LT_DATA_BUCKET }} - YC_LT_OUTPUT_DIR: ${{github.workspace}}/performance-test/output - jobs: run-unit-tests: name: Unit Tests @@ -28,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu] + os: [ubuntu, macOS] env: OS: ${{ matrix.os }}-latest GO: ${{ matrix.go-version }} @@ -37,64 +28,18 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Parse to Release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_ID: 161082234 - run: | - cat < imbalance.txt - ## Performance tests - - pandora-perf-grpc-300inst-sleep0ms: 12344 - - pandora-perf-grpc-3000inst-sleep0ms: 15555 - EOF - - sed G imbalance.txt > imbalance.md - - - name: Update release - id: update_release - uses: tubone24/update_release@v1.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG_NAME: v0.5.27.alpha13 + - name: Install Go + uses: actions/setup-go@v3 with: - body_path: ./imbalance.md - is_append_body: true + go-version: 1.21.x + cache: true -# build-and-upload: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: ./.github/actions/setup-yc -# - name: Install Go -# uses: actions/setup-go@v3 -# with: -# go-version: 1.21.x -# cache: true -# - name: Test -# run: go test -race -covermode atomic ./... -# - name: Build -# run: | -# export GOOS=linux -# export CGO_ENABLED=0 -# go build -o pandora_perf_2 -# - name: Upload -# run: | -# source performance-test/automation/_functions.sh && source performance-test/automation/_variables.sh; yc_s3_upload ./pandora_perf_2 pandora-perf ${YC_LT_DATA_BUCKET} -# -# test-pandora-perf: -# needs: [ build-and-upload ] -# runs-on: ubuntu-latest -# concurrency: { group: loadtesting } -# steps: -# - uses: actions/checkout@v4 -# - uses: ./.github/actions/setup-yc -# - name: Run Test HTTP-300inst-sleep0ms -# run: | -# stripped_tag="${{ github.event.release.tag_name }}" -# STRIPPED_TAG=${stripped_tag:1} -# YC_LT_VERBOSE=2 YC_LT_TEST_EXTRA_LABELS="version=${STRIPPED_TAG}" ./performance-test/automation/test.sh ./performance-test/test-config/pandora-perf-http-uri-300inst-sleep0ms -# - name: Upload Artifacts GRPC 300inst-sleep0ms -# uses: actions/upload-artifact@v4 -# with: -# name: pandora-perf-grpc-300inst-sleep0ms -# path: ${{ env.YC_LT_OUTPUT_DIR }} + - name: Test + run: go test -race -coverprofile unit.txt -covermode atomic ./... + + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./unit.txt + flags: unit,${{ matrix.os }},go-${{ matrix.go-version }} + name: unit diff --git a/.mapping.json b/.mapping.json index 2b9242b50..15e8ad0d8 100644 --- a/.mapping.json +++ b/.mapping.json @@ -25,6 +25,7 @@ ".changes/v0.5.25.md":"load/projects/pandora/.changes/v0.5.25.md", ".changes/v0.5.26.md":"load/projects/pandora/.changes/v0.5.26.md", ".changes/v0.5.27.md":"load/projects/pandora/.changes/v0.5.27.md", + ".changes/v0.5.28.md":"load/projects/pandora/.changes/v0.5.28.md", ".changie.yaml":"load/projects/pandora/.changie.yaml", ".github/actions/setup-yc/action.yml":"load/projects/pandora/.github/actions/setup-yc/action.yml", ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml", @@ -293,7 +294,6 @@ "docs/index.md":"load/projects/pandora/docs/index.md", "docs/rus/architecture.md":"load/projects/pandora/docs/rus/architecture.md", "docs/rus/best-practices.md":"load/projects/pandora/docs/rus/best-practices.md", - "docs/rus/best_practices.md":"load/projects/pandora/docs/rus/best_practices.md", "docs/rus/best_practices/discard-overflow.md":"load/projects/pandora/docs/rus/best_practices/discard-overflow.md", "docs/rus/best_practices/rps-per-instance.md":"load/projects/pandora/docs/rus/best_practices/rps-per-instance.md", "docs/rus/best_practices/shared-client.md":"load/projects/pandora/docs/rus/best_practices/shared-client.md", diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b960e620..ef093dc65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## v0.5.28 - 2024-06-24 +### Changed +* `discard_overflow` logic. Waiter wait 2 seconds sliding window before skip payload + ## v0.5.27 - 2024-06-18 ### Added * Performance test on release diff --git a/cli/cli.go b/cli/cli.go index 2572fae43..42dc6f3d7 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap/zapcore" ) -const Version = "0.5.27" +const Version = "0.5.28" const defaultConfigFile = "load" const stdinConfigSelector = "-" diff --git a/core/coreutil/waiter.go b/core/coreutil/waiter.go index 867a10ef8..3b46ff84e 100644 --- a/core/coreutil/waiter.go +++ b/core/coreutil/waiter.go @@ -7,10 +7,12 @@ import ( "github.com/yandex/pandora/core" ) +const MaxOverdueDuration = 2 * time.Second + // Waiter goroutine unsafe wrapper for efficient waiting schedule. type Waiter struct { - sched core.Schedule - slowDownItems int + sched core.Schedule + overdueDuration time.Duration // Lazy initialized. timer *time.Timer @@ -28,28 +30,29 @@ func (w *Waiter) Wait(ctx context.Context) (ok bool) { // Check, that context is not done. Very quick: 5 ns for op, due to benchmark. select { case <-ctx.Done(): - w.slowDownItems = 0 + w.overdueDuration = 0 return false default: } next, ok := w.sched.Next() if !ok { - w.slowDownItems = 0 + w.overdueDuration = 0 return false } // Get current time lazily. // For once schedule, for example, we need to get it only once. - if next.Before(w.lastNow) { - w.slowDownItems++ + waitFor := next.Sub(w.lastNow) + if waitFor <= 0 { + w.overdueDuration = 0 - waitFor return true } w.lastNow = time.Now() - waitFor := next.Sub(w.lastNow) + waitFor = next.Sub(w.lastNow) if waitFor <= 0 { - w.slowDownItems++ + w.overdueDuration = 0 - waitFor return true } - w.slowDownItems = 0 + w.overdueDuration = 0 // Lazy init. We don't need timer for unlimited and once schedule. if w.timer == nil { w.timer = time.NewTimer(waitFor) @@ -70,7 +73,7 @@ func (w *Waiter) IsSlowDown(ctx context.Context) (ok bool) { case <-ctx.Done(): return false default: - return w.slowDownItems >= 2 + return w.overdueDuration >= MaxOverdueDuration } } diff --git a/docs/eng/best_practices/discard-overflow.md b/docs/eng/best_practices/discard-overflow.md index d57a1d2c0..a8ff89aee 100644 --- a/docs/eng/best_practices/discard-overflow.md +++ b/docs/eng/best_practices/discard-overflow.md @@ -23,7 +23,8 @@ The instance setting `discard_overflow` determines which behavior to follow. instances. 2. `discard_overflow: true` - Strict adherence to the request schedule by the generator. Requests that do not fit into the schedule are discarded. The test duration is predetermined. Requests that fail to meet the schedule are marked as - failed (with a net error `777`, and also tagged as discarded). + failed (with a net error `777`, and also tagged as discarded). Pandora considers a test to have failed schedule, if + the time of the request is 2 seconds behind. That is 2 second sliding window is used. By default, starting from version pandora@0.5.24, the setting `discard_overflow: true` is enabled. diff --git a/docs/rus/best_practices.md b/docs/rus/best_practices.md deleted file mode 100644 index 65bc489bb..000000000 --- a/docs/rus/best_practices.md +++ /dev/null @@ -1,3 +0,0 @@ -# Практики использования - -- [RPS per instance](./best_practices/rps_per_instance.md) diff --git a/docs/rus/best_practices/discard-overflow.md b/docs/rus/best_practices/discard-overflow.md index d95b856f1..f349d3976 100644 --- a/docs/rus/best_practices/discard-overflow.md +++ b/docs/rus/best_practices/discard-overflow.md @@ -22,7 +22,8 @@ и количества инстансов. 2. `discard_overflow: true` - строгое следование генератором расписания запросов. Запросы, не уложившиеся в расписание, отбрасываются. Время выполнения теста предопределено. Запросы, которые не укладываются в расписание, - помечаются неудавшимися (ошибка net `777`, а так же добавляется tag:discarded). + помечаются неудавшимися (ошибка net `777`, а так же добавляется tag:discarded). Пандора считает, что тест не уложился + в расписание, если время запроса отстало на 2 сек. То есть используется 2 секундное скользящее окно. По-умолчанию, начиная с версии pandora@0.5.24 настройка `discard_overflow: true` diff --git a/performance-test/automation/_agent_create.sh b/performance-test/automation/_agent_create.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_agent_delete.sh b/performance-test/automation/_agent_delete.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_compose_test_create_args.sh b/performance-test/automation/_compose_test_create_args.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_functions.sh b/performance-test/automation/_functions.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_test_check.sh b/performance-test/automation/_test_check.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_test_run.sh b/performance-test/automation/_test_run.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/_variables.sh b/performance-test/automation/_variables.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/agent.sh b/performance-test/automation/agent.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/default_check_report.sh b/performance-test/automation/default_check_report.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/default_check_summary.sh b/performance-test/automation/default_check_summary.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/s3_upload.sh b/performance-test/automation/s3_upload.sh old mode 100644 new mode 100755 diff --git a/performance-test/automation/test.sh b/performance-test/automation/test.sh old mode 100644 new mode 100755 From 14d9dd0dba607a396bcb7323a9d09fbd8de02d04 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Thu, 27 Jun 2024 12:55:26 +0300 Subject: [PATCH 3/3] HTTP scenario var/header postprocessor use multiple pipes 803e9ad581cd4a1790bd6c50c41f58c27724f6ac --- .changes/v0.5.29.md | 3 + .mapping.json | 1 + CHANGELOG.md | 4 + cli/cli.go | 2 +- .../scenario/http/postprocessor/var_header.go | 21 +- .../http/postprocessor/var_header_test.go | 359 +++++++++--------- docs/eng/scenario-grpc-generator.md | 2 +- docs/eng/scenario-http-generator.md | 76 ++-- docs/rus/scenario-grpc-generator.md | 2 +- docs/rus/scenario-http-generator.md | 72 ++-- examples/http/server/server.go | 3 +- .../testdata/http_scenario/http_payload.hcl | 15 +- tests/http_scenario/testdata/http_payload.hcl | 18 +- 13 files changed, 327 insertions(+), 251 deletions(-) create mode 100644 .changes/v0.5.29.md diff --git a/.changes/v0.5.29.md b/.changes/v0.5.29.md new file mode 100644 index 000000000..bea0eac09 --- /dev/null +++ b/.changes/v0.5.29.md @@ -0,0 +1,3 @@ +## v0.5.29 - 2024-06-25 +### Added +* HTTP scenario var/header postprocessor use multiple pipes diff --git a/.mapping.json b/.mapping.json index 15e8ad0d8..4434395ce 100644 --- a/.mapping.json +++ b/.mapping.json @@ -26,6 +26,7 @@ ".changes/v0.5.26.md":"load/projects/pandora/.changes/v0.5.26.md", ".changes/v0.5.27.md":"load/projects/pandora/.changes/v0.5.27.md", ".changes/v0.5.28.md":"load/projects/pandora/.changes/v0.5.28.md", + ".changes/v0.5.29.md":"load/projects/pandora/.changes/v0.5.29.md", ".changie.yaml":"load/projects/pandora/.changie.yaml", ".github/actions/setup-yc/action.yml":"load/projects/pandora/.github/actions/setup-yc/action.yml", ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml", diff --git a/CHANGELOG.md b/CHANGELOG.md index ef093dc65..15fdc93d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## v0.5.29 - 2024-06-25 +### Added +* HTTP scenario var/header postprocessor use multiple pipes + ## v0.5.28 - 2024-06-24 ### Changed * `discard_overflow` logic. Waiter wait 2 seconds sliding window before skip payload diff --git a/cli/cli.go b/cli/cli.go index 42dc6f3d7..9a0e82813 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap/zapcore" ) -const Version = "0.5.28" +const Version = "0.5.29" const defaultConfigFile = "load" const stdinConfigSelector = "-" diff --git a/components/providers/scenario/http/postprocessor/var_header.go b/components/providers/scenario/http/postprocessor/var_header.go index 0746fe4c0..7a25a465b 100644 --- a/components/providers/scenario/http/postprocessor/var_header.go +++ b/components/providers/scenario/http/postprocessor/var_header.go @@ -46,15 +46,22 @@ func (p *VarHeaderPostprocessor) parseValue(v string) (value string, modifier fu if len(vals) == 1 { return vals[0], func(in string) string { return in }, nil } - if len(vals) > 2 { - return "", nil, fmt.Errorf("VarHeaderPostprocessor supports only one modifier yet") - } - modifier, err = p.parseModifier(vals[1]) - if err != nil { - return "", nil, fmt.Errorf("failed to parse modifier %s: %w", vals[1], err) + + value = vals[0] + modifier = func(in string) string { return in } + + for _, modStr := range vals[1:] { + mod, err := p.parseModifier(modStr) + if err != nil { + return "", nil, fmt.Errorf("failed to parse modifier %s: %w", modStr, err) + } + previousModifier := modifier + modifier = func(in string) string { + return mod(previousModifier(in)) + } } - return vals[0], modifier, nil + return value, modifier, nil } func (p *VarHeaderPostprocessor) parseModifier(s string) (func(in string) string, error) { diff --git a/components/providers/scenario/http/postprocessor/var_header_test.go b/components/providers/scenario/http/postprocessor/var_header_test.go index d45add8d7..0eb308c66 100644 --- a/components/providers/scenario/http/postprocessor/var_header_test.go +++ b/components/providers/scenario/http/postprocessor/var_header_test.go @@ -13,8 +13,8 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { name string mappings map[string]string respHeaders map[string]string - expectedMap map[string]any - expectErr bool + wantMap map[string]any + wantErr bool }{ { name: "No Headers", @@ -23,7 +23,7 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { "key2": "header2", }, respHeaders: map[string]string{}, - expectedMap: map[string]any{}, + wantMap: map[string]any{}, }, { name: "No Fields", @@ -31,7 +31,7 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { respHeaders: map[string]string{ "key1": "header1", "key2": "header2"}, - expectedMap: nil, + wantMap: nil, }, { name: "Error in Fields", @@ -39,8 +39,8 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { "key1": "header1||", }, respHeaders: map[string]string{}, - expectedMap: map[string]any{}, - expectErr: true, + wantMap: map[string]any{}, + wantErr: true, }, { name: "Headers Exist", @@ -49,18 +49,24 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { "key2": "header2|lower", "key3": "header3|upper", "key4": "header4|substr(1,3)", + "key5": "header5|lower|replace(s,x)|substr(1,3)", + "auth": "Authorization|lower|replace(=,)|substr(6)", }, respHeaders: map[string]string{ - "header1": "Value1", - "header2": "Value2", - "header3": "Value3", - "header4": "Value4", + "header1": "Value1", + "header2": "Value2", + "header3": "Value3", + "header4": "Value4", + "header5": "aSdFgHjKl", + "Authorization": "Basic Ym9zY236Ym9zY28=", }, - expectedMap: map[string]any{ + wantMap: map[string]any{ "key1": "Value1", "key2": "value2", "key3": "VALUE3", "key4": "al", + "key5": "xd", + "auth": "ym9zy236ym9zy28", }, }, } @@ -76,81 +82,79 @@ func TestVarHeaderPostprocessor_Process(t *testing.T) { } reqMap, err := p.Process(resp, nil) - if tt.expectErr { + if tt.wantErr { assert.Error(t, err) return } assert.NoError(t, err) - assert.Equal(t, tt.expectedMap, reqMap) + assert.Equal(t, tt.wantMap, reqMap) }) } } func TestVarHeaderPostprocessor_ParseValue(t *testing.T) { - tests := []struct { - name string - input string - modifierVal string - expectedValue string - expectedModifierVal string - expectedError error + name string + input string + wantVal string + wantValAfterModifier string + wantErr error }{ { - name: "No Modifier", - input: "hello", - modifierVal: "asdf", - expectedValue: "hello", - expectedModifierVal: "asdf", - expectedError: nil, + name: "No Modifier", + input: "hello", + wantVal: "hello", + wantValAfterModifier: "hello", + wantErr: nil, + }, + { + name: "Lowercase Modifier", + input: "fOOt|lower", + wantVal: "fOOt", + wantValAfterModifier: "foot", + wantErr: nil, }, { - name: "Lowercase Modifier", - input: "foo|lower", - modifierVal: "ASDF", - expectedValue: "foo", - expectedModifierVal: "asdf", - expectedError: nil, + name: "Uppercase Modifier", + input: "bar|upper", + wantVal: "bar", + wantValAfterModifier: "BAR", + wantErr: nil, }, { - name: "Uppercase Modifier", - input: "bar|upper", - modifierVal: "upper", - expectedValue: "bar", - expectedModifierVal: "UPPER", - expectedError: nil, + name: "Substring Modifier", + input: "asdfghjkl|substr(1,3)", + wantVal: "asdfghjkl", + wantValAfterModifier: "sd", + wantErr: nil, }, { - name: "Substring Modifier", - input: "baz|substr(1,3)", - modifierVal: "asdfghjkl", - expectedValue: "baz", - expectedModifierVal: "sd", - expectedError: nil, + name: "Multiple Modifiers", + input: "aSdFgHjKl|lower|replace(s,x)|substr(1,3)", + wantVal: "aSdFgHjKl", + wantValAfterModifier: "xd", + wantErr: nil, }, { - name: "Multiple Modifiers", - input: "test|lower|upper", - modifierVal: "lower|upper", - expectedValue: "", // The method should return an empty string when multiple modifiers are provided. - expectedModifierVal: "", - expectedError: fmt.Errorf("VarHeaderPostprocessor supports only one modifier yet"), + name: "Multiple Modifiers 2", + input: "aPPliCation-JSONbro|lower|replace(-, /)|substr(0, 16)", + wantVal: "aPPliCation-JSONbro", + wantValAfterModifier: "application/json", + wantErr: nil, }, { - name: "Invalid Modifier", - input: "invalid|unknown", - modifierVal: "unknown", - expectedValue: "", // The method should return an empty string when the modifier is unknown. - expectedModifierVal: "", - expectedError: fmt.Errorf("failed to parse modifier unknown: unknown modifier unknown"), + name: "Invalid Modifier", + input: "invalid|unknown", + wantVal: "", // The method should return an empty string when the modifier is unknown. + wantValAfterModifier: "", + wantErr: fmt.Errorf("failed to parse modifier unknown: unknown modifier unknown"), }, { - name: "Invalid Modifier Arguments", - input: "invalid|substr(abc)", - modifierVal: "substr(abc)", - expectedValue: "", // The method should return an empty string when the modifier arguments are invalid. - expectedModifierVal: "", - expectedError: fmt.Errorf("failed to parse modifier substr(abc): substr modifier requires integer as first argument, got abc"), + name: "Invalid Modifier Arguments", + input: "invalid|substr(abc)", + wantVal: "", // The method should return an empty string when the modifier arguments are invalid. + wantValAfterModifier: "", + wantErr: fmt.Errorf("failed to parse modifier substr(abc): substr modifier requires integer as first argument, got abc"), }, } @@ -159,15 +163,15 @@ func TestVarHeaderPostprocessor_ParseValue(t *testing.T) { p := &VarHeaderPostprocessor{} value, modifier, err := p.parseValue(tt.input) if err != nil { - assert.EqualError(t, err, tt.expectedError.Error()) + assert.EqualError(t, err, tt.wantErr.Error()) return } assert.NoError(t, err) - assert.Equal(t, tt.expectedValue, value) + assert.Equal(t, tt.wantVal, value) - gotModifierVal := modifier(tt.modifierVal) - assert.Equal(t, tt.expectedModifierVal, gotModifierVal) + gotModifierVal := modifier(value) + assert.Equal(t, tt.wantValAfterModifier, gotModifierVal) }) } @@ -177,81 +181,81 @@ func TestVarHeaderPostprocessor_ParseModifier(t *testing.T) { p := &VarHeaderPostprocessor{} tests := []struct { - name string - input string - value string - expectedRes string - expectedError error + name string + input string + value string + want string + wantErr error }{ { - name: "Lowercase Modifier", - input: "lower", - value: "HELLO", - expectedRes: "hello", - expectedError: nil, + name: "Lowercase Modifier", + input: "lower", + value: "HELLO", + want: "hello", + wantErr: nil, }, { - name: "Uppercase Modifier", - input: "upper", - value: "world", - expectedRes: "WORLD", - expectedError: nil, + name: "Uppercase Modifier", + input: "upper", + value: "world", + want: "WORLD", + wantErr: nil, }, { - name: "Substring Modifier - Normal Case", - input: "substr(1,4)", - value: "abcdefgh", - expectedRes: "bcd", - expectedError: nil, + name: "Substring Modifier - Normal Case", + input: "substr(1,4)", + value: "abcdefgh", + want: "bcd", + wantErr: nil, }, { - name: "Substring Modifier - Start Index Out of Range (Negative)", - input: "substr(-2,4)", - value: "abcdefgh", - expectedRes: "ef", - expectedError: nil, + name: "Substring Modifier - Start Index Out of Range (Negative)", + input: "substr(-2,4)", + value: "abcdefgh", + want: "ef", + wantErr: nil, }, { - name: "Substring Modifier - Start Index Greater Than End Index", - input: "substr(5,3)", - value: "abcdefgh", - expectedRes: "de", - expectedError: nil, + name: "Substring Modifier - Start Index Greater Than End Index", + input: "substr(5,3)", + value: "abcdefgh", + want: "de", + wantErr: nil, }, { - name: "Substring Modifier - End Index Beyond Length", - input: "substr(2,100)", - value: "abcdefgh", - expectedRes: "cdefgh", // End index is beyond the length of the input value, so the modifier should return the substring from index 2 to the end. - expectedError: nil, + name: "Substring Modifier - End Index Beyond Length", + input: "substr(2,100)", + value: "abcdefgh", + want: "cdefgh", // End index is beyond the length of the input value, so the modifier should return the substring from index 2 to the end. + wantErr: nil, }, { - name: "Replace Modifier", - input: "replace(a,x)", - value: "banana", - expectedRes: "bxnxnx", - expectedError: nil, + name: "Replace Modifier", + input: "replace(a,x)", + value: "banana", + want: "bxnxnx", + wantErr: nil, }, { - name: "Invalid Modifier", - input: "invalid", - value: "test", - expectedRes: "", // The modFunc will be nil, so expectedRes should be an empty string. - expectedError: fmt.Errorf("unknown modifier invalid"), + name: "Invalid Modifier", + input: "invalid", + value: "test", + want: "", // The modFunc will be nil, so want should be an empty string. + wantErr: fmt.Errorf("unknown modifier invalid"), }, { - name: "Substring Modifier with Invalid Arguments", - input: "substr(2)", - value: "abc", - expectedRes: "c", - expectedError: nil, + name: "Substring Modifier with Invalid Arguments", + input: "substr(2)", + value: "abc", + want: "c", + wantErr: nil, }, { - name: "Replace Modifier with Invalid Arguments", - input: "replace(x)", - value: "abc", - expectedRes: "", // The modFunc will be nil, so expectedRes should be an empty string. - expectedError: fmt.Errorf("replace modifier requires 2 arguments"), + name: "Replace Modifier with Invalid Arguments", + input: "replace(x)", + value: "abc", + want: "", // The modFunc will be nil, so want should be an empty string. + wantErr: fmt.Errorf("replace modifier requires 2 arguments"), }, } @@ -262,13 +266,13 @@ func TestVarHeaderPostprocessor_ParseModifier(t *testing.T) { // If there is an error, modFunc should be nil, and the result should be an empty string. if err != nil { assert.Nil(t, modFunc) - assert.EqualError(t, err, tt.expectedError.Error()) + assert.EqualError(t, err, tt.wantErr.Error()) return } // If there is no error, apply the modFunc and check the result. res := modFunc(tt.value) - assert.Equal(t, tt.expectedRes, res) + assert.Equal(t, tt.want, res) assert.NoError(t, err) }) } @@ -276,74 +280,74 @@ func TestVarHeaderPostprocessor_ParseModifier(t *testing.T) { func TestVarHeaderPostprocessor_Substr(t *testing.T) { tests := []struct { - name string - args []string - value string - expectedRes string - expectedError error + name string + args []string + value string + want string + wantErr error }{ { - name: "Substring Modifier - Normal Case", - args: []string{"1", "4"}, - value: "abcdefgh", - expectedRes: "bcd", - expectedError: nil, + name: "Substring Modifier - Normal Case", + args: []string{"1", "4"}, + value: "abcdefgh", + want: "bcd", + wantErr: nil, }, { - name: "Substring Modifier - Start Index Out of Range (Negative)", - args: []string{"-2", "4"}, - value: "abcdefgh", - expectedRes: "ef", // Start index is negative, so it should count from the end of the string. - expectedError: nil, + name: "Substring Modifier - Start Index Out of Range (Negative)", + args: []string{"-2", "4"}, + value: "abcdefgh", + want: "ef", // Start index is negative, so it should count from the end of the string. + wantErr: nil, }, { - name: "Substring Modifier - End Index Out of Range (Negative)", - args: []string{"1", "-2"}, - value: "abcdefgh", - expectedRes: "bcdef", // End index is negative, so it should count from the end of the string. - expectedError: nil, + name: "Substring Modifier - End Index Out of Range (Negative)", + args: []string{"1", "-2"}, + value: "abcdefgh", + want: "bcdef", // End index is negative, so it should count from the end of the string. + wantErr: nil, }, { - name: "Substring Modifier - Start Index Greater Than End Index", - args: []string{"5", "3"}, - value: "abcdefgh", - expectedRes: "de", // Start index is greater than end index, so the modifier should return the substring from index 3 to 5. - expectedError: nil, + name: "Substring Modifier - Start Index Greater Than End Index", + args: []string{"5", "3"}, + value: "abcdefgh", + want: "de", // Start index is greater than end index, so the modifier should return the substring from index 3 to 5. + wantErr: nil, }, { - name: "Substring Modifier - End Index Beyond Length", - args: []string{"2", "100"}, - value: "abcdefgh", - expectedRes: "cdefgh", // End index is beyond the length of the input value, so the modifier should return the substring from index 2 to the end. - expectedError: nil, + name: "Substring Modifier - End Index Beyond Length", + args: []string{"2", "100"}, + value: "abcdefgh", + want: "cdefgh", // End index is beyond the length of the input value, so the modifier should return the substring from index 2 to the end. + wantErr: nil, }, { - name: "Substring Modifier with Invalid Arguments", - args: []string{"2"}, - value: "abc", - expectedRes: "c", - expectedError: nil, + name: "Substring Modifier with Invalid Arguments", + args: []string{"2"}, + value: "abc", + want: "c", + wantErr: nil, }, { - name: "Substring Modifier with Empty Arguments", - args: []string{}, - value: "abc", - expectedRes: "", // The modFunc will be nil, so expectedRes should be an empty string. - expectedError: fmt.Errorf("substr modifier requires one or two arguments"), + name: "Substring Modifier with Empty Arguments", + args: []string{}, + value: "abc", + want: "", // The modFunc will be nil, so want should be an empty string. + wantErr: fmt.Errorf("substr modifier requires one or two arguments"), }, { - name: "Substring Modifier with Non-Integer Arguments", - args: []string{"abc", "xyz"}, - value: "abc", - expectedRes: "", // The modFunc will be nil, so expectedRes should be an empty string. - expectedError: fmt.Errorf("substr modifier requires integer as first argument, got abc"), + name: "Substring Modifier with Non-Integer Arguments", + args: []string{"abc", "xyz"}, + value: "abc", + want: "", // The modFunc will be nil, so want should be an empty string. + wantErr: fmt.Errorf("substr modifier requires integer as first argument, got abc"), }, { - name: "Substring Modifier with Non-Integer second Argument", - args: []string{"1", "xyz"}, - value: "abc", - expectedRes: "", // The modFunc will be nil, so expectedRes should be an empty string. - expectedError: fmt.Errorf("substr modifier requires integer as second argument, got xyz"), + name: "Substring Modifier with Non-Integer second Argument", + args: []string{"1", "xyz"}, + value: "abc", + want: "", // The modFunc will be nil, so want should be an empty string. + wantErr: fmt.Errorf("substr modifier requires integer as second argument, got xyz"), }, } @@ -351,18 +355,13 @@ func TestVarHeaderPostprocessor_Substr(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &VarHeaderPostprocessor{} modFunc, err := p.substr(tt.args) - - // If there is an error, modFunc should be nil, and the result should be an empty string. if err != nil { assert.Nil(t, modFunc) - assert.EqualError(t, err, tt.expectedError.Error()) + assert.EqualError(t, err, tt.wantErr.Error()) return } - - // If there is no error, apply the modFunc and check the result. - res := modFunc(tt.value) - assert.Equal(t, tt.expectedRes, res) assert.NoError(t, err) + assert.Equal(t, tt.want, modFunc(tt.value)) }) } } diff --git a/docs/eng/scenario-grpc-generator.md b/docs/eng/scenario-grpc-generator.md index cdc2add3e..3e222972d 100644 --- a/docs/eng/scenario-grpc-generator.md +++ b/docs/eng/scenario-grpc-generator.md @@ -317,7 +317,7 @@ Follow - [Variable sources](scenario/variable_source.md) # References -- [HTTP generator](http-generator.md) +- [gRPC generator](grpc-generator.md) - Best practices - [RPS per instance](best_practices/rps-per-instance.md) - [Shared client](best_practices/shared-client.md) diff --git a/docs/eng/scenario-http-generator.md b/docs/eng/scenario-http-generator.md index 1b458282b..984f66744 100644 --- a/docs/eng/scenario-http-generator.md +++ b/docs/eng/scenario-http-generator.md @@ -297,9 +297,11 @@ For more details about randomization functions, see [more](scenario/functions.md HCL example ```terraform -postprocessor "var/jsonpath" { - mapping = { - token = "$.auth_key" +request "your_request_name" { + postprocessor "var/jsonpath" { + mapping = { + token = "$.auth_key" + } } } ``` @@ -307,33 +309,61 @@ postprocessor "var/jsonpath" { ##### var/xpath ```terraform -postprocessor "var/xpath" { - mapping = { - data = "//div[@class='data']" +request "your_request_name" { + postprocessor "var/xpath" { + mapping = { + data = "//div[@class='data']" + } } } ``` ##### var/header -Creates a new variable from response headers +Creates a new variable from the response headers. -It is possible to specify simple string manipulations via pipe +It is possible to specify simple string manipulations via pipe. +Modifiers: - lower - upper -- substr(from, length) +- substr(from, length) - where `length` is optional - replace(search, replace) +Modifiers can be chained. + +**Example:** + +If you get a response with the headers +```http request +X-Trace-ID: we1fswe284awsfewf +Authorization: Basic Ym9zY236Ym9zY28= +``` + +And you need to save for future use `traceID=we1fswe284awsfewf` & `auth=ym9zy236ym9zy28` + +you can use the postprocessor with the modifiers + ```terraform -postprocessor "var/header" { - mapping = { - ContentType = "Content-Type|upper" - httpAuthorization = "Http-Authorization" +request "your_request_name" { + postprocessor "var/header" { + mapping = { + traceID = "X-Trace-ID" + auth = "Authorization|lower|replace(=,)|substr(6)" + } } } ``` +In templates you can use the result of this postprocessor as + +Translated with DeepL.com (free version) +```gotemplate +`{% raw %}{{.request.your_request_name.postprocessor.auth}}{% endraw %}` +`{% raw %}{{.request.your_request_name.postprocessor.traceID}}{% endraw %}` +``` + + ##### assert/response Checks header and body content @@ -341,16 +371,18 @@ Checks header and body content Upon assertion, further scenario execution is dropped ```terraform -postprocessor "assert/response" { - headers = { - "Content-Type" = "application/json" - } - body = ["token"] - status_code = 200 +request "your_request_name" { + postprocessor "assert/response" { + headers = { + "Content-Type" = "application/json" + } + body = ["token"] + status_code = 200 - size { - val = 10000 - op = ">" + size { + val = 10000 + op = ">" + } } } ``` diff --git a/docs/rus/scenario-grpc-generator.md b/docs/rus/scenario-grpc-generator.md index 3032476e4..c387af08a 100644 --- a/docs/rus/scenario-grpc-generator.md +++ b/docs/rus/scenario-grpc-generator.md @@ -319,7 +319,7 @@ scenario "scenario_name" { # Смотри так же -- [HTTP генератор](http-generator.md) +- [gRPC генератор](grpc-generator.md) - Практики использования - [RPS на инстанс](best_practices/rps-per-instance.md) - [Общий транспорт](best_practices/shared-client.md) diff --git a/docs/rus/scenario-http-generator.md b/docs/rus/scenario-http-generator.md index bc1d325f0..7339b2e2b 100644 --- a/docs/rus/scenario-http-generator.md +++ b/docs/rus/scenario-http-generator.md @@ -300,9 +300,11 @@ request "req_name" { Пример hcl ```terraform -postprocessor "var/jsonpath" { - mapping = { - token = "$.auth_key" +request "your_request_name" { + postprocessor "var/jsonpath" { + mapping = { + token = "$.auth_key" + } } } ``` @@ -310,33 +312,57 @@ postprocessor "var/jsonpath" { ##### var/xpath ```terraform -postprocessor "var/xpath" { - mapping = { - data = "//div[@class='data']" +request "your_request_name" { + postprocessor "var/xpath" { + mapping = { + data = "//div[@class='data']" + } } } ``` ##### var/header -Создает новую переменную из заголовков ответа +Создает новую переменную из заголовков ответа. -Есть возможность через pipe указывать простейшие строковые манипуляции +Есть возможность через pipe указывать простейшие строковые манипуляции. +Модификаторы: - lower - upper -- substr(from, length) +- substr(from, length) - где `length` - опционально - replace(search, replace) +Модификаторы можно выстраивать в цепочку. + +**Пример:** + +Если вам приходит ответ с заголовками +```http request +X-Trace-ID: we1fswe284awsfewf +Authorization: Basic Ym9zY236Ym9zY28= +``` + +И вам требуется сохранить для дальнейшего использования `traceID=we1fswe284awsfewf` & `auth=ym9zy236ym9zy28` +вы можете использовать постпроцессор с модификаторами + ```terraform -postprocessor "var/header" { - mapping = { - ContentType = "Content-Type|upper" - httpAuthorization = "Http-Authorization" +request "your_request_name" { + postprocessor "var/header" { + mapping = { + traceID = "X-Trace-ID" + auth = "Authorization|lower|replace(=,)|substr(6)" + } } } ``` +В шаблонах вы можете использовать результать данного постпроцессора как +```gotemplate +`{% raw %}{{.request.your_request_name.postprocessor.auth}}{% endraw %}` +`{% raw %}{{.request.your_request_name.postprocessor.traceID}}{% endraw %}` +``` + ##### assert/response Проверяет значения заголовков и тела @@ -344,16 +370,18 @@ postprocessor "var/header" { Если матчинг не срабатывает, прекращает дальнейшее выполнение сценария ```terraform -postprocessor "assert/response" { - headers = { - "Content-Type" = "application/json" - } - body = ["token"] - status_code = 200 +request "your_request_name" { + postprocessor "assert/response" { + headers = { + "Content-Type" = "application/json" + } + body = ["token"] + status_code = 200 - size { - val = 10000 - op = ">" + size { + val = 10000 + op = ">" + } } } ``` diff --git a/examples/http/server/server.go b/examples/http/server/server.go index 655853256..22fd6adff 100644 --- a/examples/http/server/server.go +++ b/examples/http/server/server.go @@ -108,7 +108,8 @@ func (s *Server) authHandler(w http.ResponseWriter, r *http.Request) { s.mu.RUnlock() w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(fmt.Sprintf(`{"auth_key": "%s"}`, authKey))) + w.Header().Set("Authorization", "Bearer "+authKey) + _, _ = w.Write([]byte(`{"result":"ok"}`)) } func (s *Server) listHandler(w http.ResponseWriter, r *http.Request) { diff --git a/tests/acceptance/testdata/http_scenario/http_payload.hcl b/tests/acceptance/testdata/http_scenario/http_payload.hcl index 596cbf0b0..7fc103d54 100644 --- a/tests/acceptance/testdata/http_scenario/http_payload.hcl +++ b/tests/acceptance/testdata/http_scenario/http_payload.hcl @@ -39,31 +39,32 @@ EOF mapping = { Content-Type = "Content-Type|upper" httpAuthorization = "Http-Authorization" + auth = "Authorization|substr(7)" } } postprocessor "var/jsonpath" { mapping = { - token = "$.auth_key" + result = "$.result" } } postprocessor "assert/response" { headers = { Content-Type = "json" } - body = ["key"] + body = ["{\"result\":\"ok\"}"] size { - val = 40 op = ">" + val = 10 } } postprocessor "assert/response" { - body = ["auth"] + body = ["result"] } } request "list_req" { method = "GET" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" } @@ -81,7 +82,7 @@ request "order_req" { method = "POST" uri = "/order" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" } @@ -101,7 +102,7 @@ request "order_req2" { method = "POST" uri = "/order" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" } diff --git a/tests/http_scenario/testdata/http_payload.hcl b/tests/http_scenario/testdata/http_payload.hcl index a031697de..042f5f59d 100644 --- a/tests/http_scenario/testdata/http_payload.hcl +++ b/tests/http_scenario/testdata/http_payload.hcl @@ -37,33 +37,33 @@ EOF } postprocessor "var/header" { mapping = { - Content-Type = "Content-Type|upper" - httpAuthorization = "Http-Authorization" + Content-Type = "Content-Type|upper" + auth = "Authorization|substr(7)" } } postprocessor "var/jsonpath" { mapping = { - token = "$.auth_key" + result = "$.result" } } postprocessor "assert/response" { headers = { Content-Type = "json" } - body = ["key"] + body = ["{\"result\":\"ok\"}"] size { - val = 40 op = ">" + val = 10 } } postprocessor "assert/response" { - body = ["auth"] + body = ["result"] } } request "list_req" { method = "GET" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" } @@ -81,7 +81,7 @@ request "order_req" { method = "POST" uri = "/order" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" } @@ -101,7 +101,7 @@ request "order_req2" { method = "POST" uri = "/order" headers = { - Authorization = "Bearer {{.request.auth_req.postprocessor.token}}" + Authorization = "Bearer {{.request.auth_req.postprocessor.auth}}" Content-Type = "application/json" Useragent = "Yandex" }