Large Scale Performance Tests

.github/workflows/large_scale_performance_tests.yml

@@ -0,0 +1,137 @@
+name: Large Scale Performance Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_suite:
+        type: choice
+        required: true
+        options:
+          - large_scale_performance_locustfile.py
+        description: >
+          Location of the Locust file that contains the test suite.
+
+      number_of_users:
+        type: string
+        required: true
+        default: "100"
+        description: >
+          Total number of simulated users.
+
+      user_spawn_rate:
+        type: string
+        required: true
+        default: "10"
+        description: >
+          Users spawned per second.
+
+      duration:
+        type: string
+        required: true
+        default: "1h"
+        description: >
+          Duration of the test (e.g., 1h, 24h, 72h).
+
+      marqo_host:
+        type: string
+        required: true
+        default: "https://your-marqo-instance.com"
+        description: >
+          Marqo host to run performance test on.
+
+      index_name:
+        type: string
+        required: true
+        default: "locust-test"
+        description: >
+          Index name to test.
+
+      marqo_cloud_api_key:
+        type: string
+        required: true
+        description: >
+          Marqo Cloud API key to use for the test.
+
+jobs:
+  Extract-Data:
+    name: Extract Required Data
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install Dependencies
+        run: |
+          pip install pandas
+
+      - name: Extract Required Data
+        run: |
+          python perf_tests/extract_required_data.py
+
+      - name: Upload Extracted Data as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: extracted-data
+          path: perf_tests/data/extracted_*.csv
+
+  Run-Performance-Test:
+    name: Run Large Scale Performance Tests
+    needs: Extract-Data
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Download Extracted Data
+        uses: actions/download-artifact@v3
+        with:
+          name: extracted-data
+          path: perf_tests/data/
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install Dependencies
+        run: |
+          pip install marqo locust numpy pandas
+
+      - name: Prepare Test Environment
+        run: |
+          mkdir -p perf_tests/data
+          cp perf_tests/large_scale_performance_locustfile.py perf_tests/
+          cp perf_tests/bquxjob_5a025798_19241877e37.csv perf_tests/
+          # Extracted data is already downloaded to perf_tests/data/
+
+      - name: Run Locust Test
+        working-directory: perf_tests
+        run: |
+          locust -f ${{ github.event.inputs.test_suite }} \
+            -u ${{ github.event.inputs.number_of_users }} \
+            -r ${{ github.event.inputs.user_spawn_rate }} \
+            --run-time ${{ github.event.inputs.duration }} \
+            --headless \
+            --host ${{ github.event.inputs.marqo_host }}
+        env:
+          MARQO_INDEX_NAME: ${{ github.event.inputs.index_name }}
+          MARQO_CLOUD_API_KEY: ${{ github.event.inputs.marqo_cloud_api_key }}
+
+      - name: Upload Telemetry Data
+        uses: actions/upload-artifact@v3
+        with:
+          name: telemetry-data
+          path: perf_tests/telemetry_data_*.json
+
+      - name: Upload Telemetry Summaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: telemetry-summaries
+          path: perf_tests/telemetry_summary_*.json

.gitignore

@@ -150,4 +150,10 @@ dump.rdb
 .DS_Store
 
 # Tester app for unit tests
-scripts/vespa_local/vespa_tester_app.zip
+scripts/vespa_local/vespa_tester_app.zip
+
+# Large scale performance testing
+/perf_tests/*.csv
+/perf_tests/*/*.csv
+/perf_tests/*.json
+/perf_tests/*.json1

perf_tests/analyze_telemetry.py

@@ -0,0 +1,60 @@
+import json
+import glob
+import numpy as np
+import pandas as pd
+import os
+from datetime import datetime
+
+def load_telemetry_data(pattern='telemetry_data_*.json'):
+    telemetry_files = glob.glob(pattern)
+    all_data = []
+
+    for file in telemetry_files:
+        with open(file, 'r') as f:
+            data = json.load(f)
+            all_data.extend(data)
+
+    return all_data
+
+def analyze_telemetry(all_data):
+    if not all_data:
+        print("No telemetry data to analyze.")
+        return
+
+    # Convert to DataFrame
+    df = pd.DataFrame(all_data)
+
+    # Ensure 'total_time_ms' is numeric
+    df['total_time_ms'] = pd.to_numeric(df['total_time_ms'], errors='coerce')
+
+    # Calculate metrics
+    avg_time = df['total_time_ms'].mean()
+    median_time = df['total_time_ms'].median()
+    p95_time = df['total_time_ms'].quantile(0.95)
+    total_requests = len(df)
+    successful_requests = df['total_time_ms'].notna().sum()
+    error_rate = ((total_requests - successful_requests) / total_requests) * 100 if total_requests > 0 else 0.0
+
+    summary = {
+        'Average Response Time (ms)': avg_time,
+        'Median Response Time (ms)': median_time,
+        '95th Percentile Response Time (ms)': p95_time,
+        'Total Requests': total_requests,
+        'Successful Requests': successful_requests,
+        'Error Rate (%)': error_rate,
+    }
+
+    # Save summary to JSON
+    timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
+    summary_filename = f"telemetry_summary_{timestamp}.json"
+    with open(summary_filename, 'w') as f:
+        json.dump(summary, f, indent=4)
+
+    # Print summary
+    print("Telemetry Summary:")
+    for key, value in summary.items():
+        print(f"{key}: {value}")
+
+if __name__ == "__main__":
+    all_data = load_telemetry_data()
+    analyze_telemetry(all_data)