run-on-runner.yml

name: Run Benchmarks on EC2 Runner

on:
  workflow_dispatch:
    inputs:
      runner_name:
        required: true
        type: string
      instance_type:
        required: true
        type: string
      enable_gpu:
        required: true
        type: string
      provers:
        required: false
        type: string
        default: 'sp1'
      programs:
        required: false
        type: string
        default: 'loop,fibonacci,tendermint,reth1,reth2'
      filename:
        required: false
        type: string
        default: 'benchmark'
      trials:
        required: false
        type: string
        default: '1'
      sp1_ref:
        required: false
        type: string
      additional_params:
        required: false
        type: string
        default: '{"hashfns":"poseidon","shard_sizes":"22"}'

jobs:
  run-benchmark:
    name: Run Benchmark on ${{ inputs.runner_name }}
    runs-on: ${{ inputs.runner_name }}
    steps:
      - name: Echo Workflow Inputs
        run: |
          echo "Runner Name: ${{ inputs.runner_name }}"
          echo "Instance Type: ${{ inputs.instance_type }}"
          echo "Enable GPU: ${{ inputs.enable_gpu }}"
          echo "Provers: ${{ inputs.provers }}"
          echo "Programs: ${{ inputs.programs }}"
          echo "Filename: ${{ inputs.filename }}"
          echo "Trials: ${{ inputs.trials }}"
          echo "SP1 Ref: ${{ inputs.sp1_ref }}"
          echo "Additional Params: ${{ inputs.additional_params }}"
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Set up Docker
        uses: docker/setup-buildx-action@v2

      - name: Parse additional parameters
        id: parse-params
        run: |
          echo 'ADDITIONAL_PARAMS<<EOF' >> $GITHUB_ENV
          echo '${{ inputs.additional_params }}' >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v3

      # - name: Cache Docker layers
      #   uses: actions/cache@v3
      #   with:
      #     path: /tmp/.buildx-cache
      #     key: ${{ runner.os }}-buildx-${{ github.sha }}
      #     restore-keys: |
      #       ${{ runner.os }}-buildx-

      # - name: Build and cache Docker image
      #   uses: docker/build-push-action@v5
      #   with:
      #     context: .
      #     file: ./Dockerfile.gpu
      #     push: false
      #     load: true
      #     tags: zkvm-perf
      #     build-args: |
      #       SP1_REF=${{ inputs.sp1_ref }}
      #     platforms: linux/amd64
      #     cache-from: type=local,src=/tmp/.buildx-cache
      #     cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max

      # # This step is necessary to prevent the cache from growing indefinitely
      # - name: Move cache
      #   run: |
      #     rm -rf /tmp/.buildx-cache
      #     mv /tmp/.buildx-cache-new /tmp/.buildx-cache

      - name: Build Docker image
        run: |
          docker build -t zkvm-perf --platform linux/amd64 -f Dockerfile.gpu --build-arg SP1_REF=${{ inputs.sp1_ref }} .

      - name: Make benchmarks directory
        run: mkdir -p ${{ github.workspace }}/benchmarks

      - name: rust-cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
            ~/.rustup/
          key: rust-1.79.0-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: rust-1.79.0-

      - name: Run benchmark
        run: |
          docker run ${{ inputs.enable_gpu == 'true' && '--gpus all' || '' }} --platform linux/amd64 \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v ${{ github.workspace }}/benchmarks:/usr/src/app/benchmarks \
            -v ~/.cargo/registry:/root/.cargo/registry \
            -v ~/.cargo/git:/root/.cargo/git \
            -v ~/.rustup:/root/.rustup \
            -e RUST_BACKTRACE=full \
            -e INSTANCE_TYPE=${{ inputs.instance_type }} \
            --network host \
            zkvm-perf \
            "python3 sweep.py --filename ${{ inputs.filename }} --trials ${{ inputs.trials }} --programs $(echo ${{ inputs.programs }} | sed 's/,/ /g') --provers ${{ inputs.provers }} --hashfns ${{ fromJson(env.ADDITIONAL_PARAMS).hashfns }} --shard-sizes ${{ fromJson(env.ADDITIONAL_PARAMS).shard_sizes }}"

      - name: List benchmark results
        run: ls -la ${{ github.workspace }}/benchmarks

      - name: Add instance type to CSV
        run: |
          for file in ${{ github.workspace }}/benchmarks/*.csv; do
            sed -i '1s/^/instance_type,/' "$file"
            sed -i "2,\$s/^/${{ inputs.instance_type }},/" "$file"
          done

      - name: Upload benchmark results
        uses: actions/upload-artifact@v3
        with:
          name: benchmark-results-${{ inputs.instance_type }}-${{ github.run_id }}
          path: ${{ github.workspace }}/benchmarks/*.csv

      - name: Print Results
        run: |
          cat ${{ github.workspace }}/benchmarks/*.csv