WIP: Prod Testing Workflows

.github/actions/taskcluster-artifact/action.yml

@@ -0,0 +1,34 @@
+name: Download taskcluster artifact
+descrion: Download artifacts from a Taskcluster task
+inputs:
+  taskid:
+    description: 'Taskcluster task identifier to fetch artifacts from'
+    required: true
+    type: string
+  artifact:
+    description: "Artifact to download from the task"
+    required: true
+    type: string
+  path:
+    description: Location to save the artifact
+    default: ${{ github.workspace }}
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Wait for taskcluster job
+      id: taskgraph
+      shell: bash
+      run: |
+        pip install requests
+        ${{ github.action_path }}/await-taskcluster-status.py -t 3600 ${{ inputs.taskid }} | tee ${{ runner.temp }}/taskcluster-job-status.json
+        echo run-id=$(jq '.runs[] | select(.state == "completed").runId' ${{ runner.temp }}/taskcluster-job-status.json) >> $GITHUB_OUTPUT
+
+    - name: Download build artifact
+      shell: bash
+      env:
+        TASKGRAPH_TASK_URL: https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/${{ inputs.taskid }}
+      run: |
+        curl -sSL -o ${{ inputs.path }}/$(basename ${{ inputs.artifact }}) \
+          ${TASKGRAPH_TASK_URL}/runs/${{ steps.taskgraph.outputs.run-id }}/artifacts/${{ inputs.artifact }}

.github/actions/taskcluster-artifact/await-taskcluster-status.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+import argparse
+import requests
+import json
+import os
+import sys
+import time
+
+def get_job_status(taskid, api):
+    # Request the job status from taskcluster
+    r = requests.post(api, json={"taskIds": [taskid]})
+    if r.status_code != 200:
+        r.raise_for_status()
+
+    # We only requested one job status, so return it.
+    return r.json()["statuses"]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Await a external check to complete")
+    parser.add_argument("taskid", metavar="TASKID", type=str, action="store",
+                        help="Task ID to wait for")
+    parser.add_argument("-i", "--interval", metavar="SEC", type=int, action="store", default=15,
+                        help="Interval between retries")
+    parser.add_argument("-t", "--timeout", metavar="SEC", type=int, action="store", default=300,
+                        help="Task timeout in seconds")
+    parser.add_argument("-a", "--api", metavar="URL", type=str, action="store",
+                        default='https://firefox-ci-tc.services.mozilla.com/api/queue/v1/tasks/status',
+                        help="Taskgraph status API")
+    args = parser.parse_args()
+
+    expiration = time.monotonic() + args.timeout
+    while time.monotonic() < expiration:
+        try:
+            js = get_job_status(args.taskid, args.api)
+            if len(js) == 0:
+                raise Exception("not found")
+            status = js[0]["status"]
+        except Exception as e:
+            status = {
+                "taskId": args.taskid,
+                "state": str(e),
+                "runs": [],
+            }
+
+        if status["state"] == 'completed':
+            print(f'Task {args.taskid} - {status["state"]} - exiting', file=sys.stderr)
+            print(f'{json.dumps(status, indent=3)}')
+            sys.exit(0)
+
+        # Wait and try again        
+        maxsleep = expiration - time.monotonic()
+        if maxsleep > args.interval:
+            maxsleep = args.interval
+
+        print(f'Task {args.taskid} - {status["state"]} - retrying in {maxsleep}', file=sys.stderr)
+        time.sleep(maxsleep)
+
+    # Otherwise a timeout occurred.
+    sys.exit(1)

.github/scripts/await-taskcluster-job.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import argparse
+import requests
+import json
+import os
+import sys
+
+GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
+
+def fetch(url, version='2022-11-28', timeout=None):
+    headers = {
+        'Accept': 'application/vnd.github+json',
+        'X-GitHub-Api-Version': version,
+    }
+    if GITHUB_TOKEN is not None:
+        headers['Authorization'] = f"Bearer {GITHUB_TOKEN}"
+
+    r = requests.get(url, headers)
+    if r.status_code != 200:
+        r.raise_for_status()
+    return r.json()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Await a external check to complete")
+    parser.add_argument("ref", metavar="REF", type=str, action="store",
+                        help="Github reference fetch checks for")
+    parser.add_argument("name", metavar="NAME", type=str, action="store",
+                        help="Name of the check to wait for")
+    parser.add_argument("-a", "--app", metavar="SUITE", type=str, action="store",
+                        help="App name to fetch checks from")
+    args = parser.parse_args()
+
+    # Find the URL to fetch checks from.
+    checks_url = None
+    if args.app:
+        js = fetch(f"https://api.github.com/repos/mozilla-mobile/mozilla-vpn-client/commits/{args.ref}/check-suites")
+        for suite in js['check_suites']:
+            if suite["app"]["name"] == args.app:
+                checks_url = suite["check_runs_url"]
+        if not checks_url:
+            raise KeyError(f"No suite {args.app} found for ${args.ref}")
+    else:
+        checks_url = f"https://api.github.com/repos/mozilla-mobile/mozilla-vpn-client/commits/{args.ref}/check-runs"
+
+    # Fetch the JSON for the desired check.
+    js = fetch(checks_url)
+    for check in js['check_runs']:
+        if check["name"] == args.name:
+            json.dump(check, sys.stdout)
+
+    print(f"DEBUG: {checks_url}")

.github/workflows/prod-test-suite.yaml

@@ -0,0 +1,179 @@
+# A reusable workflow to build addons
+on:
+  workflow_call:
+    inputs:
+      artifact-name:
+        description: 'Name of the taskcluster artifact to download and test'
+        required: true
+        type: string
+      ref:
+        description: 'The branch, tag or SHA of functional tests to checkout.'
+        default: ${{ github.ref }}
+        type: string
+      runs-on:
+        description: 'The type of machine to run the job on.'
+        default: 'ubuntu-latest'
+        type: string
+      group-name:
+        description: 'Group name for intermediate test artifacts'
+        default: 'test'
+        type: string
+      task-id:
+        description: 'Taskcluster task to fetch artifacts from'
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: Build Test Artifacts
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.enumerate.outputs.tests }}
+      runners: ${{ steps.enumerate.outputs.runners }}
+      uniqueid: ${{ steps.enumerate.outputs.uniqueid }}
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Setup addons cache
+        id: addons-cache
+        uses: actions/cache@v4
+        with:
+          path: build-addons/
+          key: cache-addons-${{ hashFiles('addons/', 'test/functional/addons/') }}
+
+      - name: Enumerate tests and runners
+        id: enumerate
+        run: |
+          echo -n "tests=" >> $GITHUB_OUTPUT
+          for test in $(find tests/functional -name 'test*.js' | sort); do
+            printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test
+          done | jq -s -c >> $GITHUB_OUTPUT
+
+          echo -n "runners=" >> $GITHUB_OUTPUT
+          echo "${{ inputs.runs-on }}" | jq --raw-input -c '[., inputs] | map(select(length > 0))' >> $GITHUB_OUTPUT
+
+      - name: Install build dependencies
+        if: steps.addons-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          git submodule init
+          git submodule update 3rdparty/i18n
+          sudo apt-get update
+          sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control)
+
+      - name: Build test addons
+        if: steps.addons-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          mkdir -p build-addons/
+          cmake -S $(pwd)/tests/functional/addons -B build-addons/
+          cmake --build build-addons/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.group-name }}-addons
+          path: build-addons/
+
+      - name: Fetch taskgraph artifact
+        uses: ./.github/actions/taskcluster-artifact
+        with:
+          taskid: ${{ inputs.task-id }}
+          artifact: public/build/${{ inputs.artifact-name }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.group-name }}-artifact-${{ inputs.artifact-name }}
+          path: ${{ inputs.artifact-name }}
+
+  run-tests:
+    needs:
+      - build
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false # Don't cancel other jobs if a test fails
+      matrix:
+        runner: ${{ fromJson(needs.build.outputs.runners) }}
+        test: ${{ fromJson(needs.build.outputs.tests) }}
+
+    runs-on: ${{ matrix.runner }}
+    name: Test ${{ matrix.test.name }} (${{ matrix.runner }})
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.group-name }}-artifact-${{ inputs.artifact-name }}
+          path: build/
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.group-name }}-addons
+          path: build/addons/
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+          cache: "pip"
+      - run: pip install -r requirements.txt
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: "npm"
+      - run: npm install
+
+      - name: Install VPN client
+        if: ${{ runner.os != 'Windows' }}
+        shell: bash
+        run: |
+          case ${{ inputs.artifact-name }} in
+            *.deb)
+              sudo apt install ${{ github.workspace }}/build/${{ inputs.artifact-name }}
+              echo "MVPN_BIN=$(which mozillavpn)" >> $GITHUB_ENV
+              ;;
+
+            *.pkg)
+              sudo installer -pkg ${{ github.workspace }}/build/${{ inputs.artifact-name }} -target /
+              echo "MVPN_BIN=/Applications/Mozilla VPN.app/Contents/MacOS/Mozilla VPN" >> $GITHUB_ENV
+              ;;
+
+            *.tar.gz)
+              tar -xzvf ${{ github.workspace }}/build/${{ inputs.artifact-name }} -C ${{ github.workspace }}/build --strip-components=1
+              ls -al ${{ github.workspace }}/build
+              echo MVPN_BIN="${{ github.workspace }}/build/Mozilla VPN.app/Contents/MacOS/Mozilla VPN" >> $GITHUB_ENV
+              ;;
+
+            *)
+              echo "Unsupported installation file type: ${{ inputs.artifact-name }}"
+              exit 1
+              ;;
+          esac
+          echo "MVPN_ADDONS_PATH=${{ github.workspace }}/build/addons" >> $GITHUB_ENV
+
+      - name: Install VPN client
+        if: ${{ runner.os == 'Windows' }}
+        shell: powershell
+        run: |
+          New-Item -Path . -Name "msiexec.log" -ItemType "file"
+          $pmsi = Start-Process "msiexec" "/i ${{ github.workspace }}\build\${{ inputs.artifact-name }} /qn /l*! msiexec.log" -NoNewWindow -PassThru
+          $plog = Start-Process "powershell" "Get-Content -Path msiexec.log -Wait" -NoNewWindow -PassThru
+          $pmsi.WaitForExit()
+          $plog.Kill()
+
+          Add-Content -Path $env:GITHUB_ENV -Value "MVPN_BIN=C:\Program Files\Mozilla\Mozilla VPN\Mozilla VPN.exe"
+          Add-Content -Path $env:GITHUB_ENV -Value "MVPN_ADDONS_PATH=${{ github.workspace }}\build\addons"
+
+          taskkill /f /fi "USERNAME eq $env:username" /im "Mozilla VPN.exe"
+
+      - name: Running ${{ matrix.test.name }} Tests
+        shell: bash
+        env:
+          TZ: Europe/London
+          HEADLESS: yes
+        run: npm run functionalTest --  --retries 3 ${{ matrix.test.path }}