diff --git a/.github/actions/parse-ci-config/README.md b/.github/actions/parse-ci-config/README.md new file mode 100644 index 00000000..6b26bec9 --- /dev/null +++ b/.github/actions/parse-ci-config/README.md @@ -0,0 +1,40 @@ +# parse-ci-config + +This action parses the CI testing configuration file. The caller of the action needs to checkout the branch where the config file is defined before running this action. + +## Inputs + +| Name | Type | Description | Required | Example | +| ---- | ---- | ----------- | -------- | ------- | +| check | `string` | The type of check/test to run | true | `scheduled` | +| branch-or-tag | `string` | The name of git branch or tag | true | `release-1deg_jra55_ryf-2.0` | +| config-filepath | `string` | Path to configuration file | true | `config/ci.json` | + +## Outputs + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | -------- | +| markers | `string` | Markers used for the pytest checks, in the python format | `checksum` | +| model-config-tests-version | `string` | The version of the model-config-tests | `0.0.1` | +| python-version | `string` | The python version used to create test virtual environment | `3.11.0` | + +## Example usage + +```yaml +# --------- + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Read scheduled test config + id: scheduled-config + uses: access-nri/access-om2-configs/.github/actions/parse-ci-config@main + with: + check: scheduled + branch-or-tag: "release-1deg_jra55_ryf-2.0" + config-filepath: "config/ci.json" +``` + + diff --git a/.github/actions/parse-ci-config/action.yml b/.github/actions/parse-ci-config/action.yml new file mode 100644 index 00000000..69483203 --- /dev/null +++ b/.github/actions/parse-ci-config/action.yml @@ -0,0 +1,53 @@ +name: Parse CI Config File +description: Action to parse model-config-tests configurations for CI tests +inputs: + check: + required: true + description: Type of check/test to run (e.g. "reproducibility", "qa" or "scheduled") + branch-or-tag: + required: true + description: Name of Git branch or tag to run CI testing on + config-filepath: + required: true + description: Path to CI configuration file +outputs: + model-config-tests-version: + value: ${{ steps.read-config.outputs.model-config-tests-version }} + description: A version of the model-config-tests package + python-version: + value: ${{ steps.read-config.outputs.python-version }} + description: The python version used to create test virtual environment + markers: + value: ${{ steps.read-config.outputs.markers }} + description: A python expression of markers to pass to model-config-tests pytests +runs: + using: "composite" + steps: + - name: Read Configuration File + shell: bash + id: read-config + run: | + # Fall back to default config values if not defined for a given branch or tag + output=$(jq --arg branch "${{ inputs.branch-or-tag }}" --arg check "${{ inputs.check }}" ' + { + "model-config-tests-version": ( + .[$check][$branch]["model-config-tests-version"] // + .[$check].default["model-config-tests-version"] // + .default["model-config-tests-version"] + ), + "python-version": ( + .[$check][$branch]["python-version"] // + .[$check].default["python-version"] // + .default["python-version"] + ), + "markers": ( + .[$check][$branch].markers // + .[$check].default.markers // + .default.markers + ), + } + ' "${{ inputs.config-filepath }}") + + echo "markers=$(echo "$output" | jq -r '.["markers"]')" >> $GITHUB_OUTPUT + echo "python-version=$(echo "$output" | jq -r '.["python-version"]')" >> $GITHUB_OUTPUT + echo "model-config-tests-version=$(echo "$output" | jq -r '.["model-config-tests-version"]')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/generate-initial-checksums.yml b/.github/workflows/generate-initial-checksums.yml index d1f7567b..49b01489 100644 --- a/.github/workflows/generate-initial-checksums.yml +++ b/.github/workflows/generate-initial-checksums.yml @@ -28,10 +28,31 @@ jobs: - run: | echo '::notice::This deployment is using the following inputs: `config-branch-name`=`${{ inputs.config-branch-name }}`, `commit-checksums`=`${{ inputs.commit-checksums }}`, `committed-checksum-location`=`${{ inputs.committed-checksum-location }}`, `committed-checksum-tag-version`=`${{ inputs.committed-checksum-tag-version }}`.' + config: + name: Read Testing Configuration + runs-on: ubuntu-latest + outputs: + python-version: ${{ steps.repro-config.outputs.python-version }} + model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Read reproducibility tests config + id: repro-config + uses: access-nri/access-om2-configs/.github/actions/parse-ci-config@main + with: + check: reproducibility + branch-or-tag: ${{ inputs.config-branch-name }} + config-filepath: "config/ci.json" + generate-checksums: name: Generate Checksums needs: - log-inputs + - config uses: access-nri/reproducibility/.github/workflows/generate-initial-checksums.yml@main with: model-name: access-om2 @@ -40,6 +61,8 @@ jobs: committed-checksum-location: ${{ inputs.committed-checksum-location }} committed-checksum-tag: "${{ inputs.config-branch-name }}-${{ inputs.committed-checksum-tag-version }}" environment-name: "Gadi Initial Checksum" + model-config-tests-version: ${{ needs.config.outputs.model-config-tests-version }} + python-version: ${{ needs.config.outputs.python-version }} permissions: contents: write secrets: inherit diff --git a/.github/workflows/pr-1-ci.yml b/.github/workflows/pr-1-ci.yml index 605a05e8..5ceda00c 100644 --- a/.github/workflows/pr-1-ci.yml +++ b/.github/workflows/pr-1-ci.yml @@ -1,22 +1,6 @@ name: PR Checks on: workflow_call: - inputs: - qa-pytest-markers: - type: string - required: false - default: config - description: List of markers for the pytest QA CI checks - qa-pytest-add-model-markers: - type: boolean - required: false - default: true - description: Whether to run model-specific tests - repro-pytest-markers: - type: string - required: false - default: checksum - description: List of markers for the pytest repro CI checks # Workflows that call this workflow use the following triggers: # pull_request: # branches: @@ -88,11 +72,44 @@ jobs: Rename the Source branch or check the Target branch, and try again. run: gh pr comment --body '${{ env.BODY }}' + config: + name: Read CI Testing Configuration + runs-on: ubuntu-latest + outputs: + qa-markers: ${{ steps.qa-config.outputs.markers }} + qa-python-version: ${{ steps.qa-config.outputs.python-version }} + qa-model-config-tests-version: ${{ steps.qa-config.outputs.model-config-tests-version }} + repro-markers: ${{ steps.repro-config.outputs.markers }} + repro-python-version: ${{ steps.repro-config.outputs.python-version }} + repro-model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Read QA tests config + id: qa-config + uses: access-nri/access-om2-configs/.github/actions/parse-ci-config@main + with: + check: qa + branch-or-tag: ${{ github.base_ref }} + config-filepath: "config/ci.json" + + - name: Read reproducibility tests config + id: repro-config + uses: access-nri/access-om2-configs/.github/actions/parse-ci-config@main + with: + check: reproducibility + branch-or-tag: ${{ github.base_ref }} + config-filepath: "config/ci.json" + qa-ci: # Run quick, non-HPC tests on the runner. name: QA CI Checks needs: - commit-check + - config if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME runs-on: ubuntu-latest permissions: @@ -102,45 +119,24 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - path: pr - - name: Checkout Tests - uses: actions/checkout@v4 - with: - repository: access-nri/model-config-tests - # Temporarily pin model-config-tests to a commit - # TODO: Update to use model-config-tests package - ref: b086103bb4a2ff87f7d6bb82cd767ab173b1e089 - path: pytest - - - name: Setup Python 3.11 + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: ${{ needs.config.outputs.qa-python-version }} cache: pip - - name: Install requirements.txt - working-directory: ./pytest - run: pip install -r ./requirements.txt - - - name: Get Final List of Markers - id: pytest-markers - run: | - if [[ ${{ inputs.qa-pytest-add-model-markers }} == "true" ]]; then - echo "markers=${{ inputs.qa-pytest-markers }} or access_om2" >> $GITHUB_OUTPUT - else - echo "markers=${{ inputs.qa-pytest-markers }}" >> $GITHUB_OUTPUT - fi + - name: Install model-config-tests + run: pip install model-config-tests=='${{ needs.config.outputs.qa-model-config-tests-version }}' - name: Invoke Simple CI Pytests # We continue on error because we will let the checks generated in # the next step speak to the state of the testing continue-on-error: true - working-directory: ./pr run: | - echo "Running pytest using '-m ${{ steps.pytest-markers.outputs.markers }}'" - pytest ../pytest \ - -m '${{ steps.pytest-markers.outputs.markers }}' \ + echo "Running pytest using '-m ${{ needs.config.outputs.qa-markers }}'" + model-config-tests \ + -m '${{ needs.config.outputs.qa-markers }}' \ --target-branch '${{ github.base_ref }}' \ --junitxml=./test_report.xml @@ -148,7 +144,7 @@ jobs: id: tests uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 with: - files: ./pr/test_report.xml + files: ./test_report.xml comment_mode: off check_run: true check_name: QA Test Results @@ -162,13 +158,17 @@ jobs: needs: - commit-check - branch-check + - config if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && needs.branch-check.result == 'success' uses: access-nri/reproducibility/.github/workflows/checks.yml@main with: model-name: access-om2 environment-name: Gadi config-tag: ${{ github.head_ref }} - test-markers: ${{ inputs.repro-pytest-markers }} + test-markers: ${{ needs.config.outputs.repro-markers }} + model-config-tests-version: ${{ needs.config.outputs.repro-model-config-tests-version }} + python-version: ${{ needs.config.outputs.repro-python-version }} + secrets: inherit permissions: contents: write diff --git a/.github/workflows/schedule-1-ci.yml b/.github/workflows/schedule-1-ci.yml index 55ed60bb..9721b378 100644 --- a/.github/workflows/schedule-1-ci.yml +++ b/.github/workflows/schedule-1-ci.yml @@ -16,7 +16,7 @@ jobs: - name: Get all released configs id: get-released-config - run: echo "tags=$(jq --compact-output --raw-output '.tags' config/released-configs.json)" >> $GITHUB_OUTPUT + run: echo "tags=$(jq --compact-output --raw-output '.scheduled | del(.default) | keys[]' config/ci.json)" >> $GITHUB_OUTPUT repro-ci: # We use this reusable workflow with a matrix strategy rather than calling repro-ci.yml, as diff --git a/.github/workflows/schedule-2-start.yml b/.github/workflows/schedule-2-start.yml index f782ab39..48ba89bd 100644 --- a/.github/workflows/schedule-2-start.yml +++ b/.github/workflows/schedule-2-start.yml @@ -7,15 +7,40 @@ on: required: true description: Tag associated with a config branch that is used in the reproducibility run jobs: + config: + name: Read CI Testing Configuration + runs-on: ubuntu-latest + outputs: + markers: ${{ steps.scheduled-config.outputs.markers }} + python-version: ${{ steps.scheduled-config.outputs.python-version }} + model-config-tests-version: ${{ steps.scheduled-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Read scheduled tests config + id: scheduled-config + uses: access-nri/access-om2-configs/.github/actions/parse-ci-config@main + with: + check: scheduled + branch-or-tag: ${{ inputs.config-tag }} + config-filepath: "config/ci.json" + repro-ci: # Run the given config on the deployment Github Environment (`environment-name`) and upload # the test results and checksum. + needs: + - config uses: access-nri/reproducibility/.github/workflows/checks.yml@main with: model-name: access-om2 environment-name: Gadi config-tag: ${{ inputs.config-tag }} - test-markers: checksum + test-markers: ${{ needs.config.outputs.markers }} + model-config-tests-version: ${{ needs.config.outputs.model-config-tests-version }} + python-version: ${{ needs.config.outputs.python-version }} secrets: inherit permissions: contents: write diff --git a/README-DEV.md b/README-DEV.md index 8f814645..2607fd8b 100644 --- a/README-DEV.md +++ b/README-DEV.md @@ -22,13 +22,6 @@ Merge: `pr-3-bump-tag.yml` #### The PR CI Lifecycle: `pr-1-ci.yml` This file does the bulk of the handling of the PR. -It contains the following inputs, when the PR-triggered `call-pr-1-ci.yml` calls it: - -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `qa-pytest-markers` | `string` | Markers used for the pytest QA CI checks, in the python format | `false` | `config or metadata` | `config or metadata or highres` | -| `qa-pytest-add-model-markers` | `boolean` | Markers used for the pytest QA CI checks, in the python format | `false` | `false` | `true` | -| `repro-pytest-markers` | `string` | Markers used for the pytest repro CI checks, in the python format | `false` | `checksum` | `checksum or performance` | ##### `commit-check` @@ -40,6 +33,10 @@ The first job, `commit-check`, is used to short-circuit execution of this workfl This job is used as a check before running [`repro-ci`](#repro-ci) checks, which are only run on PRs `dev-*` -> `release-*`. It also makes sure that the branches are formatted correctly. +##### `config` + +This job reads configuration file `config/ci.json` to obtain pytest markers, `model-config-tests` version and python version for running QA and reproducibility tests. + ##### `qa-ci` These checks are runner-hosted, quick configuration sanity checks that will always fire on PRs into `release-*` or `dev-*`. @@ -99,12 +96,15 @@ schedule-1-ci ---- schedule-2-start [release-1deg_jra55_iaf-1.1] #### Matrix Creation: `schedule-1-ci.yml` -This workflow is responsible for getting all the config tags that require monthly checks (defined in `config/released-configs.json`) and spawning a matrix job for each of those. +This workflow is responsible for getting all the config tags that require monthly checks (defined in `config/ci.json` under `scheduled`) and spawning a matrix job for each of those. As an aside, the reason that we call the reusable workflow `schedule-2-start.yml` with a matrix strategy is that matrix strategies only work at the `job` level. So, if you need a matrix to work across multiple jobs (for example, one job does a task with a particular matrix value, and one reports the result of that task), you need a job that calls a reusable workflow, which in turn contains multiple jobs. #### Config Tag Specific Checks: `schedule-2-start.yml` +##### `config` +Similar to the [`pr-1-ci.yml` counterpart](#config) - parses the CI configuration file for markers, `model-config-tests` and python versions for scheduled reproducibility checks. + ##### `repro-ci` Exactly the same as the [`pr-1-ci.yml` counterpart](#repro-ci) - run the config associated with the given config tag and upload the checksum. @@ -126,3 +126,20 @@ This workflow is used to check that modifications to `*.json` files are in line #### Initial Checksum Creation: `generate-initial-checksums.yml` This `workflow_dispatch`-triggered workflow generates checksums of a given config branch, and optionally commits them. This is useful for generating checksums for an entirely new config branch, so the workflows above have something to compare against. + +### CI Configuration File + +This is the `config/ci.json` configuration file for specifying different test markers, or test versions based on type of the test to run, and the name of the git branch or tag. The different types of test are defined as: +- `scheduled`: These are scheduled monthly reproducibility tests that are run as part of [`schedule-2-start.yml`](#repro-ci-1). The keys under these tests represent released config tags to run scheduled checks on. +- `reproducibility`: These are reproducibility tests are run as part of [`pr-1-ci.yml`](#repro-ci). The keys under these tests represent the target branches into which pull requests are being merged. +- `qa` - These are quick QA tests are run as part of [`pr-1-ci.yml`](#qa-ci). The keys under these tests represent the target branches into which pull requests are being merged. + +The configuration properties needed to run the tests are: + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | -------- | +| markers | `string` | Markers used for the pytest checks, in the python format | `checksum` | +| model-config-tests-version | `string` | The version of the model-config-tests | `0.0.1` | +| python-version | `string` | The python version used to create test virtual environment | `3.11.0` | + +As most of the tests use the same test and python versions, and similar markers, there are two levels of defaults. There's a default at test type level which is useful for defining test markers - this selects certain pytests to run in `model-config-tests`. There is an outer global default, which is used if a property is not defined for a given branch/tag, and it is not defined for the test default. The `parse-ci-config` action applies the fall-back default logic. For more information on using this action see [`parse-ci-config` README.md](./.github/actions/parse-ci-config/README.md). \ No newline at end of file diff --git a/config/ci.json b/config/ci.json new file mode 100644 index 00000000..171133d4 --- /dev/null +++ b/config/ci.json @@ -0,0 +1,24 @@ +{ + "$schema": "./ci.schema.json", + "scheduled": { + "release-1deg_jra55_ryf-2.0": {}, + "release-1deg_jra55_iaf_bgc-2.0": {}, + "default": { + "markers": "checksum" + } + }, + "reproducibility": { + "default": { + "markers": "checksum" + } + }, + "qa": { + "default": { + "markers": "access_om2 or config" + } + }, + "default": { + "model-config-tests-version": "0.0.1", + "python-version": "3.11.0" + } +} \ No newline at end of file diff --git a/config/ci.schema.json b/config/ci.schema.json new file mode 100644 index 00000000..6151f47d --- /dev/null +++ b/config/ci.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Model Configuration CI Testing Configuration", + "description": "Settings for Continuous Integration (CI) tests for model configurations repositories", + "type": "object", + "$defs": { + "config": { + "type": "object", + "properties": { + "markers": { + "type": "string", + "description": "A python expression of markers to pass to model-config-tests pytests" + }, + "model-config-tests-version": { + "type": "string", + "description": "A version of the model-config-tests package" + }, + "python-version": { + "type": "string", + "description": "The python version used to create test virtual environment" + } + }, + "additionalProperties": false + }, + "check": { + "type": "object", + "patternProperties": { + "^.*$": { + "$ref": "#/$defs/config", + "description": "The name of the git branch or tag" + }, + "default": { + "$ref": "#/$defs/config", + "required": [ + "markers" + ], + "description": "The default configuration for this check" + } + }, + "required": [ + "default" + ], + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "scheduled": { + "$ref": "#/$defs/check", + "description": "Scheduled reproducibility checks. The keys are config tags to run scheduled checks on" + }, + "reproducibility": { + "$ref": "#/$defs/check", + "description": "Reproducibility checks. The keys are the target branch names for pull requests" + }, + "qa": { + "$ref": "#/$defs/check", + "description": "Quick quality assurance checks. The keys are the target branch names for pull requests" + }, + "default": { + "$ref": "#/$defs/config", + "required": [ + "model-config-tests-version", + "python-version" + ], + "description": "Global default configuration" + } + }, + "required": [ + "$schema", + "scheduled", + "reproducibility", + "qa", + "default" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/config/released-configs.json b/config/released-configs.json deleted file mode 100644 index 019a85a9..00000000 --- a/config/released-configs.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "./released-configs.schema.json", - "tags": [ - "release-1deg_jra55_ryf-2.0", - "release-1deg_jra55_iaf_bgc-2.0" - ] -} \ No newline at end of file diff --git a/config/released-configs.schema.json b/config/released-configs.schema.json deleted file mode 100644 index 528b0d25..00000000 --- a/config/released-configs.schema.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Deployment Environments", - "description": "A list of deployment targets that are supported", - "type": "object", - "properties": { - "$schema": { - "type": "string" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": ["tags"], - "additionalProperties": false -} \ No newline at end of file