From 6ba960eea8bee4ee094282fa24efd1ddccda603d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Jeanneret?= Date: Fri, 17 May 2024 21:41:53 +0200 Subject: [PATCH] Generate build jobs for all automations This patch provides a script generating all of the zuul.d content, based on the automation files. Each scenario will get its own job definition, and will be triggered only when its files are changed. Note that common files, such as "lib", will trigger all the jobs. This patch also introduces a new github workflow, to ensure all of the automation scenarios have a job, by re-running the script and ensuring no chance happened in zuul.d directory. --- .github/workflows/linter.yaml | 20 ++++ .yamllint.yml | 1 + automation/mocks/ovs-dpdk.yaml | 1 + automation/mocks/sriov.yaml | 1 + create-zuul-jobs.py | 48 ++++++++++ zuul.d/{validate-jobs.yaml => base-jobs.yaml} | 18 +--- zuul.d/projects.yaml | 13 ++- zuul.d/validations.yaml | 96 +++++++++++++++++++ 8 files changed, 177 insertions(+), 21 deletions(-) create mode 120000 automation/mocks/ovs-dpdk.yaml create mode 120000 automation/mocks/sriov.yaml create mode 100755 create-zuul-jobs.py rename zuul.d/{validate-jobs.yaml => base-jobs.yaml} (52%) create mode 100644 zuul.d/validations.yaml diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 35ad1b934..6baf6df88 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -41,3 +41,23 @@ jobs: run: | kustomize version ./test-kustomizations.sh examples/ + + test-zuul: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install yaml + run: pip install pyyaml + + - name: Run create-zuul-jobs + run: | + sha256sum zuul.d/* > checksum; + ./create-zuul-jobs.py; + sha256sum -c checksum; diff --git a/.yamllint.yml b/.yamllint.yml index 6e869be18..5d7667aa0 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -2,6 +2,7 @@ extends: default ignore: - '*.md' + - 'zuul.d/' rules: line-length: diff --git a/automation/mocks/ovs-dpdk.yaml b/automation/mocks/ovs-dpdk.yaml new file mode 120000 index 000000000..5149849d9 --- /dev/null +++ b/automation/mocks/ovs-dpdk.yaml @@ -0,0 +1 @@ +ovs-dpdk-sriov.yaml \ No newline at end of file diff --git a/automation/mocks/sriov.yaml b/automation/mocks/sriov.yaml new file mode 120000 index 000000000..5149849d9 --- /dev/null +++ b/automation/mocks/sriov.yaml @@ -0,0 +1 @@ +ovs-dpdk-sriov.yaml \ No newline at end of file diff --git a/create-zuul-jobs.py b/create-zuul-jobs.py new file mode 100755 index 000000000..b3b6f446d --- /dev/null +++ b/create-zuul-jobs.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import os +import yaml + +_AUTO_PATH = 'automation/vars/' +_JOBS = [] +_PROJECTS = ['noop'] +_BASE_TRIGGER_FILES = [ + 'lib', + ] + +def create_job(name, data): + j_name = 'rhoso-architecture-validate-{}'.format(name) + _paths = [p['path'] for p in data['stages']] + _BASE_TRIGGER_FILES + for i in ['va', 'dt']: + if os.path.exists(os.path.join(i, name)): + _paths.append(os.path.join(i, name)) + _mock = os.path.join('automation', 'mocks', '{}.yaml'.format(name)) + if os.path.exists(_mock): + _paths.append(_mock) + _paths.sort() + job = { + 'job': { + 'name': j_name, + 'parent': 'rhoso-architecture-base-job', + 'vars': { + 'cifmw_architecture_scenario': name, + }, + 'files': _paths, + } + } + _JOBS.append(job) + _PROJECTS.append(j_name) + + +for fname in [f for f in os.listdir(_AUTO_PATH) if f.endswith('.yaml')]: + with open(os.path.join(_AUTO_PATH, fname), 'r') as y_file: + scenarios = yaml.safe_load(y_file) + for scenario, stages in scenarios['vas'].items(): + create_job(scenario, stages) + +with open('./zuul.d/validations.yaml', 'w+') as zuul_jobs: + yaml.dump(_JOBS, zuul_jobs) + +with open('./zuul.d/projects.yaml', 'w+') as zuul_projects: + struct = [{'project': {'github-check': {'jobs': _PROJECTS}}}] + yaml.dump(struct, zuul_projects) diff --git a/zuul.d/validate-jobs.yaml b/zuul.d/base-jobs.yaml similarity index 52% rename from zuul.d/validate-jobs.yaml rename to zuul.d/base-jobs.yaml index ef5cb3d03..f0d116ef9 100644 --- a/zuul.d/validate-jobs.yaml +++ b/zuul.d/base-jobs.yaml @@ -1,6 +1,6 @@ --- # How to create a new job: -# - check the syntax defined for the existing jobs +# - run .ci/create-zuul-jobs.py script to regenerate based on automations # - if your scenario needs mocked data: # - push a proper YAML file in automation/mocks/SCENARIO_NAME.yaml # - that's all! @@ -14,19 +14,3 @@ parent: cifmw-architecture-validate-base required-projects: - openstack-k8s-operators/ci-framework - -- job: - name: rhoso-architecture-validate-hci - parent: rhoso-architecture-base-job - vars: - cifmw_architecture_scenario: hci - files: - - examples/va/hci - -- job: - name: rhoso-architecture-validate-ovs-dpdk-sriov - parent: rhoso-architecture-base-job - vars: - cifmw_architecture_scenario: ovs-dpdk-sriov - files: - - examples/va/nfv diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index da4df8cdf..1da60002f 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -1,7 +1,12 @@ ---- - project: github-check: jobs: - - noop - - rhoso-architecture-validate-hci - - rhoso-architecture-validate-ovs-dpdk-sriov + - noop + - rhoso-architecture-validate-uni01alpha + - rhoso-architecture-validate-bgp + - rhoso-architecture-validate-uni02beta + - rhoso-architecture-validate-uni06zeta + - rhoso-architecture-validate-hci + - rhoso-architecture-validate-sriov + - rhoso-architecture-validate-ovs-dpdk + - rhoso-architecture-validate-ovs-dpdk-sriov diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml new file mode 100644 index 000000000..741b0eb7c --- /dev/null +++ b/zuul.d/validations.yaml @@ -0,0 +1,96 @@ +- job: + files: + - dt/uni01alpha + - examples/dt/uni01alpha + - examples/dt/uni01alpha/control-plane + - examples/dt/uni01alpha/control-plane/nncp + - examples/dt/uni01alpha/networker + - lib + name: rhoso-architecture-validate-uni01alpha + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: uni01alpha +- job: + files: + - dt/bgp + - examples/dt/bgp/control-plane + - examples/dt/bgp/control-plane/nncp + - examples/dt/bgp/edpm/deployment + - examples/dt/bgp/edpm/nodeset + - lib + name: rhoso-architecture-validate-bgp + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: bgp +- job: + files: + - dt/uni02beta + - examples/dt/uni02beta + - examples/dt/uni02beta/control-plane + - examples/dt/uni02beta/control-plane/nncp + - lib + name: rhoso-architecture-validate-uni02beta + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: uni02beta +- job: + files: + - dt/uni06zeta + - examples/dt/uni06zeta + - examples/dt/uni06zeta/control-plane + - examples/dt/uni06zeta/control-plane/nncp + - lib + name: rhoso-architecture-validate-uni06zeta + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: uni06zeta +- job: + files: + - examples/va/hci + - examples/va/hci/control-plane + - examples/va/hci/control-plane/nncp + - examples/va/hci/deployment + - examples/va/hci/edpm-pre-ceph/deployment + - examples/va/hci/edpm-pre-ceph/nodeset + - lib + - va/hci + name: rhoso-architecture-validate-hci + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: hci +- job: + files: + - automation/mocks/sriov.yaml + - examples/va/nfv/sriov + - examples/va/nfv/sriov/edpm/deployment + - examples/va/nfv/sriov/edpm/nodeset + - examples/va/nfv/sriov/nncp + - lib + name: rhoso-architecture-validate-sriov + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: sriov +- job: + files: + - automation/mocks/ovs-dpdk.yaml + - examples/va/nfv/ovs-dpdk + - examples/va/nfv/ovs-dpdk/edpm/deployment + - examples/va/nfv/ovs-dpdk/edpm/nodeset + - examples/va/nfv/ovs-dpdk/nncp + - lib + name: rhoso-architecture-validate-ovs-dpdk + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: ovs-dpdk +- job: + files: + - automation/mocks/ovs-dpdk-sriov.yaml + - examples/va/nfv/ovs-dpdk-sriov + - examples/va/nfv/ovs-dpdk-sriov/edpm/deployment + - examples/va/nfv/ovs-dpdk-sriov/edpm/nodeset + - examples/va/nfv/ovs-dpdk-sriov/nncp + - lib + name: rhoso-architecture-validate-ovs-dpdk-sriov + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: ovs-dpdk-sriov