Generate build jobs for all automations

This patch provides a script generating all of the zuul.d content, based
on the automation files.

Each scenario will get its own job definition, and will be triggered
only when its files are changed.

Note that common files, such as "lib", will trigger all the jobs.

This patch also introduces a new github workflow, to ensure all of the
automation scenarios have a job, by re-running the script and ensuring
no chance happened in zuul.d directory.

.github/workflows/linter.yaml

@@ -41,3 +41,23 @@ jobs:
         run: |
           kustomize version
           ./test-kustomizations.sh examples/
+
+  test-zuul:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install yaml
+        run: pip install pyyaml
+
+      - name: Run create-zuul-jobs
+        run: |
+          sha256sum zuul.d/* > checksum;
+          ./create-zuul-jobs.py;
+          sha256sum -c checksum;

.yamllint.yml

@@ -2,6 +2,7 @@
 extends: default
 ignore:
   - '*.md'
+  - 'zuul.d/'
 
 rules:
   line-length:

automation/mocks/ovs-dpdk.yaml

@@ -0,0 +1 @@
+ovs-dpdk-sriov.yaml

automation/mocks/sriov.yaml

@@ -0,0 +1 @@
+ovs-dpdk-sriov.yaml

create-zuul-jobs.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+_AUTO_PATH = 'automation/vars/'
+_JOBS = []
+_PROJECTS = ['noop']
+_BASE_TRIGGER_FILES = [
+        'lib',
+        ]
+
+def create_job(name, data):
+    j_name = 'rhoso-architecture-validate-{}'.format(name)
+    _paths = [p['path'] for p in data['stages']] + _BASE_TRIGGER_FILES
+    for i in ['va', 'dt']:
+        if os.path.exists(os.path.join(i, name)):
+            _paths.append(os.path.join(i, name))
+    _mock = os.path.join('automation', 'mocks', '{}.yaml'.format(name))
+    if os.path.exists(_mock):
+        _paths.append(_mock)
+    _paths.sort()
+    job = {
+            'job': {
+                'name': j_name,
+                'parent': 'rhoso-architecture-base-job',
+                'vars': {
+                    'cifmw_architecture_scenario': name,
+                    },
+                'files': _paths,
+                }
+            }
+    _JOBS.append(job)
+    _PROJECTS.append(j_name)
+
+
+for fname in [f for f in os.listdir(_AUTO_PATH) if f.endswith('.yaml')]:
+    with open(os.path.join(_AUTO_PATH, fname), 'r') as y_file:
+        scenarios = yaml.safe_load(y_file)
+    for scenario, stages in scenarios['vas'].items():
+        create_job(scenario, stages)
+
+with open('./zuul.d/validations.yaml', 'w+') as zuul_jobs:
+    yaml.dump(_JOBS, zuul_jobs)
+
+with open('./zuul.d/projects.yaml', 'w+') as zuul_projects:
+    struct = [{'project': {'github-check': {'jobs': _PROJECTS}}}]
+    yaml.dump(struct, zuul_projects)

zuul.d/validate-jobs.yaml → zuul.d/base-jobs.yaml

@@ -1,6 +1,6 @@
 ---
 # How to create a new job:
-# - check the syntax defined for the existing jobs
+# - run .ci/create-zuul-jobs.py script to regenerate based on automations
 # - if your scenario needs mocked data:
 #     - push a proper YAML file in automation/mocks/SCENARIO_NAME.yaml
 #     - that's all!
@@ -14,19 +14,3 @@
     parent: cifmw-architecture-validate-base
     required-projects:
       - openstack-k8s-operators/ci-framework
-
-- job:
-    name: rhoso-architecture-validate-hci
-    parent: rhoso-architecture-base-job
-    vars:
-      cifmw_architecture_scenario: hci
-    files:
-      - examples/va/hci
-
-- job:
-    name: rhoso-architecture-validate-ovs-dpdk-sriov
-    parent: rhoso-architecture-base-job
-    vars:
-      cifmw_architecture_scenario: ovs-dpdk-sriov
-    files:
-      - examples/va/nfv

zuul.d/projects.yaml

@@ -1,7 +1,12 @@
----
 - project:
     github-check:
       jobs:
-        - noop
-        - rhoso-architecture-validate-hci
-        - rhoso-architecture-validate-ovs-dpdk-sriov
+      - noop
+      - rhoso-architecture-validate-uni01alpha
+      - rhoso-architecture-validate-bgp
+      - rhoso-architecture-validate-uni02beta
+      - rhoso-architecture-validate-uni06zeta
+      - rhoso-architecture-validate-hci
+      - rhoso-architecture-validate-sriov
+      - rhoso-architecture-validate-ovs-dpdk
+      - rhoso-architecture-validate-ovs-dpdk-sriov

zuul.d/validations.yaml

@@ -0,0 +1,96 @@
+- job:
+    files:
+    - dt/uni01alpha
+    - examples/dt/uni01alpha
+    - examples/dt/uni01alpha/control-plane
+    - examples/dt/uni01alpha/control-plane/nncp
+    - examples/dt/uni01alpha/networker
+    - lib
+    name: rhoso-architecture-validate-uni01alpha
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: uni01alpha
+- job:
+    files:
+    - dt/bgp
+    - examples/dt/bgp/control-plane
+    - examples/dt/bgp/control-plane/nncp
+    - examples/dt/bgp/edpm/deployment
+    - examples/dt/bgp/edpm/nodeset
+    - lib
+    name: rhoso-architecture-validate-bgp
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: bgp
+- job:
+    files:
+    - dt/uni02beta
+    - examples/dt/uni02beta
+    - examples/dt/uni02beta/control-plane
+    - examples/dt/uni02beta/control-plane/nncp
+    - lib
+    name: rhoso-architecture-validate-uni02beta
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: uni02beta
+- job:
+    files:
+    - dt/uni06zeta
+    - examples/dt/uni06zeta
+    - examples/dt/uni06zeta/control-plane
+    - examples/dt/uni06zeta/control-plane/nncp
+    - lib
+    name: rhoso-architecture-validate-uni06zeta
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: uni06zeta
+- job:
+    files:
+    - examples/va/hci
+    - examples/va/hci/control-plane
+    - examples/va/hci/control-plane/nncp
+    - examples/va/hci/deployment
+    - examples/va/hci/edpm-pre-ceph/deployment
+    - examples/va/hci/edpm-pre-ceph/nodeset
+    - lib
+    - va/hci
+    name: rhoso-architecture-validate-hci
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: hci
+- job:
+    files:
+    - automation/mocks/sriov.yaml
+    - examples/va/nfv/sriov
+    - examples/va/nfv/sriov/edpm/deployment
+    - examples/va/nfv/sriov/edpm/nodeset
+    - examples/va/nfv/sriov/nncp
+    - lib
+    name: rhoso-architecture-validate-sriov
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: sriov
+- job:
+    files:
+    - automation/mocks/ovs-dpdk.yaml
+    - examples/va/nfv/ovs-dpdk
+    - examples/va/nfv/ovs-dpdk/edpm/deployment
+    - examples/va/nfv/ovs-dpdk/edpm/nodeset
+    - examples/va/nfv/ovs-dpdk/nncp
+    - lib
+    name: rhoso-architecture-validate-ovs-dpdk
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: ovs-dpdk
+- job:
+    files:
+    - automation/mocks/ovs-dpdk-sriov.yaml
+    - examples/va/nfv/ovs-dpdk-sriov
+    - examples/va/nfv/ovs-dpdk-sriov/edpm/deployment
+    - examples/va/nfv/ovs-dpdk-sriov/edpm/nodeset
+    - examples/va/nfv/ovs-dpdk-sriov/nncp
+    - lib
+    name: rhoso-architecture-validate-ovs-dpdk-sriov
+    parent: rhoso-architecture-base-job
+    vars:
+      cifmw_architecture_scenario: ovs-dpdk-sriov