From 2db0c416a46b60486f2aa069a36acbd9cdc301f5 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 16 Jan 2025 14:03:52 +0000 Subject: [PATCH] feat: add `gitlab` role to enable GitLab CI/CD Add a new role `gitlab` to enable GitLab CI/CD for use in constructing a CI/CD pipeline that is compatible with `kayobe-automation`. Supports key features and workflows found within the `GitHub` role. --- roles/gitlab/README.md | 120 +++++ roles/gitlab/defaults/main.yml | 576 ++++++++++++++++++++++++ roles/gitlab/handlers/main.yml | 2 + roles/gitlab/tasks/main.yml | 32 ++ roles/gitlab/templates/gitlab-ci.yml.j2 | 6 + roles/gitlab/templates/prelude.yml.j2 | 115 +++++ roles/gitlab/templates/stage.yml.j2 | 7 + roles/gitlab/vars/main.yml | 2 + tests/test.yml | 1 + 9 files changed, 861 insertions(+) create mode 100644 roles/gitlab/README.md create mode 100644 roles/gitlab/defaults/main.yml create mode 100644 roles/gitlab/handlers/main.yml create mode 100644 roles/gitlab/tasks/main.yml create mode 100644 roles/gitlab/templates/gitlab-ci.yml.j2 create mode 100644 roles/gitlab/templates/prelude.yml.j2 create mode 100644 roles/gitlab/templates/stage.yml.j2 create mode 100644 roles/gitlab/vars/main.yml diff --git a/roles/gitlab/README.md b/roles/gitlab/README.md new file mode 100644 index 0000000..4661bc6 --- /dev/null +++ b/roles/gitlab/README.md @@ -0,0 +1,120 @@ +Kayobe Automation Pipeline (GitLab) +=================================== + +This Ansible role is capable of generating Gitlab pipeline files for performing CI/CD related activities with OpenStack via Kayobe. +See the table below for a full list of all the currently supported kayobe automation tasks. + +| **Name** | **Description** | **Stage** | +|:---:|:---:|:---:| +| **build kayobe docker image** | Build a new kayobe docker image whenever a new tag is pushed to the repository. The resulting image is then pushed to a docker registry such as [registry.gitlab.com/](registry.gitlab.com/). | build | +| **overcloud container image build** | Build overcloud container images. | overcloud | +| **overcloud container image pull** | Pull overcloud container images from a container registry. | overcloud | +| **overcloud database backup** | Perform a backup of the database used by the overcloud. | overcloud | +| **overcloud database recover** | Recover the database used by the overcloud. | overcloud | +| **overcloud deployment image build** | Build the Ironic Python Agent (IPA) image. | overcloud | +| **overcloud host command run** | Run a command against the overcloud hosts. | overcloud | +| **overcloud host configure** | Perform an overcloud host configure. | overcloud | +| **overcloud host image build** | Build the the image that would deployed to overcloud hosts during provisioning. | overcloud | +| **overcloud host package update** | Perform an overcloud host package update. | overcloud | +| **overcloud host upgrade** | Perform targeted upgrade of key services before an upgrade. | overcloud | +| **overcloud inventory discover** | Get an inventory of nodes. | overcloud | +| **overcloud provision** | Provision overcloud nodes. | overcloud | +| **overcloud service configuration generate** | Generate the overcloud service configuration. | overcloud | +| **overcloud service deploy** | Deploy overcloud services. | overcloud | +| **overcloud service reconfigure** | Reconfigure overcloud services. | overcloud | +| **overcloud service upgrade** | Perform an upgrade of overcloud services. | overcloud | +| **seed container image build** | Build container images for seed. | overcloud | +| **seed host configure** | Configure the seed host. | seed | +| **seed host package update** | Update the system packages of the seed host. | seed | +| **seed hypervisor host configure** | Configure the seed hypervisor host. | seed | +| **seed hypervisor host package update** | Perform a package update of the seed hypervisor host. | seed | +| **seed service deploy** | Deploy services on the seed. | seed | +| **seed vm provision** | Provision the seed VM. | seed | +| **infra vm host configure** | Perform a host configure of the infra VMs on demand. | infra vm | +| **infra vm host package update** | Perform a package update of the infra VMs hosts on demand. | infra vm | +| **infra vm provision** | Provision infra VMs on demand. | infra vm | +| **infra vm service deploy** | Perform a service deploy against infra VMs on demand. | infra vm | +| **network connectivity check** | Execute a network connectivity check to ensure all hosts are reachable and can reach `nc external ip ` & `nc external hostname`. | network | +| **physical network configure** | Configure the physical network. | network | +| **config diff** | When a pull request is opened generate diff showing the changes made to the configuration. | merge request | +| **tempest** | Perform tests against the deployed openstack environment with tempest. | tempest | + +Also available in Kayobe Automation for GitLab are pipelines which are stages designed to complete complex task such as upgrading all hypervisors in safe and reliable manner. + +| **Name** | **Description** | +|:---:|:---:| +| **proc-overcloud-service-upgrade** | Perform an upgrade of overcloud services; pull containers, upgrade host, run tempest, backup database, upgrade services, run tempest and prune unused docker images. | +| **proc-hypervisor-host-upgrade** | Perform a host upgrade of hypervisors; disable compute services, drain hypervisor, host configure, package update, reboot and renable compute services. | + + +Role Variables +-------------- + +The following variables can be used to make small adjustments to the composition of the workflows. + +`gitlab_output_directory`: control the location where the workflows shall be written to. + +`gitlab_kayobe_environments`: list of environments the workflows should target. + +`gitlab_image_name`: name of the kayobe image defaults to `kayobe`. + +`gitlab_image_tag`: tag used to select kayobe image defaults to `latest` + +`gitlab_registry`: a string that either points to a registry or is `$CI_REGISTRY_URL` in the case of multiple environments that do not share the same registry. + +`gitlab_kayobe_base_image`: select the base image used when building the kayobe docker image. Default is `quay.io/rockylinux/rockylinux:9`. + +`gitlab_tempest_test_suites`: provide a list of load lists to be made available within the drop-down list for running tempest. Defaults to `default` and `tempest-full`. + +If you wish to make more impactful changes such as which jobs are built and what they contain then see the list of dictionaries called `jobs` in `defaults/main.yml` + +`gitlab_jobs:` is a dictionary of dictionary that contain string blocks defining the job within a given stage. Any job maybe overwritten by editing the `gitlab_JOB_NAME` or a stage can be extended adding to the `gitlab_STAGE_NAME_extra` dictionary. + +``` +gitlab_infra_vm_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true +``` + +``` +gitlab_infra_vm_something_new: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/does_something_new.sh + allow_failure: true + +gitlab_stage_infra_vm_extra: + infra_vm_something_new: "{{ gitlab_infra_vm_something_new }}" +``` + +Example Playbook +---------------- + +The following example playbook will generate a `reference` pipeline which can be found under `.gitlab` and `.gitlab/..` + +```yaml +- name: Write Kayobe Automation Pipeline for gitlab + hosts: localhost + roles: + - stackhpc.kayobe_workflows.gitlab +``` + +License +------- + +Apache License 2.0 + +Author Information +------------------ + +[StackHPC](https://www.stackhpc.com/) diff --git a/roles/gitlab/defaults/main.yml b/roles/gitlab/defaults/main.yml new file mode 100644 index 0000000..f7b3f7a --- /dev/null +++ b/roles/gitlab/defaults/main.yml @@ -0,0 +1,576 @@ +--- +gitlab_output_directory: output + +gitlab_kayobe_environments: [] + +gitlab_registry: "registry.gitlab.com" + +gitlab_image_name: kayobe + +gitlab_image_tag: "$OPENSTACK_RELEASE-latest" + +gitlab_kayobe_base_image: "quay.io/rockylinux/rockylinux:9" + +gitlab_tempest_test_suites: | + - default + - tempest-full + +gitlab_openstack_release: "2024.1" + +gitlab_stages: + build: + "{{ gitlab_stage_build }}" + infra-vm: + "{{ gitlab_stage_infra_vm }}" + merge-request: + "{{ gitlab_stage_merge_request }}" + network: + "{{ gitlab_stage_network }}" + overcloud: + "{{ gitlab_stage_overcloud }}" + seed: + "{{ gitlab_stage_seed }}" + tempest: + "{{ gitlab_stage_tempest }}" + proc-overcloud-service-upgrade: + container_image_pull: "{{ gitlab_container_image_pull }}" + host_upgrade: "{{ gitlab_host_upgrade }}" + tempest_run_before: "{{ gitlab_tempest_run_before }}" + backup_database: "{{ gitlab_backup_database }}" + upgrade_overcloud_services: "{{ gitlab_upgrade_overcloud_services }}" + tempest_run_after: "{{ gitlab_tempest_run_after }}" + proc-hypervisor-host-upgrade: + disable_compute_services: "{{ gitlab_disable_compute_services }}" + drain_hypervisor: "{{ gitlab_drain_hypervisor }}" + enable_hypervisor: "{{ gitlab_enable_hypervisor }}" + +gitlab_stage_build: "{{ gitlab_stage_build_default | combine(gitlab_stage_build_extra) }}" + +gitlab_stage_build_default: + build_kayobe_image: "{{ gitlab_build_kayobe_image }}" + +gitlab_stage_build_extra: [] + +gitlab_build_kayobe_image: | + image: docker:{{ gitlab_docker_image_version }} + rules: + - !reference [.active_stage_web_rule, rules] + stage: build + script: + - DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE="{{ gitlab_kayobe_base_image }}" --network=host --cache-from \ + {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest -f .automation/docker/kayobe/Dockerfile -t \ + {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$CI_COMMIT_SHA \ + -t {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest . + - docker image push --all-tags {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe + allow_failure: true + +gitlab_stage_infra_vm: "{{ gitlab_stage_infra_vm_default | combine(gitlab_stage_infra_vm_extra) }}" + +gitlab_stage_infra_vm_default: + infra_vm_host_configure: "{{ gitlab_infra_vm_host_configure }}" + infra_vm_host_package_update: "{{ gitlab_infra_vm_host_package_update }}" + infra_vm_provision: "{{ gitlab_infra_vm_provision }}" + infra_vm_service_deploy: "{{ gitlab_infra_vm_service_deploy }}" + +gitlab_stage_infra_vm_extra: [] + +gitlab_infra_vm_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true + +gitlab_infra_vm_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-host-configure.sh + allow_failure: true + +gitlab_infra_vm_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-provision.sh + allow_failure: true + +gitlab_infra_vm_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: infra-vm + resource_group: infra-vm + script: + - !reference [.get_secrets, script] + - .automation/pipeline/infra-vm-service-deploy.sh + allow_failure: true + +gitlab_stage_merge_request: "{{ gitlab_stage_merge_request_default | combine(gitlab_stage_merge_request_extra) }}" + +gitlab_stage_merge_request_default: + kolla_diff: "{{ gitlab_kolla_diff }}" + +gitlab_stage_merge_request_extra: [] + +gitlab_kolla_diff: | + stage: merge-request + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + parallel: + matrix: + - KAYOBE_ENVIRONMENT: [production, test] + script: + - !reference [.get_secrets, script] + - .automation/pipeline/config-diff.sh origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + artifacts: + when: always + paths: + - /tmp/kayobe-config-diff + - /tmp/target-kayobe.log + - /tmp/source-kayobe.log + +gitlab_stage_network: "{{ gitlab_stage_network_default | combine(gitlab_stage_network_extra) }}" + +gitlab_stage_network_default: + network_connectivity_check: "{{ gitlab_network_connectivity_check }}" + physical_network_configure: "{{ gitlab_physical_network_configure }}" + +gitlab_stage_network_extra: [] + +gitlab_network_connectivity_check: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: network + resource_group: network + script: + - !reference [.get_secrets, script] + - .automation/pipeline/network-connectivity-check.sh + allow_failure: true + +gitlab_physical_network_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: network + resource_group: network + script: + - !reference [.get_secrets, script] + - .automation/pipeline/physical-network-configure.sh + allow_failure: true + +gitlab_stage_overcloud: "{{ gitlab_stage_overcloud_default | combine(gitlab_stage_overcloud_extra) }}" + +gitlab_stage_overcloud_default: + overcloud_container_image_build: "{{ gitlab_overcloud_container_image_build }}" + overcloud_container_image_pull: "{{ gitlab_overcloud_container_image_pull }}" + overcloud_database_backup: "{{ gitlab_overcloud_database_backup }}" + overcloud_database_recover: "{{ gitlab_overcloud_database_recover }}" + overcloud_deployment_image_build: "{{ gitlab_overcloud_deployment_image_build }}" + overcloud_host_command_run: "{{ gitlab_overcloud_host_command_run }}" + overcloud_host_configure: "{{ gitlab_overcloud_host_configure }}" + overcloud_host_image_build: "{{ gitlab_overcloud_host_image_build }}" + overcloud_host_package_update: "{{ gitlab_overcloud_host_package_update }}" + overcloud_host_upgrade: "{{ gitlab_overcloud_host_upgrade }}" + overcloud_inventory_discover: "{{ gitlab_overcloud_inventory_discover }}" + overcloud_provision: "{{ gitlab_overcloud_provision }}" + overcloud_service_configuration_generate: "{{ gitlab_overcloud_service_configuration_generate }}" + overcloud_service_deploy: "{{ gitlab_overcloud_service_deploy }}" + overcloud_service_reconfigure: "{{ gitlab_overcloud_service_reconfigure }}" + overcloud_service_upgrade: "{{ gitlab_overcloud_service_upgrade }}" + +gitlab_stage_overcloud_extra: [] + +gitlab_overcloud_container_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-container-image-build.sh + allow_failure: true + +gitlab_overcloud_container_image_pull: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-container-image-pull.sh + allow_failure: true + +gitlab_overcloud_database_backup: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-database-backup.sh + allow_failure: true + +gitlab_overcloud_database_recover: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-database-recover.sh + allow_failure: true + +gitlab_overcloud_deployment_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-deployment-image-build.sh + allow_failure: true + +gitlab_overcloud_host_command_run: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-command-run.sh + allow_failure: true + +gitlab_overcloud_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-configure.sh + allow_failure: true + +gitlab_overcloud_host_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-image-build.sh + allow_failure: true + +gitlab_overcloud_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-package-update.sh + allow_failure: true + +gitlab_overcloud_host_upgrade: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-host-upgrade.sh + allow_failure: true + +gitlab_overcloud_inventory_discover: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-inventory-discover.sh + allow_failure: true + +gitlab_overcloud_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-provision.sh + allow_failure: true + +gitlab_overcloud_service_configuration_generate: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-configuration-generate.sh /tmp/kolla + allow_failure: true + +gitlab_overcloud_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-deploy.sh + allow_failure: true + +gitlab_overcloud_service_reconfigure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-reconfigure.sh + allow_failure: true + +gitlab_overcloud_service_upgrade: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: overcloud + resource_group: overcloud + script: + - !reference [.get_secrets, script] + - .automation/pipeline/overcloud-service-upgrade.sh + allow_failure: true + +gitlab_stage_seed: "{{ gitlab_stage_seed_default | combine(gitlab_stage_seed_extra) }}" + +gitlab_stage_seed_default: + seed_container_image_build: "{{ gitlab_seed_container_image_build }}" + seed_host_configure: "{{ gitlab_seed_host_configure }}" + seed_host_package_update: "{{ gitlab_seed_host_package_update }}" + seed_hypervisor_host_configure: "{{ gitlab_seed_hypervisor_host_configure }}" + seed_hypervisor_host_package_update: "{{ gitlab_seed_hypervisor_host_package_update }}" + seed_service_deploy: "{{ gitlab_seed_service_deploy }}" + seed_vm_provision: "{{ gitlab_seed_vm_provision }}" + +gitlab_stage_seed_extra: [] + +gitlab_seed_container_image_build: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-container-image-build.sh + allow_failure: true + +gitlab_seed_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-host-configure.sh + allow_failure: true + +gitlab_seed_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-host-package-update.sh + allow_failure: true + +gitlab_seed_hypervisor_host_configure: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-hypervisor-host-configure.sh + allow_failure: true + +gitlab_seed_hypervisor_host_package_update: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-hypervisor-host-package-update.sh + allow_failure: true + +gitlab_seed_service_deploy: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-service-deploy.sh + allow_failure: true + +gitlab_seed_vm_provision: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: seed + resource_group: seed + script: + - !reference [.get_secrets, script] + - .automation/pipeline/seed-vm-provision.sh + allow_failure: true + +gitlab_stage_tempest: "{{ gitlab_stage_tempest_default | combine(gitlab_stage_tempest_extra) }}" + +gitlab_stage_tempest_default: + tempest_run: "{{ gitlab_tempest_run }}" + +gitlab_stage_tempest_extra: [] + +gitlab_tempest_run: | + rules: + - !reference [.active_stage_web_rule, rules] + - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_JOB == 'tempest' + when: always + stage: tempest + resource_group: tempest + script: + - !reference [.get_secrets, script] + - !reference [.tempest_openrc, script] + - KAYOBE_AUTOMATION_TEMPEST_LOADLIST=${KAYOBE_AUTOMATION_TEMPEST_LOADLIST} .automation/pipeline/tempest.sh -e ansible_user=stack -e rally_force_pull=false + - mkdir ${CI_PROJECT_DIR}/artifacts + - cp $HOME/tempest-artifacts/* ${CI_PROJECT_DIR}/artifacts + - test $(wc -l < ${CI_PROJECT_DIR}/artifacts/failed-tests) -lt 1 + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_container_image_pull: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-overcloud-service-upgrade + resource_group: overcloud + script: + - !reference [overcloud_container_image_pull, script] + +gitlab_host_upgrade: | + rules: + - !reference [.active_stage_web_rule, rules] + needs: [container_image_pull] + stage: proc-overcloud-service-upgrade + resource_group: overcloud + script: + - !reference [overcloud_host_upgrade, script] + +gitlab_tempest_run_before: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-overcloud-service-upgrade + resource_group: tempest + needs: [host_upgrade] + script: + - !reference [tempest_run, script] + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_backup_database: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-overcloud-service-upgrade + resource_group: overcloud + needs: [tempest_run_before] + script: + - !reference [overcloud_database_backup, script] + +gitlab_upgrade_overcloud_services: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-overcloud-service-upgrade + resource_group: overcloud + needs: [backup_database] + script: + - !reference [overcloud_service_upgrade, script] + +gitlab_tempest_run_after: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-overcloud-service-upgrade + resource_group: tempest + needs: [upgrade_overcloud_services] + script: + - !reference [tempest_run, script] + artifacts: + when: always + paths: + - artifacts/rally-junit.xml + - artifacts/rally-verify-report.html + - artifacts/stderr.log + - artifacts/stdout.log + - artifacts/tempest.log + reports: + junit: artifacts/rally-junit.xml + allow_failure: true + +gitlab_disable_compute_services: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-hypervisor-host-upgrade + resource_group: proc-hypervisor-host-upgrade + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + allow_failure: true + +gitlab_drain_hypervisor: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-hypervisor-host-upgrade + resource_group: proc-hypervisor-host-upgrade + needs: [disable_compute_services] + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + allow_failure: true + +gitlab_enable_hypervisor: | + rules: + - !reference [.active_stage_web_rule, rules] + stage: proc-hypervisor-host-upgrade + resource_group: proc-hypervisor-host-upgrade + needs: [drain_hypervisor] + script: + - !reference [.kayobe_public_openrc, script] + - !reference [.get_secrets, script] + - .automation/pipeline/playbook-run.sh etc/kayobe/ansible/nova-compute-disable.yml + allow_failure: true diff --git a/roles/gitlab/handlers/main.yml b/roles/gitlab/handlers/main.yml new file mode 100644 index 0000000..a923c25 --- /dev/null +++ b/roles/gitlab/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for kayobe-automation-workflows diff --git a/roles/gitlab/tasks/main.yml b/roles/gitlab/tasks/main.yml new file mode 100644 index 0000000..b00564d --- /dev/null +++ b/roles/gitlab/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Ensure output directory exists + ansible.builtin.file: + dest: "{{ item }}" + state: directory + mode: "755" + loop: + - "{{ gitlab_output_directory }}" + - "{{ gitlab_output_directory }}/.gitlab" + +- name: Ensure gitlab-ci.yml entry point exists + ansible.builtin.template: + src: "gitlab-ci.yml.j2" + dest: "{{ gitlab_output_directory }}/gitlab-ci.yml" + mode: "0644" + +- name: Ensure prelude.yml exists + ansible.builtin.template: + src: "prelude.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/prelude.yml" + mode: "0644" + +- name: Ensure pipeline stage files exist + ansible.builtin.template: + src: "stage.yml.j2" + dest: "{{ gitlab_output_directory }}/.gitlab/{{ stage }}.yml" + mode: "0644" + vars: + current_stage: "{{ stage }}" + loop: "{{ gitlab_stages.keys() }}" + loop_control: + loop_var: stage diff --git a/roles/gitlab/templates/gitlab-ci.yml.j2 b/roles/gitlab/templates/gitlab-ci.yml.j2 new file mode 100644 index 0000000..f0ba6e5 --- /dev/null +++ b/roles/gitlab/templates/gitlab-ci.yml.j2 @@ -0,0 +1,6 @@ +--- +include: + - .gitlab/prelude.yml +{% for stage in gitlab_stages.keys() %} + - .gitlab/{{ stage }}.yml +{% endfor %} diff --git a/roles/gitlab/templates/prelude.yml.j2 b/roles/gitlab/templates/prelude.yml.j2 new file mode 100644 index 0000000..a65e3f5 --- /dev/null +++ b/roles/gitlab/templates/prelude.yml.j2 @@ -0,0 +1,115 @@ +--- +stages: +{%- for stage in gitlab_stages.keys() +%} + - {{ stage }} +{%- endfor +%} + +variables: +{% if gitlab_kayobe_environments %} + KAYOBE_ENVIRONMENT: + value: {{ gitlab_kayobe_environments | first }} + description: "Name of environment to apply against" + options: +{% for environment in gitlab_kayobe_environments %} + - {{ environment}} +{% endfor %} +{% endif %} + ACTIVE_STAGE: + value: overcloud + description: "Show only jobs belonging to the active stage" + options: + - all + {%- for stage in gitlab_stages.keys() +%} + - {{ stage }} + {%- endfor +%} + KAYOBE_AUTOMATION_TEMPEST_LOADLIST: + value: "default" + description: "Name of tempest load list to use" + options: + {{ gitlab_tempest_test_suites | indent(6) | trim }} + KOLLA_TAGS: + value: "" + description: "List of tasks to limit when running kolla-ansible (-kt)" + KOLLA_LIMIT: + value: "" + description: "List of hosts or groups to limit when running kolla-ansible (-kl)" + KAYOBE_TAGS: + value: "" + description: "List of tags to limit when running kayobe (-t)" + KAYOBE_LIMIT: + value: "" + description: "List of hosts or groups to limit when running kayobe (-l)" + OPENSTACK_RELEASE: + value: {{ gitlab_openstack_release }} + description: "OpenStack release to deploy" + GIT_DEPTH: 0 + GIT_SUBMODULE_STRATEGY: recursive + +default: + image: {{ gitlab_registry | default("$CI_REGISTRY") }}/kayobe:$OPENSTACK_RELEASE-latest + tags: + - kayobe + - openstack +{% if gitlab_kayobe_environments | length > 1 %} + - $KAYOBE_ENVIRONMENT +{% endif %} + id_tokens: + VAULT_AUTH_TOKEN: + aud: http://127.0.0.1:8200 + before_script: + - | + # Workaround for lack of default variables: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1867 + # CI_COMMIT_BRANCH is not set for merge requests + if [ ! -z "$CI_COMMIT_BRANCH" ]; then + echo "Setting target branch to: $CI_COMMIT_BRANCH" + export KAYOBE_AUTOMATION_PR_TARGET_BRANCH="$CI_COMMIT_BRANCH" + elif [ ! -z "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then + echo "Setting target branch to: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + export KAYOBE_AUTOMATION_PR_TARGET_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + fi + echo $KAYOBE_AUTOMATION_TEMPEST_LOADLIST + export KAYOBE_ENVIRONMENT=$KAYOBE_ENVIRONMENT + export TEMPEST_OPENRC=$TEMPEST_OPENRC + export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=$KAYOBE_AUTOMATION_TEMPEST_LOADLIST + - >- + export BAO_TOKEN=$(curl --silent --request POST + --data '{"role": "gitlab", "jwt": "'"$VAULT_AUTH_TOKEN"'"}' + http://127.0.0.1:8200/v1/auth/jwt/login | jq -r '.auth.client_token') + +.kayobe_automation_ssh_private_key: &kayobe_automation_ssh_private_key + - >- + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_automation_ssh_private_key') + +.kayobe_vault_password: &kayobe_vault_password + - >- + export KAYOBE_VAULT_PASSWORD=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_vault_password') + +.get_secrets: &get_secrets + script: + - *kayobe_automation_ssh_private_key + - *kayobe_vault_password + +.kayobe_public_openrc: &kayobe_public_openrc + script: >- + source <(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_public_openrc') + +.tempest_openrc: &tempest_openrc + script: >- + export TEMPEST_OPENRC=$(curl + --silent + --header "X-Vault-Token: $BAO_TOKEN" + --request GET http://127.0.0.1:8200/v1/kayobe-automation/$KAYOBE_ENVIRONMENT | jq -r '.data.kayobe_public_openrc') + +.active_stage_web_rule: &active_stage_web_rule + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $ACTIVE_STAGE == $CI_JOB_STAGE || $ACTIVE_STAGE == "all" + when: manual diff --git a/roles/gitlab/templates/stage.yml.j2 b/roles/gitlab/templates/stage.yml.j2 new file mode 100644 index 0000000..898fa74 --- /dev/null +++ b/roles/gitlab/templates/stage.yml.j2 @@ -0,0 +1,7 @@ +--- +{%- for job_name in gitlab_stages[stage].keys() +%} +{%- if gitlab_stages[stage][job_name] +%} +{{ job_name }}: + {{ gitlab_stages[stage][job_name] | indent(2) }} +{%- endif -%} +{%- endfor -%} diff --git a/roles/gitlab/vars/main.yml b/roles/gitlab/vars/main.yml new file mode 100644 index 0000000..8cd296c --- /dev/null +++ b/roles/gitlab/vars/main.yml @@ -0,0 +1,2 @@ +--- +gitlab_docker_image_version: "27.4.1" diff --git a/tests/test.yml b/tests/test.yml index 17926f5..fcd0519 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -3,3 +3,4 @@ hosts: all roles: - github + - gitlab