Multiple environments: part 2 (antelope)

ansible/inventory/group_vars/all/globals

@@ -16,6 +16,10 @@ kayobe_environment: "{{ lookup('env', 'KAYOBE_ENVIRONMENT') }}"
 # environment path appended if kayobe_environment is set.
 kayobe_env_config_path: "{{ kayobe_config_path ~ ('/environments/' ~ kayobe_environment if kayobe_environment else '') }}"
 
+# Ordered list of paths containing kayobe_env_config_path and all its dependent
+# environments.
+kayobe_env_search_paths: "{{ query('cached', 'kayobe_environments') }}"
+
 ###############################################################################
 # Remote path configuration (seed, seed-hypervisor and overcloud hosts).
 

ansible/inventory/group_vars/all/kolla

@@ -574,6 +574,16 @@ kolla_enable_vitrage: "no"
 kolla_enable_watcher: "no"
 kolla_enable_zun: "no"
 
+###############################################################################
+# Kolla custom config generation.
+
+# Feature flag to add $KAYOBE_CONFIG_PATH to the list of search paths used
+# when searching for Kolla custom service configuration. Only has an effect in
+# a multiple environments setup. This allows you to configure merging between
+# your environment and the base layer. Defaults to true. Set to false for
+# backwards compatibility.
+kolla_openstack_custom_config_environment_merging_enabled: true
+
 ###############################################################################
 # Passwords and credentials.
 

ansible/kolla-ansible.yml

@@ -91,9 +91,9 @@
         kolla_external_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy.pem"
         kolla_internal_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy-internal.pem"
         kolla_ansible_passwords_path: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
-        kolla_overcloud_inventory_search_paths:
+        kolla_overcloud_inventory_search_paths_static:
           - "{{ kayobe_config_path }}"
-          - "{{ kayobe_env_config_path }}"
+        kolla_overcloud_inventory_search_paths: "{{ kolla_overcloud_inventory_search_paths_static + kayobe_env_search_paths }}"
         kolla_ansible_certificates_path: "{{ kayobe_env_config_path }}/kolla/certificates"
         # NOTE: This differs from the default SELinux mode in kolla ansible,
         # which is permissive. The justification for using this mode is twofold:
@@ -109,9 +109,9 @@
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
         kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
         kolla_enable_host_ntp: false
-        kolla_globals_paths_extra:
+        kolla_globals_paths_static:
           - "{{ kayobe_config_path }}"
-          - "{{ kayobe_env_config_path }}"
+        kolla_globals_paths_extra: "{{ kolla_globals_paths_static + kayobe_env_search_paths }}"
 
 - name: Generate Kolla Ansible host vars for the seed host
   hosts: seed

ansible/kolla-bifrost.yml

@@ -14,6 +14,6 @@
       kolla_bifrost_dnsmasq_dns_servers: "{{ resolv_nameservers | default([]) }}"
       kolla_bifrost_domain: "{{ resolv_domain | default }}"
       kolla_bifrost_download_ipa: "{{ not ipa_build_images | bool }}"
-      kolla_bifrost_config_paths_extra:
+      kolla_bifrost_config_paths_static:
         - "{{ kayobe_config_path }}"
-        - "{{ kayobe_env_config_path }}"
+      kolla_bifrost_config_paths_extra: "{{ kolla_bifrost_config_paths_static + kayobe_env_search_paths }}"

ansible/kolla-build.yml

@@ -7,7 +7,7 @@
     - role: kolla
       kolla_install_epel: "{{ dnf_install_epel }}"
     - role: kolla-build
-      kolla_build_config_paths_extra:
+      kolla_build_config_paths_static:
         - "{{ kayobe_config_path }}"
-        - "{{ kayobe_env_config_path }}"
+      kolla_build_config_paths_extra: "{{ kolla_build_config_paths_static + kayobe_env_search_paths }}"
       kolla_base_tag: "{{ kolla_base_distro_version }}"

ansible/kolla-openstack.yml

@@ -100,53 +100,6 @@
     ipa_image_name: "ipa"
   pre_tasks:
     - block:
-        - name: Check whether Kolla extra configuration files exist
-          stat:
-            path: "{{ kayobe_env_config_path }}/kolla/config/{{ item.file }}"
-            get_checksum: False
-            get_md5: False
-            mime: False
-          register: stat_result
-          with_items:
-            - { name: aodh, file: aodh.conf }
-            - { name: barbican, file: barbican.conf }
-            - { name: blazar, file: blazar.conf }
-            - { name: ceilometer, file: ceilometer.conf }
-            - { name: cinder, file: cinder.conf }
-            - { name: cloudkitty, file: cloudkitty.conf }
-            - { name: designate, file: designate.conf }
-            - { name: glance, file: glance.conf }
-            - { name: global, file: global.conf }
-            - { name: gnocchi, file: gnocchi.conf }
-            - { name: grafana, file: grafana.ini }
-            - { name: heat, file: heat.conf }
-            - { name: inspector, file: ironic-inspector.conf }
-            - { name: ironic, file: ironic.conf }
-            - { name: keystone, file: keystone.conf }
-            - { name: magnum, file: magnum.conf }
-            - { name: manila, file: manila.conf }
-            - { name: mariabackup, file: backup.my.cnf }
-            - { name: mariadb, file: galera.cnf }
-            - { name: masakari, file: masakari.conf }
-            - { name: multipathd, file: multipath.conf }
-            - { name: murano, file: murano.conf }
-            - { name: neutron, file: neutron.conf }
-            - { name: neutron_ml2, file: neutron/ml2_conf.ini }
-            - { name: nova, file: nova.conf }
-            - { name: octavia, file: octavia.conf }
-            - { name: placement, file: placement.conf }
-            - { name: sahara, file: sahara.conf }
-
-        - name: Initialise a fact containing extra configuration
-          set_fact:
-            kolla_extra_config: {}
-
-        - name: Update a fact containing extra configuration
-          set_fact:
-            kolla_extra_config: "{{ kolla_extra_config | combine({item.item.name: lookup('template', '{{ item.stat.path }}')}) }}"
-          with_items: "{{ stat_result.results }}"
-          when: item.stat.exists
-
         - name: Validate switch configuration for Neutron ML2 genericswitch driver
           fail:
             msg: >
@@ -217,35 +170,11 @@
       kolla_inspector_swift_auth:
         auth_type: none
         endpoint_override: "http://{% raw %}{{ api_interface_address }}{% endraw %}:{{ inspector_store_port }}"
-      # Extra free-form user-provided configuration.
-      kolla_extra_aodh: "{{ kolla_extra_config.aodh | default }}"
-      kolla_extra_barbican: "{{ kolla_extra_config.barbican | default }}"
-      kolla_extra_blazar: "{{ kolla_extra_config.blazar | default }}"
-      kolla_extra_ceilometer: "{{ kolla_extra_config.ceilometer | default }}"
-      kolla_extra_cinder: "{{ kolla_extra_config.cinder | default }}"
-      kolla_extra_cloudkitty: "{{ kolla_extra_config.cloudkitty | default }}"
-      kolla_extra_designate: "{{ kolla_extra_config.designate | default }}"
-      kolla_extra_glance: "{{ kolla_extra_config.glance | default }}"
-      kolla_extra_global: "{{ kolla_extra_config.global | default }}"
-      kolla_extra_gnocchi: "{{ kolla_extra_config.gnocchi | default }}"
-      kolla_extra_grafana: "{{ kolla_extra_config.grafana | default }}"
-      kolla_extra_heat: "{{ kolla_extra_config.heat | default }}"
-      kolla_extra_inspector: "{{ kolla_extra_config.inspector | default }}"
-      kolla_extra_ironic: "{{ kolla_extra_config.ironic | default }}"
-      kolla_extra_keystone: "{{ kolla_extra_config.keystone | default }}"
-      kolla_extra_magnum: "{{ kolla_extra_config.magnum | default }}"
-      kolla_extra_manila: "{{ kolla_extra_config.manila | default }}"
-      kolla_extra_mariabackup: "{{ kolla_extra_config.mariabackup | default }}"
-      kolla_extra_mariadb: "{{ kolla_extra_config.mariadb | default }}"
-      kolla_extra_masakari: "{{ kolla_extra_config.masakari | default }}"
-      kolla_extra_multipathd: "{{ kolla_extra_config.multipathd | default }}"
-      kolla_extra_murano: "{{ kolla_extra_config.murano | default }}"
-      kolla_extra_neutron: "{{ kolla_extra_config.neutron | default }}"
-      kolla_extra_neutron_ml2: "{{ kolla_extra_config.neutron_ml2 | default }}"
-      kolla_extra_nova: "{{ kolla_extra_config.nova | default }}"
-      kolla_extra_octavia: "{{ kolla_extra_config.octavia | default }}"
-      kolla_extra_placement: "{{ kolla_extra_config.placement | default }}"
-      kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
-      kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config"
+      kolla_openstack_custom_config_paths_extra_multi_env_static:
+        - "{{ kayobe_config_path }}"
+      kolla_openstack_custom_config_paths_extra_multi_env: "{{ kolla_openstack_custom_config_paths_extra_multi_env_static + kayobe_env_search_paths }}"
+      kolla_openstack_custom_config_paths_extra_legacy:
+        - "{{ kayobe_env_config_path }}"
+      kolla_openstack_custom_config_paths_extra: "{{ kolla_openstack_custom_config_paths_extra_multi_env if kolla_openstack_custom_config_environment_merging_enabled | bool else kolla_openstack_custom_config_paths_extra_legacy }}"
       kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
       kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt"

ansible/lookup_plugins/cached.py

@@ -0,0 +1 @@
+from ansible_cached_lookup import LookupModule

ansible/lookup_plugins/kayobe_environments.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2023 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__metaclass__ = type
+
+import kayobe.plugins.lookup.environments
+
+LookupModule = kayobe.plugins.lookup.environments.LookupModule

ansible/roles/image-download/tasks/main.yml

@@ -1,4 +1,9 @@
 ---
+- name: Ensure destination directory exists
+  file:
+    state: directory
+    path: "{{ image_download_dest | dirname }}"
+
 - block:
     - block:
         - name: Fail if the checksum algorithm is not set