Configure Ceph RGW with a object storage network

When Ceph RGW is used, an endpoint for object storage is
hosted not in a pod on k8s but on an EDPM node. Thus, a
service hosted on an EDPM node will need to be accessed
from a separate network.

This patch adds the object storage network (storageobj)
with VLAN 25 and range 172.22.0.0/24 in the reproducers
networking-definition.yml. It also adds a multus range
for this network so that the Tempest pod can access this
network for testing. The object storage network is added
to the OCP nodes for the same reason.

The Ceph playbook cifmw_cephadm_rgw_network parameter may
then be set to the new network so that object storage may
be configured on a separate network. If this network is
parameter is not set, then the storage network is used.

Signed-off-by: John Fulton <[email protected]>

playbooks/ceph.yml

@@ -338,27 +338,43 @@
     # public network always exist because is provided by the ceph_spec role
     - name: Get Storage network range
       ansible.builtin.set_fact:
-        cifmw_cephadm_rgw_network: "{{ lookup('ansible.builtin.ini', 'public_network section=global file=' ~ cifmw_cephadm_bootstrap_conf) }}"
+        cifmw_cephadm_storage_network: "{{ lookup('ansible.builtin.ini', 'public_network section=global file=' ~ cifmw_cephadm_bootstrap_conf) }}"
+
+    - name: Set RGW network range to storage network only if it was not provided
+      ansible.builtin.set_fact:
+        cifmw_cephadm_rgw_network: "{{ cifmw_cephadm_storage_network }}"
+      when:
+        - cifmw_cephadm_rgw_network is not defined or
+          cifmw_cephadm_rgw_network | length == 0
 
     - name: Set IP address of first monitor
       ansible.builtin.set_fact:
-        cifmw_cephadm_first_mon_ip: "{{ hostvars[this_host][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first }}"
+        cifmw_cephadm_first_mon_ip: "{{ hostvars[this_host][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_storage_network) | first }}"
       vars:
         this_host: "{{ _target_hosts | first }}"
 
     - name: Assert if any EDPM nodes n/w interface is missing in storage network
+      ansible.builtin.assert:
+        that:
+          - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_storage_network) | length > 0
+        fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_storage_network }}"
+      loop: "{{ _target_hosts }}"
+
+    - name: Assert if any EDPM nodes n/w interface is missing in RGW network
       ansible.builtin.assert:
         that:
           - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | length > 0
         fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_rgw_network }}"
       loop: "{{ _target_hosts }}"
+      when:
+        - cifmw_cephadm_rgw_network != cifmw_cephadm_storage_network
 
-    - name: Get already assigned IP addresses
+    - name: Get already assigned RGW IP addresses
       ansible.builtin.set_fact:
         ips: "{{ ips | default([]) + [ hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first ] }}"
       loop: "{{ _target_hosts }}"
 
-    # cifmw_cephadm_vip is the VIP reserved in the Storage network
+    # cifmw_cephadm_vip is the VIP reserved in the RGW network
     - name: Set VIP var as empty string
       ansible.builtin.set_fact:
         cifmw_cephadm_vip: ""

roles/cifmw_cephadm/tasks/check_vip.yml

@@ -22,7 +22,7 @@
       ansible.builtin.set_fact:
         count: "{{ 2 if count is undefined else count | int + 2 }}"
 
-    - name: Get an IP address from the Storage network
+    - name: Get an IP address from the RGW network
       ansible.builtin.set_fact:
         cur_ip: "{{ cifmw_cephadm_rgw_network | ansible.utils.next_nth_usable(count) }}"
 

scenarios/reproducers/networking-definition.yml

@@ -91,6 +91,19 @@ cifmw_networking_definition:
               end: 250
       vlan: 23
       mtu: 1500
+    storageobj:
+      network: "172.22.0.0/24"
+      tools:
+        netconfig:
+          ranges:
+            - start: 100
+              end: 250
+        multus:
+          ranges:
+            - start: 30
+              end: 70
+      vlan: 25
+      mtu: 1500
 
   group-templates:
     ocps:
@@ -106,6 +119,8 @@ cifmw_networking_definition:
           trunk-parent: ctlplane
         storage:
           trunk-parent: ctlplane
+        storageobj:
+          trunk-parent: ctlplane
     ocp_workers:
       network-template:
         range:
@@ -127,6 +142,8 @@ cifmw_networking_definition:
           trunk-parent: ctlplane
         storagemgmt:
           trunk-parent: ctlplane
+        storageobj:
+          trunk-parent: ctlplane
     cephs:
       network-template:
         range: