From 053914a1ea42a428d53bc0e2ba505c4849436caf Mon Sep 17 00:00:00 2001 From: Pragadeeswaran Sathyanarayanan Date: Wed, 1 May 2024 19:37:11 +0530 Subject: [PATCH] [shiftonstack] Type 3 - external ceph support Deployed Topology 03 type for OpenShift on OpenStack. Signed-off-by: Pragadeeswaran Sathyanarayanan --- automation/vars/shiftonstack.yaml | 100 ++++++++ dt/shiftonstack03/README.md | 11 + dt/shiftonstack03/ceph/kustomization.yaml | 34 +++ .../ceph/nodeset/kustomization.yaml | 38 +++ dt/shiftonstack03/compute/bm_password.yaml | 10 + dt/shiftonstack03/compute/extra_mounts.yaml | 19 ++ dt/shiftonstack03/compute/kustomization.yaml | 101 ++++++++ dt/shiftonstack03/compute/nova_ceph.yaml | 29 +++ .../control-plane/ceph_secret.yaml | 10 + .../control-plane/kustomization.yaml | 146 +++++++++++ .../control-plane/pre-ceph/kustomization.yaml | 220 ++++++++++++++++ .../control-plane/pre-ceph/nad.yaml | 8 + .../pre-ceph/nncp/kustomization.yaml | 141 +++++++++++ dt/shiftonstack03/kustomization.yaml | 34 +++ examples/dt/shiftonstack03/.gitignore | 1 + examples/dt/shiftonstack03/README.md | 65 +++++ examples/dt/shiftonstack03/ceph.md | 40 +++ examples/dt/shiftonstack03/ceph/.gitignore | 1 + .../dt/shiftonstack03/ceph/kustomization.yaml | 9 + .../dt/shiftonstack03/ceph/nodeset/.gitignore | 1 + .../ceph/nodeset/kustomization.yaml | 9 + .../shiftonstack03/ceph/nodeset/values.yaml | 186 ++++++++++++++ examples/dt/shiftonstack03/ceph/values.yaml | 10 + examples/dt/shiftonstack03/compute/.gitignore | 1 + .../shiftonstack03/compute/kustomization.yaml | 9 + .../dt/shiftonstack03/compute/values.yaml | 168 +++++++++++++ examples/dt/shiftonstack03/control-plane.md | 38 +++ .../shiftonstack03/control-plane/.gitignore | 1 + .../control-plane/kustomization.yaml | 11 + .../control-plane/pre-ceph/.gitignore | 1 + .../control-plane/pre-ceph/kustomization.yaml | 10 + .../control-plane/pre-ceph/nncp/.gitignore | 1 + .../pre-ceph/nncp/kustomization.yaml | 23 ++ .../control-plane/pre-ceph/nncp/values.yaml | 234 ++++++++++++++++++ .../pre-ceph/service-values.yaml | 68 +++++ .../control-plane/service-values.yaml | 98 ++++++++ examples/dt/shiftonstack03/data-plane.md | 49 ++++ examples/dt/shiftonstack03/kustomization.yaml | 9 + examples/dt/shiftonstack03/values.yaml | 10 + 39 files changed, 1954 insertions(+) create mode 100644 automation/vars/shiftonstack.yaml create mode 100644 dt/shiftonstack03/README.md create mode 100644 dt/shiftonstack03/ceph/kustomization.yaml create mode 100644 dt/shiftonstack03/ceph/nodeset/kustomization.yaml create mode 100644 dt/shiftonstack03/compute/bm_password.yaml create mode 100644 dt/shiftonstack03/compute/extra_mounts.yaml create mode 100644 dt/shiftonstack03/compute/kustomization.yaml create mode 100644 dt/shiftonstack03/compute/nova_ceph.yaml create mode 100644 dt/shiftonstack03/control-plane/ceph_secret.yaml create mode 100644 dt/shiftonstack03/control-plane/kustomization.yaml create mode 100644 dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml create mode 100644 dt/shiftonstack03/control-plane/pre-ceph/nad.yaml create mode 100644 dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml create mode 100644 dt/shiftonstack03/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/.gitignore create mode 100644 examples/dt/shiftonstack03/README.md create mode 100644 examples/dt/shiftonstack03/ceph.md create mode 100644 examples/dt/shiftonstack03/ceph/.gitignore create mode 100644 examples/dt/shiftonstack03/ceph/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/ceph/nodeset/.gitignore create mode 100644 examples/dt/shiftonstack03/ceph/nodeset/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/ceph/nodeset/values.yaml create mode 100644 examples/dt/shiftonstack03/ceph/values.yaml create mode 100644 examples/dt/shiftonstack03/compute/.gitignore create mode 100644 examples/dt/shiftonstack03/compute/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/compute/values.yaml create mode 100644 examples/dt/shiftonstack03/control-plane.md create mode 100644 examples/dt/shiftonstack03/control-plane/.gitignore create mode 100644 examples/dt/shiftonstack03/control-plane/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/.gitignore create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/.gitignore create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/values.yaml create mode 100644 examples/dt/shiftonstack03/control-plane/pre-ceph/service-values.yaml create mode 100644 examples/dt/shiftonstack03/control-plane/service-values.yaml create mode 100644 examples/dt/shiftonstack03/data-plane.md create mode 100644 examples/dt/shiftonstack03/kustomization.yaml create mode 100644 examples/dt/shiftonstack03/values.yaml diff --git a/automation/vars/shiftonstack.yaml b/automation/vars/shiftonstack.yaml new file mode 100644 index 000000000..fb881f5de --- /dev/null +++ b/automation/vars/shiftonstack.yaml @@ -0,0 +1,100 @@ +--- +# Stores the different automation workflow steps required by various +# shiftonstack deployed topologies. +vas: + shiftonstack03: + stages: + - path: examples/dt/shiftonstack03/control-plane/pre-ceph/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=60s + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - path: examples/dt/shiftonstack03/control-plane/pre-ceph + wait_conditions: + - >- + oc -n openstack wait openstackcontrolplane + controlplane + --for condition=Ready + --timeout=30m + values: + - name: network-values + src_file: nncp/values.yaml + - name: service-values + src_file: service-values.yaml + build_output: control-plane.yaml + + - path: examples/dt/shiftonstack03/ceph/nodeset + wait_conditions: + - >- + oc -n openstack wait openstackdataplanenodeset + ceph-nodes + --for condition=SetupReady + --timeout=600s + values: + - name: edpm-nodeset-values + src_file: values.yaml + build_output: ceph-nodeset.yaml + + - path: examples/dt/shiftonstack03/ceph + wait_conditions: + - >- + oc -n openstack wait openstackdataplanedeployment + ceph-deploy + --for condition=Ready + --timeout=1500s + values: + - name: edpm-deployment-values + src_file: values.yaml + build_output: ceph-deploy.yaml + post_stage_run: + - name: Deploy ceph + type: playbook + source: "../../playbooks/ceph.yml" + inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml" + + - path: examples/dt/shiftonstack03/control-plane + wait_conditions: + - >- + oc -n openstack wait openstackcontrolplane + controlplane + --for condition=Ready + --timeout=30m + values: + - name: network-values + src_file: pre-ceph/nncp/values.yaml + - name: service-values + src_file: pre-ceph/service-values.yaml + - name: service-values-post-ceph + src_file: service-values.yaml + build_output: control-plane-post-ceph.yaml + + - path: examples/dt/shiftonstack03/compute + wait_conditions: + - >- + oc -n openstack wait openstackdataplanenodeset + compute-nodes + --for condition=SetupReady + --timeout=600s + values: + - name: edpm-nodeset-values + src_file: values.yaml + build_output: compute-nodes.yaml + + - path: examples/dt/shiftonstack03/compute + wait_conditions: + - >- + oc -n openstack wait openstackdataplanedeployment + compute-deploy + --for condition=Ready + --timeout=1500s + values: + - name: edpm-deployment-values + src_file: values.yaml + build_output: compute-deploy.yaml diff --git a/dt/shiftonstack03/README.md b/dt/shiftonstack03/README.md new file mode 100644 index 000000000..1b6002f77 --- /dev/null +++ b/dt/shiftonstack03/README.md @@ -0,0 +1,11 @@ +# OpenShift on OpenStack - Deployment topology type 3 + +Please refer the [README](../../examples/dt/shiftonstack03/README.md) if you +are looking for information on how to deploy the type 3 deployment topology +of OpenShift on OpenStack. + +This directory `dt/shiftonstack03/`, exists so that the +[kustomization.yaml](../../examples/dt/shiftonstack03/kustomization.yaml) in +the examples directory of shiftonstack03, reference it by path as a +component. It's contents are likely uninteresting unless you want to understand +how kustomize was implemented in this repository. diff --git a/dt/shiftonstack03/ceph/kustomization.yaml b/dt/shiftonstack03/ceph/kustomization.yaml new file mode 100644 index 000000000..9e00d2e6e --- /dev/null +++ b/dt/shiftonstack03/ceph/kustomization.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../lib/dataplane/deployment + +patches: + - target: + kind: OpenStackDataPlaneDeployment + name: .* + patch: |- + - op: replace + path: /metadata/name + value: ceph-deploy + + - op: replace + path: /spec/nodeSets + value: + - ceph-nodes diff --git a/dt/shiftonstack03/ceph/nodeset/kustomization.yaml b/dt/shiftonstack03/ceph/nodeset/kustomization.yaml new file mode 100644 index 000000000..2f5481b12 --- /dev/null +++ b/dt/shiftonstack03/ceph/nodeset/kustomization.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/dataplane/nodeset + +patches: + - target: + kind: OpenStackDataPlaneNodeSet + name: .* + patch: |- + - op: replace + path: /metadata/name + value: ceph-nodes + + - target: + kind: Secret + name: nova-migration-ssh-key + patch: |- + - op: add + path: /metadata/annotations + value: + config.kubernetes.io/local-config: true diff --git a/dt/shiftonstack03/compute/bm_password.yaml b/dt/shiftonstack03/compute/bm_password.yaml new file mode 100644 index 000000000..be0c73a7a --- /dev/null +++ b/dt/shiftonstack03/compute/bm_password.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +type: Opaque + +metadata: + name: baremetalset-password-secret + +data: + NodeRootPassword: _replaced_ diff --git a/dt/shiftonstack03/compute/extra_mounts.yaml b/dt/shiftonstack03/compute/extra_mounts.yaml new file mode 100644 index 000000000..e288003a5 --- /dev/null +++ b/dt/shiftonstack03/compute/extra_mounts.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneNodeSet + +metadata: + name: compute-nodes + +spec: + nodeTemplate: + extraMounts: + - extraVolType: Ceph + mounts: + - mountPath: /etc/ceph + name: ceph + readOnly: true + volumes: + - name: ceph + secret: + secretName: ceph-conf-files diff --git a/dt/shiftonstack03/compute/kustomization.yaml b/dt/shiftonstack03/compute/kustomization.yaml new file mode 100644 index 000000000..4811809d6 --- /dev/null +++ b/dt/shiftonstack03/compute/kustomization.yaml @@ -0,0 +1,101 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../lib/dataplane/nodeset + +resources: + - bm_password.yaml + - nova_ceph.yaml + +patches: + - target: + kind: OpenStackDataPlaneNodeSet + name: .* + patch: |- + - op: replace + path: /metadata/name + value: compute-nodes + + - target: + kind: OpenStackDataPlaneNodeSet + name: .* + path: extra_mounts.yaml + +replacements: + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nova.ceph.conf + targets: + - select: + kind: ConfigMap + name: ceph-nova + fieldPaths: + - data.03-ceph-nova\.conf + options: + create: true + + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nodeset.services + targets: + - select: + kind: OpenStackDataPlaneNodeSet + fieldPaths: + - spec.services + options: + create: true + + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.root_password + targets: + - select: + kind: Secret + name: baremetalset-password-secret + fieldPaths: + - data.NodeRootPassword + options: + create: true + + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.preProvisioned + targets: + - select: + kind: OpenStackDataPlaneNodeSet + fieldPaths: + - spec.preProvisioned + options: + create: true + + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalSetTemplate + targets: + - select: + kind: OpenStackDataPlaneNodeSet + fieldPaths: + - spec.baremetalSetTemplate + options: + create: true diff --git a/dt/shiftonstack03/compute/nova_ceph.yaml b/dt/shiftonstack03/compute/nova_ceph.yaml new file mode 100644 index 000000000..700f555dc --- /dev/null +++ b/dt/shiftonstack03/compute/nova_ceph.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-nova +data: + 03-ceph-nova.conf: _replaced_ + +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneService +metadata: + name: nova-custom-ceph +spec: + label: dataplane-deployment-nova-custom-ceph + configMaps: + - ceph-nova + secrets: + - nova-cell1-compute-config + - nova-migration-ssh-key + playbook: osp.edpm.nova + tlsCert: + contents: + - dnsnames + - ips + networks: + - ctlplane + issuer: osp-rootca-issuer-internal + caCerts: combined-ca-bundle diff --git a/dt/shiftonstack03/control-plane/ceph_secret.yaml b/dt/shiftonstack03/control-plane/ceph_secret.yaml new file mode 100644 index 000000000..c148f8ed2 --- /dev/null +++ b/dt/shiftonstack03/control-plane/ceph_secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: ceph-conf-files + namespace: openstack +type: Opaque +data: + ceph.client.openstack.keyring: _replaced_ + ceph.conf: _replaced_ diff --git a/dt/shiftonstack03/control-plane/kustomization.yaml b/dt/shiftonstack03/control-plane/kustomization.yaml new file mode 100644 index 000000000..d3fe740a9 --- /dev/null +++ b/dt/shiftonstack03/control-plane/kustomization.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../lib/control-plane + +resources: + - ceph_secret.yaml + +replacements: + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.cinderBackup.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.cinder.template.cinderBackup.customServiceConfig + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.cinderVolumes.ceph + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.cinder.template.cinderVolumes.ceph + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.glance.glanceAPIs.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.manila.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.manila.enabled + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.manila.manilaAPI.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.manila.template.manilaAPI.customServiceConfig + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.manila.manilaShares.share1.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.manila.template.manilaShares.share1.customServiceConfig + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.extraMounts + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.extraMounts + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.ceph.keyring + targets: + - select: + kind: Secret + name: ceph-conf-files + fieldPaths: + - data.ceph\.client\.openstack\.keyring + options: + create: true + + - source: + kind: ConfigMap + name: service-values-post-ceph + fieldPath: data.ceph.conf + targets: + - select: + kind: Secret + name: ceph-conf-files + fieldPaths: + - data.ceph\.conf + options: + create: true diff --git a/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml b/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml new file mode 100644 index 000000000..526548e4a --- /dev/null +++ b/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml @@ -0,0 +1,220 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +secretGenerator: + - name: octavia-ca-passphrase + literals: + - server-ca-passphrase=12345678 + options: + disableNameSuffixHash: true + +components: + - ../../../../lib/networking/metallb + - ../../../../lib/networking/netconfig + - ../../../../lib/networking/nad + - ../../../../lib/control-plane + +resources: + - nad.yaml + +patches: + - target: + version: v1beta1 + kind: NetConfig + name: netconfig + patch: |- + - op: add + path: /spec/networks/- + value: + dnsDomain: _replaced_ + name: storagemgmt + subnets: + - _replaced_ + mtu: 1500 + +replacements: + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.dnsDomain + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].dnsDomain + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.mtu + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].mtu + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.subnets + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].subnets + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.net-attach-def + targets: + - select: + kind: NetworkAttachmentDefinition + name: octavia + fieldPaths: + - spec.config + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.enabled + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.amphoraImageContainerImage + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.amphoraImageContainerImage + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.apacheContainerImage + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.apacheContainerImage + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.octaviaAPI.networkAttachments + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.octaviaAPI.networkAttachments + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.octaviaHousekeeping.networkAttachments + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.octaviaHousekeeping.networkAttachments + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.octaviaHealthManager.networkAttachments + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.octaviaHealthManager.networkAttachments + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.octavia.octaviaWorker.networkAttachments + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.octavia.template.octaviaWorker.networkAttachments + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.heat.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.heat.enabled + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.ovn.ovnController.nicMappings + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.ovn.template.ovnController.nicMappings + options: + create: true + + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true diff --git a/dt/shiftonstack03/control-plane/pre-ceph/nad.yaml b/dt/shiftonstack03/control-plane/pre-ceph/nad.yaml new file mode 100644 index 000000000..d59e60095 --- /dev/null +++ b/dt/shiftonstack03/control-plane/pre-ceph/nad.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: octavia + labels: + osp/net: octavia + osp/net-attach-def-type: standard diff --git a/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml b/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml new file mode 100644 index 000000000..69eb31c7b --- /dev/null +++ b/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml @@ -0,0 +1,141 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../lib/nncp + +patches: + - target: + kind: NodeNetworkConfigurationPolicy + name: master-0 + patch: &octavia_vlan_template |- + - op: add + path: /spec/desiredState/interfaces/- + value: + description: Octavia vlan host interface + name: octavia + state: up + type: vlan + vlan: + base-iface: _replaced_ + id: _replaced_ + + - target: + kind: NodeNetworkConfigurationPolicy + name: master-0 + patch: &octavia_br_template |- + - op: add + path: /spec/desiredState/interfaces/- + value: + description: Octavia bridge + mtu: 1500 + name: octbr + type: linux-bridge + bridge: + options: + stp: + enabled: false + port: + - name: octavia + + - target: + kind: NodeNetworkConfigurationPolicy + name: master-1 + patch: *octavia_vlan_template + + - target: + kind: NodeNetworkConfigurationPolicy + name: master-1 + patch: *octavia_br_template + + - target: + kind: NodeNetworkConfigurationPolicy + name: master-2 + patch: *octavia_vlan_template + + - target: + kind: NodeNetworkConfigurationPolicy + name: master-2 + patch: *octavia_br_template + +replacements: + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-0 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.base-iface + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-0 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.id + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-1 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.base-iface + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-1 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.id + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-2 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.base-iface + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-2 + fieldPaths: + - spec.desiredState.interfaces.[name=octavia].vlan.id diff --git a/dt/shiftonstack03/kustomization.yaml b/dt/shiftonstack03/kustomization.yaml new file mode 100644 index 000000000..2fe844b4a --- /dev/null +++ b/dt/shiftonstack03/kustomization.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../lib/dataplane/deployment + +patches: + - target: + kind: OpenStackDataPlaneDeployment + name: .* + patch: |- + - op: replace + path: /metadata/name + value: compute-deploy + + - op: replace + path: /spec/nodeSets + value: + - compute-nodes diff --git a/examples/dt/shiftonstack03/.gitignore b/examples/dt/shiftonstack03/.gitignore new file mode 100644 index 000000000..6ab6d30c9 --- /dev/null +++ b/examples/dt/shiftonstack03/.gitignore @@ -0,0 +1 @@ +compute-deploy.yaml diff --git a/examples/dt/shiftonstack03/README.md b/examples/dt/shiftonstack03/README.md new file mode 100644 index 000000000..5ca624f8d --- /dev/null +++ b/examples/dt/shiftonstack03/README.md @@ -0,0 +1,65 @@ +# Deployed Topology - OpenShift on OpenStack - Model 03 + +This document contains and overview about the deployment topology and the list +of tests used for evaluating the deployment. It also contains a collection of +custom resources (CRs) for deploying the test environment. + +## Purpose + +This topology is used for evaluating the deployment of OpenShift platforms using +OpenStack services running on OpenShift container platform. + +## Environment + +### Nodes + +| Role | Machine Type | Count | +| ------------------ | :-: | :-: | +| Compact OpenShift | VM | 3 | +| OpenStack Compute | BM | 1 | +| Ceph Storage | VM | 3 | + +### Networks + +| Name | Type | Interface | CIDR | +| --- | --- | :-: | --------------- | +| OpenShift Machine Network | untagged | nic2 | 10.46.22.128/26 | +| OpenStack Trunk Network | trunk | nic3 | | + +#### Networks in OpenStack Trunk + +| Name | Type | CIDR | +| --- | :-: | :-: | +| ctlplane | untagged | 192.168.122.0/24 | +| internalapi | VLAN 120 | 172.17.0.0/24 | +| storage | VLAN 121 | 172.18.0.0/24 | +| tenant | VLAN 122 | 172.19.0.0/24 | +| storagemgmt | VLAN 123 | 172.20.0.0/24 | +| octavia | VLAN 124 | 172.21.0.0/24 | + +### OpenStack services + +| Service | Configuration | +| --- | --- | +| Cinder | Backend - Ceph | +| Cinder Backup | Backend - Ceph | +| Glance | Backend - Ceph | +| Swift | default | +| Octavia | Backend - amphora | +| Manila | Backend - NFS-Ganesha | + +#### Support services + +| Service | Reason | +| --- | --- | +| Horizon | needed by other services. | +| Keystone | needed by all services. | +| Nova | needed by testing. | +| Neutron | needed by other services. | + +## Workflow + +1. [Install the OpenStack K8S operators and their dependencies](../../common/README.md) +2. [Configure and deploy the OpenStack control plane](control-plane.md) +3. [Configure and deploy Ceph nodes using EDPM](ceph.md) +4. [Configure and deploy the OpenStack data plane](data-plane.md) diff --git a/examples/dt/shiftonstack03/ceph.md b/examples/dt/shiftonstack03/ceph.md new file mode 100644 index 000000000..a6890c2f0 --- /dev/null +++ b/examples/dt/shiftonstack03/ceph.md @@ -0,0 +1,40 @@ +# Configuring and deploying EDPM Ceph nodes + +## Assumptions + +- The [control-plane](control-plane.md) has been created and successfully + deployed. + +## Steps + +```bash +# Switch to openstack namespace +$ oc project openstack + +# Change the working directory to shiftonstack +$ cd architecture/examples/dt/shiftonstack03 + +# Modify ceph nodeset values accordingly +$ pushd ceph/nodeset +$ vi values.yaml + +# Generate the nodeset CR. +$ kustomize build > ceph-nodeset.yaml + +# Apply the resource. +$ oc apply -f ceph-nodeset.yaml + +# Wait for the nodeset to be ready state. +$ oc wait osdpns ceph-nodes --for condition=SetupReady --timeout=300s +$ popd + +# Generate the deployment plan +$ pushd ceph +$ kustomize build > ceph-deploy.yaml + +# Apply the deployment resource. +$ oc apply -f ceph-deploy.yaml + +# Wait for the deployment to be completed. +$ oc wait osdpd ceph-deploy --for condition=Ready --timeout=600s +``` diff --git a/examples/dt/shiftonstack03/ceph/.gitignore b/examples/dt/shiftonstack03/ceph/.gitignore new file mode 100644 index 000000000..78a6b7d74 --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/.gitignore @@ -0,0 +1 @@ +ceph-deploy.yaml diff --git a/examples/dt/shiftonstack03/ceph/kustomization.yaml b/examples/dt/shiftonstack03/ceph/kustomization.yaml new file mode 100644 index 000000000..bdb45fdd0 --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../dt/shiftonstack03/ceph + +resources: + - values.yaml diff --git a/examples/dt/shiftonstack03/ceph/nodeset/.gitignore b/examples/dt/shiftonstack03/ceph/nodeset/.gitignore new file mode 100644 index 000000000..592d32a80 --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/nodeset/.gitignore @@ -0,0 +1 @@ +ceph-nodeset.yaml diff --git a/examples/dt/shiftonstack03/ceph/nodeset/kustomization.yaml b/examples/dt/shiftonstack03/ceph/nodeset/kustomization.yaml new file mode 100644 index 000000000..5a60d5aa7 --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/nodeset/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../dt/shiftonstack03/ceph/nodeset + +resources: + - values.yaml diff --git a/examples/dt/shiftonstack03/ceph/nodeset/values.yaml b/examples/dt/shiftonstack03/ceph/nodeset/values.yaml new file mode 100644 index 000000000..4e547476d --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/nodeset/values.yaml @@ -0,0 +1,186 @@ +--- +# local-config: referenced, but not emitted by kustomize +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" + +data: + ssh_keys: + authorized: CHANGEME + private: CHANGEME2 + public: CHANGEME3 + + # Do not remove the nova key. It allows us to reuse dataplane component. + nova: + migration: + ssh_keys: + private: PLACEHOLDER + public: PLACEHOLDER + + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + timesync_ntp_servers: + - hostname: clock.redhat.com + + gather_facts: false + enable_debug: false + + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - '192.168.122.0/24' + + edpm_network_config_template: | + --- + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: interface + name: nic1 + mtu: {{ external_mtu }} + dns_servers: {{ external_dns_nameservers }} + domain: {{ dns_search_domains }} + use_dhcp: false + addresses: + - ip_netmask: {{ external_ip }}/{{ external_cidr }} + routes: {{ external_host_routes }} + - type: ovs_bridge + name: osp_trunk + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + # force the MAC address of the bridge to this interface + primary: true + {% for network in nodeset_networks %} + {% if network != 'external' %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: >- + {{ + lookup('vars', networks_lower[network] ~ '_ip') + }}/{{ + lookup('vars', networks_lower[network] ~ '_cidr') + }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endif %} + {% endfor %} + + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: eth1 + + edpm_ceph_hci_pre_enabled_services: + - ceph_mon + - ceph_mgr + - ceph_osd + - ceph_rgw + - ceph_nfs + - ceph_rgw_frontend + - ceph_nfs_frontend + + storage_mtu: 9000 + storage_mgmt_mtu: 9000 + storage_mgmt_vlan_id: 123 + storage_mgmt_cidr: "24" + storage_mgmt_host_routes: [] + + networks: + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: storagemgmt + subnetName: subnet1 + - name: external + subnetName: subnet1 + + nodes: + edpm-ceph-0: + hostName: edpm-ceph-0 + ansible: + ansibleHost: 192.168.122.106 + networks: + - name: ctlplane + subnetName: subnet1 + defaultRoute: true + fixedIP: 192.168.122.106 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: storagemgmt + subnetName: subnet1 + - name: external + subnetName: subnet1 + fixedIP: 10.46.22.135 + edpm-ceph-1: + hostName: edpm-ceph-1 + ansible: + ansibleHost: 192.168.122.107 + networks: + - name: ctlplane + subnetName: subnet1 + defaultRoute: true + fixedIP: 192.168.122.107 + - name: internalapi + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + - name: storagemgmt + subnetName: subnet1 + - name: external + subnetName: subnet1 + fixedIP: 10.46.22.136 + edpm-ceph-2: + hostName: edpm-networker-2 + ansible: + ansibleHost: 192.168.122.108 + networks: + - name: ctlplane + subnetName: subnet1 + defaultRoute: true + fixedIP: 192.168.122.108 + - name: internalapi + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + - name: storagemgt + subnetName: subnet1 + - name: external + subnetName: subnet1 + fixedIP: 10.46.22.137 + + services: + - bootstrap + - configure-network + - validate-network + - install-os + - ceph-hci-pre + - configure-os + - ssh-known-hosts + - run-os + - reboot-os + - install-certs diff --git a/examples/dt/shiftonstack03/ceph/values.yaml b/examples/dt/shiftonstack03/ceph/values.yaml new file mode 100644 index 000000000..e44f7937e --- /dev/null +++ b/examples/dt/shiftonstack03/ceph/values.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: edpm-deployment-values + annotations: + config.kubernetes.io/local-config: "true" + +data: {} diff --git a/examples/dt/shiftonstack03/compute/.gitignore b/examples/dt/shiftonstack03/compute/.gitignore new file mode 100644 index 000000000..975a3f848 --- /dev/null +++ b/examples/dt/shiftonstack03/compute/.gitignore @@ -0,0 +1 @@ +compute-nodes.yaml diff --git a/examples/dt/shiftonstack03/compute/kustomization.yaml b/examples/dt/shiftonstack03/compute/kustomization.yaml new file mode 100644 index 000000000..3c4f6223b --- /dev/null +++ b/examples/dt/shiftonstack03/compute/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../dt/shiftonstack03/compute + +resources: + - values.yaml diff --git a/examples/dt/shiftonstack03/compute/values.yaml b/examples/dt/shiftonstack03/compute/values.yaml new file mode 100644 index 000000000..8aa08438a --- /dev/null +++ b/examples/dt/shiftonstack03/compute/values.yaml @@ -0,0 +1,168 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" + +data: + ssh_keys: + authorized: _replaced_ + private: _replaced_ + public: _replaced_ + + nova: + migration: + ssh_keys: + private: _replaced_ + public: _replaced_ + + ceph: + conf: _replaced_ + + neutron: + conf: | + [l3_agent] + agent_mode = dvr + + root_password: cmVkaGF0Cg== + preProvisioned: false + + baremetalSetTemplate: + ctlplaneInterface: ens1f1 + provisioningInterface: ens1f0 + + cloudUserName: cloud-admin + bmhLabelSelector: + app: openstack + + passwordSecret: baremetalset-password-secret + namespace: openstack + + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + service_net_map: + nova_api_network: internalapi + nova_libvirt_network: internalapi + timesync_ntp_servers: + - hostname: clock.redhat.com + edpm_network_config_hide_sensitive_logs: false + edpm_network_config_template: | + --- + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: ovs_bridge + name: {{ neutron_physical_bridge_name }} + mtu: {{ external_mtu }} + use_dhcp: false + dns_servers: {{ external_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ external_ip }}/{{ external_cidr }} + routes: {{ external_host_routes }} + members: + - type: interface + name: nic1 + mtu: {{ external_mtu }} + primary: true + - type: ovs_bridge + name: br-osp + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + primary: true + {% for network in nodeset_networks %} + {% if network != 'external' %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: >- + {{ + lookup('vars', networks_lower[network] ~ '_ip') + }}/{{ + lookup('vars', networks_lower[network] ~ '_cidr') + }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endif %} + {% endfor %} + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: eth0 + + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + + edpm_selinux_mode: enforcing + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - 192.168.122.0/24 + + enable_debug: false + gather_facts: false + + networks: + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + - name: external + subnetName: external + + nodes: + edpm-compute-0: + ansible: + ansibleHost: 192.168.122.100 + hostName: edpm-compute-0 + networks: + - defaultRoute: true + fixedIP: 192.168.122.100 + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + - name: external + subnetName: subnet1 + fixedIP: 10.46.22.138 + + services: + - bootstrap + - configure-network + - validate-network + - install-os + - configure-os + - ssh-known-hosts + - run-os + - reboot-os + - install-certs + - ceph-client + - ovn + - neutron-metadata + - libvirt + - nova-custom-ceph + - telemetry diff --git a/examples/dt/shiftonstack03/control-plane.md b/examples/dt/shiftonstack03/control-plane.md new file mode 100644 index 000000000..7adab0d25 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane.md @@ -0,0 +1,38 @@ +# Configuring networking and deploying the OpenStack Control Plane + +## Assumption + +- A storage class called `local-storage` exists. + +## Network configuration + +```bash +# Switch to the openstack namespace +$ oc project openstack + +# Change the working directory +$ cd architecture/examples/dt/shiftonstack03 + +# Ensure the network values are modified based on the environment. +$ pushd control-plane/nncp +$ vim values.yaml + +# Apply network configuration +$ kustomize build > nncp.yaml +$ oc apply -f nncp.yaml + +# Wait for the network configurations to be applied. +$ oc wait nncp \ + -l osp/nncm-config-type=standard \ + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured \ + --timeout=300s +# Ensure the service values are accurate +$ popd + +# Apply the control-plane customizations +$ kustomize build > control-plane.yaml +$ oc apply -f control-plane.yaml + +# Wait for the control-plane setup to be ready. +$ oc wait osctlplane controlplane --for condition=Ready --timeout=600s +``` diff --git a/examples/dt/shiftonstack03/control-plane/.gitignore b/examples/dt/shiftonstack03/control-plane/.gitignore new file mode 100644 index 000000000..8272729a7 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/.gitignore @@ -0,0 +1 @@ +control-plane-post-ceph.yaml diff --git a/examples/dt/shiftonstack03/control-plane/kustomization.yaml b/examples/dt/shiftonstack03/control-plane/kustomization.yaml new file mode 100644 index 000000000..1f3a064c2 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../dt/shiftonstack03/control-plane + +resources: + - pre-ceph/nncp/values.yaml + - pre-ceph/service-values.yaml + - service-values.yaml diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/.gitignore b/examples/dt/shiftonstack03/control-plane/pre-ceph/.gitignore new file mode 100644 index 000000000..3df8f53be --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/.gitignore @@ -0,0 +1 @@ +control-plane.yaml diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml b/examples/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml new file mode 100644 index 000000000..a00ec01cd --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../dt/shiftonstack03/control-plane/pre-ceph + +resources: + - nncp/values.yaml + - service-values.yaml diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/.gitignore b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/.gitignore new file mode 100644 index 000000000..51ed8a956 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/.gitignore @@ -0,0 +1 @@ +nncp.yaml diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml new file mode 100644 index 000000000..f16316672 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../../dt/shiftonstack03/control-plane/pre-ceph/nncp + +resources: + - values.yaml diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/values.yaml b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/values.yaml new file mode 100644 index 000000000..bffd3b05c --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/nncp/values.yaml @@ -0,0 +1,234 @@ +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: network-values + annotations: + config.kubernetes.io/local-config: "true" + +data: + openstack-operator-image: "quay.io/openstack-k8s-operators/openstack-operator-index:latest" + + node_0: + name: master-0 + ctlplane_ip: 192.168.122.10 + internalapi_ip: 172.17.0.10 + storage_ip: 172.18.0.10 + tenant_ip: 172.19.0.10 + node_1: + name: master-1 + ctlplane_ip: 192.168.122.11 + internalapi_ip: 172.17.0.11 + storage_ip: 172.18.0.11 + tenant_ip: 172.19.0.11 + node_2: + name: master-2 + ctlplane_ip: 192.168.122.12 + internalapi_ip: 172.17.0.12 + storage_ip: 172.18.0.12 + tenant_ip: 172.19.0.12 + + ctlplane: + dnsDomain: ctlplane.openstack.lab + subnets: + - allocationRanges: + - start: 192.168.122.100 + end: 192.168.122.120 + cidr: 192.168.122.0/24 + gateway: 192.168.122.1 + name: subnet1 + prefix-length: 24 + iface: enp6s0 + mtu: 9000 + lb_addresses: + - 192.168.122.80-192.168.122.90 + endpoint_annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "ctlplane", + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "192.168.122.0/24", + "range_start": "192.168.122.30", + "range_end": "192.168.122.70" + } + } + + internalapi: + dnsDomain: internalapi.openstack.lab + subnets: + - allocationRanges: + - start: 172.17.0.100 + end: 172.17.0.250 + cidr: 172.17.0.0/24 + name: subnet1 + vlan: 120 + mtu: 1500 + prefix-length: 24 + iface: internalapi + vlan: 120 + base_iface: enp6s0 + lb_addresses: + - 172.17.0.80-172.17.0.90 + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "internalapi", + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "range_start": "172.17.0.30", + "range_end": "172.17.0.70" + } + } + + storage: + dnsDomain: storage.openstack.lab + subnets: + - allocationRanges: + - start: 172.18.0.100 + end: 172.18.0.250 + cidr: 172.18.0.0/24 + name: subnet1 + vlan: 121 + mtu: 9000 + prefix-length: 24 + iface: storage + vlan: 121 + base_iface: enp6s0 + lb_addresses: + - 172.18.0.80-172.18.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.30", + "range_end": "172.18.0.70" + } + } + + tenant: + dnsDomain: tenant.openstack.lab + subnets: + - allocationRanges: + - start: 172.19.0.100 + end: 172.19.0.250 + cidr: 172.19.0.0/24 + name: subnet1 + vlan: 122 + mtu: 1500 + prefix-length: 24 + iface: tenant + vlan: 122 + base_iface: enp6s0 + lb_addresses: + - 172.19.0.80-172.19.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "tenant", + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "172.19.0.0/24", + "range_start": "172.19.0.30", + "range_end": "172.19.0.70" + } + } + + storagemgmt: + dnsDomain: storagemgmt.openstack.lab + subnets: + - allocationRanges: + - start: 172.20.0.100 + end: 172.20.0.250 + cidr: 172.20.0.0/24 + name: subnet1 + vlan: 123 + mtu: 9000 + + octavia: + dnsDomain: octavia.openstack.lab + mtu: 1500 + vlan: 124 + base_iface: enp6s0 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "octavia", + "type": "bridge", + "bridge": "octbr", + "ipam": { + "type": "whereabouts", + "range": "172.21.0.0/24", + "range_start": "172.21.0.30", + "range_end": "172.21.0.70" + } + } + + external: + dnsDomain: openstack.lab + subnets: + - allocationRanges: + - start: 10.46.22.150 + end: 10.46.22.187 + cidr: 10.46.22.128/26 + gateway: 10.46.22.189 + name: subnet1 + mtu: 1600 + + datacentre: + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "datacentre", + "type": "bridge", + "bridge": "ospbr", + "ipam": {} + } + + dns-resolver: + config: + server: + - 192.168.122.1 + search: [] + options: + - key: server + values: + - 192.168.122.1 + + routes: + config: [] + + rabbitmq: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.85 + rabbitmq-cell1: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.86 + + lbServiceType: LoadBalancer + storageClass: local-storage + bridgeName: ospbr diff --git a/examples/dt/shiftonstack03/control-plane/pre-ceph/service-values.yaml b/examples/dt/shiftonstack03/control-plane/pre-ceph/service-values.yaml new file mode 100644 index 000000000..b515e5f9e --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/pre-ceph/service-values.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" + +data: + redis: + enabled: true + + heat: + enabled: true + + swift: + enabled: true + + ovn: + ovnController: + nicMappings: + datacentre: ocpbr + octavia: octbr + + neutron: + customServiceConfig: | + [DEFAULT] + vlan_transparent = true + agent_down_time = 600 + router_distributed = true + l3_ha = true + router_scheduler_driver = neutron.scheduler.l3_agent_scheduler.ChanceScheduler + allow_automatic_l3agent_failover = true + debug = true + + [oslo_policy] + policy_file = /etc/neutron/policy.d/policy.yaml + + [ovs] + igmp_snooping_enable = true + + [ovn] + ovsdb_probe_interval = 60000 + ovn_emit_need_to_frag = true + + [ml2] + mechanism_drivers = openvswitch,l2population + type_drivers = geneve,vxlan,vlan,flat + tenant_network_types = geneve,flat + + # ToDo: Enable Octavia after resolution of OSPCIX-278. + octavia: + enabled: false + amphoraImageContainerImage: quay.io/gthiemonge/octavia-amphora-image + apacheContainerImage: registry.redhat.io/ubi9/httpd-24:latest + octaviaAPI: + networkAttachments: + - internalapi + octaviaHousekeeping: + networkAttachments: + - octavia + octaviaHealthManager: + networkAttachments: + - octavia + octaviaWorker: + networkAttachments: + - octavia diff --git a/examples/dt/shiftonstack03/control-plane/service-values.yaml b/examples/dt/shiftonstack03/control-plane/service-values.yaml new file mode 100644 index 000000000..e1cfe5f92 --- /dev/null +++ b/examples/dt/shiftonstack03/control-plane/service-values.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: service-values-post-ceph + annotations: + config.kubernetes.io/local-config: "true" + +data: + ceph: + conf: _replaced_ + keyring: _replaced_ + + cinderBackup: + customServiceConfig: | + [DEFAULT] + backup_driver = cinder.backup.drivers.ceph.CephBackupDriver + backup_ceph_pool = backups + backup_ceph_user = openstack + + cinderVolumes: + ceph: + customServiceConfig: | + [DEFAULT] + enabled_backends = ceph + + [ceph] + volume_backend_name = ceph + volume_driver = cinder.volume.drivers.rbd.RBDDriver + rbd_ceph_conf = /etc/ceph/ceph.conf + rbd_user = openstack + rbd_pool = volumes + rbd_flatten_volume_from_snapshot = False + rbd_secret_uuid = _replaced_ + + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:rbd + + [glance_store] + default_backend = default_backend + + [default_backend] + rbd_store_ceph_conf = /etc/ceph/ceph.conf + store_description = "RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + + glanceAPIs: + default: + replicas: 3 + + manila: + enabled: true + manilaAPI: + customServiceConfig: | + [DEFAULT] + enabled_share_protocols = nfs + + manilaShares: + share1: + customServiceConfig: | + [DEFAULT] + enabled_share_backends = cephfsnfs + enabled_share_protocols = cephfs + + [cephfsnfs] + driver_handles_share_servers = False + share_backend_name = cephfs + share_driver = manila.share.drivers.cephfs.driver.CephFSDriver + cephfs_auth_id = openstack + cephfs_cluster_name = ceph + cephfs_nfs_cluster_id = cephfs + cephfs_protocol_helper_type = NFS + + extraMounts: + - name: v1 + region: r1 + extraVol: + - propagation: + - CinderVolume + - CinderBackup + - GlanceAPI + - ManilaShare + extraVolType: Ceph + volumes: + - name: ceph + projected: + sources: + - secret: + name: ceph-conf-files + mounts: + - name: ceph + mountPath: /etc/ceph + readOnly: true diff --git a/examples/dt/shiftonstack03/data-plane.md b/examples/dt/shiftonstack03/data-plane.md new file mode 100644 index 000000000..3ebe917c2 --- /dev/null +++ b/examples/dt/shiftonstack03/data-plane.md @@ -0,0 +1,49 @@ +# Configuring and deploying OpenStack compute nodes + +## Assumption + +- The [control-plane](control-plane.md) has been successfully deployed. +- The [edpm-ceph](ceph.md) has been successfully deployed. + +## Configuration + +Ensure the following files are modified according to the deployed environment + +- [values.yaml](compute/values.yaml) with the following information + - SSH keys for accessing the deployed compute nodes + - SSH keys to be used for Nova migration + - MAC addresses of the compute systems + +## Deployment + +```bash +# Switch to openstack namespace +$ oc project openstack + +# Change to shiftonstack directory +$ cd architecture/examples/dt/shiftonstack03 + +# Modify nodeset values accordingly +$ pushd compute +$ vi values.yaml + +# Generate the nodeset CR. +$ kustomize build > compute-nodeset.yaml + +# Apply the resource. +$ oc apply -f compute-nodeset.yaml + +# Wait for the nodeset to be ready state. +$ oc wait osdpns compute-nodes --for condition=SetupReady --timeout=300s +$ popd + +# Generate the deployment plan +$ popd +$ kustomize build > compute-deploy.yaml + +# Apply the deployment resource. +$ oc apply -f compute-deploy.yaml + +# Wait for the deployment to be completed. +$ oc wait osdpd compute-deploy --for condition=Ready --timeout=600s +``` diff --git a/examples/dt/shiftonstack03/kustomization.yaml b/examples/dt/shiftonstack03/kustomization.yaml new file mode 100644 index 000000000..1d3454072 --- /dev/null +++ b/examples/dt/shiftonstack03/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../dt/shiftonstack03 + +resources: + - values.yaml diff --git a/examples/dt/shiftonstack03/values.yaml b/examples/dt/shiftonstack03/values.yaml new file mode 100644 index 000000000..e44f7937e --- /dev/null +++ b/examples/dt/shiftonstack03/values.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: edpm-deployment-values + annotations: + config.kubernetes.io/local-config: "true" + +data: {}