From 854172966653138a60c329215fe9bb18726353e6 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson <39007539+assumptionsandg@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:08:31 +0100 Subject: [PATCH] Add support for Manila CSI using CephFS (#329) * add manila support * cleanup component labels * cleanup merge conflict * fix scope in storageclass manila * fix storageclass names * fix storageclass names * fix manila * fix manila secret * Refactor Manila CSI support --------- Co-authored-by: Elias Wimmer Co-authored-by: Matt Pryor --- .../cluster-addons/templates/csi-cephfs.yaml | 46 +++++ .../templates/openstack/csi-cinder.yaml | 32 +-- .../templates/openstack/csi-manila.yaml | 188 ++++++++++++++++++ charts/cluster-addons/values.yaml | 84 ++++++-- 4 files changed, 316 insertions(+), 34 deletions(-) create mode 100644 charts/cluster-addons/templates/csi-cephfs.yaml create mode 100644 charts/cluster-addons/templates/openstack/csi-manila.yaml diff --git a/charts/cluster-addons/templates/csi-cephfs.yaml b/charts/cluster-addons/templates/csi-cephfs.yaml new file mode 100644 index 00000000..f914e919 --- /dev/null +++ b/charts/cluster-addons/templates/csi-cephfs.yaml @@ -0,0 +1,46 @@ +{{- if .Values.csi.cephfs.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cluster-addons.componentName" (list . "csi-cephfs") }}-config + labels: + {{- include "cluster-addons.componentLabels" (list . "csi-cephfs") | nindent 4 }} + addons.stackhpc.com/watch: "" +stringData: + defaults: | + # Adjust the provisioner settings to allow it to deploy on a single node + provisioner: + replicaCount: 1 + # Allow the node plugin to run on the control plane nodes + nodeplugin: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + overrides: | + {{- toYaml .Values.csi.cephfs.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "cluster-addons.componentName" (list . "csi-cephfs") }} + labels: {{ include "cluster-addons.componentLabels" (list . "csi-cephfs") | nindent 4 }} + annotations: + # Tell Argo to ignore the non-controller owner references for this object + argocd.argoproj.io/sync-options: "ControllerReferencesOnly=true" +spec: + clusterName: {{ include "cluster-addons.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.csi.cephfs.chart | nindent 4 }} + targetNamespace: {{ .Values.csi.cephfs.release.namespace }} + releaseName: csi-cephfs + valuesSources: + - secret: + name: {{ include "cluster-addons.componentName" (list . "csi-cephfs") }}-config + key: defaults + - secret: + name: {{ include "cluster-addons.componentName" (list . "csi-cephfs") }}-config + key: overrides +{{- end }} diff --git a/charts/cluster-addons/templates/openstack/csi-cinder.yaml b/charts/cluster-addons/templates/openstack/csi-cinder.yaml index 8c40c272..b603c7c3 100644 --- a/charts/cluster-addons/templates/openstack/csi-cinder.yaml +++ b/charts/cluster-addons/templates/openstack/csi-cinder.yaml @@ -58,13 +58,17 @@ spec: - secret: name: {{ include "cluster-addons.componentName" (list . "csi-cinder") }}-config key: overrides -{{- if .Values.openstack.csiCinder.defaultStorageClass.enabled }} +{{- + if or + .Values.openstack.csiCinder.defaultStorageClass.enabled + .Values.openstack.csiCinder.additionalStorageClasses +}} --- apiVersion: addons.stackhpc.com/v1alpha1 kind: Manifests metadata: - name: {{ include "cluster-addons.componentName" (list . "csi-cinder-storageclass") }} - labels: {{ include "cluster-addons.componentLabels" (list . "csi-cinder-storageclass") | nindent 4 }} + name: {{ include "cluster-addons.componentName" (list . "csi-cinder") }}-storageclass + labels: {{ include "cluster-addons.componentLabels" (list . "csi-cinder") | nindent 4 }} annotations: # Tell Argo to ignore the non-controller owner references for this object argocd.argoproj.io/sync-options: "ControllerReferencesOnly=true" @@ -74,14 +78,17 @@ spec: targetNamespace: {{ .Values.openstack.targetNamespace }} releaseName: csi-cinder-storageclass manifestSources: +{{- if .Values.openstack.csiCinder.defaultStorageClass.enabled }} - template: | {{- with .Values.openstack.csiCinder.defaultStorageClass }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: {{ .name }} + {{- if .isClusterDefault }} annotations: storageclass.kubernetes.io/is-default-class: "true" + {{- end }} provisioner: cinder.csi.openstack.org parameters: availability: {{ .availabilityZone }} @@ -98,25 +105,26 @@ spec: allowedTopologies: {{ toYaml . | nindent 6 }} {{- end }} {{- end }} -{{- range $rangeItem := .Values.openstack.csiCinder.additionalStorageClasses }} +{{- end }} +{{- range .Values.openstack.csiCinder.additionalStorageClasses }} - template: | apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: {{ $rangeItem.name }} + name: {{ required "name is required for storage classes" .name }} provisioner: cinder.csi.openstack.org parameters: - availability: {{ $rangeItem.availabilityZone }} - {{- with $rangeItem.volumeType }} + availability: {{ default "nova" .availabilityZone }} + {{- with .volumeType }} type: {{ . }} {{- end }} - {{- with $rangeItem.fstype }} + {{- with .fstype }} fstype: {{ . }} {{- end }} - reclaimPolicy: {{ $rangeItem.reclaimPolicy }} - allowVolumeExpansion: {{ $rangeItem.allowVolumeExpansion }} - volumeBindingMode: {{ $rangeItem.volumeBindingMode }} - {{- with $rangeItem.allowedTopologies }} + reclaimPolicy: {{ default "Delete" .reclaimPolicy }} + allowVolumeExpansion: {{ dig "allowVolumeExpansion" true . | ternary "true" "false" }} + volumeBindingMode: {{ default "WaitForFirstConsumer" .volumeBindingMode }} + {{- with .allowedTopologies }} allowedTopologies: {{ toYaml . | nindent 6 }} {{- end }} {{- end }} diff --git a/charts/cluster-addons/templates/openstack/csi-manila.yaml b/charts/cluster-addons/templates/openstack/csi-manila.yaml new file mode 100644 index 00000000..cfb3aa8e --- /dev/null +++ b/charts/cluster-addons/templates/openstack/csi-manila.yaml @@ -0,0 +1,188 @@ +{{- define "cluster-addons.openstack.csiManila.storageClass" -}} +{{- $ctx := index . 0 -}} +{{- $sc := index . 1 -}} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ required "name is required for storage classes" $sc.name }} + {{- if $sc.isClusterDefault }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" + {{- end }} +{{- if $sc.provisioner }} +provisioner: {{ $sc.provisioner }} +{{- else if $ctx.Values.csi.cephfs.enabled }} +provisioner: cephfs.manila.csi.openstack.org +{{- else }} +{{- printf "Unable to determine provisioner for storage class - %s" $sc.name | fail }} +{{- end }} +parameters: + csi.storage.k8s.io/provisioner-secret-name: csi-manila-credentials + csi.storage.k8s.io/provisioner-secret-namespace: {{ $ctx.Values.openstack.targetNamespace }} + csi.storage.k8s.io/controller-expand-secret-name: csi-manila-credentials + csi.storage.k8s.io/controller-expand-secret-namespace: {{ $ctx.Values.openstack.targetNamespace }} + csi.storage.k8s.io/node-stage-secret-name: csi-manila-credentials + csi.storage.k8s.io/node-stage-secret-namespace: {{ $ctx.Values.openstack.targetNamespace }} + csi.storage.k8s.io/node-publish-secret-name: csi-manila-credentials + csi.storage.k8s.io/node-publish-secret-namespace: {{ $ctx.Values.openstack.targetNamespace }} + {{- $parameters := default dict $sc.parameters }} + {{- if $parameters.type }} + type: {{ $parameters.type }} + {{- else if $ctx.Values.csi.cephfs.enabled }} + type: cephfs + {{- else }} + {{- printf "Unable to determine share type for storage class - %s" $sc.name | fail }} + {{- end }} + {{- with (omit $parameters "type") }} + {{- toYaml . | nindent 2 }} + {{- end }} +reclaimPolicy: {{ default "Delete" $sc.reclaimPolicy }} +allowVolumeExpansion: {{ dig "allowVolumeExpansion" true $sc | ternary "true" "false" }} +volumeBindingMode: {{ default "WaitForFirstConsumer" $sc.volumeBindingMode }} +{{- with $sc.allowedTopologies }} +allowedTopologies: {{ toYaml . | nindent 2 }} +{{- end }} +{{- end }} + +{{- if and .Values.openstack.enabled .Values.openstack.csiManila.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cluster-addons.componentName" (list . "csi-manila") }}-config + labels: + {{- include "cluster-addons.componentLabels" (list . "csi-manila") | nindent 4 }} + addons.stackhpc.com/watch: "" +stringData: + defaults: | + csimanila: + clusterID: {{ include "cluster-addons.clusterName" . }} + {{- if .Values.csi.cephfs.enabled }} + shareProtocols: + - protocolSelector: CEPHFS + fsGroupPolicy: None + fwdNodePluginEndpoint: + dir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com + sockFile: csi.sock + {{- end }} + # Allow the node plugin to run on the control plane nodes + nodeplugin: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + overrides: | + {{- toYaml .Values.openstack.csiManila.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "cluster-addons.componentName" (list . "csi-manila") }} + labels: {{ include "cluster-addons.componentLabels" (list . "csi-manila") | nindent 4 }} + annotations: + # Tell Argo to ignore the non-controller owner references for this object + argocd.argoproj.io/sync-options: "ControllerReferencesOnly=true" +spec: + clusterName: {{ include "cluster-addons.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.openstack.csiManila.chart | nindent 4 }} + targetNamespace: {{ .Values.openstack.targetNamespace }} + releaseName: csi-manila + valuesSources: + - secret: + name: {{ include "cluster-addons.componentName" (list . "csi-manila") }}-config + key: defaults + - secret: + name: {{ include "cluster-addons.componentName" (list . "csi-manila") }}-config + key: overrides +{{- + if or + .Values.openstack.csiManila.defaultStorageClass.enabled + .Values.openstack.csiManila.additionalStorageClasses +}} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: Manifests +metadata: + name: {{ include "cluster-addons.componentName" (list . "csi-manila") }}-storageclass + labels: {{ include "cluster-addons.componentLabels" (list . "csi-manila") | nindent 4 }} + annotations: + # Tell Argo to ignore the non-controller owner references for this object + argocd.argoproj.io/sync-options: "ControllerReferencesOnly=true" +spec: + clusterName: {{ include "cluster-addons.clusterName" . }} + bootstrap: true + targetNamespace: {{ .Values.openstack.targetNamespace }} + releaseName: csi-manila-storageclass + manifestSources: + - template: | + {%- set identity_data = cloud_identity.data["clouds.yaml"] | b64decode | fromyaml -%} + {%- set cloud_data = identity_data.clouds.openstack -%} + {%- set tls_verify = cloud_data.verify | default(True) %} + apiVersion: v1 + kind: Secret + metadata: + name: csi-manila-credentials + stringData: + os-authURL: >- + {{ "{{" }} cloud_data.auth.auth_url {{ "}}" }} + os-region: >- + {{ "{{" }} cloud_data.region_name {{ "}}" }} + {%- if cloud_data.auth_type == "v3applicationcredential" %} + os-applicationCredentialID: >- + {{ "{{" }} cloud_data.auth.application_credential_id {{ "}}" }} + os-applicationCredentialSecret: >- + {{ "{{" }} cloud_data.auth.application_credential_secret {{ "}}" }} + {%- elif cloud_data.auth_type == "v3password" %} + os-password: >- + {{ "{{" }} cloud_data.auth.password {{ "}}" }} + {%- if "user_id" in cloud_data.auth %} + os-userID: >- + {{ "{{" }} cloud_data.auth.user_id {{ "}}" }} + {%- else %} + os-userName: >- + {{ "{{" }} cloud_data.auth.username {{ "}}" }} + {%- endif %} + {%- if "domain_id" in cloud_data.auth %} + os-domainID: >- + {{ "{{" }} cloud_data.auth.domain_id {{ "}}" }} + {%- else %} + os-domainName: >- + {{ "{{" }} cloud_data.auth.domain_name {{ "}}" }} + {%- endif %} + {%- if "project_id" in cloud_data.auth %} + os-projectID: >- + {{ "{{" }} cloud_data.auth.project_id {{ "}}" }} + {%- else %} + os-projectName: >- + {{ "{{" }} cloud_data.auth.project_name {{ "}}" }} + {%- endif %} + {%- if "project_domain_id" in cloud_data.auth %} + os-projectDomainID: >- + {{ "{{" }} cloud_data.auth.project_domain_id {{ "}}" }} + {%- elif "project_domain_name" in cloud_data.auth %} + os-projectDomainName: >- + {{ "{{" }} cloud_data.auth.project_domain_name {{ "}}" }} + {%- endif %} + {%- if "user_domain_id" in cloud_data.auth %} + os-userDomainID: >- + {{ "{{" }} cloud_data.auth.user_domain_id {{ "}}" }} + {%- elif "user_domain_name" in cloud_data.auth %} + os-userDomainName: >- + {{ "{{" }} cloud_data.auth.user_domain_name {{ "}}" }} + {%- endif %} + {%- endif %} + os-TLSInsecure: "{{ "{{" }} "false" if tls_verify else "true" {{ "}}" }}" + {{- if .Values.openstack.csiManila.defaultStorageClass.enabled }} + {{- with .Values.openstack.csiManila.defaultStorageClass }} + - template: | + {{- include "cluster-addons.openstack.csiManila.storageClass" (list $ .) | nindent 8 }} + {{- end }} + {{- end }} + {{- range .Values.openstack.csiManila.additionalStorageClasses }} + - template: | + {{- include "cluster-addons.openstack.csiManila.storageClass" (list $ .) | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/cluster-addons/values.yaml b/charts/cluster-addons/values.yaml index e34c98ad..0ece3248 100644 --- a/charts/cluster-addons/values.yaml +++ b/charts/cluster-addons/values.yaml @@ -45,6 +45,19 @@ cni: namespace: kube-system values: {} +# Settings for CSI addons +csi: + # Settings for the CephFS CSI + cephfs: + enabled: false + chart: + repo: https://ceph.github.io/csi-charts + name: ceph-csi-cephfs + version: 3.11.0 + release: + namespace: csi-ceph-system + values: {} + # Settings for the OpenStack integrations openstack: # Indicates if the OpenStack integrations should be enabled @@ -81,52 +94,79 @@ openstack: name: openstack-cinder-csi version: 2.30.0 values: {} - # Variables affecting the definition of the storage class + # Definition of the default storage class for Cinder CSI defaultStorageClass: # Indicates if the storage class should be enabled enabled: true # The name of the storage class name: csi-cinder + # Indicates if the Cinder default storage class is the cluster default storage class + isClusterDefault: true # The reclaim policy for the storage class reclaimPolicy: Delete # Indicates if volume expansion is allowed allowVolumeExpansion: true + # Controls when volume binding and dynamic provisioning should occur + volumeBindingMode: WaitForFirstConsumer + # The allowed topologies for the storage class + allowedTopologies: + # Filesystem type to use for volumes provisioned with the storage class + # If not given, the default filesystem type will be used + fstype: # The Cinder availability zone to use for volumes provisioned by the storage class availabilityZone: nova # The Cinder volume type to use for volumes provisioned by the storage class # If not given, the default volume type will be used volumeType: - # The volumeBindingMode field controls when volume binding and dynamic provisioning should occur. + # Additional storage classes to create for the Cinder CSI + # For each item, the properties from the default storage class are supported (except enabled and isClusterDefault) + additionalStorageClasses: [] + # Settings for the Manila CSI plugin + csiManila: + # Indicates if the Manila CSI should be enabled + # By default, it is disabled as Manila is not commonly available + # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/charts/manila-csi-plugin/values.yaml + enabled: false + chart: + repo: https://kubernetes.github.io/cloud-provider-openstack + name: openstack-manila-csi + version: 2.30.0 + values: {} + # Definition of the default storage class for the Manila CSI + defaultStorageClass: + # Indicates if the storage class should be enabled + enabled: true + # The name of the storage class + name: csi-manila + # Indicates if the Manila default storage class is the cluster default storage class + isClusterDefault: false + # The provisioner for the storage class + # If not given and the Ceph CSI plugin is installed, cephfs.manila.csi.openstack.org is used + provisioner: + # The reclaim policy for the storage class + reclaimPolicy: Delete + # Indicates if volume expansion is allowed + allowVolumeExpansion: true + # Controls when volume binding and dynamic provisioning should occur volumeBindingMode: WaitForFirstConsumer # The allowed topologies for the storage class allowedTopologies: - # Filesystem type supported by Kubernetes. Defaults to "ext4" - # fstype: + # The parameters for the storage class + # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/manila-csi-plugin/using-manila-csi-plugin.md#controller-service-volume-parameters + parameters: + # The Manila share type to use + # If not given and the Ceph CSI plugin is installed, cephfs is used + type: + # Additional storage classes to create for the Manila CSI + # For each item, the properties from the default storage class are supported (except for "enabled") additionalStorageClasses: [] - # - name: additional-storage-classname - # The reclaim policy for the storage class - # reclaimPolicy: Delete - # The reclaim policy for the storage class - # allowVolumeExpansion: true - # The Cinder availability zone to use for volumes provisioned by the storage class - # availabilityZone: nova - # The Cinder volume type to use for volumes provisioned by the storage class - # If not given, the default volume type will be used - # volumeType: - # The volumeBindingMode field controls when volume binding and dynamic provisioning should occur. - # Required for additionalStorageClasses. - # volumeBindingMode: WaitForFirstConsumer - # The allowed topologies for the storage class - # allowedTopologies: - # Filesystem type supported by Kubernetes. Defaults to "ext4" - # fstype: k8sKeystoneAuth: enabled: false targetNamespace: kube-system chart: repo: https://catalyst-cloud.github.io/capi-plugin-helm-charts name: k8s-keystone-auth - version: 1.3.0 + version: 1.1.0 # Settings for etcd defragmentation jobs etcdDefrag: