Enable caching using memcached

Jira: OSP-26501

api/bases/cinder.openstack.org_cinderapis.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderapis.cinder.openstack.org
 spec:

api/bases/cinder.openstack.org_cinderbackups.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderbackups.cinder.openstack.org
 spec:

api/bases/cinder.openstack.org_cinders.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinders.cinder.openstack.org
 spec:
@@ -1131,6 +1131,9 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              memcachedInstance:
+                default: memcached
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string
@@ -1162,6 +1165,7 @@ spec:
             - cinderAPI
             - cinderScheduler
             - databaseInstance
+            - memcachedInstance
             - rabbitMqClusterName
             - secret
             type: object

api/bases/cinder.openstack.org_cinderschedulers.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderschedulers.cinder.openstack.org
 spec:

api/bases/cinder.openstack.org_cindervolumes.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cindervolumes.cinder.openstack.org
 spec:

api/v1beta1/cinder_types.go

@@ -58,6 +58,11 @@ type CinderSpec struct {
 	// Needed to request a transportURL that is created and used in Cinder
 	RabbitMqClusterName string `json:"rabbitMqClusterName"`
 
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default=memcached
+	// Memcached instance name.
+	MemcachedInstance string `json:"memcachedInstance"`
+
 	// +kubebuilder:validation:Optional
 	// Debug - enable debug for different deploy stages. If an init container is used, it runs and the
 	// actual action pod gets started with sleep infinity

config/crd/bases/cinder.openstack.org_cinderapis.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderapis.cinder.openstack.org
 spec:

config/crd/bases/cinder.openstack.org_cinderbackups.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderbackups.cinder.openstack.org
 spec:

config/crd/bases/cinder.openstack.org_cinders.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinders.cinder.openstack.org
 spec:
@@ -1131,6 +1131,9 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              memcachedInstance:
+                default: memcached
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string
@@ -1162,6 +1165,7 @@ spec:
             - cinderAPI
             - cinderScheduler
             - databaseInstance
+            - memcachedInstance
             - rabbitMqClusterName
             - secret
             type: object

config/crd/bases/cinder.openstack.org_cinderschedulers.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderschedulers.cinder.openstack.org
 spec:

config/crd/bases/cinder.openstack.org_cindervolumes.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cindervolumes.cinder.openstack.org
 spec:

config/rbac/role.yaml

@@ -276,6 +276,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - memcached.openstack.org
+  resources:
+  - memcacheds
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - rabbitmq.openstack.org
   resources:

controllers/cinder_controller.go

@@ -19,10 +19,12 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -35,6 +37,7 @@ import (
 	"github.com/go-logr/logr"
 	cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/cinder-operator/pkg/cinder"
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
@@ -102,6 +105,7 @@ type CinderReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch
 // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch;
 // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch
 // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
@@ -172,6 +176,7 @@ func (r *CinderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 			condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage),
 			condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage),
 			condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage),
+			condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage),
 			condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage),
 			condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage),
 			condition.UnknownCondition(cinderv1beta1.CinderAPIReadyCondition, condition.InitReason, cinderv1beta1.CinderAPIReadyInitMessage),
@@ -257,6 +262,35 @@ func (r *CinderReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return nil
 	}
 
+	memcachedFn := func(o client.Object) []reconcile.Request {
+		result := []reconcile.Request{}
+
+		// get all Cinder CRs
+		cinders := &cinderv1beta1.CinderList{}
+		listOpts := []client.ListOption{
+			client.InNamespace(o.GetNamespace()),
+		}
+		if err := r.Client.List(context.Background(), cinders, listOpts...); err != nil {
+			r.Log.Error(err, "Unable to retrieve Cinder CRs %w")
+			return nil
+		}
+
+		for _, cr := range cinders.Items {
+			if o.GetName() == cr.Spec.MemcachedInstance {
+				name := client.ObjectKey{
+					Namespace: o.GetNamespace(),
+					Name:      cr.Name,
+				}
+				r.Log.Info(fmt.Sprintf("Memcached %s is used by Cinder CR %s", o.GetName(), cr.Name))
+				result = append(result, reconcile.Request{NamespacedName: name})
+			}
+		}
+		if len(result) > 0 {
+			return result
+		}
+		return nil
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&cinderv1beta1.Cinder{}).
 		Owns(&mariadbv1.MariaDBDatabase{}).
@@ -273,6 +307,8 @@ func (r *CinderReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		// Watch for TransportURL Secrets which belong to any TransportURLs created by Cinder CRs
 		Watches(&source.Kind{Type: &corev1.Secret{}},
 			handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)).
+		Watches(&source.Kind{Type: &memcachedv1.Memcached{}},
+			handler.EnqueueRequestsFromMapFunc(memcachedFn)).
 		Complete(r)
 }
 
@@ -482,6 +518,41 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder
 
 	// end transportURL
 
+	//
+	// Check for required memcached used for caching
+	//
+	memcached, err := r.getCinderMemcached(ctx, helper, instance)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				condition.MemcachedReadyCondition,
+				condition.RequestedReason,
+				condition.SeverityInfo,
+				condition.MemcachedReadyWaitingMessage))
+			return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s not found", instance.Spec.MemcachedInstance)
+		}
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.MemcachedReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.MemcachedReadyErrorMessage,
+			err.Error()))
+		return ctrl.Result{}, err
+	}
+
+	if !memcached.IsReady() {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.MemcachedReadyCondition,
+			condition.RequestedReason,
+			condition.SeverityInfo,
+			condition.MemcachedReadyWaitingMessage))
+		return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s is not ready", memcached.Name)
+	}
+	// Mark the Memcached Service as Ready if we get to this point with no errors
+	instance.Status.Conditions.MarkTrue(
+		condition.MemcachedReadyCondition, condition.MemcachedReadyMessage)
+	// run check memcached - end
+
 	//
 	// check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map
 	//
@@ -512,7 +583,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder
 	//
 	// Create Secrets required as input for the Service and calculate an overall hash of hashes
 	//
-	err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels)
+	err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -780,6 +851,7 @@ func (r *CinderReconciler) generateServiceConfigs(
 	instance *cinderv1beta1.Cinder,
 	envVars *map[string]env.Setter,
 	serviceLabels map[string]string,
+	mc *memcachedv1.Memcached,
 ) error {
 	//
 	// create Secret required for cinder input
@@ -830,6 +902,8 @@ func (r *CinderReconciler) generateServiceConfigs(
 		string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
 		instance.Status.DatabaseHostname,
 		cinder.DatabaseName)
+	templateParameters["MemcachedServers"] = strings.Join(mc.Status.ServerList, ",")
+	templateParameters["MemcachedServersWithInet"] = strings.Join(mc.Status.ServerListWithInet, ",")
 
 	configTemplates := []util.Template{
 		{
@@ -899,6 +973,26 @@ func (r *CinderReconciler) transportURLCreateOrUpdate(
 	return transportURL, op, err
 }
 
+// getCinderMemcached - gets the Memcached instance used for Cinder cache backend
+func (r *CinderReconciler) getCinderMemcached(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *cinderv1beta1.Cinder,
+) (*memcachedv1.Memcached, error) {
+	memcached := &memcachedv1.Memcached{}
+	err := h.GetClient().Get(
+		ctx,
+		types.NamespacedName{
+			Name:      instance.Spec.MemcachedInstance,
+			Namespace: instance.Namespace,
+		},
+		memcached)
+	if err != nil {
+		return nil, err
+	}
+	return memcached, err
+}
+
 func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder) (*cinderv1beta1.CinderAPI, controllerutil.OperationResult, error) {
 
 	cinderAPISpec := cinderv1beta1.CinderAPISpec{

main.go

@@ -35,6 +35,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	keystonev1beta1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
 	mariadbv1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
@@ -53,6 +54,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
 	utilruntime.Must(cinderv1beta1.AddToScheme(scheme))
+	utilruntime.Must(memcachedv1.AddToScheme(scheme))
 	utilruntime.Must(mariadbv1beta1.AddToScheme(scheme))
 	utilruntime.Must(keystonev1beta1.AddToScheme(scheme))
 	utilruntime.Must(rabbitmqv1.AddToScheme(scheme))

templates/cinder/config/00-global-defaults.conf

@@ -30,6 +30,11 @@ osapi_volume_workers = 4
 control_exchange = openstack
 api_paste_config = /etc/cinder/api-paste.ini
 
+[cache]
+backend = dogpile.cache.pymemcache
+enabled = true
+memcache_servers = {{ .MemcachedServers }}
+
 [database]
 connection = {{ .DatabaseConnection }}
 max_retries = -1
@@ -58,8 +63,7 @@ file_event_handler=/etc/cinder
 [keystone_authtoken]
 www_authenticate_uri={{ .KeystonePublicURL }}
 auth_url = {{ .KeystoneInternalURL }}
-# TODO (mschuppert): Add memcached
-#memcached_servers = controller:11211
+memcached_servers = {{ .MemcachedServersWithInet }}
 auth_type = password
 project_domain_name = Default
 user_domain_name = Default