diff --git a/api/bases/cinder.openstack.org_cinderapis.yaml b/api/bases/cinder.openstack.org_cinderapis.yaml index 89e268ad4..bc6679531 100644 --- a/api/bases/cinder.openstack.org_cinderapis.yaml +++ b/api/bases/cinder.openstack.org_cinderapis.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderapis.cinder.openstack.org spec: diff --git a/api/bases/cinder.openstack.org_cinderbackups.yaml b/api/bases/cinder.openstack.org_cinderbackups.yaml index 724b61223..a71003ec1 100644 --- a/api/bases/cinder.openstack.org_cinderbackups.yaml +++ b/api/bases/cinder.openstack.org_cinderbackups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderbackups.cinder.openstack.org spec: diff --git a/api/bases/cinder.openstack.org_cinders.yaml b/api/bases/cinder.openstack.org_cinders.yaml index 1518ceb9b..0572098fb 100644 --- a/api/bases/cinder.openstack.org_cinders.yaml +++ b/api/bases/cinder.openstack.org_cinders.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinders.cinder.openstack.org spec: @@ -1131,6 +1131,9 @@ spec: - extraVol type: object type: array + memcachedInstance: + default: memcached + type: string nodeSelector: additionalProperties: type: string @@ -1162,6 +1165,7 @@ spec: - cinderAPI - cinderScheduler - databaseInstance + - memcachedInstance - rabbitMqClusterName - secret type: object diff --git a/api/bases/cinder.openstack.org_cinderschedulers.yaml b/api/bases/cinder.openstack.org_cinderschedulers.yaml index b702b997a..c9f221a19 100644 --- a/api/bases/cinder.openstack.org_cinderschedulers.yaml +++ b/api/bases/cinder.openstack.org_cinderschedulers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderschedulers.cinder.openstack.org spec: diff --git a/api/bases/cinder.openstack.org_cindervolumes.yaml b/api/bases/cinder.openstack.org_cindervolumes.yaml index 7e6ca6b71..a099d3d7d 100644 --- a/api/bases/cinder.openstack.org_cindervolumes.yaml +++ b/api/bases/cinder.openstack.org_cindervolumes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cindervolumes.cinder.openstack.org spec: diff --git a/api/v1beta1/cinder_types.go b/api/v1beta1/cinder_types.go index 6e05d47dc..c6bebd759 100644 --- a/api/v1beta1/cinder_types.go +++ b/api/v1beta1/cinder_types.go @@ -58,6 +58,11 @@ type CinderSpec struct { // Needed to request a transportURL that is created and used in Cinder RabbitMqClusterName string `json:"rabbitMqClusterName"` + // +kubebuilder:validation:Required + // +kubebuilder:default=memcached + // Memcached instance name. + MemcachedInstance string `json:"memcachedInstance"` + // +kubebuilder:validation:Optional // Debug - enable debug for different deploy stages. If an init container is used, it runs and the // actual action pod gets started with sleep infinity diff --git a/config/crd/bases/cinder.openstack.org_cinderapis.yaml b/config/crd/bases/cinder.openstack.org_cinderapis.yaml index 89e268ad4..bc6679531 100644 --- a/config/crd/bases/cinder.openstack.org_cinderapis.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderapis.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderapis.cinder.openstack.org spec: diff --git a/config/crd/bases/cinder.openstack.org_cinderbackups.yaml b/config/crd/bases/cinder.openstack.org_cinderbackups.yaml index 724b61223..a71003ec1 100644 --- a/config/crd/bases/cinder.openstack.org_cinderbackups.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderbackups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderbackups.cinder.openstack.org spec: diff --git a/config/crd/bases/cinder.openstack.org_cinders.yaml b/config/crd/bases/cinder.openstack.org_cinders.yaml index 1518ceb9b..0572098fb 100644 --- a/config/crd/bases/cinder.openstack.org_cinders.yaml +++ b/config/crd/bases/cinder.openstack.org_cinders.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinders.cinder.openstack.org spec: @@ -1131,6 +1131,9 @@ spec: - extraVol type: object type: array + memcachedInstance: + default: memcached + type: string nodeSelector: additionalProperties: type: string @@ -1162,6 +1165,7 @@ spec: - cinderAPI - cinderScheduler - databaseInstance + - memcachedInstance - rabbitMqClusterName - secret type: object diff --git a/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml b/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml index b702b997a..c9f221a19 100644 --- a/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cinderschedulers.cinder.openstack.org spec: diff --git a/config/crd/bases/cinder.openstack.org_cindervolumes.yaml b/config/crd/bases/cinder.openstack.org_cindervolumes.yaml index 7e6ca6b71..a099d3d7d 100644 --- a/config/crd/bases/cinder.openstack.org_cindervolumes.yaml +++ b/config/crd/bases/cinder.openstack.org_cindervolumes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 + controller-gen.kubebuilder.io/version: v0.10.0 creationTimestamp: null name: cindervolumes.cinder.openstack.org spec: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 84b24ee85..945dcb619 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -276,6 +276,14 @@ rules: - patch - update - watch +- apiGroups: + - memcached.openstack.org + resources: + - memcacheds + verbs: + - get + - list + - watch - apiGroups: - rabbitmq.openstack.org resources: diff --git a/controllers/cinder_controller.go b/controllers/cinder_controller.go index 50130afbd..0ceac47e8 100644 --- a/controllers/cinder_controller.go +++ b/controllers/cinder_controller.go @@ -19,10 +19,12 @@ package controllers import ( "context" "fmt" + "strings" "time" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,6 +37,7 @@ import ( "github.com/go-logr/logr" cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" "github.com/openstack-k8s-operators/cinder-operator/pkg/cinder" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" @@ -102,6 +105,7 @@ type CinderReconciler struct { // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch @@ -172,6 +176,7 @@ func (r *CinderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(cinderv1beta1.CinderAPIReadyCondition, condition.InitReason, cinderv1beta1.CinderAPIReadyInitMessage), @@ -257,6 +262,35 @@ func (r *CinderReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } + memcachedFn := func(o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // get all Cinder CRs + cinders := &cinderv1beta1.CinderList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(context.Background(), cinders, listOpts...); err != nil { + r.Log.Error(err, "Unable to retrieve Cinder CRs %w") + return nil + } + + for _, cr := range cinders.Items { + if o.GetName() == cr.Spec.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + r.Log.Info(fmt.Sprintf("Memcached %s is used by Cinder CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + return ctrl.NewControllerManagedBy(mgr). For(&cinderv1beta1.Cinder{}). Owns(&mariadbv1.MariaDBDatabase{}). @@ -273,6 +307,8 @@ func (r *CinderReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for TransportURL Secrets which belong to any TransportURLs created by Cinder CRs Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&source.Kind{Type: &memcachedv1.Memcached{}}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). Complete(r) } @@ -482,6 +518,41 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // end transportURL + // + // Check for required memcached used for caching + // + memcached, err := r.getCinderMemcached(ctx, helper, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s not found", instance.Spec.MemcachedInstance) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if !memcached.IsReady() { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s is not ready", memcached.Name) + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // @@ -512,7 +583,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // // Create Secrets required as input for the Service and calculate an overall hash of hashes // - err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -780,6 +851,7 @@ func (r *CinderReconciler) generateServiceConfigs( instance *cinderv1beta1.Cinder, envVars *map[string]env.Setter, serviceLabels map[string]string, + mc *memcachedv1.Memcached, ) error { // // create Secret required for cinder input @@ -830,6 +902,8 @@ func (r *CinderReconciler) generateServiceConfigs( string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]), instance.Status.DatabaseHostname, cinder.DatabaseName) + templateParameters["MemcachedServers"] = strings.Join(mc.Status.ServerList, ",") + templateParameters["MemcachedServersWithInet"] = strings.Join(mc.Status.ServerListWithInet, ",") configTemplates := []util.Template{ { @@ -899,6 +973,26 @@ func (r *CinderReconciler) transportURLCreateOrUpdate( return transportURL, op, err } +// getCinderMemcached - gets the Memcached instance used for Cinder cache backend +func (r *CinderReconciler) getCinderMemcached( + ctx context.Context, + h *helper.Helper, + instance *cinderv1beta1.Cinder, +) (*memcachedv1.Memcached, error) { + memcached := &memcachedv1.Memcached{} + err := h.GetClient().Get( + ctx, + types.NamespacedName{ + Name: instance.Spec.MemcachedInstance, + Namespace: instance.Namespace, + }, + memcached) + if err != nil { + return nil, err + } + return memcached, err +} + func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder) (*cinderv1beta1.CinderAPI, controllerutil.OperationResult, error) { cinderAPISpec := cinderv1beta1.CinderAPISpec{ diff --git a/main.go b/main.go index 94a6a42da..a05bcc1eb 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1beta1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -53,6 +54,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(cinderv1beta1.AddToScheme(scheme)) + utilruntime.Must(memcachedv1.AddToScheme(scheme)) utilruntime.Must(mariadbv1beta1.AddToScheme(scheme)) utilruntime.Must(keystonev1beta1.AddToScheme(scheme)) utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) diff --git a/templates/cinder/config/00-global-defaults.conf b/templates/cinder/config/00-global-defaults.conf index ff422280f..f5382373e 100644 --- a/templates/cinder/config/00-global-defaults.conf +++ b/templates/cinder/config/00-global-defaults.conf @@ -30,6 +30,11 @@ osapi_volume_workers = 4 control_exchange = openstack api_paste_config = /etc/cinder/api-paste.ini +[cache] +backend = dogpile.cache.pymemcache +enabled = true +memcache_servers = {{ .MemcachedServers }} + [database] connection = {{ .DatabaseConnection }} max_retries = -1 @@ -58,8 +63,7 @@ file_event_handler=/etc/cinder [keystone_authtoken] www_authenticate_uri={{ .KeystonePublicURL }} auth_url = {{ .KeystoneInternalURL }} -# TODO (mschuppert): Add memcached -#memcached_servers = controller:11211 +memcached_servers = {{ .MemcachedServersWithInet }} auth_type = password project_domain_name = Default user_domain_name = Default