feat: Limit caching of ConfigMaps and Secrets

feat: Toggle caching of ConfigMaps and Secrets with CommonLabels

deploy/helm/grafana-operator/README.md

@@ -104,5 +104,6 @@ It's easier to just manage this configuration outside of the operator.
 | serviceMonitor.targetLabels | list | `[]` | Set of labels to transfer from the Kubernetes Service onto the target |
 | serviceMonitor.telemetryPath | string | `"/metrics"` | Set path to metrics path |
 | tolerations | list | `[]` | pod tolerations |
+| watchLabeledReferencesOnly | bool | `false` | Sets the `WATCH_LABELED_REFERENCES_ONLY` environment variable, it enables the caching of ConfigMaps and Secrets labeled with `"app.kubernetes.io/managed-by": "grafana-operator"` to reduce requests to the api. By default, ConfigMaps and Secrets are not cached to reduce the memory usage of the operator in large clusters. WARNING This will hide unlabeled ConfigMaps and Secrets from the Operator |
 | watchNamespaceSelector | string | `""` | Sets the `WATCH_NAMESPACE_SELECTOR` environment variable, it defines which namespaces the operator should be listening for based on a namespace label (e.g. `"environment: dev"`). By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead. |
 | watchNamespaces | string | `""` | Sets the `WATCH_NAMESPACE` environment variable, it defines which namespaces the operator should be listening for (e.g. `"grafana, foo"`). By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead. |

deploy/helm/grafana-operator/templates/deployment.yaml

@@ -51,6 +51,12 @@ spec:
               {{ else }}
               value: {{quote .Values.watchNamespaceSelector }}
               {{- end }}
+            - name: WATCH_LABELED_REFERENCES_ONLY
+              {{- if .Values.watchLabeledReferencesOnly }}
+              value: "true"
+              {{ else }}
+              value: ""
+              {{- end }}
             {{- with .Values.env }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

deploy/helm/grafana-operator/values.yaml

@@ -15,6 +15,12 @@ watchNamespaces: ""
 # By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead.
 watchNamespaceSelector: ""
 
+# -- Sets the `WATCH_LABELED_REFERENCES_ONLY` environment variable,
+# it enables the caching of ConfigMaps and Secrets labeled with `"app.kubernetes.io/managed-by": "grafana-operator"` to reduce requests to the api.
+# By default, ConfigMaps and Secrets are not cached to reduce the memory usage of the operator in large clusters.
+# WARNING This will hide unlabeled ConfigMaps and Secrets from the Operator
+watchLabeledReferencesOnly: false
+
 # -- Determines if the target cluster is OpenShift. Additional rbac permissions for routes will be added on OpenShift
 isOpenShift: false
 

main.go

@@ -67,6 +67,10 @@ const (
 	// eg: "environment: dev"
 	// If empty or undefined, the operator will run in cluster scope.
 	watchNamespaceEnvSelector = "WATCH_NAMESPACE_SELECTOR"
+	// Enable caching of ConfigMaps and Secrets to reduce API read requests
+	// If empty or undefined, the operator will disable caching
+	// This will hide all referenced ConfigMaps and Secrets not labeled with: app.kubernetes.io/managed-by: grafana-operator
+	watchLabeledReferencesOnlyEnvVar = "WATCH_LABELED_REFERENCES_ONLY"
 )
 
 var (
@@ -105,8 +109,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Detect environment variables
 	watchNamespace, _ := os.LookupEnv(watchNamespaceEnvVar)
 	watchNamespaceSelector, _ := os.LookupEnv(watchNamespaceEnvSelector)
+	_, watchLabeledReferencesOnly := os.LookupEnv(watchLabeledReferencesOnlyEnvVar)
 
 	// Fetch k8s api credentials and detect platform
 	restConfig := ctrl.GetConfigOrDie()
@@ -137,12 +143,25 @@ func main() {
 			&corev1.ServiceAccount{}:        cacheLabels,
 			&networkingv1.Ingress{}:         cacheLabels,
 			&corev1.PersistentVolumeClaim{}: cacheLabels,
+			&corev1.ConfigMap{}:             cacheLabels, // Matching just labeled ConfigMaps and Secrets greatly reduces cache size
+			&corev1.Secret{}:                cacheLabels, // Omitting labels or supporting custom labels would require changes in Grafana Reconciler
 		}},
 	}
 	if isOpenShift {
 		controllerOptions.Cache.ByObject[&routev1.Route{}] = cacheLabels
 	}
 
+	// Disable ConfigMap and Secret cache lookups per default
+	// all reads will hit the api
+	if !watchLabeledReferencesOnly {
+		controllerOptions.Client = client.Options{
+			Cache: &client.CacheOptions{
+				DisableFor: []client.Object{&corev1.ConfigMap{}, &corev1.Secret{}},
+			},
+		}
+	}
+
+	// Determine Operator scope
 	switch {
 	case strings.Contains(watchNamespace, ","):
 		// multi namespace scoped
@@ -171,6 +190,7 @@ func main() {
 		os.Exit(1) //nolint
 	}
 
+	// Register controllers
 	if err = (&controllers.GrafanaReconciler{
 		Client:      mgr.GetClient(),
 		Scheme:      mgr.GetScheme(),