Fix podwebhook would be revert when helm upgrade

Using helm templates to rendering podwebhook config rather than controller genarated, which can advoid the webhook cert would be revert when helm upgrade, it can leads to pod fail to start.

Signed-off-by: Cyclinder Kuo <[email protected]>

charts/spiderpool/templates/tls.yaml

@@ -144,6 +144,48 @@ webhooks:
     - spidercoordinators
   sideEffects: None
 {{- end }}
+{{- if .Values.spiderpoolController.podResourceInject.enabled }}
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: {{ .Values.spiderpoolController.name | trunc 63 | trimSuffix "-" }}
+      namespace: {{ .Release.Namespace }}
+      path: /mutate--v1-pod
+      port: {{ .Values.spiderpoolController.webhookPort }}
+    {{- if (eq .Values.spiderpoolController.tls.method "provided") }}
+    caBundle: {{ .Values.spiderpoolController.tls.provided.tlsCa | required "missing spiderpoolController.tls.provided.tlsCa" }}
+    {{- else if (eq .Values.spiderpoolController.tls.method "auto") }}
+    caBundle: {{ .ca.Cert | b64enc }}
+    {{- end }}
+  failurePolicy: Fail
+  name: pods.spiderpool.spidernet.io
+  {{- if or .Values.spiderpoolController.podResourceInject.namespacesExclude .Values.spiderpoolController.podResourceInject.namespacesInclude }}
+  namespaceSelector:
+    matchExpressions:
+    {{- if .Values.spiderpoolController.podResourceInject.namespacesExclude }}
+    - key: kubernetes.io/metadata.name
+      operator: NotIn
+      values: {{ toYaml .Values.spiderpoolController.podResourceInject.namespacesExclude | nindent 8 }}
+    {{- end }}
+    {{- if .Values.spiderpoolController.podResourceInject.namespacesInclude }}
+    - key: kubernetes.io/metadata.name
+      operator: In
+      values: {{ toYaml .Values.spiderpoolController.podResourceInject.namespacesInclude | nindent 8 }}
+    {{- end }}
+  {{- end }}
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None
+{{- end }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

cmd/spiderpool-controller/cmd/daemon.go

@@ -266,19 +266,12 @@ func initControllerServiceManagers(ctx context.Context) {
 	controllerContext.PodManager = podManager
 
 	if controllerContext.Cfg.PodResourceInjectConfig.Enabled {
-		logger.Debug("Begin to init Pod MutatingWebhook")
-		if err := podmanager.InitPodWebhook(controllerContext.ClientSet.AdmissionregistrationV1(),
-			controllerContext.CRDManager, controllerContext.Cfg.ControllerDeploymentName,
-			controllerContext.Cfg.PodResourceInjectConfig.NamespacesExclude,
-			controllerContext.Cfg.PodResourceInjectConfig.NamespacesInclude); err != nil {
+		logger.Info("Begin to init Pod MutatingWebhook")
+		if err := podmanager.InitPodWebhook(controllerContext.CRDManager); err != nil {
 			logger.Fatal(err.Error())
 		}
 	} else {
-		logger.Debug("InjectPodNetworkResource is disabled, try to remove the pod part in the MutatingWebhook")
-		if err := podmanager.RemovePodMutatingWebhook(controllerContext.ClientSet.AdmissionregistrationV1(),
-			controllerContext.Cfg.ControllerDeploymentName); err != nil {
-			logger.Error(err.Error())
-		}
+		logger.Info("Pod MutatingWebhook is disabled")
 	}
 
 	logger.Info("Begin to initialize StatefulSet manager")

pkg/podmanager/pod_webhook.go

@@ -12,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	admissionregistrationv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
@@ -38,17 +37,10 @@ type podWebhook struct {
 // InitPodWebhook initializes the pod webhook.
 // It sets up the mutating webhook for pods and registers it with the manager.
 // Parameters:
-//   - client: The Kubernetes client
 //   - mgr: The controller manager
-//   - mutatingWebhookName: The name of the mutating webhook
 //
 // Returns an error if initialization fails.
-func InitPodWebhook(
-	admissionClient admissionregistrationv1.AdmissionregistrationV1Interface,
-	mgr ctrl.Manager,
-	mutatingWebhookName string,
-	webhookNamespaceExclude []string,
-	webhookNamespaceInclude []string) error {
+func InitPodWebhook(mgr ctrl.Manager) error {
 	spiderClient, err := crdclientset.NewForConfig(ctrl.GetConfigOrDie())
 	if err != nil {
 		return err
@@ -58,14 +50,6 @@ func InitPodWebhook(
 		spiderClient: spiderClient,
 	}
 
-	if len(webhookNamespaceExclude) != 0 {
-		PodWebhookExcludeNamespaces = webhookNamespaceExclude
-	}
-
-	if err = AddPodMutatingWebhook(admissionClient, mutatingWebhookName, webhookNamespaceInclude); err != nil {
-		return err
-	}
-
 	// setup mutating webhook for pods
 	if err = ctrl.NewWebhookManagedBy(mgr).
 		For(&corev1.Pod{}).

pkg/podmanager/utils.go

@@ -8,14 +8,10 @@ import (
 	"fmt"
 
 	crdclientset "github.com/spidernet-io/spiderpool/pkg/k8s/client/clientset/versioned"
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8s_resource "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	admissionClientv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
-	"k8s.io/client-go/util/retry"
-	"k8s.io/utils/ptr"
 	kubevirtv1 "kubevirt.io/api/core/v1"
 
 	"github.com/spidernet-io/spiderpool/pkg/constant"
@@ -191,141 +187,6 @@ func InjectRdmaResourceToPod(resourceMap map[string]bool, pod *corev1.Pod) {
 	}
 }
 
-// InitPodMutatingWebhook initializes a mutating webhook for pods based on a template webhook.
-// It sets up the webhook configuration including name, admission review versions, failure policy,
-// object selector, client config, and rules for pod creation and update operations.
-//
-// Parameters:
-//   - from: An admissionregistrationv1.MutatingWebhook object to use as a template
-//
-// Returns:
-//   - A new admissionregistrationv1.MutatingWebhook object configured for pod mutation
-func InitPodMutatingWebhook(from admissionregistrationv1.MutatingWebhook, webhookNamespaceInclude []string) admissionregistrationv1.MutatingWebhook {
-	wb := admissionregistrationv1.MutatingWebhook{
-		Name:                    constant.PodMutatingWebhookName,
-		AdmissionReviewVersions: from.AdmissionReviewVersions,
-		FailurePolicy:           ptr.To(admissionregistrationv1.Fail),
-		NamespaceSelector:       &metav1.LabelSelector{},
-		ClientConfig: admissionregistrationv1.WebhookClientConfig{
-			CABundle: from.ClientConfig.CABundle,
-		},
-		Rules: []admissionregistrationv1.RuleWithOperations{
-			{
-				Operations: []admissionregistrationv1.OperationType{
-					admissionregistrationv1.Create,
-					admissionregistrationv1.Update,
-				},
-				Rule: admissionregistrationv1.Rule{
-					APIGroups:   []string{""},
-					APIVersions: []string{"v1"},
-					Resources:   []string{"pods"},
-				},
-			},
-		},
-		SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone),
-	}
-
-	if from.ClientConfig.Service != nil {
-		wb.ClientConfig.Service = &admissionregistrationv1.ServiceReference{
-			Name:      from.ClientConfig.Service.Name,
-			Namespace: from.ClientConfig.Service.Namespace,
-			Port:      from.ClientConfig.Service.Port,
-			// format: /mutate-<group>-<apiVersion>-<resource>
-			Path: ptr.To("/mutate--v1-pod"),
-		}
-	}
-
-	if len(PodWebhookExcludeNamespaces) != 0 {
-		wb.NamespaceSelector.MatchExpressions = []metav1.LabelSelectorRequirement{
-			{
-				Key:      corev1.LabelMetadataName,
-				Operator: metav1.LabelSelectorOpNotIn,
-				Values:   PodWebhookExcludeNamespaces,
-			},
-		}
-	}
-
-	if len(webhookNamespaceInclude) != 0 {
-		wb.NamespaceSelector.MatchExpressions = append(wb.NamespaceSelector.MatchExpressions, metav1.LabelSelectorRequirement{
-			Key:      corev1.LabelMetadataName,
-			Operator: metav1.LabelSelectorOpIn,
-			Values:   webhookNamespaceInclude,
-		})
-	}
-	return wb
-}
-
-// addPodMutatingWebhook updates the MutatingWebhookConfiguration for pods.
-// It retrieves the existing configuration, adds a new webhook for pods,
-// and updates the configuration in the Kubernetes API server.
-func AddPodMutatingWebhook(admissionClient admissionClientv1.AdmissionregistrationV1Interface, mutatingWebhookName string, webhookNamespaceInclude []string) error {
-	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		mwc, err := admissionClient.MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
-		if err != nil {
-			return fmt.Errorf("failed to get MutatingWebhookConfiguration: %v", err)
-		}
-
-		if len(mwc.Webhooks) == 0 {
-			return fmt.Errorf("no any mutating webhook found in MutatingWebhookConfiguration %s", mutatingWebhookName)
-		}
-
-		var newWebhooks []admissionregistrationv1.MutatingWebhook
-		for _, wb := range mwc.Webhooks {
-			// if the webhook already exists, do nothing
-			if wb.Name == constant.PodMutatingWebhookName {
-				continue
-			}
-			newWebhooks = append(newWebhooks, wb)
-		}
-
-		podWebhook := InitPodMutatingWebhook(*mwc.Webhooks[0].DeepCopy(), webhookNamespaceInclude)
-		newWebhooks = append(newWebhooks, podWebhook)
-		mwc.Webhooks = newWebhooks
-
-		_, updateErr := admissionClient.MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{})
-		return updateErr
-	})
-	if retryErr != nil {
-		return fmt.Errorf("update MutatingWebhookConfiguration %s failed: %v", mutatingWebhookName, retryErr)
-	}
-
-	return nil
-}
-
-// RemovePodMutatingWebhook removes the mutating webhook for pods.
-// It retrieves the existing configuration, removes the webhook for pods,
-// and updates the configuration in the Kubernetes API server.
-func RemovePodMutatingWebhook(admissionClient admissionClientv1.AdmissionregistrationV1Interface, mutatingWebhookName string) error {
-	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		mwc, err := admissionClient.MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		var newWebhooks []admissionregistrationv1.MutatingWebhook
-		for _, wb := range mwc.Webhooks {
-			if wb.Name != constant.PodMutatingWebhookName {
-				newWebhooks = append(newWebhooks, wb)
-			}
-		}
-
-		if len(newWebhooks) == len(mwc.Webhooks) {
-			return nil
-		}
-
-		mwc.Webhooks = newWebhooks
-		_, err = admissionClient.MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{})
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if retryErr != nil {
-		return fmt.Errorf("removes the mutating webhook for pods: %v", retryErr)
-	}
-	return nil
-}
-
 func doValidateRdmaResouceAndIPPools(mc v2beta1.SpiderMultusConfig) error {
 	doValidateIPPools := func(name, namespace string, ippools *v2beta1.SpiderpoolPools) error {
 		if ippools == nil {