Merge pull request #4451 from cyclinder/1.0_cp/fix_cert_upgrade

Fix: the pod fails to run because the certificate of the pod webhook is not up to data after helm upgrade
spidernet-io · Dec 30, 2024 · dfc289e · dfc289e
2 parents c50fe51 + 83bb4b3
commit dfc289e
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 425 deletions.
diff --git a/charts/spiderpool/templates/tls.yaml b/charts/spiderpool/templates/tls.yaml
@@ -144,6 +144,48 @@ webhooks:
     - spidercoordinators
   sideEffects: None
 {{- end }}
+{{- if .Values.spiderpoolController.podResourceInject.enabled }}
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: {{ .Values.spiderpoolController.name | trunc 63 | trimSuffix "-" }}
+      namespace: {{ .Release.Namespace }}
+      path: /mutate--v1-pod
+      port: {{ .Values.spiderpoolController.webhookPort }}
+    {{- if (eq .Values.spiderpoolController.tls.method "provided") }}
+    caBundle: {{ .Values.spiderpoolController.tls.provided.tlsCa | required "missing spiderpoolController.tls.provided.tlsCa" }}
+    {{- else if (eq .Values.spiderpoolController.tls.method "auto") }}
+    caBundle: {{ .ca.Cert | b64enc }}
+    {{- end }}
+  failurePolicy: Fail
+  name: pods.spiderpool.spidernet.io
+  {{- if or .Values.spiderpoolController.podResourceInject.namespacesExclude .Values.spiderpoolController.podResourceInject.namespacesInclude }}
+  namespaceSelector:
+    matchExpressions:
+    {{- if .Values.spiderpoolController.podResourceInject.namespacesExclude }}
+    - key: kubernetes.io/metadata.name
+      operator: NotIn
+      values: {{ toYaml .Values.spiderpoolController.podResourceInject.namespacesExclude | nindent 8 }}
+    {{- end }}
+    {{- if .Values.spiderpoolController.podResourceInject.namespacesInclude }}
+    - key: kubernetes.io/metadata.name
+      operator: In
+      values: {{ toYaml .Values.spiderpoolController.podResourceInject.namespacesInclude | nindent 8 }}
+    {{- end }}
+  {{- end }}
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None
+{{- end }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

diff --git a/cmd/spiderpool-controller/cmd/daemon.go b/cmd/spiderpool-controller/cmd/daemon.go
@@ -266,19 +266,12 @@ func initControllerServiceManagers(ctx context.Context) {
 	controllerContext.PodManager = podManager
 
 	if controllerContext.Cfg.PodResourceInjectConfig.Enabled {
-		logger.Debug("Begin to init Pod MutatingWebhook")
-		if err := podmanager.InitPodWebhook(controllerContext.ClientSet.AdmissionregistrationV1(),
-			controllerContext.CRDManager, controllerContext.Cfg.ControllerDeploymentName,
-			controllerContext.Cfg.PodResourceInjectConfig.NamespacesExclude,
-			controllerContext.Cfg.PodResourceInjectConfig.NamespacesInclude); err != nil {
+		logger.Info("Begin to init Pod MutatingWebhook")
+		if err := podmanager.InitPodWebhook(controllerContext.CRDManager); err != nil {
 			logger.Fatal(err.Error())
 		}
 	} else {
-		logger.Debug("InjectPodNetworkResource is disabled, try to remove the pod part in the MutatingWebhook")
-		if err := podmanager.RemovePodMutatingWebhook(controllerContext.ClientSet.AdmissionregistrationV1(),
-			controllerContext.Cfg.ControllerDeploymentName); err != nil {
-			logger.Error(err.Error())
-		}
+		logger.Info("Pod MutatingWebhook is disabled")
 	}
 
 	logger.Info("Begin to initialize StatefulSet manager")

diff --git a/pkg/podmanager/pod_webhook.go b/pkg/podmanager/pod_webhook.go
@@ -12,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	admissionregistrationv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
@@ -38,17 +37,10 @@ type podWebhook struct {
 // InitPodWebhook initializes the pod webhook.
 // It sets up the mutating webhook for pods and registers it with the manager.
 // Parameters:
-//   - client: The Kubernetes client
 //   - mgr: The controller manager
-//   - mutatingWebhookName: The name of the mutating webhook
 //
 // Returns an error if initialization fails.
-func InitPodWebhook(
-	admissionClient admissionregistrationv1.AdmissionregistrationV1Interface,
-	mgr ctrl.Manager,
-	mutatingWebhookName string,
-	webhookNamespaceExclude []string,
-	webhookNamespaceInclude []string) error {
+func InitPodWebhook(mgr ctrl.Manager) error {
 	spiderClient, err := crdclientset.NewForConfig(ctrl.GetConfigOrDie())
 	if err != nil {
 		return err
@@ -58,14 +50,6 @@ func InitPodWebhook(
 		spiderClient: spiderClient,
 	}
 
-	if len(webhookNamespaceExclude) != 0 {
-		PodWebhookExcludeNamespaces = webhookNamespaceExclude
-	}
-
-	if err = AddPodMutatingWebhook(admissionClient, mutatingWebhookName, webhookNamespaceInclude); err != nil {
-		return err
-	}
-
 	// setup mutating webhook for pods
 	if err = ctrl.NewWebhookManagedBy(mgr).
 		For(&corev1.Pod{}).

diff --git a/pkg/podmanager/utils.go b/pkg/podmanager/utils.go
@@ -8,14 +8,10 @@ import (
 	"fmt"
 
 	crdclientset "github.com/spidernet-io/spiderpool/pkg/k8s/client/clientset/versioned"
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8s_resource "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	admissionClientv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
-	"k8s.io/client-go/util/retry"
-	"k8s.io/utils/ptr"
 	kubevirtv1 "kubevirt.io/api/core/v1"
 
 	"github.com/spidernet-io/spiderpool/pkg/constant"
@@ -191,141 +187,6 @@ func InjectRdmaResourceToPod(resourceMap map[string]bool, pod *corev1.Pod) {
 	}
 }
 
-// InitPodMutatingWebhook initializes a mutating webhook for pods based on a template webhook.
-// It sets up the webhook configuration including name, admission review versions, failure policy,
-// object selector, client config, and rules for pod creation and update operations.
-//
-// Parameters:
-//   - from: An admissionregistrationv1.MutatingWebhook object to use as a template
-//
-// Returns:
-//   - A new admissionregistrationv1.MutatingWebhook object configured for pod mutation
-func InitPodMutatingWebhook(from admissionregistrationv1.MutatingWebhook, webhookNamespaceInclude []string) admissionregistrationv1.MutatingWebhook {
-	wb := admissionregistrationv1.MutatingWebhook{
-		Name:                    constant.PodMutatingWebhookName,
-		AdmissionReviewVersions: from.AdmissionReviewVersions,
-		FailurePolicy:           ptr.To(admissionregistrationv1.Fail),
-		NamespaceSelector:       &metav1.LabelSelector{},
-		ClientConfig: admissionregistrationv1.WebhookClientConfig{
-			CABundle: from.ClientConfig.CABundle,
-		},
-		Rules: []admissionregistrationv1.RuleWithOperations{
-			{
-				Operations: []admissionregistrationv1.OperationType{
-					admissionregistrationv1.Create,
-					admissionregistrationv1.Update,
-				},
-				Rule: admissionregistrationv1.Rule{
-					APIGroups:   []string{""},
-					APIVersions: []string{"v1"},
-					Resources:   []string{"pods"},
-				},
-			},
-		},
-		SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone),
-	}
-
-	if from.ClientConfig.Service != nil {
-		wb.ClientConfig.Service = &admissionregistrationv1.ServiceReference{
-			Name:      from.ClientConfig.Service.Name,
-			Namespace: from.ClientConfig.Service.Namespace,
-			Port:      from.ClientConfig.Service.Port,
-			// format: /mutate-<group>-<apiVersion>-<resource>
-			Path: ptr.To("/mutate--v1-pod"),
-		}
-	}
-
-	if len(PodWebhookExcludeNamespaces) != 0 {
-		wb.NamespaceSelector.MatchExpressions = []metav1.LabelSelectorRequirement{
-			{
-				Key:      corev1.LabelMetadataName,
-				Operator: metav1.LabelSelectorOpNotIn,
-				Values:   PodWebhookExcludeNamespaces,
-			},
-		}
-	}
-
-	if len(webhookNamespaceInclude) != 0 {
-		wb.NamespaceSelector.MatchExpressions = append(wb.NamespaceSelector.MatchExpressions, metav1.LabelSelectorRequirement{
-			Key:      corev1.LabelMetadataName,
-			Operator: metav1.LabelSelectorOpIn,
-			Values:   webhookNamespaceInclude,
-		})
-	}
-	return wb
-}
-
-// addPodMutatingWebhook updates the MutatingWebhookConfiguration for pods.
-// It retrieves the existing configuration, adds a new webhook for pods,
-// and updates the configuration in the Kubernetes API server.
-func AddPodMutatingWebhook(admissionClient admissionClientv1.AdmissionregistrationV1Interface, mutatingWebhookName string, webhookNamespaceInclude []string) error {
-	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		mwc, err := admissionClient.MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
-		if err != nil {
-			return fmt.Errorf("failed to get MutatingWebhookConfiguration: %v", err)
-		}
-
-		if len(mwc.Webhooks) == 0 {
-			return fmt.Errorf("no any mutating webhook found in MutatingWebhookConfiguration %s", mutatingWebhookName)
-		}
-
-		var newWebhooks []admissionregistrationv1.MutatingWebhook
-		for _, wb := range mwc.Webhooks {
-			// if the webhook already exists, do nothing
-			if wb.Name == constant.PodMutatingWebhookName {
-				continue
-			}
-			newWebhooks = append(newWebhooks, wb)
-		}
-
-		podWebhook := InitPodMutatingWebhook(*mwc.Webhooks[0].DeepCopy(), webhookNamespaceInclude)
-		newWebhooks = append(newWebhooks, podWebhook)
-		mwc.Webhooks = newWebhooks
-
-		_, updateErr := admissionClient.MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{})
-		return updateErr
-	})
-	if retryErr != nil {
-		return fmt.Errorf("update MutatingWebhookConfiguration %s failed: %v", mutatingWebhookName, retryErr)
-	}
-
-	return nil
-}
-
-// RemovePodMutatingWebhook removes the mutating webhook for pods.
-// It retrieves the existing configuration, removes the webhook for pods,
-// and updates the configuration in the Kubernetes API server.
-func RemovePodMutatingWebhook(admissionClient admissionClientv1.AdmissionregistrationV1Interface, mutatingWebhookName string) error {
-	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		mwc, err := admissionClient.MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		var newWebhooks []admissionregistrationv1.MutatingWebhook
-		for _, wb := range mwc.Webhooks {
-			if wb.Name != constant.PodMutatingWebhookName {
-				newWebhooks = append(newWebhooks, wb)
-			}
-		}
-
-		if len(newWebhooks) == len(mwc.Webhooks) {
-			return nil
-		}
-
-		mwc.Webhooks = newWebhooks
-		_, err = admissionClient.MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{})
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if retryErr != nil {
-		return fmt.Errorf("removes the mutating webhook for pods: %v", retryErr)
-	}
-	return nil
-}
-
 func doValidateRdmaResouceAndIPPools(mc v2beta1.SpiderMultusConfig) error {
 	doValidateIPPools := func(name, namespace string, ippools *v2beta1.SpiderpoolPools) error {
 		if ippools == nil {