feat: Add cert generator

vesoft-inc · Jun 6, 2024 · ba7229b · ba7229b
1 parent bae83a2
commit ba7229b
Show file tree

Hide file tree

Showing 13 changed files with 520 additions and 29 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -12,6 +12,7 @@ RUN if [ "$USERNAME" = "ng-user" ]; then \
 ADD bin/${TARGETDIR}/controller-manager /usr/local/bin/controller-manager
 ADD bin/${TARGETDIR}/autoscaler /usr/local/bin/autoscaler
 ADD bin/${TARGETDIR}/scheduler /usr/local/bin/scheduler
+ADD bin/${TARGETDIR}/certificate-generator /usr/local/bin/certificate-generator
 
 # [Optional] Set the default user. Omit if you want to keep the default as root.
 USER $USERNAME
diff --git a/Makefile b/Makefile
@@ -79,6 +79,7 @@ build-operator: ## Build operator related binary.
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/controller-manager cmd/controller-manager/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/autoscaler cmd/autoscaler/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/scheduler cmd/scheduler/main.go
+	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/certificate-generator cmd/certificate-generator/main.go
 
 build-provisioner: ## Build provisioner binary.
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/local-pv-provisioner cmd/provisioner/main.go

diff --git a/charts/nebula-operator/templates/_helpers.tpl b/charts/nebula-operator/templates/_helpers.tpl
@@ -97,6 +97,20 @@ Admission webhook name of the chart.
 {{ include "nebula-operator.name" . }}-webhook
 {{- end }}
 
+{{/*
+Controller Manager Admission webhook name.
+*/}}
+{{- define "controller-manager-admission-webhook.name" -}}
+controller-manager-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
+{{/*
+Autoscaler Admission webhook name.
+*/}}
+{{- define "autoscaler-admission-webhook.name" -}}
+autoscaler-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
 {{/*
 Admission webhook selector labels
 */}}

diff --git a/charts/nebula-operator/templates/admission-webhook-certificate.yaml b/charts/nebula-operator/templates/admission-webhook-certificate.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) .Values.admissionWebhook.useCertManager }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -9,8 +9,8 @@ metadata:
   {{- include "admission-webhook.labels" . | nindent 4 }}
 spec:
   dnsNames:
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
     - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
     - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
   issuerRef:

diff --git a/charts/nebula-operator/templates/admission-webhook-registration.yaml b/charts/nebula-operator/templates/admission-webhook-registration.yaml
@@ -1,10 +1,12 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  {{- if .Values.admissionWebhook.useCertManager }}
   annotations:
     cert-manager.io/inject-ca-from: {{ template "nebula-operator.namespace" . }}/{{ template "admission-webhook.name" . }}-cert
+  {{- end }}
   name: {{ template "admission-webhook.name" . }}-validating
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}
@@ -14,7 +16,7 @@ webhooks:
       - v1
     clientConfig:
       service:
-        name: {{ template "admission-webhook.name" . }}-service
+        name: {{ template "controller-manager-admission-webhook.name" . }}-service
         namespace: {{ template "nebula-operator.namespace" . }}
         path: /validate-nebulacluster
     failurePolicy: Fail

diff --git a/charts/nebula-operator/templates/autoscaler-admission-webhook-service.yaml b/charts/nebula-operator/templates/autoscaler-admission-webhook-service.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
 ---
 apiVersion: v1
 kind: Service

diff --git a/.../templates/admission-webhook-service.yaml → ...er-manager-admission-webhook-service.yaml b/.../templates/admission-webhook-service.yaml → ...er-manager-admission-webhook-service.yaml
@@ -1,9 +1,9 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "admission-webhook.name" . }}-service
+  name: {{ template "controller-manager-admission-webhook.name" . }}-service
   namespace: {{ template "nebula-operator.namespace" . }}
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}

diff --git a/charts/nebula-operator/templates/controller-manager-deployment.yaml b/charts/nebula-operator/templates/controller-manager-deployment.yaml
@@ -43,8 +43,8 @@ spec:
             - --concurrent-nebulabackup-syncs={{ .Values.concurrentNebulaBackupSyncs }}
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
-            - --enable-admission-webhook={{ .Values.admissionWebhook.create }}
-            - --webhook-secure-port={{ .Values.admissionWebhook.webhookBindPort }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort }}
             - --enable-kruise-scheme={{ .Values.enableKruiseScheme }}
             - --v={{ .Values.controllerManager.verbosity }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
@@ -55,9 +55,9 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           ports:
-            - containerPort: {{ .Values.admissionWebhook.webhookBindPort | default 9443 }}
+            - containerPort: {{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort | default 9443 }}
               name: webhook-server
               protocol: TCP
           {{- end }}
@@ -77,9 +77,9 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
-          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.create }}
+          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           volumeMounts:
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
@@ -100,8 +100,8 @@ spec:
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
             - --v={{ .Values.controllerManager.verbosity }}
-            - --enable-admission-webhook={{ .Values.admissionWebhook.create }}
-            - --webhook-secure-port={{ .Values.autoscalerAdmissionWebhook.webhookBindPort }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
           env:
           {{- if .Values.kubernetesClusterDomain }}
@@ -110,9 +110,9 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
           ports:
-            - containerPort: {{ .Values.autoscalerAdmissionWebhook.webhookBindPort | default 9448 }}
+            - containerPort: {{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort | default 9448 }}
               name: webhook-server
               protocol: TCP
           {{- end }}
@@ -132,14 +132,38 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
           volumeMounts:
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
           {{- end }}
           {{- end }}
+        {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+        - name: cert-rotation
+          image: {{ .Values.image.nebulaOperator.image }}
+          imagePullPolicy: {{ .Values.image.nebulaOperator.imagePullPolicy }}
+          command:
+            - /usr/local/bin/certificate-generator
+          args:
+            - --leader-elect
+            - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
+            - --v={{ .Values.controllerManager.verbosity }}
+            - --webhook-namespace={{ template "nebula-operator.namespace" . }}
+            - --webhook-name={{ template "admission-webhook.name" . }}-validating
+            - --certificate-dir=/tmp/k8s-webhook-server/serving-certs
+            - --certificate-validity=1
+          env:
+            {{- if .Values.kubernetesClusterDomain }}
+            - name: KUBERNETES_CLUSTER_DOMAIN
+              value: {{ .Values.kubernetesClusterDomain }}
+            {{- end }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: false
+        {{- end}}
     {{- with .Values.controllerManager.sidecarContainers }}
       {{- range $name, $spec :=  $.Values.controllerManager.sidecarContainers }}
         - name: {{ $name }}
@@ -169,13 +193,18 @@ spec:
       tolerations:
       {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if or .Values.controllerManager.extraVolumes .Values.admissionWebhook.create}}
+      {{- if or .Values.controllerManager.extraVolumes (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) }}
       volumes:
-      {{- if .Values.admissionWebhook.create}}
+      {{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create}}
+      {{- if .Values.admissionWebhook.useCertManager }}
       - name: cert
         secret:
           defaultMode: 420
           secretName: {{ template "admission-webhook.name" . }}-secret
+      {{- else}}
+      - name: cert
+        emptyDir: {}
+      {{- end}}
       {{- end }}
       {{- if .Values.controllerManager.extraVolumes }}
       {{- toYaml .Values.controllerManager.extraVolumes | nindent 8 }}

diff --git a/charts/nebula-operator/templates/controller-manager-rbac.yaml b/charts/nebula-operator/templates/controller-manager-rbac.yaml
@@ -387,6 +387,16 @@ rules:
     verbs:
       - get
       - list
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/charts/nebula-operator/values.yaml b/charts/nebula-operator/values.yaml
@@ -64,13 +64,15 @@ controllerManager:
 #    runAsNonRoot: true
 
 admissionWebhook:
-  create: false
-  # The TCP port the Webhook server binds to. (default 9443)
-  webhookBindPort: 9443
-
-autoscalerAdmissionWebhook:
-  # The TCP port the Webhook server binds to. (default 9443)
-  webhookBindPort: 9448
+  contollerManagerAdmissionWebhook:
+    create: false
+    # The TCP port the Webhook server binds to. (default 9443)
+    webhookBindPort: 9443
+  autoscalerAdmissionWebhook:
+    create: true
+    # The TCP port the Webhook server binds to. (default 9448)
+    webhookBindPort: 9448
+  useCertManager: false
 
 scheduler:
   create: true