From b3a54ef4b1aeab287ddf0dd2a051b078f6072a76 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 11:52:30 +0100 Subject: [PATCH 1/7] Use latest kubebuilder version Signed-off-by: Richard Wall --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be06099..0fd7f90 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ You will need the following command line tools installed on your PATH: * [Docker v17.03+](https://docs.docker.com/install/) * [Kind v0.18.0+](https://kind.sigs.k8s.io/docs/user/quick-start/) * [Kubectl v1.26.3+](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -* [Kubebuilder v2.3.1+](https://book.kubebuilder.io/quick-start.html#installation) +* [Kubebuilder v3.9.1+](https://book.kubebuilder.io/quick-start.html#installation) * [Kustomize v3.8.1+](https://kustomize.io/) You may also want to read: the [Kubebuilder Book] and the [cert-manager Concepts Documentation] for further background From 8c4aaf3ef2c8ae331d5d191b68d56d1ac96b5df9 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 12:26:50 +0100 Subject: [PATCH 2/7] kubebuilder init --domain example.com --owner 'The cert-manager Authors' Signed-off-by: Richard Wall --- .dockerignore | 5 +++ .gitignore | 2 ++ Dockerfile | 11 ++++-- PROJECT | 8 +++++ README.md | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 2 +- hack/boilerplate.go.txt | 4 +-- main.go | 48 ++++++++++++++++++--------- 8 files changed, 60 insertions(+), 22 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..38f5432 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ +build/ diff --git a/.gitignore b/.gitignore index 45c2b47..52f0ed7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ *.so *.dylib bin +testbin/* +Dockerfile.cross # Test binary, build with `go test -c` *.test diff --git a/Dockerfile b/Dockerfile index 374e33b..0999178 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # Build the manager binary FROM golang:1.20 as builder +ARG TARGETOS +ARG TARGETARCH WORKDIR /workspace # Copy the Go Modules manifests @@ -14,9 +16,13 @@ COPY main.go main.go COPY api/ api/ COPY internal/ internal/ +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. ENV CGO_ENABLED=0 -ENV GOOS=linux -ENV GOARCH=amd64 +ENV GOOS=${TARGETOS:-linux} +ENV GOARCH=${TARGETARCH} ENV GO111MODULE=on # Do an initial compilation before setting the version so that there is less to @@ -36,5 +42,6 @@ RUN go build \ FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . +USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/PROJECT b/PROJECT index 7a8e96c..16af1c6 100644 --- a/PROJECT +++ b/PROJECT @@ -1,4 +1,11 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: example.com +layout: +- go.kubebuilder.io/v3 +projectName: sample-external-issuer repo: github.com/cert-manager/sample-external-issuer resources: - group: sample-issuer @@ -8,3 +15,4 @@ resources: kind: ClusterIssuer version: v1alpha1 version: "2" +version: "3" diff --git a/README.md b/README.md index 0fd7f90..d355e45 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ You will also need to add the cert-manager API types to the `Scheme`: ```go func init() { ... - _ = cmapi.AddToScheme(scheme) + utilruntime.Must(cmapi.AddToScheme(scheme)) ... } ``` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index add97ba..9352539 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 44d81dc..a086083 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ +*/ \ No newline at end of file diff --git a/main.go b/main.go index f59b719..e0ffb1d 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,10 +23,14 @@ import ( "io/ioutil" "os" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -35,7 +39,7 @@ import ( "github.com/cert-manager/sample-external-issuer/internal/controllers" "github.com/cert-manager/sample-external-issuer/internal/issuer/signer" "github.com/cert-manager/sample-external-issuer/internal/version" - // +kubebuilder:scaffold:imports + //+kubebuilder:scaffold:imports ) const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" @@ -46,23 +50,23 @@ var ( ) func init() { - _ = clientgoscheme.AddToScheme(scheme) - - _ = sampleissuerv1alpha1.AddToScheme(scheme) - // +kubebuilder:scaffold:scheme - - _ = cmapi.AddToScheme(scheme) + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(cmapi.AddToScheme(scheme)) + utilruntime.Must(sampleissuerv1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme } func main() { var metricsAddr string + var probeAddr string var enableLeaderElection bool var clusterResourceNamespace string var printVersion bool var disableApprovedCheck bool - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&clusterResourceNamespace, "cluster-resource-namespace", "", "The namespace for secrets in which cluster-scoped resources are found.") @@ -105,11 +109,23 @@ func main() { ) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - LeaderElection: enableLeaderElection, - LeaderElectionID: "54c549fd.example.com", + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "54c549fd.example.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, }) if err != nil { setupLog.Error(err, "unable to start manager") From b4106a68fd09793a17974b1d0744f9e8903f9139 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 12:32:47 +0100 Subject: [PATCH 3/7] kubebuilder init --domain example.com --owner 'The cert-manager Authors' (config/) Apply all the config/ changes Signed-off-by: Richard Wall --- config/default/kustomization.yaml | 18 +++--- config/default/manager_auth_proxy_patch.yaml | 40 +++++++++++-- config/default/manager_config_patch.yaml | 10 ++++ config/manager/kustomization.yaml | 6 -- config/manager/manager.yaml | 59 +++++++++++++++++-- config/prometheus/monitor.yaml | 10 ++++ .../rbac/auth_proxy_client_clusterrole.yaml | 13 +++- config/rbac/auth_proxy_role.yaml | 19 ++++-- config/rbac/auth_proxy_role_binding.yaml | 9 ++- config/rbac/auth_proxy_service.yaml | 7 +++ config/rbac/kustomization.yaml | 6 ++ config/rbac/leader_election_role.yaml | 25 ++++---- config/rbac/leader_election_role_binding.yaml | 9 ++- config/rbac/role_binding.yaml | 9 ++- config/rbac/service_account.yaml | 12 ++++ 15 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 config/default/manager_config_patch.yaml create mode 100644 config/rbac/service_account.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 53ac760..4ed0209 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -16,21 +16,23 @@ bases: - ../crd - ../rbac - ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus patchesStrategicMerge: - # Protect the /metrics endpoint by putting it behind auth. - # If you want your controller-manager to expose the /metrics - # endpoint w/o any authn/z, please comment the following line. +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. - manager_auth_proxy_patch.yaml -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in + + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- manager_webhook_patch.yaml @@ -46,7 +48,7 @@ vars: # objref: # kind: Certificate # group: cert-manager.io -# version: v1alpha2 +# version: v1 # name: serving-cert # this name should match the one in certificate.yaml # fieldref: # fieldpath: metadata.namespace @@ -54,7 +56,7 @@ vars: # objref: # kind: Certificate # group: cert-manager.io -# version: v1alpha2 +# version: v1 # name: serving-cert # this name should match the one in certificate.yaml #- name: SERVICE_NAMESPACE # namespace of the service # objref: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index a2dc7c4..b751266 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -1,4 +1,4 @@ -# This patch inject a sidecar container which is a HTTP proxy for the +# This patch inject a sidecar container which is a HTTP proxy for the # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. apiVersion: apps/v1 kind: Deployment @@ -8,18 +8,48 @@ metadata: spec: template: spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux containers: - name: kube-rbac-proxy - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 args: - "--secure-listen-address=0.0.0.0:8443" - "--upstream=http://127.0.0.1:8080/" - "--logtostderr=true" - - "--v=10" + - "--v=0" ports: - containerPort: 8443 + protocol: TCP name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi - name: manager args: - - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 0000000..f6f5891 --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ad13e96..5c5f0b8 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,8 +1,2 @@ resources: - manager.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: controller - newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c908b36..42064d1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -3,6 +3,12 @@ kind: Namespace metadata: labels: control-plane: controller-manager + app.kubernetes.io/name: namespace + app.kubernetes.io/instance: system + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: system --- apiVersion: apps/v1 @@ -12,6 +18,12 @@ metadata: namespace: system labels: control-plane: controller-manager + app.kubernetes.io/name: deployment + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize spec: selector: matchLabels: @@ -19,23 +31,60 @@ spec: replicas: 1 template: metadata: + annotations: + kubectl.kubernetes.io/default-container: manager labels: control-plane: controller-manager spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux securityContext: runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault containers: - command: - /manager args: - - --enable-leader-election + - --leader-elect image: controller:latest name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: - cpu: 100m - memory: 30Mi + cpu: 500m + memory: 128Mi requests: - cpu: 100m - memory: 20Mi + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 9b8047b..61e09f3 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -5,12 +5,22 @@ kind: ServiceMonitor metadata: labels: control-plane: controller-manager + app.kubernetes.io/name: servicemonitor + app.kubernetes.io/instance: controller-manager-metrics-monitor + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: controller-manager-metrics-monitor namespace: system spec: endpoints: - path: /metrics port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml index bd4af13..ec1e6d3 100644 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -1,7 +1,16 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: metrics-reader rules: -- nonResourceURLs: ["/metrics"] - verbs: ["get"] +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml index 618f5e4..f11211a 100644 --- a/config/rbac/auth_proxy_role.yaml +++ b/config/rbac/auth_proxy_role.yaml @@ -1,13 +1,24 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: proxy-role rules: -- apiGroups: ["authentication.k8s.io"] +- apiGroups: + - authentication.k8s.io resources: - tokenreviews - verbs: ["create"] -- apiGroups: ["authorization.k8s.io"] + verbs: + - create +- apiGroups: + - authorization.k8s.io resources: - subjectaccessreviews - verbs: ["create"] + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml index 48ed1e4..a007e4b 100644 --- a/config/rbac/auth_proxy_role_binding.yaml +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -1,6 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: proxy-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io @@ -8,5 +15,5 @@ roleRef: name: proxy-role subjects: - kind: ServiceAccount - name: default + name: controller-manager namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 6cf656b..fbe2a9e 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -3,12 +3,19 @@ kind: Service metadata: labels: control-plane: controller-manager + app.kubernetes.io/name: service + app.kubernetes.io/instance: controller-manager-metrics-service + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: controller-manager-metrics-service namespace: system spec: ports: - name: https port: 8443 + protocol: TCP targetPort: https selector: control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index b1481b7..e844a30 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,4 +1,10 @@ resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml - role.yaml - role_binding.yaml - leader_election_role.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index 7c6ec78..f7d8a97 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -2,6 +2,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: + labels: + app.kubernetes.io/name: role + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: leader-election-role rules: - apiGroups: @@ -17,25 +24,21 @@ rules: - patch - delete - apiGroups: - - "" + - coordination.k8s.io resources: - - configmaps/status + - leases verbs: - get + - list + - watch + - create - update - patch + - delete - apiGroups: - "" resources: - events verbs: - create -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index eed1690..a622fdd 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -1,6 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: + labels: + app.kubernetes.io/name: rolebinding + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: leader-election-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io @@ -8,5 +15,5 @@ roleRef: name: leader-election-role subjects: - kind: ServiceAccount - name: default + name: controller-manager namespace: system diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 8f26587..6bf1ddb 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,6 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io @@ -8,5 +15,5 @@ roleRef: name: manager-role subjects: - kind: ServiceAccount - name: default + name: controller-manager namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..e4da264 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: sample-external-issuer + app.kubernetes.io/part-of: sample-external-issuer + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system From 423eea2d1ae24476aa0cf911645cdbee8283ce62 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 13:08:21 +0100 Subject: [PATCH 4/7] kubebuilder init --domain example.com --owner 'The cert-manager Authors' (Makefile) Apply the Makefile changes Signed-off-by: Richard Wall --- Makefile | 250 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 156 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index fe27f35..e8510d1 100644 --- a/Makefile +++ b/Makefile @@ -14,133 +14,79 @@ DOCKER_IMAGE_NAME ?= cert-manager/sample-external-issuer/controller # Image URL to use all building/pushing image targets IMG ?= ${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${VERSION} -# BIN is the directory where tools will be installed -export BIN ?= ${CURDIR}/bin - -OS := $(shell go env GOOS) -ARCH := $(shell go env GOARCH) - -# Kind -KIND_VERSION := 0.18.0 -KIND := ${BIN}/kind-${KIND_VERSION} -K8S_CLUSTER_NAME := sample-external-issuer-e2e # cert-manager CERT_MANAGER_VERSION ?= 1.11.1 -# Controller tools -CONTROLLER_GEN_VERSION := 0.11.3 -CONTROLLER_GEN := ${BIN}/controller-gen INSTALL_YAML ?= build/install.yaml .PHONY: all -all: manager +all: build -# Run tests -.PHONY: test -test: generate fmt vet manifests - go test ./... -coverprofile cover.out +##@ General -# Build manager binary -.PHONY: manager -manager: generate fmt vet - go build -o bin/manager main.go +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php -# Run against the configured Kubernetes cluster in ~/.kube/config -.PHONY: run -run: generate fmt vet manifests - go run ./main.go - -# Install CRDs into a cluster -.PHONY: install -install: manifests - kustomize build config/crd | kubectl apply -f - - -# Uninstall CRDs from a cluster -.PHONY: uninstall -uninstall: manifests - kustomize build config/crd | kubectl delete -f - - -# TODO(wallrj): .PHONY ensures that the install file is always regenerated, -# because I this really depends on the checksum of the Docker image and all the -# base Kustomize files. -.PHONY: ${INSTALL_YAML} -${INSTALL_YAML}: - mkdir -p $(dir $@) - rm -rf build/kustomize - mkdir -p build/kustomize - cd build/kustomize - kustomize create --resources ../../config/default - kustomize edit set image controller=${IMG} - cd ${CURDIR} - kustomize build build/kustomize > $@ +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -.PHONY: deploy -deploy: ${INSTALL_YAML} - kubectl apply -f ${INSTALL_YAML} +##@ Development -# Generate manifests e.g. CRD, RBAC etc. .PHONY: manifests -manifests: ${CONTROLLER_GEN} - $(CONTROLLER_GEN) rbac:roleName=manager-role webhook crd paths="./..." output:crd:artifacts:config=config/crd/bases +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -# Run go fmt against code .PHONY: fmt -fmt: +fmt: ## Run go fmt against code. go fmt ./... -# Run go vet against code .PHONY: vet -vet: +vet: ## Run go vet against code. go vet ./... -# Generate code -generate: ${CONTROLLER_GEN} - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -# Build the docker image -.PHONY: docker-build -docker-build: - docker build \ - --build-arg VERSION=$(VERSION) \ - --tag ${IMG} \ - --file Dockerfile \ - ${CURDIR} +.PHONY: test +test: manifests generate fmt vet ## Run tests + go test ./... -coverprofile cover.out -# Push the docker image -.PHONY: docker-push -docker-push: - docker push ${IMG} +##@ E2E testing -${CONTROLLER_GEN}: | ${BIN} - GOBIN=${BIN} go install sigs.k8s.io/controller-tools/cmd/controller-gen@v${CONTROLLER_GEN_VERSION} +K8S_CLUSTER_NAME := sample-external-issuer-e2e -# ================================== -# E2E testing -# ================================== .PHONY: kind-cluster kind-cluster: ## Use Kind to create a Kubernetes cluster for E2E tests -kind-cluster: ${KIND} +kind-cluster: kind ${KIND} get clusters | grep ${K8S_CLUSTER_NAME} || ${KIND} create cluster --name ${K8S_CLUSTER_NAME} .PHONY: kind-load -kind-load: ## Load all the Docker images into Kind +kind-load: kind ## Load all the Docker images into Kind ${KIND} load docker-image --name ${K8S_CLUSTER_NAME} ${IMG} .PHONY: kind-export-logs -kind-export-logs: +kind-export-logs: kind ## Export Kind logs ${KIND} export logs --name ${K8S_CLUSTER_NAME} ${E2E_ARTIFACTS_DIRECTORY} - .PHONY: deploy-cert-manager deploy-cert-manager: ## Deploy cert-manager in the configured Kubernetes cluster in ~/.kube/config kubectl apply --filename=https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VERSION}/cert-manager.yaml kubectl wait --for=condition=Available --timeout=300s apiservice v1.cert-manager.io .PHONY: e2e -e2e: +e2e: ## Run E2E tests kubectl apply --filename config/samples kubectl wait --for=condition=Ready --timeout=5s issuers.sample-issuer.example.com issuer-sample @@ -153,12 +99,128 @@ e2e: kubectl delete --filename config/samples -# ================================== -# Download: tools in ${BIN} -# ================================== -${BIN}: - mkdir -p ${BIN} +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary + go build -o bin/manager main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +# If you wish built the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + docker build \ + --build-arg VERSION=$(VERSION) \ + --tag ${IMG} \ + --file Dockerfile \ + ${CURDIR} + +# Push the docker image +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ +# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=> then the export will fail) +# To properly provided solutions that supports more than one platform you should use this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: test ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - docker buildx create --name project-v3-builder + docker buildx use project-v3-builder + - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - docker buildx rm project-v3-builder + rm Dockerfile.cross + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +# TODO(wallrj): .PHONY ensures that the install file is always regenerated, +# because I this really depends on the checksum of the Docker image and all the +# base Kustomize files. +.PHONY: ${INSTALL_YAML} +${INSTALL_YAML}: manifests kustomize + mkdir -p $(dir $@) + rm -rf build/kustomize + mkdir -p build/kustomize + cd build/kustomize + $(KUSTOMIZE) create --resources ../../config/default + $(KUSTOMIZE) edit set image controller=${IMG} + cd ${CURDIR} + $(KUSTOMIZE) build build/kustomize > $@ + +.PHONY: deploy +deploy: ${INSTALL_YAML} ## Deploy controller to the K8s cluster specified in ~/.kube/config. + kubectl apply -f ${INSTALL_YAML} + +.PHONY: undeploy +undeploy: ${INSTALL_YAML} ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + kubectl delete -f ${INSTALL_YAML} --ignore-not-found=$(ignore-not-found) + +##@ Build Dependencies + +LOCAL_OS := $(shell go env GOOS) +LOCAL_ARCH := $(shell go env GOARCH) + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +KIND ?= $(LOCALBIN)/kind + +## Tool Versions +KUSTOMIZE_VERSION ?= v3.8.7 +CONTROLLER_TOOLS_VERSION ?= v0.11.3 +KIND_VERSION := 0.18.0 -${KIND}: ${BIN} - curl -fsSL -o ${KIND} https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-${OS}-${ARCH} +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. +$(KUSTOMIZE): $(LOCALBIN) + @if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \ + echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \ + rm -rf $(LOCALBIN)/kustomize; \ + fi + test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. +$(CONTROLLER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + +.PHONY: kind +kind: $(LOCALBIN) ## Download Kind locally if necessary. + curl -fsSL -o ${KIND} https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-${LOCAL_OS}-${LOCAL_ARCH} chmod +x ${KIND} From 283d83a366a055374a454794a0b8b482be799140 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 13:22:13 +0100 Subject: [PATCH 5/7] kubebuilder create api --group sample-issuer --kind Issuer --version v1alpha1 Signed-off-by: Richard Wall --- api/v1alpha1/groupversion_info.go | 2 +- api/v1alpha1/issuer_types.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 444d22a..ad02187 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/api/v1alpha1/issuer_types.go b/api/v1alpha1/issuer_types.go index 035fc29..6bc4896 100644 --- a/api/v1alpha1/issuer_types.go +++ b/api/v1alpha1/issuer_types.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -42,8 +42,8 @@ type IssuerStatus struct { Conditions []IssuerCondition `json:"conditions,omitempty"` } -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status // Issuer is the Schema for the issuers API type Issuer struct { @@ -54,7 +54,7 @@ type Issuer struct { Status IssuerStatus `json:"status,omitempty"` } -// +kubebuilder:object:root=true +//+kubebuilder:object:root=true // IssuerList contains a list of Issuer type IssuerList struct { From d6c0afa5c3304b2e204e24fe8dfe3cf72065a761 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 13:25:43 +0100 Subject: [PATCH 6/7] kubebuilder create api --group sample-issuer --kind ClusterIssuer --version v1alpha1 --namespaced=false Signed-off-by: Richard Wall --- api/v1alpha1/clusterissuer_types.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/clusterissuer_types.go b/api/v1alpha1/clusterissuer_types.go index 8aef409..feb6c0e 100644 --- a/api/v1alpha1/clusterissuer_types.go +++ b/api/v1alpha1/clusterissuer_types.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The cert-manager Authors +Copyright 2023 The cert-manager Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Cluster -// +kubebuilder:subresource:status +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster // ClusterIssuer is the Schema for the clusterissuers API type ClusterIssuer struct { @@ -33,7 +33,7 @@ type ClusterIssuer struct { Status IssuerStatus `json:"status,omitempty"` } -// +kubebuilder:object:root=true +//+kubebuilder:object:root=true // ClusterIssuerList contains a list of ClusterIssuer type ClusterIssuerList struct { From 778259ccde340c532631fc69644de8a2c55c3417 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Fri, 14 Apr 2023 13:52:53 +0100 Subject: [PATCH 7/7] Remove the instruction to add Status subresource markers Because they are now added by default. Fixes: https://github.com/cert-manager/sample-external-issuer/issues/22 Signed-off-by: Richard Wall --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index d355e45..be8236b 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,6 @@ The values we pass to these commands specify the GVK (group, version, kind): These commands will have created some boilerplate files and directories: `api/` and `controllers/`, which we now need to edit as follows: -* `api/v1alpha1/{cluster}issuer_types.go`: - Add [Kubebuilder CRD Markers](https://book.kubebuilder.io/reference/markers/crd.html) to allow modification of IssuerStatus - as a [Status Subresource](https://book-v1.book.kubebuilder.io/basics/status_subresource.html): `// +kubebuilder:subresource:status` - * `api/v1alpha1/clusterissuer_types.go`: Remove the `ClusterIssuerSpec` and `ClusterIssuerStatus` and replace them with `IssuerSpec` and `IssuerStatus`. This is because both types of issuers share the same configuration and status reporting.