diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index fe34e1a..c9a83b0 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,5 +1,5 @@ # Adds namespace to all resources. -namespace: global-load-balancer-operator-system +namespace: global-load-balancer-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -22,13 +22,13 @@ bases: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus +- ../prometheus patchesStrategicMerge: # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. -# - manager_auth_proxy_patch.yaml +- manager_auth_proxy_patch.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type @@ -45,6 +45,19 @@ patchesStrategicMerge: # the following config is for teaching kustomize how to do var substitution vars: +- name: METRICS_SERVICE_NAME + objref: + kind: Service + version: v1 + name: controller-manager-metrics +- name: METRICS_SERVICE_NAMESPACE + objref: + kind: Service + version: v1 + name: controller-manager-metrics + fieldref: + fieldpath: metadata.namespace + # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR # objref: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 49b1f1a..beb218a 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -16,6 +16,11 @@ spec: - "--upstream=http://127.0.0.1:8080/" - "--logtostderr=true" - "--v=10" + - "--tls-cert-file=/etc/certs/tls/tls.crt" + - "--tls-private-key-file=/etc/certs/tls/tls.key" + volumeMounts: + - mountPath: /etc/certs/tls + name: tls-cert ports: - containerPort: 8443 name: https @@ -24,3 +29,8 @@ spec: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" + volumes: + - name: tls-cert + secret: + defaultMode: 420 + secretName: global-load-balancer-operator-certs diff --git a/config/helmchart/kustomization.yaml b/config/helmchart/kustomization.yaml index 6366f99..4484d4e 100644 --- a/config/helmchart/kustomization.yaml +++ b/config/helmchart/kustomization.yaml @@ -14,3 +14,18 @@ namePrefix: global-load-balancer-operator- bases: - ../rbac +- ../prometheus + +vars: +- name: METRICS_SERVICE_NAME + objref: + kind: Service + version: v1 + name: controller-manager-metrics +- name: METRICS_SERVICE_NAMESPACE + objref: + kind: Service + version: v1 + name: controller-manager-metrics + fieldref: + fieldpath: metadata.namespace diff --git a/config/helmchart/templates/manager.yaml b/config/helmchart/templates/manager.yaml index b8bdfad..54dc8d8 100644 --- a/config/helmchart/templates/manager.yaml +++ b/config/helmchart/templates/manager.yaml @@ -17,6 +17,7 @@ spec: {{- end }} labels: {{- include "global-load-balancer-operator.selectorLabels" . | nindent 8 }} + operator: global-load-balancer-operator spec: serviceAccountName: global-load-balancer-operator-controller-manager {{- with .Values.imagePullSecrets }} @@ -24,6 +25,21 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --tls-cert-file=/etc/certs/tls/tls.crt + - --tls-private-key-file=/etc/certs/tls/tls.key + - --v=10 + image: quay.io/coreos/kube-rbac-proxy:v0.5.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + volumeMounts: + - mountPath: /etc/certs/tls + name: tls-cert - command: - /manager args: @@ -61,4 +77,9 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} + volumes: + - name: tls-cert + secret: + defaultMode: 420 + secretName: global-load-balancer-operator-certs \ No newline at end of file diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 08ed4c1..f4f9164 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -3,6 +3,7 @@ kind: Namespace metadata: labels: control-plane: controller-manager + openshift.io/cluster-monitoring: "true" name: system --- apiVersion: apps/v1 @@ -11,16 +12,16 @@ metadata: name: controller-manager namespace: system labels: - control-plane: controller-manager + operator: global-load-balancer-operator spec: selector: matchLabels: - control-plane: controller-manager + operator: global-load-balancer-operator replicas: 1 template: metadata: labels: - control-plane: controller-manager + operator: global-load-balancer-operator spec: serviceAccountName: controller-manager containers: diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..81dff32 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,5 @@ resources: - monitor.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/prometheus/kustomizeconfig.yaml b/config/prometheus/kustomizeconfig.yaml new file mode 100644 index 0000000..c85f72a --- /dev/null +++ b/config/prometheus/kustomizeconfig.yaml @@ -0,0 +1,4 @@ +--- +varReference: +- path: spec/endpoints/tlsConfig/serverName + kind: ServiceMonitor \ No newline at end of file diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 9b8047b..888d4e2 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -4,13 +4,18 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: - control-plane: controller-manager + operator: global-load-balancer-operator name: controller-manager-metrics-monitor namespace: system spec: endpoints: - - path: /metrics + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 30s port: https + scheme: https + tlsConfig: + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + serverName: $(METRICS_SERVICE_NAME).$(METRICS_SERVICE_NAMESPACE).svc selector: matchLabels: - control-plane: controller-manager + operator: global-load-balancer-operator diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 6cf656b..403edb7 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -2,8 +2,10 @@ apiVersion: v1 kind: Service metadata: labels: - control-plane: controller-manager - name: controller-manager-metrics-service + operator: global-load-balancer-operator + annotations: + service.alpha.openshift.io/serving-cert-secret-name: global-load-balancer-operator-certs + name: controller-manager-metrics namespace: system spec: ports: @@ -11,4 +13,4 @@ spec: port: 8443 targetPort: https selector: - control-plane: controller-manager + operator: global-load-balancer-operator diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 1c7b2a3..27cb1d1 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,7 +7,7 @@ resources: # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. -# - auth_proxy_service.yaml -# - auth_proxy_role.yaml -# - auth_proxy_role_binding.yaml -# - auth_proxy_client_clusterrole.yaml +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/readme.md b/readme.md index f3069fa..cfee20b 100644 --- a/readme.md +++ b/readme.md @@ -255,6 +255,14 @@ helm repo update helm upgrade global-load-balancer-operator global-load-balancer-operator/global-load-balancer-operator ``` +## Metrics + +Prometheus compatible metrics are exposed by the Operator and can be integrated into OpenShift's default cluster monitoring. To enable OpenShift cluster monitoring, label the namespace the operator is deployed in with the label `openshift.io/cluster-monitoring="true"`. + +```shell +oc label namespace openshift.io/cluster-monitoring="true" +``` + ## Development ### Running the operator locally @@ -312,10 +320,22 @@ docker login quay.io/$repo/global-load-balancer-operator-bundle podman push quay.io/$repo/global-load-balancer-operator-bundle:latest operator-sdk bundle validate quay.io/$repo/global-load-balancer-operator-bundle:latest --select-optional name=operatorhub oc new-project global-load-balancer-operator +oc label namespace global-load-balancer-operator openshift.io/cluster-monitoring="true" operator-sdk cleanup global-load-balancer-operator -n global-load-balancer-operator operator-sdk run bundle --install-mode AllNamespaces -n global-load-balancer-operator quay.io/$repo/global-load-balancer-operator-bundle:latest ``` +### Testing + +#### Testing metrics + +```sh +export operatorNamespace=resource-locker-operator-local # or resource-locker-operator +oc label namespace ${operatorNamespace} openshift.io/cluster-monitoring="true" +oc rsh -n openshift-monitoring -c prometheus prometheus-k8s-0 /bin/bash +curl -v -s -k -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://resource-locker-operator-controller-manager-metrics.${operatorNamespace}.svc.cluster.local:8443/metrics +``` + ### Releasing ```shell