enabled metrics (#32)

* enabled metrics Signed-off-by: raffaelespazzoli <[email protected]> * fixed selector Signed-off-by: raffaelespazzoli <[email protected]>
redhat-cop · May 31, 2021 · d2dd558 · d2dd558
1 parent 6409ab6
commit d2dd558
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 17 deletions.
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -1,5 +1,5 @@
 # Adds namespace to all resources.
-namespace: global-load-balancer-operator-system
+namespace: global-load-balancer-operator
 
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named
@@ -22,13 +22,13 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-# - manager_auth_proxy_patch.yaml
+- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
@@ -45,6 +45,19 @@ patchesStrategicMerge:
 
 # the following config is for teaching kustomize how to do var substitution
 vars:
+- name: METRICS_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+- name: METRICS_SERVICE_NAMESPACE
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+  fieldref:
+    fieldpath: metadata.namespace
+
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
 #  objref:

diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
@@ -16,6 +16,11 @@ spec:
         - "--upstream=http://127.0.0.1:8080/"
         - "--logtostderr=true"
         - "--v=10"
+        - "--tls-cert-file=/etc/certs/tls/tls.crt"
+        - "--tls-private-key-file=/etc/certs/tls/tls.key"
+        volumeMounts:
+          - mountPath: /etc/certs/tls
+            name: tls-cert         
         ports:
         - containerPort: 8443
           name: https
@@ -24,3 +29,8 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+      volumes:
+      - name: tls-cert
+        secret:
+          defaultMode: 420
+          secretName: global-load-balancer-operator-certs        
diff --git a/config/helmchart/kustomization.yaml b/config/helmchart/kustomization.yaml
@@ -14,3 +14,18 @@ namePrefix: global-load-balancer-operator-
 
 bases:
 - ../rbac
+- ../prometheus
+
+vars:
+- name: METRICS_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+- name: METRICS_SERVICE_NAMESPACE
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+  fieldref:
+    fieldpath: metadata.namespace
diff --git a/config/helmchart/templates/manager.yaml b/config/helmchart/templates/manager.yaml
@@ -17,13 +17,29 @@ spec:
     {{- end }}
       labels:
         {{- include "global-load-balancer-operator.selectorLabels" . | nindent 8 }}
+        operator: global-load-balancer-operator
     spec:
       serviceAccountName: global-load-balancer-operator-controller-manager
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}    
       containers:
+      - args:
+        - --secure-listen-address=0.0.0.0:8443
+        - --upstream=http://127.0.0.1:8080/
+        - --logtostderr=true
+        - --tls-cert-file=/etc/certs/tls/tls.crt
+        - --tls-private-key-file=/etc/certs/tls/tls.key
+        - --v=10
+        image: quay.io/coreos/kube-rbac-proxy:v0.5.0
+        name: kube-rbac-proxy
+        ports:
+        - containerPort: 8443
+          name: https
+        volumeMounts:
+        - mountPath: /etc/certs/tls
+          name: tls-cert      
       - command:
         - /manager
         args:
@@ -61,4 +77,9 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}
+      volumes:
+      - name: tls-cert
+        secret:
+          defaultMode: 420
+          secretName: global-load-balancer-operator-certs      
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -3,6 +3,7 @@ kind: Namespace
 metadata:
   labels:
     control-plane: controller-manager
+    openshift.io/cluster-monitoring: "true"
   name: system
 ---
 apiVersion: apps/v1
@@ -11,16 +12,16 @@ metadata:
   name: controller-manager
   namespace: system
   labels:
-    control-plane: controller-manager
+    operator: global-load-balancer-operator
 spec:
   selector:
     matchLabels:
-      control-plane: controller-manager
+      operator: global-load-balancer-operator
   replicas: 1
   template:
     metadata:
       labels:
-        control-plane: controller-manager
+        operator: global-load-balancer-operator
     spec:
       serviceAccountName: controller-manager
       containers:

diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
@@ -1,2 +1,5 @@
 resources:
 - monitor.yaml
+
+configurations:
+- kustomizeconfig.yaml
diff --git a/config/prometheus/kustomizeconfig.yaml b/config/prometheus/kustomizeconfig.yaml
@@ -0,0 +1,4 @@
+---
+varReference:
+- path: spec/endpoints/tlsConfig/serverName
+  kind: ServiceMonitor
diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml
@@ -4,13 +4,18 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   labels:
-    control-plane: controller-manager
+    operator: global-load-balancer-operator
   name: controller-manager-metrics-monitor
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
       port: https
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: $(METRICS_SERVICE_NAME).$(METRICS_SERVICE_NAMESPACE).svc
   selector:
     matchLabels:
-      control-plane: controller-manager
+      operator: global-load-balancer-operator
diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml
@@ -2,13 +2,15 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-service
+    operator: global-load-balancer-operator
+  annotations:
+    service.alpha.openshift.io/serving-cert-secret-name: global-load-balancer-operator-certs
+  name: controller-manager-metrics
   namespace: system
 spec:
   ports:
   - name: https
     port: 8443
     targetPort: https
   selector:
-    control-plane: controller-manager
+    operator: global-load-balancer-operator
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -7,7 +7,7 @@ resources:
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-# - auth_proxy_service.yaml
-# - auth_proxy_role.yaml
-# - auth_proxy_role_binding.yaml
-# - auth_proxy_client_clusterrole.yaml
+- auth_proxy_service.yaml
+- auth_proxy_role.yaml
+- auth_proxy_role_binding.yaml
+- auth_proxy_client_clusterrole.yaml
diff --git a/readme.md b/readme.md
@@ -255,6 +255,14 @@ helm repo update
 helm upgrade global-load-balancer-operator global-load-balancer-operator/global-load-balancer-operator
 ```
 
+## Metrics
+
+Prometheus compatible metrics are exposed by the Operator and can be integrated into OpenShift's default cluster monitoring. To enable OpenShift cluster monitoring, label the namespace the operator is deployed in with the label `openshift.io/cluster-monitoring="true"`.
+
+```shell
+oc label namespace <namespace> openshift.io/cluster-monitoring="true"
+```
+
 ## Development
 
 ### Running the operator locally
@@ -312,10 +320,22 @@ docker login quay.io/$repo/global-load-balancer-operator-bundle
 podman push quay.io/$repo/global-load-balancer-operator-bundle:latest
 operator-sdk bundle validate quay.io/$repo/global-load-balancer-operator-bundle:latest --select-optional name=operatorhub
 oc new-project global-load-balancer-operator
+oc label namespace global-load-balancer-operator openshift.io/cluster-monitoring="true"
 operator-sdk cleanup global-load-balancer-operator -n global-load-balancer-operator
 operator-sdk run bundle --install-mode AllNamespaces -n global-load-balancer-operator quay.io/$repo/global-load-balancer-operator-bundle:latest
 ```
 
+### Testing
+
+#### Testing metrics
+
+```sh
+export operatorNamespace=resource-locker-operator-local # or resource-locker-operator
+oc label namespace ${operatorNamespace} openshift.io/cluster-monitoring="true"
+oc rsh -n openshift-monitoring -c prometheus prometheus-k8s-0 /bin/bash
+curl -v -s -k -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://resource-locker-operator-controller-manager-metrics.${operatorNamespace}.svc.cluster.local:8443/metrics
+```
+
 ### Releasing
 
 ```shell