feat: upgrade Apache NiFi to v2.1.0

deployment/nifi/ssl-configmap.yml → deployment/nifi/configmap-ssl.yml

@@ -9,30 +9,31 @@ data:
 
     ORGANISATION_UNIT=${ORGANISATION_UNIT:-'Cloud Services Application'}
     ORGANISATION=${ORGANISATION:-'Cloud Services'}
-    PUBLIC_DNS=${POD_NAME:-'nifi.tld'}
+    PUBLIC_DNS=${NIFI_CLUSTER_NODE_ADDRESS:-'nifi.tld'}
     CITY=${CITY:-'London'}
     STATE=${STATE:-'London'}
     COUNTRY_CODE=${COUNTRY_CODE:-'GB'}
-    KEY_PASS=${KEY_PASS:-$KEYSTORE_PASS}
+    KEY_PASS=${NIFI_SENSITIVE_PROPS_KEY:-$KEYSTORE_PASS}
     KEYSTORE_PASS=${KEYSTORE_PASS:-$NIFI_SENSITIVE_PROPS_KEY}
     KEYSTORE_PASSWORD=${KEYSTORE_PASSWORD:-$NIFI_SENSITIVE_PROPS_KEY}
     KEYSTORE_PATH=${NIFI_HOME}/keytool/keystore.p12
-    KEYSTORE_TYPE=jks
-    TRUSTSTORE_PASS=${TRUSTSTORE_PASS:-$NIFI_SENSITIVE_PROPS_KEY}
+    KEYSTORE_TYPE=pkcs12
+    TRUSTSTORE_PASS=${TRUSTSTORE_PASSWORD:-$NIFI_SENSITIVE_PROPS_KEY}
     TRUSTSTORE_PASSWORD=${TRUSTSTORE_PASSWORD:-$NIFI_SENSITIVE_PROPS_KEY}
-    TRUSTSTORE_PATH=${NIFI_HOME}/keytool/truststore.jks
+    TRUSTSTORE_PATH=${NIFI_HOME}/keytool/truststore.p12
     TRUSTSTORE_TYPE=pkcs12
 
     if [[ ! -f "${NIFI_HOME}/keytool/keystore.p12" ]]
     then
       echo "Creating keystore"
       keytool -genkey -noprompt -alias nifi-keystore \
-      -dname "CN=${PUBLIC_DNS},OU=${ORGANISATION_UNIT},O=${ORGANISATION},L=${CITY},S=${STATE},C=${COUNTRY_CODE}" \
+      -dname "CN=${NIFI_CLUSTER_NODE_ADDRESS},OU=${ORGANISATION_UNIT},O=${ORGANISATION},L=${CITY},S=${STATE},C=${COUNTRY_CODE}" \
       -keystore ${NIFI_HOME}/keytool/keystore.p12 \
       -storepass ${KEYSTORE_PASS:-$NIFI_SENSITIVE_PROPS_KEY} \
-      -KeySize 2048 \
+      -keysize 2048 \
       -keypass "${KEY_PASS:-$NIFI_SENSITIVE_PROPS_KEY}" \
       -keyalg RSA \
+      -ext "SAN=dns:localhost,dns:${NIFI_CLUSTER_NODE_ADDRESS},dns:${NIFI_LOAD_BALANCER},dns:${PUBLIC_DNS},ip:${POD_IP},ip:127.0.0.1,dns:${NIFI_WEB_HTTP_HOST},dns:${POD_NAME},dns:${NODE_IDENTITY}" \
       -storetype pkcs12
 
       echo "Exporting the self-signed certificate from the keystore"
@@ -43,9 +44,9 @@ data:
       -rfc
     fi
 
-    if [[ ! -f "${NIFI_HOME}/keytool/truststore.jks" ]]
+    if [[ ! -f "${NIFI_HOME}/keytool/truststore.p12" ]]
     then
-      echo "Creating truststore"
+      echo "Importing truststore"
       keytool -import -noprompt -alias nifi-truststore \
       -file "${NIFI_HOME}/keytool/nifi-cert.cer" \
       -keystore "${TRUSTSTORE_PATH}" \
@@ -55,4 +56,4 @@ data:
     fi
 
     #/usr/bin/bash ${NIFI_HOME}/../scripts/secure.sh 
-    #eval ${NIFI_HOME}/../scripts/secure.sh 
+    # eval ${NIFI_HOME}/../scripts/secure.sh

deployment/nifi/configmap.yml

@@ -3,12 +3,11 @@ kind: ConfigMap
 metadata:
   name: nifi-cm
 data:
-  JAVA_OPTS: "-XX:UseAVX=0 -Djavax.net.debug=ssl,handshake"
+  JAVA_OPTS: "-XX:UseAVX=0 -Djavax.net.debug=ssl,handshake,address=8000"
   KEYSTORE_PASSWORD: "th1s1s3up34e5r37"
-  KEYSTORE_PATH: "${NIFI_HOME}/keytool/keystore.p12"
   KEYSTORE_TYPE: "PKCS12"
   NIFI_ANALYTICS_PREDICT_ENABLED: "true"
-  NIFI_CLUSTER_IS_NODE: "true"
+  NIFI_CLUSTER_IS_NODE: "false"
   # NIFI_CLUSTER_LOAD_BALANCE_HOST: "nifi"
   NIFI_CLUSTER_NODE_CONNECTION_TIMEOUT: "5 min"
   NIFI_CLUSTER_NODE_EVENT_HISTORY_SIZE: "25"
@@ -34,5 +33,4 @@ data:
   NIFI_ZK_CONNECT_STRING: "zookeeper:2181"
   NIFI_ZOOKEEPER_CONNECT_STRING: "zookeeper:2181"
   TRUSTSTORE_PASSWORD: "th1s1s3up34e5r37"
-  TRUSTSTORE_PATH: "${NIFI_HOME}/keytool/truststore.jks"
   TRUSTSTORE_TYPE: "pkcs12"

deployment/nifi/kustomization.yml

@@ -8,7 +8,7 @@ commonAnnotations:
 resources:
 - ingress.yml
 - configmap.yml
-- ssl-configmap.yml
+- configmap-ssl.yml
 - nifi.yml
 - hpa.yml
 - secrets.yml
@@ -17,7 +17,7 @@ resources:
 images:
 - name: nifi
   newName: docker.io/apache/nifi
-  newTag: "1.24.0"
+  newTag: "2.1.0"
 - name: wait
   newName: docker.io/busybox
-  newTag: "1.36"
+  newTag: "1.37"

deployment/nifi/nifi.yml

@@ -64,7 +64,7 @@ spec:
                 - ALL
       containers:
       - name: nifi
-        image: docker.io/apache/nifi:1.28.1
+        image: docker.io/apache/nifi:2.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8080
@@ -88,10 +88,16 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace # Use pod namespace
+          - name: NIFI_LOAD_BALANCER
+            value: "nifi.$(POD_NAMESPACE).svc.cluster.local" # Use nifi service fqdn
+          - name: NIFI_HOME
+            value: "/opt/nifi/nifi-current"
           - name: NIFI_UI_BANNER_TEXT
             value: $(POD_NAME) # Use pod name for banner
           - name: NIFI_WEB_HTTP_HOST
             value: $(POD_NAME).nifi.$(POD_NAMESPACE).svc.cluster.local # Use pod fqdn as web host
+          - name: NIFI_WEB_HTTPS_HOST
+            value: $(POD_NAME).nifi.$(POD_NAMESPACE).svc.cluster.local # Use pod fqdn as web host
           - name: NIFI_CLUSTER_NODE_ADDRESS
             value: $(POD_NAME).nifi.$(POD_NAMESPACE).svc.cluster.local # Use pod fqdn as node address
           - name: NIFI_REMOTE_INPUT_SOCKET_HOST
@@ -100,8 +106,16 @@ spec:
             value: $(POD_NAME).nifi.$(POD_NAMESPACE).svc.cluster.local # Use pod fqdn as input host address
           - name: HOSTNAME
             value: $(POD_IP) # Use pod ip as hostname
+          # - name: NIFI_WEB_PROXY_HOST
+          #   value: $(POD_NAME).nifi.$(POD_NAMESPACE).svc.cluster.local # Use pod fqdn as input host address
           - name: NODE_IDENTITY
             value: $(POD_NAME) # Use pod name as identity
+          - name: NIFI_CLUSTER_NODE_PROTOCOL_ADDRESS
+            value: $(POD_IP) # Use pod ip as hostname
+          - name: KEYSTORE_PATH
+            value: "$(NIFI_HOME)/keytool/keystore.p12"
+          - name: TRUSTSTORE_PATH
+            value: "$(NIFI_HOME)/keytool/truststore.p12"
         envFrom:
           - configMapRef:
               name: nifi-cm
@@ -137,7 +151,7 @@ spec:
         readinessProbe:
           tcpSocket:
             port: cluster
-          initialDelaySeconds: 60
+          initialDelaySeconds: 90
           periodSeconds: 30
           timeoutSeconds: 20
           failureThreshold: 3

deployment/zookeeper-cluster/zookeeper.yml

@@ -29,7 +29,7 @@ spec:
       containers:
       - name: zookeeper
         imagePullPolicy: Always
-        image: "registry.k8s.io/kubernetes-zookeeper:1.0-3.4.10"
+        image: "docker.io/apache/nifi:2.1.0"
         ports:
         - containerPort: 2181
           name: client

deployment/zookeeper/kustomization.yml

@@ -13,4 +13,4 @@ resources:
 images:
 - name: zookeeper
   newName: docker.io/zookeeper
-  newTag: "3.8"
+  newTag: "3.9"