From b091c898255896963a7291602ab15fa0459ed4c3 Mon Sep 17 00:00:00 2001
From: "Kipchumba C. Bett" <kbett68@gmail.com>
Date: Thu, 15 Feb 2024 11:21:21 +0300
Subject: [PATCH 1/4] OZ-438: Use shared GitHub workflows.

---
 .github/workflows/batch-image.yaml          | 75 ---------------------
 .github/workflows/ci.yml                    | 68 +++++++++++++++++++
 .github/workflows/parquet-export-image.yaml | 75 ---------------------
 .github/workflows/setup-connect-image.yaml  | 73 --------------------
 .github/workflows/streaming-image.yaml      | 75 ---------------------
 5 files changed, 68 insertions(+), 298 deletions(-)
 delete mode 100644 .github/workflows/batch-image.yaml
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/parquet-export-image.yaml
 delete mode 100644 .github/workflows/setup-connect-image.yaml
 delete mode 100644 .github/workflows/streaming-image.yaml

diff --git a/.github/workflows/batch-image.yaml b/.github/workflows/batch-image.yaml
deleted file mode 100644
index 59589d1..0000000
--- a/.github/workflows/batch-image.yaml
+++ /dev/null
@@ -1,75 +0,0 @@
----
-name: 'CI batch image'
-
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare
-        id: prep
-        run: |
-          #!/bin/bash
-          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME }}/ozone-flink-jobs-batch
-          VERSION=latest
-          SHORTREF=${GITHUB_SHA::8}
-
-          # If this is git tag, use the tag name as a docker tag
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
-
-          # If the VERSION looks like a version number, assume that
-          # this is the most recent version of the image and also
-          # tag it 'latest'.
-          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
-            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
-          fi
-
-          # Set output parameters.
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=docker_image::${DOCKER_IMAGE}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Get version
-        run: |
-          VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-          echo $VERSION
-          echo ::set-output name=version::$VERSION
-        id: get_version
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: .
-          file: ./Dockerfile_batch
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
-          build-args: JAR_VERSION=${{ steps.get_version.outputs.version }}
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..1468ce9
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,68 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  release:
+    types: [published]
+
+jobs:
+  validate:
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/maven-build-test.yml@main
+    with:
+      java-version: 11
+    secrets:
+      NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+      NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+
+  spotless:
+    needs: validate
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/maven-spotless-check.yml@main
+    with:
+      java-version: 11
+
+  docker-ozone-flink-parquet-export:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    needs: spotless
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
+    with:
+      dockerfile: Dockerfile_parquet_export
+      image-name: "ozone-flink-parquet-export"
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  docker-ozone-flink-jobs-batch:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    needs: spotless
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
+    with:
+      dockerfile: Dockerfile_batch
+      image-name: "ozone-flink-jobs-batch"
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  docker-ozone-flink-jobs-streaming:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    needs: spotless
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
+    with:
+      image-name: "ozone-flink-jobs" # ozone-flink-jobs-streaming
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  docker-ozone-analytics-setup-connect:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    needs: spotless
+    uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
+    with:
+      context: ./development/setup-connect
+      dockerfile: Dockerfile
+      image-name: "ozone-analytics-setup-connect"
+    secrets:
+      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
diff --git a/.github/workflows/parquet-export-image.yaml b/.github/workflows/parquet-export-image.yaml
deleted file mode 100644
index 0bdcb9d..0000000
--- a/.github/workflows/parquet-export-image.yaml
+++ /dev/null
@@ -1,75 +0,0 @@
----
-name: 'CI Parquet Export image'
-
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare
-        id: prep
-        run: |
-          #!/bin/bash
-          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME }}/ozone-flink-parquet-export
-          VERSION=latest
-          SHORTREF=${GITHUB_SHA::8}
-
-          # If this is git tag, use the tag name as a docker tag
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
-
-          # If the VERSION looks like a version number, assume that
-          # this is the most recent version of the image and also
-          # tag it 'latest'.
-          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
-            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
-          fi
-
-          # Set output parameters.
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=docker_image::${DOCKER_IMAGE}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Get version
-        run: |
-          VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-          echo $VERSION
-          echo ::set-output name=version::$VERSION
-        id: get_version
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: .
-          file: ./Dockerfile_parquet_export
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
-          build-args: JAR_VERSION=${{ steps.get_version.outputs.version }}
\ No newline at end of file
diff --git a/.github/workflows/setup-connect-image.yaml b/.github/workflows/setup-connect-image.yaml
deleted file mode 100644
index 65dd930..0000000
--- a/.github/workflows/setup-connect-image.yaml
+++ /dev/null
@@ -1,73 +0,0 @@
----
-name: 'CI Setup Connect'
-
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare
-        id: prep
-        run: |
-          #!/bin/bash
-          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME }}/ozone-analytics-setup-connect
-          VERSION=latest
-          SHORTREF=${GITHUB_SHA::8}
-
-          # If this is git tag, use the tag name as a docker tag
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
-
-          # If the VERSION looks like a version number, assume that
-          # this is the most recent version of the image and also
-          # tag it 'latest'.
-          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
-            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
-          fi
-
-          # Set output parameters.
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=docker_image::${DOCKER_IMAGE}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Get version
-        run: |
-          VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-          echo $VERSION
-          echo ::set-output name=version::$VERSION
-        id: get_version
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: ./development/setup-connect
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
\ No newline at end of file
diff --git a/.github/workflows/streaming-image.yaml b/.github/workflows/streaming-image.yaml
deleted file mode 100644
index 2559d95..0000000
--- a/.github/workflows/streaming-image.yaml
+++ /dev/null
@@ -1,75 +0,0 @@
----
-name: 'CI Streaming image'
-
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare
-        id: prep
-        run: |
-          #!/bin/bash
-          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME }}/ozone-flink-jobs
-          VERSION=latest
-          SHORTREF=${GITHUB_SHA::8}
-
-          # If this is git tag, use the tag name as a docker tag
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
-
-          # If the VERSION looks like a version number, assume that
-          # this is the most recent version of the image and also
-          # tag it 'latest'.
-          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
-            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
-          fi
-
-          # Set output parameters.
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=docker_image::${DOCKER_IMAGE}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Get version
-        run: |
-          VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
-          echo $VERSION
-          echo ::set-output name=version::$VERSION
-        id: get_version
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
-          build-args: JAR_VERSION=${{ steps.get_version.outputs.version }}
\ No newline at end of file

From 4d6e299e9ae7365405620e9944aaa929fc096b07 Mon Sep 17 00:00:00 2001
From: "Kipchumba C. Bett" <kbett68@gmail.com>
Date: Mon, 19 Feb 2024 09:57:59 +0300
Subject: [PATCH 2/4] Optimize dockerfile batch build time

---
 .github/workflows/ci.yml |  2 +-
 Dockerfile_batch         | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1468ce9..a828d11 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
       DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
   docker-ozone-flink-jobs-batch:
-    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    #if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
     needs: spotless
     uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
     with:
diff --git a/Dockerfile_batch b/Dockerfile_batch
index f2d3113..166bdde 100644
--- a/Dockerfile_batch
+++ b/Dockerfile_batch
@@ -1,9 +1,13 @@
 FROM  maven:3.8.6-eclipse-temurin-11 as builder
-# add pom.xml and source code
-ADD ./pom.xml pom.xml
-RUN mvn dependency:go-offline
-ADD ./src src/
-RUN mvn clean install -Pbatch
+
+ENV JAVA_OPTS="-Xmx2g"
+
+# Copy pom.xml and source code
+COPY ./pom.xml pom.xml
+COPY ./src src/
+
+#cache dependencies
+RUN mvn dependency:go-offline && mvn clean install -Pbatch
 
 FROM eclipse-temurin:11-jre
 ARG JAR_VERSION=2.2.0-SNAPSHOT

From c2981b55f29dbdc1911f6a8b5d2e580d1ec8dca4 Mon Sep 17 00:00:00 2001
From: "Kipchumba C. Bett" <kbett68@gmail.com>
Date: Mon, 19 Feb 2024 11:04:11 +0300
Subject: [PATCH 3/4] Set MAVEN_OPTS

---
 Dockerfile_batch | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Dockerfile_batch b/Dockerfile_batch
index 166bdde..dd26fb9 100644
--- a/Dockerfile_batch
+++ b/Dockerfile_batch
@@ -1,13 +1,12 @@
 FROM  maven:3.8.6-eclipse-temurin-11 as builder
 
-ENV JAVA_OPTS="-Xmx2g"
+ENV MAVEN_OPTS="-Xmx2g"
 
 # Copy pom.xml and source code
 COPY ./pom.xml pom.xml
 COPY ./src src/
 
-#cache dependencies
-RUN mvn dependency:go-offline && mvn clean install -Pbatch
+RUN mvn clean install -Pbatch
 
 FROM eclipse-temurin:11-jre
 ARG JAR_VERSION=2.2.0-SNAPSHOT

From 59e151d39f10b20aba88485dfaf685cefdb544fb Mon Sep 17 00:00:00 2001
From: "Kipchumba C. Bett" <kbett68@gmail.com>
Date: Wed, 20 Mar 2024 12:37:52 +0300
Subject: [PATCH 4/4] Enable spotless maven profile

---
 .github/workflows/ci.yml | 3 ++-
 README.md                | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a828d11..3734e65 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -22,6 +22,7 @@ jobs:
     uses: mekomsolutions/shared-github-workflow/.github/workflows/maven-spotless-check.yml@main
     with:
       java-version: 11
+      maven-args: -Pspotless
 
   docker-ozone-flink-parquet-export:
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
@@ -35,7 +36,7 @@ jobs:
       DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
   docker-ozone-flink-jobs-batch:
-    #if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
     needs: spotless
     uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
     with:
diff --git a/README.md b/README.md
index be2a94a..7974f31 100644
--- a/README.md
+++ b/README.md
@@ -183,3 +183,4 @@ The project includes spotless-maven-plugin to enforce code formatting. To format
 ```bash
 mvn clean install -Pspotless
 ```
+