Apache spark images automated build pipeline

.build/ci-versions.yml

@@ -0,0 +1,51 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+########### ~~~ CI TEST MATRIX VERSIONS ~~~ ######################################################################################
+########### DEFINE A CI REFERENCE COMBINATIONS TO TEST AND PREVENT TESTING ALL THE COMBINATION WITCH LEADS TO TAKE A LOT OF TIME #
+#### PUT THE SPARK VERSIONS TO TEST IN CORRESPONDANCE WITH 'reference-versions.yml' FILE #########################################
+#### !!! ANY DECLARED TEST VERSION WHICH IS NOT PRESENT IN 'reference-versions.yml' FILE IS SKIPPED DURING BUILD !!! #############
+#### REMOVE, UPDATE OR ADD VERSIONS TO TEST ######################################################################################
+versions:
+  # Maximum python version supported by spark-3.2.x: 3.9
+  # Java support: 8/11
+  - python_version: 3.9
+    spark_version: [3.2.4]
+    java_version: [11]
+    scala_version: [2.12]
+    hadoop_version: 3.2
+  # Maximum python version supported by spark-3.3.x: 3.10
+  # Java support: 8/11/17
+  - python_version: '3.10'
+    spark_version: [3.3.4]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # Maximum python version supported by spark-3.4.x: 3.11
+  # Java support: 8/11/17
+  - python_version: 3.11
+    spark_version: [3.4.2]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # https://spark.apache.org/releases/spark-release-3-5-0.html
+  # Minimum supported java version: 17/21
+  - python_version: 3.11
+    spark_version: [3.5.1]
+    java_version: [17]
+    scala_version: [2.13]
+    hadoop_version: 3
+

.build/images.yml

@@ -0,0 +1,60 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+images:
+  - name: docker.io/eclipse-temurin
+    tags:
+      - ${java_version}-jre-jammy
+  - name: spark-base
+    dependsOn: docker.io/eclipse-temurin
+    tags:
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-${git_release_version}
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')-${git_release_version}
+      #- spark-${spark_version}-scala-${scala_version}-java-${java_version}-${git_commit_short_sha}
+  - name: spark
+    dependsOn: spark-base
+    tags:
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-${git_release_version}
+      - spark-${spark_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')-${git_release_version}
+      #- spark-${spark_version}-scala-${scala_version}-java-${java_version}-${git_commit_short_sha}
+  - name: spark-py
+    dependsOn: spark
+    tags:
+      - spark-${spark_version}-python-${python_version}-scala-${scala_version}-java-${java_version}
+      - spark-${spark_version}-python-${python_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')
+      - spark-${spark_version}-python-${python_version}-scala-${scala_version}-java-${java_version}-${git_release_version}
+      - spark-${spark_version}-python-${python_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')-${git_release_version}
+      #- spark-${spark_version}-python-${python_version}-scala-${scala_version}-java-${java_version}-${git_commit_short_sha}
+  - name: spark-r
+    dependsOn: spark
+    tags:
+      - spark-${spark_version}-r-${r_version}-scala-${scala_version}-java-${java_version}
+      - spark-${spark_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')
+      - spark-${spark_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-${git_release_version}
+      - spark-${spark_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')-${git_release_version}
+      #- spark-${spark_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-${git_commit_short_sha}
+  - name: spark-py-r
+    dependsOn: spark-py
+    tags:
+      - spark-${spark_version}-python-${python_version}-r-${r_version}-scala-${scala_version}-java-${java_version}
+      - spark-${spark_version}-python-${python_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')
+      - spark-${spark_version}-python-${python_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-${git_release_version}
+      - spark-${spark_version}-python-${python_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-$(date '+%Y-%m-%d')-${git_release_version}
+      #- spark-${spark_version}-python-${python_version}-r-${r_version}-scala-${scala_version}-java-${java_version}-${git_commit_short_sha}

.build/reference-versions.yml

@@ -0,0 +1,50 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### REFERENCE MATRIX VERSIONS ##############################
+#### !!! DOT NOT DELETE ANY ELEMENT !!! ####################
+######## APPEND ONLY WHEN NEW SPARK VERSION IS REALEASED ###
+############ USED AS REFERENCE DURING BUILD ################
+versions:
+  # Maximum python version supported by spark-3.2.x: 3.9
+  # Java support: 8/11
+  - python_version: 3.9
+    spark_version: [3.2.1, 3.2.2, 3.2.3, 3.2.4]
+    java_version: [11]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3.2
+  # Maximum python version supported by spark-3.3.x: 3.10
+  # Java support: 8/11/17
+  - python_version: '3.10'
+    spark_version: [3.3.1, 3.3.2, 3.3.3, 3.3.4]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # Maximum python version supported by spark-3.4.x: 3.11
+  # Java support: 8/11/17
+  - python_version: 3.11
+    spark_version: [3.4.1, 3.4.2]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # https://spark.apache.org/releases/spark-release-3-5-0.html
+  # Minimum supported java version: 17/21
+  - python_version: 3.11
+    spark_version: [3.5.1]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+

.build/release-versions.yml

@@ -0,0 +1,50 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+########### CURRENT MATRIX VERSIONS ################################################################################
+#### PUT THE SPARK VERSIONS TO BUILD IN CORRESPONDANCE WITH 'reference-versions.yml' FILE ##########################
+#### !!! ANY DECLARED VERSION WHICH IS NOT PRESENT IN 'reference-versions.yml' FILE IS SKIPPED DURING BUILD !!! ####
+#### REMOVE, UPDATE OR ADD VERSIONS ################################################################################
+versions:
+  # Maximum python version supported by spark-3.2.x: 3.9
+  # Java support: 8/11
+  - python_version: 3.9
+    spark_version: [3.2.1, 3.2.2, 3.2.3, 3.2.4]
+    java_version: [11]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3.2
+  # Maximum python version supported by spark-3.3.x: 3.10
+  # Java support: 8/11/17
+  - python_version: '3.10'
+    spark_version: [3.3.1, 3.3.2, 3.3.3, 3.3.4]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # Maximum python version supported by spark-3.4.x: 3.11
+  # Java support: 8/11/17
+  - python_version: 3.11
+    spark_version: [3.4.1, 3.4.2]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+  # https://spark.apache.org/releases/spark-release-3-5-0.html
+  # Minimum supported java version: 17/21
+  - python_version: 3.11
+    spark_version: [3.5.1]
+    java_version: [17]
+    scala_version: [2.12, 2.13]
+    hadoop_version: 3
+

.github/actions/free-disk-space/action.yml

@@ -0,0 +1,40 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Free disk space
+description: Free Github runnner disk space
+
+runs:
+  using: composite
+  steps:
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
+
+
+

.github/actions/setup-buildx/action.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Set up QEMU and Docker Buildx
+description: Set up Docker Buildx
+
+runs:
+  using: composite
+  steps:
+    - name: Set up QEMU 📦
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx 📦
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: network=host

.github/actions/setup-kind/action.yaml

@@ -0,0 +1,37 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Setup kind
+description: Deploy kind cluster
+
+runs:
+  using: composite
+  steps:
+    - name: Create k8s Kind Cluster
+      uses: helm/kind-action@v1
+      with:
+        # https://github.com/helm/kind-action?tab=readme-ov-file#inputs
+        verbosity: 10
+        cluster_name: "kind-ci-${{ github.job }}"
+        ignore_failed_clean: true # Ignore the post delete cluster action failing
+        wait: "180s" # Max timeout to wait Kind becomes ready
+
+    - name: Print Kind cluster state
+      run: |
+        kubectl cluster-info
+        kubectl get pods -A
+        kubectl describe node
+      shell: bash