Adding e2e to the CI pipeline

Signed-off-by: dwertent <[email protected]>

.github/workflows/build-chart.yaml

@@ -73,31 +73,84 @@ jobs:
           path: /tmp # download all docker images to /tmp
           pattern: paladin-*
           merge-multiple: true
-      
+
       - name: Load image
         run: |
           docker load --input /tmp/paladin-operator-${{ github.sha }}.tar
           docker load --input /tmp/paladin-${{ github.sha }}.tar
           docker image ls -a  
-      
+        
       # The makefile uses kustomize
       - uses: imranismail/setup-kustomize@v2
 
-      - name: Deploy Operator
+      - name: E2E Test
         run: |
-          ./gradlew deploy \
+          ./gradlew --no-daemon --parallel e2e \
             -PclusterName=${{ env.CLUSTER_NAME }} \
             -Pnamespace=${{ env.NAMESPACE }} \
             -PbuildOperator=${{ inputs.build-operator }} \
             -PbuildPaladin=${{ inputs.build-paladin }} \
             -PoperatorImageName=${{ inputs.operator-image }} \
             -PoperatorImageTag=${{ inputs.operator-image-tag }} \
             -PpaladinImageName=${{ inputs.paladin-image }} \
-            -PpaladinImageTag=${{ inputs.paladin-image-tag }} 
+            -PpaladinImageTag=${{ inputs.paladin-image-tag }} \
+            -PdeleteCluster=false
 
-      - name: Uninstall Operator
+      - name: Port-forward pprof endpoints
+        if: failure()
         run: |
-          ./gradlew clean \
-            -PclusterName=${{ env.CLUSTER_NAME }} \
-            -Pnamespace=${{ env.NAMESPACE }} \
-            -PdeleteCluster=true
+          for i in 1 2 3; do
+            POD_NAME="paladin-node$i-0"
+            echo "Port-forwarding $POD_NAME on port $((6060 + i))"
+            kubectl port-forward pod/$POD_NAME $((6060 + i)):6060 -n ${{ env.NAMESPACE }} &
+          done
+          
+          sleep 10
+
+          for i in 1 2 3; do
+            POD_NAME="paladin-node$i-0"
+            curl "http://localhost:$((6060 + i))/debug/pprof/goroutine?debug=2" > goroutine_dump_node$i.txt
+            curl "http://localhost:$((6060 + i))/debug/javadump"
+          done
+
+      - name: Save logs
+        if: failure()
+        run: |
+          kubectl logs deployment/paladin-operator -n ${{ env.NAMESPACE }} > operator.log
+          kubectl logs statefulset/paladin-node1 -n ${{ env.NAMESPACE }} > sfs-paladin-1.log
+          kubectl logs statefulset/paladin-node2 -n ${{ env.NAMESPACE }} > sfs-paladin-2.log
+          kubectl logs statefulset/paladin-node3 -n ${{ env.NAMESPACE }} > sfs-paladin-3.log
+          kubectl logs statefulset/besu-node1 -n ${{ env.NAMESPACE }} > sfs-besu-1.log
+          kubectl logs statefulset/besu-node2 -n ${{ env.NAMESPACE }} > sfs-besu-2.log
+          kubectl logs statefulset/besu-node3 -n ${{ env.NAMESPACE }} > sfs-besu-3.log
+          kubectl get paladin node1 -n ${{ env.NAMESPACE }} -o json > crd-paladin-1.json
+          kubectl get paladin node2 -n ${{ env.NAMESPACE }} -o json > crd-paladin-2.json
+          kubectl get paladin node3 -n ${{ env.NAMESPACE }} -o json > crd-paladin-3.json
+          kubectl get cm paladin-node1 -n ${{ env.NAMESPACE }} -o json > cm-1.json
+          kubectl get cm paladin-node2 -n ${{ env.NAMESPACE }} -o json > cm-2.json
+          kubectl get cm paladin-node3 -n ${{ env.NAMESPACE }} -o json > cm-3.json
+
+ 
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: paladin-logs
+          path: |
+            operator.log
+            sfs-paladin-1.log
+            sfs-paladin-2.log
+            sfs-paladin-3.log
+            sfs-besu-1.log
+            sfs-besu-2.log
+            sfs-besu-3.log
+            crd-paladin-1.json
+            crd-paladin-2.json
+            crd-paladin-3.json
+            cm-1.json
+            cm-2.json
+            cm-3.json
+            goroutine_dump_node1.txt
+            goroutine_dump_node2.txt
+            goroutine_dump_node3.txt
+            

.github/workflows/paladin-PR-build.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
-
+          
       - name: Install pre-requisites
         uses: ./.github/actions/setup
 
@@ -59,25 +59,25 @@ jobs:
       dockerfile: ./Dockerfile
       registry: paladin.io
       image: paladin
-      image_tag: test
+      image_tag: ${{ github.sha }}
       push: false
       upload-artifact: true
       platforms: linux/amd64
       runs-on: ubuntu-latest
 
   operator-image-build:
-      # run only on PRs
-      if: github.event_name == 'pull_request'
-      uses: ./.github/workflows/build-image.yaml
-      with:
-        dockerfile: ./operator/Dockerfile
-        registry: paladin.io
-        image: paladin-operator
-        image_tag: test
-        push: false
-        upload-artifact: true
-        platforms: linux/amd64
-        runs-on: ubuntu-latest
+    # run only on PRs
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/build-image.yaml
+    with:
+      dockerfile: ./operator/Dockerfile
+      registry: paladin.io
+      image: paladin-operator
+      image_tag: ${{ github.sha }}
+      push: false
+      upload-artifact: true
+      platforms: linux/amd64
+      runs-on: ubuntu-latest
 
   chart-build:
     # run only if pull_request and the path operator/** was changed
@@ -86,9 +86,9 @@ jobs:
     uses: ./.github/workflows/build-chart.yaml
     with:
       paladin-image: paladin.io/paladin
-      paladin-image-tag: test
+      paladin-image-tag: ${{ github.sha }}
       operator-image: paladin.io/paladin-operator
-      operator-image-tag: test
+      operator-image-tag: ${{ github.sha }}
 
   image-release:
     # run only on pushes to main or manual triggers

.github/workflows/release-charts.yaml

@@ -86,18 +86,24 @@ jobs:
           repository: ${{ github.repository_owner }}
           registry: ${{ inputs.registry }}
 
+      - name: Finalize image names
+        shell: bash
+        run: |
+          echo "IMG_PALADIN=${{ inputs.registry }}/${{ steps.docker_repo.outputs.repository }}/paladin" >> $GITHUB_ENV
+          echo "IMG_OPERATOR=${{ inputs.registry }}/${{ steps.docker_repo.outputs.repository }}/paladin-operator" >> $GITHUB_ENV
+      
       - name: Override image tags
         uses: ./.github/actions/update-yaml
         with:
           file: 'operator/charts/paladin-operator/values.yaml'
           updates: |
             operator:
               image:
-                repository: ${{ inputs.registry }}/${{ steps.docker_repo.outputs.repository }}/paladin-operator
+                repository: ${{ env.IMG_OPERATOR }}
                 tag: ${{ inputs.images_tag }}
             paladin:
               image:
-                repository: ${{ inputs.registry }}/${{ steps.docker_repo.outputs.repository }}/paladin
+                repository: ${{ env.IMG_PALADIN }}
                 tag: ${{ inputs.images_tag }}
       
       # Remove 'v' prefix from the tag
@@ -125,7 +131,18 @@ jobs:
           rm -rf charts
           rm Chart.lock
 
-      # TODO: Add e2e tests for the helm chart
+      # e2e tests 
+      - name: E2E Test
+        run: |
+          ./gradlew e2e \
+            -PloadImages=false \
+            -PbuildOperator=false \
+            -PbuildPaladin=false \
+            -PoperatorImageName=${{ env.IMG_OPERATOR }} \
+            -PoperatorImageTag=${{ inputs.images_tag }} \
+            -PpaladinImageName=${{ env.IMG_PALADIN }} \
+            -PpaladinImageTag=${{ inputs.images_tag }} \
+            -PdeleteCluster=true
 
       # set CR_TOKEN env with the GitHub token secret when it is workflow_dispatch action
       - name: Set CR_TOKEN

Dockerfile

@@ -185,8 +185,11 @@ ENV PATH=$PATH:/usr/local/java/bin
 # Define the entry point for running the application
 ENTRYPOINT [                         \
     "java",                          \
+    "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", \
+    "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", \
+    "--add-opens", "java.base/java.nio=ALL-UNNAMED", \
+    "-Dio.netty.tryReflectionSetAccessible=true", \
     "-Djna.library.path=/app/libs",  \
     "-jar",                          \
     "/app/libs/paladin.jar"          \
 ]
-

config/pkg/pldconf/httpclient.go

@@ -16,14 +16,25 @@
 
 package pldconf
 
+import (
+	"github.com/kaleido-io/paladin/config/pkg/confutil"
+)
+
 type HTTPBasicAuthConfig struct {
 	Username string `json:"username"`
 	Password string `json:"password"`
 }
 
 type HTTPClientConfig struct {
-	URL         string                 `json:"url"`
-	HTTPHeaders map[string]interface{} `json:"httpHeaders"`
-	Auth        HTTPBasicAuthConfig    `json:"auth"`
-	TLS         TLSConfig              `json:"tls"`
+	URL               string                 `json:"url"`
+	HTTPHeaders       map[string]interface{} `json:"httpHeaders"`
+	Auth              HTTPBasicAuthConfig    `json:"auth"`
+	TLS               TLSConfig              `json:"tls"`
+	RequestTimeout    *string                `json:"requestTimeout,omitempty"`
+	ConnectionTimeout *string                `json:"connectionTimeout,omitempty"`
+}
+
+var DefaultHTTPConfig = &HTTPClientConfig{
+	ConnectionTimeout: confutil.P("30s"),
+	RequestTimeout:    confutil.P("30s"),
 }

core/go/pkg/bootstrap/instance.go

@@ -17,6 +17,8 @@ package bootstrap
 
 import (
 	"context"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"strconv"
@@ -78,6 +80,16 @@ func (i *instance) signalHandler() {
 	}
 }
 
+func (i *instance) startProfiler() {
+	go func() {
+		address := "localhost:6060"
+		log.L(i.ctx).Infof("Starting profiler on '%s'", address)
+		if err := http.ListenAndServe(address, nil); err != nil {
+			log.L(i.ctx).Errorf("Profiler failed: %s", err)
+		}
+	}()
+}
+
 func (i *instance) run() RC {
 	defer func() {
 		close(i.done)
@@ -111,6 +123,11 @@ func (i *instance) run() RC {
 	// From this point need to ensure we stop the component manager
 	defer cm.Stop()
 
+	// Notice: The logger is initialized in the ManagerFactory constructor
+	if log.IsDebugEnabled() {
+		i.startProfiler()
+	}
+
 	// Start it up
 	err = cm.Init()
 	if err == nil {

core/java/src/main/java/io/kaleido/paladin/Main.java

@@ -19,15 +19,14 @@
 import io.kaleido.paladin.configlight.RuntimeInfo;
 import io.kaleido.paladin.configlight.YamlConfig;
 import io.kaleido.paladin.loader.PluginLoader;
-import org.apache.logging.log4j.LogManager;
+import io.kaleido.paladin.logging.PaladinLogging;
 import org.apache.logging.log4j.Logger;
 
-import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 
 public class Main {
 
-    private static final Logger LOGGER = LogManager.getLogger(Main.class);
+    private static final Logger LOGGER = PaladinLogging.getLogger(Main.class);
 
     static CoreJNA.PaladinGo paladinGo;
 
@@ -50,6 +49,7 @@ private static synchronized RuntimeInfo setRunning(RuntimeInfo newInstance) {
     }
 
     public static synchronized void stop() {
+        LOGGER.info("Stopping paladin");
         final RuntimeInfo runningInstance = instance;
         if (runningInstance != null) {
             CompletableFuture.runAsync(() -> ensureLoaded().Stop());
@@ -72,7 +72,7 @@ public static int run(String[] args) {
         PluginLoader loader = null;
 
         if (args.length < 2) {
-            throw new Error("usage: <config.paladin.yaml> <node|testbed>");
+            throw new Error("usage: <config.paladin.yaml> <engine|testbed>");
         }
         try {
             final String configFile = args[0];
@@ -85,13 +85,16 @@ public static int run(String[] args) {
 
             loader = new PluginLoader(runtimeInfo.socketFilename(), runtimeInfo.instanceId());
 
-            return ensureLoaded().Run(
+            var rc = ensureLoaded().Run(
                     runtimeInfo.socketFilename(),
                     runtimeInfo.instanceId().toString(),
                     configFile,
                     engineName
             );
+            LOGGER.error("Paladin shutting down with rc={}", rc);
+            return rc;
         } catch(Throwable e) {
+            LOGGER.error("Paladin shutting down due to error", e);
             throw new RuntimeException(e.getMessage(), e);
         } finally {
             if (loader != null) {