changes for ha

.github/workflows/release.yaml

@@ -43,7 +43,7 @@ jobs:
 
       - name: "Sign and Publish the image"
         run: |
-          zarf package publish build/zarf-init-amd64-v${ZARF_VERSION}.tar.zst oci://ghcr.io/radiusmethod/zarf-init-bigbang --signing-key awskms:///alias/zarf-init-bigbang
+          zarf package publish build/zarf-init-amd64-v${ZARF_VERSION}.tar.zst oci://ghcr.io/radiusmethod/zarf-init-bigbang-k3s-ha --signing-key awskms:///alias/zarf-init-bigbang
         env:
           AWS_REGION: ${{ secrets.AWS_REGION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_GOV }}

README.md

@@ -1,4 +1,6 @@
-# zarf-init-bigbang
+# zarf-init-bigbang-k3s-ha
+
+> ⚠️This has a custom k3s package that allows for deploying in an HA configuration.  This repository is implementing what could be merged into the upstream Zarf repository [here](https://github.com/defenseunicorns/zarf/pull/2495),
 
 This package uses Iron Bank images for as many things as possible.  This package also assumes you will use a logging stack from within Big Bang.
 

k3s/common/k3s.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Zarf K3s Runner
+Documentation=https://zarf.dev
+Wants=network-online.target
+After=network-online.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service'
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/sbin/k3s server --write-kubeconfig-mode=700 --write-kubeconfig /root/.kube/config ###ZARF_VAR_K3S_ARGS###

k3s/common/zarf-clean-k3s.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+
+echo -e '\033[0;31m'
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+	[ -d $bin ] && export PATH=$PATH:$bin:$bin/aux
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+	[ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+	[ -x $service ] && $service stop
+done
+
+pschildren() {
+	ps -e -o ppid= -o pid= | \
+	sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+	grep -w "^$1" | \
+	cut -f2
+}
+
+pstree() {
+	for pid in $@; do
+		echo $pid
+		for child in $(pschildren $pid); do
+			pstree $child
+		done
+	done
+}
+
+killtree() {
+	kill -9 $(
+		{ set +x; } 2>/dev/null;
+		pstree $@;
+		set -x;
+	) 2>/dev/null
+}
+
+getshims() {
+	ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
+}
+
+killtree $({ set +x; } 2>/dev/null; getshims; set -x)
+
+do_unmount_and_remove() {
+	awk -v path="$1" '$2 ~ ("^" path) { print $2 }' /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"'
+}
+
+do_unmount_and_remove '/run/k3s'
+do_unmount_and_remove '/var/lib/rancher/k3s'
+do_unmount_and_remove '/var/lib/kubelet/pods'
+do_unmount_and_remove '/var/lib/kubelet/plugins'
+do_unmount_and_remove '/run/netns/cni-'
+
+# Remove CNI namespaces
+ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
+
+# Delete network interface(s) that match 'master cni0'
+ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+	iface=${iface%%@*}
+	[ -z "$iface" ] || ip link delete $iface
+done
+ip link delete cni0
+ip link delete flannel.1
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
+
+if command -v systemctl; then
+	systemctl disable k3s
+	systemctl reset-failed k3s
+	systemctl daemon-reload
+fi
+
+rm -f /etc/systemd/system/k3s.service
+
+for cmd in kubectl crictl ctr; do
+	if [ -L /usr/sbin/$cmd ]; then
+		rm -f /usr/sbin/$cmd
+	fi
+done
+
+rm -rf /etc/rancher/k3s
+rm -rf /run/k3s
+rm -rf /run/flannel
+rm -rf /var/lib/rancher/k3s
+rm -rf /var/lib/kubelet
+rm -f /usr/sbin/k3s
+rm -f /usr/sbin/ctr
+rm -f /usr/sbin/crictl
+rm -f /usr/sbin/kubectl
+rm -f /opt/zarf/k3s-remove.sh
+rm -fr zarf-pki
+
+echo -e '\033[0m'

k3s/common/zarf.yaml

@@ -0,0 +1,55 @@
+kind: ZarfInitConfig
+metadata:
+  name: distro-k3s
+
+variables:
+  - name: K3S_ARGS
+    description: Arguments to pass to K3s
+    default: --disable traefik
+
+components:
+  - name: k3s
+    only:
+      localOS: linux
+    description: >
+      *** REQUIRES ROOT (not sudo) ***
+      Install K3s, a certified Kubernetes distribution built for IoT & Edge computing.
+      K3s provides the cluster need for Zarf running in Appliance Mode as well as can
+      host a low-resource Gitops Service if not using an existing Kubernetes platform.
+    actions:
+      onDeploy:
+        defaults:
+          maxRetries: 5
+        before:
+          - cmd: ./zarf internal is-valid-hostname
+            maxRetries: 0
+            description: Check if the current system has a, RFC1123 compliant hostname
+          # If running RHEL variant, disable firewalld
+          # https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-red-hat-centos-enterprise-linux
+          # NOTE: The empty echo prevents infinite retry loops on non-RHEL systems where the exit code would be an error
+          - cmd: "[ -e /etc/redhat-release ] && systemctl disable firewalld --now || echo ''"
+            description: If running a RHEL variant, disable 'firewalld' per k3s docs
+        after:
+          # Configure K3s systemd service
+          - cmd: systemctl daemon-reload
+            description: Reload the system services
+          - cmd: systemctl enable k3s
+            description: Enable 'k3s' to run at system boot
+          - cmd: systemctl restart k3s
+            description: Start the 'k3s' system service
+      onRemove:
+        before:
+          - cmd: /opt/zarf/zarf-clean-k3s.sh
+            description: Remove 'k3s' from the system
+          - cmd: rm /opt/zarf/zarf-clean-k3s.sh
+            description: Remove the cleanup script
+    files:
+      # K3s removal script
+      - source: zarf-clean-k3s.sh
+        target: /opt/zarf/zarf-clean-k3s.sh
+        executable: true
+      # The K3s systemd service definition
+      - source: k3s.service
+        target: /etc/systemd/system/k3s.service
+        symlinks:
+          - /etc/systemd/system/multi-user.target.wants/k3s.service

k3s/zarf.yaml

@@ -0,0 +1,65 @@
+kind: ZarfInitConfig
+metadata:
+  name: distro-k3s
+  description: Used to establish a new Zarf cluster
+
+components:
+  # AMD-64 version of the K3s stack
+  - name: k3s
+    import:
+      path: common
+      name: k3s
+    only:
+      cluster:
+        architecture: amd64
+    files:
+      # Include the actual K3s binary
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s
+        shasum: 9014535a4cd20c788282d60398a06279983562093455b53ab76701539ce67acf
+        target: /usr/sbin/k3s
+        executable: true
+        # K3s magic provides these tools when symlinking
+        symlinks:
+          - /usr/sbin/kubectl
+          - /usr/sbin/ctr
+          - /usr/sbin/crictl
+      # Transfer the K3s images for containerd to pick them up
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-airgap-images-amd64.tar.zst
+        shasum: bc4d05bad56a583c80ff443d60e8277a136cc4357dc8527702d38b5cca28880d
+        target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst
+    actions:
+      onDeploy:
+        before:
+          - cmd: if [ "$(uname -m)" != "x86_64" ]; then echo "this package architecture is amd64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi
+            description: Check that the host architecture matches the package architecture
+            maxRetries: 0
+
+  # ARM-64 version of the K3s stack
+  - name: k3s
+    import:
+      path: common
+      name: k3s
+    only:
+      cluster:
+        architecture: arm64
+    files:
+      # Include the actual K3s binary
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-arm64
+        shasum: 1ae72ca06d3302f3e86ef92e6e8f84e14a084da69564e87d6e2e75f62e72388d
+        target: /usr/sbin/k3s
+        executable: true
+        # K3s magic provides these tools when symlinking
+        symlinks:
+          - /usr/sbin/kubectl
+          - /usr/sbin/ctr
+          - /usr/sbin/crictl
+      # Transfer the K3s images for containerd to pick them up
+      - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-airgap-images-arm64.tar.zst
+        shasum: 50621ae1391aec7fc66ca66a46a0e9fd48ce373a58073000efdc278233adc64b
+        target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst
+    actions:
+      onDeploy:
+        before:
+          - cmd: if [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "arm64" ]; then echo "this package architecture is arm64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi
+            description: Check that the host architecture matches the package architecture
+            maxRetries: 0

zarf.yaml

@@ -6,8 +6,7 @@ metadata:
 components:
   - name: k3s
     import:
-      # renovate: depName=defenseunicorns/zarf
-      url: oci://ghcr.io/defenseunicorns/packages/init:v0.33.2
+      path: k3s
 
   - name: zarf-injector
     required: true