diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 55a80bb..3866ac5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -43,7 +43,7 @@ jobs: - name: "Sign and Publish the image" run: | - zarf package publish build/zarf-init-amd64-v${ZARF_VERSION}.tar.zst oci://ghcr.io/radiusmethod/zarf-init-bigbang --signing-key awskms:///alias/zarf-init-bigbang + zarf package publish build/zarf-init-amd64-v${ZARF_VERSION}.tar.zst oci://ghcr.io/radiusmethod/zarf-init-bigbang-k3s-ha --signing-key awskms:///alias/zarf-init-bigbang env: AWS_REGION: ${{ secrets.AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_GOV }} diff --git a/README.md b/README.md index 4c24ea6..894166b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# zarf-init-bigbang +# zarf-init-bigbang-k3s-ha + +> ⚠️This has a custom k3s package that allows for deploying in an HA configuration. This repository is implementing what could be merged into the upstream Zarf repository [here](https://github.com/defenseunicorns/zarf/pull/2495), This package uses Iron Bank images for as many things as possible. This package also assumes you will use a logging stack from within Big Bang. diff --git a/k3s/common/k3s.service b/k3s/common/k3s.service new file mode 100644 index 0000000..7c9cdbe --- /dev/null +++ b/k3s/common/k3s.service @@ -0,0 +1,28 @@ +[Unit] +Description=Zarf K3s Runner +Documentation=https://zarf.dev +Wants=network-online.target +After=network-online.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=notify +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/sbin/k3s server --write-kubeconfig-mode=700 --write-kubeconfig /root/.kube/config ###ZARF_VAR_K3S_ARGS### diff --git a/k3s/common/zarf-clean-k3s.sh b/k3s/common/zarf-clean-k3s.sh new file mode 100644 index 0000000..14204dc --- /dev/null +++ b/k3s/common/zarf-clean-k3s.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +echo -e '\033[0;31m' + +for bin in /var/lib/rancher/k3s/data/**/bin/; do + [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux +done + +set -x + +for service in /etc/systemd/system/k3s*.service; do + [ -s $service ] && systemctl stop $(basename $service) +done + +for service in /etc/init.d/k3s*; do + [ -x $service ] && $service stop +done + +pschildren() { + ps -e -o ppid= -o pid= | \ + sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ + grep -w "^$1" | \ + cut -f2 +} + +pstree() { + for pid in $@; do + echo $pid + for child in $(pschildren $pid); do + pstree $child + done + done +} + +killtree() { + kill -9 $( + { set +x; } 2>/dev/null; + pstree $@; + set -x; + ) 2>/dev/null +} + +getshims() { + ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 +} + +killtree $({ set +x; } 2>/dev/null; getshims; set -x) + +do_unmount_and_remove() { + awk -v path="$1" '$2 ~ ("^" path) { print $2 }' /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' +} + +do_unmount_and_remove '/run/k3s' +do_unmount_and_remove '/var/lib/rancher/k3s' +do_unmount_and_remove '/var/lib/kubelet/pods' +do_unmount_and_remove '/var/lib/kubelet/plugins' +do_unmount_and_remove '/run/netns/cni-' + +# Remove CNI namespaces +ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete + +# Delete network interface(s) that match 'master cni0' +ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do + iface=${iface%%@*} + [ -z "$iface" ] || ip link delete $iface +done +ip link delete cni0 +ip link delete flannel.1 +rm -rf /var/lib/cni/ +iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore + +if command -v systemctl; then + systemctl disable k3s + systemctl reset-failed k3s + systemctl daemon-reload +fi + +rm -f /etc/systemd/system/k3s.service + +for cmd in kubectl crictl ctr; do + if [ -L /usr/sbin/$cmd ]; then + rm -f /usr/sbin/$cmd + fi +done + +rm -rf /etc/rancher/k3s +rm -rf /run/k3s +rm -rf /run/flannel +rm -rf /var/lib/rancher/k3s +rm -rf /var/lib/kubelet +rm -f /usr/sbin/k3s +rm -f /usr/sbin/ctr +rm -f /usr/sbin/crictl +rm -f /usr/sbin/kubectl +rm -f /opt/zarf/k3s-remove.sh +rm -fr zarf-pki + +echo -e '\033[0m' diff --git a/k3s/common/zarf.yaml b/k3s/common/zarf.yaml new file mode 100644 index 0000000..0a2b6b2 --- /dev/null +++ b/k3s/common/zarf.yaml @@ -0,0 +1,55 @@ +kind: ZarfInitConfig +metadata: + name: distro-k3s + +variables: + - name: K3S_ARGS + description: Arguments to pass to K3s + default: --disable traefik + +components: + - name: k3s + only: + localOS: linux + description: > + *** REQUIRES ROOT (not sudo) *** + Install K3s, a certified Kubernetes distribution built for IoT & Edge computing. + K3s provides the cluster need for Zarf running in Appliance Mode as well as can + host a low-resource Gitops Service if not using an existing Kubernetes platform. + actions: + onDeploy: + defaults: + maxRetries: 5 + before: + - cmd: ./zarf internal is-valid-hostname + maxRetries: 0 + description: Check if the current system has a, RFC1123 compliant hostname + # If running RHEL variant, disable firewalld + # https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-red-hat-centos-enterprise-linux + # NOTE: The empty echo prevents infinite retry loops on non-RHEL systems where the exit code would be an error + - cmd: "[ -e /etc/redhat-release ] && systemctl disable firewalld --now || echo ''" + description: If running a RHEL variant, disable 'firewalld' per k3s docs + after: + # Configure K3s systemd service + - cmd: systemctl daemon-reload + description: Reload the system services + - cmd: systemctl enable k3s + description: Enable 'k3s' to run at system boot + - cmd: systemctl restart k3s + description: Start the 'k3s' system service + onRemove: + before: + - cmd: /opt/zarf/zarf-clean-k3s.sh + description: Remove 'k3s' from the system + - cmd: rm /opt/zarf/zarf-clean-k3s.sh + description: Remove the cleanup script + files: + # K3s removal script + - source: zarf-clean-k3s.sh + target: /opt/zarf/zarf-clean-k3s.sh + executable: true + # The K3s systemd service definition + - source: k3s.service + target: /etc/systemd/system/k3s.service + symlinks: + - /etc/systemd/system/multi-user.target.wants/k3s.service diff --git a/k3s/zarf.yaml b/k3s/zarf.yaml new file mode 100644 index 0000000..0813b1e --- /dev/null +++ b/k3s/zarf.yaml @@ -0,0 +1,65 @@ +kind: ZarfInitConfig +metadata: + name: distro-k3s + description: Used to establish a new Zarf cluster + +components: + # AMD-64 version of the K3s stack + - name: k3s + import: + path: common + name: k3s + only: + cluster: + architecture: amd64 + files: + # Include the actual K3s binary + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s + shasum: 9014535a4cd20c788282d60398a06279983562093455b53ab76701539ce67acf + target: /usr/sbin/k3s + executable: true + # K3s magic provides these tools when symlinking + symlinks: + - /usr/sbin/kubectl + - /usr/sbin/ctr + - /usr/sbin/crictl + # Transfer the K3s images for containerd to pick them up + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-airgap-images-amd64.tar.zst + shasum: bc4d05bad56a583c80ff443d60e8277a136cc4357dc8527702d38b5cca28880d + target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst + actions: + onDeploy: + before: + - cmd: if [ "$(uname -m)" != "x86_64" ]; then echo "this package architecture is amd64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi + description: Check that the host architecture matches the package architecture + maxRetries: 0 + + # ARM-64 version of the K3s stack + - name: k3s + import: + path: common + name: k3s + only: + cluster: + architecture: arm64 + files: + # Include the actual K3s binary + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-arm64 + shasum: 1ae72ca06d3302f3e86ef92e6e8f84e14a084da69564e87d6e2e75f62e72388d + target: /usr/sbin/k3s + executable: true + # K3s magic provides these tools when symlinking + symlinks: + - /usr/sbin/kubectl + - /usr/sbin/ctr + - /usr/sbin/crictl + # Transfer the K3s images for containerd to pick them up + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.4+k3s2/k3s-airgap-images-arm64.tar.zst + shasum: 50621ae1391aec7fc66ca66a46a0e9fd48ce373a58073000efdc278233adc64b + target: /var/lib/rancher/k3s/agent/images/k3s.tar.zst + actions: + onDeploy: + before: + - cmd: if [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "arm64" ]; then echo "this package architecture is arm64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi + description: Check that the host architecture matches the package architecture + maxRetries: 0 diff --git a/zarf.yaml b/zarf.yaml index f36d6e2..dca6786 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -6,8 +6,7 @@ metadata: components: - name: k3s import: - # renovate: depName=defenseunicorns/zarf - url: oci://ghcr.io/defenseunicorns/packages/init:v0.33.2 + path: k3s - name: zarf-injector required: true