Skip to content

Commit

Permalink
feat!: separate templating and bootstrap process connotations (#1674)
Browse files Browse the repository at this point in the history
  • Loading branch information
onedr0p authored Jan 4, 2025
1 parent 70fbd00 commit abb8323
Show file tree
Hide file tree
Showing 149 changed files with 309 additions and 347 deletions.
4 changes: 2 additions & 2 deletions .github/labeler.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
area/bootstrap:
area/templates:
- changed-files:
- any-glob-to-any-file: bootstrap/**/*
- any-glob-to-any-file: templates/**/*
area/github:
- changed-files:
- any-glob-to-any-file: .github/**/*
Expand Down
6 changes: 2 additions & 4 deletions .github/labels.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
---
# Area
- { name: "area/bootstrap", color: "0e8a16" }
- { name: "area/templates", color: "0e8a16" }
- { name: "area/github", color: "0e8a16" }
- { name: "area/kubernetes", color: "0e8a16" }
- { name: "area/taskfile", color: "0e8a16" }
# Distro
- { name: "distro/talos", color: "ffc300" }
# Renovate
- { name: "renovate/container", color: "027fa0" }
- { name: "renovate/github-action", color: "027fa0" }
Expand All @@ -17,4 +15,4 @@
- { name: "type/major", color: "f6412d" }
- { name: "type/break", color: "f6412d" }
# Uncategorized
- { name: "hold/upstream", color: "ee0701" }
- { name: "hold", color: "ee0701" }
34 changes: 17 additions & 17 deletions .github/tests/config-talos.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
---
skip_tests: true

bootstrap_cluster_name: fake
bootstrap_schematic_id: "376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba"
bootstrap_node_network: 10.10.10.0/24
bootstrap_node_default_gateway: 10.10.10.1
bootstrap_node_inventory:
cluster_name: fake
schematic_id: "376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba"
node_network: 10.10.10.0/24
node_default_gateway: 10.10.10.1
node_inventory:
- name: k8s-controller-0
address: 10.10.10.100
controller: true
Expand All @@ -16,19 +16,19 @@ bootstrap_node_inventory:
controller: false
disk: fake
mac_addr: fake
bootstrap_dns_servers: ["1.1.1.1", "1.0.0.1"]
bootstrap_ntp_servers: ["time.cloudflare.com"]
bootstrap_pod_network: 10.69.0.0/16
bootstrap_service_network: 10.96.0.0/16
bootstrap_controller_vip: 10.10.10.254
bootstrap_tls_sans: ["fake"]
bootstrap_age_pubkey: $BOOTSTRAP_AGE_PUBLIC_KEY
bootstrap_bgp:
dns_servers: ["1.1.1.1", "1.0.0.1"]
ntp_servers: ["time.cloudflare.com"]
pod_network: 10.69.0.0/16
service_network: 10.96.0.0/16
controller_vip: 10.10.10.254
tls_sans: ["fake"]
age_pubkey: $TEMPLATE_AGE_PUBLIC_KEY
bgp:
enabled: false
bootstrap_github_address: https://github.com/onedr0p/cluster-template
bootstrap_github_branch: main
bootstrap_github_webhook_token: fake
bootstrap_cloudflare:
github_address: https://github.com/onedr0p/cluster-template
github_branch: main
github_webhook_token: fake
cloudflare:
enabled: true
domain: fake
token: take
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,21 @@ jobs:
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

- name: Install Python dependencies
run: mise run pip
- name: Install dependencies
run: mise run deps

- name: Run age task
run: task bootstrap:age-keygen
run: task template:generate-age-key

- name: Run init task
run: |
task init
cp ./.github/tests/config-${{ matrix.config-files }}.yaml ./config.yaml
export BOOTSTRAP_AGE_PUBLIC_KEY=$(sed -n 's/# public key: //gp' age.key)
export TEMPLATE_AGE_PUBLIC_KEY=$(sed -n 's/# public key: //gp' age.key)
envsubst < ./config.yaml > ./config.yaml.tmp && mv ./config.yaml.tmp ./config.yaml
- name: Run configure task
run: task configure --yes

- name: Run clean task
run: task bootstrap:clean
- name: Run cleanup task
run: task template:cleanup --yes
28 changes: 0 additions & 28 deletions .github/workflows/kubeconform.yaml

This file was deleted.

8 changes: 4 additions & 4 deletions .mise.toml
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
[env]
_.python.venv = { path = "{{config_root}}/.venv", create = true }
KUBECONFIG = "{{config_root}}/kubeconfig"
PYTHONDONTWRITEBYTECODE = "1"
SOPS_AGE_KEY_FILE = "{{config_root}}/age.key"
TALOSCONFIG = "{{config_root}}/kubernetes/bootstrap/talos/clusterconfig/talosconfig"
TASK_X_ENV_PRECEDENCE = 1
TASK_X_MAP_VARIABLES = 0

[tasks.pip]
description = "Install Python dependencies"
run = "pip install -r requirements.txt"
[tasks.deps]
description = "Install dependencies"
run = "uv pip install -r requirements.txt"

[tools]
python = "3.13"
uv = "latest"
"aqua:budimanjojo/talhelper" = "latest"
"aqua:cloudflare/cloudflared" = "latest"
"aqua:FiloSottile/age" = "latest"
Expand Down
52 changes: 0 additions & 52 deletions .taskfiles/bootstrap/Taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,55 +63,3 @@ tasks:
- test -f {{.KUBECONFIG}}
- test -f {{.SOPS_AGE_KEY_FILE}}
- which flux kubectl sops

age-keygen:
desc: Bootstrap the Sops Age key
cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
status:
- test -f {{.SOPS_AGE_KEY_FILE}}
preconditions:
- which age-keygen

template:
internal: true
cmd: makejinja
preconditions:
- test -f {{.BOOTSTRAP_CONFIG_FILE}}
- test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py
- test -f {{.MAKEJINJA_CONFIG_FILE}}
- which makejinja

secrets:
internal: true
cmds:
- for: { var: SECRET_FILES }
cmd: |
if sops filestatus "{{.ITEM}}" | jq --exit-status ".encrypted == false" &>/dev/null; then
sops --encrypt --in-place "{{.ITEM}}"
fi
vars:
SECRET_FILES:
sh: find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*"
preconditions:
- test -f {{.SOPS_AGE_KEY_FILE}}
- test -f {{.SOPS_CONFIG_FILE}}
- which sops

clean:
desc: Clean files and directories no longer needed after cluster bootstrap
cmds:
- mkdir -p {{.ROOT_DIR}}/.private/{{.TS}}
- mv {{.BOOTSTRAP_DIR}} {{.ROOT_DIR}}/.private/{{.TS}}/bootstrap
- mv {{.MAKEJINJA_CONFIG_FILE}} {{.ROOT_DIR}}/.private/{{.TS}}/makejinja.toml
- mv {{.ROOT_DIR}}/requirements.txt {{.ROOT_DIR}}/.private/{{.TS}}/requirements.txt
- rm -rf {{.ROOT_DIR}}/.github/tests
- rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
- sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
vars:
TS: '{{now | unixEpoch}}'
preconditions:
- test -d {{.BOOTSTRAP_DIR}}
- test -d {{.ROOT_DIR}}/.github/tests
- test -f {{.MAKEJINJA_CONFIG_FILE}}
- test -f {{.ROOT_DIR}}/.github/renovate.json5
- test -f {{.ROOT_DIR}}/requirements.txt
30 changes: 1 addition & 29 deletions .taskfiles/kubernetes/Taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,15 @@
# yaml-language-server: $schema=https://taskfile.dev/schema.json
version: '3'

vars:
KUBECONFORM_SCRIPT: "{{.SCRIPTS_DIR}}/kubeconform.sh"

tasks:

ks-*:
desc: Apply a Flux KS [PATH=required]
cmd: >
flux build --namespace flux-system ks {{base .PATH}}
--kustomization-file {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
--path {{.KUBERNETES_DIR}}/apps/{{.PATH}}{{if contains .KUSTOMIZATION "not found"}} --dry-run{{end}}
| yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "flux-system")' -
| kubectl {{.OPERATION}} {{if eq .OPERATION "apply"}}--server-side --field-manager=kustomize-controller{{end}} --filename -
requires:
vars: [PATH]
vars:
OPERATION: '{{index .MATCH 0}}'
KUSTOMIZATION:
sh: flux --namespace flux-system get kustomizations {{base .PATH}}
preconditions:
- test -f {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
- which flux kubectl yq

reconcile:
desc: Force update Flux to pull in changes from your Git repository
desc: Force Flux to pull in changes from your Git repository
cmd: flux --namespace flux-system reconcile kustomization cluster --with-source
preconditions:
- test -f {{.KUBECONFIG}}
- which flux

kubeconform:
desc: Validate Kubernetes manifests with kubeconform
cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
preconditions:
- test -f {{.KUBECONFORM_SCRIPT}}
- which kubeconform

resources:
desc: Gather common resources in your cluster, useful when asking for support
cmds:
Expand Down
96 changes: 96 additions & 0 deletions .taskfiles/template/Taskfile.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
---
# yaml-language-server: $schema=https://taskfile.dev/schema.json
version: '3'

vars:
MAKEJINJA_CONFIG_FILE: '{{.ROOT_DIR}}/makejinja.toml'
TEMPLATE_DIR: '{{.ROOT_DIR}}/templates'
TEMPLATE_CONFIG_FILE: '{{.ROOT_DIR}}/config.yaml'
TEMPLATE_RESOURCES_DIR: '{{.ROOT_DIR}}/.taskfiles/template/resources'

tasks:

:init:
desc: Initialize configuration files
cmd: mv {{.TEMPLATE_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.TEMPLATE_CONFIG_FILE}}
status:
- test -f {{.TEMPLATE_CONFIG_FILE}}

:configure:
desc: Render and validate configuration files
prompt: Any conflicting files in the kubernetes directory will be overwritten... continue?
cmds:
- task: render-configs
- task: encrypt-secrets
- task: kubeconform
preconditions:
- msg: |
An existing Age key interferes with age key in this repository.
Rename or delete ~/.config/sops/age/keys.txt
sh: '! test -f ~/.config/sops/age/keys.txt'
generate-age-key:
desc: Create the Sops Age key
cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
status:
- test -f {{.SOPS_AGE_KEY_FILE}}
preconditions:
- which age-keygen

render-configs:
desc: Template the configuration files
cmd: makejinja
env:
PYTHONDONTWRITEBYTECODE: '1'
preconditions:
- test -f {{.TEMPLATE_CONFIG_FILE}}
- test -f {{.TEMPLATE_DIR}}/scripts/plugin.py
- test -f {{.MAKEJINJA_CONFIG_FILE}}
- which makejinja

encrypt-secrets:
desc: Encrypt secrets with Sops
cmds:
- for: { var: SECRET_FILES }
cmd: |
if sops filestatus "{{.ITEM}}" | jq --exit-status ".encrypted == false" &>/dev/null; then
sops --encrypt --in-place "{{.ITEM}}"
fi
vars:
SECRET_FILES:
sh: find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*"
preconditions:
- test -f {{.SOPS_AGE_KEY_FILE}}
- test -f {{.SOPS_CONFIG_FILE}}
- which sops

kubeconform:
desc: Validate Kubernetes manifests with kubeconform
cmd: bash {{.TEMPLATE_RESOURCES_DIR}}/kubeconform.sh {{.KUBERNETES_DIR}}
preconditions:
- test -f {{.TEMPLATE_RESOURCES_DIR}}/kubeconform.sh
- which kubeconform

cleanup:
desc: Remove or archive template files and directories no longer needed after cluster bootstrap
prompt: All files and directories related to the templating process will be removed... continue?
cmds:
- mkdir -p {{.ROOT_DIR}}/.private/{{.TS}}/{.taskfiles}
- rm -rf {{.ROOT_DIR}}/.github/tests
- rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
- sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
- mv {{.TEMPLATE_DIR}} {{.ROOT_DIR}}/.private/{{.TS}}/templates
- mv {{.MAKEJINJA_CONFIG_FILE}} {{.ROOT_DIR}}/.private/{{.TS}}/makejinja.toml
- mv {{.ROOT_DIR}}/requirements.txt {{.ROOT_DIR}}/.private/{{.TS}}/requirements.txt
- mv {{.TEMPLATE_CONFIG_FILE}} {{.ROOT_DIR}}/.private/{{.TS}}/config.yaml
- sed -i {{if eq OS "darwin"}}''{{end}} '/template:/d' {{.ROOT_DIR}}/Taskfile.yaml
- mv {{.ROOT_DIR}}/.taskfiles/template {{.ROOT_DIR}}/.private/{{.TS}}/.taskfiles/
vars:
TS: '{{now | unixEpoch}}'
preconditions:
- test -d {{.ROOT_DIR}}/.github/tests
- test -d {{.ROOT_DIR}}/.taskfiles/template
- test -d {{.TEMPLATE_DIR}}
- test -f {{.MAKEJINJA_CONFIG_FILE}}
- test -f {{.ROOT_DIR}}/.github/renovate.json5
- test -f {{.ROOT_DIR}}/requirements.txt
File renamed without changes.
Loading

0 comments on commit abb8323

Please sign in to comment.