From 8dcda1c465545ff600efb4558ed578d63e76a370 Mon Sep 17 00:00:00 2001 From: nadavbuc Date: Mon, 15 Apr 2024 10:47:22 +0300 Subject: [PATCH] Improvements --- README.md | 4 +- charts/karpenter_nodes/Chart.yaml | 2 +- charts/karpenter_nodes/README.md | 7 +- .../karpenter_nodes/templates/headroom.yaml | 2 + .../karpenter_nodes/templates/nodeclass.yaml | 8 +-- .../karpenter_nodes/templates/nodepool.yaml | 19 ++---- .../tests/headroom_nodes_default_test.yaml | 6 -- .../tests/nodepool_nodes_default_test.yaml | 36 +++++++--- .../tests/nodepool_nodes_workers_test.yaml | 65 ++++++++++++++++--- charts/karpenter_nodes/tests/values.yaml | 12 +++- charts/karpenter_nodes/values.yaml | 4 ++ 11 files changed, 112 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index ec8d0f6..95cfcc0 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ You can then run `helm search repo fiverr_public` to see the charts. ## License - -[Apache 2.0 License](https://github.com/fiverr/public_charts/blob/main/LICENSE). + +[Apache 2.0 License](https://github.com/fiverr/public_charts/blob/master/LICENSE). ## Helm charts build status diff --git a/charts/karpenter_nodes/Chart.yaml b/charts/karpenter_nodes/Chart.yaml index b86b2b1..6388195 100644 --- a/charts/karpenter_nodes/Chart.yaml +++ b/charts/karpenter_nodes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: karpenter_nodes -version: 0.0.2 +version: 0.1.0 description: A Helm chart for generating NodeClasses and NodePools for Karpenter maintainers: - name: nadavbuc diff --git a/charts/karpenter_nodes/README.md b/charts/karpenter_nodes/README.md index ad6a4b5..c37eaba 100644 --- a/charts/karpenter_nodes/README.md +++ b/charts/karpenter_nodes/README.md @@ -23,13 +23,13 @@ The `UserData` field supports templating and your own values. You can take a loo ### Testing Your Changes After making changes you will probably want to see the new output. Run `helm template` with the relevant example files:
-`helm template . -f values.yaml` +`helm template . -f values.yaml` ### Unit Tests Make sure you have `helm-unittest` plugin installed. [helm-unittest](https://github.com/helm-unittest/helm-unittest) Unit tests are written in `tests` directory. To run the tests, use the following command:
-`helm unittest --helm3 karpenter_nodes -f "tests/$value/*_test.yaml"` +`helm unittest --helm3 karpenter_nodes -f "tests/*_test.yaml"` ## Configuration keys @@ -72,6 +72,7 @@ Note - Most of the values can be overridden per nodegroup (If not specified, it | `consolidateAfter` | Specify how long to wait before consolidating nodes [Documentation](https://karpenter.sh/docs/concepts/nodepools/) | `String` | ✓ | ✓ | | `excludeInstanceSize` | Exclude specific instance sizes | `List` | ✓ | ✓ | | `headRoom` | Generate Ultra Low Priority Class for Headroom (see below) | `String` | ✓ | x | +| `additionalRequirements` | add NodePool requirements which are not covered by this chart | `List(map)` | ✓ | ✓ | ### NodeGroup Configuration | Key Name | Description | Type | Optional? | Optional Per NodeGroup? | @@ -96,7 +97,7 @@ The pods will be configured with ultra-low priority, and will be terminated and | `nodegroups.{}.headRoom` | List of headroom configurations for the nodePool | `List(Map)` | ✓ | ✓ | | `nodegroups.{}.headRoom.size` | `small`, `medium`, `large`, `xlarge` - see below | `String` | ✓ | ✓ | | `nodegroups.{}.headRoom.count` | Number of headroom pod replicas to schedule | `Integer` | ✓ | ✓ | -| `nodegroups.{}.headRoom.antiAffinitySpec` | Required - set antiaffinity to match against all running workloads | `LabelSelectorSpec` | ✓ | ✓ | +| `nodegroups.{}.headRoom.antiAffinitySpec` | Optional - set antiaffinity to match against running workloads | `LabelSelectorSpec` | ✓ | ✓ | | `nodegroups.{}.headRoom.nameSpaces` | Specify list of namespaces to match again (default `all`) | `List(String)` | ✓ | ✓ | ### Headroom Sizing diff --git a/charts/karpenter_nodes/templates/headroom.yaml b/charts/karpenter_nodes/templates/headroom.yaml index 99f2722..e312dfb 100644 --- a/charts/karpenter_nodes/templates/headroom.yaml +++ b/charts/karpenter_nodes/templates/headroom.yaml @@ -38,7 +38,9 @@ spec: operator: In values: - headroom-{{ $k }}-{{ $v.instances.architecture | default $.Values.instances.architecture }}-{{ $nhr.size }} + {{- if hasKey $nhr "antiAffinitySpec" }} {{- $nhr.antiAffinitySpec | toYaml | nindent 14 }} + {{- end }} topologyKey: kubernetes.io/hostname {{- if hasKey $nhr "nameSpaces" }} namespaces: diff --git a/charts/karpenter_nodes/templates/nodeclass.yaml b/charts/karpenter_nodes/templates/nodeclass.yaml index 8d83d9b..6191a28 100644 --- a/charts/karpenter_nodes/templates/nodeclass.yaml +++ b/charts/karpenter_nodes/templates/nodeclass.yaml @@ -20,10 +20,8 @@ spec: {{- end }} amiFamily: {{ $v.amiFamily | default $.Values.amiFamily }} amiSelectorTerms: - {{- if hasKey $v "amiSelectorTerms" }} - {{- toYaml $v.amiSelectorTerms | nindent 4 }} - {{- else }} - {{- toYaml $.Values.amiSelectorTerms | nindent 4 }} + {{- if or (hasKey $v "amiSelectorTerms") (hasKey $.Values "amiSelectorTerms") }} + {{- toYaml ($v.amiSelectorTerms | default $.Values.amiSelectorTerms) | nindent 4 }} {{- end }} subnetSelectorTerms: {{- if hasKey $v "subnetSelectorTerms" }} @@ -44,7 +42,7 @@ spec: {{- toYaml ($v.nodeTags | default $.Values.nodeTags) | nindent 4 }} {{- end }} {{- if hasKey $v "additionalNodeTags" }} - {{- toYaml $v.tags | nindent 4 }} + {{- toYaml $v.additionalNodeTags | nindent 4 }} {{- end }} managed_by: karpenter blockDeviceMappings: diff --git a/charts/karpenter_nodes/templates/nodepool.yaml b/charts/karpenter_nodes/templates/nodepool.yaml index 23ad2a4..d982497 100644 --- a/charts/karpenter_nodes/templates/nodepool.yaml +++ b/charts/karpenter_nodes/templates/nodepool.yaml @@ -102,20 +102,6 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if or (hasKey $.Values "excludeCpuManufacturer") (hasKey $v "excludeCpuManufacturer") }} - - key: "karpenter.k8s.aws/instance-cpu-manufacturer" - operator: NotIn - values: - {{- if hasKey $v "excludeCpuManufacturer" }} - {{- range $v .excludeCpuManufacturer }} - - {{ . }} - {{- end }} - {{- else }} - {{- range $.Values.excludeCpuManufacturer }} - - {{ . }} - {{- end }} - {{- end }} - {{- end }} {{- if or (hasKey $.Values "excludeInstanceSize") (hasKey $v "excludeInstanceSize") }} - key: "karpenter.k8s.aws/instance-size" operator: NotIn @@ -146,6 +132,9 @@ spec: - {{ . -}} {{- end }} {{- end }} + {{- if or (hasKey $.Values "additionalRequirements") (hasKey $v "additionalRequirements") }} + {{- toYaml ($v.additionalRequirements | default $.Values.additionalRequirements) | nindent 8 }} + {{- end }} kubelet: {{- if or (hasKey $v "kubeletClusterDNS") (hasKey $.Values "kubeletClusterDNS") }} clusterDNS: @@ -209,7 +198,7 @@ spec: consolidateAfter: {{ $v.consolidateAfter | default $.Values.consolidateAfter }} {{- if $v.budgets }} budgets: - {{- $v.budgets | toYaml | nindent 6 }} + {{- toYaml $v.budgets | nindent 6 }} {{- end }} {{- if hasKey $v "limits" }} limits: diff --git a/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml b/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml index 1e45176..134bb3f 100644 --- a/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml @@ -39,12 +39,6 @@ tests: - equal: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].values[0] value: headroom-nodes-default-amd64-small - - equal: - path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[1].key - value: testlabel1 - - equal: - path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[1].operator - value: Exists - equal: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].namespaces value: {} diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml index 9d9e838..ed10bd7 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml @@ -52,7 +52,9 @@ tests: - it: Verify nodes-default requirements documentIndex: 0 asserts: - # instance-category + - equal: + path: spec.template.spec.requirements[0].key + value: "karpenter.k8s.aws/instance-category" - notContains: path: spec.template.spec.requirements[0].values content: t @@ -62,47 +64,63 @@ tests: - equal: path: spec.template.spec.requirements[0].values[2] value: c - # instance-cpu + - equal: + path: spec.template.spec.requirements[1].key + value: "karpenter.k8s.aws/instance-cpu" - equal: path: spec.template.spec.requirements[1].values[0] value: "4" - equal: path: spec.template.spec.requirements[1].values[2] value: "16" - # instance-generation + - equal: + path: spec.template.spec.requirements[2].key + value: "karpenter.k8s.aws/instance-generation" - equal: path: spec.template.spec.requirements[2].operator value: "Gt" - equal: path: spec.template.spec.requirements[2].values[0] value: "6" - # instance-zone + - equal: + path: spec.template.spec.requirements[3].key + value: "topology.kubernetes.io/zone" - equal: path: spec.template.spec.requirements[3].values[0] value: "eu-west-1a" - equal: path: spec.template.spec.requirements[3].values[2] value: "eu-west-1c" - # instance-architecture + - equal: + path: spec.template.spec.requirements[4].key + value: "kubernetes.io/arch" - equal: path: spec.template.spec.requirements[4].values[0] value: "amd64" - # instance-capacity-type + - equal: + path: spec.template.spec.requirements[5].key + value: "karpenter.sh/capacity-type" - equal: path: spec.template.spec.requirements[5].values[0] value: "spot" - equal: path: spec.template.spec.requirements[5].values[1] value: "on-demand" - # instance-OS + - equal: + path: spec.template.spec.requirements[6].key + value: "kubernetes.io/os" - equal: path: spec.template.spec.requirements[6].values[0] value: "linux" - # instance-family Exclusions + - equal: + path: spec.template.spec.requirements[7].key + value: "karpenter.k8s.aws/instance-family" - equal: path: spec.template.spec.requirements[7].values[0] value: "c6a" - # instance-size Exclusiong + - equal: + path: spec.template.spec.requirements[8].key + value: "karpenter.k8s.aws/instance-size" - equal: path: spec.template.spec.requirements[8].operator value: NotIn diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml index f0f5249..e72dc00 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml @@ -47,7 +47,9 @@ tests: - it: Verify nodes-workers requirements documentIndex: 1 asserts: - # instance-category + - equal: + path: spec.template.spec.requirements[0].key + value: "karpenter.k8s.aws/instance-category" - notContains: path: spec.template.spec.requirements[0].values content: m @@ -57,47 +59,92 @@ tests: - equal: path: spec.template.spec.requirements[0].values[1] value: x - # instance-cpu + - equal: + path: spec.template.spec.requirements[1].key + value: "karpenter.k8s.aws/instance-cpu" - equal: path: spec.template.spec.requirements[1].values[0] value: "2" - equal: path: spec.template.spec.requirements[1].values[1] value: "6" - # instance-generation + - equal: + path: spec.template.spec.requirements[2].key + value: "karpenter.k8s.aws/instance-generation" - equal: path: spec.template.spec.requirements[2].operator value: "Gt" - equal: path: spec.template.spec.requirements[2].values[0] value: "4" - # instance-zone + - equal: + path: spec.template.spec.requirements[3].key + value: "topology.kubernetes.io/zone" - equal: path: spec.template.spec.requirements[3].values[0] value: "eu-west-1g" - # instance-architecture + - equal: + path: spec.template.spec.requirements[4].key + value: "kubernetes.io/arch" - equal: path: spec.template.spec.requirements[4].values[0] value: "arm64" - # instance-capacity-type + - equal: + path: spec.template.spec.requirements[5].key + value: "karpenter.sh/capacity-type" - equal: path: spec.template.spec.requirements[5].values[0] value: "on-demand" - # instance-OS + - equal: + path: spec.template.spec.requirements[6].key + value: "kubernetes.io/os" - equal: path: spec.template.spec.requirements[6].values[0] value: "linux" - # instance-family Exclusions + - equal: + path: spec.template.spec.requirements[7].key + value: "karpenter.k8s.aws/instance-family" + - equal: + path: spec.template.spec.requirements[7].operator + value: "NotIn" - equal: path: spec.template.spec.requirements[7].values[0] value: "m6a" - # instance-size Exclusiong + - equal: + path: spec.template.spec.requirements[8].key + value: "karpenter.k8s.aws/instance-size" - equal: path: spec.template.spec.requirements[8].operator value: NotIn - equal: path: spec.template.spec.requirements[8].values[0] value: metal + - equal: + path: spec.template.spec.requirements[9].key + value: "capacity-spread" + - equal: + path: spec.template.spec.requirements[9].values[0] + value: "1" + - equal: + path: spec.template.spec.requirements[9].values[4] + value: "5" + # additional requirements + - equal: + path: spec.template.spec.requirements[10].key + value: "karpenter.k8s.aws/instance-local-nvme" + - equal: + path: spec.template.spec.requirements[10].operator + value: "Exists" + - equal: + path: spec.template.spec.requirements[11].key + value: "karpenter.k8s.aws/other" + - equal: + path: spec.template.spec.requirements[11].operator + value: "In" + - equal: + path: spec.template.spec.requirements[11].values[1] + value: "value2" + - it: Verify nodes-workers kubelet documentIndex: 1 diff --git a/charts/karpenter_nodes/tests/values.yaml b/charts/karpenter_nodes/tests/values.yaml index b950124..601c58d 100644 --- a/charts/karpenter_nodes/tests/values.yaml +++ b/charts/karpenter_nodes/tests/values.yaml @@ -8,9 +8,6 @@ nodeGroups: headRoom: - size: small count: 2 - antiAffinitySpec: - - key: testlabel1 - operator: Exists labels: testlabel1: label1 testlabel2: label2 @@ -55,6 +52,15 @@ nodeGroups: nameSpaces: - default - kube-system + additionalRequirements: + - key: "karpenter.k8s.aws/instance-local-nvme" + operator: "Exists" + - key: "karpenter.k8s.aws/other" + operator: "In" + values: + - "value1" + - "value2" + instances: architecture: "arm64" diff --git a/charts/karpenter_nodes/values.yaml b/charts/karpenter_nodes/values.yaml index 291ad44..dc3a3f4 100644 --- a/charts/karpenter_nodes/values.yaml +++ b/charts/karpenter_nodes/values.yaml @@ -135,3 +135,7 @@ headRoom: true # PlaceHolder fo NodeGroups nodeGroups: {} + +#additionalRequirements: +# - key: "karpenter.k8s.aws/instance-local-nvme" +# operator: "Exists"