From 8dcda1c465545ff600efb4558ed578d63e76a370 Mon Sep 17 00:00:00 2001 From: nadavbuc Date: Mon, 15 Apr 2024 10:47:22 +0300 Subject: [PATCH 1/4] Improvements --- README.md | 4 +- charts/karpenter_nodes/Chart.yaml | 2 +- charts/karpenter_nodes/README.md | 7 +- .../karpenter_nodes/templates/headroom.yaml | 2 + .../karpenter_nodes/templates/nodeclass.yaml | 8 +-- .../karpenter_nodes/templates/nodepool.yaml | 19 ++---- .../tests/headroom_nodes_default_test.yaml | 6 -- .../tests/nodepool_nodes_default_test.yaml | 36 +++++++--- .../tests/nodepool_nodes_workers_test.yaml | 65 ++++++++++++++++--- charts/karpenter_nodes/tests/values.yaml | 12 +++- charts/karpenter_nodes/values.yaml | 4 ++ 11 files changed, 112 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index ec8d0f6..95cfcc0 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ You can then run `helm search repo fiverr_public` to see the charts. ## License - -[Apache 2.0 License](https://github.com/fiverr/public_charts/blob/main/LICENSE). + +[Apache 2.0 License](https://github.com/fiverr/public_charts/blob/master/LICENSE). ## Helm charts build status diff --git a/charts/karpenter_nodes/Chart.yaml b/charts/karpenter_nodes/Chart.yaml index b86b2b1..6388195 100644 --- a/charts/karpenter_nodes/Chart.yaml +++ b/charts/karpenter_nodes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: karpenter_nodes -version: 0.0.2 +version: 0.1.0 description: A Helm chart for generating NodeClasses and NodePools for Karpenter maintainers: - name: nadavbuc diff --git a/charts/karpenter_nodes/README.md b/charts/karpenter_nodes/README.md index ad6a4b5..c37eaba 100644 --- a/charts/karpenter_nodes/README.md +++ b/charts/karpenter_nodes/README.md @@ -23,13 +23,13 @@ The `UserData` field supports templating and your own values. You can take a loo ### Testing Your Changes After making changes you will probably want to see the new output. Run `helm template` with the relevant example files:
-`helm template . -f values.yaml` +`helm template . -f values.yaml` ### Unit Tests Make sure you have `helm-unittest` plugin installed. [helm-unittest](https://github.com/helm-unittest/helm-unittest) Unit tests are written in `tests` directory. To run the tests, use the following command:
-`helm unittest --helm3 karpenter_nodes -f "tests/$value/*_test.yaml"` +`helm unittest --helm3 karpenter_nodes -f "tests/*_test.yaml"` ## Configuration keys @@ -72,6 +72,7 @@ Note - Most of the values can be overridden per nodegroup (If not specified, it | `consolidateAfter` | Specify how long to wait before consolidating nodes [Documentation](https://karpenter.sh/docs/concepts/nodepools/) | `String` | ✓ | ✓ | | `excludeInstanceSize` | Exclude specific instance sizes | `List` | ✓ | ✓ | | `headRoom` | Generate Ultra Low Priority Class for Headroom (see below) | `String` | ✓ | x | +| `additionalRequirements` | add NodePool requirements which are not covered by this chart | `List(map)` | ✓ | ✓ | ### NodeGroup Configuration | Key Name | Description | Type | Optional? | Optional Per NodeGroup? | @@ -96,7 +97,7 @@ The pods will be configured with ultra-low priority, and will be terminated and | `nodegroups.{}.headRoom` | List of headroom configurations for the nodePool | `List(Map)` | ✓ | ✓ | | `nodegroups.{}.headRoom.size` | `small`, `medium`, `large`, `xlarge` - see below | `String` | ✓ | ✓ | | `nodegroups.{}.headRoom.count` | Number of headroom pod replicas to schedule | `Integer` | ✓ | ✓ | -| `nodegroups.{}.headRoom.antiAffinitySpec` | Required - set antiaffinity to match against all running workloads | `LabelSelectorSpec` | ✓ | ✓ | +| `nodegroups.{}.headRoom.antiAffinitySpec` | Optional - set antiaffinity to match against running workloads | `LabelSelectorSpec` | ✓ | ✓ | | `nodegroups.{}.headRoom.nameSpaces` | Specify list of namespaces to match again (default `all`) | `List(String)` | ✓ | ✓ | ### Headroom Sizing diff --git a/charts/karpenter_nodes/templates/headroom.yaml b/charts/karpenter_nodes/templates/headroom.yaml index 99f2722..e312dfb 100644 --- a/charts/karpenter_nodes/templates/headroom.yaml +++ b/charts/karpenter_nodes/templates/headroom.yaml @@ -38,7 +38,9 @@ spec: operator: In values: - headroom-{{ $k }}-{{ $v.instances.architecture | default $.Values.instances.architecture }}-{{ $nhr.size }} + {{- if hasKey $nhr "antiAffinitySpec" }} {{- $nhr.antiAffinitySpec | toYaml | nindent 14 }} + {{- end }} topologyKey: kubernetes.io/hostname {{- if hasKey $nhr "nameSpaces" }} namespaces: diff --git a/charts/karpenter_nodes/templates/nodeclass.yaml b/charts/karpenter_nodes/templates/nodeclass.yaml index 8d83d9b..6191a28 100644 --- a/charts/karpenter_nodes/templates/nodeclass.yaml +++ b/charts/karpenter_nodes/templates/nodeclass.yaml @@ -20,10 +20,8 @@ spec: {{- end }} amiFamily: {{ $v.amiFamily | default $.Values.amiFamily }} amiSelectorTerms: - {{- if hasKey $v "amiSelectorTerms" }} - {{- toYaml $v.amiSelectorTerms | nindent 4 }} - {{- else }} - {{- toYaml $.Values.amiSelectorTerms | nindent 4 }} + {{- if or (hasKey $v "amiSelectorTerms") (hasKey $.Values "amiSelectorTerms") }} + {{- toYaml ($v.amiSelectorTerms | default $.Values.amiSelectorTerms) | nindent 4 }} {{- end }} subnetSelectorTerms: {{- if hasKey $v "subnetSelectorTerms" }} @@ -44,7 +42,7 @@ spec: {{- toYaml ($v.nodeTags | default $.Values.nodeTags) | nindent 4 }} {{- end }} {{- if hasKey $v "additionalNodeTags" }} - {{- toYaml $v.tags | nindent 4 }} + {{- toYaml $v.additionalNodeTags | nindent 4 }} {{- end }} managed_by: karpenter blockDeviceMappings: diff --git a/charts/karpenter_nodes/templates/nodepool.yaml b/charts/karpenter_nodes/templates/nodepool.yaml index 23ad2a4..d982497 100644 --- a/charts/karpenter_nodes/templates/nodepool.yaml +++ b/charts/karpenter_nodes/templates/nodepool.yaml @@ -102,20 +102,6 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if or (hasKey $.Values "excludeCpuManufacturer") (hasKey $v "excludeCpuManufacturer") }} - - key: "karpenter.k8s.aws/instance-cpu-manufacturer" - operator: NotIn - values: - {{- if hasKey $v "excludeCpuManufacturer" }} - {{- range $v .excludeCpuManufacturer }} - - {{ . }} - {{- end }} - {{- else }} - {{- range $.Values.excludeCpuManufacturer }} - - {{ . }} - {{- end }} - {{- end }} - {{- end }} {{- if or (hasKey $.Values "excludeInstanceSize") (hasKey $v "excludeInstanceSize") }} - key: "karpenter.k8s.aws/instance-size" operator: NotIn @@ -146,6 +132,9 @@ spec: - {{ . -}} {{- end }} {{- end }} + {{- if or (hasKey $.Values "additionalRequirements") (hasKey $v "additionalRequirements") }} + {{- toYaml ($v.additionalRequirements | default $.Values.additionalRequirements) | nindent 8 }} + {{- end }} kubelet: {{- if or (hasKey $v "kubeletClusterDNS") (hasKey $.Values "kubeletClusterDNS") }} clusterDNS: @@ -209,7 +198,7 @@ spec: consolidateAfter: {{ $v.consolidateAfter | default $.Values.consolidateAfter }} {{- if $v.budgets }} budgets: - {{- $v.budgets | toYaml | nindent 6 }} + {{- toYaml $v.budgets | nindent 6 }} {{- end }} {{- if hasKey $v "limits" }} limits: diff --git a/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml b/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml index 1e45176..134bb3f 100644 --- a/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/headroom_nodes_default_test.yaml @@ -39,12 +39,6 @@ tests: - equal: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].values[0] value: headroom-nodes-default-amd64-small - - equal: - path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[1].key - value: testlabel1 - - equal: - path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[1].operator - value: Exists - equal: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].namespaces value: {} diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml index 9d9e838..ed10bd7 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml @@ -52,7 +52,9 @@ tests: - it: Verify nodes-default requirements documentIndex: 0 asserts: - # instance-category + - equal: + path: spec.template.spec.requirements[0].key + value: "karpenter.k8s.aws/instance-category" - notContains: path: spec.template.spec.requirements[0].values content: t @@ -62,47 +64,63 @@ tests: - equal: path: spec.template.spec.requirements[0].values[2] value: c - # instance-cpu + - equal: + path: spec.template.spec.requirements[1].key + value: "karpenter.k8s.aws/instance-cpu" - equal: path: spec.template.spec.requirements[1].values[0] value: "4" - equal: path: spec.template.spec.requirements[1].values[2] value: "16" - # instance-generation + - equal: + path: spec.template.spec.requirements[2].key + value: "karpenter.k8s.aws/instance-generation" - equal: path: spec.template.spec.requirements[2].operator value: "Gt" - equal: path: spec.template.spec.requirements[2].values[0] value: "6" - # instance-zone + - equal: + path: spec.template.spec.requirements[3].key + value: "topology.kubernetes.io/zone" - equal: path: spec.template.spec.requirements[3].values[0] value: "eu-west-1a" - equal: path: spec.template.spec.requirements[3].values[2] value: "eu-west-1c" - # instance-architecture + - equal: + path: spec.template.spec.requirements[4].key + value: "kubernetes.io/arch" - equal: path: spec.template.spec.requirements[4].values[0] value: "amd64" - # instance-capacity-type + - equal: + path: spec.template.spec.requirements[5].key + value: "karpenter.sh/capacity-type" - equal: path: spec.template.spec.requirements[5].values[0] value: "spot" - equal: path: spec.template.spec.requirements[5].values[1] value: "on-demand" - # instance-OS + - equal: + path: spec.template.spec.requirements[6].key + value: "kubernetes.io/os" - equal: path: spec.template.spec.requirements[6].values[0] value: "linux" - # instance-family Exclusions + - equal: + path: spec.template.spec.requirements[7].key + value: "karpenter.k8s.aws/instance-family" - equal: path: spec.template.spec.requirements[7].values[0] value: "c6a" - # instance-size Exclusiong + - equal: + path: spec.template.spec.requirements[8].key + value: "karpenter.k8s.aws/instance-size" - equal: path: spec.template.spec.requirements[8].operator value: NotIn diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml index f0f5249..e72dc00 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml @@ -47,7 +47,9 @@ tests: - it: Verify nodes-workers requirements documentIndex: 1 asserts: - # instance-category + - equal: + path: spec.template.spec.requirements[0].key + value: "karpenter.k8s.aws/instance-category" - notContains: path: spec.template.spec.requirements[0].values content: m @@ -57,47 +59,92 @@ tests: - equal: path: spec.template.spec.requirements[0].values[1] value: x - # instance-cpu + - equal: + path: spec.template.spec.requirements[1].key + value: "karpenter.k8s.aws/instance-cpu" - equal: path: spec.template.spec.requirements[1].values[0] value: "2" - equal: path: spec.template.spec.requirements[1].values[1] value: "6" - # instance-generation + - equal: + path: spec.template.spec.requirements[2].key + value: "karpenter.k8s.aws/instance-generation" - equal: path: spec.template.spec.requirements[2].operator value: "Gt" - equal: path: spec.template.spec.requirements[2].values[0] value: "4" - # instance-zone + - equal: + path: spec.template.spec.requirements[3].key + value: "topology.kubernetes.io/zone" - equal: path: spec.template.spec.requirements[3].values[0] value: "eu-west-1g" - # instance-architecture + - equal: + path: spec.template.spec.requirements[4].key + value: "kubernetes.io/arch" - equal: path: spec.template.spec.requirements[4].values[0] value: "arm64" - # instance-capacity-type + - equal: + path: spec.template.spec.requirements[5].key + value: "karpenter.sh/capacity-type" - equal: path: spec.template.spec.requirements[5].values[0] value: "on-demand" - # instance-OS + - equal: + path: spec.template.spec.requirements[6].key + value: "kubernetes.io/os" - equal: path: spec.template.spec.requirements[6].values[0] value: "linux" - # instance-family Exclusions + - equal: + path: spec.template.spec.requirements[7].key + value: "karpenter.k8s.aws/instance-family" + - equal: + path: spec.template.spec.requirements[7].operator + value: "NotIn" - equal: path: spec.template.spec.requirements[7].values[0] value: "m6a" - # instance-size Exclusiong + - equal: + path: spec.template.spec.requirements[8].key + value: "karpenter.k8s.aws/instance-size" - equal: path: spec.template.spec.requirements[8].operator value: NotIn - equal: path: spec.template.spec.requirements[8].values[0] value: metal + - equal: + path: spec.template.spec.requirements[9].key + value: "capacity-spread" + - equal: + path: spec.template.spec.requirements[9].values[0] + value: "1" + - equal: + path: spec.template.spec.requirements[9].values[4] + value: "5" + # additional requirements + - equal: + path: spec.template.spec.requirements[10].key + value: "karpenter.k8s.aws/instance-local-nvme" + - equal: + path: spec.template.spec.requirements[10].operator + value: "Exists" + - equal: + path: spec.template.spec.requirements[11].key + value: "karpenter.k8s.aws/other" + - equal: + path: spec.template.spec.requirements[11].operator + value: "In" + - equal: + path: spec.template.spec.requirements[11].values[1] + value: "value2" + - it: Verify nodes-workers kubelet documentIndex: 1 diff --git a/charts/karpenter_nodes/tests/values.yaml b/charts/karpenter_nodes/tests/values.yaml index b950124..601c58d 100644 --- a/charts/karpenter_nodes/tests/values.yaml +++ b/charts/karpenter_nodes/tests/values.yaml @@ -8,9 +8,6 @@ nodeGroups: headRoom: - size: small count: 2 - antiAffinitySpec: - - key: testlabel1 - operator: Exists labels: testlabel1: label1 testlabel2: label2 @@ -55,6 +52,15 @@ nodeGroups: nameSpaces: - default - kube-system + additionalRequirements: + - key: "karpenter.k8s.aws/instance-local-nvme" + operator: "Exists" + - key: "karpenter.k8s.aws/other" + operator: "In" + values: + - "value1" + - "value2" + instances: architecture: "arm64" diff --git a/charts/karpenter_nodes/values.yaml b/charts/karpenter_nodes/values.yaml index 291ad44..dc3a3f4 100644 --- a/charts/karpenter_nodes/values.yaml +++ b/charts/karpenter_nodes/values.yaml @@ -135,3 +135,7 @@ headRoom: true # PlaceHolder fo NodeGroups nodeGroups: {} + +#additionalRequirements: +# - key: "karpenter.k8s.aws/instance-local-nvme" +# operator: "Exists" From 29794da5793e1ac50d0338eace934a3b35964bff Mon Sep 17 00:00:00 2001 From: nadavbuc Date: Mon, 15 Apr 2024 10:51:16 +0300 Subject: [PATCH 2/4] fix lint --- charts/karpenter_nodes/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/karpenter_nodes/values.yaml b/charts/karpenter_nodes/values.yaml index dc3a3f4..526c7c1 100644 --- a/charts/karpenter_nodes/values.yaml +++ b/charts/karpenter_nodes/values.yaml @@ -136,6 +136,6 @@ headRoom: true # PlaceHolder fo NodeGroups nodeGroups: {} -#additionalRequirements: +# additionalRequirements: # - key: "karpenter.k8s.aws/instance-local-nvme" # operator: "Exists" From d2d48f068509ec78900619544405059e427bfa87 Mon Sep 17 00:00:00 2001 From: nadavbuc Date: Mon, 15 Apr 2024 11:14:31 +0300 Subject: [PATCH 3/4] cover everything in tests --- charts/karpenter_nodes/README.md | 2 +- .../karpenter_nodes/templates/nodeclass.yaml | 2 ++ .../karpenter_nodes/tests/nodeclass_test.yaml | 20 ++++++++++++++++++- .../tests/nodepool_nodes_default_test.yaml | 2 +- .../tests/nodepool_nodes_workers_test.yaml | 20 +++++++++++++++++-- charts/karpenter_nodes/tests/values.yaml | 17 +++++++++++++++- 6 files changed, 57 insertions(+), 6 deletions(-) diff --git a/charts/karpenter_nodes/README.md b/charts/karpenter_nodes/README.md index c37eaba..1717e84 100644 --- a/charts/karpenter_nodes/README.md +++ b/charts/karpenter_nodes/README.md @@ -46,7 +46,6 @@ Note - Most of the values can be overridden per nodegroup (If not specified, it | `securityGroupSelectorTerms` | Selector for Security Groups [Documentation](https://karpenter.sh/docs/concepts/nodeclasses/#specsecuritygroupselectorterms) | `List(Map)` | x | ✓ | | `nodeGroupLabelName` | The Name of the label for each nodegroup (default is `nodegroup`) | `String` | x | ✓ | | `nodeTags` | Tags to add to the instances ``: `` | `Map` | ✓ | ✓ | -| `additionalNodeTags` | Additional Tags to add to the instances ``: `` | `Map` | ✓ | ✓ | | `nodegroups.{}` | each will be used to setup a provisioner and template based on the nodegrup name key | `List[Maps]` | x | ✓ | | `blockDeviceMappings` | Block Device Mappings [Documentation](https://karpenter.sh/docs/concepts/nodeclasses/#specblockdevicemappings) | `List(Map)` | x | ✓ | | `detailedMonitoring` | Detailed Monitoring [Documentation](https://karpenter.sh/docs/concepts/nodeclasses/#specdetailedmonitoring) | `Boolean` | x | ✓ | @@ -78,6 +77,7 @@ Note - Most of the values can be overridden per nodegroup (If not specified, it | Key Name | Description | Type | Optional? | Optional Per NodeGroup? | | ------------------------------ | ----------- | ---- | --------- | ----------------------- | | `nodegroups.{}.labels` | Labels to add to nodes ``: `` | `Map` | ✓ | ✓ | +| `nodegroups.{}.additionalNodeTags` | Additional Tags to add to the instances ``: `` | `Map` | ✓ | ✓ | | `nodegroups.{}.annotations` | Annotations to add to nodes ``: `` | `Map` | ✓ | ✓ | | `nodegroups.{}.nodeClassRef` | If you wish to use your own nodeClass, specify it [Documentation](https://karpenter.sh/docs/concepts/nodeclasses/) | `Map` | ✓ | ✓ | | `nodegroups.{}.taints` | Taints to add to nodes `- `: ``: `` | `List(Map)` | ✓ | ✓ | diff --git a/charts/karpenter_nodes/templates/nodeclass.yaml b/charts/karpenter_nodes/templates/nodeclass.yaml index 6191a28..fd02ddd 100644 --- a/charts/karpenter_nodes/templates/nodeclass.yaml +++ b/charts/karpenter_nodes/templates/nodeclass.yaml @@ -1,6 +1,7 @@ {{- range $k, $v := $.Values.nodeGroups }} {{- $_ := set $ "key" $k }} {{- $_ := set $ "value" $v }} +{{- if not (hasKey $v "nodeClassRef") }} --- apiVersion: karpenter.k8s.aws/{{ $.Values.ApiVersion }} kind: EC2NodeClass @@ -72,3 +73,4 @@ spec: {{- end }} {{- end}} {{- end }} +{{- end }} diff --git a/charts/karpenter_nodes/tests/nodeclass_test.yaml b/charts/karpenter_nodes/tests/nodeclass_test.yaml index 393de8c..3c0a0e9 100644 --- a/charts/karpenter_nodes/tests/nodeclass_test.yaml +++ b/charts/karpenter_nodes/tests/nodeclass_test.yaml @@ -1,10 +1,13 @@ suite: test nodeclasses templates: - nodeclass.yaml - values: - values.yaml tests: + - it: should only render 2 nodeclasses + asserts: + - hasDocuments: + count: 2 - it: Verify nodes-default documentIndex: 0 asserts: @@ -80,6 +83,15 @@ tests: - equal: path: spec.amiFamily value: BottleRocket + - equal: + path: spec.amiSelectorTerms[0].tags.environment + value: test + - equal: + path: spec.amiSelectorTerms[1].name + value: my-ami + - equal: + path: spec.amiSelectorTerms[2].id + value: ami-123 - equal: path: spec.subnetSelectorTerms[0].tags.Name value: eks-subnet-workers-1 @@ -95,6 +107,12 @@ tests: - equal: path: spec.tags.cluster value: eks-dev + - equal: + path: spec.tags.testtag1 + value: tag1 + - equal: + path: spec.tags.testtag2 + value: tag2 - equal: path: spec.blockDeviceMappings[0].deviceName value: /dev/xvda diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml index ed10bd7..5431804 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml @@ -151,7 +151,7 @@ tests: path: spec.disruption.consolidateAfter value: 5m - isNull: - path: spec.budgets + path: spec.disruption.budgets - isNull: path: spec.limits - equal: diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml index e72dc00..a10abc0 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml @@ -22,6 +22,12 @@ tests: value: nodes-workers - isNull: path: spec.template.metadata.labels.testlabel1 + - equal: + path: spec.template.metadata.annotations.testannotation1 + value: annotation1 + - equal: + path: spec.template.metadata.annotations.testannotation2 + value: annotation2 - equal: path: spec.template.spec.nodeClassRef.name value: nodes-workers-arm64 @@ -174,8 +180,18 @@ tests: - equal: path: spec.disruption.consolidateAfter value: 10m - - isNull: - path: spec.budgets + - equal: + path: spec.disruption.budgets[0].nodes + value: "5" + - equal: + path: spec.disruption.budgets[1].nodes + value: "0" + - equal: + path: spec.disruption.budgets[1].schedule + value: "@daily" + - equal: + path: spec.disruption.budgets[1].duration + value: "10m" - equal: path: spec.limits.cpu value: 100 diff --git a/charts/karpenter_nodes/tests/values.yaml b/charts/karpenter_nodes/tests/values.yaml index 601c58d..c59c6f3 100644 --- a/charts/karpenter_nodes/tests/values.yaml +++ b/charts/karpenter_nodes/tests/values.yaml @@ -22,6 +22,12 @@ nodeGroups: - c6a nodes-workers: + additionalNodeTags: + testtag1: tag1 + testtag2: tag2 + annotations: + testannotation1: annotation1 + testannotation2: annotation2 weight: 3 amiFamily: BottleRocket consolidationPolicy: "WhenEmpty" @@ -60,7 +66,16 @@ nodeGroups: values: - "value1" - "value2" - + budgets: + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m + amiSelectorTerms: + - tags: + environment: test + - name: my-ami + - id: ami-123 instances: architecture: "arm64" From 491d490e18fa6bb173cb4793028d0d5af40cb33e Mon Sep 17 00:00:00 2001 From: nadavbuc Date: Mon, 15 Apr 2024 11:35:29 +0300 Subject: [PATCH 4/4] cover all in tests --- .../karpenter_nodes/templates/nodepool.yaml | 2 +- .../tests/nodepool_nodes_default_test.yaml | 1 - .../tests/nodepool_nodes_noclass_test.yaml | 88 +++++++++++++++++++ .../tests/nodepool_nodes_workers_test.yaml | 9 +- charts/karpenter_nodes/tests/values.yaml | 6 +- 5 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 charts/karpenter_nodes/tests/nodepool_nodes_noclass_test.yaml diff --git a/charts/karpenter_nodes/templates/nodepool.yaml b/charts/karpenter_nodes/templates/nodepool.yaml index d982497..4214e89 100644 --- a/charts/karpenter_nodes/templates/nodepool.yaml +++ b/charts/karpenter_nodes/templates/nodepool.yaml @@ -24,7 +24,7 @@ spec: spec: nodeClassRef: {{- if hasKey $v "nodeClassRef" }} - {{ $v.nodeClassRef | toYaml | nindent 8 }} + {{- $v.nodeClassRef | toYaml | nindent 8 }} {{- else }} name: {{ $k }}-{{ $v.instances.architecture | default $.Values.instances.architecture }} {{- end }} diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml index 5431804..bdf270b 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_default_test.yaml @@ -1,7 +1,6 @@ suite: test nodepool - nodes-default templates: - nodepool.yaml - values: - values.yaml diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_noclass_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_noclass_test.yaml new file mode 100644 index 0000000..b432124 --- /dev/null +++ b/charts/karpenter_nodes/tests/nodepool_nodes_noclass_test.yaml @@ -0,0 +1,88 @@ +suite: test nodepool - nodes-noclass +templates: + - nodepool.yaml +values: + - values.yaml + +tests: + - it: Verify nodes-noclass metadata + documentIndex: 1 + asserts: + - isKind: + of: NodePool + - equal: + path: metadata.name + value: nodes-noclass-amd64 + - equal: + path: spec.template.metadata.labels.cluster + value: eks-dev + - equal: + path: spec.template.metadata.labels.nodegroup + value: nodes-noclass + - equal: + path: spec.template.spec.nodeClassRef.name + value: default + + - it: Verify nodes-noclass requirements + documentIndex: 1 + asserts: + - equal: + path: spec.template.spec.requirements[0].key + value: "karpenter.k8s.aws/instance-category" + - notContains: + path: spec.template.spec.requirements[0].values + content: t + - equal: + path: spec.template.spec.requirements[0].values[0] + value: m + - equal: + path: spec.template.spec.requirements[0].values[2] + value: c + - equal: + path: spec.template.spec.requirements[1].key + value: "karpenter.k8s.aws/instance-cpu" + - equal: + path: spec.template.spec.requirements[1].values[0] + value: "4" + - equal: + path: spec.template.spec.requirements[1].values[2] + value: "16" + - equal: + path: spec.template.spec.requirements[2].key + value: "karpenter.k8s.aws/instance-generation" + - equal: + path: spec.template.spec.requirements[2].operator + value: "Gt" + - equal: + path: spec.template.spec.requirements[2].values[0] + value: "4" + - equal: + path: spec.template.spec.requirements[3].key + value: "topology.kubernetes.io/zone" + - equal: + path: spec.template.spec.requirements[3].values[0] + value: "eu-west-1a" + - equal: + path: spec.template.spec.requirements[3].values[2] + value: "eu-west-1c" + - equal: + path: spec.template.spec.requirements[4].key + value: "kubernetes.io/arch" + - equal: + path: spec.template.spec.requirements[4].values[0] + value: "amd64" + - equal: + path: spec.template.spec.requirements[5].key + value: "karpenter.sh/capacity-type" + - equal: + path: spec.template.spec.requirements[5].values[0] + value: "spot" + - equal: + path: spec.template.spec.requirements[5].values[1] + value: "on-demand" + - equal: + path: spec.template.spec.requirements[6].key + value: "kubernetes.io/os" + - equal: + path: spec.template.spec.requirements[6].values[0] + value: "linux" diff --git a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml index a10abc0..9ae4919 100644 --- a/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml +++ b/charts/karpenter_nodes/tests/nodepool_nodes_workers_test.yaml @@ -1,13 +1,12 @@ suite: test nodepool - nodes-workers templates: - nodepool.yaml - values: - values.yaml tests: - it: Verify nodes-workers metadata - documentIndex: 1 + documentIndex: 2 asserts: - isKind: of: NodePool @@ -51,7 +50,7 @@ tests: value: noSchedule - it: Verify nodes-workers requirements - documentIndex: 1 + documentIndex: 2 asserts: - equal: path: spec.template.spec.requirements[0].key @@ -153,7 +152,7 @@ tests: - it: Verify nodes-workers kubelet - documentIndex: 1 + documentIndex: 2 asserts: - equal: path: spec.template.spec.kubelet.systemReserved.cpu @@ -169,7 +168,7 @@ tests: value: "2.2.2.2" - it: Verify nodes-workers Options - documentIndex: 1 + documentIndex: 2 asserts: - equal: path: spec.disruption.expireAfter diff --git a/charts/karpenter_nodes/tests/values.yaml b/charts/karpenter_nodes/tests/values.yaml index c59c6f3..d1273dc 100644 --- a/charts/karpenter_nodes/tests/values.yaml +++ b/charts/karpenter_nodes/tests/values.yaml @@ -21,6 +21,11 @@ nodeGroups: excludeFamilies: - c6a + nodes-noclass: + instances: {} + nodeClassRef: + name: default + nodes-workers: additionalNodeTags: testtag1: tag1 @@ -76,7 +81,6 @@ nodeGroups: environment: test - name: my-ami - id: ami-123 - instances: architecture: "arm64" minGeneration: 5