From 3bdfcac1cdb0e62a4f01eb566cdb83a58a4f7756 Mon Sep 17 00:00:00 2001 From: Aidan Hilt Date: Thu, 16 Jan 2025 10:52:52 -0500 Subject: [PATCH 1/5] Let's add the new cost-tracking tags to workflows --- kube/services/argo-events/workflows/configmap.yaml | 3 +++ .../argo-events/workflows/sensor-created.yaml | 11 +++++++++-- .../karpenter-reconciler-cronjob.yaml | 12 ++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index c084533fe..f0b1b4742 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -42,6 +42,9 @@ data: karpenter.sh/discovery: ENVIRONMENT purpose: workflow workflowname: WORKFLOW_NAME + "vadc:cost-type": user-based-variable + "vadc:usage-type": user-infrastructure + "vadc:environment-type": "ENVIRONMENT_TYPE_CODE" userData: | MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="BOUNDARY" diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml index 05da3bc38..d98dc6a10 100644 --- a/kube/services/argo-events/workflows/sensor-created.yaml +++ b/kube/services/argo-events/workflows/sensor-created.yaml @@ -64,6 +64,13 @@ spec: - "-c" - | #!/bin/bash + if [ "$ENVIRONMENT" = "vhdcprod" ]; then + ENVIRONMENT_TYPE_CODE="production" + elif [ "$ENVIRONMENT" = "va-testing" ]; then + ENVIRONMENT_TYPE_CODE="testing" + else; + ENVIRONMENT_TYPE_CODE="environment" + fi if [ -z "$NODEPOOL_TEMPLATE" ]; then NODEPOOL_TEMPLATE="/manifests/nodepool.yaml" @@ -74,11 +81,11 @@ spec: fi if ! kubectl get ec2nodeclass workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - fi if ! kubectl get nodepool workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - fi env: - name: WORKFLOW_NAME diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 3c6d58768..89968f24d 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -33,6 +33,14 @@ spec: - "-c" - | #!/bin/bash + if [ "$ENVIRONMENT" = "vhdcprod" ]; then + ENVIRONMENT_TYPE_CODE="production" + elif [ "$ENVIRONMENT" = "va-testing" ]; then + ENVIRONMENT_TYPE_CODE="testing" + else; + ENVIRONMENT_TYPE_CODE="environment" + fi + if [ -z "$NODEPOOL_TEMPLATE" ]; then NODEPOOL_TEMPLATE="/manifests/provisioner.yaml" fi @@ -62,12 +70,12 @@ spec: if [ ! -z "$workflow_name" ]; then if ! kubectl get ec2nodeclass workflow-$workflow_name >/dev/null 2>&1; then echo "No awsnodetemplate found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - fi if ! kubectl get nodepool workflow-$workflow_name >/dev/null 2>&1; then echo "No provisioner found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - fi fi From 2a3b9d6fa69167e55dba641b46647cb5425ce3c9 Mon Sep 17 00:00:00 2001 From: Aidan Hilt Date: Thu, 16 Jan 2025 12:47:55 -0500 Subject: [PATCH 2/5] Bash syntax errors --- kube/services/argo-events/workflows/sensor-created.yaml | 4 ++-- .../karpenter-reconciler/karpenter-reconciler-cronjob.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml index d98dc6a10..351432872 100644 --- a/kube/services/argo-events/workflows/sensor-created.yaml +++ b/kube/services/argo-events/workflows/sensor-created.yaml @@ -68,8 +68,8 @@ spec: ENVIRONMENT_TYPE_CODE="production" elif [ "$ENVIRONMENT" = "va-testing" ]; then ENVIRONMENT_TYPE_CODE="testing" - else; - ENVIRONMENT_TYPE_CODE="environment" + else + ENVIRONMENT_TYPE_CODE="$ENVIRONMENT" fi if [ -z "$NODEPOOL_TEMPLATE" ]; then diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 89968f24d..4f5388e63 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -37,7 +37,7 @@ spec: ENVIRONMENT_TYPE_CODE="production" elif [ "$ENVIRONMENT" = "va-testing" ]; then ENVIRONMENT_TYPE_CODE="testing" - else; + else ENVIRONMENT_TYPE_CODE="environment" fi From 913523b1762feb84802b6047b4ccfc35435b2a97 Mon Sep 17 00:00:00 2001 From: Aidan Hilt Date: Thu, 16 Jan 2025 12:50:42 -0500 Subject: [PATCH 3/5] Sed syntax error --- kube/services/argo-events/workflows/sensor-created.yaml | 4 ++-- .../karpenter-reconciler/karpenter-reconciler-cronjob.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml index 351432872..9d5fd7082 100644 --- a/kube/services/argo-events/workflows/sensor-created.yaml +++ b/kube/services/argo-events/workflows/sensor-created.yaml @@ -81,11 +81,11 @@ spec: fi if ! kubectl get ec2nodeclass workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE/" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - fi if ! kubectl get nodepool workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE/" -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/GEN3_TEAMNAME/$GEN3_TEAMNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - fi env: - name: WORKFLOW_NAME diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 4f5388e63..948e24932 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -70,12 +70,12 @@ spec: if [ ! -z "$workflow_name" ]; then if ! kubectl get ec2nodeclass workflow-$workflow_name >/dev/null 2>&1; then echo "No awsnodetemplate found for ${workflow_name}, creating one" - sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE/" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODECLASS_TEMPLATE" | kubectl apply -f - fi if ! kubectl get nodepool workflow-$workflow_name >/dev/null 2>&1; then echo "No provisioner found for ${workflow_name}, creating one" - sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - + sed -e "s/ENVIRONMENT_TYPE_CODE/$ENVIRONMENT_TYPE_CODE/" -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_TEAMNAME/$workflow_team/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$NODEPOOL_TEMPLATE" | kubectl apply -f - fi fi From 07960839ec909fafc7e7a6e5f71264b0b399d0d4 Mon Sep 17 00:00:00 2001 From: Aidan Hilt Date: Thu, 16 Jan 2025 13:27:04 -0500 Subject: [PATCH 4/5] Bad order --- .../karpenter-reconciler/karpenter-reconciler-cronjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 948e24932..549bc77ae 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -33,6 +33,8 @@ spec: - "-c" - | #!/bin/bash + ENVIRONMENT=$(kubectl -n default get configmap global -o jsonpath="{.data.environment}") + if [ "$ENVIRONMENT" = "vhdcprod" ]; then ENVIRONMENT_TYPE_CODE="production" elif [ "$ENVIRONMENT" = "va-testing" ]; then @@ -49,8 +51,6 @@ spec: NODECLASS_TEMPLATE="/manifests/nodeclass.yaml" fi - ENVIRONMENT=$(kubectl -n default get configmap global -o jsonpath="{.data.environment}") - WORKFLOWS=$(kubectl get workflows -n argo -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.gen3username}{" "}{.metadata.labels.gen3teamproject}{"\n"}') WORKFLOW_ARRAY=() From 338afd4eb7c8332370692316bc1b04bd0a14c232 Mon Sep 17 00:00:00 2001 From: Aidan Hilt Date: Thu, 16 Jan 2025 13:33:46 -0500 Subject: [PATCH 5/5] Better substitution --- .../karpenter-reconciler/karpenter-reconciler-cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 549bc77ae..86330f3a1 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -40,7 +40,7 @@ spec: elif [ "$ENVIRONMENT" = "va-testing" ]; then ENVIRONMENT_TYPE_CODE="testing" else - ENVIRONMENT_TYPE_CODE="environment" + ENVIRONMENT_TYPE_CODE="$ENVIRONMENT" fi if [ -z "$NODEPOOL_TEMPLATE" ]; then