[release] set gcp artifact cleanup policies on deploy (#14198)

Adds ci step to set gcloud artifact registry cleaup policies on every deploy. fixes: #14154
hail-is · Feb 6, 2024 · 977eb75 · 977eb75
1 parent 14c9457
commit 977eb75
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 11 deletions.
diff --git a/build.yaml b/build.yaml
@@ -3934,3 +3934,26 @@ steps:
     dependsOn:
       - ci_utils_image
       - default_ns
+  - kind: runImage
+    name: set_gcp_ar_cleanup_policies
+    resources:
+      memory: standard
+      cpu: '0.25'
+    image:
+      valueFrom: ci_utils_image.image
+    scopes:
+      - deploy
+    script: |
+      gcloud artifacts repositories set-cleanup-policies hail \
+        --project={{ global.gcp_project }} \
+        --location=us \
+        --policy=/io/repo/infra/gcp-broad/gcp-ar-cleanup-policy.txt \
+        --no-dry-run
+    inputs:
+      - from: /repo/infra
+        to: /io/repo/infra
+    clouds:
+      - gcp
+    dependsOn:
+      - test_gcp_ar_cleanup_policies
+      - ci_utils_image
diff --git a/hail/Makefile b/hail/Makefile
@@ -60,7 +60,6 @@ GRADLE_ARGS += -Dscala.version=$(SCALA_VERSION) -Dspark.version=$(SPARK_VERSION)
 
 TEST_STORAGE_URI = $(shell kubectl get secret global-config --template={{.data.test_storage_uri}} | base64 --decode)
 HAIL_TEST_GCS_BUCKET = $(shell kubectl get secret global-config --template={{.data.hail_test_gcs_bucket}} | base64 --decode)
-GCP_PROJECT = $(shell kubectl get secret global-config --template={{.data.gcp_project}} | base64 --decode)
 CLOUD_HAIL_TEST_RESOURCES_PREFIX = $(TEST_STORAGE_URI)/$(shell whoami)/hail-test-resources
 CLOUD_HAIL_TEST_RESOURCES_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/test/resources/
 CLOUD_HAIL_DOCTEST_DATA_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/doctest/data/
@@ -452,7 +451,21 @@ test-dataproc-38: install-hailctl
 #
 .PHONY: release
 release: test-dataproc $(WHEEL)
-	bash scripts/release.sh $(HAIL_PIP_VERSION) $(HAIL_VERSION) $(REVISION) $(DEPLOY_REMOTE) $(WHEEL) $(GITHUB_OAUTH_HEADER_FILE) $(HAIL_GENETICS_HAIL_IMAGE) $(HAIL_GENETICS_HAIL_IMAGE_PY_3_10) $(HAIL_GENETICS_HAIL_IMAGE_PY_3_11) $(HAIL_GENETICS_HAILTOP_IMAGE) $(HAIL_GENETICS_VEP_GRCH37_85_IMAGE) $(HAIL_GENETICS_VEP_GRCH38_95_IMAGE) $(WHEEL_FOR_AZURE) $(WEBSITE_TAR) $(GCP_PROJECT) $(HAIL_HAIL_DIR)/../infra/gcp-broad/gcp-ar-cleanup-policy.txt
+	bash scripts/release.sh \
+		$(HAIL_PIP_VERSION) \
+		$(HAIL_VERSION) \
+		$(REVISION) \
+		$(DEPLOY_REMOTE) \
+		$(WHEEL) \
+		$(GITHUB_OAUTH_HEADER_FILE) \
+		$(HAIL_GENETICS_HAIL_IMAGE) \
+		$(HAIL_GENETICS_HAIL_IMAGE_PY_3_10) \
+		$(HAIL_GENETICS_HAIL_IMAGE_PY_3_11) \
+		$(HAIL_GENETICS_HAILTOP_IMAGE) \
+		$(HAIL_GENETICS_VEP_GRCH37_85_IMAGE) \
+		$(HAIL_GENETICS_VEP_GRCH38_95_IMAGE) \
+		$(WHEEL_FOR_AZURE) \
+		$(WEBSITE_TAR)
 
 python/hail/docs/change_log.rst: python/hail/docs/change_log.md
 	sed -E "s/\(hail\#([0-9]+)\)/(\[#\1](https:\/\/github.com\/hail-is\/hail\/pull\/\1))/g" \

diff --git a/hail/scripts/release.sh b/hail/scripts/release.sh
@@ -4,13 +4,34 @@ set -ex
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+usage() {
+cat << EOF
+USAGE:
+$(basename "$0") \\
+  HAIL_PIP_VERSION \\
+  HAIL_VERSION \\
+  GIT_VERSION \\
+  REMOTE \\
+  WHEEL \\
+  GITHUB_OAUTH_HEADER_FILE \\
+  HAIL_GENETICS_HAIL_IMAGE \\
+  HAIL_GENETICS_HAIL_IMAGE_PY_3_10 \\
+  HAIL_GENETICS_HAIL_IMAGE_PY_3_11 \\
+  HAIL_GENETICS_HAILTOP_IMAGE \\
+  HAIL_GENETICS_VEP_GRCH37_85_IMAGE \\
+  HAIL_GENETICS_VEP_GRCH38_95_IMAGE \\
+  WHEEL_FOR_AZURE \\
+  WEBSITE_TAR
+EOF
+}
+
 retry() {
     "$@" ||
         (sleep 2 && "$@") ||
         (sleep 5 && "$@");
 }
 
-[[ $# -eq 16 ]] || (echo "./release.sh HAIL_PIP_VERSION HAIL_VERSION GIT_VERSION REMOTE WHEEL GITHUB_OAUTH_HEADER_FILE HAIL_GENETICS_HAIL_IMAGE HAIL_GENETICS_HAIL_IMAGE_PY_3_10 HAIL_GENETICS_HAIL_IMAGE_PY_3_11 HAIL_GENETICS_HAILTOP_IMAGE HAIL_GENETICS_VEP_GRCH37_85_IMAGE HAIL_GENETICS_VEP_GRCH38_95_IMAGE WHEEL_FOR_AZURE WEBSITE_TAR GCP_PROJECT GCP_AR_CLEANUP_POLICY" ; exit 1)
+[[ $# -eq 14 ]] || usage; exit 1
 
 HAIL_PIP_VERSION=$1
 HAIL_VERSION=$2
@@ -26,8 +47,6 @@ HAIL_GENETICS_VEP_GRCH37_85_IMAGE=${11}
 HAIL_GENETICS_VEP_GRCH38_95_IMAGE=${12}
 WHEEL_FOR_AZURE=${13}
 WEBSITE_TAR=${14}
-GCP_PROJECT=${15}
-GCP_AR_CLEANUP_POLICY=${16}
 
 retry skopeo inspect $HAIL_GENETICS_HAIL_IMAGE || (echo "could not pull $HAIL_GENETICS_HAIL_IMAGE" ; exit 1)
 retry skopeo inspect $HAIL_GENETICS_HAIL_IMAGE_PY_3_10 || (echo "could not pull $HAIL_GENETICS_HAIL_IMAGE_PY_3_10" ; exit 1)
@@ -160,9 +179,3 @@ make_pr_for() {
 
 make_pr_for terra-jupyter-hail
 make_pr_for terra-jupyter-aou
-
-gcloud artifacts repositories set-cleanup-policies hail \
-    --project=$GCP_PROJECT \
-    --location=us \
-    --policy=$GCP_AR_CLEANUP_POLICY \
-    --no-dry-run