Migrate Buildkite CI queues from AWS to GKE

- Hard code the ssh volume mount name - Downgrade update-deps pipeline docker compose version - Necessary since it will otherwise fail with [/tmp/github-com-buildkite-plugins-docker-compose-buildkite-plugin/hooks/../commands/run.sh: line 99: BUILDKITE_AGENT_ID: unbound variable ](https://buildkite.com/uberopensource/dubstep-core-gcp/builds/10#018ef7c9-9cc5-49e5-b83d-69f1157cc57c/62-64) - Opened a thread with Buildkite support in Slack, but this pipeline isn't using any of the newer docker-compose plugin features so I don't think it needs to be on latest
yarpc · Apr 22, 2024 · 23c069f · 23c069f
1 parent 407d76c
commit 23c069f
Show file tree

Hide file tree

Showing 3 changed files with 164 additions and 66 deletions.
diff --git a/.buildkite/pipeline-default.yml b/.buildkite/pipeline-default.yml
@@ -1,60 +1,122 @@
+container:
+  kubernetes: &kubernetes
+    gitEnvFrom:
+      - secretRef:
+          name: oss-github-ssh-credentials
+    sidecars:
+    - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
+      volumeMounts:
+        - mountPath: /var/run/
+          name: docker-sock
+      securityContext:
+        privileged: true
+        allowPrivilegeEscalation: true
+    mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars
+    podSpec: &podSpec
+      containers:
+        - &commandContainer
+          image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2
+          command:
+          - |-
+            echo "Command step was not overridden."
+            exit 1
+          volumeMounts:
+            - mountPath: /var/run/
+              name: docker-sock
+          resources:
+            requests:
+              cpu: 7500m
+              memory: 30G
+      volumes:
+      - name: docker-sock
+        emptyDir: {}
+
+agents:
+  queue: buildkite-gcp
+
 steps:
-  - name: ":docker: :package: 1.21"
-    plugins:
-      docker-compose#v5.2.0:
-        build: yarpc-go-1.21
-        push: yarpc-go-1.21:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:latest
-    agents:
-      queue: builders
-  - name: ":docker: :package: 1.22"
-    plugins:
-      docker-compose#v5.2.0:
-        build: yarpc-go-1.22
-        push: yarpc-go-1.22:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:latest
-    agents:
-      queue: builders
-  - wait
   - name: ":go: 1.21 test - %n"
-    command: "make test"
     parallelism: 2
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.21
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make test
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.21
+
   - name: ":go: 1.21 examples"
-    command: "make examples"
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.21
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make examples
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.21
+
   - name: ":go: 1.22 test - %n"
-    command: "make codecov"
     parallelism: 6
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.22
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make codecov
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.22
+
   - name: ":go: 1.22 crossdock"
-    command: "make crossdock-codecov"
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.22
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make crossdock-codecov
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.22
+
   - name: ":go: 1.22 lint"
-    command: "make lint"
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.22
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make lint
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.22
+
   - name: ":go: 1.22 examples"
-    command: "make examples"
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.22
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make examples
+      - docker-compose#v3.13.0:
+          run: yarpc-go-1.22
diff --git a/.buildkite/pipeline-update-deps.yml b/.buildkite/pipeline-update-deps.yml
@@ -1,23 +1,57 @@
+container:
+  kubernetes: &kubernetes
+    gitEnvFrom:
+      - secretRef:
+          name: oss-github-ssh-credentials
+    sidecars:
+    - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
+      volumeMounts:
+        - mountPath: /var/run/
+          name: docker-sock
+      securityContext:
+        privileged: true
+        allowPrivilegeEscalation: true
+    mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars
+    podSpec: &podSpec
+      containers:
+        - &commandContainer
+          image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2
+          command:
+          - |-
+            echo "Command step was not overridden."
+            exit 1
+          volumeMounts:
+            - mountPath: /var/run/
+              name: docker-sock
+          resources:
+            requests:
+              cpu: 7500m
+              memory: 30G
+      volumes:
+      - name: docker-sock
+        emptyDir: {}
+
+agents:
+  queue: buildkite-gcp
+
 steps:
-  - name: ":docker: :package: 1.22"
-    plugins:
-      docker-compose#v5.2.0:
-        build: yarpc-go-1.22
-        push: yarpc-go-1.22:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:latest
-    agents:
-      queue: builders
-  - wait
   - name: ":go: 1.22 update-deps"
-    command: "etc/bin/update-deps.sh"
     plugins:
-      docker-compose#v5.2.0:
-        run: yarpc-go-1.22
-        env:
-          # The script needs the following environment variables in addition
-          # to those provided by the docker-compose.
-          - GITHUB_USER
-          - GITHUB_EMAIL
-          - GITHUB_TOKEN
-          - GITHUB_REPO
-    agents:
-      queue: workers
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  etc/bin/update-deps.sh
+      - docker-compose#v3.0.0:
+          run: yarpc-go-1.22
+          env:
+            # The script needs the following environment variables in addition
+            # to those provided by the docker-compose.
+            - GITHUB_USER
+            - GITHUB_EMAIL
+            - GITHUB_TOKEN
+            - GITHUB_REPO
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -12,6 +12,7 @@ services:
       - CODECOV_TOKEN
       - CI=true
       - BUILDKITE
+      - BUILDKITE_AGENT_ID
       - BUILDKITE_BRANCH
       - BUILDKITE_BUILD_NUMBER
       - BUILDKITE_BUILD_URL
@@ -22,7 +23,7 @@ services:
       - GO111MODULE=on
       - SSH_AUTH_SOCK=/ssh-agent
     volumes:
-      - $SSH_AUTH_SOCK:/ssh-agent
+      - /ssh-agent:/ssh-agent
     # We mount the host's SSH Agent unix socket at /ssh-agent in the container
     # and tell the container where to find it so that the container can
     # actually push commits.
@@ -38,6 +39,7 @@ services:
       - CODECOV_TOKEN
       - CI=true
       - BUILDKITE
+      - BUILDKITE_AGENT_ID
       - BUILDKITE_BRANCH
       - BUILDKITE_BUILD_NUMBER
       - BUILDKITE_BUILD_URL