misc ci updates

don't checkout to the same repo on linux - that actually saves time

--tags-from-gcp for easy targeting and agent info

containers/buildkite-linux/Dockerfile

@@ -84,4 +84,4 @@ COPY --chown=buildkite-agent:buildkite-agent pre-checkout /etc/buildkite-agent/h
 VOLUME /var/lib/buildkite-agent
 
 ENTRYPOINT ["entrypoint.sh"]
-CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"]
+CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color", "--tags-from-gcp"]

containers/buildkite-linux/pre-checkout

@@ -21,7 +21,7 @@ set -o pipefail
 
 # Convert https://github.com/llvm-premerge-tests/llvm-project.git -> llvm-project
 # to use the same directory for fork and origin.
-BUILDKITE_BUILD_CHECKOUT_PATH="${BUILDKITE_BUILD_PATH}/$(echo $BUILDKITE_REPO | sed -E "s#.*/([^/]*)#\1#" | sed "s/.git$//")"
+# BUILDKITE_BUILD_CHECKOUT_PATH="${BUILDKITE_BUILD_PATH}/$(echo $BUILDKITE_REPO | sed -E "s#.*/([^/]*)#\1#" | sed "s/.git$//")"
 gc_counter="$BUILDKITE_BUILD_PATH/git_gc_counter"
 echo "BUILDKITE_REPO: $BUILDKITE_REPO"
 if [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then
@@ -30,7 +30,7 @@ if [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then
     echo "current remote URL: $remoteUrl"
     if [ "$remoteUrl" != "$BUILDKITE_REPO" ]; then
         echo "Remote URL does not match. Deleting and recreating the directory."
-        cd /c/
+        cd $BUILDKITE_BUILD_PATH
         rm -rf "$BUILDKITE_BUILD_CHECKOUT_PATH"
         rm -rf "$gc_counter"
     else

kubernetes/buildkite/linux-56.yaml

@@ -1,82 +0,0 @@
-# Copyright 2021 Google LLC
-#
-# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://llvm.org/LICENSE.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: linux-56
-  namespace: buildkite
-spec:
-  replicas: 3
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 1
-      maxSurge: 0
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: linux-56
-  template:
-    metadata:
-      labels:
-        app: linux-56
-    spec:
-      containers:
-        - name: runner
-          image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
-          resources:
-            limits:
-              cpu: 55
-              memory: 200Gi
-            requests:
-              cpu: 55
-              memory: 200Gi
-          volumeMounts:
-            - name: github-ssh
-              mountPath: /mnt/ssh
-            - name: workdir
-              mountPath: /var/lib/buildkite-agent
-          env:
-            - name: BUILDKITE_AGENT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: buildkite-agent-token
-                  key: token
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                 fieldPath: metadata.name
-            - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux,name=$(POD_NAME),cluster=google,node-pool=linux-56"
-            - name: BUILDKITE_BUILD_PATH
-              value: "/var/lib/buildkite-agent/builds"
-            - name: CONDUIT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: conduit-api-token
-                  key: token
-            - name: BUILDKITE_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: buildkite-api-token-readonly
-                  key: token
-      volumes:
-        - name: github-ssh
-          secret:
-            secretName: github-ssh
-        - name: workdir
-          emptyDir: {}
-      nodeSelector:
-        cloud.google.com/gke-nodepool: linux-56
-      terminationGracePeriodSeconds: 30

kubernetes/buildkite/linux-test.yaml

@@ -58,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux-test,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2"
+              value: "queue=linux-test,pod-name=$(POD_NAME)"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN

kubernetes/buildkite/linux.yaml

@@ -15,10 +15,10 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: linux
+  name: linux-56
   namespace: buildkite
 spec:
-  replicas: 2
+  replicas: 3
   strategy:
     rollingUpdate:
       maxUnavailable: 1
@@ -26,22 +26,22 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: linux
+      app: linux-56
   template:
     metadata:
       labels:
-        app: linux
+        app: linux-56
     spec:
       containers:
         - name: runner
           image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
           resources:
             limits:
-              cpu: 30
+              cpu: 55
-              memory: 80Gi
+              memory: 200Gi
             requests:
-              cpu: 30
+              cpu: 55
-              memory: 80Gi
+              memory: 200Gi
           volumeMounts:
             - name: github-ssh
               mountPath: /mnt/ssh
@@ -58,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2"
+              value: "queue=linux,pod-name=$(POD_NAME)"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN
@@ -78,5 +78,5 @@ spec:
         - name: workdir
           emptyDir: {}
       nodeSelector:
-        cloud.google.com/gke-nodepool: linux-agents-2
+        cloud.google.com/gke-nodepool: linux-56
       terminationGracePeriodSeconds: 30

kubernetes/buildkite/service-test.yaml

@@ -56,7 +56,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=service-test,name=$(POD_NAME),cluster=google"
+              value: "queue=service-test,pod-name=$(POD_NAME)"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN

kubernetes/buildkite/service.yaml

@@ -56,7 +56,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=service,name=$(POD_NAME),cluster=google"
+              value: "queue=service,pod-name=$(POD_NAME)"
             - name: CONDUIT_TOKEN
               valueFrom:
                 secretKeyRef:

kubernetes/buildkite/windows-test-32.yaml

@@ -52,7 +52,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=windows-test-32,cluster=google"
+              value: "queue=windows-test-32,pod-name=$(POD_NAME)"
             - name: BUILDKITE_API_TOKEN
               valueFrom:
                 secretKeyRef:

kubernetes/buildkite/windows.yaml

@@ -52,7 +52,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=windows,cluster=google"
+              value: "queue=windows,pod-name=$(POD_NAME)"
             - name: BUILDKITE_API_TOKEN
               valueFrom:
                 secretKeyRef: