From 47d20a3302e7a93e1324f18a297bec44bf00b7cf Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Thu, 7 Sep 2023 13:08:28 +0200 Subject: [PATCH] misc ci updates don't checkout to the same repo on linux - that actually saves time --tags-from-gcp for easy targeting and agent info --- containers/buildkite-linux/Dockerfile | 2 +- containers/buildkite-linux/pre-checkout | 4 +- kubernetes/buildkite/linux-56.yaml | 82 ----------------------- kubernetes/buildkite/linux-test.yaml | 2 +- kubernetes/buildkite/linux.yaml | 20 +++--- kubernetes/buildkite/service-test.yaml | 2 +- kubernetes/buildkite/service.yaml | 2 +- kubernetes/buildkite/windows-test-32.yaml | 2 +- kubernetes/buildkite/windows.yaml | 2 +- 9 files changed, 18 insertions(+), 100 deletions(-) delete mode 100644 kubernetes/buildkite/linux-56.yaml diff --git a/containers/buildkite-linux/Dockerfile b/containers/buildkite-linux/Dockerfile index 7066e68..6ed5e44 100644 --- a/containers/buildkite-linux/Dockerfile +++ b/containers/buildkite-linux/Dockerfile @@ -84,4 +84,4 @@ COPY --chown=buildkite-agent:buildkite-agent pre-checkout /etc/buildkite-agent/h VOLUME /var/lib/buildkite-agent ENTRYPOINT ["entrypoint.sh"] -CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"] +CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color", "--tags-from-gcp"] diff --git a/containers/buildkite-linux/pre-checkout b/containers/buildkite-linux/pre-checkout index 0ce2562..b25b751 100755 --- a/containers/buildkite-linux/pre-checkout +++ b/containers/buildkite-linux/pre-checkout @@ -21,7 +21,7 @@ set -o pipefail # Convert https://github.com/llvm-premerge-tests/llvm-project.git -> llvm-project # to use the same directory for fork and origin. -BUILDKITE_BUILD_CHECKOUT_PATH="${BUILDKITE_BUILD_PATH}/$(echo $BUILDKITE_REPO | sed -E "s#.*/([^/]*)#\1#" | sed "s/.git$//")" +# BUILDKITE_BUILD_CHECKOUT_PATH="${BUILDKITE_BUILD_PATH}/$(echo $BUILDKITE_REPO | sed -E "s#.*/([^/]*)#\1#" | sed "s/.git$//")" gc_counter="$BUILDKITE_BUILD_PATH/git_gc_counter" echo "BUILDKITE_REPO: $BUILDKITE_REPO" if [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then @@ -30,7 +30,7 @@ if [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then echo "current remote URL: $remoteUrl" if [ "$remoteUrl" != "$BUILDKITE_REPO" ]; then echo "Remote URL does not match. Deleting and recreating the directory." - cd /c/ + cd $BUILDKITE_BUILD_PATH rm -rf "$BUILDKITE_BUILD_CHECKOUT_PATH" rm -rf "$gc_counter" else diff --git a/kubernetes/buildkite/linux-56.yaml b/kubernetes/buildkite/linux-56.yaml deleted file mode 100644 index 4284acb..0000000 --- a/kubernetes/buildkite/linux-56.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: linux-56 - namespace: buildkite -spec: - replicas: 3 - strategy: - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - type: RollingUpdate - selector: - matchLabels: - app: linux-56 - template: - metadata: - labels: - app: linux-56 - spec: - containers: - - name: runner - image: gcr.io/llvm-premerge-checks/buildkite-linux:latest - resources: - limits: - cpu: 55 - memory: 200Gi - requests: - cpu: 55 - memory: 200Gi - volumeMounts: - - name: github-ssh - mountPath: /mnt/ssh - - name: workdir - mountPath: /var/lib/buildkite-agent - env: - - name: BUILDKITE_AGENT_TOKEN - valueFrom: - secretKeyRef: - name: buildkite-agent-token - key: token - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: BUILDKITE_AGENT_TAGS - value: "queue=linux,name=$(POD_NAME),cluster=google,node-pool=linux-56" - - name: BUILDKITE_BUILD_PATH - value: "/var/lib/buildkite-agent/builds" - - name: CONDUIT_TOKEN - valueFrom: - secretKeyRef: - name: conduit-api-token - key: token - - name: BUILDKITE_API_TOKEN - valueFrom: - secretKeyRef: - name: buildkite-api-token-readonly - key: token - volumes: - - name: github-ssh - secret: - secretName: github-ssh - - name: workdir - emptyDir: {} - nodeSelector: - cloud.google.com/gke-nodepool: linux-56 - terminationGracePeriodSeconds: 30 diff --git a/kubernetes/buildkite/linux-test.yaml b/kubernetes/buildkite/linux-test.yaml index b590399..b5891c5 100644 --- a/kubernetes/buildkite/linux-test.yaml +++ b/kubernetes/buildkite/linux-test.yaml @@ -58,7 +58,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux-test,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2" + value: "queue=linux-test,pod-name=$(POD_NAME)" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN diff --git a/kubernetes/buildkite/linux.yaml b/kubernetes/buildkite/linux.yaml index 9b238d1..ccac87c 100644 --- a/kubernetes/buildkite/linux.yaml +++ b/kubernetes/buildkite/linux.yaml @@ -15,10 +15,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: linux + name: linux-56 namespace: buildkite spec: - replicas: 2 + replicas: 3 strategy: rollingUpdate: maxUnavailable: 1 @@ -26,22 +26,22 @@ spec: type: RollingUpdate selector: matchLabels: - app: linux + app: linux-56 template: metadata: labels: - app: linux + app: linux-56 spec: containers: - name: runner image: gcr.io/llvm-premerge-checks/buildkite-linux:latest resources: limits: - cpu: 30 - memory: 80Gi + cpu: 55 + memory: 200Gi requests: - cpu: 30 - memory: 80Gi + cpu: 55 + memory: 200Gi volumeMounts: - name: github-ssh mountPath: /mnt/ssh @@ -58,7 +58,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2" + value: "queue=linux,pod-name=$(POD_NAME)" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN @@ -78,5 +78,5 @@ spec: - name: workdir emptyDir: {} nodeSelector: - cloud.google.com/gke-nodepool: linux-agents-2 + cloud.google.com/gke-nodepool: linux-56 terminationGracePeriodSeconds: 30 diff --git a/kubernetes/buildkite/service-test.yaml b/kubernetes/buildkite/service-test.yaml index 065a32f..bb708d9 100644 --- a/kubernetes/buildkite/service-test.yaml +++ b/kubernetes/buildkite/service-test.yaml @@ -56,7 +56,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=service-test,name=$(POD_NAME),cluster=google" + value: "queue=service-test,pod-name=$(POD_NAME)" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN diff --git a/kubernetes/buildkite/service.yaml b/kubernetes/buildkite/service.yaml index 6b0571d..1276c1e 100644 --- a/kubernetes/buildkite/service.yaml +++ b/kubernetes/buildkite/service.yaml @@ -56,7 +56,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=service,name=$(POD_NAME),cluster=google" + value: "queue=service,pod-name=$(POD_NAME)" - name: CONDUIT_TOKEN valueFrom: secretKeyRef: diff --git a/kubernetes/buildkite/windows-test-32.yaml b/kubernetes/buildkite/windows-test-32.yaml index f1bd12b..124809e 100644 --- a/kubernetes/buildkite/windows-test-32.yaml +++ b/kubernetes/buildkite/windows-test-32.yaml @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=windows-test-32,cluster=google" + value: "queue=windows-test-32,pod-name=$(POD_NAME)" - name: BUILDKITE_API_TOKEN valueFrom: secretKeyRef: diff --git a/kubernetes/buildkite/windows.yaml b/kubernetes/buildkite/windows.yaml index 5f636a7..5f7e7f5 100644 --- a/kubernetes/buildkite/windows.yaml +++ b/kubernetes/buildkite/windows.yaml @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=windows,cluster=google" + value: "queue=windows,pod-name=$(POD_NAME)" - name: BUILDKITE_API_TOKEN valueFrom: secretKeyRef: