update containers for linux + container for win

containers/buildkite-linux/Dockerfile

@@ -1,24 +1,28 @@
-FROM debian:stable
+FROM ubuntu:latest
 
 RUN echo 'intall packages'; \
     apt-get update; \
+    apt-get upgrade; \
     apt-get install -y --no-install-recommends \
+        gosu \
         locales openssh-client gnupg ca-certificates  \
-        zip wget git \
+        zip wget curl git \
         gdb build-essential  \
         ninja-build \
-        libelf-dev libffi-dev gcc-multilib \
+        libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \
         # for llvm-libc tests that build mpfr and gmp from source
         autoconf automake libtool \
-        ccache \
+        python3 python3-psutil python3-pip python3-setuptools \
-        python3 python3-psutil \
-        python3-pip python3-setuptools \
         lsb-release software-properties-common \
         swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq \
         # for llvm installation script
-        sudo;
+        sudo \
+        # build scripts
+        nodejs ccache \
+	# shell users
+	less vim
 
-# debian stable cmake is 3.18, we need to install a more recent version.
+# Install cmake 3.23+ from source.
 RUN wget --no-verbose -O /cmake.sh https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3-linux-x86_64.sh; \
     chmod +x /cmake.sh; \
     mkdir -p /etc/cmake; \
@@ -27,36 +31,42 @@ RUN wget --no-verbose -O /cmake.sh https://github.com/Kitware/CMake/releases/dow
     cmake --version; \
     rm /cmake.sh
 
+# Install sccache.
+RUN curl -o sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz \
+    && echo "4bf3ce366aa02599019093584a5cbad4df783f8d6e3610548c2044daa595d40b  sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz" | shasum -a 256 -c \
+    && tar xzf ./sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz \
+    && mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/bin \
+    && chown root:root /usr/bin/sccache \
+    && ls -la /usr/bin/sccache \
+    && sccache --version
+
 # LLVM must be installed after prerequsite packages.
 ENV LLVM_VERSION=16
-RUN echo 'install llvm ${LLVM_VERSION}'; \
+RUN echo 'install llvm ${LLVM_VERSION}' && \
-    wget --no-verbose https://apt.llvm.org/llvm.sh; \
+    wget --no-verbose https://apt.llvm.org/llvm.sh && \
-    chmod +x llvm.sh; \
+    chmod +x llvm.sh && \
-    ./llvm.sh ${LLVM_VERSION};\
+    ./llvm.sh ${LLVM_VERSION} && \
-    apt-get update; \
+    apt-get update && \
-    apt install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION}; \
+    apt-get install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION} && \
-    ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang;\
+    ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang && \
-    ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++;\
+    ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ && \
-    ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy;\
+    ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy && \
-    ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff;\
+    ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff && \
-    ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format;\
+    ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format && \
-    ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff;\
+    ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff && \
-    ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld;\
+    ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld && \
-    ln -s /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb;\
+    ln -s /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb && \
-    ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld
+    ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld && \
+    clang --version
 
-RUN echo 'configure locale'; \
+RUN echo 'configure locale' && \
-    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
+    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen && \
-    locale-gen ;\
+    locale-gen
-    echo 'make python 3 default'; \
-    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \
-    pip3 install wheel
-
-# Configure locale
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+
 RUN echo 'install buildkite' ;\
     apt-get install -y apt-transport-https gnupg;\
     sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\
@@ -64,6 +74,7 @@ RUN echo 'install buildkite' ;\
     apt-get update ;\
     apt-get install -y buildkite-agent tini gosu; \
     apt-get clean;
+
 COPY *.sh /usr/local/bin/
 RUN chmod og+rx /usr/local/bin/*.sh
 COPY --chown=buildkite-agent:buildkite-agent pre-checkout /etc/buildkite-agent/hooks

kubernetes/buildkite/linux-agents-big.yaml

@@ -0,0 +1,77 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-linux-buildkite
+  namespace: buildkite
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: test-linux-buildkite
+  template:
+    metadata:
+      labels:
+        app: test-linux-buildkite
+    spec:
+      containers:
+        - name: buildkite-linux
+          image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
+          resources:
+            limits:
+              cpu: 110
+              memory: 420Gi
+            requests:
+              cpu: 110
+              memory: 420Gi
+          volumeMounts:
+            - name: github-ssh
+              mountPath: /mnt/ssh
+            - name: workdir
+              mountPath: /var/lib/buildkite-agent
+          env:
+            - name: BUILDKITE_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-agent-token
+                  key: token
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                 fieldPath: metadata.name
+            - name: BUILDKITE_AGENT_TAGS
+              value: "queue=linux-test,name=$(POD_NAME),cluster=google"
+            - name: BUILDKITE_BUILD_PATH
+              value: "/var/lib/buildkite-agent/builds"
+            - name: BUILDKITE_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-api-token-readonly
+                  key: token
+      volumes:
+        - name: github-ssh
+          secret:
+            secretName: github-ssh
+        - name: workdir
+          emptyDir: {}
+      nodeSelector:
+        cloud.google.com/gke-nodepool: linux-big
+      terminationGracePeriodSeconds: 30

kubernetes/buildkite/linux-agents-test.yaml

@@ -15,32 +15,33 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: linux-agents-test
+  name: buildkite-linux-agents-test
   namespace: buildkite
 spec:
-  replicas: 0
+  replicas: 1
   strategy:
     rollingUpdate:
-      maxSurge: 25%
+      maxUnavailable: 1
-      maxUnavailable: 50%
+      maxSurge: 0
+    type: RollingUpdate
   selector:
     matchLabels:
-      app: agent-premerge-debian
+      app: linux-agents-test
   template:
     metadata:
       labels:
-        app: agent-premerge-debian
+        app: linux-agents-test
     spec:
       containers:
-        - name: test-debian
+        - name: runner
-          image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest
+          image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
           resources:
             limits:
-              cpu: 30
+              cpu: 110
-              memory: 80Gi
+              memory: 420Gi
             requests:
-              cpu: 30
+              cpu: 110
-              memory: 80Gi
+              memory: 420Gi
           volumeMounts:
             - name: github-ssh
               mountPath: /mnt/ssh
@@ -57,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux-google-test,name=$(POD_NAME)"
+              value: "queue=linux-test,name=$(POD_NAME),cluster=google"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN
@@ -77,5 +78,5 @@ spec:
         - name: workdir
           emptyDir: {}
       nodeSelector:
-        cloud.google.com/gke-nodepool: linux-agents-2
+        cloud.google.com/gke-nodepool: linux-big
-      terminationGracePeriodSeconds: 3600
+      terminationGracePeriodSeconds: 30

kubernetes/buildkite/service-agents-test.yaml

@@ -15,21 +15,26 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: service-agents-test
+  name: buildkite-service-agents-test
   namespace: buildkite
 spec:
   replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+    type: RollingUpdate
   selector:
     matchLabels:
-      app: agent-premerge-debian
+      app: buildkite-service-agents
   template:
     metadata:
       labels:
-        app: agent-premerge-debian
+        app: buildkite-service-agents
     spec:
       containers:
-        - name: buildkite-premerge-debian
+        - name: runner
-          image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest
+          image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
           resources:
             limits:
               cpu: 2
@@ -70,4 +75,4 @@ spec:
             secretName: github-ssh
       nodeSelector:
         cloud.google.com/gke-nodepool: default-pool
-      terminationGracePeriodSeconds: 1200
+      terminationGracePeriodSeconds: 60

kubernetes/buildkite/service-agents.yaml

@@ -75,4 +75,4 @@ spec:
             secretName: github-ssh
       nodeSelector:
         cloud.google.com/gke-nodepool: default-pool
-      terminationGracePeriodSeconds: 1200
+      terminationGracePeriodSeconds: 30

kubernetes/buildkite/windows-agents.yaml

@@ -16,7 +16,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: test-windows-buildkite
-  namespace: buildkite
 spec:
   replicas: 1
   strategy:
@@ -39,6 +38,9 @@ spec:
             requests:
               cpu: 110
               memory: 420Gi
+          volumeMounts:
+            - name: workdir
+              mountPath: "C:\\ws"
           env:
             - name: buildkiteAgentToken
               valueFrom:
@@ -50,14 +52,20 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=windows-test-pr,name=$(POD_NAME),cluster=google"
+              value: "queue=windows-test,name=w,cluster=google"
+            - name: BUILDKITE_BUILD_CHECKOUT_PATH
+              # TODO: need to drop it if BUILDKITE_REPO is different.
+              value: "c:\\src"
             - name: BUILDKITE_API_TOKEN
               valueFrom:
                 secretKeyRef:
                   name: buildkite-api-token-readonly
                   key: token
+      volumes:
+        - name: workdir
+          emptyDir: {}
       nodeSelector:
-        cloud.google.com/gke-nodepool: linux-big
+        cloud.google.com/gke-nodepool: windows
       tolerations:
       - key: "node.kubernetes.io/os"
         operator: "Equal"