From 80978127eeb898598b5f261c0fe2aa624ff4a2dd Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Mon, 28 Aug 2023 12:05:01 +0200 Subject: [PATCH] update containers for linux + container for win --- .../.dockerignore | 0 .../Dockerfile | 73 ++++++++++-------- .../cloudbuild.yaml | 0 .../entrypoint.sh | 0 .../post-checkout | 0 .../pre-checkout | 0 kubernetes/buildkite/linux-agents-big.yaml | 77 +++++++++++++++++++ kubernetes/buildkite/linux-agents-test.yaml | 31 ++++---- kubernetes/buildkite/service-agents-test.yaml | 17 ++-- kubernetes/buildkite/service-agents.yaml | 2 +- kubernetes/buildkite/windows-agents.yaml | 14 +++- 11 files changed, 158 insertions(+), 56 deletions(-) rename containers/{buildkite-premerge-debian => buildkite-linux}/.dockerignore (100%) rename containers/{buildkite-premerge-debian => buildkite-linux}/Dockerfile (54%) rename containers/{buildkite-premerge-debian => buildkite-linux}/cloudbuild.yaml (100%) rename containers/{buildkite-premerge-debian => buildkite-linux}/entrypoint.sh (100%) rename containers/{buildkite-premerge-debian => buildkite-linux}/post-checkout (100%) rename containers/{buildkite-premerge-debian => buildkite-linux}/pre-checkout (100%) create mode 100644 kubernetes/buildkite/linux-agents-big.yaml diff --git a/containers/buildkite-premerge-debian/.dockerignore b/containers/buildkite-linux/.dockerignore similarity index 100% rename from containers/buildkite-premerge-debian/.dockerignore rename to containers/buildkite-linux/.dockerignore diff --git a/containers/buildkite-premerge-debian/Dockerfile b/containers/buildkite-linux/Dockerfile similarity index 54% rename from containers/buildkite-premerge-debian/Dockerfile rename to containers/buildkite-linux/Dockerfile index 1b00797..3886a53 100644 --- a/containers/buildkite-premerge-debian/Dockerfile +++ b/containers/buildkite-linux/Dockerfile @@ -1,24 +1,28 @@ -FROM debian:stable +FROM ubuntu:latest RUN echo 'intall packages'; \ apt-get update; \ + apt-get upgrade; \ apt-get install -y --no-install-recommends \ + gosu \ locales openssh-client gnupg ca-certificates \ - zip wget git \ + zip wget curl git \ gdb build-essential \ ninja-build \ - libelf-dev libffi-dev gcc-multilib \ + libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \ # for llvm-libc tests that build mpfr and gmp from source autoconf automake libtool \ - ccache \ - python3 python3-psutil \ - python3-pip python3-setuptools \ + python3 python3-psutil python3-pip python3-setuptools \ lsb-release software-properties-common \ swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq \ # for llvm installation script - sudo; + sudo \ + # build scripts + nodejs ccache \ + # shell users + less vim -# debian stable cmake is 3.18, we need to install a more recent version. +# Install cmake 3.23+ from source. RUN wget --no-verbose -O /cmake.sh https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3-linux-x86_64.sh; \ chmod +x /cmake.sh; \ mkdir -p /etc/cmake; \ @@ -27,36 +31,42 @@ RUN wget --no-verbose -O /cmake.sh https://github.com/Kitware/CMake/releases/dow cmake --version; \ rm /cmake.sh +# Install sccache. +RUN curl -o sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz \ + && echo "4bf3ce366aa02599019093584a5cbad4df783f8d6e3610548c2044daa595d40b sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz" | shasum -a 256 -c \ + && tar xzf ./sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz \ + && mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/bin \ + && chown root:root /usr/bin/sccache \ + && ls -la /usr/bin/sccache \ + && sccache --version + # LLVM must be installed after prerequsite packages. ENV LLVM_VERSION=16 -RUN echo 'install llvm ${LLVM_VERSION}'; \ - wget --no-verbose https://apt.llvm.org/llvm.sh; \ - chmod +x llvm.sh; \ - ./llvm.sh ${LLVM_VERSION};\ - apt-get update; \ - apt install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION}; \ - ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang;\ - ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++;\ - ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy;\ - ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff;\ - ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format;\ - ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff;\ - ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld;\ - ln -s /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb;\ - ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld +RUN echo 'install llvm ${LLVM_VERSION}' && \ + wget --no-verbose https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} && \ + apt-get update && \ + apt-get install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION} && \ + ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang && \ + ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ && \ + ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy && \ + ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff && \ + ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format && \ + ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff && \ + ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld && \ + ln -s /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb && \ + ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld && \ + clang --version -RUN echo 'configure locale'; \ - sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen ;\ - echo 'make python 3 default'; \ - rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \ - pip3 install wheel - -# Configure locale +RUN echo 'configure locale' && \ + sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen && \ + locale-gen ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 + RUN echo 'install buildkite' ;\ apt-get install -y apt-transport-https gnupg;\ sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\ @@ -64,6 +74,7 @@ RUN echo 'install buildkite' ;\ apt-get update ;\ apt-get install -y buildkite-agent tini gosu; \ apt-get clean; + COPY *.sh /usr/local/bin/ RUN chmod og+rx /usr/local/bin/*.sh COPY --chown=buildkite-agent:buildkite-agent pre-checkout /etc/buildkite-agent/hooks diff --git a/containers/buildkite-premerge-debian/cloudbuild.yaml b/containers/buildkite-linux/cloudbuild.yaml similarity index 100% rename from containers/buildkite-premerge-debian/cloudbuild.yaml rename to containers/buildkite-linux/cloudbuild.yaml diff --git a/containers/buildkite-premerge-debian/entrypoint.sh b/containers/buildkite-linux/entrypoint.sh similarity index 100% rename from containers/buildkite-premerge-debian/entrypoint.sh rename to containers/buildkite-linux/entrypoint.sh diff --git a/containers/buildkite-premerge-debian/post-checkout b/containers/buildkite-linux/post-checkout similarity index 100% rename from containers/buildkite-premerge-debian/post-checkout rename to containers/buildkite-linux/post-checkout diff --git a/containers/buildkite-premerge-debian/pre-checkout b/containers/buildkite-linux/pre-checkout similarity index 100% rename from containers/buildkite-premerge-debian/pre-checkout rename to containers/buildkite-linux/pre-checkout diff --git a/kubernetes/buildkite/linux-agents-big.yaml b/kubernetes/buildkite/linux-agents-big.yaml new file mode 100644 index 0000000..733df15 --- /dev/null +++ b/kubernetes/buildkite/linux-agents-big.yaml @@ -0,0 +1,77 @@ +# Copyright 2023 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-linux-buildkite + namespace: buildkite +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + type: RollingUpdate + selector: + matchLabels: + app: test-linux-buildkite + template: + metadata: + labels: + app: test-linux-buildkite + spec: + containers: + - name: buildkite-linux + image: gcr.io/llvm-premerge-checks/buildkite-linux:latest + resources: + limits: + cpu: 110 + memory: 420Gi + requests: + cpu: 110 + memory: 420Gi + volumeMounts: + - name: github-ssh + mountPath: /mnt/ssh + - name: workdir + mountPath: /var/lib/buildkite-agent + env: + - name: BUILDKITE_AGENT_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-agent-token + key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: BUILDKITE_AGENT_TAGS + value: "queue=linux-test,name=$(POD_NAME),cluster=google" + - name: BUILDKITE_BUILD_PATH + value: "/var/lib/buildkite-agent/builds" + - name: BUILDKITE_API_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-api-token-readonly + key: token + volumes: + - name: github-ssh + secret: + secretName: github-ssh + - name: workdir + emptyDir: {} + nodeSelector: + cloud.google.com/gke-nodepool: linux-big + terminationGracePeriodSeconds: 30 diff --git a/kubernetes/buildkite/linux-agents-test.yaml b/kubernetes/buildkite/linux-agents-test.yaml index 436060f..c3f541a 100644 --- a/kubernetes/buildkite/linux-agents-test.yaml +++ b/kubernetes/buildkite/linux-agents-test.yaml @@ -15,32 +15,33 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: linux-agents-test + name: buildkite-linux-agents-test namespace: buildkite spec: - replicas: 0 + replicas: 1 strategy: rollingUpdate: - maxSurge: 25% - maxUnavailable: 50% + maxUnavailable: 1 + maxSurge: 0 + type: RollingUpdate selector: matchLabels: - app: agent-premerge-debian + app: linux-agents-test template: metadata: labels: - app: agent-premerge-debian + app: linux-agents-test spec: containers: - - name: test-debian - image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest + - name: runner + image: gcr.io/llvm-premerge-checks/buildkite-linux:latest resources: limits: - cpu: 30 - memory: 80Gi + cpu: 110 + memory: 420Gi requests: - cpu: 30 - memory: 80Gi + cpu: 110 + memory: 420Gi volumeMounts: - name: github-ssh mountPath: /mnt/ssh @@ -57,7 +58,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux-google-test,name=$(POD_NAME)" + value: "queue=linux-test,name=$(POD_NAME),cluster=google" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN @@ -77,5 +78,5 @@ spec: - name: workdir emptyDir: {} nodeSelector: - cloud.google.com/gke-nodepool: linux-agents-2 - terminationGracePeriodSeconds: 3600 + cloud.google.com/gke-nodepool: linux-big + terminationGracePeriodSeconds: 30 diff --git a/kubernetes/buildkite/service-agents-test.yaml b/kubernetes/buildkite/service-agents-test.yaml index acaebd8..dd0284c 100644 --- a/kubernetes/buildkite/service-agents-test.yaml +++ b/kubernetes/buildkite/service-agents-test.yaml @@ -15,21 +15,26 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: service-agents-test + name: buildkite-service-agents-test namespace: buildkite spec: replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + type: RollingUpdate selector: matchLabels: - app: agent-premerge-debian + app: buildkite-service-agents template: metadata: labels: - app: agent-premerge-debian + app: buildkite-service-agents spec: containers: - - name: buildkite-premerge-debian - image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest + - name: runner + image: gcr.io/llvm-premerge-checks/buildkite-linux:latest resources: limits: cpu: 2 @@ -70,4 +75,4 @@ spec: secretName: github-ssh nodeSelector: cloud.google.com/gke-nodepool: default-pool - terminationGracePeriodSeconds: 1200 \ No newline at end of file + terminationGracePeriodSeconds: 60 \ No newline at end of file diff --git a/kubernetes/buildkite/service-agents.yaml b/kubernetes/buildkite/service-agents.yaml index 34ebded..44caf4d 100644 --- a/kubernetes/buildkite/service-agents.yaml +++ b/kubernetes/buildkite/service-agents.yaml @@ -75,4 +75,4 @@ spec: secretName: github-ssh nodeSelector: cloud.google.com/gke-nodepool: default-pool - terminationGracePeriodSeconds: 1200 \ No newline at end of file + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/kubernetes/buildkite/windows-agents.yaml b/kubernetes/buildkite/windows-agents.yaml index a31420d..b8bc00e 100644 --- a/kubernetes/buildkite/windows-agents.yaml +++ b/kubernetes/buildkite/windows-agents.yaml @@ -16,7 +16,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: test-windows-buildkite - namespace: buildkite spec: replicas: 1 strategy: @@ -39,6 +38,9 @@ spec: requests: cpu: 110 memory: 420Gi + volumeMounts: + - name: workdir + mountPath: "C:\\ws" env: - name: buildkiteAgentToken valueFrom: @@ -50,14 +52,20 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=windows-test-pr,name=$(POD_NAME),cluster=google" + value: "queue=windows-test,name=w,cluster=google" + - name: BUILDKITE_BUILD_CHECKOUT_PATH + # TODO: need to drop it if BUILDKITE_REPO is different. + value: "c:\\src" - name: BUILDKITE_API_TOKEN valueFrom: secretKeyRef: name: buildkite-api-token-readonly key: token + volumes: + - name: workdir + emptyDir: {} nodeSelector: - cloud.google.com/gke-nodepool: linux-big + cloud.google.com/gke-nodepool: windows tolerations: - key: "node.kubernetes.io/os" operator: "Equal"