use one dockerfile, upgrade to clang 15

successfully run https://buildkite.com/llvm-project/llvm-main/builds/6204 based on this image
2022-12-25 14:57:47 +01:00 · 2022-12-25 14:57:47 +01:00 · 76d6f29154
commit 76d6f29154
parent f96092cfb1
7 changed files with 74 additions and 71 deletions
--- a/2
+++ b/2
@ -19,4 +19,4 @@ pytest = "*"
 [dev-packages]

 [requires]
-python_version = "3.9"
+python_version = "3.10"
--- a/containers/base-debian/Dockerfile
+++ b/containers/base-debian/Dockerfile
@ -1,52 +0,0 @@
-# debian:unstable to get a go version > 1.11 (migth have other packages with different version, please check before updating)
-FROM debian:unstable
-
-
-
-RUN echo 'intall packages'; \
-    apt-get update; \
-    apt-get install -y --no-install-recommends \
-        locales openssh-client gnupg ca-certificates  \
-        zip wget git \
-        cmake gdb build-essential  \
-        ninja-build \
-        libelf-dev libffi-dev gcc-multilib \
-# for llvm-libc tests that build mpfr and gmp from source
-        autoconf automake libtool \
-# for bolt subproject
-        libc6-dev-i386 \
-        ccache \
-        python3 python3-psutil \
-        python3-pip python3-setuptools \
-        lsb-release software-properties-common \
-        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq;
-
-# LLVM must be installed after prerequsite packages.
-RUN echo 'install llvm 13'; \
-    wget https://apt.llvm.org/llvm.sh; \
-    chmod +x llvm.sh; \
-    ./llvm.sh 13;\
-    apt-get update; \
-    apt install -y clang-format-13 clang-tidy-13;
-
-RUN echo 'configure locale'; \
-    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
-    locale-gen ;\
-    echo 'make python 3 default'; \
-    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \
-    pip3 install wheel
-
-# Configure locale
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-RUN ln -s /usr/bin/clang-13 /usr/bin/clang;\
-    ln -s /usr/bin/clang++-13 /usr/bin/clang++;\
-    ln -s /usr/bin/clang-tidy-13 /usr/bin/clang-tidy;\
-    ln -s /usr/bin/clang-tidy-diff-13.py /usr/bin/clang-tidy-diff;\
-    ln -s /usr/bin/clang-format-13 /usr/bin/clang-format;\
-    ln -s /usr/bin/clang-format-diff-13 /usr/bin/clang-format-diff;\
-    ln -s /usr/bin/lld-13 /usr/bin/lld;\
-    ln -s /usr/bin/ld.lld-13 /usr/bin/ld.lld
-
--- a/containers/buildkite-premerge-debian/Dockerfile
+++ b/containers/buildkite-premerge-debian/Dockerfile
@ -1,4 +1,49 @@
-FROM gcr.io/llvm-premerge-checks/base-debian:latest
+FROM debian:stable
+
+RUN echo 'intall packages'; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        locales openssh-client gnupg ca-certificates  \
+        zip wget git \
+        cmake gdb build-essential  \
+        ninja-build \
+        libelf-dev libffi-dev gcc-multilib \
+# for llvm-libc tests that build mpfr and gmp from source
+        autoconf automake libtool \
+        ccache \
+        python3 python3-psutil \
+        python3-pip python3-setuptools \
+        lsb-release software-properties-common \
+        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq;
+
+# LLVM must be installed after prerequsite packages.
+ENV LLVM_VERSION=15
+RUN echo 'install llvm ${LLVM_VERSION}'; \
+    wget https://apt.llvm.org/llvm.sh; \
+    chmod +x llvm.sh; \
+    ./llvm.sh ${LLVM_VERSION};\
+    apt-get update; \
+    apt install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION}; \
+    ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang;\
+    ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++;\
+    ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy;\
+    ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff;\
+    ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format;\
+    ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff;\
+    ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld;\
+    ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld
+
+RUN echo 'configure locale'; \
+    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
+    locale-gen ;\
+    echo 'make python 3 default'; \
+    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \
+    pip3 install wheel
+
+# Configure locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8

 RUN echo 'install buildkite' ;\
    apt-get install -y apt-transport-https gnupg;\
@ -16,4 +61,4 @@ COPY --chown=buildkite-agent:buildkite-agent post-checkout /etc/buildkite-agent/
 VOLUME /var/lib/buildkite-agent

 ENTRYPOINT ["entrypoint.sh"]
-CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"]
+CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"]
--- a/containers/buildkite-premerge-debian/entrypoint.sh
+++ b/containers/buildkite-premerge-debian/entrypoint.sh
@ -13,21 +13,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-set -euo pipefail
+set -eo pipefail

 USER=buildkite-agent
-mkdir -p "${BUILDKITE_BUILD_PATH}"
-chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}"
+P="${BUILDKITE_BUILD_PATH:-/var/lib/buildkite-agent}"
+set -u
+mkdir -p "$P"
+chown -R ${USER}:${USER} "$P"

-export CCACHE_DIR="${BUILDKITE_BUILD_PATH}"/ccache
+export CCACHE_DIR="${P}"/ccache
 export CCACHE_MAXSIZE=20G
 mkdir -p "${CCACHE_DIR}"
 chown -R ${USER}:${USER} "${CCACHE_DIR}"

 # /mnt/ssh should contain known_hosts, id_rsa and id_rsa.pub .
 mkdir -p /var/lib/buildkite-agent/.ssh
-cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh
-chmod 700 /var/lib/buildkite-agent/.ssh
-chmod 600 /var/lib/buildkite-agent/.ssh/*
-chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.ssh/
+if [ -d /mnt/ssh ]; then
+  cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh || echo "no "
+  chmod 700 /var/lib/buildkite-agent/.ssh
+  chmod 600 /var/lib/buildkite-agent/.ssh/*
+  chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.ssh/
+else
+  echo "/mnt/ssh is not mounted"
+fi
 exec /usr/bin/tini -g -- $@
--- a/kubernetes/buildkite/linux-agents-test.yaml
+++ b/kubernetes/buildkite/linux-agents-test.yaml
@ -32,8 +32,8 @@ spec:
        app: agent-premerge-debian
    spec:
      containers:
-        - name: buildkite-premerge-debian
-          image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest
+        - name: test-debian
+          image: gcr.io/llvm-premerge-checks/test:latest
          resources:
            limits:
              cpu: 30
@ -57,7 +57,7 @@ spec:
                fieldRef:
                 fieldPath: metadata.name
            - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux-test,name=$(POD_NAME)"
+              value: "queue=linux-google-test,name=$(POD_NAME)"
            - name: BUILDKITE_BUILD_PATH
              value: "/var/lib/buildkite-agent/builds"
            - name: CONDUIT_TOKEN
--- a/scripts/buildkite/sample_build.py
+++ b/scripts/buildkite/sample_build.py
@ -12,17 +12,21 @@ if __name__ == '__main__':

  d = json.dumps({
      'branch': 'main',
-      'commit': '20ba079dda7be1a72d64cebc9f55d909bf29f6c1',
+      'commit': '40dd8ff3310ba69247c911d1b5fd7ad7cfde81b9',
      'env': {
          'ph_skip_generated': 'skip',
          'ph_log_level': 'DEBUG',
-          'ph_skip_linux': 'skip',
-          'ph_linux_agents': '{"queue": "linux-test"}',
+          #'ph_skip_linux': 'skip',
+          #'ph_linux_agents': '{"queue": "linux-google-test"}',
+          'ph_linux_agents': '{"queue": "linux"}',
+          # 'ph_linux_agents': '{"queue": "linux-clang15-test"}',
          # 'ph_skip_windows': 'skip',
          'ph_windows_agents': f'{{"name": "win-dev", "queue": "windows-test"}}',
          # 'ph_windows_agents': f'{{"queue": "windows"}}',
          # 'ph_scripts_refspec': 'windows-vscmd',
-          'ph_projects': 'clang',
+          'ph_projects': 'all',
+          'ph_skip_windows': 'skip',
+          # 'ph_windows_agents': f'{{"name": "", "queue": "{queue}"}}',
      }})
  print(d)
  if (args.dryrun):
--- a/scripts/etc/run_sample_linux_build.sh
+++ b/scripts/etc/run_sample_linux_build.sh
@ -9,7 +9,7 @@ cd build
 export CC="clang"
 export CXX="clang++"
 export LD="LLD"
-cmake ../llvm -D LLVM_ENABLE_PROJECTS="bolt;compiler-rt;libc;pstl;polly;llvm;libclc;clang;mlir;clang-tools-extra;flang;lld" -G Ninja -D CMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_ASSERTIONS=ON -D LLVM_BUILD_EXAMPLES=ON -D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml" -D LLVM_ENABLE_LLD=ON -D CMAKE_CXX_FLAGS=-gmlt -DBOLT_CLANG_EXE=/usr/bin/clang
+cmake ../llvm -D LLVM_ENABLE_PROJECTS="bolt;compiler-rt;libc;pstl;polly;llvm;libclc;clang;mlir;clang-tools-extra;flang;lld" -G Ninja -D CMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_ASSERTIONS=ON -D LLVM_BUILD_EXAMPLES=ON -D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml --resultdb-output resultdb.json" -D LLVM_ENABLE_LLD=ON -D CMAKE_CXX_FLAGS=-gmlt -DBOLT_CLANG_EXE=/usr/bin/clang
 # ^note that compiler cache arguments are omitted
 ln -s $PWD/compile_commands.json ../compile_commands.json
 ninja check-all