From 76d6f29154d5cf439fc4b5891a634c8aaf43eff8 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Sun, 25 Dec 2022 14:57:47 +0100 Subject: [PATCH] use one dockerfile, upgrade to clang 15 successfully run https://buildkite.com/llvm-project/llvm-main/builds/6204 based on this image --- Pipfile | 2 +- containers/base-debian/Dockerfile | 52 ------------------- .../buildkite-premerge-debian/Dockerfile | 49 ++++++++++++++++- .../buildkite-premerge-debian/entrypoint.sh | 22 +++++--- kubernetes/buildkite/linux-agents-test.yaml | 6 +-- scripts/buildkite/sample_build.py | 12 +++-- scripts/etc/run_sample_linux_build.sh | 2 +- 7 files changed, 74 insertions(+), 71 deletions(-) delete mode 100644 containers/base-debian/Dockerfile mode change 100644 => 100755 scripts/etc/run_sample_linux_build.sh diff --git a/Pipfile b/Pipfile index da0cabb..49179e2 100644 --- a/Pipfile +++ b/Pipfile @@ -19,4 +19,4 @@ pytest = "*" [dev-packages] [requires] -python_version = "3.9" +python_version = "3.10" diff --git a/containers/base-debian/Dockerfile b/containers/base-debian/Dockerfile deleted file mode 100644 index 7fbee35..0000000 --- a/containers/base-debian/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# debian:unstable to get a go version > 1.11 (migth have other packages with different version, please check before updating) -FROM debian:unstable - - - -RUN echo 'intall packages'; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - locales openssh-client gnupg ca-certificates \ - zip wget git \ - cmake gdb build-essential \ - ninja-build \ - libelf-dev libffi-dev gcc-multilib \ -# for llvm-libc tests that build mpfr and gmp from source - autoconf automake libtool \ -# for bolt subproject - libc6-dev-i386 \ - ccache \ - python3 python3-psutil \ - python3-pip python3-setuptools \ - lsb-release software-properties-common \ - swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; - -# LLVM must be installed after prerequsite packages. -RUN echo 'install llvm 13'; \ - wget https://apt.llvm.org/llvm.sh; \ - chmod +x llvm.sh; \ - ./llvm.sh 13;\ - apt-get update; \ - apt install -y clang-format-13 clang-tidy-13; - -RUN echo 'configure locale'; \ - sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen ;\ - echo 'make python 3 default'; \ - rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \ - pip3 install wheel - -# Configure locale -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -RUN ln -s /usr/bin/clang-13 /usr/bin/clang;\ - ln -s /usr/bin/clang++-13 /usr/bin/clang++;\ - ln -s /usr/bin/clang-tidy-13 /usr/bin/clang-tidy;\ - ln -s /usr/bin/clang-tidy-diff-13.py /usr/bin/clang-tidy-diff;\ - ln -s /usr/bin/clang-format-13 /usr/bin/clang-format;\ - ln -s /usr/bin/clang-format-diff-13 /usr/bin/clang-format-diff;\ - ln -s /usr/bin/lld-13 /usr/bin/lld;\ - ln -s /usr/bin/ld.lld-13 /usr/bin/ld.lld - diff --git a/containers/buildkite-premerge-debian/Dockerfile b/containers/buildkite-premerge-debian/Dockerfile index 4db4f4c..eb13fa8 100644 --- a/containers/buildkite-premerge-debian/Dockerfile +++ b/containers/buildkite-premerge-debian/Dockerfile @@ -1,4 +1,49 @@ -FROM gcr.io/llvm-premerge-checks/base-debian:latest +FROM debian:stable + +RUN echo 'intall packages'; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + locales openssh-client gnupg ca-certificates \ + zip wget git \ + cmake gdb build-essential \ + ninja-build \ + libelf-dev libffi-dev gcc-multilib \ +# for llvm-libc tests that build mpfr and gmp from source + autoconf automake libtool \ + ccache \ + python3 python3-psutil \ + python3-pip python3-setuptools \ + lsb-release software-properties-common \ + swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; + +# LLVM must be installed after prerequsite packages. +ENV LLVM_VERSION=15 +RUN echo 'install llvm ${LLVM_VERSION}'; \ + wget https://apt.llvm.org/llvm.sh; \ + chmod +x llvm.sh; \ + ./llvm.sh ${LLVM_VERSION};\ + apt-get update; \ + apt install -y clang-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} lld-${LLVM_VERSION}; \ + ln -s /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang;\ + ln -s /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++;\ + ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy;\ + ln -s /usr/bin/clang-tidy-diff-${LLVM_VERSION}.py /usr/bin/clang-tidy-diff;\ + ln -s /usr/bin/clang-format-${LLVM_VERSION} /usr/bin/clang-format;\ + ln -s /usr/bin/clang-format-diff-${LLVM_VERSION} /usr/bin/clang-format-diff;\ + ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld;\ + ln -s /usr/bin/ld.lld-${LLVM_VERSION} /usr/bin/ld.lld + +RUN echo 'configure locale'; \ + sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ + locale-gen ;\ + echo 'make python 3 default'; \ + rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \ + pip3 install wheel + +# Configure locale +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 RUN echo 'install buildkite' ;\ apt-get install -y apt-transport-https gnupg;\ @@ -16,4 +61,4 @@ COPY --chown=buildkite-agent:buildkite-agent post-checkout /etc/buildkite-agent/ VOLUME /var/lib/buildkite-agent ENTRYPOINT ["entrypoint.sh"] -CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"] \ No newline at end of file +CMD ["gosu", "buildkite-agent", "buildkite-agent", "start", "--no-color"] diff --git a/containers/buildkite-premerge-debian/entrypoint.sh b/containers/buildkite-premerge-debian/entrypoint.sh index b86ab15..e0360cf 100755 --- a/containers/buildkite-premerge-debian/entrypoint.sh +++ b/containers/buildkite-premerge-debian/entrypoint.sh @@ -13,21 +13,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail +set -eo pipefail USER=buildkite-agent -mkdir -p "${BUILDKITE_BUILD_PATH}" -chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}" +P="${BUILDKITE_BUILD_PATH:-/var/lib/buildkite-agent}" +set -u +mkdir -p "$P" +chown -R ${USER}:${USER} "$P" -export CCACHE_DIR="${BUILDKITE_BUILD_PATH}"/ccache +export CCACHE_DIR="${P}"/ccache export CCACHE_MAXSIZE=20G mkdir -p "${CCACHE_DIR}" chown -R ${USER}:${USER} "${CCACHE_DIR}" # /mnt/ssh should contain known_hosts, id_rsa and id_rsa.pub . mkdir -p /var/lib/buildkite-agent/.ssh -cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh -chmod 700 /var/lib/buildkite-agent/.ssh -chmod 600 /var/lib/buildkite-agent/.ssh/* -chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.ssh/ +if [ -d /mnt/ssh ]; then + cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh || echo "no " + chmod 700 /var/lib/buildkite-agent/.ssh + chmod 600 /var/lib/buildkite-agent/.ssh/* + chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.ssh/ +else + echo "/mnt/ssh is not mounted" +fi exec /usr/bin/tini -g -- $@ diff --git a/kubernetes/buildkite/linux-agents-test.yaml b/kubernetes/buildkite/linux-agents-test.yaml index 543e540..b5f5111 100644 --- a/kubernetes/buildkite/linux-agents-test.yaml +++ b/kubernetes/buildkite/linux-agents-test.yaml @@ -32,8 +32,8 @@ spec: app: agent-premerge-debian spec: containers: - - name: buildkite-premerge-debian - image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian:latest + - name: test-debian + image: gcr.io/llvm-premerge-checks/test:latest resources: limits: cpu: 30 @@ -57,7 +57,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux-test,name=$(POD_NAME)" + value: "queue=linux-google-test,name=$(POD_NAME)" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN diff --git a/scripts/buildkite/sample_build.py b/scripts/buildkite/sample_build.py index da76fff..79e62f8 100644 --- a/scripts/buildkite/sample_build.py +++ b/scripts/buildkite/sample_build.py @@ -12,17 +12,21 @@ if __name__ == '__main__': d = json.dumps({ 'branch': 'main', - 'commit': '20ba079dda7be1a72d64cebc9f55d909bf29f6c1', + 'commit': '40dd8ff3310ba69247c911d1b5fd7ad7cfde81b9', 'env': { 'ph_skip_generated': 'skip', 'ph_log_level': 'DEBUG', - 'ph_skip_linux': 'skip', - 'ph_linux_agents': '{"queue": "linux-test"}', + #'ph_skip_linux': 'skip', + #'ph_linux_agents': '{"queue": "linux-google-test"}', + 'ph_linux_agents': '{"queue": "linux"}', + # 'ph_linux_agents': '{"queue": "linux-clang15-test"}', # 'ph_skip_windows': 'skip', 'ph_windows_agents': f'{{"name": "win-dev", "queue": "windows-test"}}', # 'ph_windows_agents': f'{{"queue": "windows"}}', # 'ph_scripts_refspec': 'windows-vscmd', - 'ph_projects': 'clang', + 'ph_projects': 'all', + 'ph_skip_windows': 'skip', + # 'ph_windows_agents': f'{{"name": "", "queue": "{queue}"}}', }}) print(d) if (args.dryrun): diff --git a/scripts/etc/run_sample_linux_build.sh b/scripts/etc/run_sample_linux_build.sh old mode 100644 new mode 100755 index 07f14c2..2eadfe1 --- a/scripts/etc/run_sample_linux_build.sh +++ b/scripts/etc/run_sample_linux_build.sh @@ -9,7 +9,7 @@ cd build export CC="clang" export CXX="clang++" export LD="LLD" -cmake ../llvm -D LLVM_ENABLE_PROJECTS="bolt;compiler-rt;libc;pstl;polly;llvm;libclc;clang;mlir;clang-tools-extra;flang;lld" -G Ninja -D CMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_ASSERTIONS=ON -D LLVM_BUILD_EXAMPLES=ON -D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml" -D LLVM_ENABLE_LLD=ON -D CMAKE_CXX_FLAGS=-gmlt -DBOLT_CLANG_EXE=/usr/bin/clang +cmake ../llvm -D LLVM_ENABLE_PROJECTS="bolt;compiler-rt;libc;pstl;polly;llvm;libclc;clang;mlir;clang-tools-extra;flang;lld" -G Ninja -D CMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_ASSERTIONS=ON -D LLVM_BUILD_EXAMPLES=ON -D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml --resultdb-output resultdb.json" -D LLVM_ENABLE_LLD=ON -D CMAKE_CXX_FLAGS=-gmlt -DBOLT_CLANG_EXE=/usr/bin/clang # ^note that compiler cache arguments are omitted ln -s $PWD/compile_commands.json ../compile_commands.json ninja check-all