From f3ac1b41dd04ed65740130041b2114272848cc08 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 26 Jan 2022 18:40:18 +0100 Subject: [PATCH] enable bolt premerge checks --- containers/base-debian/Dockerfile | 16 ++++++-- .../buildkite-premerge-debian/entrypoint.sh | 2 +- docs/playbooks.md | 38 +++++++++++++++++++ scripts/llvm-dependencies.yaml | 4 ++ scripts/run_cmake_config.yaml | 2 + 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/containers/base-debian/Dockerfile b/containers/base-debian/Dockerfile index 6e12d9f..8fbcb6a 100644 --- a/containers/base-debian/Dockerfile +++ b/containers/base-debian/Dockerfile @@ -1,23 +1,31 @@ # debian:unstable to get a go version > 1.11 (migth have other packages with different version, please check before updating) FROM debian:unstable -RUN echo 'intall build dependencies'; \ - apt-get update ;\ + + +RUN echo 'intall packages'; \ + apt-get update; \ apt-get install -y --no-install-recommends \ locales openssh-client gnupg ca-certificates \ zip wget git \ cmake gdb build-essential \ ninja-build \ libelf-dev libffi-dev gcc-multilib \ +# for bolt subproject + libc6-dev-i386 \ ccache \ python3 python3-psutil \ python3-pip python3-setuptools \ lsb-release software-properties-common \ swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; -RUN wget https://apt.llvm.org/llvm.sh; \ + +# LLVM must be installed after prerequsite packages. +RUN echo 'install llvm 13'; \ + wget https://apt.llvm.org/llvm.sh; \ chmod +x llvm.sh; \ ./llvm.sh 13;\ - apt install -y clang-format-13 clang-tidy-13 + apt-get update; \ + apt install -y clang-format-13 clang-tidy-13; RUN echo 'configure locale'; \ sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ diff --git a/containers/buildkite-premerge-debian/entrypoint.sh b/containers/buildkite-premerge-debian/entrypoint.sh index 222c99d..b86ab15 100755 --- a/containers/buildkite-premerge-debian/entrypoint.sh +++ b/containers/buildkite-premerge-debian/entrypoint.sh @@ -30,4 +30,4 @@ cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh chmod 700 /var/lib/buildkite-agent/.ssh chmod 600 /var/lib/buildkite-agent/.ssh/* chown -R buildkite-agent:buildkite-agent /var/lib/buildkite-agent/.ssh/ -exec /usr/bin/tini -g -- $@ \ No newline at end of file +exec /usr/bin/tini -g -- $@ diff --git a/docs/playbooks.md b/docs/playbooks.md index 9612daa..de4f6fa 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -88,6 +88,24 @@ These are the steps to set up the build server on a clean infrastructure: ``` 1. Configure it +## Running Docker containers + +`./containers/build_run.sh` provides a good way to run a container once to check that it builds and contains good package versions. To work with a container and +persist it state do (taking 'base-debian' as example): + +``` +cd ./containers/base-debian +sudo docker build -t base-debian . +sudo docker run -it base-debian /bin/bash +``` + +to resume it from where you left + +``` +sudo docker start -a -i $(sudo docker ps -a -f ancestor=base-debian -q -l) +``` + + ## Creating docker containers on Windows If you want to build/update/test docker container for Windows, you need to do this on a Windows machine. @@ -218,3 +236,23 @@ htpasswd -b auth kubectl delete secret http-auth -n buildkite kubectl create secret generic http-auth -n buildkite --from-file=./auth ``` + +## Update github-ssh secret + +This Kubernetes stores github ssh access files for llvm.premerge.tests@gmail.com user. + +To access it run `kubectl get secret github-ssh -n buildkite -o yaml | tee github-ssh.yam`. +It has 3 entries: "id_rsa", "id_rsa.pub", "known_hosts". +If you need to updated it, edit resulting yaml above with updated base 64 values. + +To updated "known_hosts" run `ssh-keyscan github.com` and store base64 encoded output in "known_hosts". +Then run `kubectl apply -f github-ssh.yaml`. + +To create this secret from scratch run + +`kubectl create secret generic github-ssh --namespace buildkite --from-file ./id_rsa --from-file ./id_rsa.pub --from-file ./known_hosts` + +providing all files needed. + +Note that containers have to be restarted as ssh settings are copied at startup +(entrypoint.sh). diff --git a/scripts/llvm-dependencies.yaml b/scripts/llvm-dependencies.yaml index 73eef15..42c04ff 100644 --- a/scripts/llvm-dependencies.yaml +++ b/scripts/llvm-dependencies.yaml @@ -4,6 +4,8 @@ dependencies: llvm: [] + bolt: + - llvm clang: - llvm clang-tools-extra: @@ -47,6 +49,7 @@ dependencies: # llvm/CMakeLists.txt in "set(LLVM_ALL_PROJECTS ..." # The value for all project is the list of ninja targets to run. allprojects: + bolt: ["check-bolt"] clang: ["check-clang"] clang-tools-extra: ["check-clang-tools"] compiler-rt: ["check-all"] # check-compiler-rt seems to exist only in standalone builds. @@ -68,6 +71,7 @@ allprojects: # projects excluded from automatic configuration as they could not be built excludedProjects: windows: + - bolt # tests are not supported yet - lldb # failing tests - libunwind # no windows support - libcxx # no windows support diff --git a/scripts/run_cmake_config.yaml b/scripts/run_cmake_config.yaml index 5cd44f7..a138e49 100644 --- a/scripts/run_cmake_config.yaml +++ b/scripts/run_cmake_config.yaml @@ -27,4 +27,6 @@ arguments: # CCACHE is enabled in script iff environment variable `CCACHE_DIR` is set - '-D LLVM_ENABLE_LLD=ON' - '-D CMAKE_CXX_FLAGS=-gmlt' + # Bolt https://github.com/google/llvm-premerge-checks/issues/364#issuecomment-1013952831. + - '-DBOLT_CLANG_EXE=/usr/bin/clang -DBOLT_LLD_EXE=/usr/bin/lld' windows: []