From 1fa24a55ec9d8e51c8bfe859a0f070456e656265 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 8 Jul 2020 09:45:22 +0200 Subject: [PATCH 1/4] Add option to disable cache --- scripts/buildkite/build_master_pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py index a9fdda6..6936ffb 100755 --- a/scripts/buildkite/build_master_pipeline.py +++ b/scripts/buildkite/build_master_pipeline.py @@ -19,11 +19,13 @@ import yaml if __name__ == '__main__': script_branch = os.getenv("scripts_branch", "master") queue_prefix = os.getenv("ph_queue_prefix", "") + no_cache = os.getenv('ph_no_cache', '') != '' steps = [] linux_buld_step = { 'label': ':linux: build and test linux', 'key': 'linux', 'commands': [ + 'ccache --clear' if no_cache else '', 'mkdir -p artifacts', 'dpkg -l >> artifacts/packages.txt', 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', @@ -36,10 +38,14 @@ if __name__ == '__main__': 'agents': {'queue': f'{queue_prefix}linux'}, 'timeout_in_minutes': 120, } + clear_sccache = 'powershell -command "sccache --stop-server; ' \ + 'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \ + 'sccache --start-server"' windows_buld_step = { 'label': ':windows: build and test windows', 'key': 'windows', 'commands': [ + clear_sccache if no_cache else '', 'sccache --zero-stats', 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', 'rm -rf %SRC%', From 5e9a7be29561b79e947c94182e90cd8418506aca Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 8 Jul 2020 10:25:05 +0200 Subject: [PATCH 2/4] add perl to windows image --- containers/agent-windows-vs2019/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/containers/agent-windows-vs2019/Dockerfile b/containers/agent-windows-vs2019/Dockerfile index 692826b..967ca75 100644 --- a/containers/agent-windows-vs2019/Dockerfile +++ b/containers/agent-windows-vs2019/Dockerfile @@ -32,11 +32,8 @@ RUN powershell -NoProfile -InputFormat None -Command ` # install tools as described in https://llvm.org/docs/GettingStartedVS.html # and a few more that were not documented... -RUN choco install -y ninja -RUN choco install -y git +RUN choco install -y ninja git python3 gnuwin activeperl RUN choco install -y cmake --version 3.15.4 -RUN choco install -y python3 -RUN choco install -y gnuwin RUN choco install -y sccache RUN pip install psutil From f1257289089a0391728be08a0f779efd9ac85f29 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 8 Jul 2020 12:01:17 +0200 Subject: [PATCH 3/4] remove compiler-rt and flang temporarily --- scripts/buildkite/build_master_pipeline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py index 6936ffb..d513cf5 100755 --- a/scripts/buildkite/build_master_pipeline.py +++ b/scripts/buildkite/build_master_pipeline.py @@ -20,6 +20,7 @@ if __name__ == '__main__': script_branch = os.getenv("scripts_branch", "master") queue_prefix = os.getenv("ph_queue_prefix", "") no_cache = os.getenv('ph_no_cache', '') != '' + projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly' steps = [] linux_buld_step = { 'label': ':linux: build and test linux', @@ -32,7 +33,7 @@ if __name__ == '__main__': 'rm -rf ${SRC}', 'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}', '${SRC}/scripts/premerge_checks.py ' - '--projects="clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;flang;openmp;polly;compiler-rt"', + f'--projects="{projects}"', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], 'agents': {'queue': f'{queue_prefix}linux'}, @@ -51,7 +52,7 @@ if __name__ == '__main__': 'rm -rf %SRC%', 'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%', 'powershell -command "%SRC%/scripts/premerge_checks.py ' - '--projects=\'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;flang;openmp;polly;compiler-rt\'; ' + f'--projects=\'{projects}\'; ' '\\$exit=\\$?;' 'sccache --show-stats;' 'if (\\$exit) {' From 3f52fb7969b56aca1bf8c94258128393e22f9da7 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 8 Jul 2020 12:20:31 +0200 Subject: [PATCH 4/4] Show stats on linux / windows --- scripts/buildkite/build_branch_pipeline.py | 1 + scripts/buildkite/build_master_pipeline.py | 12 ++++++++---- scripts/windows_agent_start_buildkite.ps1 | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/buildkite/build_branch_pipeline.py b/scripts/buildkite/build_branch_pipeline.py index 7357ae7..a3bd48e 100755 --- a/scripts/buildkite/build_branch_pipeline.py +++ b/scripts/buildkite/build_branch_pipeline.py @@ -20,6 +20,7 @@ if __name__ == '__main__': script_branch = os.getenv("scripts_branch", "master") queue_prefix = os.getenv("ph_queue_prefix", "") diff_id = os.getenv("ph_buildable_diff", "") + steps = [] linux_buld_step = { 'label': ':linux: build and test linux', diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py index d513cf5..17d632a 100755 --- a/scripts/buildkite/build_master_pipeline.py +++ b/scripts/buildkite/build_master_pipeline.py @@ -27,13 +27,17 @@ if __name__ == '__main__': 'key': 'linux', 'commands': [ 'ccache --clear' if no_cache else '', + 'ccache --zero-stats', 'mkdir -p artifacts', 'dpkg -l >> artifacts/packages.txt', 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', 'rm -rf ${SRC}', 'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}', - '${SRC}/scripts/premerge_checks.py ' - f'--projects="{projects}"', + f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}"', + 'EXIT_STATUS=\\$?', + 'echo "--- ccache stats"', + 'ccache --show-stats', + 'exit \\$EXIT_STATUS', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], 'agents': {'queue': f'{queue_prefix}linux'}, @@ -51,9 +55,9 @@ if __name__ == '__main__': 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', 'rm -rf %SRC%', 'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%', - 'powershell -command "%SRC%/scripts/premerge_checks.py ' - f'--projects=\'{projects}\'; ' + f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\'; ' '\\$exit=\\$?;' + 'echo \'--- sccache stats\';' 'sccache --show-stats;' 'if (\\$exit) {' ' echo success;' diff --git a/scripts/windows_agent_start_buildkite.ps1 b/scripts/windows_agent_start_buildkite.ps1 index 70361a8..542b6b8 100644 --- a/scripts/windows_agent_start_buildkite.ps1 +++ b/scripts/windows_agent_start_buildkite.ps1 @@ -27,7 +27,7 @@ $IMAGE="gcr.io/llvm-premerge-checks/${NAME}:${version}" Write-Output "Authenticating docker..." Write-Output "y`n" | gcloud auth configure-docker -Write-Output "Pulling new image..." +Write-Output "Pulling new image '${IMAGE}'..." docker pull ${IMAGE} Write-Output "Stopping old container..." docker stop ${NAME}