From 770f6726bc40aa4c2b97c3318e38507ee2d0280d Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Thu, 4 Jun 2020 08:40:04 +0200 Subject: [PATCH] Add new pipeline to build a master branch --- docs/playbooks.md | 3 +- scripts/buildkite/build_master_pipeline.py | 65 ++++++++++++++++++++++ scripts/premerge_checks.py | 7 ++- 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100755 scripts/buildkite/build_master_pipeline.py diff --git a/docs/playbooks.md b/docs/playbooks.md index e54c367..ff23360 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -119,7 +119,7 @@ To spawn a new windows agent: Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1 ./windows_agent_bootstrap.ps1 ``` - Ignore the pop-up to format the new disk andw wait for the machine to reboot. + Ignore the pop-up to format the new disk and wait for the machine to reboot. ### Buildkite @@ -131,6 +131,7 @@ To spawn a new windows agent: ``` 1. Run ```powershell + git clone https://github.com/google/llvm-premerge-checks.git C:\llvm-premerge-checks C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 [-workdir D:/] [-testing] [-version latest] ``` diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py new file mode 100755 index 0000000..8d06aa3 --- /dev/null +++ b/scripts/buildkite/build_master_pipeline.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +if __name__ == '__main__': + script_branch = os.getenv("scripts_branch", "master") + queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default") + diff_id = os.getenv("ph_buildable_diff", "") + steps = [] + linux_buld_step = { + 'label': ':linux: build and test linux', + 'key': 'linux', + 'commands': [ + 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', + 'rm -rf ${SRC}', + 'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}', + # Add link in review to the build. + '${SRC}/scripts/phabtalk/add_url_artifact.py ' + '--phid="$ph_target_phid" ' + '--url="$BUILDKITE_BUILD_URL" ' + '--name="Buildkite build"', + '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy --projects=default', + ], + 'artifact_paths': ['artifacts/**/*', '*_result.json'], + 'agents': {'queue': queue, 'os': 'linux'} + } + windows_buld_step = { + 'label': ':windows: build and test windows', + 'key': 'windows', + 'commands': [ + 'sccache --show-stats', + 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', + 'rm -rf %SRC%', + 'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%', + 'powershell -command "%SRC%/scripts/premerge_checks.py --projects=default; ' + '\\$exit=\\$?;' + 'sccache --show-stats;' + 'if (\\$exit) {' + ' echo "success";' + ' exit 0; } ' + 'else {' + ' echo "failure";' + ' exit 1;' + '}', + ], + 'artifact_paths': ['artifacts/**/*', '*_result.json'], + 'agents': {'queue': queue, 'os': 'windows'}, + } + steps.append(linux_buld_step) + steps.append(windows_buld_step) + print(yaml.dump({'steps': steps})) diff --git a/scripts/premerge_checks.py b/scripts/premerge_checks.py index a1f0a21..2bfcdb8 100755 --- a/scripts/premerge_checks.py +++ b/scripts/premerge_checks.py @@ -67,9 +67,9 @@ def run_step(name: str, report: Report, thunk: Callable[[Step, Report], None]) - return step -def cmake_report(step: Step, _: Report): +def cmake_report(projects: str, step: Step, _: Report): global build_dir - cmake_result, build_dir, cmake_artifacts = run_cmake.run('detect', os.getcwd()) + cmake_result, build_dir, cmake_artifacts = run_cmake.run(projects, os.getcwd()) for file in cmake_artifacts: if os.path.exists(file): shutil.copy2(file, artifacts_dir) @@ -88,6 +88,7 @@ if __name__ == '__main__': parser.add_argument('--log-level', type=str, default='WARNING') parser.add_argument('--check-clang-format', action='store_true') parser.add_argument('--check-clang-tidy', action='store_true') + parser.add_argument('--projects', type=str, default='detect', choices=['detect', 'default']) args = parser.parse_args() logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') build_dir = '' @@ -104,7 +105,7 @@ if __name__ == '__main__': with open(report_path, 'w') as f: json.dump(report.__dict__, f, default=as_dict) report.success = True - cmake = run_step('cmake', report, cmake_report) + cmake = run_step('cmake', report, lambda s, r: cmake_report(args.projects, s, r)) if cmake.success: ninja_all = run_step('ninja all', report, ninja_all_report) if ninja_all.success: