From c350101a23037d39e99f3bd0878d726bc772c6f2 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Fri, 2 Oct 2020 16:18:33 +0200 Subject: [PATCH] Allow projects to define custom steps Now it's possible to allow sub-projects to define own checks and skip "generic" ones. To properly accomodate affected projects that might not have special treatment we: 1. extend the set of affected projecs with dependent (e.g. add 'libc' if 'clang' was modified) 2. add custom steps for projects that define own workflow. At the moment it's only libcxx and it has a custom trigger pipeline so it's noop. 3. add dependent projects and run generic check on them. To illustrate: imagine that we have a dependency graph: llvm -> clang -> openmp and only clang was modified in a diff; also clang defines own checks. Thus list of affected projects will be [clang, openmp]. After adding custom checks and removing their projecst: [openmp]. After adding dependencies: [llvm, clang, openmp]. Generic linux / windows checks will be run on thouse 3 projects. So as you can see in some scenarios projects with custom checks will still go through generic checks. Note that clang-format and clang-tidy checks are run only for "generic" checks at the moment. --- scripts/choose_projects.py | 4 +- scripts/metrics/analyze_jobs.ipynb | 2 +- scripts/pipeline_create_branch.py | 8 +- scripts/pipeline_master.py | 105 ++----------------- scripts/pipeline_premerge.py | 162 +++++++---------------------- scripts/steps.py | 147 ++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 228 deletions(-) create mode 100644 scripts/steps.py diff --git a/scripts/choose_projects.py b/scripts/choose_projects.py index 893c76e..3c4581f 100755 --- a/scripts/choose_projects.py +++ b/scripts/choose_projects.py @@ -73,6 +73,8 @@ class ChooseProjects: return 'linux' def choose_projects(self, patch: str = None) -> List[str]: + """List all touched project with all projects that they depend on and also + all projects that depend on them""" if self.llvm_dir is None: raise ValueError('path to llvm folder must be set in ChooseProject.') @@ -119,7 +121,7 @@ class ChooseProjects: @staticmethod def get_changed_files(patch_str: str = None) -> Set[str]: - """get list of changed files from the patch from STDIN.""" + """get list of changed files from the patch or from STDIN.""" if patch_str is None: patch_str = sys.stdin patch = PatchSet(patch_str) diff --git a/scripts/metrics/analyze_jobs.ipynb b/scripts/metrics/analyze_jobs.ipynb index 8fd65a8..4a1dd54 100644 --- a/scripts/metrics/analyze_jobs.ipynb +++ b/scripts/metrics/analyze_jobs.ipynb @@ -295,7 +295,7 @@ "...a: as is D:/ is a workdir (ealy run with full cache build)\n", "...b: C:/ws as a workdir, no cache\n", "...c: c:/ws as workdir, no av\n", - "```\n" + "```\n", ] } ], diff --git a/scripts/pipeline_create_branch.py b/scripts/pipeline_create_branch.py index 5ffc295..bcf1022 100755 --- a/scripts/pipeline_create_branch.py +++ b/scripts/pipeline_create_branch.py @@ -17,7 +17,6 @@ import os import yaml if __name__ == '__main__': - queue_prefix = os.getenv("ph_queue_prefix", "") diff_id = os.getenv("ph_buildable_diff") log_level = os.getenv('ph_log_level', 'INFO') base_commit = os.getenv('ph_base_commit', 'auto') @@ -27,17 +26,17 @@ if __name__ == '__main__': trigger = 'premerge-checks' steps = [] - create_branch_step = { + steps.append({ 'label': 'create branch', 'key': 'create-branch', 'commands': ['scripts/apply_patch.sh'], - 'agents': {'queue': f'{queue_prefix}linux'}, + 'agents': {'queue': 'linux'}, 'timeout_in_minutes': 20, 'env': { 'LOG_LEVEL': log_level, 'BASE_COMMIT': base_commit, } - } + }) if run_build: trigger_build_step = { 'trigger': trigger, @@ -56,5 +55,4 @@ if __name__ == '__main__': if 'ph_scripts_refspec' not in trigger_build_step['build']['env']: trigger_build_step['build']['env']['ph_scripts_refspec'] = '${BUILDKITE_BRANCH}' steps.append(trigger_build_step) - steps.append(create_branch_step) print(yaml.dump({'steps': steps})) diff --git a/scripts/pipeline_master.py b/scripts/pipeline_master.py index e483a8c..4c8ced2 100755 --- a/scripts/pipeline_master.py +++ b/scripts/pipeline_master.py @@ -13,113 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os + +from steps import generic_linux, generic_windows import yaml if __name__ == '__main__': scripts_refspec = os.getenv("ph_scripts_refspec", "master") - queue_prefix = os.getenv("ph_queue_prefix", "") no_cache = os.getenv('ph_no_cache') is not None filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else '' projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly') log_level = os.getenv('ph_log_level', 'WARNING') notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(','))) steps = [] - linux_agents = {'queue': f'{queue_prefix}linux'} - t = os.getenv('ph_linux_agents') - if t is not None: - linux_agents = json.loads(t) - linux_buld_step = { - 'label': ':linux: build and test linux', - 'key': 'linux', - 'commands': [ - 'set -euo pipefail', - 'ccache --clear' if no_cache else '', - 'ccache --zero-stats', - 'ccache --show-config', - 'mkdir -p artifacts', - 'dpkg -l >> artifacts/packages.txt', - # Clone scripts. - 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', - 'rm -rf ${SRC}', - 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"', - 'cd ${SRC}', - f'git fetch origin "{scripts_refspec}":x', - 'git checkout x', - 'echo "llvm-premerge-checks commit"', - 'git rev-parse HEAD', - 'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"', - - 'set +e', - f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level} {filter_output}', - 'EXIT_STATUS=\\$?', - 'echo "--- ccache stats"', - 'ccache --print-stats', - 'ccache --show-stats', - 'exit \\$EXIT_STATUS', - ], - 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': linux_agents, - 'timeout_in_minutes': 120, - 'retry': {'automatic': [ - {'exit_status': -1, 'limit': 2}, # Agent lost - {'exit_status': 255, 'limit': 2}, # Forced agent shutdown - ]}, - } - clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \ - 'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \ - 'sccache --start-server"' - # FIXME: openmp is removed as it constantly fails. Make this project list be evaluated through "choose_projects". - projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly') - win_agents = {'queue': f'{queue_prefix}windows'} - t = os.getenv('ph_windows_agents') - if t is not None: - win_agents = json.loads(t) - windows_buld_step = { - 'label': ':windows: build and test windows', - 'key': 'windows', - 'commands': [ - clear_sccache if no_cache else '', - 'sccache --zero-stats', - - # Clone scripts. - 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', - 'rm -rf %SRC%', - 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%', - 'cd %SRC%', - f'git fetch origin "{scripts_refspec}":x', - 'git checkout x', - 'echo llvm-premerge-checks commit:', - 'git rev-parse HEAD', - 'cd %BUILDKITE_BUILD_CHECKOUT_PATH%', - - 'powershell -command "' - f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; ' - '\\$exit=\\$?;' - 'echo \'--- sccache stats\';' - 'sccache --show-stats;' - 'if (\\$exit) {' - ' echo success;' - ' exit 0; } ' - 'else {' - ' echo failure;' - ' exit 1;' - '}"', - ], - 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': win_agents, - 'timeout_in_minutes': 120, - 'retry': {'automatic': [ - {'exit_status': -1, 'limit': 2}, # Agent lost - {'exit_status': 255, 'limit': 2}, # Forced agent shutdown - ]}, - } - if os.getenv('ph_skip_linux') is None: - steps.append(linux_buld_step) - # TODO: windows builds are temporary disabled #243 - # if os.getenv('ph_skip_windows') is None: - # steps.append(windows_buld_step) + steps.extend(generic_linux( + os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'), + False)) + # FIXME: openmp is removed as it constantly fails. + # TODO: Make this project list be evaluated through "choose_projects". + steps.extend(generic_windows( + os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'))) notify = [] for e in notify_emails: notify.append({'email': e}) diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index 937a56b..c6b3259 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -13,155 +13,68 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import os from choose_projects import ChooseProjects import git +from steps import libcxx, generic_linux, generic_windows import yaml if __name__ == '__main__': scripts_refspec = os.getenv("ph_scripts_refspec", "master") - queue_prefix = os.getenv("ph_queue_prefix", "") diff_id = os.getenv("ph_buildable_diff", "") no_cache = os.getenv('ph_no_cache') is not None filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else '' projects = os.getenv('ph_projects', 'detect') - log_level = os.getenv('ph_log_level', 'WARNING') + log_level = os.getenv('ph_log_level', 'INFO') logging.basicConfig(level=log_level, format='%(levelname)-7s %(message)s') # List all affected projects. repo = git.Repo('.') patch = repo.git.diff("HEAD~1") cp = ChooseProjects('.') - affected_projects = cp.choose_projects(patch) - print('# all affected projects') - for p in affected_projects: - print(f'# {p}') + modified_files = cp.get_changed_files(patch) + modified_projects, unmapped_changes = cp.get_changed_projects(modified_files) + if unmapped_changes: + logging.warning('There were changes that could not be mapped to a project. Checking everything') + modified_projects = cp.all_projects + logging.info(f'modified projects: {modified_projects}') + # Add projects that depend on modified. + affected_projects = cp.get_affected_projects(modified_projects) + # Handle special checks. + checked = set() + generic_projects = set() + logging.info(f'all affected projects: {affected_projects}') steps = [] + for p in affected_projects: + if p == 'libcxx' or p == 'libcxxabi': + if 'libcxx' not in checked: + logging.info('Adding custom steps for "libc++"') + checked.add('libcxx') + steps.extend(libcxx()) + else: + generic_projects.add(p) + + if len(generic_projects) > 0: + # Add dependencies + projects = ';'.join(sorted(cp.add_dependencies(generic_projects))) + logging.info(f'Projects for default checks: {projects}') + steps.extend(generic_linux(projects, True)) + steps.extend(generic_windows(projects)) + else: + logging.info('No projects for default checks') + deps = [] - if 'libcxx' in affected_projects or 'libcxxabi' in affected_projects: - start_libcxx_step = { - 'trigger': 'libcxx-ci', - 'label': 'libcxx', - 'key': 'libcxx', - 'async': False, - 'build': { - 'branch': os.getenv('BUILDKITE_BRANCH'), - 'env': {}, - }, - } - for e in os.environ: - if e.startswith('ph_'): - start_libcxx_step['build']['env'][e] = os.getenv(e) - steps.append(start_libcxx_step) - deps.append(start_libcxx_step['key']) + steps.append({ + 'wait': '~', + 'continue_on_failure': True, + }) - linux_agents = {'queue': f'{queue_prefix}linux'} - t = os.getenv('ph_linux_agents') - if t is not None: - linux_agents = json.loads(t) - linux_buld_step = { - 'label': ':linux: build and test linux', - 'key': 'linux', - 'commands': [ - 'set -euo pipefail', - 'ccache --clear' if no_cache else '', - 'ccache --zero-stats', - 'ccache --show-config', - 'mkdir -p artifacts', - 'dpkg -l >> artifacts/packages.txt', - # Clone scripts. - 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', - 'rm -rf ${SRC}', - 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"', - 'cd ${SRC}', - f'git fetch origin "{scripts_refspec}":x', - 'git checkout x', - 'echo "llvm-premerge-checks commit"', - 'git rev-parse HEAD', - 'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"', - - 'set +e', - # Add link in review to the build. - '${SRC}/scripts/add_phabricator_artifact.py ' - '--phid="$ph_target_phid" ' - '--url="$BUILDKITE_BUILD_URL" ' - '--name="Buildkite build"', - '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy ' - f'--projects="{projects}" --log-level={log_level} {filter_output}', - 'EXIT_STATUS=\\$?', - 'echo "--- ccache stats"', - 'ccache --print-stats', - 'ccache --show-stats', - 'exit \\$EXIT_STATUS', - ], - 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': linux_agents, - 'timeout_in_minutes': 120, - 'retry': {'automatic': [ - {'exit_status': -1, 'limit': 2}, # Agent lost - {'exit_status': 255, 'limit': 2}, # Forced agent shutdown - ]}, - } - clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \ - 'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \ - 'sccache --start-server"' - win_agents = {'queue': f'{queue_prefix}windows'} - t = os.getenv('ph_windows_agents') - if t is not None: - win_agents = json.loads(t) - windows_buld_step = { - 'label': ':windows: build and test windows', - 'key': 'windows', - 'commands': [ - clear_sccache if no_cache else '', - 'sccache --zero-stats', - - # Clone scripts. - 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', - 'rm -rf %SRC%', - 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%', - 'cd %SRC%', - f'git fetch origin "{scripts_refspec}":x', - 'git checkout x', - 'echo llvm-premerge-checks commit:', - 'git rev-parse HEAD', - 'cd %BUILDKITE_BUILD_CHECKOUT_PATH%', - - 'powershell -command "' - f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; ' - '\\$exit=\\$?;' - 'sccache --show-stats;' - 'if (\\$exit) {' - ' echo success;' - ' exit 0; } ' - 'else {' - ' echo failure;' - ' exit 1;' - '}"', - ], - 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': win_agents, - 'timeout_in_minutes': 120, - 'retry': {'automatic': [ - {'exit_status': -1, 'limit': 2}, # Agent lost - {'exit_status': 255, 'limit': 2}, # Forced agent shutdown - ]}, - } - if os.getenv('ph_skip_linux') is None: - steps.append(linux_buld_step) - deps.append(linux_buld_step['key']) - # TODO: windows builds are temporary disabled #243 - # if os.getenv('ph_skip_windows') is None: - # steps.append(windows_buld_step) - # deps.append(windows_buld_step['key']) report_step = { 'label': ':spiral_note_pad: report', - 'depends_on': deps, 'commands': [ 'mkdir -p artifacts', 'buildkite-agent artifact download "*_result.json" .', @@ -178,9 +91,8 @@ if __name__ == '__main__': 'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"', '${SRC}/scripts/summary.py', ], - 'allow_dependency_failure': True, 'artifact_paths': ['artifacts/**/*'], - 'agents': {'queue': f'{queue_prefix}linux'}, + 'agents': {'queue': 'linux'}, 'timeout_in_minutes': 10, } steps.append(report_step) diff --git a/scripts/steps.py b/scripts/steps.py new file mode 100644 index 0000000..f5017df --- /dev/null +++ b/scripts/steps.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from typing import List + + +def libcxx(): + # TODO: read steps from the YAML file from the repo. + return [] + + +def generic_linux(projects: str, check_diff: bool) -> List: + if os.getenv('ph_skip_linux') is not None: + return [] + scripts_refspec = os.getenv("ph_scripts_refspec", "master") + no_cache = os.getenv('ph_no_cache') is not None + filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else '' + log_level = os.getenv('ph_log_level', 'WARNING') + linux_agents = {'queue': 'linux'} + t = os.getenv('ph_linux_agents') + if t is not None: + linux_agents = json.loads(t) + commands = [ + 'set -euo pipefail', + 'ccache --clear' if no_cache else '', + 'ccache --zero-stats', + 'ccache --show-config', + 'mkdir -p artifacts', + 'dpkg -l >> artifacts/packages.txt', + # Clone scripts. + 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', + 'rm -rf ${SRC}', + 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"', + 'cd ${SRC}', + f'git fetch origin "{scripts_refspec}":x', + 'git checkout x', + 'echo "llvm-premerge-checks commit"', + 'git rev-parse HEAD', + 'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"', + 'set +e', + ] + if check_diff: + commands.extend([ + # Add link in review to the build. + '${SRC}/scripts/add_phabricator_artifact.py ' + '--phid="$ph_target_phid" ' + '--url="$BUILDKITE_BUILD_URL" ' + '--name="Buildkite build"' if check_diff else '', + '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy ' + f'--projects="{projects}" --log-level={log_level} {filter_output}', + ]) + else: + commands.extend([ + f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level} {filter_output}' + ]) + commands.extend([ + 'EXIT_STATUS=\\$?', + 'echo "--- ccache stats"', + 'ccache --print-stats', + 'ccache --show-stats', + 'exit \\$EXIT_STATUS', + ]) + + linux_buld_step = { + 'label': ':linux: build and test linux', + 'key': 'linux', + 'commands': commands, + 'artifact_paths': ['artifacts/**/*', '*_result.json'], + 'agents': linux_agents, + 'timeout_in_minutes': 120, + 'retry': {'automatic': [ + {'exit_status': -1, 'limit': 2}, # Agent lost + {'exit_status': 255, 'limit': 2}, # Forced agent shutdown + ]}, + } + return [linux_buld_step] + + +def generic_windows(projects: str) -> List: + # TODO: windows builds are temporary disabled #243 + return [] + if os.getenv('ph_skip_windows') is not None: + return [] + scripts_refspec = os.getenv("ph_scripts_refspec", "master") + no_cache = os.getenv('ph_no_cache') is not None + log_level = os.getenv('ph_log_level', 'WARNING') + filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else '' + clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \ + 'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \ + 'sccache --start-server"' + win_agents = {'queue': 'windows'} + t = os.getenv('ph_windows_agents') + if t is not None: + win_agents = json.loads(t) + windows_buld_step = { + 'label': ':windows: build and test windows', + 'key': 'windows', + 'commands': [ + clear_sccache if no_cache else '', + 'sccache --zero-stats', + + # Clone scripts. + 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', + 'rm -rf %SRC%', + 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%', + 'cd %SRC%', + f'git fetch origin "{scripts_refspec}":x', + 'git checkout x', + 'echo llvm-premerge-checks commit:', + 'git rev-parse HEAD', + 'cd %BUILDKITE_BUILD_CHECKOUT_PATH%', + + 'powershell -command "' + f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; ' + '\\$exit=\\$?;' + 'sccache --show-stats;' + 'if (\\$exit) {' + ' echo success;' + ' exit 0; } ' + 'else {' + ' echo failure;' + ' exit 1;' + '}"', + ], + 'artifact_paths': ['artifacts/**/*', '*_result.json'], + 'agents': win_agents, + 'timeout_in_minutes': 120, + 'retry': {'automatic': [ + {'exit_status': -1, 'limit': 2}, # Agent lost + {'exit_status': 255, 'limit': 2}, # Forced agent shutdown + ]}, + } + return [windows_buld_step]