revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

Allow projects to define custom steps

Browse source 
Now it's possible to allow sub-projects to define own checks and skip
"generic" ones.

To properly accomodate affected projects that might not have special
treatment we:

1. extend the set of affected projecs with dependent (e.g. add 'libc' if
'clang' was modified)

2. add custom steps for projects that define own workflow. At the moment
it's only libcxx and it has a custom trigger pipeline so it's noop.

3. add dependent projects and run generic check on them.

To illustrate: imagine that we have a dependency graph:

llvm -> clang -> openmp

and only clang was modified in a diff; also clang defines own checks.
Thus list of affected projects will be [clang, openmp].
After adding custom checks and removing their projecst: [openmp].
After adding dependencies: [llvm, clang, openmp]. Generic linux /
windows checks will be run on thouse 3 projects.
So as you can see in some scenarios projects with custom checks will
still go through generic checks.

Note that clang-format and clang-tidy checks are run only for "generic"
checks at the moment.
This commit is contained in:
Mikhail Goncharov 2020-10-02 16:18:33 +02:00
parent c082f1aa9b
commit c350101a23
6 changed files with 200 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
scripts/choose_projects.py
View file

@ -73,6 +73,8 @@ class ChooseProjects:
return 'linux'
def choose_projects(self, patch: str = None) -> List[str]:
"""List all touched project with all projects that they depend on and also
all projects that depend on them"""
if self.llvm_dir is None:
raise ValueError('path to llvm folder must be set in ChooseProject.')
@ -119,7 +121,7 @@ class ChooseProjects:
@staticmethod
def get_changed_files(patch_str: str = None) -> Set[str]:
"""get list of changed files from the patch from STDIN."""
"""get list of changed files from the patch or from STDIN."""
if patch_str is None:
patch_str = sys.stdin
patch = PatchSet(patch_str)

2
scripts/metrics/analyze_jobs.ipynb
View file

@ -295,7 +295,7 @@
"...a: as is D:/ is a workdir (ealy run with full cache build)\n",
"...b: C:/ws as a workdir, no cache\n",
"...c: c:/ws as workdir, no av\n",
"```\n"
"```\n",
]
}
],

8
scripts/pipeline_create_branch.py
View file

@ -17,7 +17,6 @@ import os
import yaml
if __name__ == '__main__':
queue_prefix = os.getenv("ph_queue_prefix", "")
diff_id = os.getenv("ph_buildable_diff")
log_level = os.getenv('ph_log_level', 'INFO')
base_commit = os.getenv('ph_base_commit', 'auto')
@ -27,17 +26,17 @@ if __name__ == '__main__':
trigger = 'premerge-checks'
steps = []
create_branch_step = {
steps.append({
'label': 'create branch',
'key': 'create-branch',
'commands': ['scripts/apply_patch.sh'],
'agents': {'queue': f'{queue_prefix}linux'},
'agents': {'queue': 'linux'},
'timeout_in_minutes': 20,
'env': {
'LOG_LEVEL': log_level,
'BASE_COMMIT': base_commit,
}
}
})
if run_build:
trigger_build_step = {
'trigger': trigger,
@ -56,5 +55,4 @@ if __name__ == '__main__':
if 'ph_scripts_refspec' not in trigger_build_step['build']['env']:
trigger_build_step['build']['env']['ph_scripts_refspec'] = '${BUILDKITE_BRANCH}'
steps.append(trigger_build_step)
steps.append(create_branch_step)
print(yaml.dump({'steps': steps}))

105
scripts/pipeline_master.py
View file

@ -13,113 +13,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from steps import generic_linux, generic_windows
import yaml
if __name__ == '__main__':
scripts_refspec = os.getenv("ph_scripts_refspec", "master")
queue_prefix = os.getenv("ph_queue_prefix", "")
no_cache = os.getenv('ph_no_cache') is not None
filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly')
log_level = os.getenv('ph_log_level', 'WARNING')
notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(',')))
steps = []
linux_agents = {'queue': f'{queue_prefix}linux'}
t = os.getenv('ph_linux_agents')
if t is not None:
linux_agents = json.loads(t)
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
'commands': [
'set -euo pipefail',
'ccache --clear' if no_cache else '',
'ccache --zero-stats',
'ccache --show-config',
'mkdir -p artifacts',
'dpkg -l >> artifacts/packages.txt',
# Clone scripts.
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
'rm -rf ${SRC}',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"',
'cd ${SRC}',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo "llvm-premerge-checks commit"',
'git rev-parse HEAD',
'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"',
'set +e',
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level} {filter_output}',
'EXIT_STATUS=\\$?',
'echo "--- ccache stats"',
'ccache --print-stats',
'ccache --show-stats',
'exit \\$EXIT_STATUS',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': linux_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \
'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \
'sccache --start-server"'
# FIXME: openmp is removed as it constantly fails. Make this project list be evaluated through "choose_projects".
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
win_agents = {'queue': f'{queue_prefix}windows'}
t = os.getenv('ph_windows_agents')
if t is not None:
win_agents = json.loads(t)
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
'commands': [
clear_sccache if no_cache else '',
'sccache --zero-stats',
# Clone scripts.
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
'rm -rf %SRC%',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%',
'cd %SRC%',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo llvm-premerge-checks commit:',
'git rev-parse HEAD',
'cd %BUILDKITE_BUILD_CHECKOUT_PATH%',
'powershell -command "'
f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
'\\$exit=\\$?;'
'echo \'--- sccache stats\';'
'sccache --show-stats;'
'if (\\$exit) {'
' echo success;'
' exit 0; } '
'else {'
' echo failure;'
' exit 1;'
'}"',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': win_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
if os.getenv('ph_skip_linux') is None:
steps.append(linux_buld_step)
# TODO: windows builds are temporary disabled #243
# if os.getenv('ph_skip_windows') is None:
# steps.append(windows_buld_step)
steps.extend(generic_linux(
os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'),
False))
# FIXME: openmp is removed as it constantly fails.
# TODO: Make this project list be evaluated through "choose_projects".
steps.extend(generic_windows(
os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')))
notify = []
for e in notify_emails:
notify.append({'email': e})

162
scripts/pipeline_premerge.py
View file

@ -13,155 +13,68 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import os
from choose_projects import ChooseProjects
import git
from steps import libcxx, generic_linux, generic_windows
import yaml
if __name__ == '__main__':
scripts_refspec = os.getenv("ph_scripts_refspec", "master")
queue_prefix = os.getenv("ph_queue_prefix", "")
diff_id = os.getenv("ph_buildable_diff", "")
no_cache = os.getenv('ph_no_cache') is not None
filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
projects = os.getenv('ph_projects', 'detect')
log_level = os.getenv('ph_log_level', 'WARNING')
log_level = os.getenv('ph_log_level', 'INFO')
logging.basicConfig(level=log_level, format='%(levelname)-7s %(message)s')
# List all affected projects.
repo = git.Repo('.')
patch = repo.git.diff("HEAD~1")
cp = ChooseProjects('.')
affected_projects = cp.choose_projects(patch)
print('# all affected projects')
for p in affected_projects:
print(f'# {p}')
modified_files = cp.get_changed_files(patch)
modified_projects, unmapped_changes = cp.get_changed_projects(modified_files)
if unmapped_changes:
logging.warning('There were changes that could not be mapped to a project. Checking everything')
modified_projects = cp.all_projects
logging.info(f'modified projects: {modified_projects}')
# Add projects that depend on modified.
affected_projects = cp.get_affected_projects(modified_projects)
# Handle special checks.
checked = set()
generic_projects = set()
logging.info(f'all affected projects: {affected_projects}')
steps = []
for p in affected_projects:
if p == 'libcxx' or p == 'libcxxabi':
if 'libcxx' not in checked:
logging.info('Adding custom steps for "libc++"')
checked.add('libcxx')
steps.extend(libcxx())
else:
generic_projects.add(p)
if len(generic_projects) > 0:
# Add dependencies
projects = ';'.join(sorted(cp.add_dependencies(generic_projects)))
logging.info(f'Projects for default checks: {projects}')
steps.extend(generic_linux(projects, True))
steps.extend(generic_windows(projects))
else:
logging.info('No projects for default checks')
deps = []
if 'libcxx' in affected_projects or 'libcxxabi' in affected_projects:
start_libcxx_step = {
'trigger': 'libcxx-ci',
'label': 'libcxx',
'key': 'libcxx',
'async': False,
'build': {
'branch': os.getenv('BUILDKITE_BRANCH'),
'env': {},
},
}
for e in os.environ:
if e.startswith('ph_'):
start_libcxx_step['build']['env'][e] = os.getenv(e)
steps.append(start_libcxx_step)
deps.append(start_libcxx_step['key'])
steps.append({
'wait': '~',
'continue_on_failure': True,
})
linux_agents = {'queue': f'{queue_prefix}linux'}
t = os.getenv('ph_linux_agents')
if t is not None:
linux_agents = json.loads(t)
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
'commands': [
'set -euo pipefail',
'ccache --clear' if no_cache else '',
'ccache --zero-stats',
'ccache --show-config',
'mkdir -p artifacts',
'dpkg -l >> artifacts/packages.txt',
# Clone scripts.
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
'rm -rf ${SRC}',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"',
'cd ${SRC}',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo "llvm-premerge-checks commit"',
'git rev-parse HEAD',
'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"',
'set +e',
# Add link in review to the build.
'${SRC}/scripts/add_phabricator_artifact.py '
'--phid="$ph_target_phid" '
'--url="$BUILDKITE_BUILD_URL" '
'--name="Buildkite build"',
'${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
f'--projects="{projects}" --log-level={log_level} {filter_output}',
'EXIT_STATUS=\\$?',
'echo "--- ccache stats"',
'ccache --print-stats',
'ccache --show-stats',
'exit \\$EXIT_STATUS',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': linux_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \
'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \
'sccache --start-server"'
win_agents = {'queue': f'{queue_prefix}windows'}
t = os.getenv('ph_windows_agents')
if t is not None:
win_agents = json.loads(t)
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
'commands': [
clear_sccache if no_cache else '',
'sccache --zero-stats',
# Clone scripts.
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
'rm -rf %SRC%',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%',
'cd %SRC%',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo llvm-premerge-checks commit:',
'git rev-parse HEAD',
'cd %BUILDKITE_BUILD_CHECKOUT_PATH%',
'powershell -command "'
f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
'\\$exit=\\$?;'
'sccache --show-stats;'
'if (\\$exit) {'
' echo success;'
' exit 0; } '
'else {'
' echo failure;'
' exit 1;'
'}"',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': win_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
if os.getenv('ph_skip_linux') is None:
steps.append(linux_buld_step)
deps.append(linux_buld_step['key'])
# TODO: windows builds are temporary disabled #243
# if os.getenv('ph_skip_windows') is None:
# steps.append(windows_buld_step)
# deps.append(windows_buld_step['key'])
report_step = {
'label': ':spiral_note_pad: report',
'depends_on': deps,
'commands': [
'mkdir -p artifacts',
'buildkite-agent artifact download "*_result.json" .',
@ -178,9 +91,8 @@ if __name__ == '__main__':
'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"',
'${SRC}/scripts/summary.py',
],
'allow_dependency_failure': True,
'artifact_paths': ['artifacts/**/*'],
'agents': {'queue': f'{queue_prefix}linux'},
'agents': {'queue': 'linux'},
'timeout_in_minutes': 10,
}
steps.append(report_step)

147
scripts/steps.py Normal file
View file

@ -0,0 +1,147 @@
#!/usr/bin/env python3
# Copyright 2020 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from typing import List
def libcxx():
# TODO: read steps from the YAML file from the repo.
return []
def generic_linux(projects: str, check_diff: bool) -> List:
if os.getenv('ph_skip_linux') is not None:
return []
scripts_refspec = os.getenv("ph_scripts_refspec", "master")
no_cache = os.getenv('ph_no_cache') is not None
filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
log_level = os.getenv('ph_log_level', 'WARNING')
linux_agents = {'queue': 'linux'}
t = os.getenv('ph_linux_agents')
if t is not None:
linux_agents = json.loads(t)
commands = [
'set -euo pipefail',
'ccache --clear' if no_cache else '',
'ccache --zero-stats',
'ccache --show-config',
'mkdir -p artifacts',
'dpkg -l >> artifacts/packages.txt',
# Clone scripts.
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
'rm -rf ${SRC}',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"',
'cd ${SRC}',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo "llvm-premerge-checks commit"',
'git rev-parse HEAD',
'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"',
'set +e',
]
if check_diff:
commands.extend([
# Add link in review to the build.
'${SRC}/scripts/add_phabricator_artifact.py '
'--phid="$ph_target_phid" '
'--url="$BUILDKITE_BUILD_URL" '
'--name="Buildkite build"' if check_diff else '',
'${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
f'--projects="{projects}" --log-level={log_level} {filter_output}',
])
else:
commands.extend([
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level} {filter_output}'
])
commands.extend([
'EXIT_STATUS=\\$?',
'echo "--- ccache stats"',
'ccache --print-stats',
'ccache --show-stats',
'exit \\$EXIT_STATUS',
])
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
'commands': commands,
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': linux_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
return [linux_buld_step]
def generic_windows(projects: str) -> List:
# TODO: windows builds are temporary disabled #243
return []
if os.getenv('ph_skip_windows') is not None:
return []
scripts_refspec = os.getenv("ph_scripts_refspec", "master")
no_cache = os.getenv('ph_no_cache') is not None
log_level = os.getenv('ph_log_level', 'WARNING')
filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \
'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \
'sccache --start-server"'
win_agents = {'queue': 'windows'}
t = os.getenv('ph_windows_agents')
if t is not None:
win_agents = json.loads(t)
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
'commands': [
clear_sccache if no_cache else '',
'sccache --zero-stats',
# Clone scripts.
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
'rm -rf %SRC%',
'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%',
'cd %SRC%',
f'git fetch origin "{scripts_refspec}":x',
'git checkout x',
'echo llvm-premerge-checks commit:',
'git rev-parse HEAD',
'cd %BUILDKITE_BUILD_CHECKOUT_PATH%',
'powershell -command "'
f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
'\\$exit=\\$?;'
'sccache --show-stats;'
'if (\\$exit) {'
' echo success;'
' exit 0; } '
'else {'
' echo failure;'
' exit 1;'
'}"',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': win_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
]},
}
return [windows_buld_step]