2020-04-24 12:49:07 +02:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
# Copyright 2020 Google LLC
|
|
|
|
#
|
|
|
|
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# https://llvm.org/LICENSE.txt
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2020-07-24 12:08:10 +02:00
|
|
|
import json
|
2020-04-24 15:43:09 +02:00
|
|
|
import os
|
2020-05-14 12:57:38 +02:00
|
|
|
import yaml
|
2020-04-24 15:43:09 +02:00
|
|
|
|
2020-04-24 12:49:07 +02:00
|
|
|
if __name__ == '__main__':
|
2020-07-10 17:53:22 +02:00
|
|
|
scripts_branch = os.getenv("scripts_branch", "master")
|
2020-06-17 13:52:33 +02:00
|
|
|
queue_prefix = os.getenv("ph_queue_prefix", "")
|
2020-05-14 12:57:38 +02:00
|
|
|
diff_id = os.getenv("ph_buildable_diff", "")
|
2020-07-10 17:53:22 +02:00
|
|
|
no_cache = os.getenv('ph_no_cache') is not None
|
|
|
|
filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
|
|
|
|
projects = os.getenv('ph_projects', 'detect')
|
|
|
|
log_level = os.getenv('ph_log_level', 'WARNING')
|
2020-05-14 12:57:38 +02:00
|
|
|
steps = []
|
2020-07-24 12:08:10 +02:00
|
|
|
linux_agents = {'queue': f'{queue_prefix}linux'}
|
|
|
|
t = os.getenv('ph_linux_agents')
|
|
|
|
if t is not None:
|
|
|
|
linux_agents = json.loads(t)
|
2020-05-14 12:57:38 +02:00
|
|
|
linux_buld_step = {
|
2020-06-03 13:40:22 +02:00
|
|
|
'label': ':linux: build and test linux',
|
|
|
|
'key': 'linux',
|
2020-05-25 16:42:40 +02:00
|
|
|
'commands': [
|
2020-07-10 17:53:22 +02:00
|
|
|
'set -euo pipefail',
|
|
|
|
'ccache --clear' if no_cache else '',
|
2020-07-24 12:08:10 +02:00
|
|
|
'ccache --zero-stats',
|
2020-08-26 12:00:50 +02:00
|
|
|
'ccache --show-config',
|
2020-06-18 11:54:28 +02:00
|
|
|
'mkdir -p artifacts',
|
2020-06-18 10:54:57 +02:00
|
|
|
'dpkg -l >> artifacts/packages.txt',
|
2020-06-03 13:40:22 +02:00
|
|
|
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
|
|
|
|
'rm -rf ${SRC}',
|
2020-07-10 17:53:22 +02:00
|
|
|
f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git '
|
|
|
|
'${SRC}',
|
|
|
|
'echo "llvm-premerge-checks commit"',
|
2020-07-22 17:04:02 +02:00
|
|
|
'git --git-dir ${SRC}/.git rev-parse HEAD',
|
2020-07-10 17:53:22 +02:00
|
|
|
'set +e',
|
2020-06-03 13:40:22 +02:00
|
|
|
# Add link in review to the build.
|
|
|
|
'${SRC}/scripts/phabtalk/add_url_artifact.py '
|
|
|
|
'--phid="$ph_target_phid" '
|
|
|
|
'--url="$BUILDKITE_BUILD_URL" '
|
|
|
|
'--name="Buildkite build"',
|
2020-07-10 17:53:22 +02:00
|
|
|
'${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
|
|
|
|
f'--projects="{projects}" --log-level={log_level} {filter_output}',
|
|
|
|
'EXIT_STATUS=\\$?',
|
|
|
|
'echo "--- ccache stats"',
|
2020-08-26 12:00:50 +02:00
|
|
|
'ccache --print-stats',
|
2020-07-10 17:53:22 +02:00
|
|
|
'ccache --show-stats',
|
|
|
|
'exit \\$EXIT_STATUS',
|
2020-05-25 16:42:40 +02:00
|
|
|
],
|
2020-06-03 13:40:22 +02:00
|
|
|
'artifact_paths': ['artifacts/**/*', '*_result.json'],
|
2020-07-24 12:08:10 +02:00
|
|
|
'agents': linux_agents,
|
2020-06-29 12:17:03 +02:00
|
|
|
'timeout_in_minutes': 120,
|
2020-07-21 10:26:00 +02:00
|
|
|
'retry': {'automatic': [
|
|
|
|
{'exit_status': -1, 'limit': 2}, # Agent lost
|
|
|
|
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
|
|
|
|
]},
|
2020-05-14 12:57:38 +02:00
|
|
|
}
|
2020-07-28 14:32:13 +02:00
|
|
|
clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \
|
|
|
|
'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \
|
2020-07-10 17:53:22 +02:00
|
|
|
'sccache --start-server"'
|
2020-07-24 12:08:10 +02:00
|
|
|
win_agents = {'queue': f'{queue_prefix}windows'}
|
|
|
|
t = os.getenv('ph_windows_agents')
|
|
|
|
if t is not None:
|
|
|
|
win_agents = json.loads(t)
|
2020-06-03 13:40:22 +02:00
|
|
|
windows_buld_step = {
|
|
|
|
'label': ':windows: build and test windows',
|
|
|
|
'key': 'windows',
|
|
|
|
'commands': [
|
2020-07-10 17:53:22 +02:00
|
|
|
clear_sccache if no_cache else '',
|
2020-06-04 12:36:46 +02:00
|
|
|
'sccache --zero-stats',
|
2020-06-03 13:40:22 +02:00
|
|
|
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
|
|
|
|
'rm -rf %SRC%',
|
2020-07-10 17:53:22 +02:00
|
|
|
f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git %SRC%',
|
2020-07-22 17:04:02 +02:00
|
|
|
'echo llvm-premerge-checks commit:',
|
|
|
|
'git --git-dir %SRC%/.git rev-parse HEAD',
|
2020-07-10 17:53:22 +02:00
|
|
|
'powershell -command "'
|
|
|
|
f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
|
2020-06-03 13:40:22 +02:00
|
|
|
'\\$exit=\\$?;'
|
|
|
|
'sccache --show-stats;'
|
|
|
|
'if (\\$exit) {'
|
2020-07-07 15:13:38 +02:00
|
|
|
' echo success;'
|
2020-06-03 13:40:22 +02:00
|
|
|
' exit 0; } '
|
|
|
|
'else {'
|
2020-07-07 15:13:38 +02:00
|
|
|
' echo failure;'
|
2020-06-03 13:40:22 +02:00
|
|
|
' exit 1;'
|
2020-07-07 15:13:38 +02:00
|
|
|
'}"',
|
2020-06-03 13:40:22 +02:00
|
|
|
],
|
|
|
|
'artifact_paths': ['artifacts/**/*', '*_result.json'],
|
2020-07-24 12:08:10 +02:00
|
|
|
'agents': win_agents,
|
2020-06-29 12:17:03 +02:00
|
|
|
'timeout_in_minutes': 120,
|
2020-07-21 10:26:00 +02:00
|
|
|
'retry': {'automatic': [
|
|
|
|
{'exit_status': -1, 'limit': 2}, # Agent lost
|
|
|
|
{'exit_status': 255, 'limit': 2}, # Forced agent shutdown
|
|
|
|
]},
|
2020-06-03 13:40:22 +02:00
|
|
|
}
|
2020-07-14 18:26:53 +02:00
|
|
|
deps = []
|
|
|
|
if os.getenv('ph_skip_linux') is None:
|
|
|
|
steps.append(linux_buld_step)
|
|
|
|
deps.append(linux_buld_step['key'])
|
2020-09-28 10:38:19 +02:00
|
|
|
# TODO: temporary disable windows step
|
|
|
|
# if os.getenv('ph_skip_windows') is None:
|
|
|
|
# steps.append(windows_buld_step)
|
|
|
|
# deps.append(windows_buld_step['key'])
|
2020-06-03 13:40:22 +02:00
|
|
|
report_step = {
|
|
|
|
'label': ':spiral_note_pad: report',
|
2020-07-14 18:26:53 +02:00
|
|
|
'depends_on': deps,
|
2020-06-03 13:40:22 +02:00
|
|
|
'commands': [
|
|
|
|
'mkdir -p artifacts',
|
|
|
|
'buildkite-agent artifact download "*_result.json" .',
|
|
|
|
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
|
|
|
|
'rm -rf ${SRC}',
|
2020-07-10 17:53:22 +02:00
|
|
|
f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git '
|
|
|
|
'${SRC}',
|
2020-06-03 13:40:22 +02:00
|
|
|
'${SRC}/scripts/buildkite/summary.py',
|
|
|
|
],
|
|
|
|
'allow_dependency_failure': True,
|
|
|
|
'artifact_paths': ['artifacts/**/*'],
|
2020-06-29 12:17:03 +02:00
|
|
|
'agents': {'queue': f'{queue_prefix}linux'},
|
|
|
|
'timeout_in_minutes': 10,
|
2020-06-03 13:40:22 +02:00
|
|
|
}
|
|
|
|
steps.append(report_step)
|
2020-05-25 16:42:40 +02:00
|
|
|
print(yaml.dump({'steps': steps}))
|