revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

Merge pull request #210 from google/print-reproduce

Browse source 
Print commands to reproduce build locally
This commit is contained in:
Mikhail Goncharov 2020-07-09 15:35:49 +02:00 committed by GitHub
commit 01ccfca7b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
scripts/buildkite/build_master_pipeline.py
View file

@ -20,12 +20,14 @@ if __name__ == '__main__':
script_branch = os.getenv("scripts_branch", "master")
queue_prefix = os.getenv("ph_queue_prefix", "")
no_cache = os.getenv('ph_no_cache', '') != ''
projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly')
log_level = os.getenv('ph_log_level', 'WARNING')
steps = []
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
'commands': [
'set -euo pipefail',
'ccache --clear' if no_cache else '',
'ccache --zero-stats',
'mkdir -p artifacts',
@ -33,7 +35,8 @@ if __name__ == '__main__':
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
'rm -rf ${SRC}',
'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}"',
'set -eo pipefail',
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}',
'EXIT_STATUS=\\$?',
'echo "--- ccache stats"',
'ccache --show-stats',
@ -47,7 +50,7 @@ if __name__ == '__main__':
'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
'sccache --start-server"'
# FIXME: openmp is removed as it constantly fails.
projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
@ -57,7 +60,7 @@ if __name__ == '__main__':
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
'rm -rf %SRC%',
'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%',
f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\'; '
f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; '
'\\$exit=\\$?;'
'echo \'--- sccache stats\';'
'sccache --show-stats;'

1
scripts/phabtalk/phabtalk.py
View file

@ -199,6 +199,7 @@ class Step:
self.success = True
self.duration = 0.0
self.messages = []
self.reproduce_commands = []
def set_status_from_exit_code(self, exit_code: int):
if exit_code != 0:

20
scripts/premerge_checks.py
View file

@ -35,6 +35,7 @@ from phabtalk.phabtalk import Report, PhabTalk, Step
def ninja_all_report(step: Step, _: Report):
print('Full log will be available in Artifacts "ninja-all.log"', flush=True)
step.reproduce_commands.append('ninja all')
r = subprocess.run(f'ninja all | '
f'tee {artifacts_dir}/ninja-all.log | '
f'grep -vE "\\[.*] (Linking|Linting|Copying|Generating|Creating)"',
@ -45,8 +46,9 @@ def ninja_all_report(step: Step, _: Report):
def ninja_check_all_report(step: Step, _: Report):
print('Full log will be available in Artifacts "ninja-check-all.log"', flush=True)
step.reproduce_commands.append('ninja check-all')
r = subprocess.run(f'ninja check-all | tee {artifacts_dir}/ninja-check-all.log | '
f'grep -vE "^\\[.*] (Building|Linking)" | '
f'grep -vE "^\\[.*] (Building|Linking|Generating)" | '
f'grep -vE "^(PASS|XFAIL|UNSUPPORTED):"', shell=True, cwd=build_dir)
logging.debug(f'ninja check-all: returned {r.returncode}, stderr: "{r.stderr}"')
step.set_status_from_exit_code(r.returncode)
@ -69,11 +71,12 @@ def run_step(name: str, report: Report, thunk: Callable[[Step, Report], None]) -
def cmake_report(projects: str, step: Step, _: Report):
global build_dir
cmake_result, build_dir, cmake_artifacts = run_cmake.run(projects, os.getcwd())
cmake_result, build_dir, cmake_artifacts, commands = run_cmake.run(projects, os.getcwd())
for file in cmake_artifacts:
if os.path.exists(file):
shutil.copy2(file, artifacts_dir)
step.set_status_from_exit_code(cmake_result)
step.reproduce_commands = commands
def as_dict(obj):
@ -120,6 +123,7 @@ if __name__ == '__main__':
run_step('clang-format', report,
lambda s, r: clang_format_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-format.ignore'), s, r))
logging.debug(report)
print('+++ Summary', flush=True)
for s in report.steps:
mark = 'OK '
if not s.success:
@ -128,7 +132,17 @@ if __name__ == '__main__':
msg = ''
if len(s.messages):
msg = ': ' + '\n '.join(s.messages)
print(f'{mark} {s.name}{msg}')
print(f'{mark} {s.name}{msg}', flush=True)
print('--- Reproduce build locally', flush=True)
print(f'git clone {os.getenv("BUILDKITE_REPO")}')
print(f'git checkout {os.getenv("BUILDKITE_BRANCH")}')
for s in report.steps:
if len(s.reproduce_commands) == 0:
continue
print('\n'.join(s.reproduce_commands), flush=True)
print('', flush=True)
if not report.success:
print('^^^ +++', flush=True)
ph_target_phid = os.getenv('ph_target_phid')
ph_buildable_diff = os.getenv('ph_buildable_diff')

50
scripts/run_cmake.py
View file

@ -104,7 +104,7 @@ def _create_env(config: Configuration) -> Dict[str, str]:
return env
def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
def _create_args(config: Configuration, llvm_enable_projects: str, use_cache: bool) -> List[str]:
"""Generate the command line arguments for cmake."""
arguments = [
os.path.join('..', 'llvm'),
@ -113,21 +113,21 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
arguments.extend(config.general_cmake_arguments)
arguments.extend(config.specific_cmake_arguments)
# enable sccache
if 'SCCACHE_DIR' in os.environ:
logging.info("using sccache")
arguments.extend([
'-DCMAKE_C_COMPILER_LAUNCHER=sccache',
'-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
])
# enable ccache if the path is set in the environment
elif 'CCACHE_PATH' in os.environ:
logging.info("using ccache")
arguments.extend([
'-D LLVM_CCACHE_BUILD=ON',
'-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
'-D LLVM_CCACHE_MAXSIZE=20G',
])
if use_cache:
if 'SCCACHE_DIR' in os.environ:
logging.info("using sccache")
arguments.extend([
'-DCMAKE_C_COMPILER_LAUNCHER=sccache',
'-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
])
# enable ccache if the path is set in the environment
elif 'CCACHE_PATH' in os.environ:
logging.info("using ccache")
arguments.extend([
'-D LLVM_CCACHE_BUILD=ON',
'-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
'-D LLVM_CCACHE_MAXSIZE=20G',
])
return arguments
@ -136,8 +136,11 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
Returns build directory and path to created artifacts.
This version works on all operating systems.
This version works on Linux and Windows.
Returns: exit code of cmake command, build directory, path to CMakeCache.txt, commands.
"""
commands = []
if config_file_path is None:
script_dir = os.path.dirname(__file__)
config_file_path = os.path.join(script_dir, 'run_cmake_config.yaml')
@ -147,21 +150,26 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
if not dry_run:
secure_delete(build_dir)
os.makedirs(build_dir)
commands.append("rm -rf build")
commands.append("mkdir build")
commands.append("cd build")
for k, v in config.environment.items():
commands.append(f'export {k}="{v}"')
env = _create_env(config)
llvm_enable_projects = _select_projects(config, projects, repo_path)
print('Enabled projects: {}'.format(llvm_enable_projects), flush=True)
arguments = _create_args(config, llvm_enable_projects)
arguments = _create_args(config, llvm_enable_projects, True)
cmd = 'cmake ' + ' '.join(arguments)
print('Running cmake with these arguments:\n{}'.format(cmd), flush=True)
if dry_run:
print('Dry run, not invoking CMake!')
return 0, build_dir, []
result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir)
commands.append('cmake ' + ' '.join(_create_args(config, llvm_enable_projects, False)))
commands.append('# ^note that compiler cache arguments are omitted')
_link_compile_commands(config, repo_path, build_dir)
return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')]
return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')], commands
def secure_delete(path: str):

2
scripts/run_cmake_config.yaml
View file

@ -26,6 +26,6 @@ arguments:
linux:
# CCACHE is enabled in script iff environment variable `CCACHE_PATH` is set
- '-D LLVM_ENABLE_LLD=ON'
- '-DCMAKE_CXX_FLAGS=-gmlt'
- '-D CMAKE_CXX_FLAGS=-gmlt'
windows:
- '-D LLVM_ENABLE_DIA_SDK=OFF'