Print commands to reproduce build locally
- additional arguments to set enabled projects, log level and reset cache.
This commit is contained in:
Mikhail Goncharov
parent
7667fc8004
commit
6bf3c85cc6
5 changed files with 55 additions and 29 deletions
|
|
@ -20,12 +20,14 @@ if __name__ == '__main__':
|
|||
script_branch = os.getenv("scripts_branch", "master")
|
||||
queue_prefix = os.getenv("ph_queue_prefix", "")
|
||||
no_cache = os.getenv('ph_no_cache', '') != ''
|
||||
projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'
|
||||
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly')
|
||||
log_level = os.getenv('ph_log_level', 'WARNING')
|
||||
steps = []
|
||||
linux_buld_step = {
|
||||
'label': ':linux: build and test linux',
|
||||
'key': 'linux',
|
||||
'commands': [
|
||||
'set -euo pipefail',
|
||||
'ccache --clear' if no_cache else '',
|
||||
'ccache --zero-stats',
|
||||
'mkdir -p artifacts',
|
||||
|
|
@ -33,7 +35,8 @@ if __name__ == '__main__':
|
|||
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
|
||||
'rm -rf ${SRC}',
|
||||
'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
|
||||
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}"',
|
||||
'set -eo pipefail',
|
||||
f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}',
|
||||
'EXIT_STATUS=\\$?',
|
||||
'echo "--- ccache stats"',
|
||||
'ccache --show-stats',
|
||||
|
|
@ -47,7 +50,7 @@ if __name__ == '__main__':
|
|||
'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
|
||||
'sccache --start-server"'
|
||||
# FIXME: openmp is removed as it constantly fails.
|
||||
projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'
|
||||
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
|
||||
windows_buld_step = {
|
||||
'label': ':windows: build and test windows',
|
||||
'key': 'windows',
|
||||
|
|
@ -57,7 +60,7 @@ if __name__ == '__main__':
|
|||
'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
|
||||
'rm -rf %SRC%',
|
||||
'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%',
|
||||
f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\'; '
|
||||
f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; '
|
||||
'\\$exit=\\$?;'
|
||||
'echo \'--- sccache stats\';'
|
||||
'sccache --show-stats;'
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ class Step:
|
|||
self.success = True
|
||||
self.duration = 0.0
|
||||
self.messages = []
|
||||
self.reproduce_commands = []
|
||||
|
||||
def set_status_from_exit_code(self, exit_code: int):
|
||||
if exit_code != 0:
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from phabtalk.phabtalk import Report, PhabTalk, Step
|
|||
|
||||
def ninja_all_report(step: Step, _: Report):
|
||||
print('Full log will be available in Artifacts "ninja-all.log"', flush=True)
|
||||
step.reproduce_commands.append('ninja all')
|
||||
r = subprocess.run(f'ninja all | '
|
||||
f'tee {artifacts_dir}/ninja-all.log | '
|
||||
f'grep -vE "\\[.*] (Linking|Linting|Copying|Generating|Creating)"',
|
||||
|
|
@ -45,8 +46,9 @@ def ninja_all_report(step: Step, _: Report):
|
|||
|
||||
def ninja_check_all_report(step: Step, _: Report):
|
||||
print('Full log will be available in Artifacts "ninja-check-all.log"', flush=True)
|
||||
step.reproduce_commands.append('ninja check-all')
|
||||
r = subprocess.run(f'ninja check-all | tee {artifacts_dir}/ninja-check-all.log | '
|
||||
f'grep -vE "^\\[.*] (Building|Linking)" | '
|
||||
f'grep -vE "^\\[.*] (Building|Linking|Generating)" | '
|
||||
f'grep -vE "^(PASS|XFAIL|UNSUPPORTED):"', shell=True, cwd=build_dir)
|
||||
logging.debug(f'ninja check-all: returned {r.returncode}, stderr: "{r.stderr}"')
|
||||
step.set_status_from_exit_code(r.returncode)
|
||||
|
|
@ -69,11 +71,12 @@ def run_step(name: str, report: Report, thunk: Callable[[Step, Report], None]) -
|
|||
|
||||
def cmake_report(projects: str, step: Step, _: Report):
|
||||
global build_dir
|
||||
cmake_result, build_dir, cmake_artifacts = run_cmake.run(projects, os.getcwd())
|
||||
cmake_result, build_dir, cmake_artifacts, commands = run_cmake.run(projects, os.getcwd())
|
||||
for file in cmake_artifacts:
|
||||
if os.path.exists(file):
|
||||
shutil.copy2(file, artifacts_dir)
|
||||
step.set_status_from_exit_code(cmake_result)
|
||||
step.reproduce_commands = commands
|
||||
|
||||
|
||||
def as_dict(obj):
|
||||
|
|
@ -120,6 +123,7 @@ if __name__ == '__main__':
|
|||
run_step('clang-format', report,
|
||||
lambda s, r: clang_format_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-format.ignore'), s, r))
|
||||
logging.debug(report)
|
||||
print('+++ Summary', flush=True)
|
||||
for s in report.steps:
|
||||
mark = 'OK '
|
||||
if not s.success:
|
||||
|
|
@ -128,7 +132,17 @@ if __name__ == '__main__':
|
|||
msg = ''
|
||||
if len(s.messages):
|
||||
msg = ': ' + '\n '.join(s.messages)
|
||||
print(f'{mark} {s.name}{msg}')
|
||||
print(f'{mark} {s.name}{msg}', flush=True)
|
||||
print('--- Reproduce build locally', flush=True)
|
||||
print(f'git clone {os.getenv("BUILDKITE_REPO")}')
|
||||
print(f'git checkout {os.getenv("BUILDKITE_BRANCH")}')
|
||||
for s in report.steps:
|
||||
if len(s.reproduce_commands) == 0:
|
||||
continue
|
||||
print('\n'.join(s.reproduce_commands), flush=True)
|
||||
print('', flush=True)
|
||||
if not report.success:
|
||||
print('^^^ +++', flush=True)
|
||||
|
||||
ph_target_phid = os.getenv('ph_target_phid')
|
||||
ph_buildable_diff = os.getenv('ph_buildable_diff')
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ def _create_env(config: Configuration) -> Dict[str, str]:
|
|||
return env
|
||||
|
||||
|
||||
def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
|
||||
def _create_args(config: Configuration, llvm_enable_projects: str, use_cache: bool) -> List[str]:
|
||||
"""Generate the command line arguments for cmake."""
|
||||
arguments = [
|
||||
os.path.join('..', 'llvm'),
|
||||
|
|
@ -113,21 +113,21 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
|
|||
arguments.extend(config.general_cmake_arguments)
|
||||
arguments.extend(config.specific_cmake_arguments)
|
||||
|
||||
# enable sccache
|
||||
if 'SCCACHE_DIR' in os.environ:
|
||||
logging.info("using sccache")
|
||||
arguments.extend([
|
||||
'-DCMAKE_C_COMPILER_LAUNCHER=sccache',
|
||||
'-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
|
||||
])
|
||||
# enable ccache if the path is set in the environment
|
||||
elif 'CCACHE_PATH' in os.environ:
|
||||
logging.info("using ccache")
|
||||
arguments.extend([
|
||||
'-D LLVM_CCACHE_BUILD=ON',
|
||||
'-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
|
||||
'-D LLVM_CCACHE_MAXSIZE=20G',
|
||||
])
|
||||
if use_cache:
|
||||
if 'SCCACHE_DIR' in os.environ:
|
||||
logging.info("using sccache")
|
||||
arguments.extend([
|
||||
'-DCMAKE_C_COMPILER_LAUNCHER=sccache',
|
||||
'-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
|
||||
])
|
||||
# enable ccache if the path is set in the environment
|
||||
elif 'CCACHE_PATH' in os.environ:
|
||||
logging.info("using ccache")
|
||||
arguments.extend([
|
||||
'-D LLVM_CCACHE_BUILD=ON',
|
||||
'-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
|
||||
'-D LLVM_CCACHE_MAXSIZE=20G',
|
||||
])
|
||||
return arguments
|
||||
|
||||
|
||||
|
|
@ -136,8 +136,11 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
|
|||
|
||||
Returns build directory and path to created artifacts.
|
||||
|
||||
This version works on all operating systems.
|
||||
This version works on Linux and Windows.
|
||||
|
||||
Returns: exit code of cmake command, build directory, path to CMakeCache.txt, commands.
|
||||
"""
|
||||
commands = []
|
||||
if config_file_path is None:
|
||||
script_dir = os.path.dirname(__file__)
|
||||
config_file_path = os.path.join(script_dir, 'run_cmake_config.yaml')
|
||||
|
|
@ -147,21 +150,26 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
|
|||
if not dry_run:
|
||||
secure_delete(build_dir)
|
||||
os.makedirs(build_dir)
|
||||
|
||||
commands.append("rm -rf build")
|
||||
commands.append("mkdir build")
|
||||
commands.append("cd build")
|
||||
for k, v in config.environment.items():
|
||||
commands.append(f'export {k}="{v}"')
|
||||
env = _create_env(config)
|
||||
llvm_enable_projects = _select_projects(config, projects, repo_path)
|
||||
print('Enabled projects: {}'.format(llvm_enable_projects), flush=True)
|
||||
arguments = _create_args(config, llvm_enable_projects)
|
||||
arguments = _create_args(config, llvm_enable_projects, True)
|
||||
cmd = 'cmake ' + ' '.join(arguments)
|
||||
|
||||
print('Running cmake with these arguments:\n{}'.format(cmd), flush=True)
|
||||
if dry_run:
|
||||
print('Dry run, not invoking CMake!')
|
||||
return 0, build_dir, []
|
||||
|
||||
result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir)
|
||||
commands.append('cmake ' + ' '.join(_create_args(config, llvm_enable_projects, False)))
|
||||
commands.append('# ^note that compiler cache arguments are omitted')
|
||||
_link_compile_commands(config, repo_path, build_dir)
|
||||
return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')]
|
||||
return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')], commands
|
||||
|
||||
|
||||
def secure_delete(path: str):
|
||||
|
|
|
|||
|
|
@ -26,6 +26,6 @@ arguments:
|
|||
linux:
|
||||
# CCACHE is enabled in script iff environment variable `CCACHE_PATH` is set
|
||||
- '-D LLVM_ENABLE_LLD=ON'
|
||||
- '-DCMAKE_CXX_FLAGS=-gmlt'
|
||||
- '-D CMAKE_CXX_FLAGS=-gmlt'
|
||||
windows:
|
||||
- '-D LLVM_ENABLE_DIA_SDK=OFF'
|
||||
|
|
|
|||
Loading…
Reference in a new issue