Print commands to reproduce build locally

- additional arguments to set enabled projects, log level and reset
cache.

scripts/buildkite/build_master_pipeline.py

@@ -20,12 +20,14 @@ if __name__ == '__main__':
     script_branch = os.getenv("scripts_branch", "master")
     queue_prefix = os.getenv("ph_queue_prefix", "")
     no_cache = os.getenv('ph_no_cache', '') != ''
-    projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'
+    projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly')
+    log_level = os.getenv('ph_log_level', 'WARNING')
     steps = []
     linux_buld_step = {
         'label': ':linux: build and test linux',
         'key': 'linux',
         'commands': [
+            'set -euo pipefail',
             'ccache --clear' if no_cache else '',
             'ccache --zero-stats',
             'mkdir -p artifacts',
@@ -33,7 +35,8 @@ if __name__ == '__main__':
             'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
             'rm -rf ${SRC}',
             'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
-            f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}"',
+            'set -eo pipefail',
+            f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}',
             'EXIT_STATUS=\\$?',
             'echo "--- ccache stats"',
             'ccache --show-stats',
@@ -47,7 +50,7 @@ if __name__ == '__main__':
                     'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
                     'sccache --start-server"'
     # FIXME: openmp is removed as it constantly fails.
-    projects = 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'
+    projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
     windows_buld_step = {
         'label': ':windows: build and test windows',
         'key': 'windows',
@@ -57,7 +60,7 @@ if __name__ == '__main__':
             'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
             'rm -rf %SRC%',
             'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%',
-            f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\'; '
+            f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; '
             '\\$exit=\\$?;'
             'echo \'--- sccache stats\';'
             'sccache --show-stats;'

scripts/phabtalk/phabtalk.py

@@ -199,6 +199,7 @@ class Step:
         self.success = True
         self.duration = 0.0
         self.messages = []
+        self.reproduce_commands = []
 
     def set_status_from_exit_code(self, exit_code: int):
         if exit_code != 0:

scripts/premerge_checks.py

@@ -35,6 +35,7 @@ from phabtalk.phabtalk import Report, PhabTalk, Step
 
 def ninja_all_report(step: Step, _: Report):
     print('Full log will be available in Artifacts "ninja-all.log"', flush=True)
+    step.reproduce_commands.append('ninja all')
     r = subprocess.run(f'ninja all | '
                        f'tee {artifacts_dir}/ninja-all.log | '
                        f'grep -vE "\\[.*] (Linking|Linting|Copying|Generating|Creating)"',
@@ -45,8 +46,9 @@ def ninja_all_report(step: Step, _: Report):
 
 def ninja_check_all_report(step: Step, _: Report):
     print('Full log will be available in Artifacts "ninja-check-all.log"', flush=True)
+    step.reproduce_commands.append('ninja check-all')
     r = subprocess.run(f'ninja check-all | tee {artifacts_dir}/ninja-check-all.log | '
-                       f'grep -vE "^\\[.*] (Building|Linking)" | '
+                       f'grep -vE "^\\[.*] (Building|Linking|Generating)" | '
                        f'grep -vE "^(PASS|XFAIL|UNSUPPORTED):"', shell=True, cwd=build_dir)
     logging.debug(f'ninja check-all: returned {r.returncode}, stderr: "{r.stderr}"')
     step.set_status_from_exit_code(r.returncode)
@@ -69,11 +71,12 @@ def run_step(name: str, report: Report, thunk: Callable[[Step, Report], None]) -
 
 def cmake_report(projects: str, step: Step, _: Report):
     global build_dir
-    cmake_result, build_dir, cmake_artifacts = run_cmake.run(projects, os.getcwd())
+    cmake_result, build_dir, cmake_artifacts, commands = run_cmake.run(projects, os.getcwd())
     for file in cmake_artifacts:
         if os.path.exists(file):
             shutil.copy2(file, artifacts_dir)
     step.set_status_from_exit_code(cmake_result)
+    step.reproduce_commands = commands
 
 
 def as_dict(obj):
@@ -120,6 +123,7 @@ if __name__ == '__main__':
         run_step('clang-format', report,
                  lambda s, r: clang_format_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-format.ignore'), s, r))
     logging.debug(report)
+    print('+++ Summary', flush=True)
     for s in report.steps:
         mark = 'OK   '
         if not s.success:
@@ -128,7 +132,17 @@ if __name__ == '__main__':
         msg = ''
         if len(s.messages):
             msg = ': ' + '\n  '.join(s.messages)
-        print(f'{mark} {s.name}{msg}')
+        print(f'{mark} {s.name}{msg}', flush=True)
+    print('--- Reproduce build locally', flush=True)
+    print(f'git clone {os.getenv("BUILDKITE_REPO")}')
+    print(f'git checkout {os.getenv("BUILDKITE_BRANCH")}')
+    for s in report.steps:
+        if len(s.reproduce_commands) == 0:
+            continue
+        print('\n'.join(s.reproduce_commands), flush=True)
+    print('', flush=True)
+    if not report.success:
+        print('^^^ +++', flush=True)
 
     ph_target_phid = os.getenv('ph_target_phid')
     ph_buildable_diff = os.getenv('ph_buildable_diff')

scripts/run_cmake.py

@@ -104,7 +104,7 @@ def _create_env(config: Configuration) -> Dict[str, str]:
     return env
 
 
-def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
+def _create_args(config: Configuration, llvm_enable_projects: str, use_cache: bool) -> List[str]:
     """Generate the command line arguments for cmake."""
     arguments = [
         os.path.join('..', 'llvm'),
@@ -113,21 +113,21 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
     arguments.extend(config.general_cmake_arguments)
     arguments.extend(config.specific_cmake_arguments)
 
-    # enable sccache
+    if use_cache:
-    if 'SCCACHE_DIR' in os.environ:
+        if 'SCCACHE_DIR' in os.environ:
-        logging.info("using sccache")
+            logging.info("using sccache")
-        arguments.extend([
+            arguments.extend([
-            '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
+                '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
-            '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
+                '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
-        ])
+            ])
-    # enable ccache if the path is set in the environment
+        # enable ccache if the path is set in the environment
-    elif 'CCACHE_PATH' in os.environ:
+        elif 'CCACHE_PATH' in os.environ:
-        logging.info("using ccache")
+            logging.info("using ccache")
-        arguments.extend([
+            arguments.extend([
-            '-D LLVM_CCACHE_BUILD=ON',
+                '-D LLVM_CCACHE_BUILD=ON',
-            '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
+                '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
-            '-D LLVM_CCACHE_MAXSIZE=20G',
+                '-D LLVM_CCACHE_MAXSIZE=20G',
-        ])
+            ])
     return arguments
 
 
@@ -136,8 +136,11 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
 
     Returns build directory and path to created artifacts.
 
-    This version works on all operating systems.
+    This version works on Linux and Windows.
+
+    Returns: exit code of cmake command, build directory, path to CMakeCache.txt, commands.
     """
+    commands = []
     if config_file_path is None:
         script_dir = os.path.dirname(__file__)
         config_file_path = os.path.join(script_dir, 'run_cmake_config.yaml')
@@ -147,21 +150,26 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
     if not dry_run:
         secure_delete(build_dir)
         os.makedirs(build_dir)
-
+        commands.append("rm -rf build")
+        commands.append("mkdir build")
+        commands.append("cd build")
+    for k, v in config.environment.items():
+        commands.append(f'export {k}="{v}"')
     env = _create_env(config)
     llvm_enable_projects = _select_projects(config, projects, repo_path)
     print('Enabled projects: {}'.format(llvm_enable_projects), flush=True)
-    arguments = _create_args(config, llvm_enable_projects)
+    arguments = _create_args(config, llvm_enable_projects, True)
     cmd = 'cmake ' + ' '.join(arguments)
 
     print('Running cmake with these arguments:\n{}'.format(cmd), flush=True)
     if dry_run:
         print('Dry run, not invoking CMake!')
         return 0, build_dir, []
-
     result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir)
+    commands.append('cmake ' + ' '.join(_create_args(config, llvm_enable_projects, False)))
+    commands.append('# ^note that compiler cache arguments are omitted')
     _link_compile_commands(config, repo_path, build_dir)
-    return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')]
+    return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')], commands
 
 
 def secure_delete(path: str):

scripts/run_cmake_config.yaml

@@ -26,6 +26,6 @@ arguments:
   linux:
     # CCACHE is enabled in script iff environment variable `CCACHE_PATH` is set
     - '-D LLVM_ENABLE_LLD=ON'
-    - '-DCMAKE_CXX_FLAGS=-gmlt'
+    - '-D CMAKE_CXX_FLAGS=-gmlt'
   windows:
     - '-D LLVM_ENABLE_DIA_SDK=OFF'