Merge pull request #211 from google/fix-exit-status

Fix exit status

scripts/buildkite/build_branch_pipeline.py

@@ -17,40 +17,62 @@ import os
 import yaml
 
 if __name__ == '__main__':
-    script_branch = os.getenv("scripts_branch", "master")
+    scripts_branch = os.getenv("scripts_branch", "master")
     queue_prefix = os.getenv("ph_queue_prefix", "")
     diff_id = os.getenv("ph_buildable_diff", "")
-
+    no_cache = os.getenv('ph_no_cache') is not None
+    filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
+    projects = os.getenv('ph_projects', 'detect')
+    log_level = os.getenv('ph_log_level', 'WARNING')
     steps = []
     linux_buld_step = {
         'label': ':linux: build and test linux',
         'key': 'linux',
         'commands': [
+            'set -euo pipefail',
+            'ccache --clear' if no_cache else '',
+             'ccache --zero-stats',
             'mkdir -p artifacts',
             'dpkg -l >> artifacts/packages.txt',
             'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
             'rm -rf ${SRC}',
-            'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
+            f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git '
+            '${SRC}',
+            'echo "llvm-premerge-checks commit"',
+            'git rev-parse HEAD',
+            'set +e',
             # Add link in review to the build.
             '${SRC}/scripts/phabtalk/add_url_artifact.py '
             '--phid="$ph_target_phid" '
             '--url="$BUILDKITE_BUILD_URL" '
             '--name="Buildkite build"',
-            '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy',
+            '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
+            f'--projects="{projects}" --log-level={log_level} {filter_output}',
+            'EXIT_STATUS=\\$?',
+            'echo "--- ccache stats"',
+            'ccache --show-stats',
+            'exit \\$EXIT_STATUS',
         ],
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
         'agents': {'queue': f'{queue_prefix}linux'},
         'timeout_in_minutes': 120,
     }
+    clear_sccache = 'powershell -command "sccache --stop-server; ' \
+                    'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
+                    'sccache --start-server"'
     windows_buld_step = {
         'label': ':windows: build and test windows',
         'key': 'windows',
         'commands': [
+            clear_sccache if no_cache else '',
             'sccache --zero-stats',
             'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
             'rm -rf %SRC%',
-            'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%',
+            f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git %SRC%',
-            'powershell -command "%SRC%/scripts/premerge_checks.py; '
+            'echo "llvm-premerge-checks commit"',
+            'git rev-parse HEAD',
+            'powershell -command "'
+            f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
             '\\$exit=\\$?;'
             'sccache --show-stats;'
             'if (\\$exit) {'
@@ -75,7 +97,8 @@ if __name__ == '__main__':
             'buildkite-agent artifact download "*_result.json" .',
             'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
             'rm -rf ${SRC}',
-            'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
+            f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git '
+            '${SRC}',
             '${SRC}/scripts/buildkite/summary.py',
         ],
         'allow_dependency_failure': True,

scripts/buildkite/build_master_pipeline.py

@@ -17,9 +17,10 @@ import os
 import yaml
 
 if __name__ == '__main__':
-    script_branch = os.getenv("scripts_branch", "master")
+    scripts_branch = os.getenv("scripts_branch", "master")
     queue_prefix = os.getenv("ph_queue_prefix", "")
-    no_cache = os.getenv('ph_no_cache', '') != ''
+    no_cache = os.getenv('ph_no_cache') is not None
+    filter_output = '--filter-output' if os.getenv('ph_no_filter_output') is None else ''
     projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly')
     log_level = os.getenv('ph_log_level', 'WARNING')
     steps = []
@@ -34,9 +35,12 @@ if __name__ == '__main__':
             'dpkg -l >> artifacts/packages.txt',
             'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
             'rm -rf ${SRC}',
-            'git clone --depth 1 --branch ${scripts_branch} https://github.com/google/llvm-premerge-checks.git ${SRC}',
+            f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git '
-            'set -eo pipefail',
+            '${SRC}',
-            f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}',
+            'echo "llvm-premerge-checks commit"',
+            'git rev-parse HEAD',
+            'set +e',
+            f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level} {filter_output}',
             'EXIT_STATUS=\\$?',
             'echo "--- ccache stats"',
             'ccache --show-stats',
@@ -59,8 +63,11 @@ if __name__ == '__main__':
             'sccache --zero-stats',
             'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks',
             'rm -rf %SRC%',
-            'git clone --depth 1 --branch %scripts_branch% https://github.com/google/llvm-premerge-checks.git %SRC%',
+            f'git clone --depth 1 --branch {scripts_branch} https://github.com/google/llvm-premerge-checks.git %SRC%',
-            f'powershell -command "%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; '
+            'echo "llvm-premerge-checks commit"',
+            'git rev-parse HEAD',
+            'powershell -command "'
+            f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level} {filter_output}; '
             '\\$exit=\\$?;'
             'echo \'--- sccache stats\';'
             'sccache --show-stats;'
@@ -76,6 +83,8 @@ if __name__ == '__main__':
         'agents': {'queue': f'{queue_prefix}windows'},
         'timeout_in_minutes': 120,
     }
-    steps.append(linux_buld_step)
+    if os.getenv('ph_skip_linux') is None:
-    steps.append(windows_buld_step)
+        steps.append(linux_buld_step)
+    if os.getenv('ph_skip_windows') is None:
+        steps.append(windows_buld_step)
     print(yaml.dump({'steps': steps}))

scripts/exec_utils.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import logging
+import os
+from asyncio.subprocess import PIPE
+from typing import Callable, AnyStr
+
+
+async def read_stream_and_display(stream, display):
+    while True:
+        line = await stream.readline()
+        if not line:
+            break
+        display(line)  # assume it doesn't block
+
+
+async def read_and_display(write_stdout, write_stderr, *cmd, **kwargs):
+    logging.debug(f'subprocess called with {cmd}; {kwargs}')
+    process = await asyncio.create_subprocess_shell(*cmd, stdout=PIPE, stderr=PIPE, **kwargs)
+    try:
+        await asyncio.gather(
+            read_stream_and_display(process.stdout, write_stdout),
+            read_stream_and_display(process.stderr, write_stderr))
+    except Exception:
+        process.kill()
+        raise
+    finally:
+        return await process.wait()
+
+
+def tee(s: AnyStr, write1: Callable[[AnyStr], None], write2: Callable[[AnyStr], None]):
+    write1(s)
+    write2(s)
+
+
+def if_not_matches(s: AnyStr, regexp, write: Callable[[AnyStr], None]):
+    x = s
+    if isinstance(s, (bytes, bytearray)):
+        x = s.decode()
+    if regexp.match(x) is None:
+        write(s)
+
+
+def watch_shell(write_stdout, write_stderr, *cmd, **kwargs):
+    if os.name == 'nt':
+        loop = asyncio.ProactorEventLoop()  # Windows
+        asyncio.set_event_loop(loop)
+    else:
+        loop = asyncio.get_event_loop()
+    rc = loop.run_until_complete(read_and_display(write_stdout, write_stderr, *cmd, **kwargs))
+    return rc

scripts/premerge_checks.py

@@ -19,9 +19,11 @@ import json
 import logging
 import os
 import pathlib
+import re
 import shutil
-import subprocess
+import sys
 import time
+from functools import partial
 from typing import Callable
 
 import clang_format_report
@@ -29,29 +31,41 @@ import clang_tidy_report
 import run_cmake
 import test_results_report
 from buildkite.utils import upload_file
+from exec_utils import watch_shell, if_not_matches, tee
 from phabtalk.add_url_artifact import maybe_add_url_artifact
 from phabtalk.phabtalk import Report, PhabTalk, Step
 
 
-def ninja_all_report(step: Step, _: Report):
+def ninja_all_report(step: Step, _: Report, filter_output: bool):
     print('Full log will be available in Artifacts "ninja-all.log"', flush=True)
     step.reproduce_commands.append('ninja all')
-    r = subprocess.run(f'ninja all | '
+    with open(f'{artifacts_dir}/ninja-all.log', 'wb') as f:
-                       f'tee {artifacts_dir}/ninja-all.log | '
+        w = sys.stdout.buffer.write
-                       f'grep -vE "\\[.*] (Linking|Linting|Copying|Generating|Creating)"',
+        if filter_output:
-                       shell=True, cwd=build_dir)
+            r = re.compile(r'^\[.*] (Building|Linking|Linting|Copying|Generating|Creating)')
-    logging.debug(f'ninja all: returned {r.returncode}, stderr: "{r.stderr}"')
+            w = partial(if_not_matches, write=sys.stdout.buffer.write, regexp=r)
-    step.set_status_from_exit_code(r.returncode)
+        rc = watch_shell(
+            partial(tee, write1=w, write2=f.write),
+            partial(tee, write1=sys.stderr.buffer.write, write2=f.write),
+            'ninja all', cwd=build_dir)
+        logging.debug(f'ninja all: returned {rc}')
+        step.set_status_from_exit_code(rc)
 
 
-def ninja_check_all_report(step: Step, _: Report):
+def ninja_check_all_report(step: Step, _: Report, filter_output: bool):
     print('Full log will be available in Artifacts "ninja-check-all.log"', flush=True)
     step.reproduce_commands.append('ninja check-all')
-    r = subprocess.run(f'ninja check-all | tee {artifacts_dir}/ninja-check-all.log | '
+    with open(f'{artifacts_dir}/ninja-check-all.log', 'wb') as f:
-                       f'grep -vE "^\\[.*] (Building|Linking|Generating)" | '
+        w = sys.stdout.buffer.write
-                       f'grep -vE "^(PASS|XFAIL|UNSUPPORTED):"', shell=True, cwd=build_dir)
+        if filter_output:
-    logging.debug(f'ninja check-all: returned {r.returncode}, stderr: "{r.stderr}"')
+            r = re.compile(r'^(\[.*] (Building|Linking|Generating)|(PASS|XFAIL|UNSUPPORTED):)')
-    step.set_status_from_exit_code(r.returncode)
+            w = partial(if_not_matches, write=sys.stdout.buffer.write, regexp=r)
+        rc = watch_shell(
+            partial(tee, write1=w, write2=f.write),
+            partial(tee, write1=sys.stderr.buffer.write, write2=f.write),
+            'ninja check-all', cwd=build_dir)
+        logging.debug(f'ninja check-all: returned {rc}')
+        step.set_status_from_exit_code(rc)
     test_results_report.run(build_dir, 'test-results.xml', step, report)
 
 
@@ -91,6 +105,7 @@ if __name__ == '__main__':
     parser.add_argument('--log-level', type=str, default='WARNING')
     parser.add_argument('--check-clang-format', action='store_true')
     parser.add_argument('--check-clang-tidy', action='store_true')
+    parser.add_argument('--filter-output', action='store_true')
     parser.add_argument('--projects', type=str, default='detect',
                         help="Projects to select, either a list or projects like 'clang;libc', or "
                              "'detect' to automatically infer proejcts from the diff, or "
@@ -113,9 +128,9 @@ if __name__ == '__main__':
     report.success = True
     cmake = run_step('cmake', report, lambda s, r: cmake_report(args.projects, s, r))
     if cmake.success:
-        ninja_all = run_step('ninja all', report, ninja_all_report)
+        ninja_all = run_step('ninja all', report, partial(ninja_all_report, filter_output=args.filter_output))
         if ninja_all.success:
-            run_step('ninja check-all', report, ninja_check_all_report)
+            run_step('ninja check-all', report, partial(ninja_check_all_report, filter_output=args.filter_output))
         if args.check_clang_tidy:
             run_step('clang-tidy', report,
                      lambda s, r: clang_tidy_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-tidy.ignore'), s, r))
@@ -159,6 +174,7 @@ if __name__ == '__main__':
         logging.warning('No phabricator phid is specified. Will not update the build status in Phabricator')
     with open(report_path, 'w') as f:
         json.dump(report.__dict__, f, default=as_dict)
+
     if not report.success:
         print('Build completed with failures', flush=True)
         exit(1)