From 8eee5355ea5a567b5d90951a396408bb8dac0ef9 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Mon, 25 May 2020 16:42:40 +0200 Subject: [PATCH] building with buildkite --- .editorconfig | 148 +++++++++++++++++ phabricator-proxy/main.py | 2 +- scripts/buildkite/build_branch_pipeline.py | 17 +- scripts/clang_format_report.py | 109 +++++++++++++ scripts/clang_tidy_report.py | 117 ++++++++++++++ scripts/ignore_diff.py | 37 +++-- scripts/lint.sh | 4 +- scripts/phabtalk/phabtalk.py | 67 ++++++-- scripts/premerge_checks.py | 175 +++++++++++++++++++++ scripts/run_cmake.py | 29 ++-- scripts/run_ninja.py | 15 +- scripts/test_results_report.py | 70 +++++++++ 12 files changed, 734 insertions(+), 56 deletions(-) create mode 100644 .editorconfig create mode 100755 scripts/clang_format_report.py create mode 100755 scripts/clang_tidy_report.py create mode 100755 scripts/premerge_checks.py create mode 100755 scripts/test_results_report.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4cb7dd4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,148 @@ +[*] +charset = utf-8 +end_of_line = lf +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on +ij_formatter_tags_enabled = false +ij_smart_tabs = false + + +[{*.pyw,*.py,*.pi}] +indent_style = space +insert_final_newline = false +indent_size = 4 +max_line_length = 120 +tab_width = 4 +ij_continuation_indent_size = 8 +ij_wrap_on_typing = false +ij_python_align_collections_and_comprehensions = true +ij_python_align_multiline_imports = true +ij_python_align_multiline_parameters = true +ij_python_align_multiline_parameters_in_calls = true +ij_python_blank_line_at_file_end = true +ij_python_blank_lines_after_imports = 1 +ij_python_blank_lines_after_local_imports = 0 +ij_python_blank_lines_around_class = 1 +ij_python_blank_lines_around_method = 1 +ij_python_blank_lines_around_top_level_classes_functions = 2 +ij_python_blank_lines_before_first_method = 0 +ij_python_dict_alignment = 0 +ij_python_dict_new_line_after_left_brace = false +ij_python_dict_new_line_before_right_brace = false +ij_python_dict_wrapping = 1 +ij_python_from_import_new_line_after_left_parenthesis = false +ij_python_from_import_new_line_before_right_parenthesis = false +ij_python_from_import_parentheses_force_if_multiline = false +ij_python_from_import_trailing_comma_if_multiline = false +ij_python_from_import_wrapping = 1 +ij_python_hang_closing_brackets = false +ij_python_keep_blank_lines_in_code = 1 +ij_python_keep_blank_lines_in_declarations = 1 +ij_python_keep_indents_on_empty_lines = false +ij_python_keep_line_breaks = true +ij_python_new_line_after_colon = false +ij_python_new_line_after_colon_multi_clause = true +ij_python_optimize_imports_always_split_from_imports = false +ij_python_optimize_imports_case_insensitive_order = false +ij_python_optimize_imports_join_from_imports_with_same_source = false +ij_python_optimize_imports_sort_by_type_first = true +ij_python_optimize_imports_sort_imports = true +ij_python_optimize_imports_sort_names_in_from_imports = false +ij_python_space_after_comma = true +ij_python_space_after_number_sign = true +ij_python_space_after_py_colon = true +ij_python_space_before_backslash = true +ij_python_space_before_comma = false +ij_python_space_before_for_semicolon = false +ij_python_space_before_lbracket = false +ij_python_space_before_method_call_parentheses = false +ij_python_space_before_method_parentheses = false +ij_python_space_before_number_sign = true +ij_python_space_before_py_colon = false +ij_python_space_within_empty_method_call_parentheses = false +ij_python_space_within_empty_method_parentheses = false +ij_python_spaces_around_additive_operators = true +ij_python_spaces_around_assignment_operators = true +ij_python_spaces_around_bitwise_operators = true +ij_python_spaces_around_eq_in_keyword_argument = false +ij_python_spaces_around_eq_in_named_parameter = false +ij_python_spaces_around_equality_operators = true +ij_python_spaces_around_multiplicative_operators = true +ij_python_spaces_around_power_operator = true +ij_python_spaces_around_relational_operators = true +ij_python_spaces_around_shift_operators = true +ij_python_spaces_within_braces = false +ij_python_spaces_within_brackets = false +ij_python_spaces_within_method_call_parentheses = false +ij_python_spaces_within_method_parentheses = false +ij_python_use_continuation_indent_for_arguments = false +ij_python_use_continuation_indent_for_collection_and_comprehensions = false +ij_python_wrap_long_lines = false + +[{*.sht,*.htm,*.html,*.shtm,*.ng,*.shtml}] +ij_html_add_new_line_before_tags = body,div,p,form,h1,h2,h3 +ij_html_align_attributes = true +ij_html_align_text = false +ij_html_attribute_wrap = normal +ij_html_block_comment_at_first_column = true +ij_html_do_not_align_children_of_min_lines = 0 +ij_html_do_not_break_if_inline_tags = title,h1,h2,h3,h4,h5,h6,p +ij_html_do_not_indent_children_of_tags = html,body,thead,tbody,tfoot +ij_html_enforce_quotes = false +ij_html_inline_tags = a,abbr,acronym,b,basefont,bdo,big,br,cite,cite,code,dfn,em,font,i,img,input,kbd,label,q,s,samp,select,small,span,strike,strong,sub,sup,textarea,tt,u,var +ij_html_keep_blank_lines = 2 +ij_html_keep_indents_on_empty_lines = false +ij_html_keep_line_breaks = true +ij_html_keep_line_breaks_in_text = true +ij_html_keep_whitespaces = false +ij_html_keep_whitespaces_inside = span,pre,textarea +ij_html_line_comment_at_first_column = true +ij_html_new_line_after_last_attribute = never +ij_html_new_line_before_first_attribute = never +ij_html_quote_style = double +ij_html_remove_new_line_before_tags = br +ij_html_space_after_tag_name = false +ij_html_space_around_equality_in_attribute = false +ij_html_space_inside_empty_tag = false +ij_html_text_wrap = normal + +[{*.vsl,*.vm,*.ft}] +ij_vtl_keep_indents_on_empty_lines = false + +[{*.xjsp,*.tag,*.jsp,*.jsf,*.jspf,*.tagf}] +ij_jsp_jsp_prefer_comma_separated_import_list = false +ij_jsp_keep_indents_on_empty_lines = false + +[{*.yml,*.yaml}] +indent_size = 2 +ij_yaml_keep_indents_on_empty_lines = false +ij_yaml_keep_line_breaks = true + +[{*.zsh,*.bash,*.sh}] +indent_size = 2 +tab_width = 2 +ij_shell_binary_ops_start_line = false +ij_shell_keep_column_alignment_padding = false +ij_shell_minify_program = false +ij_shell_redirect_followed_by_space = false +ij_shell_switch_cases_indented = false + +[{.eslintrc,.babelrc,.stylelintrc,jest.config,bowerrc,*.jsb3,*.jsb2,*.json}] +indent_size = 2 +ij_json_keep_blank_lines_in_code = 0 +ij_json_keep_indents_on_empty_lines = false +ij_json_keep_line_breaks = true +ij_json_space_after_colon = true +ij_json_space_after_comma = true +ij_json_space_before_colon = true +ij_json_space_before_comma = false +ij_json_spaces_within_braces = false +ij_json_spaces_within_brackets = false +ij_json_wrap_long_lines = false + +[{BUILD,WORKSPACE,*.bzl}] +ij_continuation_indent_size = 4 +ij_build_keep_indents_on_empty_lines = false + +[{spring.schemas,spring.handlers,*.properties}] +ij_properties_align_group_field_declarations = false diff --git a/phabricator-proxy/main.py b/phabricator-proxy/main.py index 3c48349..49b6036 100644 --- a/phabricator-proxy/main.py +++ b/phabricator-proxy/main.py @@ -49,7 +49,7 @@ def build(): headers=headers) app.logger.info('buildkite response: %s %s', response.status_code, response.text) rjs = json.loads(response.text) - return rjs['id'] + return rjs['web_url'] else: return "expected POST request" diff --git a/scripts/buildkite/build_branch_pipeline.py b/scripts/buildkite/build_branch_pipeline.py index 3226d22..a41f6d4 100755 --- a/scripts/buildkite/build_branch_pipeline.py +++ b/scripts/buildkite/build_branch_pipeline.py @@ -23,14 +23,13 @@ if __name__ == '__main__': steps = [] # SCRIPT_DIR is defined in buildkite pipeline step. linux_buld_step = { - 'label': 'build linux', - 'key': 'build-linux', - 'commands': [ - '${SCRIPT_DIR}/run_cmake.py detect', - '${SCRIPT_DIR}/run_ninja.py all', - '${SCRIPT_DIR}/run_ninja.py check-all', - '${SCRIPT_DIR}/lint.sh HEAD~1 ./'], - 'agents': {'queue': queue, 'os': 'linux'} + 'label': 'build linux', + 'key': 'build-linux', + 'commands': [ + '${SCRIPT_DIR}/premerge_checks.py', + ], + 'artifact_paths': ['artifacts/**/*'], + 'agents': {'queue': queue, 'os': 'linux'} } steps.append(linux_buld_step) - print(yaml.dump({'steps': steps})) \ No newline at end of file + print(yaml.dump({'steps': steps})) diff --git a/scripts/clang_format_report.py b/scripts/clang_format_report.py new file mode 100755 index 0000000..55d75ad --- /dev/null +++ b/scripts/clang_format_report.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import subprocess +import logging + +import pathspec +import unidiff + +from typing import Tuple, Optional +from phabtalk.phabtalk import Report, CheckResult + + +def get_diff(base_commit) -> Tuple[bool, str]: + r = subprocess.run(f'git-clang-format {base_commit}', shell=True) + logging.debug(f'git-clang-format {r}') + if r.returncode != 0: + logging.error(f'git-clang-format returned an non-zero exit code {r.returncode}') + r = subprocess.run(f'git checkout -- .', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + logging.debug(f'git reset {r}') + return False, '' + diff_run = subprocess.run(f'git diff -U0 --no-prefix --exit-code', capture_output=True, shell=True) + logging.debug(f'git diff {diff_run}') + r = subprocess.run(f'git checkout -- .', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + logging.debug(f'git reset {r}') + return True, diff_run.stdout.decode() + + +def run(base_commit, ignore_config, report: Optional[Report]): + """Apply clang-format and return if no issues were found.""" + if report is None: + report = Report() # For debugging. + r, patch = get_diff(base_commit) + if not r: + report.add_step('clang-format', CheckResult.FAILURE, '') + return + add_artifact = False + patches = unidiff.PatchSet(patch) + ignore_lines = [] + if ignore_config is not None and os.path.exists(ignore_config): + ignore_lines = open(ignore_config, 'r').readlines() + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_lines) + patched_file: unidiff.PatchedFile + success = True + for patched_file in patches: + add_artifact = True + if ignore.match_file(patched_file.source_file) or ignore.match_file(patched_file.target_file): + logging.info(f'patch of {patched_file.patch_info} is ignored') + continue + hunk: unidiff.Hunk + for hunk in patched_file: + lines = [str(x) for x in hunk] + success = False + m = 10 # max number of lines to report. + description = 'please reformat the code\n```\n' + n = len(lines) + cut = n > m + 1 + if cut: + lines = lines[:m] + description += ''.join(lines) + '\n```' + if cut: + description += f'\n{n - m} diff lines are omitted. See full path.' + report.add_lint({ + 'name': 'clang-format', + 'severity': 'autofix', + 'code': 'clang-format', + 'path': patched_file.source_file, + 'line': hunk.source_start, + 'char': 1, + 'description': description, + }) + if add_artifact: + patch_file = 'clang-format.patch' + with open(patch_file, 'w') as f: + f.write(patch) + report.add_artifact(os.getcwd(), patch_file, 'clang-format') + if success: + report.add_step('clang-format', CheckResult.SUCCESS, message='') + else: + report.add_step( + 'clang-format', + CheckResult.FAILURE, + 'Please format your changes with clang-format by running `git-clang-format HEAD^` or applying patch.') + logging.debug(f'report: {report}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Runs clang-format against given diff with given commit. ' + 'Produces patch and attaches linter comments to a review.') + parser.add_argument('--base', default='HEAD~1') + parser.add_argument('--ignore-config', default=None, help='path to file with patters of files to ignore') + parser.add_argument('--log-level', type=str, default='INFO') + args = parser.parse_args() + logging.basicConfig(level=args.log_level) + run(args.base, args.ignore_config, None) diff --git a/scripts/clang_tidy_report.py b/scripts/clang_tidy_report.py new file mode 100755 index 0000000..cc0d93f --- /dev/null +++ b/scripts/clang_tidy_report.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import re +import subprocess +from typing import Optional +import pathspec + +import ignore_diff +from phabtalk.phabtalk import Report, CheckResult + + +def run(base_commit, ignore_config, report: Optional[Report]): + """Apply clang-format and return if no issues were found.""" + r = subprocess.run(f'git diff -U0 --no-prefix {base_commit}', shell=True, capture_output=True) + logging.debug(f'git diff {r}') + diff = r.stdout.decode() + if ignore_config is not None and os.path.exists(ignore_config): + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, + open(ignore_config, 'r').readlines()) + diff = ignore_diff.remove_ignored(diff.splitlines(keepends=True), open(ignore_config, 'r')) + logging.debug(f'filtered diff: {diff}') + else: + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, []) + p = subprocess.Popen(['clang-tidy-diff', '-p0', '-quiet'], stdout=subprocess.PIPE, stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + a = ''.join(diff) + logging.info(f'clang-tidy input: {a}') + out = p.communicate(input=a.encode())[0].decode() + logging.debug(f'clang-tidy-diff {p}: {out}') + if report is None: + report = Report() # For debugging. + # Typical finding looks like: + # [cwd/]clang/include/clang/AST/DeclCXX.h:3058:20: error: ... [clang-diagnostic-error] + pattern = '^([^:]*):(\\d+):(\\d+): (.*): (.*)' + add_artifact = False + logging.debug("cwd", os.getcwd()) + errors_count = 0 + warn_count = 0 + inline_comments = 0 + for line in out.splitlines(keepends=False): + line = line.strip() + line = line.replace(os.getcwd() + os.sep, '') + logging.debug(line) + if len(line) == 0 or line == 'No relevant changes found.': + continue + add_artifact = True + match = re.search(pattern, line) + if match: + file_name = match.group(1) + line_pos = match.group(2) + char_pos = match.group(3) + severity = match.group(4) + text = match.group(5) + text += '\n[[{} | not useful]] '.format( + 'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not-useful') + if severity in ['warning', 'error']: + if severity == 'warning': + warn_count += 1 + if severity == 'error': + errors_count += 1 + if ignore.match_file(file_name): + print('{} is ignored by pattern and no comment will be added'.format(file_name)) + else: + inline_comments += 1 + report.add_lint({ + 'name': 'clang-tidy', + 'severity': 'warning', + 'code': 'clang-tidy', + 'path': file_name, + 'line': int(line_pos), + 'char': int(char_pos), + 'description': '{}: {}'.format(severity, text), + }) + else: + logging.debug('does not match pattern') + if add_artifact: + p = 'clang-tidy.txt' + with open(p, 'w') as f: + f.write(out) + report.add_artifact(os.getcwd(), p, 'clang-tidy') + if errors_count + warn_count == 0: + report.add_step('clang-tidy', CheckResult.SUCCESS, message='') + else: + report.add_step( + 'clang-tidy', + CheckResult.FAILURE, + f'clang-tidy found {errors_count} errors and {warn_count} warnings. {inline_comments} of them are added ' + f'as review comments. See' + f'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review-comments.') + logging.debug(f'report: {report}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Runs clang-format against given diff with given commit. ' + 'Produces patch and attaches linter comments to a review.') + parser.add_argument('--base', default='HEAD~1') + parser.add_argument('--ignore-config', default=None, help='path to file with patters of files to ignore') + parser.add_argument('--log-level', type=str, default='INFO') + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') + run(args.base, args.ignore_config, None) diff --git a/scripts/ignore_diff.py b/scripts/ignore_diff.py index 5f4faee..20a156c 100755 --- a/scripts/ignore_diff.py +++ b/scripts/ignore_diff.py @@ -12,31 +12,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import argparse +import logging import re import sys import pathspec -# Takes an output of git diff and removes files ignored by patten specified by ignore file. -def main(): - # FIXME: use argparse for parsing commandline parameters - # Maybe FIXME: Replace path to file with flags for tidy/format, use paths relative to `__file__` - argv = sys.argv[1:] - if not argv: - print("Please provide a path to .ignore file.") - sys.exit(1) - ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, - open(argv[0], 'r').readlines()) +def remove_ignored(diff_lines, ignore_patterns_lines): + logging.debug(f'ignore pattern {ignore_patterns_lines}') + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_patterns_lines) good = True - for line in sys.stdin: - match = re.search(r'^diff --git a/(.*) b/(.*)$', line) + result = [] + for line in diff_lines: + match = re.search(r'^diff --git (.*) (.*)$', line) if match: good = not (ignore.match_file(match.group(1)) and ignore.match_file(match.group(2))) if not good: + logging.debug(f'skip {line.rstrip()}') continue - sys.stdout.write(line) + result.append(line) + return result if __name__ == "__main__": - main() + # Maybe FIXME: Replace this tool usage with flags for tidy/format, use paths relative to `__file__` + parser = argparse.ArgumentParser(description='Takes an output of git diff and removes files ignored by patten ' + 'specified by ignore file') + parser.add_argument('ignore_config', default=None, + help='path to file with patters of files to ignore') + parser.add_argument('--log-level', type=str, default='WARNING') + args = parser.parse_args() + logging.basicConfig(level=args.log_level) + filtered = remove_ignored([x for x in sys.stdin], open(args.ignore_config, 'r').readlines()) + for x in filtered: + sys.stdout.write(x) diff --git a/scripts/lint.sh b/scripts/lint.sh index 2b1e71f..308d96c 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -39,12 +39,12 @@ fi # Let clang format apply patches --diff doesn't produces results in the format we want. git-clang-format "${COMMIT}" set +e -git diff -U0 --exit-code | "${DIR}/ignore_diff.py" "${DIR}/clang-format.ignore" > "${OUTPUT_DIR}"/clang-format.patch +git diff -U0 --exit-code --no-prefix | "${DIR}/ignore_diff.py" "${DIR}/clang-format.ignore" > "${OUTPUT_DIR}"/clang-format.patch set -e # Revert changes of git-clang-format. git checkout -- . # clang-tidy -git diff -U0 "${COMMIT}" | "${DIR}/ignore_diff.py" "${DIR}/clang-tidy.ignore" | clang-tidy-diff -p1 -quiet | sed "/^[[:space:]]*$/d" > "${OUTPUT_DIR}"/clang-tidy.txt +git diff -U0 --no-prefix "${COMMIT}" | "${DIR}/ignore_diff.py" "${DIR}/clang-tidy.ignore" | clang-tidy-diff -p0 -quiet | sed "/^[[:space:]]*$/d" > "${OUTPUT_DIR}"/clang-tidy.txt echo "linters completed ======================================" diff --git a/scripts/phabtalk/phabtalk.py b/scripts/phabtalk/phabtalk.py index 1267743..1c7d5ec 100755 --- a/scripts/phabtalk/phabtalk.py +++ b/scripts/phabtalk/phabtalk.py @@ -25,10 +25,10 @@ import time import urllib import uuid from typing import Optional, List, Dict - import pathspec from lxml import etree from phabricator import Phabricator +from enum import Enum class PhabTalk: @@ -126,24 +126,28 @@ class PhabTalk: print('Uploaded build status {}, {} test results and {} lint results'.format( result_type, len(unit), len(lint_messages))) + # TODO: deprecate def add_artifact(self, phid: str, file: str, name: str, results_url: str): artifactKey = str(uuid.uuid4()) artifactType = 'uri' artifactData = {'uri': '{}/{}'.format(results_url, file), 'ui.external': True, 'name': name} + self.create_artefact(phid, artifactKey, artifactType, artifactData) + print('Created artifact "{}"'.format(name)) + + def create_artifact(self, phid, artifact_key, artifact_type, artifact_data): if self.dryrun: print('harbormaster.createartifact =================') - print('artifactKey: {}'.format(artifactKey)) - print('artifactType: {}'.format(artifactType)) - print('artifactData: {}'.format(artifactData)) + print('artifactKey: {}'.format(artifact_key)) + print('artifactType: {}'.format(artifact_type)) + print('artifactData: {}'.format(artifact_data)) return _try_call(lambda: self._phab.harbormaster.createartifact( buildTargetPHID=phid, - artifactKey=artifactKey, - artifactType=artifactType, - artifactData=artifactData)) - print('Created artifact "{}"'.format(name)) + artifactKey=artifact_key, + artifactType=artifact_type, + artifactData=artifact_data)) def _parse_patch(patch) -> List[Dict[str, str]]: @@ -189,6 +193,47 @@ def _parse_patch(patch) -> List[Dict[str, str]]: return entries +class CheckResult(Enum): + UNKNOWN = 0 + SUCCESS = 1 + FAILURE = 2 + + +class Report: + def __init__(self): + self.comments = [] + self.success = True + self.working = False + self.unit = [] # type: List + self.lint = {} + self.test_stats = { + 'pass': 0, + 'fail': 0, + 'skip': 0 + } # type: Dict[str, int] + self.steps = [] # type: List + self.artifacts = [] # type: List + + def __str__(self): + return str(self.__dict__) + + def add_lint(self, m): + key = '{}:{}'.format(m['path'], m['line']) + if key not in self.lint: + self.lint[key] = [] + self.lint[key].append(m) + + def add_step(self, title: str, result: CheckResult, message: str): + self.steps.append({ + 'title': title, + 'result': result, + 'message': message, + }) + + def add_artifact(self, dir: str, file: str, name: str): + self.artifacts.append({'dir': dir, 'file': file, 'name': name}) + + class BuildReport: def __init__(self, args): @@ -295,7 +340,7 @@ class BuildReport: diffs = _parse_patch(open(p, 'r')) success = len(diffs) == 0 for d in diffs: - lines = d['diff'].splitlines(True) + lines = d['diff'].splitlines(keepends=True) m = 10 # max number of lines to report. description = 'please reformat the code\n```\n' n = len(lines) @@ -325,10 +370,10 @@ class BuildReport: self.success = success and self.success def add_clang_tidy(self): - # Typical message looks like - # [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error] if self.clang_tidy_result is None: return + # Typical message looks like + # [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error] pattern = '^{}/([^:]*):(\\d+):(\\d+): (.*): (.*)'.format(self.workspace) errors_count = 0 warn_count = 0 diff --git a/scripts/premerge_checks.py b/scripts/premerge_checks.py new file mode 100755 index 0000000..fc60b18 --- /dev/null +++ b/scripts/premerge_checks.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs all check on buildkite agent. +import json +import logging +import os +import pathlib +import re +import shutil +import subprocess +import time +import uuid +from typing import Callable, Optional + +import clang_format_report +import clang_tidy_report +import run_cmake +import test_results_report +from phabtalk.phabtalk import Report, CheckResult, PhabTalk + + +def upload_file(base_dir: str, file: str): + """ + Uploads artifact to buildkite and returns URL to it + """ + r = subprocess.run(f'buildkite-agent artifact upload "{file}"', shell=True, capture_output=True, cwd=base_dir) + logging.debug(f'upload-artifact {r}') + match = re.search('Uploading artifact ([^ ]*) ', r.stderr.decode()) + logging.debug(f'match {match}') + if match: + url = f'https://buildkite.com/organizations/llvm-project/pipelines/premerge-checks/builds/{os.getenv("BUILDKITE_BUILD_NUMBER")}/jobs/{os.getenv("BUILDKITE_JOB_ID")}/artifacts/{match.group(1)}' + logging.info(f'uploaded {file} to {url}') + return url + else: + logging.warning(f'could not find artifact {base_dir}/{file}') + return None + + +def maybe_add_url_artifact(phab: PhabTalk, url: str, name: str): + phid = os.getenv('ph_target_phid') + if phid is None: + return + phab.create_artifact(phid, str(uuid.uuid4()), 'uri', {'uri': url, 'ui.external': True, 'name': name}) + + +def add_shell_result(report: Report, name: str, exit_code: int) -> CheckResult: + logging.info(f'"{name}" exited with {exit_code}') + z = CheckResult.SUCCESS + if exit_code != 0: + z = CheckResult.FAILURE + report.add_step(name, z, '') + return z + + +def ninja_all_report(report: Report) -> CheckResult: + print('Full will be available in Artifacts "ninja-all.log"') + r = subprocess.run(f'ninja all | ' + f'tee {artifacts_dir}/ninja-all.log | ' + f'grep -vE "\\[.*] (Building|Linking|Copying|Generating|Creating)"', + shell=True, cwd=build_dir) + return add_shell_result(report, 'ninja all', r.returncode) + + +def ninja_check_all_report(report: Report) -> CheckResult: + # TODO: merge running ninja check all and analysing results in one step? + print('Full will be available in Artifacts "ninja-check-all.log"') + r = subprocess.run(f'ninja check-all | tee {artifacts_dir}/ninja-check-all.log | ' + f'grep -vE "^\\[.*] (Building|Linking)" | ' + f'grep -vE "^(PASS|XFAIL|UNSUPPORTED):"', shell=True, cwd=build_dir) + z = add_shell_result(report, 'ninja check all', r.returncode) + # TODO: check if test-results are present. + report.add_artifact(build_dir, 'test-results.xml', 'test results') + test_results_report.run(os.path.join(build_dir, 'test-results.xml'), report) + return z + + +def run_step(name: str, report: Report, thunk: Callable[[Report], CheckResult]) -> CheckResult: + global timings + start = time.time() + print(f'--- {name}') # New section in Buildkite log. + result = thunk(report) + timings[name] = time.time() - start + # Expand section if it failed. + if result == CheckResult.FAILURE: + print('^^^ +++') + return result + + +def cmake_report(report: Report) -> CheckResult: + global build_dir + cmake_result, build_dir, cmake_artifacts = run_cmake.run('detect', os.getcwd()) + for file in cmake_artifacts: + if os.path.exists(file): + shutil.copy2(file, artifacts_dir) + return add_shell_result(report, 'cmake', cmake_result) + + +def furl(url: str, name: Optional[str] = None): + if name is None: + name = url + return f"\033]1339;url='{url}';content='{name}'\a\n" + + +if __name__ == '__main__': + build_dir = '' + logging.basicConfig(level=logging.WARNING, format='%(levelname)-7s %(message)s') + scripts_dir = pathlib.Path(__file__).parent.absolute() + phab = PhabTalk(os.getenv('CONDUIT_TOKEN'), 'https://reviews.llvm.org/api/', False) + maybe_add_url_artifact(phab, os.getenv('BUILDKITE_BUILD_URL'), 'Buildkite build') + artifacts_dir = os.path.join(os.getcwd(), 'artifacts') + os.makedirs(artifacts_dir, exist_ok=True) + report = Report() + timings = {} + cmake_result = run_step('cmake', report, cmake_report) + if cmake_result == CheckResult.SUCCESS: + compile_result = run_step('ninja all', report, ninja_all_report) + if compile_result == CheckResult.SUCCESS: + run_step('ninja check all', report, ninja_check_all_report) + run_step('clang-tidy', report, + lambda x: clang_tidy_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-tidy.ignore'), x)) + run_step('clang-format', report, + lambda x: clang_format_report.run('HEAD~1', os.path.join(scripts_dir, 'clang-format.ignore'), x)) + print('+++ summary') + print(f'Branch {os.getenv("BUILDKITE_BRANCH")} at {os.getenv("BUILDKITE_REPO")}') + ph_buildable_diff = os.getenv('ph_buildable_diff') + if ph_buildable_diff is not None: + url = f'https://reviews.llvm.org/D{os.getenv("ph_buildable_revision")}?id={ph_buildable_diff}' + print(f'Review: {furl(url)}') + if os.getenv('BUILDKITE_TRIGGERED_FROM_BUILD_NUMBER') is not None: + url = f'https://buildkite.com/llvm-project/' \ + f'{os.getenv("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG")}/'\ + f'builds/{os.getenv("BUILDKITE_TRIGGERED_FROM_BUILD_NUMBER")}' + print(f'Triggered from build {furl(url)}') + logging.debug(report) + success = True + for s in report.steps: + mark = 'V' + if s['result'] == CheckResult.UNKNOWN: + mark = '?' + if s['result'] == CheckResult.FAILURE: + success = False + mark = 'X' + msg = s['message'] + if len(msg): + msg = ': ' + msg + print(f'{mark} {s["title"]}{msg}') + + # TODO: dump the report and deduplicate tests and other reports later (for multiple OS) in a separate step. + ph_target_phid = os.getenv('ph_target_phid') + if ph_target_phid is not None: + build_url = f'https://reviews.llvm.org/harbormaster/build/{os.getenv("ph_build_id")}' + print(f'Reporting results to Phabricator build {furl(build_url)}') + phab.update_build_status(ph_buildable_diff, ph_target_phid, False, success, report.lint, report.unit) + for a in report.artifacts: + url = upload_file(a['dir'], a['file']) + if url is not None: + maybe_add_url_artifact(phab, url, a['name']) + else: + logging.warning('No phabricator phid is specified. Will not update the build status in Phabricator') + # TODO: add link to report issue on github + with open(os.path.join(artifacts_dir, 'step-timings.json'), 'w') as f: + f.write(json.dumps(timings)) diff --git a/scripts/run_cmake.py b/scripts/run_cmake.py index 973b9a9..4bc7678 100755 --- a/scripts/run_cmake.py +++ b/scripts/run_cmake.py @@ -21,6 +21,7 @@ import platform import shutil import subprocess import stat +import sys from typing import List, Dict import yaml @@ -44,7 +45,8 @@ class Configuration: config = yaml.load(config_file, Loader=yaml.SafeLoader) self._environment = config['environment'] # type: Dict[OperatingSystem, Dict[str, str]] self.general_cmake_arguments = config['arguments']['general'] # type: List[str] - self._specific_cmake_arguments = config['arguments'] # type: Dict[OperatingSystem, List[str]] + self._specific_cmake_arguments = config[ + 'arguments'] # type: Dict[OperatingSystem, List[str]] self.operating_system = self._detect_os() # type: OperatingSystem @property @@ -126,8 +128,10 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]: return arguments -def run_cmake(projects: str, repo_path: str, config_file_path: str = None, *, dryrun: bool = False): - """Use cmake to configure the project. +def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run: bool = False): + """Use cmake to configure the project and create build directory. + + Returns build directory and path to created artifacts. This version works on all operating systems. """ @@ -137,7 +141,7 @@ def run_cmake(projects: str, repo_path: str, config_file_path: str = None, *, dr config = Configuration(config_file_path) build_dir = os.path.abspath(os.path.join(repo_path, 'build')) - if not dryrun: + if not dry_run: secure_delete(build_dir) os.makedirs(build_dir) @@ -146,13 +150,15 @@ def run_cmake(projects: str, repo_path: str, config_file_path: str = None, *, dr print('Enabled projects: {}'.format(llvm_enable_projects)) arguments = _create_args(config, llvm_enable_projects) cmd = 'cmake ' + ' '.join(arguments) - + print('Running cmake with these arguments:\n{}'.format(cmd), flush=True) - if dryrun: - print('Dryrun, not invoking CMake!') - else: - subprocess.check_call(cmd, env=env, shell=True, cwd=build_dir) - _link_compile_commands(config, repo_path, build_dir) + if dry_run: + print('Dry run, not invoking CMake!') + return 0, build_dir, [] + + result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir) + _link_compile_commands(config, repo_path, build_dir) + return result, build_dir, [os.path.join(build_dir, 'CMakeCache.txt')] def secure_delete(path: str): @@ -187,4 +193,5 @@ if __name__ == '__main__': parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd()) parser.add_argument('--dryrun', action='store_true') args = parser.parse_args() - run_cmake(args.projects, args.repo_path, dryrun=args.dryrun) + result, _, _ = run(args.projects, args.repo_path, dry_run=args.dryrun) + sys.exit(result) diff --git a/scripts/run_ninja.py b/scripts/run_ninja.py index 07ee5e6..435ca78 100755 --- a/scripts/run_ninja.py +++ b/scripts/run_ninja.py @@ -18,9 +18,10 @@ import os import platform import shutil import subprocess +import sys -def check_sccache(dryrun:bool): +def check_sccache(dryrun: bool): """check if sccache can be started Wipe local cache folder if it fails with a timeout. @@ -40,18 +41,18 @@ def check_sccache(dryrun:bool): print('sccache failed with timeout. Wiping local cache dir {}'.format(sccache_dir)) if dryrun: print('Dryrun. Not deleting anything.') - else: + else: shutil.rmtree(sccache_dir) -def run_ninja(target: str, repo_path: str, *, dryrun:bool = False): - check_sccache(dryrun) - build_dir = os.path.join(repo_path, 'build') +def run_ninja(target: str, work_dir: str, *, dryrun: bool = False): + check_sccache(dryrun) cmd = 'ninja {}'.format(target) if dryrun: print('Dryrun. Command would have been:\n{}'.format(cmd)) + return 0 else: - subprocess.check_call(cmd, shell=True, cwd=build_dir) + return subprocess.call(cmd, shell=True, cwd=work_dir) if __name__ == '__main__': @@ -60,4 +61,4 @@ if __name__ == '__main__': parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd()) parser.add_argument('--dryrun', action='store_true') args = parser.parse_args() - run_ninja(args.target, args.repo_path, dryrun=args.dryrun) + sys.exit(run_ninja(args.target, os.path.join(args.repo_path, 'build'), dryrun=args.dryrun)) diff --git a/scripts/test_results_report.py b/scripts/test_results_report.py new file mode 100755 index 0000000..75cee6a --- /dev/null +++ b/scripts/test_results_report.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import logging +from typing import Optional +from lxml import etree +from phabtalk.phabtalk import Report, CheckResult + + +def run(test_results, report: Optional[Report]): + """Apply clang-format and return if no issues were found.""" + if report is None: + report = Report() # For debugging. + if not os.path.exists(test_results): + logging.warning(f'{test_results} not found') + report.add_step('clang-format', CheckResult.UNKNOWN, 'test report is not found') + return + success = True + root_node = etree.parse(test_results) + for test_case in root_node.xpath('//testcase'): + test_result = 'pass' + if test_case.find('failure') is not None: + test_result = 'fail' + if test_case.find('skipped') is not None: + test_result = 'skip' + report.test_stats[test_result] += 1 + if test_result == 'fail': + success = False + failure = test_case.find('failure') + test_result = { + 'name': test_case.attrib['name'], + 'namespace': test_case.attrib['classname'], + 'result': test_result, + 'duration': float(test_case.attrib['time']), + 'details': failure.text + } + report.unit.append(test_result) + + msg = f'{report.test_stats["pass"]} tests passed, {report.test_stats["fail"]} failed and' \ + f'{report.test_stats["skip"]} were skipped.\n' + if success: + report.add_step('test results', CheckResult.SUCCESS, msg) + else: + for test_case in report.unit: + if test_case['result'] == 'fail': + msg += f'{test_case["namespace"]}/{test_case["name"]}\n' + report.add_step('unit tests', CheckResult.FAILURE, msg) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Processes results from xml report') + parser.add_argument('test_report', default='build/test-results.xml') + parser.add_argument('--log-level', type=str, default='INFO') + args = parser.parse_args() + logging.basicConfig(level=args.log_level) + run(args.test_report, None)