From 3925a2a84473e26a086d884281cb1a5f4a27a94f Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Fri, 9 Oct 2020 11:05:19 +0200 Subject: [PATCH] get libcxx steps from sh output --- scripts/llvm-dependencies.yaml | 8 ++++--- scripts/pipeline_master.py | 14 ++++++++++-- scripts/pipeline_premerge.py | 42 +++++++++++++--------------------- scripts/steps.py | 39 +++++++++++++++++++++++++++---- 4 files changed, 68 insertions(+), 35 deletions(-) diff --git a/scripts/llvm-dependencies.yaml b/scripts/llvm-dependencies.yaml index 237b8e3..ea98e23 100644 --- a/scripts/llvm-dependencies.yaml +++ b/scripts/llvm-dependencies.yaml @@ -70,19 +70,21 @@ allprojects: # projects excluded from automatic configuration as they could not be built excludedProjects: - # These projects are not working with Visual Studio Compiler on Windows windows: - lldb - llgo - libunwind - - libcxxabi - - openmp # blacklisting as kuhnel has trouble with the Perl installation + - libcxx # no windows support + - libcxxabi # no windows support + - openmp # TODO: check: kuhnel has trouble with the Perl installation - debuginfo-tests # test failing - polly # test failing - flang # compilation failing with invalid compile arguments # test stuck, needs to be killed manually: instrprof-multiprocess.test - compiler-rt linux: + - libcxx # has custom checks + - libcxxabi # has custom checks # tests keep failing: # debuginfo-tests :: llgdb-tests/asan-deque.cpp # debuginfo-tests :: llgdb-tests/asan.c diff --git a/scripts/pipeline_master.py b/scripts/pipeline_master.py index 4c8ced2..a571dd6 100755 --- a/scripts/pipeline_master.py +++ b/scripts/pipeline_master.py @@ -15,9 +15,13 @@ import os -from steps import generic_linux, generic_windows +from steps import generic_linux, generic_windows, from_shell_output import yaml +steps_generators = [ + '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-snapshot.sh', +] + if __name__ == '__main__': scripts_refspec = os.getenv("ph_scripts_refspec", "master") no_cache = os.getenv('ph_no_cache') is not None @@ -30,9 +34,15 @@ if __name__ == '__main__': os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'), False)) # FIXME: openmp is removed as it constantly fails. - # TODO: Make this project list be evaluated through "choose_projects". + + # TODO: Make this project list be evaluated through "choose_projects"(? as now we define "all" and exclusions in + # two placess). steps.extend(generic_windows( os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'))) + + for gen in steps_generators: + steps.extend(from_shell_output(gen)) + notify = [] for e in notify_emails: notify.append({'email': e}) diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index 01cc8cc..154acbf 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -18,9 +18,13 @@ import os from choose_projects import ChooseProjects import git -from steps import libcxx, generic_linux, generic_windows +from steps import generic_linux, generic_windows, from_shell_output import yaml +steps_generators = [ + '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-premerge.sh', +] + if __name__ == '__main__': scripts_refspec = os.getenv("ph_scripts_refspec", "master") diff_id = os.getenv("ph_buildable_diff", "") @@ -42,33 +46,18 @@ if __name__ == '__main__': logging.info(f'modified projects: {modified_projects}') # Add projects that depend on modified. affected_projects = cp.get_affected_projects(modified_projects) - - # Handle special checks. - checked = set() - generic_projects = set() - logging.info(f'all affected projects: {affected_projects}') steps = [] - for p in affected_projects: - if p == 'libcxx' or p == 'libcxxabi': - if 'libcxx' not in checked: - logging.info('Adding custom steps for "libc++"') - checked.add('libcxx') - steps.extend(libcxx()) - else: - generic_projects.add(p) + projects = cp.add_dependencies(affected_projects) + logging.info(f'projects with dependencies: {projects}') + excluded_linux = cp.get_excluded('linux') + logging.info(f'excluded for linux: {excluded_linux}') + steps.extend(generic_linux(';'.join(sorted(projects - excluded_linux)), True)) + excluded_windows = cp.get_excluded('windows') + logging.info(f'excluded for windows: {excluded_windows}') + steps.extend(generic_windows(';'.join(sorted(projects - excluded_windows)))) - if len(generic_projects) > 0: - # Add dependencies - projects = cp.add_dependencies(generic_projects) - logging.info(f'all projects {projects}') - excluded_linux = cp.get_excluded('linux') - logging.info(f'excluded for linux: {excluded_linux}') - steps.extend(generic_linux(';'.join(sorted(projects - excluded_linux)), True)) - excluded_windows = cp.get_excluded('windows') - logging.info(f'excluded for windows: {excluded_windows}') - steps.extend(generic_windows(';'.join(sorted(projects - excluded_windows)))) - else: - logging.info('No projects for default checks') + for gen in steps_generators: + steps.extend(from_shell_output(gen)) if os.getenv('ph_target_phid') is None: logging.warning('ph_target_phid is not specified. Skipping "Report" step') @@ -99,4 +88,5 @@ if __name__ == '__main__': 'timeout_in_minutes': 10, } steps.append(report_step) + print(yaml.dump({'steps': steps})) diff --git a/scripts/steps.py b/scripts/steps.py index 0b0024d..79421e7 100644 --- a/scripts/steps.py +++ b/scripts/steps.py @@ -13,14 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import json +import logging import os from typing import List - -def libcxx(): - # TODO: read steps from the YAML file from the repo. - return [] +from exec_utils import watch_shell +import yaml def generic_linux(projects: str, check_diff: bool) -> List: @@ -143,3 +143,34 @@ def generic_windows(projects: str) -> List: ]}, } return [windows_buld_step] + + +def from_shell_output(command) -> []: + """ + Executes shell command and parses stdout as multidoc yaml file, see + https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. + :param command: command, may include env variables + :return: all 'steps' that defined in the result ("env" section is ignored). + Non-zero exit code and malformed YAML produces empty result. + """ + path = os.path.expandvars(command) + logging.debug(f'invoking "{path}"') + out = io.BytesIO() + err = io.BytesIO() + rc = watch_shell(out.write, err.write, path) + logging.debug(f'exit code: {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"') + steps = [] + if rc != 0: + logging.error( + f'{path} returned non-zero code {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"') + return steps + try: + for part in yaml.safe_load_all(out.getvalue()): + part.setdefault('steps', []) + steps.extend(part['steps']) + except yaml.YAMLError as e: + logging.error(f'''"{path}" produced malformed YAML, exception: +{e} + +stdout: >>>{out.getvalue().decode()}>>>''') + return steps