#!/usr/bin/env python3 # Copyright 2020 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://llvm.org/LICENSE.txt # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import io import json import logging import os from typing import List, Set, Dict from exec_utils import watch_shell import yaml def generic_linux(projects: str, check_diff: bool) -> List: if os.getenv('ph_skip_linux') is not None: return [] scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') linux_agents = {'queue': 'linux'} t = os.getenv('ph_linux_agents') if t is not None: linux_agents = json.loads(t) commands = [ 'set -euo pipefail', 'ccache --clear' if no_cache else '', 'ccache --zero-stats', 'ccache --show-config', 'mkdir -p artifacts', 'dpkg -l >> artifacts/packages.txt', *checkout_scripts('linux', scripts_refspec), 'set +e', 'pip install -q -r ./mlir/python/requirements.txt', ] if check_diff: commands.extend([ '$${SRC}/scripts/premerge_checks.py --check-clang-format ' f'--projects="{projects}" --log-level={log_level}', ]) else: commands.extend([ f'$${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}' ]) commands.extend([ 'EXIT_STATUS=$$?', 'echo "--- ccache stats"', 'ccache --print-stats', 'exit $$EXIT_STATUS', ]) linux_buld_step = { 'label': ':linux: x64 debian', 'key': 'linux', 'commands': commands, 'artifact_paths': ['artifacts/**/*', '*_result.json', 'build/test-results.xml'], 'agents': linux_agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost {'exit_status': 255, 'limit': 2}, # Forced agent shutdown ]}, } return [linux_buld_step] def bazel(modified_files: Set[str], force: bool = False) -> List: if os.getenv('ph_skip_bazel') is not None: logging.info('bazel build is skipped as "ph_skip_bazel" is set') return [] updated_build = any(s.startswith('utils/bazel/') for s in modified_files) if not force: if updated_build: logging.info('files in utils/bazel/ modified, will trigger bazel build') else: user_projects = os.getenv('ph_user_project_slugs', '').split(',') if 'bazel_build' not in user_projects: logging.info('bazel build is skipped as "bazel_build" is not listed in user projects and no files in ' 'utils/bazel/ are modified') return [] agents = {'queue': 'llvm-bazel-premerge'} t = os.getenv('ph_bazel_agents') if t is not None: agents = json.loads(t) return [{ 'label': ':bazel: bazel', 'key': 'bazel', 'commands': [ 'set -eu', 'cd utils/bazel', 'bazel query //... + @llvm-project//... | xargs bazel test --config=ci', ], 'agents': agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost {'exit_status': 255, 'limit': 2}, # Forced agent shutdown ]}, }] def generic_windows(projects: str) -> List: if os.getenv('ph_skip_windows') is not None: return [] scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') clear_sccache = 'powershell -command "sccache --stop-server; echo $$env:SCCACHE_DIR; ' \ 'Remove-Item -Recurse -Force -ErrorAction Ignore $$env:SCCACHE_DIR; ' \ 'sccache --start-server"' win_agents = {'queue': 'windows'} t = os.getenv('ph_windows_agents') if t is not None: win_agents = json.loads(t) windows_buld_step = { 'label': ':windows: x64 windows', 'key': 'windows', 'commands': [ clear_sccache if no_cache else '', 'sccache --zero-stats', 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64', *checkout_scripts('windows', scripts_refspec), 'pip install -q -r ./mlir/python/requirements.txt', 'powershell -command "' f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; ' '$$exit=$$?;' 'sccache --show-stats;' 'if ($$exit) {' ' echo success;' ' exit 0; } ' 'else {' ' echo failure;' ' exit 1;' '}"', ], 'artifact_paths': ['artifacts/**/*', '*_result.json', 'build/test-results.xml'], 'agents': win_agents, 'timeout_in_minutes': 150, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost {'exit_status': 255, 'limit': 2}, # Forced agent shutdown ]}, } return [windows_buld_step] def from_shell_output(command, **kwargs) -> []: """ Executes shell command and parses stdout as multidoc yaml file, see https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. :param command: command, may include env variables :return: all 'steps' that defined in the result ("env" section is ignored). Non-zero exit code and malformed YAML produces empty result. """ path = os.path.expandvars(command) logging.debug(f'invoking "{path}"') out = io.BytesIO() err = io.BytesIO() rc = watch_shell(out.write, err.write, path, **kwargs) logging.debug(f'exit code: {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"') steps = [] if rc != 0: logging.error( f'{path} returned non-zero code {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"') return steps try: for part in yaml.safe_load_all(out.getvalue()): part.setdefault('steps', []) steps.extend(part['steps']) except yaml.YAMLError as e: logging.error(f'''"{path}" produced malformed YAML, exception: {e} stdout: >>>{out.getvalue().decode()}>>>''') return steps def checkout_scripts(target_os: str, scripts_refspec: str) -> []: if target_os == 'windows': return [ 'set SRC=%BUILDKITE_BUILD_PATH%/llvm-premerge-checks', 'rm -rf %SRC%', 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git %SRC%', 'cd %SRC%', f'git fetch origin "{scripts_refspec}":x', 'git checkout x', 'echo llvm-premerge-checks commit:', 'git rev-parse HEAD', 'pip install -q -r %SRC%/scripts/requirements.txt', 'cd %BUILDKITE_BUILD_CHECKOUT_PATH%', ] return [ 'export SRC=$${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', 'rm -rf $${SRC}', 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "$${SRC}"', 'cd $${SRC}', f'git fetch origin "{scripts_refspec}":x', 'git checkout x', 'echo "llvm-premerge-checks commit"', 'git rev-parse HEAD', 'pip install -q -r $${SRC}/scripts/requirements.txt', 'cd "$$BUILDKITE_BUILD_CHECKOUT_PATH"', ] def extend_dict(target: Dict, extra: Dict) -> Dict: if not target: return extra for k in extra: if k in target: continue target[k] = extra[k] return target def extend_steps_env(steps: List[Dict], env: Dict): for s in steps: if 'commands' in s: s['env'] = extend_dict(s.get('env'), env) if 'build' in s: s['build']['env'] = extend_dict(s['build'].get('env'), env)