From 49be688a27c7eb867446a1ad12aa7b5cd1ac2122 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 11 Aug 2021 16:20:03 +0200 Subject: [PATCH] script to sync fork llvm-premerge-tests/llvm-project - uniformly append env variables - resolve current HEAD and pass it as BUILDKITE_COMMIT to script generators. Add it as ph_commit_sha. --- scripts/pipeline_main.py | 23 +++++++++++- scripts/pipeline_premerge.py | 18 +++++++-- scripts/steps.py | 24 ++++++++++-- scripts/sync_fork.py | 71 ++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 8 deletions(-) create mode 100755 scripts/sync_fork.py diff --git a/scripts/pipeline_main.py b/scripts/pipeline_main.py index ee7bf2a..cc8d462 100755 --- a/scripts/pipeline_main.py +++ b/scripts/pipeline_main.py @@ -13,9 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Script runs in checked out llvm-project directory. + import os -from steps import generic_linux, generic_windows, from_shell_output +from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env +from sync_fork import sync_fork +import git import yaml steps_generators = [ @@ -27,7 +31,18 @@ if __name__ == '__main__': no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(','))) + # Syncing LLVM fork so any pipelines started from upstream llvm-project# + # but then triggered a build on fork will observe the commit. + sync_fork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH'), 'llvm-project-fork'), [os.getenv('BUILDKITE_BRANCH'), 'main']) steps = [] + + env = {} + for e in os.environ: + if e.startswith('ph_'): + env[e] = os.getenv(e) + repo = git.Repo('.') + env['ph_commit_sha'] = repo.head.commit.hexsha + steps.extend(generic_linux( os.getenv('ph_projects', 'llvm;clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly;flang'), False)) @@ -39,10 +54,14 @@ if __name__ == '__main__': os.getenv('ph_projects', 'llvm;clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly;flang'))) if os.getenv('ph_skip_generated') is None: + e = os.environ.copy() + # BUILDKITE_COMMIT might be an alias, e.g. "HEAD". Resolve it to make the build hermetic. + e["BUILDKITE_COMMIT"] = repo.head.commit.hexsha for gen in steps_generators: - steps.extend(from_shell_output(gen)) + steps.extend(from_shell_output(gen, env=e)) notify = [] for e in notify_emails: notify.append({'email': e}) + extend_steps_env(steps, env) print(yaml.dump({'steps': steps, 'notify': notify})) diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index 5d09409..262a17a 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Script runs in checked out llvm-project directory. + import logging import os from buildkite_utils import annotate, feedback_url, set_metadata from choose_projects import ChooseProjects import git -from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel +from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel, extend_steps_env import yaml steps_generators = [ @@ -42,8 +44,14 @@ if __name__ == '__main__': set_metadata('ph_buildable_diff', os.getenv("ph_buildable_diff")) set_metadata('ph_buildable_revision', os.getenv('ph_buildable_revision')) set_metadata('ph_build_id', os.getenv("ph_build_id")) - # List all affected projects. + + env = {} + for e in os.environ: + if e.startswith('ph_'): + env[e] = os.getenv(e) repo = git.Repo('.') + env['ph_commit_sha'] = repo.head.commit.hexsha + # List all affected projects. patch = repo.git.diff("HEAD~1") cp = ChooseProjects('.') modified_files = cp.get_changed_files(patch) @@ -71,8 +79,11 @@ if __name__ == '__main__': steps.extend(generic_windows(';'.join(sorted(windows_projects)))) # Add custom checks. if os.getenv('ph_skip_generated') is None: + e = os.environ.copy() + # BUILDKITE_COMMIT might be an alias, e.g. "HEAD". Resolve it to make the build hermetic. + e["BUILDKITE_COMMIT"] = repo.head.commit.hexsha for gen in steps_generators: - steps.extend(from_shell_output(gen)) + steps.extend(from_shell_output(gen, env=e)) steps.extend(bazel(modified_files)) if phid is None: @@ -95,4 +106,5 @@ if __name__ == '__main__': } steps.append(report_step) + extend_steps_env(steps, env) print(yaml.dump({'steps': steps})) diff --git a/scripts/steps.py b/scripts/steps.py index a3d00d3..1b559ea 100644 --- a/scripts/steps.py +++ b/scripts/steps.py @@ -17,7 +17,7 @@ import io import json import logging import os -from typing import List, Set +from typing import List, Set, Dict from exec_utils import watch_shell import yaml @@ -153,7 +153,7 @@ def generic_windows(projects: str) -> List: return [windows_buld_step] -def from_shell_output(command) -> []: +def from_shell_output(command, **kwargs) -> []: """ Executes shell command and parses stdout as multidoc yaml file, see https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. @@ -165,7 +165,7 @@ def from_shell_output(command) -> []: logging.debug(f'invoking "{path}"') out = io.BytesIO() err = io.BytesIO() - rc = watch_shell(out.write, err.write, path) + rc = watch_shell(out.write, err.write, path, **kwargs) logging.debug(f'exit code: {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"') steps = [] if rc != 0: @@ -210,3 +210,21 @@ def checkout_scripts(target_os: str, scripts_refspec: str) -> []: 'pip install -q -r $${SRC}/scripts/requirements.txt', 'cd "$$BUILDKITE_BUILD_CHECKOUT_PATH"', ] + + +def extend_dict(target: Dict, extra: Dict) -> Dict: + if not target: + return extra + for k in extra: + if k in target: + continue + target[k] = extra[k] + return target + + +def extend_steps_env(steps: List[Dict], env: Dict): + for s in steps: + if 'commands' in s: + s['env'] = extend_dict(s.get('env'), env) + if 'build' in s: + s['build']['env'] = extend_dict(s['build'].get('env'), env) diff --git a/scripts/sync_fork.py b/scripts/sync_fork.py new file mode 100755 index 0000000..1b0448d --- /dev/null +++ b/scripts/sync_fork.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright 2021 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +from typing import List +import backoff +import git + +"""URL of upstream LLVM repository.""" +LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' +FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' + + +@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) +def sync_fork(path: str, branches: List[str]): + if not os.path.isdir(path): + logging.info(f'{path} does not exist, cloning repository...') + repo = git.Repo.clone_from(FORK_REMOTE_URL, path) + else: + logging.info('repository exist, will reuse') + repo = git.Repo(path) # type: git.Repo + repo.remote('origin').set_url(FORK_REMOTE_URL) + os.chdir(path) + logging.info(f'working dir {os.getcwd()}') + logging.info(f'Syncing origin and upstream branches {branches}') + if 'upstream' not in repo.remotes: + repo.create_remote('upstream', url=LLVM_GITHUB_URL) + repo.git.fetch('--all') + for b in branches: + logging.info(f'syncing branch {b}') + if find_commit(repo, b) is None: + logging.info(f'new head {b}') + repo.create_head(b) + h = repo.heads[b] + h.checkout() + repo.git.reset('--hard', f'upstream/{b}') + repo.git.clean('-ffxdq') + repo.git.push('origin', h) + repo.git.push('origin', '--tags') + + +def find_commit(repo, rev): + try: + return repo.commit(rev) + except: + return None + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Sync LLVM fork with origina rpo.') + parser.add_argument('--path', type=str, help='repository path', required=True) + parser.add_argument('--branch', nargs='+', help='branch to sync (specify multiple to sync many branches)', + required=True) + parser.add_argument('--log-level', type=str, default='INFO') + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') + sync_fork(args.path, args.branch)