diff --git a/Pipfile b/Pipfile index 8770ad0..db14950 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ unidiff = "==0.6.0" python-benedict = "==0.23.2" GitPython = "==3.1.14" types-pyyaml = "*" +pytest = "*" [dev-packages] diff --git a/scripts/git_utils.py b/scripts/git_utils.py new file mode 100644 index 0000000..a0aece6 --- /dev/null +++ b/scripts/git_utils.py @@ -0,0 +1,51 @@ +# Copyright 2022 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import git +import os +import logging + +"""URL of upstream LLVM repository.""" +LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' +FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' + +def initLlvmFork(path: str) -> git.Repo: + if not os.path.isdir(path): + logging.info(f'{path} does not exist, cloning repository...') + git.Repo.clone_from(FORK_REMOTE_URL, path) + repo = git.Repo(path) + repo.remote('origin').set_url(FORK_REMOTE_URL) + if 'upstream' not in repo.remotes: + repo.create_remote('upstream', url=LLVM_GITHUB_URL) + else: + repo.remote('upstream').set_url(LLVM_GITHUB_URL) + repo.git.fetch('--all') + repo.git.clean('-ffxdq') + repo.git.reset('--hard') + repo.heads.main.checkout() + repo.git.pull('origin', 'main') + return repo + +def syncLlvmFork(repo: git.Repo): + repo.remote('origin').set_url(FORK_REMOTE_URL) + if 'upstream' not in repo.remotes: + repo.create_remote('upstream', url=LLVM_GITHUB_URL) + repo.remote('upstream').set_url(LLVM_GITHUB_URL) + syncRemotes(repo, 'upstream', 'origin') + pass + +def syncRemotes(repo: git.Repo, fromRemote, toRemote): + """sync one remote from another""" + repo.remotes[fromRemote].fetch() + repo.git.push(toRemote, '-f', f'refs/remotes/{fromRemote}/*:refs/heads/*') diff --git a/scripts/git_utils_test.py b/scripts/git_utils_test.py new file mode 100644 index 0000000..c629462 --- /dev/null +++ b/scripts/git_utils_test.py @@ -0,0 +1,109 @@ +# Copyright 2022 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import git +import os +import scripts.git_utils as git_utils + +def assertForkIsSynced(upstreamPath, forkPath): + upstream = git.Repo(path=upstreamPath) + fork = git.Repo(path=forkPath) + forkBranches = {} + for b in fork.branches: + forkBranches[b.name] = b + for b in upstream.branches: + assert b.name in forkBranches + assert b.commit.hexsha == forkBranches[b.name].commit.hexsha, f'branch {b.name} head' + +def forkIsSynced(upstreamPath, forkPath) -> bool: + upstream = git.Repo(path=upstreamPath) + fork = git.Repo(path=forkPath) + forkBranches = {} + for b in fork.branches: + forkBranches[b.name] = b + for b in upstream.branches: + if b.name not in forkBranches: + return False + if b.commit.hexsha != forkBranches[b.name].commit.hexsha: + return False + return True + +def add_simple_commit(remote, name: str): + with open(os.path.join(remote.working_tree_dir, name), 'wt') as f: + f.write('first line\n') + remote.index.add([os.path.join(remote.working_tree_dir, name)]) + remote.index.commit(name) + +def test_sync_branches(tmp_path): + upstreamRemote = os.path.join(tmp_path, 'upstreamBare') + forkRemote = os.path.join(tmp_path, 'forkBare') + git.Repo.init(path=upstreamRemote, bare=True) + git.Repo.init(path=forkRemote, bare=True) + upstreamPath = os.path.join(tmp_path, 'upstream') + forkPath = os.path.join(tmp_path, 'fork') + upstream = git.Repo.clone_from(url=upstreamRemote, to_path=upstreamPath) + add_simple_commit(upstream, '1') + upstream.git.push('origin', 'main') + fork = git.Repo.clone_from(url=forkRemote, to_path=forkPath) + fork.create_remote('upstream', url=upstreamRemote) + git_utils.syncRemotes(fork, 'upstream', 'origin') + fork.remotes.upstream.fetch() + fork.create_head('main', fork.remotes.upstream.refs.main) + fork.heads.main.checkout() + + # Sync init commit. + git_utils.syncRemotes(fork, 'upstream', 'origin') + assertForkIsSynced(upstreamRemote, forkRemote) + + # Add new change upstream. + add_simple_commit(upstream, '2') + upstream.git.push('--all') + + git_utils.syncRemotes(fork, 'upstream', 'origin') + assertForkIsSynced(upstreamRemote, forkRemote) + + # Add new branch. + upstream.create_head('branch1') + upstream.heads['branch1'].checkout() + add_simple_commit(upstream, '3') + upstream.git.push('--all') + + git_utils.syncRemotes(fork, 'upstream', 'origin') + assertForkIsSynced(upstreamRemote, forkRemote) + + # Add another branch commit. + add_simple_commit(upstream, '4') + upstream.git.push('--all') + git_utils.syncRemotes(fork, 'upstream', 'origin') + assertForkIsSynced(upstreamRemote, forkRemote) + + # Discard changes in fork. + fork.remotes.origin.pull() + fork.heads.main.checkout() + add_simple_commit(fork, '5') + fork.remotes.origin.push() + + upstream.remotes.origin.pull('main') + upstream.heads.main.checkout() + add_simple_commit(upstream, '6') + upstream.remotes.origin.push() + + assert not forkIsSynced(upstreamRemote, forkRemote) + assert os.path.isfile(os.path.join(fork.working_tree_dir, '5')) + git_utils.syncRemotes(fork, 'upstream', 'origin') + assertForkIsSynced(upstreamRemote, forkRemote) + fork.git.pull('origin', 'main') + fork.heads.main.checkout() + assert not os.path.isfile(os.path.join(fork.working_tree_dir, '5')) + assert os.path.isfile(os.path.join(fork.working_tree_dir, '6')) diff --git a/scripts/pipeline_main.py b/scripts/pipeline_main.py index 7585792..3c378c8 100755 --- a/scripts/pipeline_main.py +++ b/scripts/pipeline_main.py @@ -15,13 +15,13 @@ # Script runs in checked out llvm-project directory. -import os -from typing import Dict -from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env, bazel -from sync_fork import sync_fork -import git -import yaml from choose_projects import ChooseProjects +from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env, bazel +from typing import Dict +import git +import git_utils +import os +import yaml steps_generators = [ '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-snapshot.sh', @@ -34,7 +34,8 @@ if __name__ == '__main__': notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(','))) # Syncing LLVM fork so any pipelines started from upstream llvm-project # but then triggered a build on fork will observe the commit. - sync_fork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH', ''), 'llvm-project-fork'), [os.getenv('BUILDKITE_BRANCH'), 'main']) + repo = git_utils.initLlvmFork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH', ''), 'llvm-project-fork')) + git_utils.syncRemotes(repo, 'upstream', 'origin') steps = [] env: Dict[str, str] = {} diff --git a/scripts/summary.py b/scripts/summary.py index 8524cbf..8c51f7d 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -19,7 +19,7 @@ import os from phabtalk.phabtalk import PhabTalk from buildkite_utils import format_url, BuildkiteApi, strip_emojis -import test_results_report +import scripts.xunit_utils as xunit_utils from benedict import benedict @@ -50,7 +50,7 @@ def process_unit_test_reports(bk: BuildkiteApi, build: benedict, prefix: str) -> continue content = bk.get(a.get('download_url')).content ctx = strip_emojis(prefix + ' ' + job.get('name', build.get('pipeline.name'))) - failed_tests.extend(test_results_report.parse_failures(content, ctx)) + failed_tests.extend(xunit_utils.parse_failures(content, ctx)) return failed_tests diff --git a/scripts/sync_fork.py b/scripts/sync_fork.py deleted file mode 100755 index 1b0448d..0000000 --- a/scripts/sync_fork.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -from typing import List -import backoff -import git - -"""URL of upstream LLVM repository.""" -LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' -FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' - - -@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) -def sync_fork(path: str, branches: List[str]): - if not os.path.isdir(path): - logging.info(f'{path} does not exist, cloning repository...') - repo = git.Repo.clone_from(FORK_REMOTE_URL, path) - else: - logging.info('repository exist, will reuse') - repo = git.Repo(path) # type: git.Repo - repo.remote('origin').set_url(FORK_REMOTE_URL) - os.chdir(path) - logging.info(f'working dir {os.getcwd()}') - logging.info(f'Syncing origin and upstream branches {branches}') - if 'upstream' not in repo.remotes: - repo.create_remote('upstream', url=LLVM_GITHUB_URL) - repo.git.fetch('--all') - for b in branches: - logging.info(f'syncing branch {b}') - if find_commit(repo, b) is None: - logging.info(f'new head {b}') - repo.create_head(b) - h = repo.heads[b] - h.checkout() - repo.git.reset('--hard', f'upstream/{b}') - repo.git.clean('-ffxdq') - repo.git.push('origin', h) - repo.git.push('origin', '--tags') - - -def find_commit(repo, rev): - try: - return repo.commit(rev) - except: - return None - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Sync LLVM fork with origina rpo.') - parser.add_argument('--path', type=str, help='repository path', required=True) - parser.add_argument('--branch', nargs='+', help='branch to sync (specify multiple to sync many branches)', - required=True) - parser.add_argument('--log-level', type=str, default='INFO') - args = parser.parse_args() - logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') - sync_fork(args.path, args.branch) diff --git a/scripts/test_results_report.py b/scripts/xunit_utils.py similarity index 100% rename from scripts/test_results_report.py rename to scripts/xunit_utils.py