sync all branches for llvm-project fork

There are multiple scenarios when script expects to see same branches
and commits. E.g. when upstream commits to release branch trigger
the build.

Reported by Louis Dionne.

Pipfile

@@ -14,6 +14,7 @@ unidiff = "==0.6.0"
 python-benedict = "==0.23.2"
 GitPython = "==3.1.14"
 types-pyyaml = "*"
+pytest = "*"
 
 [dev-packages]
 

scripts/git_utils.py

@@ -0,0 +1,51 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import git
+import os
+import logging
+
+"""URL of upstream LLVM repository."""
+LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
+FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
+
+def initLlvmFork(path: str) -> git.Repo:
+  if not os.path.isdir(path):
+      logging.info(f'{path} does not exist, cloning repository...')
+      git.Repo.clone_from(FORK_REMOTE_URL, path)
+  repo = git.Repo(path)
+  repo.remote('origin').set_url(FORK_REMOTE_URL)
+  if 'upstream' not in repo.remotes:
+    repo.create_remote('upstream', url=LLVM_GITHUB_URL)
+  else:
+    repo.remote('upstream').set_url(LLVM_GITHUB_URL)
+  repo.git.fetch('--all')
+  repo.git.clean('-ffxdq')
+  repo.git.reset('--hard')
+  repo.heads.main.checkout()
+  repo.git.pull('origin', 'main')
+  return repo
+
+def syncLlvmFork(repo: git.Repo):
+  repo.remote('origin').set_url(FORK_REMOTE_URL)
+  if 'upstream' not in repo.remotes:
+    repo.create_remote('upstream', url=LLVM_GITHUB_URL)
+  repo.remote('upstream').set_url(LLVM_GITHUB_URL)
+  syncRemotes(repo, 'upstream', 'origin')
+  pass
+
+def syncRemotes(repo: git.Repo, fromRemote, toRemote):
+  """sync one remote from another"""
+  repo.remotes[fromRemote].fetch()
+  repo.git.push(toRemote, '-f', f'refs/remotes/{fromRemote}/*:refs/heads/*')

scripts/git_utils_test.py

@@ -0,0 +1,109 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import git
+import os
+import scripts.git_utils as git_utils
+
+def assertForkIsSynced(upstreamPath, forkPath):
+    upstream = git.Repo(path=upstreamPath)
+    fork = git.Repo(path=forkPath)
+    forkBranches = {}
+    for b in fork.branches:
+      forkBranches[b.name] = b
+    for b in upstream.branches:
+      assert b.name in forkBranches
+      assert b.commit.hexsha == forkBranches[b.name].commit.hexsha, f'branch {b.name} head'
+
+def forkIsSynced(upstreamPath, forkPath) -> bool:
+    upstream = git.Repo(path=upstreamPath)
+    fork = git.Repo(path=forkPath)
+    forkBranches = {}
+    for b in fork.branches:
+      forkBranches[b.name] = b
+    for b in upstream.branches:
+      if b.name not in forkBranches:
+        return False
+      if b.commit.hexsha != forkBranches[b.name].commit.hexsha:
+        return False
+    return True
+
+def add_simple_commit(remote, name: str):
+    with open(os.path.join(remote.working_tree_dir, name), 'wt') as f:
+      f.write('first line\n')
+    remote.index.add([os.path.join(remote.working_tree_dir, name)])
+    remote.index.commit(name)
+
+def test_sync_branches(tmp_path):
+    upstreamRemote = os.path.join(tmp_path, 'upstreamBare')
+    forkRemote = os.path.join(tmp_path, 'forkBare')
+    git.Repo.init(path=upstreamRemote, bare=True)
+    git.Repo.init(path=forkRemote, bare=True)
+    upstreamPath = os.path.join(tmp_path, 'upstream')
+    forkPath = os.path.join(tmp_path, 'fork')
+    upstream = git.Repo.clone_from(url=upstreamRemote, to_path=upstreamPath)
+    add_simple_commit(upstream, '1')
+    upstream.git.push('origin', 'main')
+    fork = git.Repo.clone_from(url=forkRemote, to_path=forkPath)
+    fork.create_remote('upstream', url=upstreamRemote)
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    fork.remotes.upstream.fetch()
+    fork.create_head('main', fork.remotes.upstream.refs.main)
+    fork.heads.main.checkout()
+
+    # Sync init commit.
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    assertForkIsSynced(upstreamRemote, forkRemote)
+
+    # Add new change upstream.
+    add_simple_commit(upstream, '2')
+    upstream.git.push('--all')
+
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    assertForkIsSynced(upstreamRemote, forkRemote)
+
+    # Add new branch.
+    upstream.create_head('branch1')
+    upstream.heads['branch1'].checkout()
+    add_simple_commit(upstream, '3')
+    upstream.git.push('--all')
+
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    assertForkIsSynced(upstreamRemote, forkRemote)
+
+    # Add another branch commit.
+    add_simple_commit(upstream, '4')
+    upstream.git.push('--all')
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    assertForkIsSynced(upstreamRemote, forkRemote)
+
+    # Discard changes in fork.
+    fork.remotes.origin.pull()
+    fork.heads.main.checkout()
+    add_simple_commit(fork, '5')
+    fork.remotes.origin.push()
+
+    upstream.remotes.origin.pull('main')
+    upstream.heads.main.checkout()
+    add_simple_commit(upstream, '6')
+    upstream.remotes.origin.push()
+
+    assert not forkIsSynced(upstreamRemote, forkRemote)
+    assert os.path.isfile(os.path.join(fork.working_tree_dir, '5'))
+    git_utils.syncRemotes(fork, 'upstream', 'origin')
+    assertForkIsSynced(upstreamRemote, forkRemote)
+    fork.git.pull('origin', 'main')
+    fork.heads.main.checkout()
+    assert not os.path.isfile(os.path.join(fork.working_tree_dir, '5'))
+    assert os.path.isfile(os.path.join(fork.working_tree_dir, '6'))

scripts/pipeline_main.py

@@ -15,13 +15,13 @@
 
 # Script runs in checked out llvm-project directory.
 
-import os
-from typing import Dict
-from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env, bazel
-from sync_fork import sync_fork
-import git
-import yaml
 from choose_projects import ChooseProjects
+from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env, bazel
+from typing import Dict
+import git
+import git_utils
+import os
+import yaml
 
 steps_generators = [
     '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-snapshot.sh',
@@ -34,7 +34,8 @@ if __name__ == '__main__':
     notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(',')))
     # Syncing LLVM fork so any pipelines started from upstream llvm-project
     # but then triggered a build on fork will observe the commit.
-    sync_fork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH', ''), 'llvm-project-fork'), [os.getenv('BUILDKITE_BRANCH'), 'main'])
+    repo = git_utils.initLlvmFork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH', ''), 'llvm-project-fork'))
+    git_utils.syncRemotes(repo, 'upstream', 'origin')
     steps = []
 
     env: Dict[str, str] = {}

scripts/summary.py

@@ -19,7 +19,7 @@ import os
 
 from phabtalk.phabtalk import PhabTalk
 from buildkite_utils import format_url, BuildkiteApi, strip_emojis
-import test_results_report
+import scripts.xunit_utils as xunit_utils
 from benedict import benedict
 
 
@@ -50,7 +50,7 @@ def process_unit_test_reports(bk: BuildkiteApi, build: benedict, prefix: str) ->
                 continue
             content = bk.get(a.get('download_url')).content
             ctx = strip_emojis(prefix + ' ' + job.get('name', build.get('pipeline.name')))
-            failed_tests.extend(test_results_report.parse_failures(content, ctx))
+            failed_tests.extend(xunit_utils.parse_failures(content, ctx))
     return failed_tests
 
 

scripts/sync_fork.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 Google LLC
-#
-# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://llvm.org/LICENSE.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import logging
-import os
-from typing import List
-import backoff
-import git
-
-"""URL of upstream LLVM repository."""
-LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
-FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
-
-
-@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
-def sync_fork(path: str, branches: List[str]):
-    if not os.path.isdir(path):
-        logging.info(f'{path} does not exist, cloning repository...')
-        repo = git.Repo.clone_from(FORK_REMOTE_URL, path)
-    else:
-        logging.info('repository exist, will reuse')
-        repo = git.Repo(path)  # type: git.Repo
-        repo.remote('origin').set_url(FORK_REMOTE_URL)
-    os.chdir(path)
-    logging.info(f'working dir {os.getcwd()}')
-    logging.info(f'Syncing origin and upstream branches {branches}')
-    if 'upstream' not in repo.remotes:
-        repo.create_remote('upstream', url=LLVM_GITHUB_URL)
-    repo.git.fetch('--all')
-    for b in branches:
-        logging.info(f'syncing branch {b}')
-        if find_commit(repo, b) is None:
-            logging.info(f'new head {b}')
-            repo.create_head(b)
-        h = repo.heads[b]
-        h.checkout()
-        repo.git.reset('--hard', f'upstream/{b}')
-        repo.git.clean('-ffxdq')
-        repo.git.push('origin', h)
-    repo.git.push('origin', '--tags')
-
-
-def find_commit(repo, rev):
-    try:
-        return repo.commit(rev)
-    except:
-        return None
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Sync LLVM fork with origina rpo.')
-    parser.add_argument('--path', type=str, help='repository path', required=True)
-    parser.add_argument('--branch', nargs='+', help='branch to sync (specify multiple to sync many branches)',
-                        required=True)
-    parser.add_argument('--log-level', type=str, default='INFO')
-    args = parser.parse_args()
-    logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s')
-    sync_fork(args.path, args.branch)