216 lines
9.3 KiB
Python
216 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright 2020 Google LLC
|
|
#
|
|
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# https://llvm.org/LICENSE.txt
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
from typing import Optional, Union
|
|
import git
|
|
import logging
|
|
from phab_wrapper import PhabWrapper, Revision
|
|
import subprocess
|
|
import github
|
|
import json
|
|
import sys
|
|
|
|
# TODO: move to config file
|
|
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
|
|
MY_GITHUB_URL = 'ssh://git@github.com/christiankuehnel/llvm-project'
|
|
MY_REPO = 'christiankuehnel/llvm-project'
|
|
|
|
|
|
_LOGGER = logging.getLogger()
|
|
|
|
|
|
class ApplyPatchException(Exception):
|
|
"""A patch could not be applied."""
|
|
pass
|
|
|
|
|
|
class Phab2Github:
|
|
|
|
def __init__(self, workdir: str):
|
|
self.workdir = workdir
|
|
self.llvm_dir = os.path.join(self.workdir, 'llvm-project')
|
|
self.repo = None # type: Optional[git.Repo]
|
|
self.phab_wrapper = PhabWrapper()
|
|
self.github = self._create_github()
|
|
self.github_repo = self.github.get_repo(MY_REPO) # type: github.Repository
|
|
|
|
def sync(self):
|
|
"""Sync Phabricator to Github."""
|
|
_LOGGER.info('Starting sync...')
|
|
self._refresh_main()
|
|
self._delete_phab_branches()
|
|
revisions = self.phab_wrapper.get_revisions()
|
|
pull_requests = {p.title: p for p in self.github_repo.get_pulls(state='open')}
|
|
for revision in revisions:
|
|
self.create_branches_for_revision(revision)
|
|
if self._has_branch(revision):
|
|
if self._branches_identical(revision.branch_name, 'origin/{}'.format(revision.branch_name)):
|
|
_LOGGER.info('Branch {} is identical to upstream. Not pushing.'.format(revision.branch_name))
|
|
else:
|
|
_LOGGER.info('Pushing branch {} to github...'.format(revision.branch_name))
|
|
# TODO: do we sill need to force-push?
|
|
self.repo.git.push('--force', 'origin', revision.branch_name)
|
|
if revision.pr_title in pull_requests:
|
|
_LOGGER.info('Pull request already exists: {}'.format(pull_requests[revision.pr_title].html_url))
|
|
else:
|
|
_LOGGER.info('Creating pull-request for branch {}...'.format(revision.branch_name))
|
|
pr = self.github_repo.create_pull(title=revision.pr_title,
|
|
body=revision.pr_summary,
|
|
head=revision.branch_name,
|
|
base='main')
|
|
_LOGGER.info(pr.html_url)
|
|
_LOGGER.info('Sync completed.')
|
|
|
|
def _refresh_main(self):
|
|
"""Clone/update local git repo."""
|
|
if not os.path.exists(self.workdir):
|
|
os.mkdir(self.workdir)
|
|
if os.path.exists(self.llvm_dir):
|
|
# TODO: in case of errors: delete and clone
|
|
_LOGGER.info('pulling origin and upstream...')
|
|
self.repo = git.Repo(self.llvm_dir)
|
|
self.repo.git.fetch('--all')
|
|
self.repo.git.checkout('main')
|
|
self.repo.git.pull('upstream', 'main')
|
|
self.repo.git.push('origin', 'main')
|
|
else:
|
|
_LOGGER.info('cloning repository...')
|
|
git.Repo.clone_from(MY_GITHUB_URL, self.llvm_dir)
|
|
self.repo = git.Repo(self.llvm_dir)
|
|
self.repo.create_remote('upstream', url=LLVM_GITHUB_URL)
|
|
self.repo.remotes.upstream.fetch()
|
|
_LOGGER.info('refresh of main branch completed')
|
|
|
|
def create_branches_for_revision(self, revision: Revision):
|
|
"""Create branches for a Revision and it's Diffs.
|
|
|
|
Apply changes from Phabricator to these branches.
|
|
"""
|
|
name = revision.branch_name
|
|
# TODO: only look at diffs that were not yet applied
|
|
# TODO: can diffs be modified on Phabricator and keep their ID?
|
|
for diff in revision.sorted_diffs:
|
|
if self._has_branch(diff):
|
|
continue
|
|
self.create_branch_for_diff(diff)
|
|
patch = self.phab_wrapper.get_raw_patch(diff)
|
|
try:
|
|
self.apply_patch(diff, patch)
|
|
except ApplyPatchException as e:
|
|
# TODO: retry on main if this fails
|
|
_LOGGER.error('Could not apply patch for Diff {}. Deleting branch'.format(diff.id))
|
|
_LOGGER.exception(e)
|
|
self.repo.heads['main'].checkout()
|
|
self.repo.delete_head(diff.branch_name)
|
|
|
|
diffs = [d for d in revision.sorted_diffs if self._has_branch(d)]
|
|
if len(diffs) == 0:
|
|
# TODO: handle error
|
|
_LOGGER.error('Could not create branch for Revision D{}'.format(revision.id))
|
|
return
|
|
new_branch = self.repo.create_head(revision.branch_name, diffs[0].branch_name)
|
|
new_branch.checkout()
|
|
|
|
for diff in diffs[1:]:
|
|
_LOGGER.info('Applying Diff {} onto branch {}'.format(diff.branch_name, revision.branch_name))
|
|
patch = self._create_patch(revision.branch_name, diff.branch_name)
|
|
if len(patch) == 0:
|
|
_LOGGER.warning('Diff {} is identical to last one.'.format(diff.id))
|
|
else:
|
|
try:
|
|
self.apply_patch(diff, patch)
|
|
except ApplyPatchException:
|
|
_LOGGER.error('Applying patch failed, but should not:')
|
|
_LOGGER.error(patch)
|
|
raise
|
|
|
|
_LOGGER.info('Created branch for Revision D{}'.format(revision.id))
|
|
|
|
def create_branch_for_diff(self, diff: "Diff"):
|
|
"""Create a branch for diff."""
|
|
base_hash = diff.base_hash
|
|
if base_hash is None:
|
|
base_hash = 'upstream/main'
|
|
_LOGGER.info('creating branch {} based on {}...'.format(diff.branch_name, base_hash))
|
|
try:
|
|
new_branch = self.repo.create_head(diff.branch_name, base_hash)
|
|
except ValueError:
|
|
# commit hash not found, try again with main
|
|
_LOGGER.warning('commit hash {} not found in upstream repository. '
|
|
'Trying main instead...'.format(diff.branch_name, base_hash))
|
|
base_hash = 'upstream/main'
|
|
new_branch = self.repo.create_head(diff.branch_name, base_hash)
|
|
self.repo.head.reference = new_branch
|
|
self.repo.head.reset(index=True, working_tree=True)
|
|
|
|
def apply_patch(self, diff: "Diff", raw_patch: str):
|
|
"""Apply a patch to the working copy."""
|
|
# TODO: impersonate the original author of the patch
|
|
proc = subprocess.run('git apply --ignore-whitespace --whitespace=fix -', input=raw_patch, shell=True,
|
|
text=True, cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if proc.returncode != 0:
|
|
raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
|
|
self.repo.git.add('-A')
|
|
self.repo.index.commit(message='applying Diff {} for Revision D{}\n\n Diff: {}'.format(
|
|
diff.id, diff.revision.id, diff.id))
|
|
|
|
@staticmethod
|
|
def _create_github() -> github.Github:
|
|
"""Create instance of Github client.
|
|
|
|
Reads access token from a file.
|
|
"""
|
|
with open(os.path.expanduser('~/.llvm-premerge-checks/github-token.json')) as json_file:
|
|
token = json.load(json_file)['token']
|
|
return github.Github(token)
|
|
|
|
def _has_branch(self, item: Union["Diff", "Revision"]) -> bool:
|
|
"""Check if the Diff/Revision has a local branch."""
|
|
return item.branch_name in self.repo.heads
|
|
|
|
def _delete_phab_branches(self):
|
|
"""Delete all branches sarting with 'phab-'."""
|
|
_LOGGER.info('Deleting local Phabricator-relates branches...')
|
|
self.repo.git.checkout('main')
|
|
for branch in [b for b in self.repo.heads if b.name.startswith('phab-')]:
|
|
_LOGGER.info('Deleding branch {}'.format(branch))
|
|
self.repo.git.branch('-D', branch.name)
|
|
|
|
def _create_patch(self, base: str, changes: str) -> str:
|
|
"""Create patch from two commits."""
|
|
proc = subprocess.run('git diff {} {}'.format(base, changes), shell=True, text=True,
|
|
cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if proc.returncode != 0:
|
|
raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
|
|
return proc.stdout
|
|
|
|
def _branches_identical(self, left, right) -> bool:
|
|
"""Check if two branches are identical."""
|
|
try:
|
|
patch = self.repo.git.diff(left, right)
|
|
except git.GitCommandError:
|
|
return False
|
|
if len(patch) == 0:
|
|
return True
|
|
return False
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logging.basicConfig(level=logging.INFO)
|
|
rootdir = os.path.dirname(os.path.abspath(__file__))
|
|
tmpdir = os.path.join(rootdir, 'tmp')
|
|
p2g = Phab2Github(tmpdir)
|
|
p2g.sync()
|