#!/usr/bin/env python3 # Copyright 2020 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://llvm.org/LICENSE.txt # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from typing import Optional, Union import git import logging from phab_wrapper import PhabWrapper, Revision import subprocess import github import json import sys # TODO: move to config file LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' MY_GITHUB_URL = 'ssh://git@github.com/christiankuehnel/llvm-project' MY_REPO = 'christiankuehnel/llvm-project' _LOGGER = logging.getLogger() class ApplyPatchException(Exception): """A patch could not be applied.""" pass class Phab2Github: def __init__(self, workdir: str): self.workdir = workdir self.llvm_dir = os.path.join(self.workdir, 'llvm-project') self.repo = None # type: Optional[git.Repo] self.phab_wrapper = PhabWrapper() self.github = self._create_github() self.github_repo = self.github.get_repo(MY_REPO) # type: github.Repository def sync(self): """Sync Phabricator to Github.""" _LOGGER.info('Starting sync...') self._refresh_main() self._delete_phab_branches() revisions = self.phab_wrapper.get_revisions() pull_requests = {p.title: p for p in self.github_repo.get_pulls(state='open')} for revision in revisions: self.create_branches_for_revision(revision) if self._has_branch(revision): if self._branches_identical(revision.branch_name, 'origin/{}'.format(revision.branch_name)): _LOGGER.info('Branch {} is identical to upstream. Not pushing.'.format(revision.branch_name)) else: _LOGGER.info('Pushing branch {} to github...'.format(revision.branch_name)) # TODO: do we sill need to force-push? self.repo.git.push('--force', 'origin', revision.branch_name) if revision.pr_title in pull_requests: _LOGGER.info('Pull request already exists: {}'.format(pull_requests[revision.pr_title].html_url)) else: _LOGGER.info('Creating pull-request for branch {}...'.format(revision.branch_name)) pr = self.github_repo.create_pull(title=revision.pr_title, body=revision.pr_summary, head=revision.branch_name, base='main') _LOGGER.info(pr.html_url) _LOGGER.info('Sync completed.') def _refresh_main(self): """Clone/update local git repo.""" if not os.path.exists(self.workdir): os.mkdir(self.workdir) if os.path.exists(self.llvm_dir): # TODO: in case of errors: delete and clone _LOGGER.info('pulling origin and upstream...') self.repo = git.Repo(self.llvm_dir) self.repo.git.fetch('--all') self.repo.git.checkout('main') self.repo.git.pull('upstream', 'main') self.repo.git.push('origin', 'main') else: _LOGGER.info('cloning repository...') git.Repo.clone_from(MY_GITHUB_URL, self.llvm_dir) self.repo = git.Repo(self.llvm_dir) self.repo.create_remote('upstream', url=LLVM_GITHUB_URL) self.repo.remotes.upstream.fetch() _LOGGER.info('refresh of main branch completed') def create_branches_for_revision(self, revision: Revision): """Create branches for a Revision and it's Diffs. Apply changes from Phabricator to these branches. """ name = revision.branch_name # TODO: only look at diffs that were not yet applied # TODO: can diffs be modified on Phabricator and keep their ID? for diff in revision.sorted_diffs: if self._has_branch(diff): continue self.create_branch_for_diff(diff) patch = self.phab_wrapper.get_raw_patch(diff) try: self.apply_patch(diff, patch) except ApplyPatchException as e: # TODO: retry on main if this fails _LOGGER.error('Could not apply patch for Diff {}. Deleting branch'.format(diff.id)) _LOGGER.exception(e) self.repo.heads['main'].checkout() self.repo.delete_head(diff.branch_name) diffs = [d for d in revision.sorted_diffs if self._has_branch(d)] if len(diffs) == 0: # TODO: handle error _LOGGER.error('Could not create branch for Revision D{}'.format(revision.id)) return new_branch = self.repo.create_head(revision.branch_name, diffs[0].branch_name) new_branch.checkout() for diff in diffs[1:]: _LOGGER.info('Applying Diff {} onto branch {}'.format(diff.branch_name, revision.branch_name)) patch = self._create_patch(revision.branch_name, diff.branch_name) if len(patch) == 0: _LOGGER.warning('Diff {} is identical to last one.'.format(diff.id)) else: try: self.apply_patch(diff, patch) except ApplyPatchException: _LOGGER.error('Applying patch failed, but should not:') _LOGGER.error(patch) raise _LOGGER.info('Created branch for Revision D{}'.format(revision.id)) def create_branch_for_diff(self, diff: "Diff"): """Create a branch for diff.""" base_hash = diff.base_hash if base_hash is None: base_hash = 'upstream/main' _LOGGER.info('creating branch {} based on {}...'.format(diff.branch_name, base_hash)) try: new_branch = self.repo.create_head(diff.branch_name, base_hash) except ValueError: # commit hash not found, try again with main _LOGGER.warning('commit hash {} not found in upstream repository. ' 'Trying main instead...'.format(diff.branch_name, base_hash)) base_hash = 'upstream/main' new_branch = self.repo.create_head(diff.branch_name, base_hash) self.repo.head.reference = new_branch self.repo.head.reset(index=True, working_tree=True) def apply_patch(self, diff: "Diff", raw_patch: str): """Apply a patch to the working copy.""" # TODO: impersonate the original author of the patch proc = subprocess.run('git apply --ignore-whitespace --whitespace=fix -', input=raw_patch, shell=True, text=True, cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr)) self.repo.git.add('-A') self.repo.index.commit(message='applying Diff {} for Revision D{}\n\n Diff: {}'.format( diff.id, diff.revision.id, diff.id)) @staticmethod def _create_github() -> github.Github: """Create instance of Github client. Reads access token from a file. """ with open(os.path.expanduser('~/.llvm-premerge-checks/github-token.json')) as json_file: token = json.load(json_file)['token'] return github.Github(token) def _has_branch(self, item: Union["Diff", "Revision"]) -> bool: """Check if the Diff/Revision has a local branch.""" return item.branch_name in self.repo.heads def _delete_phab_branches(self): """Delete all branches sarting with 'phab-'.""" _LOGGER.info('Deleting local Phabricator-relates branches...') self.repo.git.checkout('main') for branch in [b for b in self.repo.heads if b.name.startswith('phab-')]: _LOGGER.info('Deleding branch {}'.format(branch)) self.repo.git.branch('-D', branch.name) def _create_patch(self, base: str, changes: str) -> str: """Create patch from two commits.""" proc = subprocess.run('git diff {} {}'.format(base, changes), shell=True, text=True, cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr)) return proc.stdout def _branches_identical(self, left, right) -> bool: """Check if two branches are identical.""" try: patch = self.repo.git.diff(left, right) except git.GitCommandError: return False if len(patch) == 0: return True return False if __name__ == '__main__': logging.basicConfig(level=logging.INFO) rootdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = os.path.join(rootdir, 'tmp') p2g = Phab2Github(tmpdir) p2g.sync()