llvm-premerge-checks/scripts/phab2github/phab2github.py

#!/usr/bin/env python3
# Copyright 2020 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from typing import Optional, Union
import git
import logging
from phab_wrapper import PhabWrapper, Revision
import subprocess
import github
import json
import sys

# TODO: move to config file
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
MY_GITHUB_URL = 'ssh://git@github.com/christiankuehnel/llvm-project'
MY_REPO = 'christiankuehnel/llvm-project'


_LOGGER = logging.getLogger()


class ApplyPatchException(Exception):
    """A patch could not be applied."""
    pass


class Phab2Github:

    def __init__(self, workdir: str):
        self.workdir = workdir
        self.llvm_dir = os.path.join(self.workdir, 'llvm-project')
        self.repo = None  # type: Optional[git.Repo]
        self.phab_wrapper = PhabWrapper()
        self.github = self._create_github()
        self.github_repo = self.github.get_repo(MY_REPO)  # type: github.Repository

    def sync(self):
        """Sync Phabricator to Github."""
        _LOGGER.info('Starting sync...')
        self._refresh_main()
        self._delete_phab_branches()
        revisions = self.phab_wrapper.get_revisions()
        pull_requests = {p.title: p for p in self.github_repo.get_pulls(state='open')}
        for revision in revisions:
            self.create_branches_for_revision(revision)
            if self._has_branch(revision):
                if self._branches_identical(revision.branch_name, 'origin/{}'.format(revision.branch_name)):
                    _LOGGER.info('Branch {} is identical to upstream. Not pushing.'.format(revision.branch_name))
                else:
                    _LOGGER.info('Pushing branch {} to github...'.format(revision.branch_name))
                    # TODO: do we sill need to force-push?
                    self.repo.git.push('--force', 'origin', revision.branch_name)
                if revision.pr_title in pull_requests:
                    _LOGGER.info('Pull request already exists: {}'.format(pull_requests[revision.pr_title].html_url))
                else:
                    _LOGGER.info('Creating pull-request for branch {}...'.format(revision.branch_name))
                    pr = self.github_repo.create_pull(title=revision.pr_title,
                                                      body=revision.pr_summary,
                                                      head=revision.branch_name,
                                                      base='main')
                    _LOGGER.info(pr.html_url)
        _LOGGER.info('Sync completed.')

    def _refresh_main(self):
        """Clone/update local git repo."""
        if not os.path.exists(self.workdir):
            os.mkdir(self.workdir)
        if os.path.exists(self.llvm_dir):
            # TODO: in case of errors: delete and clone
            _LOGGER.info('pulling origin and upstream...')
            self.repo = git.Repo(self.llvm_dir)
            self.repo.git.fetch('--all')
            self.repo.git.checkout('main')
            self.repo.git.pull('upstream', 'main')
            self.repo.git.push('origin', 'main')
        else:
            _LOGGER.info('cloning repository...')
            git.Repo.clone_from(MY_GITHUB_URL, self.llvm_dir)
            self.repo = git.Repo(self.llvm_dir)
            self.repo.create_remote('upstream', url=LLVM_GITHUB_URL)
            self.repo.remotes.upstream.fetch()
        _LOGGER.info('refresh of main branch completed')

    def create_branches_for_revision(self, revision: Revision):
        """Create branches for a Revision and it's Diffs.

        Apply changes from Phabricator to these branches.
        """
        name = revision.branch_name
        # TODO: only look at diffs that were not yet applied
        # TODO: can diffs be modified on Phabricator and keep their ID?
        for diff in revision.sorted_diffs:
            if self._has_branch(diff):
                continue
            self.create_branch_for_diff(diff)
            patch = self.phab_wrapper.get_raw_patch(diff)
            try:
                self.apply_patch(diff, patch)
            except ApplyPatchException as e:
                # TODO: retry on main if this fails
                _LOGGER.error('Could not apply patch for Diff {}. Deleting branch'.format(diff.id))
                _LOGGER.exception(e)
                self.repo.heads['main'].checkout()
                self.repo.delete_head(diff.branch_name)

        diffs = [d for d in revision.sorted_diffs if self._has_branch(d)]
        if len(diffs) == 0:
            # TODO: handle error
            _LOGGER.error('Could not create branch for Revision D{}'.format(revision.id))
            return
        new_branch = self.repo.create_head(revision.branch_name, diffs[0].branch_name)
        new_branch.checkout()

        for diff in diffs[1:]:
            _LOGGER.info('Applying Diff {} onto branch {}'.format(diff.branch_name, revision.branch_name))
            patch = self._create_patch(revision.branch_name, diff.branch_name)
            if len(patch) == 0:
                _LOGGER.warning('Diff {} is identical to last one.'.format(diff.id))
            else:
                try:
                    self.apply_patch(diff, patch)
                except ApplyPatchException:
                    _LOGGER.error('Applying patch failed, but should not:')
                    _LOGGER.error(patch)
                    raise

        _LOGGER.info('Created branch for Revision D{}'.format(revision.id))

    def create_branch_for_diff(self, diff: "Diff"):
        """Create a branch for diff."""
        base_hash = diff.base_hash
        if base_hash is None:
            base_hash = 'upstream/main'
        _LOGGER.info('creating branch {} based on {}...'.format(diff.branch_name, base_hash))
        try:
            new_branch = self.repo.create_head(diff.branch_name, base_hash)
        except ValueError:
            # commit hash not found, try again with main
            _LOGGER.warning('commit hash {} not found in upstream repository. '
                            'Trying main instead...'.format(diff.branch_name, base_hash))
            base_hash = 'upstream/main'
            new_branch = self.repo.create_head(diff.branch_name, base_hash)
        self.repo.head.reference = new_branch
        self.repo.head.reset(index=True, working_tree=True)

    def apply_patch(self, diff: "Diff", raw_patch: str):
        """Apply a patch to the working copy."""
        # TODO: impersonate the original author of the patch
        proc = subprocess.run('git apply --ignore-whitespace --whitespace=fix -', input=raw_patch, shell=True,
                              text=True, cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if proc.returncode != 0:
            raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
        self.repo.git.add('-A')
        self.repo.index.commit(message='applying Diff {} for Revision D{}\n\n  Diff: {}'.format(
            diff.id, diff.revision.id, diff.id))

    @staticmethod
    def _create_github() -> github.Github:
        """Create instance of Github client.

        Reads access token from a file.
        """
        with open(os.path.expanduser('~/.llvm-premerge-checks/github-token.json')) as json_file:
            token = json.load(json_file)['token']
        return github.Github(token)

    def _has_branch(self, item: Union["Diff", "Revision"]) -> bool:
        """Check if the Diff/Revision has a local branch."""
        return item.branch_name in self.repo.heads

    def _delete_phab_branches(self):
        """Delete all branches sarting with 'phab-'."""
        _LOGGER.info('Deleting local Phabricator-relates branches...')
        self.repo.git.checkout('main')
        for branch in [b for b in self.repo.heads if b.name.startswith('phab-')]:
            _LOGGER.info('Deleding branch {}'.format(branch))
            self.repo.git.branch('-D', branch.name)

    def _create_patch(self, base: str, changes: str) -> str:
        """Create patch from two commits."""
        proc = subprocess.run('git diff {} {}'.format(base, changes), shell=True, text=True,
                              cwd=self.repo.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if proc.returncode != 0:
            raise ApplyPatchException('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
        return proc.stdout

    def _branches_identical(self, left, right) -> bool:
        """Check if two branches are identical."""
        try:
            patch = self.repo.git.diff(left, right)
        except git.GitCommandError:
            return False
        if len(patch) == 0:
            return True
        return False


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    rootdir = os.path.dirname(os.path.abspath(__file__))
    tmpdir = os.path.join(rootdir, 'tmp')
    p2g = Phab2Github(tmpdir)
    p2g.sync()