From 4aed7decc36374883ab0895258bc4f7af9d18c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=BChnel?= Date: Tue, 10 Mar 2020 16:11:41 +0100 Subject: [PATCH] creating branches working --- scripts/phab2github/phab2github.py | 39 +++++++- scripts/phab2github/phab_wrapper.py | 143 ++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 scripts/phab2github/phab_wrapper.py diff --git a/scripts/phab2github/phab2github.py b/scripts/phab2github/phab2github.py index f24698d..01f3310 100644 --- a/scripts/phab2github/phab2github.py +++ b/scripts/phab2github/phab2github.py @@ -17,6 +17,8 @@ import os from typing import Optional import git import logging +from phab_wrapper import PhabWrapper, Revision +import subprocess # TODO: move to config file LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' @@ -29,11 +31,16 @@ class Phab2Github: self.workdir = workdir self.llvm_dir = os.path.join(self.workdir, 'llvm-project') self.repo = None # type: Optional[git.Repo] + self.phab_wrapper = PhabWrapper() def sync(self): _LOGGER.info('Starting sync...') self._refresh_master() - revisions = self._get_revisions() + revisions = self.phab_wrapper.get_revisions() + for revision in revisions: + self.create_branch(revision) + self.apply_patch(revision.latest_diff, + self.phab_wrapper.get_raw_patch(revision.latest_diff)) def _refresh_master(self): if not os.path.exists(self.workdir): @@ -42,14 +49,42 @@ class Phab2Github: # TODO: in case of errors: delete and clone _LOGGER.info('pulling origin...') self.repo = git.Repo(self.llvm_dir) - self.repo.remotes.origin.pull() + self.repo.remotes.origin.fetch() else: _LOGGER.info('cloning repository...') git.Repo.clone_from(LLVM_GITHUB_URL, self.llvm_dir) + self.repo = git.Repo(self.llvm_dir) _LOGGER.info('refresh of master branch completed') + def create_branch(self, revision: Revision): + name = 'phab-D{}'.format(revision.id) + if name in self.repo.heads: + self.repo.head.reference = self.repo.heads['master'] + self.repo.head.reset(index=True, working_tree=True) + self.repo.delete_head(name) + base_hash = revision.latest_diff.base_hash + if base_hash is None: + base_hash = 'origin/master' + _LOGGER.info('creating branch {} based one {}...'.format(name, base_hash)) + try: + new_branch = self.repo.create_head(name, base_hash) + except ValueError: + # commit hash not found, try again with master + base_hash = 'origin/master' + new_branch = self.repo.create_head(name, base_hash) + self.repo.head.reference = new_branch + self.repo.head.reset(index=True, working_tree=True) + + def apply_patch(self, diff: "Diff", raw_patch: str): + proc = subprocess.run('git apply --ignore-whitespace --whitespace=fix -', input=raw_patch, shell=True, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr)) + self.repo.index.commit(message='applying diff Phabricator Revision {}\n\ndiff: {}'.format(diff.revision, diff.id)) + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) rootdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = os.path.join(rootdir, 'tmp') p2g = Phab2Github(tmpdir) diff --git a/scripts/phab2github/phab_wrapper.py b/scripts/phab2github/phab_wrapper.py new file mode 100644 index 0000000..f188dfe --- /dev/null +++ b/scripts/phab2github/phab_wrapper.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import json +from phabricator import Phabricator +from typing import List, Optional, Dict +import datetime + +_LOGGER = logging.getLogger() + + +class Revision: + """A Revision on Phabricator""" + + def __init__(self, revision_dict: Dict): + self.revision_dict = revision_dict + self.diffs = [] # type : "Diff" + + @property + def id(self) -> str: + return self.revision_dict['id'] + + @property + def phid(self) -> str: + return self.revision_dict['phid'] + + @property + def status(self) -> str: + return self.revision_dict['fields']['status']['value'] + + def __str__(self): + return 'Revision {}: {} - ({})'.format(self.id, self.status, + ','.join([str(d.id) for d in self.diffs])) + + @property + def latest_diff(self): + # TODO: make sure self.diffs is sorted + return self.diffs[-1] + + +class Diff: + """A Phabricator diff.""" + + def __init__(self, diff_dict: Dict, revision: Revision): + self.diff_dict = diff_dict + self.revision = revision + + @property + def id(self) -> str: + return self.diff_dict['id'] + + @property + def phid(self) -> str: + return self.diff_dict['phid'] + + def __str__(self): + return 'Diff {}'.format(self.id) + + @property + def base_hash(self) -> Optional[str]: + for ref in self.diff_dict['fields']['refs']: + if ref['type'] == 'base': + return ref['identifier'] + return None + + +class PhabWrapper: + """ + Wrapper around the interactions with Phabricator. + + Conduit API documentation: https://reviews.llvm.org/conduit/ + """ + + def __init__(self): + self.conduit_token = None # type: Optional[str] + self.host = None # type: Optional[str] + self._load_arcrc() + self.phab = self._create_phab() # type: Phabricator + + def _load_arcrc(self): + """Load arc configuration from file if not set.""" + _LOGGER.info('Loading configuration from ~/.arcrc file') + with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file: + arcrc = json.load(arcrc_file) + # use the first host configured in the file + self.host = next(iter(arcrc['hosts'])) + self.conduit_token = arcrc['hosts'][self.host]['token'] + + def _create_phab(self) -> Phabricator: + """Create Phabricator API instance and update it.""" + phab = Phabricator(token=self.conduit_token, host=self.host) + # TODO: retry on communication error + phab.update_interfaces() + return phab + + def get_revisions(self) -> List[Revision]: + """Get relevant revisions.""" + # TODO: figure out which revisions we really need to pull in + _LOGGER.info('Getting revisions from Phabricator') + start_date = datetime.datetime.now() - datetime.timedelta(days=3) + constraints = { + 'createdStart': int(start_date.timestamp()) + } + # TODO: handle > 100 responses + revision_response = self.phab.differential.revision.search( + constraints=constraints) + revisions = [Revision(r) for r in revision_response.response['data']] + _LOGGER.info('Got {} revisions from the server'.format(len(revisions))) + for revision in revisions: + # TODO: batch-query diffs for all revisions, reduce number of + # API calls this would be much faster. But then we need to locally + # map the diffs to the right revisions + self._get_diffs(revision) + return revisions + + def _get_diffs(self, revision: Revision): + """Get diffs for a revision from Phabricator.""" + _LOGGER.info('Downloading diffs for {}...'.format(revision.id)) + constraints = { + 'revisionPHIDs': [revision.phid] + } + diff_response = self.phab.differential.diff.search( + constraints=constraints) + revision.diffs = [Diff(d, revision) for d in diff_response.response['data']] + + def get_raw_patch(self, diff: Diff) -> str: + """Get raw patch for diff from Phabricator.""" + _LOGGER.info('Downloading patch for {}...'.format(diff.id)) + return self.phab.differential.getrawdiff(diffID=str(diff.id)).response