creating branches working

scripts/phab2github/phab2github.py

@@ -17,6 +17,8 @@ import os
 from typing import Optional
 import git
 import logging
+from phab_wrapper import PhabWrapper, Revision
+import subprocess
 
 # TODO: move to config file
 LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
@@ -29,11 +31,16 @@ class Phab2Github:
         self.workdir = workdir
         self.llvm_dir = os.path.join(self.workdir, 'llvm-project')
         self.repo = None  # type: Optional[git.Repo]
+        self.phab_wrapper = PhabWrapper()
 
     def sync(self):
         _LOGGER.info('Starting sync...')
         self._refresh_master()
-        revisions = self._get_revisions()
+        revisions = self.phab_wrapper.get_revisions()
+        for revision in revisions:
+            self.create_branch(revision)
+            self.apply_patch(revision.latest_diff,
+                             self.phab_wrapper.get_raw_patch(revision.latest_diff))
 
     def _refresh_master(self):
         if not os.path.exists(self.workdir):
@@ -42,14 +49,42 @@ class Phab2Github:
             # TODO: in case of errors: delete and clone
             _LOGGER.info('pulling origin...')
             self.repo = git.Repo(self.llvm_dir)
-            self.repo.remotes.origin.pull()
+            self.repo.remotes.origin.fetch()
         else:
             _LOGGER.info('cloning repository...')
             git.Repo.clone_from(LLVM_GITHUB_URL, self.llvm_dir)
+            self.repo = git.Repo(self.llvm_dir)
         _LOGGER.info('refresh of master branch completed')
 
+    def create_branch(self, revision: Revision):
+        name = 'phab-D{}'.format(revision.id)
+        if name in self.repo.heads:
+            self.repo.head.reference = self.repo.heads['master']
+            self.repo.head.reset(index=True, working_tree=True)
+            self.repo.delete_head(name)
+        base_hash = revision.latest_diff.base_hash
+        if base_hash is None:
+            base_hash = 'origin/master'
+        _LOGGER.info('creating branch {} based one {}...'.format(name, base_hash))
+        try:
+            new_branch = self.repo.create_head(name, base_hash)
+        except ValueError:
+            # commit hash not found, try again with master
+            base_hash = 'origin/master'
+            new_branch = self.repo.create_head(name, base_hash)
+        self.repo.head.reference = new_branch
+        self.repo.head.reset(index=True, working_tree=True)
+
+    def apply_patch(self, diff: "Diff", raw_patch: str):
+        proc = subprocess.run('git apply --ignore-whitespace --whitespace=fix -', input=raw_patch, shell=True, text=True,
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if proc.returncode != 0:
+            raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
+        self.repo.index.commit(message='applying diff Phabricator Revision {}\n\ndiff: {}'.format(diff.revision, diff.id))
+
 
 if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
     rootdir = os.path.dirname(os.path.abspath(__file__))
     tmpdir = os.path.join(rootdir, 'tmp')
     p2g = Phab2Github(tmpdir)

scripts/phab2github/phab_wrapper.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import json
+from phabricator import Phabricator
+from typing import List, Optional, Dict
+import datetime
+
+_LOGGER = logging.getLogger()
+
+
+class Revision:
+    """A Revision on Phabricator"""
+
+    def __init__(self, revision_dict: Dict):
+        self.revision_dict = revision_dict
+        self.diffs = []  # type : "Diff"
+
+    @property
+    def id(self) -> str:
+        return self.revision_dict['id']
+
+    @property
+    def phid(self) -> str:
+        return self.revision_dict['phid']
+
+    @property
+    def status(self) -> str:
+        return self.revision_dict['fields']['status']['value']
+
+    def __str__(self):
+        return 'Revision {}: {} - ({})'.format(self.id, self.status,
+                                    ','.join([str(d.id) for d in self.diffs]))
+
+    @property
+    def latest_diff(self):
+        # TODO: make sure self.diffs is sorted
+        return self.diffs[-1]
+
+
+class Diff:
+    """A Phabricator diff."""
+
+    def __init__(self, diff_dict: Dict, revision: Revision):
+        self.diff_dict = diff_dict
+        self.revision = revision
+
+    @property
+    def id(self) -> str:
+        return self.diff_dict['id']
+
+    @property
+    def phid(self) -> str:
+        return self.diff_dict['phid']
+
+    def __str__(self):
+        return 'Diff {}'.format(self.id)
+
+    @property
+    def base_hash(self) -> Optional[str]:
+        for ref in self.diff_dict['fields']['refs']:
+            if ref['type'] == 'base':
+                return ref['identifier']
+        return None
+
+
+class PhabWrapper:
+    """
+    Wrapper around the interactions with Phabricator.
+
+    Conduit API documentation: https://reviews.llvm.org/conduit/
+    """
+
+    def __init__(self):
+        self.conduit_token = None  # type: Optional[str]
+        self.host = None  # type: Optional[str]
+        self._load_arcrc()
+        self.phab = self._create_phab()  # type: Phabricator
+
+    def _load_arcrc(self):
+        """Load arc configuration from file if not set."""
+        _LOGGER.info('Loading configuration from ~/.arcrc file')
+        with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file:
+            arcrc = json.load(arcrc_file)
+        # use the first host configured in the file
+        self.host = next(iter(arcrc['hosts']))
+        self.conduit_token = arcrc['hosts'][self.host]['token']
+
+    def _create_phab(self) -> Phabricator:
+        """Create Phabricator API instance and update it."""
+        phab = Phabricator(token=self.conduit_token, host=self.host)
+        # TODO: retry on communication error
+        phab.update_interfaces()
+        return phab
+
+    def get_revisions(self) -> List[Revision]:
+        """Get relevant revisions."""
+        # TODO: figure out which revisions we really need to pull in
+        _LOGGER.info('Getting revisions from Phabricator')
+        start_date = datetime.datetime.now() - datetime.timedelta(days=3)
+        constraints = {
+            'createdStart': int(start_date.timestamp())
+        }
+        # TODO: handle > 100 responses
+        revision_response = self.phab.differential.revision.search(
+                constraints=constraints)
+        revisions = [Revision(r) for r in revision_response.response['data']]
+        _LOGGER.info('Got {} revisions from the server'.format(len(revisions)))
+        for revision in revisions:
+            # TODO: batch-query diffs for all revisions, reduce number of
+            #   API calls this would be much faster. But then we need to locally
+            #   map the diffs to the right revisions
+            self._get_diffs(revision)
+        return revisions
+
+    def _get_diffs(self, revision: Revision):
+        """Get diffs for a revision from Phabricator."""
+        _LOGGER.info('Downloading diffs for {}...'.format(revision.id))
+        constraints = {
+            'revisionPHIDs': [revision.phid]
+        }
+        diff_response = self.phab.differential.diff.search(
+            constraints=constraints)
+        revision.diffs = [Diff(d, revision) for d in diff_response.response['data']]
+
+    def get_raw_patch(self, diff: Diff) -> str:
+        """Get raw patch for diff from Phabricator."""
+        _LOGGER.info('Downloading patch for {}...'.format(diff.id))
+        return self.phab.differential.getrawdiff(diffID=str(diff.id)).response