Begin bridging GitHub objects through Doorkeeper

Summary:
Ref T10538. This sets up a Doorkeeper bridge for GitHub issues, and pulls issues from GitHub to create ExternalObject references.

Broadly, does nothing useful.

Test Plan: Put a `var_dump()` in there somewhere and saw it probably do something when running `bin/nuance update --item 44`.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10538

Differential Revision: https://secure.phabricator.com/D15447

src/__phutil_library_map__.php

@@ -840,6 +840,8 @@ phutil_register_library_map(array(
     'DoorkeeperAsanaRemarkupRule' => 'applications/doorkeeper/remarkup/DoorkeeperAsanaRemarkupRule.php',
     'DoorkeeperBridge' => 'applications/doorkeeper/bridge/DoorkeeperBridge.php',
     'DoorkeeperBridgeAsana' => 'applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php',
+    'DoorkeeperBridgeGitHub' => 'applications/doorkeeper/bridge/DoorkeeperBridgeGitHub.php',
+    'DoorkeeperBridgeGitHubIssue' => 'applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php',
     'DoorkeeperBridgeJIRA' => 'applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php',
     'DoorkeeperBridgeJIRATestCase' => 'applications/doorkeeper/bridge/__tests__/DoorkeeperBridgeJIRATestCase.php',
     'DoorkeeperDAO' => 'applications/doorkeeper/storage/DoorkeeperDAO.php',
@@ -1423,6 +1425,7 @@ phutil_register_library_map(array(
     'NuanceGitHubEventItemType' => 'applications/nuance/item/NuanceGitHubEventItemType.php',
     'NuanceGitHubImportCursor' => 'applications/nuance/cursor/NuanceGitHubImportCursor.php',
     'NuanceGitHubIssuesImportCursor' => 'applications/nuance/cursor/NuanceGitHubIssuesImportCursor.php',
+    'NuanceGitHubRawEvent' => 'applications/nuance/github/NuanceGitHubRawEvent.php',
     'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
     'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
     'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
@@ -4970,6 +4973,8 @@ phutil_register_library_map(array(
     'DoorkeeperAsanaRemarkupRule' => 'DoorkeeperRemarkupRule',
     'DoorkeeperBridge' => 'Phobject',
     'DoorkeeperBridgeAsana' => 'DoorkeeperBridge',
+    'DoorkeeperBridgeGitHub' => 'DoorkeeperBridge',
+    'DoorkeeperBridgeGitHubIssue' => 'DoorkeeperBridgeGitHub',
     'DoorkeeperBridgeJIRA' => 'DoorkeeperBridge',
     'DoorkeeperBridgeJIRATestCase' => 'PhabricatorTestCase',
     'DoorkeeperDAO' => 'PhabricatorLiskDAO',
@@ -5681,6 +5686,7 @@ phutil_register_library_map(array(
     'NuanceGitHubEventItemType' => 'NuanceItemType',
     'NuanceGitHubImportCursor' => 'NuanceImportCursor',
     'NuanceGitHubIssuesImportCursor' => 'NuanceGitHubImportCursor',
+    'NuanceGitHubRawEvent' => 'Phobject',
     'NuanceGitHubRepositoryImportCursor' => 'NuanceGitHubImportCursor',
     'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
     'NuanceImportCursor' => 'Phobject',

src/applications/doorkeeper/bridge/DoorkeeperBridge.php

@@ -3,6 +3,7 @@
 abstract class DoorkeeperBridge extends Phobject {
 
   private $viewer;
+  private $context = array();
   private $throwOnMissingLink;
 
   public function setThrowOnMissingLink($throw_on_missing_link) {
@@ -19,6 +20,15 @@ abstract class DoorkeeperBridge extends Phobject {
     return $this->viewer;
   }
 
+  final public function setContext($context) {
+    $this->context = $context;
+    return $this;
+  }
+
+  final public function getContextProperty($key, $default = null) {
+    return idx($this->context, $key, $default);
+  }
+
   public function isEnabled() {
     return true;
   }

src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHub.php

@@ -0,0 +1,50 @@
+<?php
+
+abstract class DoorkeeperBridgeGitHub extends DoorkeeperBridge {
+
+  const APPTYPE_GITHUB  = 'github';
+  const APPDOMAIN_GITHUB = 'github.com';
+
+  public function canPullRef(DoorkeeperObjectRef $ref) {
+    if ($ref->getApplicationType() != self::APPTYPE_GITHUB) {
+      return false;
+    }
+
+    if ($ref->getApplicationDomain() != self::APPDOMAIN_GITHUB) {
+      return false;
+    }
+
+    return true;
+  }
+
+  protected function getGitHubAccessToken() {
+    $context_token = $this->getContextProperty('github.token');
+    if ($context_token) {
+      return $context_token->openEnvelope();
+    }
+
+    // TODO: Do a bunch of work to fetch the viewer's linked account if
+    // they have one.
+
+    return $this->didFailOnMissingLink();
+  }
+
+  protected function parseGitHubIssueID($id) {
+    $matches = null;
+    if (!preg_match('(^([^/]+)/([^/]+)#([1-9]\d*)\z)', $id, $matches)) {
+      throw new Exception(
+        pht(
+          'GitHub Issue ID "%s" is not properly formatted. Expected an ID '.
+          'in the form "owner/repository#123".',
+          $id));
+    }
+
+    return array(
+      $matches[1],
+      $matches[2],
+      (int)$matches[3],
+    );
+  }
+
+
+}

src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php

@@ -0,0 +1,97 @@
+<?php
+
+final class DoorkeeperBridgeGitHubIssue
+  extends DoorkeeperBridgeGitHub {
+
+  const OBJTYPE_GITHUB_ISSUE = 'github.issue';
+
+  public function canPullRef(DoorkeeperObjectRef $ref) {
+    if (!parent::canPullRef($ref)) {
+      return false;
+    }
+
+    if ($ref->getObjectType() !== self::OBJTYPE_GITHUB_ISSUE) {
+      return false;
+    }
+
+    return true;
+  }
+
+  public function pullRefs(array $refs) {
+    $token = $this->getGitHubAccessToken();
+    if (!strlen($token)) {
+      return null;
+    }
+
+    $template = id(new PhutilGitHubFuture())
+      ->setAccessToken($token);
+
+    $futures = array();
+    $id_map = mpull($refs, 'getObjectID', 'getObjectKey');
+    foreach ($id_map as $key => $id) {
+      list($user, $repository, $number) = $this->parseGitHubIssueID($id);
+      $uri = "/repos/{$user}/{$repository}/issues/{$number}";
+      $data = array();
+      $futures[$key] = id(clone $template)
+        ->setRawGitHubQuery($uri, $data);
+    }
+
+    $results = array();
+    $failed = array();
+    foreach (new FutureIterator($futures) as $key => $future) {
+      try {
+        $results[$key] = $future->resolve();
+      } catch (Exception $ex) {
+        if (($ex instanceof HTTPFutureResponseStatus) &&
+            ($ex->getStatusCode() == 404)) {
+          // TODO: Do we end up here for deleted objects and invisible
+          // objects?
+        } else {
+          phlog($ex);
+          $failed[$key] = $ex;
+        }
+      }
+    }
+
+    $viewer = $this->getViewer();
+
+    foreach ($refs as $ref) {
+      $ref->setAttribute('name', pht('GitHub Issue %s', $ref->getObjectID()));
+
+      $did_fail = idx($failed, $ref->getObjectKey());
+      if ($did_fail) {
+        $ref->setSyncFailed(true);
+        continue;
+      }
+
+      $result = idx($results, $ref->getObjectKey());
+      if (!$result) {
+        continue;
+      }
+
+      $body = $result->getBody();
+
+      $ref->setIsVisible(true);
+      $ref->setAttribute('api.raw', $body);
+      $ref->setAttribute('name', $body['title']);
+
+      $obj = $ref->getExternalObject();
+      if ($obj->getID()) {
+        continue;
+      }
+
+      $this->fillObjectFromData($obj, $result);
+
+      $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+        $obj->save();
+      unset($unguarded);
+    }
+  }
+
+  public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) {
+    $body = $result->getBody();
+    $uri = $body['html_url'];
+    $obj->setObjectURI($uri);
+  }
+
+}

src/applications/doorkeeper/engine/DoorkeeperImportEngine.php

@@ -7,6 +7,7 @@ final class DoorkeeperImportEngine extends Phobject {
   private $phids = array();
   private $localOnly;
   private $throwOnMissingLink;
+  private $context = array();
 
   public function setViewer(PhabricatorUser $viewer) {
     $this->viewer = $viewer;
@@ -37,6 +38,10 @@ final class DoorkeeperImportEngine extends Phobject {
     return $this;
   }
 
+  public function setContextProperty($key, $value) {
+    $this->context[$key] = $value;
+    return $this;
+  }
 
   /**
    * Configure behavior if remote refs can not be retrieved because an
@@ -96,6 +101,7 @@ final class DoorkeeperImportEngine extends Phobject {
       foreach ($bridges as $key => $bridge) {
         $bridge->setViewer($viewer);
         $bridge->setThrowOnMissingLink($this->throwOnMissingLink);
+        $bridge->setContext($this->context);
       }
 
       $working_set = $refs;

src/applications/nuance/github/NuanceGitHubRawEvent.php

@@ -0,0 +1,72 @@
+<?php
+
+final class NuanceGitHubRawEvent extends Phobject {
+
+  private $raw;
+  private $type;
+
+  const TYPE_ISSUE = 'issue';
+  const TYPE_REPOSITORY = 'repository';
+
+  public static function newEvent($type, array $raw) {
+    $event = new self();
+    $event->type = $type;
+    $event->raw = $raw;
+    return $event;
+  }
+
+  public function getRepositoryFullName() {
+    return $this->getRepositoryFullRawName();
+  }
+
+  public function isIssueEvent() {
+    if ($this->isPullRequestEvent()) {
+      return false;
+    }
+
+    if ($this->type == self::TYPE_ISSUE) {
+      return true;
+    }
+
+    switch ($this->getIssueRawKind()) {
+      case 'IssuesEvent':
+      case 'IssuesCommentEvent':
+        return true;
+    }
+
+    return false;
+  }
+
+  public function isPullRequestEvent() {
+    return false;
+  }
+
+  public function getIssueNumber() {
+    if (!$this->isIssueEvent()) {
+      return null;
+    }
+
+    $raw = $this->raw;
+
+    if ($this->type == self::TYPE_ISSUE) {
+      return idxv($raw, array('issue', 'number'));
+    }
+
+    if ($this->type == self::TYPE_REPOSITORY) {
+      return idxv($raw, array('payload', 'issue', 'number'));
+    }
+
+    return null;
+  }
+
+  private function getRepositoryFullRawName() {
+    $raw = $this->raw;
+    return idxv($raw, array('repo', 'name'));
+  }
+
+  private function getIssueRawKind() {
+    $raw = $this->raw;
+    return idxv($raw, array('type'));
+  }
+
+}

src/applications/nuance/item/NuanceGitHubEventItemType.php

@@ -74,13 +74,74 @@ final class NuanceGitHubEventItemType
   }
 
   protected function updateItemFromSource(NuanceItem $item) {
+    $viewer = $this->getViewer();
+    $is_dirty = false;
+
     // TODO: Link up the requestor, etc.
 
+    $source = $item->getSource();
+    $token = $source->getSourceProperty('github.token');
+    $token = new PhutilOpaqueEnvelope($token);
+
+    $ref = $this->getDoorkeeperRef($item);
+    if ($ref) {
+      $ref = id(new DoorkeeperImportEngine())
+        ->setViewer($viewer)
+        ->setRefs(array($ref))
+        ->setThrowOnMissingLink(true)
+        ->setContextProperty('github.token', $token)
+        ->executeOne();
+
+      if ($ref->getSyncFailed()) {
+        $xobj = null;
+      } else {
+        $xobj = $ref->getExternalObject();
+      }
+
+      if ($xobj) {
+        $item->setItemProperty('doorkeeper.xobj.phid', $xobj->getPHID());
+        $is_dirty = true;
+      }
+    }
+
     if ($item->getStatus() == NuanceItem::STATUS_IMPORTING) {
-      $item
-        ->setStatus(NuanceItem::STATUS_ROUTING)
-        ->save();
+      $item->setStatus(NuanceItem::STATUS_ROUTING);
+      $is_dirty = true;
+    }
+
+    if ($is_dirty) {
+      $item->save();
     }
   }
 
+  private function getDoorkeeperRef(NuanceItem $item) {
+    $raw = $this->newRawEvent($item);
+
+    $full_repository = $raw->getRepositoryFullName();
+    if (!strlen($full_repository)) {
+      return null;
+    }
+
+    if ($raw->isIssueEvent()) {
+      $ref_type = DoorkeeperBridgeGitHubIssue::OBJTYPE_GITHUB_ISSUE;
+      $issue_number = $raw->getIssueNumber();
+      $full_ref = "{$full_repository}#{$issue_number}";
+    } else {
+      return null;
+    }
+
+    return id(new DoorkeeperObjectRef())
+      ->setApplicationType(DoorkeeperBridgeGitHub::APPTYPE_GITHUB)
+      ->setApplicationDomain(DoorkeeperBridgeGitHub::APPDOMAIN_GITHUB)
+      ->setObjectType($ref_type)
+      ->setObjectID($full_ref);
+  }
+
+  private function newRawEvent(NuanceItem $item) {
+    $type = $item->getItemProperty('api.type');
+    $raw = $item->getItemProperty('api.raw', array());
+
+    return NuanceGitHubRawEvent::newEvent($type, $raw);
+  }
+
 }

src/applications/nuance/worker/NuanceItemUpdateWorker.php

@@ -25,9 +25,14 @@ final class NuanceItemUpdateWorker
 
   private function updateItem(NuanceItem $item) {
     $impl = $item->getImplementation();
-    if ($impl->canUpdateItems()) {
-      $impl->updateItem($item);
+    if (!$impl->canUpdateItems()) {
+      return null;
     }
+
+    $viewer = $this->getViewer();
+
+    $impl->setViewer($viewer);
+    $impl->updateItem($item);
   }
 
   private function routeItem(NuanceItem $item) {