Add a Nuance GitHub repository source and basic polling

Summary: Ref T10537. Ref T10538. This calls GitHub, sorta?

Test Plan:
```
$ ./bin/nuance import --source poem
<cursor:events.repository> Polling GitHub Repository API endpoint "/repos/epriestley/poems/events".
<cursor:events.repository> This key has 4,988 remaining API request(s), limit resets in 1,871 second(s).
<cursor:events.repository> ETag for this request was ""4abdd3d66ad5ca38f5117b094e76f4ba"".
array(4) {
  [0]=>
  array(7) {
    ["id"]=>
    string(10) "3733510485"
...
```

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10537, T10538

Differential Revision: https://secure.phabricator.com/D15439

src/__phutil_library_map__.php

@@ -1421,6 +1421,8 @@ phutil_register_library_map(array(
     'NuanceController' => 'applications/nuance/controller/NuanceController.php',
     'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
     'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
+    'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
+    'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
     'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
     'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
     'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
@@ -5668,6 +5670,8 @@ phutil_register_library_map(array(
     'NuanceController' => 'PhabricatorController',
     'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
     'NuanceDAO' => 'PhabricatorLiskDAO',
+    'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor',
+    'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
     'NuanceImportCursor' => 'Phobject',
     'NuanceImportCursorData' => 'NuanceDAO',
     'NuanceImportCursorDataQuery' => 'NuanceQuery',

src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php

@@ -0,0 +1,114 @@
+<?php
+
+final class NuanceGitHubRepositoryImportCursor
+  extends NuanceImportCursor {
+
+  const CURSORTYPE = 'github.repository';
+
+  protected function shouldPullDataFromSource() {
+    $now = PhabricatorTime::getNow();
+
+    // Respect GitHub's poll interval header. If we made a request recently,
+    // don't make another one until we've waited long enough.
+    $ttl = $this->getCursorProperty('github.poll.ttl');
+    if ($ttl && ($ttl >= $now)) {
+      $this->logInfo(
+        pht(
+          'Respecting "%s": waiting for %s second(s) to poll GitHub.',
+          'X-Poll-Interval',
+          new PhutilNumber(1 + ($ttl - $now))));
+
+      return false;
+    }
+
+    // Respect GitHub's API rate limiting. If we've exceeded the rate limit,
+    // wait until it resets to try again.
+    $limit = $this->getCursorProperty('github.limit.ttl');
+    if ($limit && ($limit >= $now)) {
+      $this->logInfo(
+        pht(
+          'Respecting "%s": waiting for %s second(s) to poll GitHub.',
+          'X-RateLimit-Reset',
+          new PhutilNumber(1 + ($limit - $now))));
+      return false;
+    }
+
+    return true;
+  }
+
+  protected function pullDataFromSource() {
+    $source = $this->getSource();
+
+    $user = $source->getSourceProperty('github.user');
+    $repository = $source->getSourceProperty('github.repository');
+    $api_token = $source->getSourceProperty('github.token');
+
+    $uri = "/repos/{$user}/{$repository}/events";
+    $data = array();
+
+    $future = id(new PhutilGitHubFuture())
+      ->setAccessToken($api_token)
+      ->setRawGitHubQuery($uri, $data);
+
+    $etag = $this->getCursorProperty('github.poll.etag');
+    if ($etag) {
+      $future->addHeader('If-None-Match', $etag);
+    }
+
+    $this->logInfo(
+      pht(
+        'Polling GitHub Repository API endpoint "%s".',
+        $uri));
+    $response = $future->resolve();
+
+    // Do this first: if we hit the rate limit, we get a response but the
+    // body isn't valid.
+    $this->updateRateLimits($response);
+
+    // This means we hit a rate limit or a "Not Modified" because of the "ETag"
+    // header. In either case, we should bail out.
+    if ($response->getStatus()->isError()) {
+      // TODO: Save cursor data!
+      return false;
+    }
+
+    $this->updateETag($response);
+
+    var_dump($response->getBody());
+  }
+
+  private function updateRateLimits(PhutilGitHubResponse $response) {
+    $remaining = $response->getHeaderValue('X-RateLimit-Remaining');
+    $limit_reset = $response->getHeaderValue('X-RateLimit-Reset');
+    $now = PhabricatorTime::getNow();
+
+    $limit_ttl = null;
+    if (strlen($remaining)) {
+      $remaining = (int)$remaining;
+      if (!$remaining) {
+        $limit_ttl = (int)$limit_reset;
+      }
+    }
+
+    $this->setCursorProperty('github.limit.ttl', $limit_ttl);
+
+    $this->logInfo(
+      pht(
+        'This key has %s remaining API request(s), '.
+        'limit resets in %s second(s).',
+        new PhutilNumber($remaining),
+        new PhutilNumber($limit_reset - $now)));
+  }
+
+  private function updateETag(PhutilGitHubResponse $response) {
+    $etag = $response->getHeaderValue('ETag');
+
+    $this->setCursorProperty('github.poll.etag', $etag);
+
+    $this->logInfo(
+      pht(
+        'ETag for this request was "%s".',
+        $etag));
+  }
+
+}

src/applications/nuance/cursor/NuanceImportCursor.php

@@ -2,9 +2,97 @@
 
 abstract class NuanceImportCursor extends Phobject {
 
+  private $cursorData;
+  private $cursorKey;
+  private $source;
+
+  abstract protected function shouldPullDataFromSource();
+  abstract protected function pullDataFromSource();
+
+  final public function getCursorType() {
+    return $this->getPhobjectClassConstant('CURSORTYPE', 32);
+  }
+
+  public function setCursorData(NuanceImportCursorData $cursor_data) {
+    $this->cursorData = $cursor_data;
+    return $this;
+  }
+
+  public function getCursorData() {
+    return $this->cursorData;
+  }
+
+  public function setSource($source) {
+    $this->source = $source;
+    return $this;
+  }
+
+  public function getSource() {
+    return $this->source;
+  }
+
+  public function setCursorKey($cursor_key) {
+    $this->cursorKey = $cursor_key;
+    return $this;
+  }
+
+  public function getCursorKey() {
+    return $this->cursorKey;
+  }
+
   final public function importFromSource() {
-    // TODO: Perhaps, do something.
-    return false;
+    if (!$this->shouldPullDataFromSource()) {
+      return false;
+    }
+
+    $source = $this->getSource();
+    $key = $this->getCursorKey();
+
+    $parts = array(
+      'nsc',
+      $source->getID(),
+      PhabricatorHash::digestToLength($key, 20),
+    );
+    $lock_name = implode('.', $parts);
+
+    $lock = PhabricatorGlobalLock::newLock($lock_name);
+    $lock->lock(1);
+
+    try {
+      $more_data = $this->pullDataFromSource();
+    } catch (Exception $ex) {
+      $lock->unlock();
+      throw $ex;
+    }
+
+    $lock->unlock();
+
+    return $more_data;
+  }
+
+  final public function newEmptyCursorData(NuanceSource $source) {
+    return id(new NuanceImportCursorData())
+      ->setCursorKey($this->getCursorKey())
+      ->setCursorType($this->getCursorType())
+      ->setSourcePHID($source->getPHID());
+  }
+
+  final protected function logInfo($message) {
+    echo tsprintf(
+      "<cursor:%s> %s\n",
+      $this->getCursorKey(),
+      $message);
+
+    return $this;
+  }
+
+  final protected function getCursorProperty($key, $default = null) {
+    return $this->getCursorData()->getCursorProperty($key, $default);
+  }
+
+  final protected function setCursorProperty($key, $value) {
+    $this->getCursorData()->setCursorProperty($key, $value);
+    return $this;
   }
 
 }

src/applications/nuance/management/NuanceManagementImportWorkflow.php

@@ -40,9 +40,9 @@ final class NuanceManagementImportWorkflow
           $source->getName()));
     }
 
-    echo tsprintf(
-      "%s\n",
-      pht('OK, but actual importing is not implemented yet.'));
+    foreach ($cursors as $cursor) {
+      $cursor->importFromSource();
+    }
 
     return 0;
   }

src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php

@@ -0,0 +1,29 @@
+<?php
+
+final class NuanceGitHubRepositorySourceDefinition
+  extends NuanceSourceDefinition {
+
+  public function getName() {
+    return pht('GitHub Repository');
+  }
+
+  public function getSourceDescription() {
+    return pht('Import issues and pull requests from a GitHub repository.');
+  }
+
+  public function getSourceTypeConstant() {
+    return 'github.repository';
+  }
+
+  public function hasImportCursors() {
+    return true;
+  }
+
+  protected function newImportCursors() {
+    return array(
+      id(new NuanceGitHubRepositoryImportCursor())
+        ->setCursorKey('events.repository'),
+    );
+  }
+
+}

src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php

@@ -26,15 +26,6 @@ final class NuancePhabricatorFormSourceDefinition
     return $actions;
   }
 
-  public function updateItems() {
-    return null;
-  }
-
-  public function renderView() {}
-
-  public function renderListView() {}
-
-
   public function handleActionRequest(AphrontRequest $request) {
     $viewer = $request->getViewer();
 

src/applications/nuance/source/NuanceSourceDefinition.php

@@ -53,7 +53,66 @@ abstract class NuanceSourceDefinition extends Phobject {
         pht('This source has no input cursors.'));
     }
 
-    return $this->newImportCursors();
+    $source = $this->getSource();
+    $cursors = $this->newImportCursors();
+
+    $data = id(new NuanceImportCursorDataQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withSourcePHIDs(array($source->getPHID()))
+      ->execute();
+    $data = mpull($data, 'getCursorKey');
+
+    $map = array();
+    foreach ($cursors as $cursor) {
+      if (!($cursor instanceof NuanceImportCursor)) {
+        throw new Exception(
+          pht(
+            'Source "%s" (of class "%s") returned an invalid value from '.
+            'method "%s": all values must be objects of class "%s".',
+            $this->getName(),
+            get_class($this),
+            'newImportCursors()',
+            'NuanceImportCursor'));
+      }
+
+      $key = $cursor->getCursorKey();
+      if (!strlen($key)) {
+        throw new Exception(
+          pht(
+            'Source "%s" (of class "%s") returned an import cursor with '.
+            'a missing key from "%s". Each cursor must have a unique, '.
+            'nonempty key.',
+            $this->getName(),
+            get_class($this),
+            'newImportCursors()'));
+      }
+
+      $other = idx($map, $key);
+      if ($other) {
+        throw new Exception(
+          pht(
+            'Source "%s" (of class "%s") returned two cursors from method '.
+            '"%s" with the same key ("%s"). Each cursor must have a unique '.
+            'key.',
+            $this->getName(),
+            get_class($this),
+            'newImportCursors()',
+            $key));
+      }
+
+      $map[$key] = $cursor;
+
+      $cursor->setSource($source);
+
+      $cursor_data = idx($data, $key);
+      if (!$cursor_data) {
+        $cursor_data = $cursor->newEmptyCursorData($source);
+      }
+
+      $cursor->setCursorData($cursor_data);
+    }
+
+    return $cursors;
   }
 
   protected function newImportCursors() {
@@ -79,21 +138,13 @@ abstract class NuanceSourceDefinition extends Phobject {
    */
   abstract public function getSourceTypeConstant();
 
-  /**
-   * Code to create and update @{class:NuanceItem}s and
-   * @{class:NuanceRequestor}s via daemons goes here.
-   *
-   * If that does not make sense for the @{class:NuanceSource} you are
-   * defining, simply return null. For example,
-   * @{class:NuancePhabricatorFormSourceDefinition} since these are one-way
-   * contact forms.
-   */
-  abstract public function updateItems();
-
-  abstract public function renderView();
-
-  abstract public function renderListView();
+  public function renderView() {
+    return null;
+  }
 
+  public function renderListView() {
+    return null;
+  }
 
   protected function newItemFromProperties(
     NuanceRequestor $requestor,

src/applications/nuance/storage/NuanceImportCursorData.php

@@ -32,4 +32,13 @@ final class NuanceImportCursorData
       NuanceImportCursorPHIDType::TYPECONST);
   }
 
+  public function getCursorProperty($key, $default = null) {
+    return idx($this->properties, $key, $default);
+  }
+
+  public function setCursorProperty($key, $value) {
+    $this->properties[$key] = $value;
+    return $this;
+  }
+
 }

src/applications/nuance/storage/NuanceSource.php

@@ -8,7 +8,7 @@ final class NuanceSource extends NuanceDAO
 
   protected $name;
   protected $type;
-  protected $data;
+  protected $data = array();
   protected $mailKey;
   protected $viewPolicy;
   protected $editPolicy;
@@ -82,6 +82,15 @@ final class NuanceSource extends NuanceDAO
     return $this;
   }
 
+  public function getSourceProperty($key, $default = null) {
+    return idx($this->data, $key, $default);
+  }
+
+  public function setSourceProperty($key, $value) {
+    $this->data[$key] = $value;
+    return $this;
+  }
+
 
 /* -(  PhabricatorApplicationTransactionInterface  )------------------------- */