Allow Nuances sources to provide import cursors

Summary:
Ref T10537. Some sources (like the future "GitHub Repository" source) need to poll remotes.

  - Provide a mechanism for sources to emit import cursors.
  - Hook them into the trigger daemon so they'll fire periodically.
  - Provide some storage.

This diff does nothing useful or interesting, and is pure infrastructure.

Test Plan:
  - Ran `bin/storage upgrade -f`, no adjustment issues.
  - Poked around Nuance.
  - Ran the trigger daemon, verified it didn't crash and checked for Nuance stuff to do.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10537

Differential Revision: https://secure.phabricator.com/D15435

resources/sql/autopatches/20160308.nuance.01.disabled.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_source
+  ADD isDisabled BOOL NOT NULL;

resources/sql/autopatches/20160308.nuance.02.cursordata.sql

@@ -0,0 +1,12 @@
+CREATE TABLE {$NAMESPACE}_nuance.nuance_importcursordata (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARBINARY(64) NOT NULL,
+  sourcePHID VARBINARY(64) NOT NULL,
+  cursorKey VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
+  cursorType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
+  properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY `key_phid` (phid),
+  UNIQUE KEY `key_source` (sourcePHID, cursorKey)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -1421,6 +1421,10 @@ phutil_register_library_map(array(
     'NuanceController' => 'applications/nuance/controller/NuanceController.php',
     'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
     'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
+    'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
+    'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
+    'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
+    'NuanceImportCursorPHIDType' => 'applications/nuance/phid/NuanceImportCursorPHIDType.php',
     'NuanceItem' => 'applications/nuance/storage/NuanceItem.php',
     'NuanceItemEditController' => 'applications/nuance/controller/NuanceItemEditController.php',
     'NuanceItemEditor' => 'applications/nuance/editor/NuanceItemEditor.php',
@@ -5661,6 +5665,10 @@ phutil_register_library_map(array(
     'NuanceController' => 'PhabricatorController',
     'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
     'NuanceDAO' => 'PhabricatorLiskDAO',
+    'NuanceImportCursor' => 'Phobject',
+    'NuanceImportCursorData' => 'NuanceDAO',
+    'NuanceImportCursorDataQuery' => 'NuanceQuery',
+    'NuanceImportCursorPHIDType' => 'PhabricatorPHIDType',
     'NuanceItem' => array(
       'NuanceDAO',
       'PhabricatorPolicyInterface',

src/applications/nuance/cursor/NuanceImportCursor.php

@@ -0,0 +1,10 @@
+<?php
+
+abstract class NuanceImportCursor extends Phobject {
+
+  final public function importFromSource() {
+    // TODO: Perhaps, do something.
+    return false;
+  }
+
+}

src/applications/nuance/phid/NuanceImportCursorPHIDType.php

@@ -0,0 +1,38 @@
+<?php
+
+final class NuanceImportCursorPHIDType extends PhabricatorPHIDType {
+
+  const TYPECONST = 'NUAC';
+
+  public function getTypeName() {
+    return pht('Import Cursor');
+  }
+
+  public function newObject() {
+    return new NuanceImportCursorData();
+  }
+
+  public function getPHIDTypeApplicationClass() {
+    return 'PhabricatorNuanceApplication';
+  }
+
+  protected function buildQueryForObjects(
+    PhabricatorObjectQuery $query,
+    array $phids) {
+
+    return id(new NuanceImportCursorDataQuery())
+      ->withPHIDs($phids);
+  }
+
+  public function loadHandles(
+    PhabricatorHandleQuery $query,
+    array $handles,
+    array $objects) {
+
+    $viewer = $query->getViewer();
+    foreach ($handles as $phid => $handle) {
+      $item = $objects[$phid];
+    }
+  }
+
+}

src/applications/nuance/query/NuanceImportCursorDataQuery.php

@@ -0,0 +1,60 @@
+<?php
+
+final class NuanceImportCursorDataQuery
+  extends NuanceQuery {
+
+  private $ids;
+  private $phids;
+  private $sourcePHIDs;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withPHIDs(array $phids) {
+    $this->phids = $phids;
+    return $this;
+  }
+
+  public function withSourcePHIDs(array $source_phids) {
+    $this->sourcePHIDs = $source_phids;
+    return $this;
+  }
+
+  public function newResultObject() {
+    return new NuanceImportCursorData();
+  }
+
+  protected function loadPage() {
+    return $this->loadStandardPage($this->newResultObject());
+  }
+
+  protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
+    $where = parent::buildWhereClauseParts($conn);
+
+    if ($this->sourcePHIDs !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'sourcePHID IN (%Ls)',
+        $this->sourcePHIDs);
+    }
+
+    if ($this->ids !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->phids !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'phid IN (%Ls)',
+        $this->phids);
+    }
+
+    return $where;
+  }
+
+}

src/applications/nuance/query/NuanceSourceQuery.php

@@ -6,6 +6,8 @@ final class NuanceSourceQuery
   private $ids;
   private $phids;
   private $types;
+  private $isDisabled;
+  private $hasCursors;
 
   public function withIDs(array $ids) {
     $this->ids = $ids;
@@ -22,6 +24,16 @@ final class NuanceSourceQuery
     return $this;
   }
 
+  public function withIsDisabled($disabled) {
+    $this->isDisabled = $disabled;
+    return $this;
+  }
+
+  public function withHasImportCursors($has_cursors) {
+    $this->hasCursors = $has_cursors;
+    return $this;
+  }
+
   public function newResultObject() {
     return new NuanceSource();
   }
@@ -71,6 +83,44 @@ final class NuanceSourceQuery
         $this->phids);
     }
 
+    if ($this->isDisabled !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'isDisabled = %d',
+        (int)$this->isDisabled);
+    }
+
+    if ($this->hasCursors !== null) {
+      $cursor_types = array();
+
+      $definitions = NuanceSourceDefinition::getAllDefinitions();
+      foreach ($definitions as $key => $definition) {
+        if ($definition->hasImportCursors()) {
+          $cursor_types[] = $key;
+        }
+      }
+
+      if ($this->hasCursors) {
+        if (!$cursor_types) {
+          throw new PhabricatorEmptyQueryException();
+        } else {
+          $where[] = qsprintf(
+            $conn,
+            'type IN (%Ls)',
+            $cursor_types);
+        }
+      } else {
+        if (!$cursor_types) {
+          // Apply no constraint.
+        } else {
+          $where[] = qsprintf(
+            $conn,
+            'type NOT IN (%Ls)',
+            $cursor_types);
+        }
+      }
+    }
+
     return $where;
   }
 

src/applications/nuance/source/NuanceSourceDefinition.php

@@ -43,6 +43,23 @@ abstract class NuanceSourceDefinition extends Phobject {
       ->execute();
   }
 
+  public function hasImportCursors() {
+    return false;
+  }
+
+  final public function getImportCursors() {
+    if (!$this->hasImportCursors()) {
+      throw new Exception(
+        pht('This source has no input cursors.'));
+    }
+
+    return $this->newImportCursors();
+  }
+
+  protected function newImportCursors() {
+    throw new PhutilMethodNotImplementedException();
+  }
+
   /**
    * A human readable string like "Twitter" or "Phabricator Form".
    */

src/applications/nuance/storage/NuanceImportCursorData.php

@@ -0,0 +1,35 @@
+<?php
+
+final class NuanceImportCursorData
+  extends NuanceDAO {
+
+  protected $sourcePHID;
+  protected $cursorKey;
+  protected $cursorType;
+  protected $properties = array();
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_AUX_PHID => true,
+      self::CONFIG_SERIALIZATION => array(
+        'properties' => self::SERIALIZATION_JSON,
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'cursorType' => 'text32',
+        'cursorKey' => 'text32',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_source' => array(
+          'columns' => array('sourcePHID', 'cursorKey'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function generatePHID() {
+    return PhabricatorPHID::generateNewPHID(
+      NuanceImportCursorPHIDType::TYPECONST);
+  }
+
+}

src/applications/nuance/storage/NuanceSource.php

@@ -12,6 +12,7 @@ final class NuanceSource extends NuanceDAO
   protected $viewPolicy;
   protected $editPolicy;
   protected $defaultQueuePHID;
+  protected $isDisabled;
 
   private $definition = self::ATTACHABLE;
 
@@ -25,6 +26,7 @@ final class NuanceSource extends NuanceDAO
         'name' => 'text255?',
         'type' => 'text32',
         'mailKey' => 'bytes20',
+        'isDisabled' => 'bool',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_type' => array(
@@ -66,7 +68,8 @@ final class NuanceSource extends NuanceDAO
       ->setViewPolicy($view_policy)
       ->setEditPolicy($edit_policy)
       ->setType($definition->getSourceTypeConstant())
-      ->attachDefinition($definition);
+      ->attachDefinition($definition)
+      ->setIsDisabled(0);
   }
 
   public function getDefinition() {

src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php

@@ -16,6 +16,10 @@ final class PhabricatorTriggerDaemon
   private $garbageCollectors;
   private $nextCollection;
 
+  private $anyNuanceData;
+  private $nuanceSources;
+  private $nuanceCursors;
+
   protected function run() {
 
     // The trigger daemon is a low-level infrastructure daemon which schedules
@@ -99,6 +103,7 @@ final class PhabricatorTriggerDaemon
       $lock->unlock();
 
       $sleep_duration = $this->getSleepDuration();
+      $sleep_duration = $this->runNuanceImportCursors($sleep_duration);
       $sleep_duration = $this->runGarbageCollection($sleep_duration);
       $this->sleep($sleep_duration);
     } while (!$this->shouldExit());
@@ -379,4 +384,70 @@ final class PhabricatorTriggerDaemon
     return false;
   }
 
+
+/* -(  Nuance Importers  )--------------------------------------------------- */
+
+
+  private function runNuanceImportCursors($duration) {
+    $run_until = (PhabricatorTime::getNow() + $duration);
+
+    do {
+      $more_data = $this->updateNuanceImportCursors();
+      if (!$more_data) {
+        break;
+      }
+    } while (PhabricatorTime::getNow() <= $run_until);
+
+    $remaining = max(0, $run_until - PhabricatorTime::getNow());
+
+    return $remaining;
+  }
+
+
+  private function updateNuanceImportCursors() {
+    $nuance_app = 'PhabricatorNuanceApplication';
+    if (!PhabricatorApplication::isClassInstalled($nuance_app)) {
+      return false;
+    }
+
+    // If we haven't loaded sources yet, load them first.
+    if (!$this->nuanceSources) {
+      $this->anyNuanceData = false;
+
+      $sources = id(new NuanceSourceQuery())
+        ->setViewer($this->getViewer())
+        ->withIsDisabled(false)
+        ->withHasImportCursors(true)
+        ->execute();
+      if (!$sources) {
+        return false;
+      }
+
+      $this->nuanceSources = array_reverse($sources);
+    }
+
+    // If we don't have any cursors, move to the next source and generate its
+    // cursors.
+    if (!$this->nuanceCursors) {
+      $source = array_pop($this->nuanceSources);
+      $cursors = $source->getImportCursors();
+      $this->nuanceCursors = array_reverse($cursors);
+    }
+
+    // Update the next cursor.
+    $cursor = array_pop($this->nuanceCursors);
+    if ($cursor) {
+      $more_data = $cursor->importFromSource();
+      if ($more_data) {
+        $this->anyNuanceData = true;
+      }
+    }
+
+    if (!$this->nuanceSources && !$this->nuanceCursors) {
+      return $this->anyNuanceData;
+    }
+
+    return true;
+  }
+
 }