Provide bin/nuance import and ngram indexes for sources

Summary:
Ref T10537. More infrastructure:

  - Put a `bin/nuance` in place with `bin/nuance import`. This has no useful behavior yet.
  - Allow sources to be searched by substring. This supports `bin/nuance import --source whatever` so you don't have to dig up PHIDs.

Test Plan:
  - Applied migrations.
  - Ran `bin/nuance import --source ...` (no meaningful effect, but works fine).
  - Searched for sources by substring in the UI.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10537

Differential Revision: https://secure.phabricator.com/D15436

bin/nuance

@@ -0,0 +1 @@
+../scripts/setup/manage_nuance.php

resources/sql/autopatches/20160308.nuance.03.sourcen.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_nuance.nuance_sourcename_ngrams (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  objectID INT UNSIGNED NOT NULL,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  KEY `key_object` (objectID),
+  KEY `key_ngram` (ngram, objectID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20160308.nuance.04.sourcei.php

@@ -0,0 +1,11 @@
+<?php
+
+$table = new NuanceSource();
+
+foreach (new LiskMigrationIterator($table) as $source) {
+  PhabricatorSearchWorker::queueDocumentForIndexing(
+    $source->getPHID(),
+    array(
+      'force' => true,
+    ));
+}

resources/sql/autopatches/20160308.nuance.05.sourcename.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_source
+  CHANGE name name VARCHAR(255) NOT NULL COLLATE {$COLLATE_SORT};

scripts/setup/manage_nuance.php

@@ -0,0 +1,21 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline(pht('manage Nuance'));
+$args->setSynopsis(<<<EOSYNOPSIS
+**nuance** __command__ [__options__]
+  Manage and debug Nuance.
+
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+
+$workflows = id(new PhutilClassMapQuery())
+  ->setAncestorClass('NuanceManagementWorkflow')
+  ->execute();
+$workflows[] = new PhutilHelpArgumentWorkflow();
+$args->parseWorkflows($workflows);

src/__phutil_library_map__.php

@@ -1434,6 +1434,8 @@ phutil_register_library_map(array(
     'NuanceItemTransactionComment' => 'applications/nuance/storage/NuanceItemTransactionComment.php',
     'NuanceItemTransactionQuery' => 'applications/nuance/query/NuanceItemTransactionQuery.php',
     'NuanceItemViewController' => 'applications/nuance/controller/NuanceItemViewController.php',
+    'NuanceManagementImportWorkflow' => 'applications/nuance/management/NuanceManagementImportWorkflow.php',
+    'NuanceManagementWorkflow' => 'applications/nuance/management/NuanceManagementWorkflow.php',
     'NuancePhabricatorFormSourceDefinition' => 'applications/nuance/source/NuancePhabricatorFormSourceDefinition.php',
     'NuanceQuery' => 'applications/nuance/query/NuanceQuery.php',
     'NuanceQueue' => 'applications/nuance/storage/NuanceQueue.php',
@@ -1473,6 +1475,7 @@ phutil_register_library_map(array(
     'NuanceSourceEditor' => 'applications/nuance/editor/NuanceSourceEditor.php',
     'NuanceSourceListController' => 'applications/nuance/controller/NuanceSourceListController.php',
     'NuanceSourceManageCapability' => 'applications/nuance/capability/NuanceSourceManageCapability.php',
+    'NuanceSourceNameNgrams' => 'applications/nuance/storage/NuanceSourceNameNgrams.php',
     'NuanceSourcePHIDType' => 'applications/nuance/phid/NuanceSourcePHIDType.php',
     'NuanceSourceQuery' => 'applications/nuance/query/NuanceSourceQuery.php',
     'NuanceSourceSearchEngine' => 'applications/nuance/query/NuanceSourceSearchEngine.php',
@@ -5682,6 +5685,8 @@ phutil_register_library_map(array(
     'NuanceItemTransactionComment' => 'PhabricatorApplicationTransactionComment',
     'NuanceItemTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
     'NuanceItemViewController' => 'NuanceController',
+    'NuanceManagementImportWorkflow' => 'NuanceManagementWorkflow',
+    'NuanceManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'NuancePhabricatorFormSourceDefinition' => 'NuanceSourceDefinition',
     'NuanceQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'NuanceQueue' => array(
@@ -5721,6 +5726,7 @@ phutil_register_library_map(array(
       'NuanceDAO',
       'PhabricatorApplicationTransactionInterface',
       'PhabricatorPolicyInterface',
+      'PhabricatorNgramsInterface',
     ),
     'NuanceSourceActionController' => 'NuanceController',
     'NuanceSourceController' => 'NuanceController',
@@ -5733,6 +5739,7 @@ phutil_register_library_map(array(
     'NuanceSourceEditor' => 'PhabricatorApplicationTransactionEditor',
     'NuanceSourceListController' => 'NuanceSourceController',
     'NuanceSourceManageCapability' => 'PhabricatorPolicyCapability',
+    'NuanceSourceNameNgrams' => 'PhabricatorSearchNgrams',
     'NuanceSourcePHIDType' => 'PhabricatorPHIDType',
     'NuanceSourceQuery' => 'NuanceQuery',
     'NuanceSourceSearchEngine' => 'PhabricatorApplicationSearchEngine',

src/applications/almanac/storage/AlmanacDevice.php

@@ -246,7 +246,7 @@ final class AlmanacDevice
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/almanac/storage/AlmanacNamespace.php

@@ -210,7 +210,7 @@ final class AlmanacNamespace
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/almanac/storage/AlmanacNetwork.php

@@ -116,7 +116,7 @@ final class AlmanacNetwork
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/almanac/storage/AlmanacService.php

@@ -251,7 +251,7 @@ final class AlmanacService
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/drydock/storage/DrydockBlueprint.php

@@ -350,7 +350,7 @@ final class DrydockBlueprint extends DrydockDAO
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php

@@ -197,7 +197,7 @@ final class HarbormasterBuildPlan extends HarbormasterDAO
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/applications/nuance/editor/NuanceSourceEditor.php

@@ -11,6 +11,10 @@ final class NuanceSourceEditor
     return pht('Nuance Sources');
   }
 
+  protected function supportsSearch() {
+    return true;
+  }
+
   public function getTransactionTypes() {
     $types = parent::getTransactionTypes();
 

src/applications/nuance/management/NuanceManagementImportWorkflow.php

@@ -0,0 +1,50 @@
+<?php
+
+final class NuanceManagementImportWorkflow
+  extends NuanceManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('import')
+      ->setExamples('**import** [__options__]')
+      ->setSynopsis(pht('Import data from a source.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'source',
+            'param' => 'source',
+            'help' => pht('Choose which source to import.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $source = $this->loadSource($args, 'source');
+
+    $definition = $source->getDefinition()
+      ->setViewer($this->getViewer())
+      ->setSource($source);
+
+    if (!$definition->hasImportCursors()) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'This source ("%s") does not expose import cursors.',
+          $source->getName()));
+    }
+
+    $cursors = $definition->getImportCursors();
+    if (!$cursors) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'This source ("%s") does not have any import cursors.',
+          $source->getName()));
+    }
+
+    echo tsprintf(
+      "%s\n",
+      pht('OK, but actual importing is not implemented yet.'));
+
+    return 0;
+  }
+
+}

src/applications/nuance/management/NuanceManagementWorkflow.php

@@ -0,0 +1,67 @@
+<?php
+
+abstract class NuanceManagementWorkflow
+  extends PhabricatorManagementWorkflow {
+
+  protected function loadSource(PhutilArgumentParser $argv, $key) {
+    $source = $argv->getArg($key);
+    if (!strlen($source)) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Specify a source with %s.',
+          '--'.$key));
+    }
+
+    $query = id(new NuanceSourceQuery())
+      ->setViewer($this->getViewer())
+      ->setRaisePolicyExceptions(true);
+
+    $type_unknown = PhabricatorPHIDConstants::PHID_TYPE_UNKNOWN;
+
+    if (ctype_digit($source)) {
+      $kind = 'id';
+      $query->withIDs(array($source));
+    } else if (phid_get_type($source) !== $type_unknown) {
+      $kind = 'phid';
+      $query->withPHIDs($source);
+    } else {
+      $kind = 'name';
+      $query->withNameNgrams($source);
+    }
+
+    $sources = $query->execute();
+
+    if (!$sources) {
+      switch ($kind) {
+        case 'id':
+          $message = pht(
+            'No source exists with ID "%s".',
+            $source);
+          break;
+        case 'phid':
+          $message = pht(
+            'No source exists with PHID "%s".',
+            $source);
+          break;
+        default:
+          $message = pht(
+            'No source exists with a name matching "%s".',
+            $source);
+          break;
+      }
+
+      throw new PhutilArgumentUsageException($message);
+    } else if (count($sources) > 1) {
+      $message = pht(
+        'More than one source matches "%s". Choose a narrower query, or '.
+        'use an ID or PHID to select a source. Matching sources: %s.',
+        $source,
+        implode(', ', mpull($sources, 'getName')));
+
+      throw new PhutilArgumentUsageException($message);
+    }
+
+    return head($sources);
+  }
+
+}

src/applications/nuance/query/NuanceSourceQuery.php

@@ -34,10 +34,20 @@ final class NuanceSourceQuery
     return $this;
   }
 
+  public function withNameNgrams($ngrams) {
+    return $this->withNgramsConstraint(
+      new NuanceSourceNameNgrams(),
+      $ngrams);
+  }
+
   public function newResultObject() {
     return new NuanceSource();
   }
 
+  protected function getPrimaryTableAlias() {
+    return 'source';
+  }
+
   protected function loadPage() {
     return $this->loadStandardPage($this->newResultObject());
   }
@@ -65,28 +75,28 @@ final class NuanceSourceQuery
     if ($this->types !== null) {
       $where[] = qsprintf(
         $conn,
-        'type IN (%Ls)',
+        'source.type IN (%Ls)',
         $this->types);
     }
 
     if ($this->ids !== null) {
       $where[] = qsprintf(
         $conn,
-        'id IN (%Ld)',
+        'source.id IN (%Ld)',
         $this->ids);
     }
 
     if ($this->phids !== null) {
       $where[] = qsprintf(
         $conn,
-        'phid IN (%Ls)',
+        'source.phid IN (%Ls)',
         $this->phids);
     }
 
     if ($this->isDisabled !== null) {
       $where[] = qsprintf(
         $conn,
-        'isDisabled = %d',
+        'source.isDisabled = %d',
         (int)$this->isDisabled);
     }
 
@@ -106,7 +116,7 @@ final class NuanceSourceQuery
         } else {
           $where[] = qsprintf(
             $conn,
-            'type IN (%Ls)',
+            'source.type IN (%Ls)',
             $cursor_types);
         }
       } else {
@@ -115,7 +125,7 @@ final class NuanceSourceQuery
         } else {
           $where[] = qsprintf(
             $conn,
-            'type NOT IN (%Ls)',
+            'source.type NOT IN (%Ls)',
             $cursor_types);
         }
       }

src/applications/nuance/query/NuanceSourceSearchEngine.php

@@ -18,11 +18,20 @@ final class NuanceSourceSearchEngine
   protected function buildQueryFromParameters(array $map) {
     $query = $this->newQuery();
 
+    if ($map['match'] !== null) {
+      $query->withNameNgrams($map['match']);
+    }
+
     return $query;
   }
 
   protected function buildCustomSearchFields() {
-    return array();
+    return array(
+      id(new PhabricatorSearchTextField())
+        ->setLabel(pht('Name Contains'))
+        ->setKey('match')
+        ->setDescription(pht('Search for sources by name substring.')),
+    );
   }
 
   protected function getURI($path) {

src/applications/nuance/storage/NuanceSource.php

@@ -3,7 +3,8 @@
 final class NuanceSource extends NuanceDAO
   implements
     PhabricatorApplicationTransactionInterface,
-    PhabricatorPolicyInterface {
+    PhabricatorPolicyInterface,
+    PhabricatorNgramsInterface {
 
   protected $name;
   protected $type;
@@ -23,7 +24,7 @@ final class NuanceSource extends NuanceDAO
         'data' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255?',
+        'name' => 'sort255',
         'type' => 'text32',
         'mailKey' => 'bytes20',
         'isDisabled' => 'bool',
@@ -132,4 +133,15 @@ final class NuanceSource extends NuanceDAO
     return null;
   }
 
+
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
+
+
+  public function newNgrams() {
+    return array(
+      id(new NuanceSourceNameNgrams())
+        ->setValue($this->getName()),
+    );
+  }
+
 }

src/applications/nuance/storage/NuanceSourceNameNgrams.php

@@ -0,0 +1,18 @@
+<?php
+
+final class NuanceSourceNameNgrams
+  extends PhabricatorSearchNgrams {
+
+  public function getNgramKey() {
+    return 'sourcename';
+  }
+
+  public function getColumnName() {
+    return 'name';
+  }
+
+  public function getApplicationName() {
+    return 'nuance';
+  }
+
+}

src/applications/owners/storage/PhabricatorOwnersPackage.php

@@ -451,7 +451,7 @@ final class PhabricatorOwnersPackage
   }
 
 
-/* -(  PhabricatorNgramInterface  )------------------------------------------ */
+/* -(  PhabricatorNgramsInterface  )----------------------------------------- */
 
 
   public function newNgrams() {

src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php

@@ -430,7 +430,12 @@ final class PhabricatorTriggerDaemon
     // cursors.
     if (!$this->nuanceCursors) {
       $source = array_pop($this->nuanceSources);
-      $cursors = $source->getImportCursors();
+
+      $definition = $source->getDefinition()
+        ->setViewer($this->getViewer())
+        ->setSource($source);
+
+      $cursors = $definition->getImportCursors();
       $this->nuanceCursors = array_reverse($cursors);
     }