Make Portals indexable with Ferret

Summary:
Ref T13275. Add portals to the search index so that:

  - they show up in fulltext global search; and
  - the typeahead actually uses an index.

Also make them taggable with projects as an organizational aid.

Test Plan: Indexed portals with `bin/serach index`, searched for a portal with "Query", with fulltext search in main menu, with typehead on "Install Dashboard...", changed the name of a portal and searched again to check that the index updates properly.

Reviewers: amckinley

Reviewed By: amckinley

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13275

Differential Revision: https://secure.phabricator.com/D20389

resources/sql/autopatches/20190410.portals.01.ferret.doc.sql

@@ -0,0 +1,9 @@
+CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fdocument (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  objectPHID VARBINARY(64) NOT NULL,
+  isClosed BOOL NOT NULL,
+  authorPHID VARBINARY(64),
+  ownerPHID VARBINARY(64),
+  epochCreated INT UNSIGNED NOT NULL,
+  epochModified INT UNSIGNED NOT NULL
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20190410.portals.02.ferret.field.sql

@@ -0,0 +1,8 @@
+CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_ffield (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  documentID INT UNSIGNED NOT NULL,
+  fieldKey VARCHAR(4) NOT NULL COLLATE {$COLLATE_TEXT},
+  rawCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT},
+  termCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT},
+  normalCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20190410.portals.03.ferret.ngrams.sql

@@ -0,0 +1,5 @@
+CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fngrams (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  documentID INT UNSIGNED NOT NULL,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20190410.portals.04.ferret.cngrams.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -2957,6 +2957,8 @@ phutil_register_library_map(array(
     'PhabricatorDashboardPortalEditController' => 'applications/dashboard/controller/portal/PhabricatorDashboardPortalEditController.php',
     'PhabricatorDashboardPortalEditEngine' => 'applications/dashboard/editor/PhabricatorDashboardPortalEditEngine.php',
     'PhabricatorDashboardPortalEditor' => 'applications/dashboard/editor/PhabricatorDashboardPortalEditor.php',
+    'PhabricatorDashboardPortalFerretEngine' => 'applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php',
+    'PhabricatorDashboardPortalFulltextEngine' => 'applications/dashboard/engine/PhabricatorDashboardPortalFulltextEngine.php',
     'PhabricatorDashboardPortalInstallWorkflow' => 'applications/dashboard/install/PhabricatorDashboardPortalInstallWorkflow.php',
     'PhabricatorDashboardPortalListController' => 'applications/dashboard/controller/portal/PhabricatorDashboardPortalListController.php',
     'PhabricatorDashboardPortalMenuItem' => 'applications/dashboard/menuitem/PhabricatorDashboardPortalMenuItem.php',
@@ -8938,6 +8940,9 @@ phutil_register_library_map(array(
       'PhabricatorApplicationTransactionInterface',
       'PhabricatorPolicyInterface',
       'PhabricatorDestructibleInterface',
+      'PhabricatorProjectInterface',
+      'PhabricatorFulltextInterface',
+      'PhabricatorFerretInterface',
     ),
     'PhabricatorDashboardPortalController' => 'PhabricatorDashboardController',
     'PhabricatorDashboardPortalDatasource' => 'PhabricatorTypeaheadDatasource',
@@ -8945,6 +8950,8 @@ phutil_register_library_map(array(
     'PhabricatorDashboardPortalEditController' => 'PhabricatorDashboardPortalController',
     'PhabricatorDashboardPortalEditEngine' => 'PhabricatorEditEngine',
     'PhabricatorDashboardPortalEditor' => 'PhabricatorApplicationTransactionEditor',
+    'PhabricatorDashboardPortalFerretEngine' => 'PhabricatorFerretEngine',
+    'PhabricatorDashboardPortalFulltextEngine' => 'PhabricatorFulltextEngine',
     'PhabricatorDashboardPortalInstallWorkflow' => 'PhabricatorDashboardObjectInstallWorkflow',
     'PhabricatorDashboardPortalListController' => 'PhabricatorDashboardPortalController',
     'PhabricatorDashboardPortalMenuItem' => 'PhabricatorProfileMenuItem',

src/applications/dashboard/editor/PhabricatorDashboardPortalEditor.php

@@ -28,4 +28,8 @@ final class PhabricatorDashboardPortalEditor
     return $types;
   }
 
+  protected function supportsSearch() {
+    return true;
+  }
+
 }

src/applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php

@@ -0,0 +1,18 @@
+<?php
+
+final class PhabricatorDashboardPortalFerretEngine
+  extends PhabricatorFerretEngine {
+
+  public function getApplicationName() {
+    return 'dashboard';
+  }
+
+  public function getScopeName() {
+    return 'portal';
+  }
+
+  public function newSearchEngine() {
+    return new PhabricatorDashboardPortalSearchEngine();
+  }
+
+}

src/applications/dashboard/engine/PhabricatorDashboardPortalFulltextEngine.php

@@ -0,0 +1,23 @@
+<?php
+
+final class PhabricatorDashboardPortalFulltextEngine
+  extends PhabricatorFulltextEngine {
+
+  protected function buildAbstractDocument(
+    PhabricatorSearchAbstractDocument $document,
+    $object) {
+
+    $portal = $object;
+
+    $document->setDocumentTitle($portal->getName());
+
+    $document->addRelationship(
+      $portal->isArchived()
+        ? PhabricatorSearchRelationship::RELATIONSHIP_CLOSED
+        : PhabricatorSearchRelationship::RELATIONSHIP_OPEN,
+      $portal->getPHID(),
+      PhabricatorDashboardPortalPHIDType::TYPECONST,
+      PhabricatorTime::getNow());
+  }
+
+}

src/applications/dashboard/query/PhabricatorDashboardPortalQuery.php

@@ -36,21 +36,21 @@ final class PhabricatorDashboardPortalQuery
     if ($this->ids !== null) {
       $where[] = qsprintf(
         $conn,
-        'id IN (%Ld)',
+        'portal.id IN (%Ld)',
         $this->ids);
     }
 
     if ($this->phids !== null) {
       $where[] = qsprintf(
         $conn,
-        'phid IN (%Ls)',
+        'portal.phid IN (%Ls)',
         $this->phids);
     }
 
     if ($this->statuses !== null) {
       $where[] = qsprintf(
         $conn,
-        'status IN (%Ls)',
+        'portal.status IN (%Ls)',
         $this->statuses);
     }
 
@@ -61,4 +61,8 @@ final class PhabricatorDashboardPortalQuery
     return 'PhabricatorDashboardApplication';
   }
 
+  protected function getPrimaryTableAlias() {
+    return 'portal';
+  }
+
 }

src/applications/dashboard/storage/PhabricatorDashboardPortal.php

@@ -5,7 +5,10 @@ final class PhabricatorDashboardPortal
   implements
     PhabricatorApplicationTransactionInterface,
     PhabricatorPolicyInterface,
-    PhabricatorDestructibleInterface {
+    PhabricatorDestructibleInterface,
+    PhabricatorProjectInterface,
+    PhabricatorFulltextInterface,
+    PhabricatorFerretInterface {
 
   protected $name;
   protected $viewPolicy;
@@ -55,6 +58,11 @@ final class PhabricatorDashboardPortal
     return '/portal/view/'.$this->getID().'/';
   }
 
+  public function isArchived() {
+    $status_archived = PhabricatorDashboardPortalStatus::STATUS_ARCHIVED;
+    return ($this->getStatus() === $status_archived);
+  }
+
 
 /* -(  PhabricatorApplicationTransactionInterface  )------------------------- */
 
@@ -100,5 +108,16 @@ final class PhabricatorDashboardPortal
     $this->delete();
   }
 
+/* -(  PhabricatorFulltextInterface  )--------------------------------------- */
+
+  public function newFulltextEngine() {
+    return new PhabricatorDashboardPortalFulltextEngine();
+  }
+
+/* -(  PhabricatorFerretInterface  )----------------------------------------- */
+
+  public function newFerretEngine() {
+    return new PhabricatorDashboardPortalFerretEngine();
+  }
 
 }

src/applications/dashboard/typeahead/PhabricatorDashboardPortalDatasource.php

@@ -27,7 +27,11 @@ final class PhabricatorDashboardPortalDatasource
   public function buildResults() {
     $query = new PhabricatorDashboardPortalQuery();
 
-    // TODO: Actually query by name so this scales past 100 portals.
+    $this->applyFerretConstraints(
+      $query,
+      id(new PhabricatorDashboardPortal())->newFerretEngine(),
+      'title',
+      $this->getRawQuery());
 
     $portals = $this->executeQuery($query);
 

src/applications/phriction/typeahead/PhrictionDocumentDatasource.php

@@ -18,41 +18,17 @@ final class PhrictionDocumentDatasource
   public function loadResults() {
     $viewer = $this->getViewer();
 
-    $raw_query = $this->getRawQuery();
+    $query = id(new PhrictionDocumentQuery())
-
-    $engine = id(new PhrictionDocument())
-      ->newFerretEngine();
-
-    $compiler = id(new PhutilSearchQueryCompiler())
-      ->setEnableFunctions(true);
-
-    $raw_tokens = $compiler->newTokens($raw_query);
-
-    $fulltext_tokens = array();
-    foreach ($raw_tokens as $raw_token) {
-
-      // This is a little hacky and could maybe be cleaner. We're treating
-      // every search term as though the user had entered "title:dog" insead
-      // of "dog".
-
-      $alternate_token = PhutilSearchQueryToken::newFromDictionary(
-        array(
-          'quoted' => $raw_token->isQuoted(),
-          'value' => $raw_token->getValue(),
-          'operator' => PhutilSearchQueryCompiler::OPERATOR_SUBSTRING,
-          'function' => 'title',
-        ));
-
-      $fulltext_token = id(new PhabricatorFulltextToken())
-        ->setToken($alternate_token);
-      $fulltext_tokens[] = $fulltext_token;
-    }
-
-    $documents = id(new PhrictionDocumentQuery())
       ->setViewer($viewer)
-      ->withFerretConstraint($engine, $fulltext_tokens)
+      ->needContent(true);
-      ->needContent(true)
+
-      ->execute();
+    $this->applyFerretConstraints(
+      $query,
+      id(new PhrictionDocument())->newFerretEngine(),
+      'title',
+      $this->getRawQuery());
+
+    $documents = $query->execute();
 
     $results = array();
     foreach ($documents as $document) {

src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php

@@ -604,4 +604,38 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject {
     return mpull($tokens, 'getWireFormat', 'getPHID');
   }
 
+  final protected function applyFerretConstraints(
+    PhabricatorCursorPagedPolicyAwareQuery $query,
+    PhabricatorFerretEngine $engine,
+    $ferret_function,
+    $raw_query) {
+
+    $compiler = id(new PhutilSearchQueryCompiler())
+      ->setEnableFunctions(true);
+
+    $raw_tokens = $compiler->newTokens($raw_query);
+
+    $fulltext_tokens = array();
+    foreach ($raw_tokens as $raw_token) {
+      // This is a little hacky and could maybe be cleaner. We're treating
+      // every search term as though the user had entered "title:dog" instead
+      // of "dog".
+
+      $alternate_token = PhutilSearchQueryToken::newFromDictionary(
+        array(
+          'quoted' => $raw_token->isQuoted(),
+          'value' => $raw_token->getValue(),
+          'operator' => PhutilSearchQueryCompiler::OPERATOR_SUBSTRING,
+          'function' => $ferret_function,
+        ));
+
+      $fulltext_token = id(new PhabricatorFulltextToken())
+        ->setToken($alternate_token);
+      $fulltext_tokens[] = $fulltext_token;
+    }
+
+    $query->withFerretConstraint($engine, $fulltext_tokens);
+  }
+
+
 }