Skeleton support for running global fulltext queries via the Ferret engine

Summary:
Ref T12819. Provides a Ferret-engine-based fulltext engine to ultimately replace the InnoDB fulltext engine.

This is still pretty basic (hard-coded and buggy) but technically sort of works.

To activate this, you must explicitly configure it, so it isn't visible to users yet.

Test Plan: Searched for objects with global fulltext search, got a mixture of matching revisions and tasks back.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T12819

Differential Revision: https://secure.phabricator.com/D18548

src/__phutil_library_map__.php

@@ -2838,6 +2838,7 @@ phutil_register_library_map(array(
     'PhabricatorFerretEngineTestCase' => 'applications/search/ferret/__tests__/PhabricatorFerretEngineTestCase.php',
     'PhabricatorFerretField' => 'applications/search/ferret/PhabricatorFerretField.php',
     'PhabricatorFerretFulltextEngineExtension' => 'applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php',
+    'PhabricatorFerretFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorFerretFulltextStorageEngine.php',
     'PhabricatorFerretInterface' => 'applications/search/ferret/PhabricatorFerretInterface.php',
     'PhabricatorFerretNgrams' => 'applications/search/ferret/PhabricatorFerretNgrams.php',
     'PhabricatorFerretSearchEngineExtension' => 'applications/search/engineextension/PhabricatorFerretSearchEngineExtension.php',
@@ -8173,6 +8174,7 @@ phutil_register_library_map(array(
     'PhabricatorFerretEngineTestCase' => 'PhabricatorTestCase',
     'PhabricatorFerretField' => 'PhabricatorSearchDAO',
     'PhabricatorFerretFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension',
+    'PhabricatorFerretFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
     'PhabricatorFerretNgrams' => 'PhabricatorSearchDAO',
     'PhabricatorFerretSearchEngineExtension' => 'PhabricatorSearchEngineExtension',
     'PhabricatorFile' => array(

src/applications/differential/query/DifferentialRevisionQuery.php

@@ -772,7 +772,7 @@ final class DifferentialRevisionQuery
         'column' => 'dateModified',
         'type' => 'int',
       ),
-    );
+    ) + parent::getOrderableColumns();
   }
 
   protected function getPagingValueMap($cursor, array $keys) {

src/applications/differential/search/DifferentialRevisionFerretEngine.php

@@ -15,6 +15,10 @@ final class DifferentialRevisionFerretEngine
     return new DifferentialRevisionFerretField();
   }
 
+  protected function newSearchEngine() {
+    return new DifferentialRevisionSearchEngine();
+  }
+
   protected function getFunctionMap() {
     $map = parent::getFunctionMap();
 

src/applications/maniphest/search/ManiphestTaskFerretEngine.php

@@ -15,6 +15,10 @@ final class ManiphestTaskFerretEngine
     return new ManiphestTaskFerretField();
   }
 
+  protected function newSearchEngine() {
+    return new ManiphestTaskSearchEngine();
+  }
+
   protected function getFunctionMap() {
     $map = parent::getFunctionMap();
 

src/applications/search/ferret/PhabricatorFerretEngine.php

@@ -5,6 +5,7 @@ abstract class PhabricatorFerretEngine extends Phobject {
   abstract public function newNgramsObject();
   abstract public function newDocumentObject();
   abstract public function newFieldObject();
+  abstract protected function newSearchEngine();
 
   public function getDefaultFunctionKey() {
     return 'all';
@@ -69,6 +70,60 @@ abstract class PhabricatorFerretEngine extends Phobject {
     return new PhutilSearchStemmer();
   }
 
+  public function newConfiguredFulltextQuery(
+    $object,
+    PhabricatorSavedQuery $query,
+    PhabricatorUser $viewer) {
+
+    $local_query = new PhabricatorSavedQuery();
+    $local_query->setParameter('query', $query->getParameter('query'));
+
+    // TODO: Modularize this piece.
+    $project_phids = $query->getParameter('projectPHIDs');
+    if ($project_phids) {
+      $local_query->setParameter('projectPHIDs', $project_phids);
+    }
+
+    $subscriber_phids = $query->getParameter('subscriberPHIDs');
+    if ($subscriber_phids) {
+      $local_query->setParameter('subscriberPHIDs', $subscriber_phids);
+    }
+
+    $author_phids = $query->getParameter('authorPHIDs');
+    if ($author_phids) {
+      $local_query->setParameter('authorPHIDs', $author_phids);
+    }
+
+    $owner_phids = $query->getParameter('ownerPHIDs');
+    if ($owner_phids) {
+      $local_query->setParameter('ownerPHIDs', $owner_phids);
+    }
+
+    $rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
+    $rel_closed = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED;
+
+    $statuses = $query->getParameter('statuses');
+    if ($statuses) {
+      $statuses = array_fuse($statuses);
+      if (count($statuses) == 1) {
+        if (isset($statuses[$rel_open])) {
+          $local_query->setParameter('statuses', array('open()'));
+        }
+        if (isset($statuses[$rel_closed])) {
+          $local_query->setParameter('statuses', array('closed()'));
+        }
+      }
+    }
+
+    $search_engine = $this->newSearchEngine()
+      ->setViewer($viewer);
+
+    $engine_query = $search_engine->buildQueryFromSavedQuery($local_query)
+      ->setViewer($viewer);
+
+    return $engine_query;
+  }
+
   public function tokenizeString($value) {
     $value = trim($value, ' ');
     $value = preg_split('/ +/', $value);

src/applications/search/fulltextstorage/PhabricatorFerretFulltextStorageEngine.php

@@ -0,0 +1,96 @@
+<?php
+
+final class PhabricatorFerretFulltextStorageEngine
+  extends PhabricatorFulltextStorageEngine {
+
+  private $fulltextTokens = array();
+  private $engineLimits;
+
+  public function getEngineIdentifier() {
+    return 'ferret';
+  }
+
+  public function getHostType() {
+    return new PhabricatorMySQLSearchHost($this);
+  }
+
+  public function reindexAbstractDocument(
+    PhabricatorSearchAbstractDocument $doc) {
+
+    // NOTE: The Ferret engine indexes are rebuilt by an extension rather than
+    // by the main fulltext engine, and are always built regardless of
+    // configuration.
+
+    return;
+  }
+
+  public function executeSearch(PhabricatorSavedQuery $query) {
+    $all_objects = id(new PhutilClassMapQuery())
+      ->setAncestorClass('PhabricatorFerretInterface')
+      ->execute();
+
+    $type_map = array();
+    foreach ($all_objects as $object) {
+      $phid_type = phid_get_type($object->generatePHID());
+
+      $type_map[$phid_type] = array(
+        'object' => $object,
+        'engine' => $object->newFerretEngine(),
+      );
+    }
+
+    $types = $query->getParameter('types');
+    if ($types) {
+      $type_map = array_select_keys($type_map, $types);
+    }
+
+    $offset = (int)$query->getParameter('offset', 0);
+    $limit  = (int)$query->getParameter('limit', 25);
+
+    $viewer = PhabricatorUser::getOmnipotentUser();
+
+    $type_results = array();
+    foreach ($type_map as $type => $spec) {
+      $engine = $spec['engine'];
+      $object = $spec['object'];
+
+      // NOTE: For now, it's okay to query with the omnipotent viewer here
+      // because we're just returning PHIDs which we'll filter later.
+
+      $type_query = $engine->newConfiguredFulltextQuery(
+        $object,
+        $query,
+        $viewer);
+
+      $type_query
+        ->setOrder('relevance')
+        ->setLimit($offset + $limit);
+
+      $results = $type_query->execute();
+      $results = mpull($results, null, 'getPHID');
+      $type_results[$type] = $results;
+    }
+
+    $list = array();
+    foreach ($type_results as $type => $results) {
+      $list += $results;
+    }
+
+    $result_slice = array_slice($list, $offset, $limit, true);
+    return array_keys($result_slice);
+  }
+
+  public function indexExists() {
+    return true;
+  }
+
+  public function getIndexStats() {
+    return false;
+  }
+
+  public function getFulltextTokens() {
+    return $this->fulltextTokens;
+  }
+
+
+}