Give projects a proper on-demand datasource

Summary:
Fixes T5614. Ref T4420. Other than the "users" datasource and a couple of others, many datasources ignore what the user typed and just return all results, then rely on the client to filter them.

This works fine for rarely used ("legalpad documents") or always small ("task priorities", "applications") datasets, but is something we should graudally move away from as datasets get larger.

Add a token table to projects, populate it, and use it to drive the datasource query. Additionally, expose it on the applicationsearch UI.

Test Plan:
  - Ran migration.
  - Manually checked the table.
  - Searched for projects by name from ApplicationSearch.
  - Searched for projects by name from typeahead.
  - Manually checked the typeahead response.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T5614, T4420

Differential Revision: https://secure.phabricator.com/D9896

resources/sql/autopatches/20140711.pnames.1.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_project.project_datasourcetoken (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  projectID INT UNSIGNED NOT NULL,
+  token VARCHAR(128) NOT NULL COLLATE utf8_general_ci,
+  UNIQUE KEY (token, projectID),
+  KEY (projectID)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

resources/sql/autopatches/20140711.pnames.2.php

@@ -0,0 +1,11 @@
+<?php
+
+echo "Updating project datasource tokens...\n";
+
+foreach (new LiskMigrationIterator(new PhabricatorProject()) as $project) {
+  $name = $project->getName();
+  echo "Updating project '{$name}'...\n";
+  $project->updateDatasourceTokens();
+}
+
+echo "Done.\n";

src/applications/people/storage/PhabricatorUser.php

@@ -459,32 +459,18 @@ final class PhabricatorUser
     return $this;
   }
 
-  private static function tokenizeName($name) {
-    if (function_exists('mb_strtolower')) {
-      $name = mb_strtolower($name, 'UTF-8');
-    } else {
-      $name = strtolower($name);
-    }
-    $name = trim($name);
-    if (!strlen($name)) {
-      return array();
-    }
-    return preg_split('/\s+/', $name);
-  }
-
   /**
    * Populate the nametoken table, which used to fetch typeahead results. When
    * a user types "linc", we want to match "Abraham Lincoln" from on-demand
    * typeahead sources. To do this, we need a separate table of name fragments.
    */
   public function updateNameTokens() {
-    $tokens = array_merge(
-      self::tokenizeName($this->getRealName()),
-      self::tokenizeName($this->getUserName()));
-    $tokens = array_unique($tokens);
     $table  = self::NAMETOKEN_TABLE;
     $conn_w = $this->establishConnection('w');
 
+    $tokens = PhabricatorTypeaheadDatasource::tokenizeString(
+      $this->getUserName().' '.$this->getRealName());
+
     $sql = array();
     foreach ($tokens as $token) {
       $sql[] = qsprintf(

src/applications/project/editor/PhabricatorProjectTransactionEditor.php

@@ -125,6 +125,8 @@ final class PhabricatorProjectTransactionEditor
           ->setProjectPHID($object->getPHID())
           ->save();
 
+        $object->updateDatasourceTokens();
+
         // TODO -- delete all of the below once we sever automagical project
         // to phriction stuff
         if ($xaction->getOldValue() === null) {
@@ -182,6 +184,9 @@ final class PhabricatorProjectTransactionEditor
             $rem_slug->delete();
           }
         }
+
+        $object->updateDatasourceTokens();
+
         return;
       case PhabricatorTransactions::TYPE_VIEW_POLICY:
       case PhabricatorTransactions::TYPE_EDIT_POLICY:

src/applications/project/query/PhabricatorProjectQuery.php

@@ -9,6 +9,7 @@ final class PhabricatorProjectQuery
   private $slugs;
   private $phrictionSlugs;
   private $names;
+  private $datasourceQuery;
 
   private $status       = 'status-any';
   const STATUS_ANY      = 'status-any';
@@ -57,6 +58,11 @@ final class PhabricatorProjectQuery
     return $this;
   }
 
+  public function withDatasourceQuery($string) {
+    $this->datasourceQuery = $string;
+    return $this;
+  }
+
   public function needMembers($need_members) {
     $this->needMembers = $need_members;
     return $this;
@@ -286,7 +292,7 @@ final class PhabricatorProjectQuery
   }
 
   private function buildGroupClause($conn_r) {
-    if ($this->memberPHIDs) {
+    if ($this->memberPHIDs || $this->datasourceQuery) {
       return 'GROUP BY p.id';
     } else {
       return $this->buildApplicationSearchGroupClause($conn_r);
@@ -296,7 +302,7 @@ final class PhabricatorProjectQuery
   private function buildJoinClause($conn_r) {
     $joins = array();
 
-    if (!$this->needMembers) {
+    if (!$this->needMembers !== null) {
       $joins[] = qsprintf(
         $conn_r,
         'LEFT JOIN %T vm ON vm.src = p.phid AND vm.type = %d AND vm.dst = %s',
@@ -305,7 +311,7 @@ final class PhabricatorProjectQuery
         $this->getViewer()->getPHID());
     }
 
-    if ($this->memberPHIDs) {
+    if ($this->memberPHIDs !== null) {
       $joins[] = qsprintf(
         $conn_r,
         'JOIN %T e ON e.src = p.phid AND e.type = %d',
@@ -313,13 +319,32 @@ final class PhabricatorProjectQuery
         PhabricatorEdgeConfig::TYPE_PROJ_MEMBER);
     }
 
-    if ($this->slugs) {
+    if ($this->slugs !== null) {
       $joins[] = qsprintf(
         $conn_r,
         'JOIN %T slug on slug.projectPHID = p.phid',
         id(new PhabricatorProjectSlug())->getTableName());
     }
 
+    if ($this->datasourceQuery !== null) {
+      $tokens = PhabricatorTypeaheadDatasource::tokenizeString(
+        $this->datasourceQuery);
+      if (!$tokens) {
+        throw new PhabricatorEmptyQueryException();
+      }
+
+      $likes = array();
+      foreach ($tokens as $token) {
+        $likes[] = qsprintf($conn_r, 'token.token LIKE %>', $token);
+      }
+
+      $joins[] = qsprintf(
+        $conn_r,
+        'JOIN %T token ON token.projectID = p.id AND (%Q)',
+        PhabricatorProject::TABLE_DATASOURCE_TOKEN,
+        '('.implode(') OR (', $likes).')');
+    }
+
     $joins[] = $this->buildApplicationSearchJoinClause($conn_r);
 
     return implode(' ', $joins);

src/applications/project/query/PhabricatorProjectSearchEngine.php

@@ -21,7 +21,9 @@ final class PhabricatorProjectSearchEngine
     $saved->setParameter(
       'memberPHIDs',
       $this->readUsersFromRequest($request, 'members'));
+
     $saved->setParameter('status', $request->getStr('status'));
+    $saved->setParameter('name', $request->getStr('name'));
 
     $this->readCustomFieldsFromRequest($request, $saved);
 
@@ -43,6 +45,11 @@ final class PhabricatorProjectSearchEngine
       $query->withStatus($status);
     }
 
+    $name = $saved->getParameter('name');
+    if (strlen($name)) {
+      $query->withDatasourceQuery($name);
+    }
+
     $this->applyCustomFieldsToQuery($query, $saved);
 
     return $query;
@@ -59,8 +66,14 @@ final class PhabricatorProjectSearchEngine
       ->execute();
 
     $status = $saved->getParameter('status');
+    $name = $saved->getParameter('name');
 
     $form
+      ->appendChild(
+        id(new AphrontFormTextControl())
+          ->setName('name')
+          ->setLabel(pht('Name'))
+          ->setValue($name))
       ->appendChild(
         id(new AphrontFormTokenizerControl())
           ->setDatasource(new PhabricatorPeopleDatasource())

src/applications/project/storage/PhabricatorProject.php

@@ -32,6 +32,8 @@ final class PhabricatorProject extends PhabricatorProjectDAO
   const DEFAULT_ICON = 'fa-briefcase';
   const DEFAULT_COLOR = 'blue';
 
+  const TABLE_DATASOURCE_TOKEN = 'project_datasourcetoken';
+
   public static function initializeNewProject(PhabricatorUser $actor) {
     return id(new PhabricatorProject())
       ->setName('')
@@ -219,6 +221,53 @@ final class PhabricatorProject extends PhabricatorProjectDAO
     return $this->color;
   }
 
+  public function save() {
+    $this->openTransaction();
+      $result = parent::save();
+      $this->updateDatasourceTokens();
+    $this->saveTransaction();
+
+    return $result;
+  }
+
+  public function updateDatasourceTokens() {
+    $table = self::TABLE_DATASOURCE_TOKEN;
+    $conn_w = $this->establishConnection('w');
+    $id = $this->getID();
+
+    $slugs = queryfx_all(
+      $conn_w,
+      'SELECT * FROM %T WHERE projectPHID = %s',
+      id(new PhabricatorProjectSlug())->getTableName(),
+      $this->getPHID());
+
+    $all_strings = ipull($slugs, 'slug');
+    $all_strings[] = $this->getName();
+    $all_strings = implode(' ', $all_strings);
+
+    $tokens = PhabricatorTypeaheadDatasource::tokenizeString($all_strings);
+
+    $sql = array();
+    foreach ($tokens as $token) {
+      $sql[] = qsprintf($conn_w, '(%d, %s)', $id, $token);
+    }
+
+    $this->openTransaction();
+      queryfx(
+        $conn_w,
+        'DELETE FROM %T WHERE projectID = %d',
+        $table,
+        $id);
+
+      foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+        queryfx(
+          $conn_w,
+          'INSERT INTO %T (projectID, token) VALUES %Q',
+          $table,
+          $chunk);
+      }
+    $this->saveTransaction();
+  }
 
 
 /* -(  PhabricatorSubscribableInterface  )----------------------------------- */

src/applications/project/typeahead/PhabricatorProjectDatasource.php

@@ -13,22 +13,37 @@ final class PhabricatorProjectDatasource
 
   public function loadResults() {
     $viewer = $this->getViewer();
+
     $raw_query = $this->getRawQuery();
 
-    $results = array();
+    // Allow users to type "#qa" or "qa" to find "Quality Assurance".
+    $raw_query = ltrim($raw_query, '#');
+
+    if (!strlen($raw_query)) {
+      return array();
+    }
 
     $projs = id(new PhabricatorProjectQuery())
       ->setViewer($viewer)
       ->needImages(true)
+      ->needSlugs(true)
+      ->withDatasourceQuery($raw_query)
       ->execute();
+
+    $results = array();
     foreach ($projs as $proj) {
       $closed = null;
       if ($proj->isArchived()) {
         $closed = pht('Archived');
       }
 
+      $all_strings = mpull($proj->getSlugs(), 'getSlug');
+      $all_strings[] = $proj->getName();
+      $all_strings = implode(' ', $all_strings);
+
       $proj_result = id(new PhabricatorTypeaheadResult())
-        ->setName($proj->getName())
+        ->setName($all_strings)
+        ->setDisplayName($proj->getName())
         ->setDisplayType('Project')
         ->setURI('/tag/'.$proj->getPrimarySlug().'/')
         ->setPHID($proj->getPHID())

src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php

@@ -51,4 +51,15 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject {
   abstract public function getDatasourceApplicationClass();
   abstract public function loadResults();
 
+  public static function tokenizeString($string) {
+    $string = phutil_utf8_strtolower($string);
+    $string = trim($string);
+    if (!strlen($string)) {
+      return array();
+    }
+
+    $tokens = preg_split('/\s+/', $string);
+    return array_unique($tokens);
+  }
+
 }