Add a name token table so on-demand typeaheads can match last names

Summary: See T585. We currently don't match middle/last/nth names in on-demand
tokenizers. Build a table so we can match them.

Test Plan:
Ran upgrade script, verified table looks sensible. Searched for "priestley" in a
tokenizer, got a bunch of test account hits.

  mysql> select * from user_nametoken;
  +-------------------+--------+
  | token             | userID |
  +-------------------+--------+
  | evan              |      1 |
  | priestley         |      1 |
  | epriestley        |      1 |
  | epriestley2       |      2 |
  | ducks             |      4 |
  | epriestley3       |      4 |
  | asdf              |      6 |
  | epriestley99      |      6 |
  ...

Reviewers: bh, nh, jungejason, tuomaspelkonen, aran

Reviewed By: aran

CC: aran

Differential Revision: 1034

resources/sql/patches/078.nametoken.sql

@@ -0,0 +1,6 @@
+CREATE TABLE phabricator_user.user_nametoken (
+  token VARCHAR(255) NOT NULL,
+  userID INT UNSIGNED NOT NULL,
+  KEY (token),
+  key (userID)
+) ENGINE=InnoDB;

resources/sql/patches/079.nametokenindex.php

@@ -0,0 +1,30 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$conn = $schema_conn;
+
+echo "Indexing username tokens for typeaheads...\n";
+
+$users = id(new PhabricatorUser())->loadAll();
+echo count($users)." users to index";
+foreach ($users as $user) {
+  $user->updateNameTokens();
+  echo ".";
+}
+
+echo "\nDone.\n";

src/applications/people/storage/user/PhabricatorUser.php

@@ -19,6 +19,7 @@
 class PhabricatorUser extends PhabricatorUserDAO {
 
   const SESSION_TABLE = 'phabricator_session';
+  const NAMETOKEN_TABLE = 'user_nametoken';
 
   protected $phid;
   protected $userName;
@@ -91,6 +92,7 @@ class PhabricatorUser extends PhabricatorUserDAO {
     }
     $result = parent::save();
 
+    $this->updateNameTokens();
     PhabricatorSearchUserIndexer::indexUser($this);
 
     return $result;
@@ -349,4 +351,53 @@ class PhabricatorUser extends PhabricatorUserDAO {
     return $preferences;
   }
 
+  private static function tokenizeName($name) {
+    if (function_exists('mb_strtolower')) {
+      $name = mb_strtolower($name, 'UTF-8');
+    } else {
+      $name = strtolower($name);
+    }
+    $name = trim($name);
+    if (!strlen($name)) {
+      return array();
+    }
+    return preg_split('/\s+/', $name);
+  }
+
+  /**
+   * Populate the nametoken table, which used to fetch typeahead results. When
+   * a user types "linc", we want to match "Abraham Lincoln" from on-demand
+   * typeahead sources. To do this, we need a separate table of name fragments.
+   */
+  public function updateNameTokens() {
+    $tokens = array_merge(
+      self::tokenizeName($this->getRealName()),
+      self::tokenizeName($this->getUserName()));
+    $tokens = array_unique($tokens);
+    $table  = self::NAMETOKEN_TABLE;
+    $conn_w = $this->establishConnection('w');
+
+    $sql = array();
+    foreach ($tokens as $token) {
+      $sql[] = qsprintf(
+        $conn_w,
+        '(%d, %s)',
+        $this->getID(),
+        $token);
+    }
+
+    queryfx(
+      $conn_w,
+      'DELETE FROM %T WHERE userID = %d',
+      $table,
+      $this->getID());
+    if ($sql) {
+      queryfx(
+        $conn_w,
+        'INSERT INTO %T (userID, token) VALUES %Q',
+        $table,
+        implode(', ', $sql));
+    }
+  }
+
 }

src/applications/people/storage/user/__init__.php

@@ -14,6 +14,7 @@ phutil_require_module('phabricator', 'applications/phid/constants');
 phutil_require_module('phabricator', 'applications/phid/storage/phid');
 phutil_require_module('phabricator', 'applications/search/index/indexer/user');
 phutil_require_module('phabricator', 'infrastructure/env');
+phutil_require_module('phabricator', 'storage/qsprintf');
 phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'filesystem');

src/applications/typeahead/controller/common/PhabricatorTypeaheadCommonDatasourceController.php

@@ -85,12 +85,21 @@ class PhabricatorTypeaheadCommonDatasourceController
         'realName',
         'phid');
       if ($query) {
-        // TODO: We probably need to split last names here. Workaround until
+        $conn_r = id(new PhabricatorUser())->establishConnection('r');
-        // we get that up and running is to not enable server-side datasources.
+        $ids = queryfx_all(
-        $users = id(new PhabricatorUser())->loadColumnsWhere($columns,
+          $conn_r,
-          '(userName LIKE %> OR realName LIKE %>)',
+          'SELECT DISTINCT userID FROM %T WHERE token LIKE %>',
-          $query,
+          PhabricatorUser::NAMETOKEN_TABLE,
           $query);
+        $ids = ipull($ids, 'userID');
+        if ($ids) {
+          $users = id(new PhabricatorUser())->loadColumnsWhere(
+            $columns,
+            'id IN (%Ld)',
+            $ids);
+        } else {
+          $users = array();
+        }
       } else {
         $users = id(new PhabricatorUser())->loadColumns($columns);
       }

src/applications/typeahead/controller/common/__init__.php

@@ -15,6 +15,7 @@ phutil_require_module('phabricator', 'applications/project/storage/project');
 phutil_require_module('phabricator', 'applications/repository/storage/arcanistproject');
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'applications/typeahead/controller/base');
+phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'utils');