(stable) Promote 2017 Week 40

resources/sql/autopatches/20171002.cngram.01.maniphest.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_maniphest.maniphest_task_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.02.event.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_calendar.calendar_event_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.03.revision.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_differential.differential_revision_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.04.fund.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_fund.fund_initiative_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.05.owners.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_owners.owners_package_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.06.passphrase.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_passphrase.passphrase_credential_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.07.blog.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_phame.phame_blog_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.08.post.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_phame.phame_post_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.09.pholio.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_pholio.pholio_mock_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.10.phriction.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_phriction.phriction_document_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.11.project.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_project.project_project_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.12.user.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_user.user_user_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.13.repository.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_repository_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20171002.cngram.14.commit.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_commit_fngrams_common (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
+  needsCollection BOOL NOT NULL,
+  UNIQUE KEY `key_ngram` (ngram),
+  KEY `key_collect` (needsCollection)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -3941,6 +3941,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchEngineAttachment' => 'applications/search/engineextension/PhabricatorSearchEngineAttachment.php',
     'PhabricatorSearchEngineExtension' => 'applications/search/engineextension/PhabricatorSearchEngineExtension.php',
     'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php',
+    'PhabricatorSearchFerretNgramGarbageCollector' => 'applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php',
     'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
     'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php',
     'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
@@ -3948,6 +3949,8 @@ phutil_register_library_map(array(
     'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
     'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
     'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
+    'PhabricatorSearchManagementNgramsWorkflow' => 'applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php',
+    'PhabricatorSearchManagementQueryWorkflow' => 'applications/search/management/PhabricatorSearchManagementQueryWorkflow.php',
     'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
     'PhabricatorSearchNgrams' => 'applications/search/ngrams/PhabricatorSearchNgrams.php',
     'PhabricatorSearchNgramsDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchNgramsDestructionEngineExtension.php',
@@ -9521,6 +9524,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchEngineAttachment' => 'Phobject',
     'PhabricatorSearchEngineExtension' => 'Phobject',
     'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule',
+    'PhabricatorSearchFerretNgramGarbageCollector' => 'PhabricatorGarbageCollector',
     'PhabricatorSearchField' => 'Phobject',
     'PhabricatorSearchHost' => 'Phobject',
     'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
@@ -9528,6 +9532,8 @@ phutil_register_library_map(array(
     'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
     'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
+    'PhabricatorSearchManagementNgramsWorkflow' => 'PhabricatorSearchManagementWorkflow',
+    'PhabricatorSearchManagementQueryWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorSearchNgrams' => 'PhabricatorSearchDAO',
     'PhabricatorSearchNgramsDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',

src/applications/cache/storage/PhabricatorCacheSchemaSpec.php

@@ -30,6 +30,9 @@ final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec {
         'key_ttl' => array(
           'columns' => array('cacheExpires'),
         ),
+      ),
+      array(
+        'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
       ));
 
   }

src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php

@@ -261,6 +261,7 @@ final class PhabricatorConfigDatabaseStatusController
         $this->renderAttr(
           $table->getCollation(),
           $table->hasIssue($collation_issue)),
+        $table->getPersistenceTypeDisplayName(),
       );
     }
 
@@ -270,12 +271,14 @@ final class PhabricatorConfigDatabaseStatusController
           null,
           pht('Table'),
           pht('Collation'),
+          pht('Persistence'),
         ))
       ->setColumnClasses(
         array(
           null,
           'wide pri',
           null,
+          null,
         ));
 
     $title = $database_name;

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -338,6 +338,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
           $comp_table->addKey($comp_key);
         }
 
+        $comp_table->setPersistenceType($expect_table->getPersistenceType());
+
         $comp_database->addTable($comp_table);
       }
       $comp_server->addDatabase($comp_database);

src/applications/config/schema/PhabricatorConfigSchemaSpec.php

@@ -56,30 +56,52 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
   }
 
   protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
+    $index_options = array(
+      'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
+    );
+
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getDocumentTableName(),
       $engine->getDocumentSchemaColumns(),
-      $engine->getDocumentSchemaKeys());
+      $engine->getDocumentSchemaKeys(),
+      $index_options);
 
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getFieldTableName(),
       $engine->getFieldSchemaColumns(),
-      $engine->getFieldSchemaKeys());
+      $engine->getFieldSchemaKeys(),
+      $index_options);
 
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getNgramsTableName(),
       $engine->getNgramsSchemaColumns(),
-      $engine->getNgramsSchemaKeys());
+      $engine->getNgramsSchemaKeys(),
+      $index_options);
+
+    $this->buildRawSchema(
+      $engine->getApplicationName(),
+      $engine->getCommonNgramsTableName(),
+      $engine->getCommonNgramsSchemaColumns(),
+      $engine->getCommonNgramsSchemaKeys(),
+      $index_options);
   }
 
   protected function buildRawSchema(
     $database_name,
     $table_name,
     array $columns,
-    array $keys) {
+    array $keys,
+    array $options = array()) {
+
+    PhutilTypeSpec::checkMap(
+      $options,
+      array(
+        'persistence' => 'optional string',
+      ));
+
     $database = $this->getDatabase($database_name);
 
     $table = $this->newTable($table_name);
@@ -138,6 +160,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
       $table->addKey($key);
     }
 
+    $persistence_type = idx($options, 'persistence');
+    if ($persistence_type !== null) {
+      $table->setPersistenceType($persistence_type);
+    }
+
     $database->addTable($table);
   }
 

src/applications/config/schema/PhabricatorConfigTableSchema.php

@@ -7,6 +7,11 @@ final class PhabricatorConfigTableSchema
   private $engine;
   private $columns = array();
   private $keys = array();
+  private $persistenceType = self::PERSISTENCE_DATA;
+
+  const PERSISTENCE_DATA = 'data';
+  const PERSISTENCE_CACHE = 'cache';
+  const PERSISTENCE_INDEX = 'index';
 
   public function addColumn(PhabricatorConfigColumnSchema $column) {
     $key = $column->getName();
@@ -45,6 +50,27 @@ final class PhabricatorConfigTableSchema
     return idx($this->getKeys(), $key);
   }
 
+  public function setPersistenceType($persistence_type) {
+    $this->persistenceType = $persistence_type;
+    return $this;
+  }
+
+  public function getPersistenceType() {
+    return $this->persistenceType;
+  }
+
+  public function getPersistenceTypeDisplayName() {
+    $map = array(
+      self::PERSISTENCE_DATA => pht('Data'),
+      self::PERSISTENCE_CACHE => pht('Cache'),
+      self::PERSISTENCE_INDEX => pht('Index'),
+    );
+
+    $type = $this->getPersistenceType();
+
+    return idx($map, $type, $type);
+  }
+
   protected function getSubschemata() {
     // NOTE: Keys and columns may have the same name, so make sure we return
     // everything.

src/applications/differential/storage/DifferentialSchemaSpec.php

@@ -21,6 +21,9 @@ final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec {
         'dateCreated' => array(
           'columns' => array('dateCreated'),
         ),
+      ),
+      array(
+        'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
       ));
 
     $this->buildRawSchema(

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

@@ -69,10 +69,11 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
         'branches/(?P<dblob>.*)' => 'DiffusionBranchTableController',
         'refs/(?P<dblob>.*)' => 'DiffusionRefTableController',
         'lint/(?P<dblob>.*)' => 'DiffusionLintController',
-        'commit/(?P<commit>[a-z0-9]+)/branches/'
+        'commit/(?P<commit>[a-z0-9]+)' => array(
-          => 'DiffusionCommitBranchesController',
+          '/?' => 'DiffusionCommitController',
-        'commit/(?P<commit>[a-z0-9]+)/tags/'
+          '/branches/' => 'DiffusionCommitBranchesController',
-          => 'DiffusionCommitTagsController',
+          '/tags/' => 'DiffusionCommitTagsController',
+        ),
         'compare/' => 'DiffusionCompareController',
         'manage/(?:(?P<panel>[^/]+)/)?'
           => 'DiffusionRepositoryManagePanelsController',

src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php

@@ -115,6 +115,10 @@ final class DiffusionLastModifiedQueryConduitAPIMethod
     $graph_cache = new PhabricatorRepositoryGraphCache();
 
     $results = array();
+
+    // Spend no more than this many total seconds trying to satisfy queries
+    // via the graph cache.
+    $remaining_time = 10.0;
     foreach ($map as $path => $commit) {
       $path_id = idx($path_map, $path);
       if (!$path_id) {
@@ -125,13 +129,21 @@ final class DiffusionLastModifiedQueryConduitAPIMethod
         continue;
       }
 
+      $t_start = microtime(true);
       $cache_result = $graph_cache->loadLastModifiedCommitID(
         $commit_id,
-        $path_id);
+        $path_id,
+        $remaining_time);
+      $t_end = microtime(true);
 
       if ($cache_result !== false) {
         $results[$path] = $cache_result;
       }
+
+      $remaining_time -= ($t_end - $t_start);
+      if ($remaining_time <= 0) {
+        break;
+      }
     }
 
     if ($results) {

src/applications/diffusion/controller/DiffusionCommitController.php

@@ -22,17 +22,27 @@ final class DiffusionCommitController extends DiffusionController {
 
     $drequest = $this->getDiffusionRequest();
     $viewer = $request->getUser();
+    $repository = $drequest->getRepository();
+    $commit_identifier = $drequest->getCommit();
+
+    // If this page is being accessed via "/source/xyz/commit/...", redirect
+    // to the canonical URI.
+    $has_callsign = strlen($request->getURIData('repositoryCallsign'));
+    $has_id = strlen($request->getURIData('repositoryID'));
+    if (!$has_callsign && !$has_id) {
+      $canonical_uri = $repository->getCommitURI($commit_identifier);
+      return id(new AphrontRedirectResponse())
+        ->setURI($canonical_uri);
+    }
 
     if ($request->getStr('diff')) {
       return $this->buildRawDiffResponse($drequest);
     }
 
-    $repository = $drequest->getRepository();
-
     $commit = id(new DiffusionCommitQuery())
       ->setViewer($viewer)
       ->withRepository($repository)
-      ->withIdentifiers(array($drequest->getCommit()))
+      ->withIdentifiers(array($commit_identifier))
       ->needCommitData(true)
       ->needAuditRequests(true)
       ->executeOne();

src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php

@@ -102,6 +102,10 @@ final class PhabricatorRepositoryGraphCache extends Phobject {
         }
 
         // Otherwise, the rebuild gave us the data, so we can keep going.
+
+        $did_fill = true;
+      } else {
+        $did_fill = false;
       }
 
       // Sanity check so we can survive and recover from bad data.
@@ -147,12 +151,17 @@ final class PhabricatorRepositoryGraphCache extends Phobject {
         $commit_id = $parent_id;
 
         // Periodically check if we've spent too long looking for a result
-        // in the cache, and return so we can fall back to a VCS operation. This
+        // in the cache, and return so we can fall back to a VCS operation.
-        // keeps us from having a degenerate worst case if, e.g., the cache
+        // This keeps us from having a degenerate worst case if, e.g., the
-        // is cold and we need to inspect a very large number of blocks
+        // cache is cold and we need to inspect a very large number of blocks
         // to satisfy the query.
 
-        if (((++$iterations) % 64) === 0) {
+        ++$iterations;
+
+        // If we performed a cache fill in this cycle, always check the time
+        // limit, since cache fills may take a significant amount of time.
+
+        if ($did_fill || ($iterations % 64 === 0)) {
           $t_end = microtime(true);
           if (($t_end - $t_start) > $time) {
             return false;

src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php

@@ -165,21 +165,46 @@ final class PhabricatorFerretFulltextEngineExtension
             $ferret_field['normalCorpus']);
       }
 
-      $sql = array();
+      if ($ngrams) {
-      foreach ($ngrams as $ngram) {
+        $common = queryfx_all(
-        $sql[] = qsprintf(
           $conn,
-          '(%d, %s)',
+          'SELECT ngram FROM %T WHERE ngram IN (%Ls)',
-          $document_id,
+          $engine->getCommonNgramsTableName(),
-          $ngram);
+          $ngrams);
+        $common = ipull($common, 'ngram', 'ngram');
+
+        foreach ($ngrams as $key => $ngram) {
+          if (isset($common[$ngram])) {
+            unset($ngrams[$key]);
+            continue;
+          }
+
+          // NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
+          $trim_ngram = rtrim($ngram, ' ');
+          if (isset($common[$ngram])) {
+            unset($ngrams[$key]);
+            continue;
+          }
+        }
       }
 
-      foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+      if ($ngrams) {
-        queryfx(
+        $sql = array();
-          $conn,
+        foreach ($ngrams as $ngram) {
-          'INSERT INTO %T (documentID, ngram) VALUES %Q',
+          $sql[] = qsprintf(
-          $engine->getNgramsTableName(),
+            $conn,
-          $chunk);
+            '(%d, %s)',
+            $document_id,
+            $ngram);
+        }
+
+        foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+          queryfx(
+            $conn,
+            'INSERT INTO %T (documentID, ngram) VALUES %Q',
+            $engine->getNgramsTableName(),
+            $chunk);
+        }
       }
     } catch (Exception $ex) {
       $object->killTransaction();

src/applications/search/ferret/PhabricatorFerretEngine.php

@@ -295,4 +295,35 @@ abstract class PhabricatorFerretEngine extends Phobject {
     );
   }
 
+  public function getCommonNgramsTableName() {
+    $application = $this->getApplicationName();
+    $scope = $this->getScopeName();
+
+    return "{$application}_{$scope}_fngrams_common";
+  }
+
+  public function getCommonNgramsSchemaColumns() {
+    return array(
+      'id' => 'auto',
+      'ngram' => 'char3',
+      'needsCollection' => 'bool',
+    );
+  }
+
+  public function getCommonNgramsSchemaKeys() {
+    return array(
+      'PRIMARY' => array(
+        'columns' => array('id'),
+        'unique' => true,
+      ),
+      'key_ngram' => array(
+        'columns' => array('ngram'),
+        'unique' => true,
+      ),
+      'key_collect' => array(
+        'columns' => array('needsCollection'),
+      ),
+    );
+  }
+
 }

src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php

@@ -0,0 +1,55 @@
+<?php
+
+final class PhabricatorSearchFerretNgramGarbageCollector
+  extends PhabricatorGarbageCollector {
+
+  const COLLECTORCONST = 'search.ferret.ngram';
+
+  public function getCollectorName() {
+    return pht('Ferret Engine Ngrams');
+  }
+
+  public function hasAutomaticPolicy() {
+    return true;
+  }
+
+  protected function collectGarbage() {
+    $all_objects = id(new PhutilClassMapQuery())
+      ->setAncestorClass('PhabricatorFerretInterface')
+      ->execute();
+
+    $did_collect = false;
+    foreach ($all_objects as $object) {
+      $engine = $object->newFerretEngine();
+      $conn = $object->establishConnection('w');
+
+      $ngram_row = queryfx_one(
+        $conn,
+        'SELECT ngram FROM %T WHERE needsCollection = 1 LIMIT 1',
+        $engine->getCommonNgramsTableName());
+      if (!$ngram_row) {
+        continue;
+      }
+
+      $ngram = $ngram_row['ngram'];
+
+      queryfx(
+        $conn,
+        'DELETE FROM %T WHERE ngram = %s',
+        $engine->getNgramsTableName(),
+        $ngram);
+
+      queryfx(
+        $conn,
+        'UPDATE %T SET needsCollection = 0 WHERE ngram = %s',
+        $engine->getCommonNgramsTableName(),
+        $ngram);
+
+      $did_collect = true;
+      break;
+    }
+
+    return $did_collect;
+  }
+
+}

src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php

@@ -0,0 +1,109 @@
+<?php
+
+final class PhabricatorSearchManagementNgramsWorkflow
+  extends PhabricatorSearchManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('ngrams')
+      ->setSynopsis(
+        pht(
+          'Recompute common ngrams. This is an advanced workflow that '.
+          'can harm search quality if used improperly.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'reset',
+            'help' => pht('Reset all common ngram records.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $is_reset = $args->getArg('reset');
+
+    $all_objects = id(new PhutilClassMapQuery())
+      ->setAncestorClass('PhabricatorFerretInterface')
+      ->execute();
+
+    $min_documents = 4096;
+    $threshold = 0.15;
+
+    foreach ($all_objects as $object) {
+      $engine = $object->newFerretEngine();
+      $conn = $object->establishConnection('w');
+      $display_name = get_class($object);
+
+      if ($is_reset) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'Resetting common ngrams for "%s".',
+            $display_name));
+
+        queryfx(
+          $conn,
+          'DELETE FROM %T',
+          $engine->getCommonNgramsTableName());
+        continue;
+      }
+
+      $document_count = queryfx_one(
+        $conn,
+        'SELECT COUNT(*) N FROM %T',
+        $engine->getDocumentTableName());
+      $document_count = $document_count['N'];
+
+      if ($document_count < $min_documents) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'Too few documents of type "%s" for any ngrams to be common.',
+            $display_name));
+        continue;
+      }
+
+      $min_frequency = (int)ceil($document_count * $threshold);
+      $common_ngrams = queryfx_all(
+        $conn,
+        'SELECT ngram, COUNT(*) N FROM %T
+          GROUP BY ngram
+          HAVING N >= %d',
+        $engine->getNgramsTableName(),
+        $min_frequency);
+
+      if (!$common_ngrams) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'No new common ngrams exist for "%s".',
+            $display_name));
+        continue;
+      }
+
+      $sql = array();
+      foreach ($common_ngrams as $ngram) {
+        $sql[] = qsprintf(
+          $conn,
+          '(%s, 1)',
+          $ngram['ngram']);
+      }
+
+      foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+        queryfx(
+          $conn,
+          'INSERT IGNORE INTO %T (ngram, needsCollection)
+            VALUES %Q',
+          $engine->getCommonNgramsTableName(),
+          $chunk);
+      }
+
+      echo tsprintf(
+        "%s\n",
+        pht(
+          'Updated common ngrams for "%s".',
+          $display_name));
+    }
+  }
+
+}

src/applications/search/management/PhabricatorSearchManagementQueryWorkflow.php

@@ -0,0 +1,55 @@
+<?php
+
+final class PhabricatorSearchManagementQueryWorkflow
+  extends PhabricatorSearchManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('query')
+      ->setSynopsis(
+        pht('Run a search query. Intended for debugging and development.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'query',
+            'param' => 'query',
+            'help' => pht('Raw query to execute.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+    $raw_query = $args->getArg('query');
+    if (!strlen($raw_query)) {
+      throw new PhutilArgumentUsageException(
+        pht('Specify a query with --query.'));
+    }
+
+    $engine = id(new PhabricatorSearchApplicationSearchEngine())
+      ->setViewer($viewer);
+
+    $saved = $engine->newSavedQuery();
+    $saved->setParameter('query', $raw_query);
+
+    $query = $engine->buildQueryFromSavedQuery($saved);
+    $pager = $engine->newPagerForSavedQuery($saved);
+
+    $results = $engine->executeQuery($query, $pager);
+    if ($results) {
+      foreach ($results as $result) {
+        echo tsprintf(
+          "%s\t%s\n",
+          $result->getPHID(),
+          $result->getName());
+      }
+    } else {
+      echo tsprintf(
+        "%s\n",
+        pht('No results.'));
+    }
+
+    return 0;
+  }
+
+}

src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php

@@ -138,9 +138,9 @@ final class PhabricatorEmailAddressesSettingsPanel
         $editable,
       ));
 
-    $button = null;
+    $buttons = array();
     if ($editable) {
-      $button = id(new PHUIButtonView())
+      $buttons[] = id(new PHUIButtonView())
         ->setTag('a')
         ->setIcon('fa-plus')
         ->setText(pht('Add New Address'))
@@ -149,7 +149,7 @@ final class PhabricatorEmailAddressesSettingsPanel
         ->setColor(PHUIButtonView::GREY);
     }
 
-    return $this->newBox(pht('Email Addresses'), $table, array($button));
+    return $this->newBox(pht('Email Addresses'), $table, $buttons);
   }
 
   private function returnNewAddressResponse(

src/docs/user/cluster/cluster_notifications.diviner

@@ -14,7 +14,7 @@ are:
   - performance and capacity may improve.
 
 This configuration is relatively simple, but has a small impact on availability
-and does nothing to increase resitance to data loss.
+and does nothing to increase resistance to data loss.
 
 
 Clustering Design Goals

src/docs/user/configuration/configuring_backups.diviner

@@ -145,6 +145,24 @@ present a risk. If you restrict access to the Phabricator host or database, you
 should also restrict access to the backups.
 
 
+Skipping Indexes
+================
+
+By default, `bin/storage dump` does not dump all of the data in the database:
+it skips some caches which can be rebuilt automatically and do not need to be
+backed up. Some of these caches are very large, so the size of the dump may
+be significantly smaller than the size of the databases.
+
+If you have a large amount of data, you can specify `--no-indexes` when taking
+a database dump to skip additional tables which contain search indexes. This
+will reduce the size (and increase the speed) of the backup. This is an
+advanced option which most installs will not benefit from.
+
+This index data can be rebuilt after a restore, but will not be rebuilt
+automatically. If you choose to use this flag, you must manually rebuild
+indexes after a restore (for details, see ((reindex))).
+
+
 Next Steps
 ==========
 

src/infrastructure/graph/ManiphestTaskGraph.php

@@ -27,6 +27,8 @@ final class ManiphestTaskGraph
   protected function newTableRow($phid, $object, $trace) {
     $viewer = $this->getViewer();
 
+    Javelin::initBehavior('phui-hovercards');
+
     if ($object) {
       $status = $object->getStatus();
       $priority = $object->getPriority();
@@ -51,15 +53,16 @@ final class ManiphestTaskGraph
         $assigned = phutil_tag('em', array(), pht('None'));
       }
 
-      $full_title = $object->getTitle();
+      $link = javelin_tag(
-
-      $link = phutil_tag(
         'a',
         array(
           'href' => $object->getURI(),
-          'title' => $full_title,
+          'sigil' => 'hovercard',
+          'meta' => array(
+            'hoverPHID' => $object->getPHID(),
+          ),
         ),
-        $full_title);
+        $object->getTitle());
 
       $link = array(
         phutil_tag(

src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php

@@ -1700,6 +1700,34 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
       }
     }
 
+    // Remove common ngrams, like "the", which occur too frequently in
+    // documents to be useful in constraining the query. The best ngrams
+    // are obscure sequences which occur in very few documents.
+
+    if ($flat) {
+      $common_ngrams = queryfx_all(
+        $conn,
+        'SELECT ngram FROM %T WHERE ngram IN (%Ls)',
+        $engine->getCommonNgramsTableName(),
+        ipull($flat, 'ngram'));
+      $common_ngrams = ipull($common_ngrams, 'ngram', 'ngram');
+
+      foreach ($flat as $key => $spec) {
+        $ngram = $spec['ngram'];
+        if (isset($common_ngrams[$ngram])) {
+          unset($flat[$key]);
+          continue;
+        }
+
+        // NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
+        $trim_ngram = rtrim($ngram, ' ');
+        if (isset($common_ngrams[$trim_ngram])) {
+          unset($flat[$key]);
+          continue;
+        }
+      }
+    }
+
     // MySQL only allows us to join a maximum of 61 tables per query. Each
     // ngram is going to cost us a join toward that limit, so if the user
     // specified a very long query string, just pick 16 of the ngrams

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php

@@ -30,6 +30,13 @@ final class PhabricatorStorageManagementDumpWorkflow
               'With __--output__, write a compressed file to disk instead '.
               'of a plaintext file.'),
           ),
+          array(
+            'name' => 'no-indexes',
+            'help' => pht(
+              'Do not dump data in rebuildable index tables. This means '.
+              'backups are smaller and faster, but you will need to manually '.
+              'rebuild indexes after performing a restore.'),
+          ),
           array(
             'name' => 'overwrite',
             'help' => pht(
@@ -49,6 +56,8 @@ final class PhabricatorStorageManagementDumpWorkflow
 
     $console = PhutilConsole::getConsole();
 
+    $with_indexes = !$args->getArg('no-indexes');
+
     $applied = $api->getAppliedPatches();
     if ($applied === null) {
       $namespace = $api->getNamespace();
@@ -62,7 +71,64 @@ final class PhabricatorStorageManagementDumpWorkflow
       return 1;
     }
 
-    $databases = $api->getDatabaseList($patches, true);
+    $ref = $api->getRef();
+    $ref_key = $ref->getRefKey();
+
+    $schemata_query = id(new PhabricatorConfigSchemaQuery())
+      ->setAPIs(array($api))
+      ->setRefs(array($ref));
+
+    $actual_map = $schemata_query->loadActualSchemata();
+    $expect_map = $schemata_query->loadExpectedSchemata();
+
+    $schemata = $actual_map[$ref_key];
+    $expect = $expect_map[$ref_key];
+
+    $targets = array();
+    foreach ($schemata->getDatabases() as $database_name => $database) {
+      $expect_database = $expect->getDatabase($database_name);
+      foreach ($database->getTables() as $table_name => $table) {
+
+        // NOTE: It's possible for us to find tables in these database which
+        // we don't expect to be there. For example, an older version of
+        // Phabricator may have had a table that was later dropped. We assume
+        // these are data tables and always dump them, erring on the side of
+        // caution.
+
+        $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
+        if ($expect_database) {
+          $expect_table = $expect_database->getTable($table_name);
+          if ($expect_table) {
+            $persistence = $expect_table->getPersistenceType();
+          }
+        }
+
+        switch ($persistence) {
+          case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
+            // When dumping tables, leave the data in cache tables in the
+            // database. This will be automatically rebuild after the data
+            // is restored and does not need to be persisted in backups.
+            $with_data = false;
+            break;
+          case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
+            // When dumping tables, leave index data behind of the caller
+            // specified "--no-indexes". These tables can be rebuilt manually
+            // from other tables, but do not rebuild automatically.
+            $with_data = $with_indexes;
+            break;
+          case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
+          default:
+            $with_data = true;
+            break;
+        }
+
+        $targets[] = array(
+          'database' => $database_name,
+          'table' => $table_name,
+          'data' => $with_data,
+        );
+      }
+    }
 
     list($host, $port) = $this->getBareHostAndPort($api->getHost());
 
@@ -126,35 +192,46 @@ final class PhabricatorStorageManagementDumpWorkflow
       $argv[] = $port;
     }
 
-    $argv[] = '--databases';
+    $commands = array();
-    foreach ($databases as $database) {
+    foreach ($targets as $target) {
-      $argv[] = $database;
+      $target_argv = $argv;
+
+      if (!$target['data']) {
+        $target_argv[] = '--no-data';
+      }
+
+      if ($has_password) {
+        $commands[] = csprintf(
+          'mysqldump -p%P %Ls -- %R %R',
+          $password,
+          $target_argv,
+          $target['database'],
+          $target['table']);
+      } else {
+        $command = csprintf(
+          'mysqldump %Ls -- %R %R',
+          $target_argv,
+          $target['database'],
+          $target['table']);
+      }
+
+      $commands[] = $command;
     }
 
 
-    if ($has_password) {
-      $command = csprintf('mysqldump -p%P %Ls', $password, $argv);
-    } else {
-      $command = csprintf('mysqldump %Ls', $argv);
-    }
-
     // Decrease the CPU priority of this process so it doesn't contend with
     // other more important things.
     if (function_exists('proc_nice')) {
       proc_nice(19);
     }
 
-
-    // If we aren't writing to a file, just passthru the command.
-    if ($output_file === null) {
-      return phutil_passthru('%C', $command);
-    }
-
     // If we are writing to a file, stream the command output to disk. This
     // mode makes sure the whole command fails if there's an error (commonly,
     // a full disk). See T6996 for discussion.
 
-    if ($is_compress) {
+    if ($output_file === null) {
+      $file = null;
+    } else if ($is_compress) {
       $file = gzopen($output_file, 'wb1');
     } else {
       $file = fopen($output_file, 'wb');
@@ -167,41 +244,47 @@ final class PhabricatorStorageManagementDumpWorkflow
           $file));
     }
 
-    $future = new ExecFuture('%C', $command);
-
     try {
-      $iterator = id(new FutureIterator(array($future)))
+      foreach ($commands as $command) {
-        ->setUpdateInterval(0.100);
+        $future = new ExecFuture('%C', $command);
-      foreach ($iterator as $ready) {
-        list($stdout, $stderr) = $future->read();
-        $future->discardBuffers();
 
-        if (strlen($stderr)) {
+        $iterator = id(new FutureIterator(array($future)))
-          fwrite(STDERR, $stderr);
+          ->setUpdateInterval(0.100);
-        }
+        foreach ($iterator as $ready) {
+          list($stdout, $stderr) = $future->read();
+          $future->discardBuffers();
 
-        if (strlen($stdout)) {
+          if (strlen($stderr)) {
-          if ($is_compress) {
+            fwrite(STDERR, $stderr);
-            $ok = gzwrite($file, $stdout);
-          } else {
-            $ok = fwrite($file, $stdout);
           }
 
-          if ($ok !== strlen($stdout)) {
+          if (strlen($stdout)) {
-            throw new Exception(
+            if (!$file) {
-              pht(
+              $ok = fwrite(STDOUT, $stdout);
-                'Failed to write %d byte(s) to file "%s".',
+            } else if ($is_compress) {
-                new PhutilNumber(strlen($stdout)),
+              $ok = gzwrite($file, $stdout);
-                $output_file));
+            } else {
-          }
+              $ok = fwrite($file, $stdout);
-        }
+            }
 
-        if ($ready !== null) {
+            if ($ok !== strlen($stdout)) {
-          $ready->resolvex();
+              throw new Exception(
+                pht(
+                  'Failed to write %d byte(s) to file "%s".',
+                  new PhutilNumber(strlen($stdout)),
+                  $output_file));
+            }
+          }
+
+          if ($ready !== null) {
+            $ready->resolvex();
+          }
         }
       }
 
-      if ($is_compress) {
+      if (!$file) {
+        $ok = true;
+      } else if ($is_compress) {
         $ok = gzclose($file);
       } else {
         $ok = fclose($file);
@@ -218,7 +301,9 @@ final class PhabricatorStorageManagementDumpWorkflow
       // we don't leave any confusing artifacts laying around.
 
       try {
-        Filesystem::remove($output_file);
+        if ($file !== null) {
+          Filesystem::remove($output_file);
+        }
       } catch (Exception $ex) {
         // Ignore any errors we hit.
       }