Add "persistence" types (data, cache, or index) to tables, and tweak what "storage dump" dumps

Summary:
Ref T13000. This marks each table as either "data" (normal data), "cache" (automatically rebuilt, no need to ever dump) or "index" (can be manually rebuilt).

By default, `bin/storage dump` dumps data and index tables, but not cache tables.

With `--no-indexes`, it dumps only data tables. Indexes can be rebuilt after a restore with `bin/search index --all ...`.

Test Plan:
  - Ran `--no-indexes` and normal dumps with `--trace`, verified that cache and index (former case) or cache only (latter case) tables were dumped with `--no-data`.
  - Verified dump has the same number of `CREATE TABLE` statements as before the changes.
  - Reviewed persistence tags in the web UI (note Ferret engine tables are "Index"):

{F5210886}

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13000

Differential Revision: https://secure.phabricator.com/D18682

src/applications/cache/storage/PhabricatorCacheSchemaSpec.php

@@ -30,6 +30,9 @@ final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec {
         'key_ttl' => array(
           'columns' => array('cacheExpires'),
         ),
+      ),
+      array(
+        'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
       ));
 
   }

src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php

@@ -261,6 +261,7 @@ final class PhabricatorConfigDatabaseStatusController
         $this->renderAttr(
           $table->getCollation(),
           $table->hasIssue($collation_issue)),
+        $table->getPersistenceTypeDisplayName(),
       );
     }
 
@@ -270,12 +271,14 @@ final class PhabricatorConfigDatabaseStatusController
           null,
           pht('Table'),
           pht('Collation'),
+          pht('Persistence'),
         ))
       ->setColumnClasses(
         array(
           null,
           'wide pri',
           null,
+          null,
         ));
 
     $title = $database_name;

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -338,6 +338,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
           $comp_table->addKey($comp_key);
         }
 
+        $comp_table->setPersistenceType($expect_table->getPersistenceType());
+
         $comp_database->addTable($comp_table);
       }
       $comp_server->addDatabase($comp_database);

src/applications/config/schema/PhabricatorConfigSchemaSpec.php

@@ -56,36 +56,52 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
   }
 
   protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
+    $index_options = array(
+      'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
+    );
+
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getDocumentTableName(),
       $engine->getDocumentSchemaColumns(),
-      $engine->getDocumentSchemaKeys());
+      $engine->getDocumentSchemaKeys(),
+      $index_options);
 
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getFieldTableName(),
       $engine->getFieldSchemaColumns(),
-      $engine->getFieldSchemaKeys());
+      $engine->getFieldSchemaKeys(),
+      $index_options);
 
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getNgramsTableName(),
       $engine->getNgramsSchemaColumns(),
-      $engine->getNgramsSchemaKeys());
+      $engine->getNgramsSchemaKeys(),
+      $index_options);
 
     $this->buildRawSchema(
       $engine->getApplicationName(),
       $engine->getCommonNgramsTableName(),
       $engine->getCommonNgramsSchemaColumns(),
-      $engine->getCommonNgramsSchemaKeys());
+      $engine->getCommonNgramsSchemaKeys(),
+      $index_options);
   }
 
   protected function buildRawSchema(
     $database_name,
     $table_name,
     array $columns,
-    array $keys) {
+    array $keys,
+    array $options = array()) {
+
+    PhutilTypeSpec::checkMap(
+      $options,
+      array(
+        'persistence' => 'optional string',
+      ));
+
     $database = $this->getDatabase($database_name);
 
     $table = $this->newTable($table_name);
@@ -144,6 +160,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
       $table->addKey($key);
     }
 
+    $persistence_type = idx($options, 'persistence');
+    if ($persistence_type !== null) {
+      $table->setPersistenceType($persistence_type);
+    }
+
     $database->addTable($table);
   }
 

src/applications/config/schema/PhabricatorConfigTableSchema.php

@@ -7,6 +7,11 @@ final class PhabricatorConfigTableSchema
   private $engine;
   private $columns = array();
   private $keys = array();
+  private $persistenceType = self::PERSISTENCE_DATA;
+
+  const PERSISTENCE_DATA = 'data';
+  const PERSISTENCE_CACHE = 'cache';
+  const PERSISTENCE_INDEX = 'index';
 
   public function addColumn(PhabricatorConfigColumnSchema $column) {
     $key = $column->getName();
@@ -45,6 +50,27 @@ final class PhabricatorConfigTableSchema
     return idx($this->getKeys(), $key);
   }
 
+  public function setPersistenceType($persistence_type) {
+    $this->persistenceType = $persistence_type;
+    return $this;
+  }
+
+  public function getPersistenceType() {
+    return $this->persistenceType;
+  }
+
+  public function getPersistenceTypeDisplayName() {
+    $map = array(
+      self::PERSISTENCE_DATA => pht('Data'),
+      self::PERSISTENCE_CACHE => pht('Cache'),
+      self::PERSISTENCE_INDEX => pht('Index'),
+    );
+
+    $type = $this->getPersistenceType();
+
+    return idx($map, $type, $type);
+  }
+
   protected function getSubschemata() {
     // NOTE: Keys and columns may have the same name, so make sure we return
     // everything.

src/applications/differential/storage/DifferentialSchemaSpec.php

@@ -21,6 +21,9 @@ final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec {
         'dateCreated' => array(
           'columns' => array('dateCreated'),
         ),
+      ),
+      array(
+        'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
       ));
 
     $this->buildRawSchema(

src/docs/user/configuration/configuring_backups.diviner

@@ -145,6 +145,24 @@ present a risk. If you restrict access to the Phabricator host or database, you
 should also restrict access to the backups.
 
 
+Skipping Indexes
+================
+
+By default, `bin/storage dump` does not dump all of the data in the database:
+it skips some caches which can be rebuilt automatically and do not need to be
+backed up. Some of these caches are very large, so the size of the dump may
+be significantly smaller than the size of the databases.
+
+If you have a large amount of data, you can specify `--no-indexes` when taking
+a database dump to skip additional tables which contain search indexes. This
+will reduce the size (and increase the speed) of the backup. This is an
+advanced option which most installs will not benefit from.
+
+This index data can be rebuilt after a restore, but will not be rebuilt
+automatically. If you choose to use this flag, you must manually rebuild
+indexes after a restore (for details, see ((reindex))).
+
+
 Next Steps
 ==========
 

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php

@@ -30,6 +30,13 @@ final class PhabricatorStorageManagementDumpWorkflow
               'With __--output__, write a compressed file to disk instead '.
               'of a plaintext file.'),
           ),
+          array(
+            'name' => 'no-indexes',
+            'help' => pht(
+              'Do not dump data in rebuildable index tables. This means '.
+              'backups are smaller and faster, but you will need to manually '.
+              'rebuild indexes after performing a restore.'),
+          ),
           array(
             'name' => 'overwrite',
             'help' => pht(
@@ -49,6 +56,8 @@ final class PhabricatorStorageManagementDumpWorkflow
 
     $console = PhutilConsole::getConsole();
 
+    $with_indexes = !$args->getArg('no-indexes');
+
     $applied = $api->getAppliedPatches();
     if ($applied === null) {
       $namespace = $api->getNamespace();
@@ -65,18 +74,58 @@ final class PhabricatorStorageManagementDumpWorkflow
     $ref = $api->getRef();
     $ref_key = $ref->getRefKey();
 
-    $schemata_map = id(new PhabricatorConfigSchemaQuery())
+    $schemata_query = id(new PhabricatorConfigSchemaQuery())
       ->setAPIs(array($api))
-      ->setRefs(array($ref))
-      ->loadActualSchemata();
-    $schemata = $schemata_map[$ref_key];
+      ->setRefs(array($ref));
+
+    $actual_map = $schemata_query->loadActualSchemata();
+    $expect_map = $schemata_query->loadExpectedSchemata();
+
+    $schemata = $actual_map[$ref_key];
+    $expect = $expect_map[$ref_key];
 
     $targets = array();
     foreach ($schemata->getDatabases() as $database_name => $database) {
+      $expect_database = $expect->getDatabase($database_name);
       foreach ($database->getTables() as $table_name => $table) {
+
+        // NOTE: It's possible for us to find tables in these database which
+        // we don't expect to be there. For example, an older version of
+        // Phabricator may have had a table that was later dropped. We assume
+        // these are data tables and always dump them, erring on the side of
+        // caution.
+
+        $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
+        if ($expect_database) {
+          $expect_table = $expect_database->getTable($table_name);
+          if ($expect_table) {
+            $persistence = $expect_table->getPersistenceType();
+          }
+        }
+
+        switch ($persistence) {
+          case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
+            // When dumping tables, leave the data in cache tables in the
+            // database. This will be automatically rebuild after the data
+            // is restored and does not need to be persisted in backups.
+            $with_data = false;
+            break;
+          case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
+            // When dumping tables, leave index data behind of the caller
+            // specified "--no-indexes". These tables can be rebuilt manually
+            // from other tables, but do not rebuild automatically.
+            $with_data = $with_indexes;
+            break;
+          case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
+          default:
+            $with_data = true;
+            break;
+        }
+
         $targets[] = array(
           'database' => $database_name,
           'table' => $table_name,
+          'data' => $with_data,
         );
       }
     }
@@ -147,6 +196,10 @@ final class PhabricatorStorageManagementDumpWorkflow
     foreach ($targets as $target) {
       $target_argv = $argv;
 
+      if (!$target['data']) {
+        $target_argv[] = '--no-data';
+      }
+
       if ($has_password) {
         $commands[] = csprintf(
           'mysqldump -p%P %Ls -- %R %R',