diff --git a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php index 97d6e1ac81..c9e78af0bc 100644 --- a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php +++ b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php @@ -30,6 +30,9 @@ final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec { 'key_ttl' => array( 'columns' => array('cacheExpires'), ), + ), + array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE, )); } diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php index a67c57e6f9..760317ae80 100644 --- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php @@ -261,6 +261,7 @@ final class PhabricatorConfigDatabaseStatusController $this->renderAttr( $table->getCollation(), $table->hasIssue($collation_issue)), + $table->getPersistenceTypeDisplayName(), ); } @@ -270,12 +271,14 @@ final class PhabricatorConfigDatabaseStatusController null, pht('Table'), pht('Collation'), + pht('Persistence'), )) ->setColumnClasses( array( null, 'wide pri', null, + null, )); $title = $database_name; diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index 6feda7363c..2cb7763110 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -338,6 +338,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject { $comp_table->addKey($comp_key); } + $comp_table->setPersistenceType($expect_table->getPersistenceType()); + $comp_database->addTable($comp_table); } $comp_server->addDatabase($comp_database); diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index 971f5b828f..c451658682 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -56,36 +56,52 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { } protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) { + $index_options = array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX, + ); + $this->buildRawSchema( $engine->getApplicationName(), $engine->getDocumentTableName(), $engine->getDocumentSchemaColumns(), - $engine->getDocumentSchemaKeys()); + $engine->getDocumentSchemaKeys(), + $index_options); $this->buildRawSchema( $engine->getApplicationName(), $engine->getFieldTableName(), $engine->getFieldSchemaColumns(), - $engine->getFieldSchemaKeys()); + $engine->getFieldSchemaKeys(), + $index_options); $this->buildRawSchema( $engine->getApplicationName(), $engine->getNgramsTableName(), $engine->getNgramsSchemaColumns(), - $engine->getNgramsSchemaKeys()); + $engine->getNgramsSchemaKeys(), + $index_options); $this->buildRawSchema( $engine->getApplicationName(), $engine->getCommonNgramsTableName(), $engine->getCommonNgramsSchemaColumns(), - $engine->getCommonNgramsSchemaKeys()); + $engine->getCommonNgramsSchemaKeys(), + $index_options); } protected function buildRawSchema( $database_name, $table_name, array $columns, - array $keys) { + array $keys, + array $options = array()) { + + PhutilTypeSpec::checkMap( + $options, + array( + 'persistence' => 'optional string', + )); + $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); @@ -144,6 +160,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { $table->addKey($key); } + $persistence_type = idx($options, 'persistence'); + if ($persistence_type !== null) { + $table->setPersistenceType($persistence_type); + } + $database->addTable($table); } diff --git a/src/applications/config/schema/PhabricatorConfigTableSchema.php b/src/applications/config/schema/PhabricatorConfigTableSchema.php index 870c8ebddb..d6e5bfd9fa 100644 --- a/src/applications/config/schema/PhabricatorConfigTableSchema.php +++ b/src/applications/config/schema/PhabricatorConfigTableSchema.php @@ -7,6 +7,11 @@ final class PhabricatorConfigTableSchema private $engine; private $columns = array(); private $keys = array(); + private $persistenceType = self::PERSISTENCE_DATA; + + const PERSISTENCE_DATA = 'data'; + const PERSISTENCE_CACHE = 'cache'; + const PERSISTENCE_INDEX = 'index'; public function addColumn(PhabricatorConfigColumnSchema $column) { $key = $column->getName(); @@ -45,6 +50,27 @@ final class PhabricatorConfigTableSchema return idx($this->getKeys(), $key); } + public function setPersistenceType($persistence_type) { + $this->persistenceType = $persistence_type; + return $this; + } + + public function getPersistenceType() { + return $this->persistenceType; + } + + public function getPersistenceTypeDisplayName() { + $map = array( + self::PERSISTENCE_DATA => pht('Data'), + self::PERSISTENCE_CACHE => pht('Cache'), + self::PERSISTENCE_INDEX => pht('Index'), + ); + + $type = $this->getPersistenceType(); + + return idx($map, $type, $type); + } + protected function getSubschemata() { // NOTE: Keys and columns may have the same name, so make sure we return // everything. diff --git a/src/applications/differential/storage/DifferentialSchemaSpec.php b/src/applications/differential/storage/DifferentialSchemaSpec.php index e376c7f4f8..7aa76fa822 100644 --- a/src/applications/differential/storage/DifferentialSchemaSpec.php +++ b/src/applications/differential/storage/DifferentialSchemaSpec.php @@ -21,6 +21,9 @@ final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec { 'dateCreated' => array( 'columns' => array('dateCreated'), ), + ), + array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE, )); $this->buildRawSchema( diff --git a/src/docs/user/configuration/configuring_backups.diviner b/src/docs/user/configuration/configuring_backups.diviner index 9776448216..d32eebb0da 100644 --- a/src/docs/user/configuration/configuring_backups.diviner +++ b/src/docs/user/configuration/configuring_backups.diviner @@ -145,6 +145,24 @@ present a risk. If you restrict access to the Phabricator host or database, you should also restrict access to the backups. +Skipping Indexes +================ + +By default, `bin/storage dump` does not dump all of the data in the database: +it skips some caches which can be rebuilt automatically and do not need to be +backed up. Some of these caches are very large, so the size of the dump may +be significantly smaller than the size of the databases. + +If you have a large amount of data, you can specify `--no-indexes` when taking +a database dump to skip additional tables which contain search indexes. This +will reduce the size (and increase the speed) of the backup. This is an +advanced option which most installs will not benefit from. + +This index data can be rebuilt after a restore, but will not be rebuilt +automatically. If you choose to use this flag, you must manually rebuild +indexes after a restore (for details, see ((reindex))). + + Next Steps ========== diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php index f491133c67..fecc0517e3 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php @@ -30,6 +30,13 @@ final class PhabricatorStorageManagementDumpWorkflow 'With __--output__, write a compressed file to disk instead '. 'of a plaintext file.'), ), + array( + 'name' => 'no-indexes', + 'help' => pht( + 'Do not dump data in rebuildable index tables. This means '. + 'backups are smaller and faster, but you will need to manually '. + 'rebuild indexes after performing a restore.'), + ), array( 'name' => 'overwrite', 'help' => pht( @@ -49,6 +56,8 @@ final class PhabricatorStorageManagementDumpWorkflow $console = PhutilConsole::getConsole(); + $with_indexes = !$args->getArg('no-indexes'); + $applied = $api->getAppliedPatches(); if ($applied === null) { $namespace = $api->getNamespace(); @@ -65,18 +74,58 @@ final class PhabricatorStorageManagementDumpWorkflow $ref = $api->getRef(); $ref_key = $ref->getRefKey(); - $schemata_map = id(new PhabricatorConfigSchemaQuery()) + $schemata_query = id(new PhabricatorConfigSchemaQuery()) ->setAPIs(array($api)) - ->setRefs(array($ref)) - ->loadActualSchemata(); - $schemata = $schemata_map[$ref_key]; + ->setRefs(array($ref)); + + $actual_map = $schemata_query->loadActualSchemata(); + $expect_map = $schemata_query->loadExpectedSchemata(); + + $schemata = $actual_map[$ref_key]; + $expect = $expect_map[$ref_key]; $targets = array(); foreach ($schemata->getDatabases() as $database_name => $database) { + $expect_database = $expect->getDatabase($database_name); foreach ($database->getTables() as $table_name => $table) { + + // NOTE: It's possible for us to find tables in these database which + // we don't expect to be there. For example, an older version of + // Phabricator may have had a table that was later dropped. We assume + // these are data tables and always dump them, erring on the side of + // caution. + + $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA; + if ($expect_database) { + $expect_table = $expect_database->getTable($table_name); + if ($expect_table) { + $persistence = $expect_table->getPersistenceType(); + } + } + + switch ($persistence) { + case PhabricatorConfigTableSchema::PERSISTENCE_CACHE: + // When dumping tables, leave the data in cache tables in the + // database. This will be automatically rebuild after the data + // is restored and does not need to be persisted in backups. + $with_data = false; + break; + case PhabricatorConfigTableSchema::PERSISTENCE_INDEX: + // When dumping tables, leave index data behind of the caller + // specified "--no-indexes". These tables can be rebuilt manually + // from other tables, but do not rebuild automatically. + $with_data = $with_indexes; + break; + case PhabricatorConfigTableSchema::PERSISTENCE_DATA: + default: + $with_data = true; + break; + } + $targets[] = array( 'database' => $database_name, 'table' => $table_name, + 'data' => $with_data, ); } } @@ -147,6 +196,10 @@ final class PhabricatorStorageManagementDumpWorkflow foreach ($targets as $target) { $target_argv = $argv; + if (!$target['data']) { + $target_argv[] = '--no-data'; + } + if ($has_password) { $commands[] = csprintf( 'mysqldump -p%P %Ls -- %R %R',