From 0d7489da79bcbfba9d71fd740378c941d2b1c680 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 1 Oct 2014 08:22:37 -0700 Subject: [PATCH] Provide `bin/storage quickstart` to automate generation of `quickstart.sql` Summary: Ref T1191. Currently, the `quickstart.sql` gets generated in a pretty manual fashion. This is a pain, and will become more of a pain in the world of utf8mb4. Provide a workflow which does upgrade + adjust + dump + destroy, then massages the output to produce a workable `quickstart.sql`. Test Plan: Inspected output; I'll test this more throughly before actually generating a new quickstart, but that's some ways away. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T1191 Differential Revision: https://secure.phabricator.com/D10603 --- resources/sql/patches/065.sshkeys.sql | 2 +- src/__phutil_library_map__.php | 2 + .../schema/PhabricatorConfigSchemaQuery.php | 30 +---- .../fact/storage/PhabricatorFactAggregate.php | 3 +- .../fact/storage/PhabricatorFactRaw.php | 3 +- .../PhabricatorStorageManagementAPI.php | 33 +++++ ...ricatorStorageManagementAdjustWorkflow.php | 5 +- ...torStorageManagementQuickstartWorkflow.php | 122 ++++++++++++++++++ 8 files changed, 167 insertions(+), 33 deletions(-) create mode 100644 src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php diff --git a/resources/sql/patches/065.sshkeys.sql b/resources/sql/patches/065.sshkeys.sql index 766e4affba..a16c3319e4 100644 --- a/resources/sql/patches/065.sshkeys.sql +++ b/resources/sql/patches/065.sshkeys.sql @@ -4,7 +4,7 @@ CREATE TABLE {$NAMESPACE}_user.user_sshkey ( key (userPHID), name varchar(255), keyType varchar(255), - keyBody varchar(32768) BINARY, + keyBody LONGBLOB, unique key (keyBody(128)), keyComment varchar(255), dateCreated INT UNSIGNED NOT NULL, diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 3be57a0757..0a0eb45969 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2315,6 +2315,7 @@ phutil_register_library_map(array( 'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php', 'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php', 'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php', + 'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php', 'PhabricatorStorageManagementStatusWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php', 'PhabricatorStorageManagementUpgradeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php', 'PhabricatorStorageManagementWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php', @@ -5319,6 +5320,7 @@ phutil_register_library_map(array( 'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementStatusWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementUpgradeWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementWorkflow' => 'PhabricatorManagementWorkflow', diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index b830486e57..b99f40637f 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -149,28 +149,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject { $api = $this->getAPI(); - if ($api->isCharacterSetAvailable('utf8mb4')) { - // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci - // collation. This is most correct, and will sort properly. - - $utf8_charset = 'utf8mb4'; - $utf8_binary_collation = 'utf8mb4_bin'; - $utf8_sorting_collation = 'utf8mb4_unicode_ci'; - } else { - // If utf8mb4 is not available, we use binary. This allows us to store - // 4-byte unicode characters. This has some tradeoffs: - // - // Unicode characters won't sort correctly. There's nothing we can do - // about this while still supporting 4-byte characters. - // - // It's possible that strings will be truncated in the middle of a - // character on insert. We encourage users to set STRICT_ALL_TABLES - // to prevent this. - - $utf8_charset = 'binary'; - $utf8_binary_collation = 'binary'; - $utf8_sorting_collation = 'binary'; - } + $charset_info = $api->getCharsetInfo(); + list($charset, $collate_text, $collate_sort) = $charset_info; $specs = id(new PhutilSymbolLoader()) ->setAncestorClass('PhabricatorConfigSchemaSpec') @@ -179,9 +159,9 @@ final class PhabricatorConfigSchemaQuery extends Phobject { $server_schema = new PhabricatorConfigServerSchema(); foreach ($specs as $spec) { $spec - ->setUTF8Charset($utf8_charset) - ->setUTF8BinaryCollation($utf8_binary_collation) - ->setUTF8SortingCollation($utf8_sorting_collation) + ->setUTF8Charset($charset) + ->setUTF8BinaryCollation($collate_text) + ->setUTF8SortingCollation($collate_sort) ->setServer($server_schema) ->buildSchemata($server_schema); } diff --git a/src/applications/fact/storage/PhabricatorFactAggregate.php b/src/applications/fact/storage/PhabricatorFactAggregate.php index ec86c88e38..d6f2219939 100644 --- a/src/applications/fact/storage/PhabricatorFactAggregate.php +++ b/src/applications/fact/storage/PhabricatorFactAggregate.php @@ -9,12 +9,11 @@ final class PhabricatorFactAggregate extends PhabricatorFactDAO { public function getConfiguration() { return array( self::CONFIG_COLUMN_SCHEMA => array( - 'id' => null, + 'id' => 'id64', 'factType' => 'text32', 'valueX' => 'uint64', ), self::CONFIG_KEY_SCHEMA => array( - 'PRIMARY' => null, 'factType' => array( 'columns' => array('factType', 'objectPHID'), 'unique' => true, diff --git a/src/applications/fact/storage/PhabricatorFactRaw.php b/src/applications/fact/storage/PhabricatorFactRaw.php index 0d613dd9d3..d4684b61db 100644 --- a/src/applications/fact/storage/PhabricatorFactRaw.php +++ b/src/applications/fact/storage/PhabricatorFactRaw.php @@ -15,14 +15,13 @@ final class PhabricatorFactRaw extends PhabricatorFactDAO { public function getConfiguration() { return array( self::CONFIG_COLUMN_SCHEMA => array( - 'id' => null, + 'id' => 'id64', 'factType' => 'text32', 'objectA' => 'phid', 'valueX' => 'sint64', 'valueY' => 'sint64', ), self::CONFIG_KEY_SCHEMA => array( - 'PRIMARY' => null, 'objectPHID' => array( 'columns' => array('objectPHID'), ), diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php index dc9309bff5..017aa9a11f 100644 --- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php +++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php @@ -182,8 +182,14 @@ final class PhabricatorStorageManagementAPI { $conn = $this->getConn(null); + $charset_info = $this->getCharsetInfo(); + list($charset, $collate_text, $collate_sort) = $charset_info; + foreach ($queries as $query) { $query = str_replace('{$NAMESPACE}', $this->namespace, $query); + $query = str_replace('{$CHARSET}', $charset, $query); + $query = str_replace('{$COLLATE_TEXT}', $collate_text, $query); + $query = str_replace('{$COLLATE_SORT}', $collate_sort, $query); queryfx( $conn, '%Q', @@ -208,4 +214,31 @@ final class PhabricatorStorageManagementAPI { return (bool)$result; } + public function getCharsetInfo() { + if ($this->isCharacterSetAvailable('utf8mb4')) { + // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci + // collation. This is most correct, and will sort properly. + + $charset = 'utf8mb4'; + $collate_text = 'utf8mb4_bin'; + $collate_sort = 'utf8mb4_unicode_ci'; + } else { + // If utf8mb4 is not available, we use binary. This allows us to store + // 4-byte unicode characters. This has some tradeoffs: + // + // Unicode characters won't sort correctly. There's nothing we can do + // about this while still supporting 4-byte characters. + // + // It's possible that strings will be truncated in the middle of a + // character on insert. We encourage users to set STRICT_ALL_TABLES + // to prevent this. + + $charset = 'binary'; + $collate_text = 'binary'; + $collate_sort = 'binary'; + } + + return array($charset, $collate_text, $collate_sort); + } + } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php index c2f492cd54..4831a7e34b 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php @@ -17,8 +17,7 @@ final class PhabricatorStorageManagementAdjustWorkflow $force = $args->getArg('force'); $this->requireAllPatchesApplied(); - $this->adjustSchemata($force); - return 0; + return $this->adjustSchemata($force); } private function requireAllPatchesApplied() { @@ -258,7 +257,7 @@ final class PhabricatorStorageManagementAdjustWorkflow $console->writeOut( "%s\n", pht('Completed fixing all schema issues.')); - return; + return 0; } $table = id(new PhutilConsoleTable()) diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php new file mode 100644 index 0000000000..6abeedf9bc --- /dev/null +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php @@ -0,0 +1,122 @@ +setName('quickstart') + ->setExamples('**quickstart** [__options__]') + ->setSynopsis( + pht( + 'Generate a new quickstart database dump. This command is mostly '. + 'useful when developing Phabricator.')) + ->setArguments( + array( + array( + 'name' => 'output', + 'param' => 'file', + 'help' => pht('Specify output file to write.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $output = $args->getArg('output'); + if (!$output) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a file to write with `--output`.')); + } + + $namespace = 'phabricator_quickstart_'.Filesystem::readRandomCharacters(8); + + $bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage'; + + $err = phutil_passthru( + '%s upgrade --force --no-quickstart --namespace %s', + $bin, + $namespace); + if ($err) { + return $err; + } + + $err = phutil_passthru( + '%s adjust --force --namespace %s', + $bin, + $namespace); + if ($err) { + return $err; + } + + $tmp = new TempFile(); + $err = phutil_passthru( + '%s dump --namespace %s > %s', + $bin, + $namespace, + $tmp); + if ($err) { + return $err; + } + + $err = phutil_passthru( + '%s destroy --force --namespace %s', + $bin, + $namespace); + if ($err) { + return $err; + } + + $dump = Filesystem::readFile($tmp); + + $dump = str_replace( + $namespace, + '{$NAMESPACE}', + $dump); + + $dump = str_replace( + 'utf8mb4_bin', + '{$COLLATE_TEXT}', + $dump); + + $dump = str_replace( + 'utf8mb4_unicode_ci', + '{$COLLATE_SORT}', + $dump); + + $dump = str_replace( + 'utf8mb4', + '{$CHARSET}', + $dump); + + // Strip out a bunch of unnecessary commands which make the dump harder + // to handle and slower to import. + + // Remove character set adjustments and key disables. + $dump = preg_replace( + '(^/\*.*\*/;$)m', + '', + $dump); + + // Remove comments. + $dump = preg_replace('/^--.*$/m', '', $dump); + + // Remove table drops, locks, and unlocks. These are never relevant when + // performing q quickstart. + $dump = preg_replace( + '/^(DROP TABLE|LOCK TABLES|UNLOCK TABLES).*$/m', + '', + $dump); + + // Collapse adjacent newlines. + $dump = preg_replace('/\n\s*\n/', "\n", $dump); + + $dump = str_replace(';', ";\n", $dump); + $dump = trim($dump)."\n"; + + Filesystem::writeFile($output, $dump); + + return 0; + } + +}