Provide bin/storage quickstart to automate generation of quickstart.sql

Summary:
Ref T1191. Currently, the `quickstart.sql` gets generated in a pretty manual fashion. This is a pain, and will become more of a pain in the world of utf8mb4.

Provide a workflow which does upgrade + adjust + dump + destroy, then massages the output to produce a workable `quickstart.sql`.

Test Plan: Inspected output; I'll test this more throughly before actually generating a new quickstart, but that's some ways away.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10603

resources/sql/patches/065.sshkeys.sql

@@ -4,7 +4,7 @@ CREATE TABLE {$NAMESPACE}_user.user_sshkey (
   key (userPHID),
   name varchar(255),
   keyType varchar(255),
-  keyBody varchar(32768) BINARY,
+  keyBody LONGBLOB,
   unique key (keyBody(128)),
   keyComment varchar(255),
   dateCreated INT UNSIGNED NOT NULL,

src/__phutil_library_map__.php

@@ -2315,6 +2315,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php',
     'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php',
     'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php',
+    'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php',
     'PhabricatorStorageManagementStatusWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php',
     'PhabricatorStorageManagementUpgradeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php',
     'PhabricatorStorageManagementWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php',
@@ -5319,6 +5320,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow',
+    'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementStatusWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementUpgradeWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementWorkflow' => 'PhabricatorManagementWorkflow',

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -149,28 +149,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
 
     $api = $this->getAPI();
 
-    if ($api->isCharacterSetAvailable('utf8mb4')) {
+    $charset_info = $api->getCharsetInfo();
-      // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
+    list($charset, $collate_text, $collate_sort) = $charset_info;
-      // collation. This is most correct, and will sort properly.
-
-      $utf8_charset = 'utf8mb4';
-      $utf8_binary_collation = 'utf8mb4_bin';
-      $utf8_sorting_collation = 'utf8mb4_unicode_ci';
-    } else {
-      // If utf8mb4 is not available, we use binary. This allows us to store
-      // 4-byte unicode characters. This has some tradeoffs:
-      //
-      // Unicode characters won't sort correctly. There's nothing we can do
-      // about this while still supporting 4-byte characters.
-      //
-      // It's possible that strings will be truncated in the middle of a
-      // character on insert. We encourage users to set STRICT_ALL_TABLES
-      // to prevent this.
-
-      $utf8_charset = 'binary';
-      $utf8_binary_collation = 'binary';
-      $utf8_sorting_collation = 'binary';
-    }
 
     $specs = id(new PhutilSymbolLoader())
       ->setAncestorClass('PhabricatorConfigSchemaSpec')
@@ -179,9 +159,9 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
     $server_schema = new PhabricatorConfigServerSchema();
     foreach ($specs as $spec) {
       $spec
-        ->setUTF8Charset($utf8_charset)
+        ->setUTF8Charset($charset)
-        ->setUTF8BinaryCollation($utf8_binary_collation)
+        ->setUTF8BinaryCollation($collate_text)
-        ->setUTF8SortingCollation($utf8_sorting_collation)
+        ->setUTF8SortingCollation($collate_sort)
         ->setServer($server_schema)
         ->buildSchemata($server_schema);
     }

src/applications/fact/storage/PhabricatorFactAggregate.php

@@ -9,12 +9,11 @@ final class PhabricatorFactAggregate extends PhabricatorFactDAO {
   public function getConfiguration() {
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
-        'id' => null,
+        'id' => 'id64',
         'factType' => 'text32',
         'valueX' => 'uint64',
       ),
       self::CONFIG_KEY_SCHEMA => array(
-        'PRIMARY' => null,
         'factType' => array(
           'columns' => array('factType', 'objectPHID'),
           'unique' => true,

src/applications/fact/storage/PhabricatorFactRaw.php

@@ -15,14 +15,13 @@ final class PhabricatorFactRaw extends PhabricatorFactDAO {
   public function getConfiguration() {
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
-        'id' => null,
+        'id' => 'id64',
         'factType' => 'text32',
         'objectA' => 'phid',
         'valueX' => 'sint64',
         'valueY' => 'sint64',
       ),
       self::CONFIG_KEY_SCHEMA => array(
-        'PRIMARY' => null,
         'objectPHID' => array(
           'columns' => array('objectPHID'),
         ),

src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php

@@ -182,8 +182,14 @@ final class PhabricatorStorageManagementAPI {
 
     $conn = $this->getConn(null);
 
+    $charset_info = $this->getCharsetInfo();
+    list($charset, $collate_text, $collate_sort) = $charset_info;
+
     foreach ($queries as $query) {
       $query = str_replace('{$NAMESPACE}', $this->namespace, $query);
+      $query = str_replace('{$CHARSET}', $charset, $query);
+      $query = str_replace('{$COLLATE_TEXT}', $collate_text, $query);
+      $query = str_replace('{$COLLATE_SORT}', $collate_sort, $query);
       queryfx(
         $conn,
         '%Q',
@@ -208,4 +214,31 @@ final class PhabricatorStorageManagementAPI {
     return (bool)$result;
   }
 
+  public function getCharsetInfo() {
+    if ($this->isCharacterSetAvailable('utf8mb4')) {
+      // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
+      // collation. This is most correct, and will sort properly.
+
+      $charset = 'utf8mb4';
+      $collate_text = 'utf8mb4_bin';
+      $collate_sort = 'utf8mb4_unicode_ci';
+    } else {
+      // If utf8mb4 is not available, we use binary. This allows us to store
+      // 4-byte unicode characters. This has some tradeoffs:
+      //
+      // Unicode characters won't sort correctly. There's nothing we can do
+      // about this while still supporting 4-byte characters.
+      //
+      // It's possible that strings will be truncated in the middle of a
+      // character on insert. We encourage users to set STRICT_ALL_TABLES
+      // to prevent this.
+
+      $charset = 'binary';
+      $collate_text = 'binary';
+      $collate_sort = 'binary';
+    }
+
+    return array($charset, $collate_text, $collate_sort);
+  }
+
 }

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php

@@ -17,8 +17,7 @@ final class PhabricatorStorageManagementAdjustWorkflow
     $force = $args->getArg('force');
 
     $this->requireAllPatchesApplied();
-    $this->adjustSchemata($force);
+    return $this->adjustSchemata($force);
-    return 0;
   }
 
   private function requireAllPatchesApplied() {
@@ -258,7 +257,7 @@ final class PhabricatorStorageManagementAdjustWorkflow
       $console->writeOut(
         "%s\n",
         pht('Completed fixing all schema issues.'));
-      return;
+      return 0;
     }
 
     $table = id(new PhutilConsoleTable())

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php

@@ -0,0 +1,122 @@
+<?php
+
+final class PhabricatorStorageManagementQuickstartWorkflow
+  extends PhabricatorStorageManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('quickstart')
+      ->setExamples('**quickstart** [__options__]')
+      ->setSynopsis(
+        pht(
+          'Generate a new quickstart database dump. This command is mostly '.
+          'useful when developing Phabricator.'))
+      ->setArguments(
+        array(
+          array(
+            'name'  => 'output',
+            'param' => 'file',
+            'help'  => pht('Specify output file to write.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $output = $args->getArg('output');
+    if (!$output) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Specify a file to write with `--output`.'));
+    }
+
+    $namespace = 'phabricator_quickstart_'.Filesystem::readRandomCharacters(8);
+
+    $bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage';
+
+    $err = phutil_passthru(
+      '%s upgrade --force --no-quickstart --namespace %s',
+      $bin,
+      $namespace);
+    if ($err) {
+      return $err;
+    }
+
+    $err = phutil_passthru(
+      '%s adjust --force --namespace %s',
+      $bin,
+      $namespace);
+    if ($err) {
+      return $err;
+    }
+
+    $tmp = new TempFile();
+    $err = phutil_passthru(
+      '%s dump --namespace %s > %s',
+      $bin,
+      $namespace,
+      $tmp);
+    if ($err) {
+      return $err;
+    }
+
+    $err = phutil_passthru(
+      '%s destroy --force --namespace %s',
+      $bin,
+      $namespace);
+    if ($err) {
+      return $err;
+    }
+
+    $dump = Filesystem::readFile($tmp);
+
+    $dump = str_replace(
+      $namespace,
+      '{$NAMESPACE}',
+      $dump);
+
+    $dump = str_replace(
+      'utf8mb4_bin',
+      '{$COLLATE_TEXT}',
+      $dump);
+
+    $dump = str_replace(
+      'utf8mb4_unicode_ci',
+      '{$COLLATE_SORT}',
+      $dump);
+
+    $dump = str_replace(
+      'utf8mb4',
+      '{$CHARSET}',
+      $dump);
+
+    // Strip out a bunch of unnecessary commands which make the dump harder
+    // to handle and slower to import.
+
+    // Remove character set adjustments and key disables.
+    $dump = preg_replace(
+      '(^/\*.*\*/;$)m',
+      '',
+      $dump);
+
+    // Remove comments.
+    $dump = preg_replace('/^--.*$/m', '', $dump);
+
+    // Remove table drops, locks, and unlocks. These are never relevant when
+    // performing q quickstart.
+    $dump = preg_replace(
+      '/^(DROP TABLE|LOCK TABLES|UNLOCK TABLES).*$/m',
+      '',
+      $dump);
+
+    // Collapse adjacent newlines.
+    $dump = preg_replace('/\n\s*\n/', "\n", $dump);
+
+    $dump = str_replace(';', ";\n", $dump);
+    $dump = trim($dump)."\n";
+
+    Filesystem::writeFile($output, $dump);
+
+    return 0;
+  }
+
+}