Document the adjustment workflow and warn users about adjusting old MySQL

Summary: Ref T1191. Explain the adjustment workflow, how to resolve common errors, etc.

Test Plan: Read it, clicked doc links.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10758

src/__phutil_library_map__.php

@@ -4480,7 +4480,7 @@ phutil_register_library_map(array(
     'PhabricatorConfigEntryQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorConfigFileSource' => 'PhabricatorConfigProxySource',
     'PhabricatorConfigGroupController' => 'PhabricatorConfigController',
-    'PhabricatorConfigIgnoreController' => 'PhabricatorApplicationsController',
+    'PhabricatorConfigIgnoreController' => 'PhabricatorConfigController',
     'PhabricatorConfigIssueListController' => 'PhabricatorConfigController',
     'PhabricatorConfigIssueViewController' => 'PhabricatorConfigController',
     'PhabricatorConfigJSONOptionType' => 'PhabricatorConfigOptionType',

src/applications/config/controller/PhabricatorConfigDatabaseController.php

@@ -62,4 +62,20 @@ abstract class PhabricatorConfigDatabaseController
     }
   }
 
+  protected function buildHeaderWithDocumentationLink($title) {
+
+    $doc_link = PhabricatorEnv::getDoclink('Managing Storage Adjustments');
+
+    return id(new PHUIHeaderView())
+      ->setHeader($title)
+      ->addActionLink(
+        id(new PHUIButtonView())
+          ->setTag('a')
+          ->setIcon(
+            id(new PHUIIconView())
+              ->setIconFont('fa-book'))
+          ->setHref($doc_link)
+          ->setText(pht('Learn More')));
+  }
+
 }

src/applications/config/controller/PhabricatorConfigDatabaseIssueController.php

@@ -133,10 +133,6 @@ final class PhabricatorConfigDatabaseIssueController
 
     $errors = array();
 
-    $errors[] = pht(
-      'IMPORTANT: This feature is in development and the information below '.
-      'is not accurate! Ignore it for now. See T1191.');
-
     if (isset($counts[PhabricatorConfigStorageSchema::STATUS_FAIL])) {
       $errors[] = pht(
         'Detected %s serious issue(s) with the schemata.',
@@ -152,7 +148,7 @@ final class PhabricatorConfigDatabaseIssueController
     $title = pht('Database Issues');
 
     $table_box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->setFormErrors($errors)
       ->appendChild($table);
 

src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php

@@ -166,7 +166,7 @@ final class PhabricatorConfigDatabaseStatusController
       $comp->getIssues());
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->addPropertyList($properties)
       ->appendChild($table);
 
@@ -261,7 +261,7 @@ final class PhabricatorConfigDatabaseStatusController
       $database->getIssues());
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->addPropertyList($properties)
       ->appendChild($table);
 
@@ -471,7 +471,7 @@ final class PhabricatorConfigDatabaseStatusController
       $table->getIssues());
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->addPropertyList($properties)
       ->appendChild($table_view)
       ->appendChild($keys_view);
@@ -609,7 +609,7 @@ final class PhabricatorConfigDatabaseStatusController
       $column->getIssues());
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->addPropertyList($properties);
 
     return $this->buildResponse($title, $box);
@@ -702,7 +702,7 @@ final class PhabricatorConfigDatabaseStatusController
       $key->getIssues());
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeader($this->buildHeaderWithDocumentationLink($title))
       ->addPropertyList($properties);
 
     return $this->buildResponse($title, $box);

src/docs/user/configuration/storage_adjust.diviner

@@ -0,0 +1,147 @@
+@title Managing Storage Adjustments
+@group config
+
+Explains how to apply storage adjustments to the MySQL schemata.
+
+Overview
+========
+
+Phabricator uses a workflow called //storage adjustment// to make some minor
+kinds of changes to the MySQL schema. This workflow compliments the //storage
+upgrade// workflow, which makes major changes.
+
+You can perform storage adjustment by running:
+
+  phabricator/ $ ./bin/storage adjust
+
+This document describes what adjustments are, how they relate to storage
+upgrades, how to perform them, and how to troubleshoot issues with storage
+adjustment.
+
+
+Understanding Adjustments
+===================
+
+Storage adjustments make minor changes to the Phabricator MySQL schemata to
+improve consistency, unicode handling, and performance. Changes covered by
+adjustment include:
+
+  - Character set and collation settings for columns, tables, and databases.
+  - Setting and removing "Auto Increment" on columns.
+  - Adding, removing, renaming and adjusting keys.
+
+Adjustment does not make major changes to the schemata, like creating or
+removing columns or tables or migrating data. (Major changes are performed by
+the upgrade workflow.)
+
+Adjustments are separate from upgrades primarily because adjustments depend on
+the MySQL version, while upgrades do not. If you update MySQL, better collations
+may become available, and the adjustment workflow will convert your schemata to
+use them.
+
+All changes covered by adjustment are minor, and technically optional. However,
+you are strongly encouraged to apply outstanding adjustments: if you do not,
+you may encounter issues storing or sorting some unicode data, and may suffer
+poor performance on some queries.
+
+
+Reviewing Outstanding Adjustments
+=================================
+
+There are two ways to review outstanding adjustments: you can use the web UI,
+or you can use the CLI.
+
+To access the web UI, navigate to {nav Config > Database Status} or
+{nav Config > Database Issues}. The //Database Status// panel provides a general
+overview of all schemata. The //Database Issues// panel shows outstanding
+issues.
+
+These interfaces report //Errors//, which are serious issues that can not be
+resolved through adjustment, and //Warnings//, which are minor issues that the
+adjustment workflow can resolve.
+
+You can also review adjustments from the CLI, by running:
+
+  phabricator/ $ ./bin/storage adjust
+
+Before you're prompted to actually apply adjustments, you'll be given a list of
+available adjustments. You can then make a choice to apply them.
+
+
+Performing Adjustments
+======================
+
+To perform adjustments, run the `adjust` workflow:
+
+  phabricator/ $ ./bin/storage adjust
+
+For details about flags, use:
+
+  phabricator/ $ ./bin/storage help adjust
+
+You do not normally need to run this workflow manually: it will be run
+automatically after you run the `upgrade` workflow.
+
+
+History and Rationale
+=====================
+
+The primary motivation for the adjustment workflow is MySQL's handling of
+unicode character sets. Before MySQL 5.5, MySQL supports a character set called
+`utf8`. However, this character set can not store 4-byte unicode characters
+(including emoji). Inserting 4-byte characters into a `utf8` column truncates
+the data.
+
+With MySQL 5.5, a new `utf8mb4` character set was introduced. This character
+set can safely store 4-byte unicode characters.
+
+The adjustment workflow allows us to alter the schema to primarily use
+`binary` character sets on older MySQL, and primarily use `utf8mb4` character
+sets on newer MySQL. The net effect is that Phabricator works consistently and
+can store 4-byte unicode characters regardless of the MySQL version. Under
+newer MySQL, we can also take advantage of the better collation rules the
+`utf8mb4` character set offers.
+
+The adjustment workflow was introduced in November 2014. If your install
+predates its introduction, your first adjustment may take a long time (we must
+convert all of the data out of `utf8` and into the appropriate character set).
+If your install was set up after November 2014, adjustments should generally
+be very minor and complete quickly, unless you perform a major MySQL update and
+make new character sets available.
+
+If you plan to update MySQL from an older version to 5.5 or newer, it is
+advisable to update first, then run the adjustment workflow. If you adjust
+first, you'll need to adjust again after updating, so you'll end up spending
+twice as much time performing schemata adjustments.
+
+
+Troubleshooting
+===============
+
+When you apply adjustments, some adjustments may fail. The two most common
+errors you may encounter are:
+
+  - **#1406 Data Too Long**: Usually this is caused by a very long object name
+    (like a task title) which contains multibyte unicode characters. When the
+    column type is converted to `binary`, only the first part of the title still
+    fits in the column. Depending on what is failing, you may be able to find
+    the relevant object in the web UI and retitle it so the adjustment succeeds.
+    Alternatively, you can use `--unsafe` to force the adjustment to truncate
+    the title. This will destroy some data, but usually the data is not
+    important (just the end of very long titles).
+  - **#1366 Incorrect String Value**: This can occur when converting invalid
+    or truncated multibyte unicode characters to a unicode character set.
+    In both cases, the old value can not be represented under the new character
+    set. You may be able to identify the object and edit it to allow the
+    adjustment to proceed, or you can use the `--unsafe` flag to truncate the
+    data at the invalid character. Usually, the truncated data is not important.
+
+As with most commands, you can add the `--trace` flag to get more details about
+what `bin/storage adjust` is doing. This may help you diagnose or understand any
+issues you encounter, and this data is useful if you file reports in the
+upstream.
+
+In general, adjustments are not critical. If you run into issues applying
+adjustments, it is safe to file a task in the upstream describing the problem
+you've encountered and continue using Phabricator normally until the issue can
+be resolved.

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php

@@ -79,6 +79,7 @@ final class PhabricatorStorageManagementAdjustWorkflow
       pht('Verifying database schemata...'));
 
     $adjustments = $this->findAdjustments();
+    $api = $this->getAPI();
 
     if (!$adjustments) {
       $console->writeOut(
@@ -87,6 +88,36 @@ final class PhabricatorStorageManagementAdjustWorkflow
       return;
     }
 
+    if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) {
+      $message = pht(
+        "You have an old version of MySQL (older than 5.5) which does not ".
+        "support the utf8mb4 character set. If you apply adjustments now ".
+        "and later update MySQL to 5.5 or newer, you'll need to apply ".
+        "adjustments again (and they will take a long time).\n\n".
+        "You can exit this workflow, update MySQL now, and then run this ".
+        "workflow again. This is recommended, but may cause a lot of downtime ".
+        "right now.\n\n".
+        "You can exit this workflow, continue using Phabricator without ".
+        "applying adjustments, update MySQL at a later date, and then run ".
+        "this workflow again. This is also a good approach, and will let you ".
+        "delay downtime until later.\n\n".
+        "You can proceed with this workflow, and then optionally update ".
+        "MySQL at a later date. After you do, you'll need to apply ".
+        "adjustments again.\n\n".
+        "For more information, see \"Managing Storage Adjustments\" in ".
+        "the documentation.");
+
+      $console->writeOut(
+        "\n**<bg:yellow> %s </bg>**\n\n%s\n",
+        pht('OLD MySQL VERSION'),
+        phutil_console_wrap($message));
+
+      $prompt = pht('Continue with old MySQL version?');
+      if (!phutil_console_confirm($prompt, $default_no = true)) {
+        return;
+      }
+    }
+
     $table = id(new PhutilConsoleTable())
       ->addColumn('database', array('title' => pht('Database')))
       ->addColumn('table', array('title' => pht('Table')))
@@ -147,7 +178,6 @@ final class PhabricatorStorageManagementAdjustWorkflow
       "%s\n",
       pht('Fixing schema issues...'));
 
-    $api = $this->getAPI();
     $conn = $api->getConn(null);
 
     if ($unsafe) {
@@ -344,15 +374,11 @@ final class PhabricatorStorageManagementAdjustWorkflow
     $console->writeOut(
       "\n%s\n",
       pht('Failed to make some schema adjustments, detailed above.'));
-
+    $console->writeOut(
-    if (!$unsafe) {
+      "%s\n",
-      $console->writeOut(
+      pht(
-        "%s\n",
+        'For help troubleshooting adjustments, see "Managing Storage '.
-        pht(
+        'Adjustments" in the documentation.'));
-          'Migrations which fail with certain types of errors (including '.
-          '"#1406 Data Too Long" and "#1366 Incorrect String Value") can be '.
-          'forced to complete by running again with `--unsafe`.'));
-    }
 
     return 1;
   }