Add a UI for reviewing database schemata

Summary:
Ref T1191. Plan here is:

  - Build a tool showing the current schemata status (this diff).
  - Have it compare the current status to the desired status (partly here, mostly in future diffs).
  - Then add a migration tool, and eventually a setup issue to tell people to run it.

Test Plan:
Reviewed current schemata.

{F204492}

{F204493}

{F204494}

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10494

src/__phutil_library_map__.php

@@ -1328,8 +1328,11 @@ phutil_register_library_map(array(
     'PhabricatorConduitTokenController' => 'applications/conduit/controller/PhabricatorConduitTokenController.php',
     'PhabricatorConfigAllController' => 'applications/config/controller/PhabricatorConfigAllController.php',
     'PhabricatorConfigApplication' => 'applications/config/application/PhabricatorConfigApplication.php',
+    'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
     'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
     'PhabricatorConfigController' => 'applications/config/controller/PhabricatorConfigController.php',
+    'PhabricatorConfigDatabaseController' => 'applications/config/controller/PhabricatorConfigDatabaseController.php',
+    'PhabricatorConfigDatabaseSchema' => 'applications/config/schema/PhabricatorConfigDatabaseSchema.php',
     'PhabricatorConfigDatabaseSource' => 'infrastructure/env/PhabricatorConfigDatabaseSource.php',
     'PhabricatorConfigDefaultSource' => 'infrastructure/env/PhabricatorConfigDefaultSource.php',
     'PhabricatorConfigDictionarySource' => 'infrastructure/env/PhabricatorConfigDictionarySource.php',
@@ -1356,8 +1359,11 @@ phutil_register_library_map(array(
     'PhabricatorConfigOptionType' => 'applications/config/custom/PhabricatorConfigOptionType.php',
     'PhabricatorConfigProxySource' => 'infrastructure/env/PhabricatorConfigProxySource.php',
     'PhabricatorConfigResponse' => 'applications/config/response/PhabricatorConfigResponse.php',
+    'PhabricatorConfigSchemaQuery' => 'applications/config/schema/PhabricatorConfigSchemaQuery.php',
+    'PhabricatorConfigServerSchema' => 'applications/config/schema/PhabricatorConfigServerSchema.php',
     'PhabricatorConfigSource' => 'infrastructure/env/PhabricatorConfigSource.php',
     'PhabricatorConfigStackSource' => 'infrastructure/env/PhabricatorConfigStackSource.php',
+    'PhabricatorConfigTableSchema' => 'applications/config/schema/PhabricatorConfigTableSchema.php',
     'PhabricatorConfigTransaction' => 'applications/config/storage/PhabricatorConfigTransaction.php',
     'PhabricatorConfigTransactionQuery' => 'applications/config/query/PhabricatorConfigTransactionQuery.php',
     'PhabricatorConfigValidationException' => 'applications/config/exception/PhabricatorConfigValidationException.php',
@@ -4210,8 +4216,11 @@ phutil_register_library_map(array(
     'PhabricatorConduitTokenController' => 'PhabricatorConduitController',
     'PhabricatorConfigAllController' => 'PhabricatorConfigController',
     'PhabricatorConfigApplication' => 'PhabricatorApplication',
+    'PhabricatorConfigColumnSchema' => 'Phobject',
     'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
     'PhabricatorConfigController' => 'PhabricatorController',
+    'PhabricatorConfigDatabaseController' => 'PhabricatorConfigController',
+    'PhabricatorConfigDatabaseSchema' => 'Phobject',
     'PhabricatorConfigDatabaseSource' => 'PhabricatorConfigProxySource',
     'PhabricatorConfigDefaultSource' => 'PhabricatorConfigProxySource',
     'PhabricatorConfigDictionarySource' => 'PhabricatorConfigSource',
@@ -4242,7 +4251,10 @@ phutil_register_library_map(array(
     ),
     'PhabricatorConfigProxySource' => 'PhabricatorConfigSource',
     'PhabricatorConfigResponse' => 'AphrontHTMLResponse',
+    'PhabricatorConfigSchemaQuery' => 'Phobject',
+    'PhabricatorConfigServerSchema' => 'Phobject',
     'PhabricatorConfigStackSource' => 'PhabricatorConfigSource',
+    'PhabricatorConfigTableSchema' => 'Phobject',
     'PhabricatorConfigTransaction' => 'PhabricatorApplicationTransaction',
     'PhabricatorConfigTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
     'PhabricatorConfigValidationException' => 'Exception',

src/applications/config/application/PhabricatorConfigApplication.php

@@ -42,6 +42,8 @@ final class PhabricatorConfigApplication extends PhabricatorApplication {
         'edit/(?P<key>[\w\.\-]+)/' => 'PhabricatorConfigEditController',
         'group/(?P<key>[^/]+)/' => 'PhabricatorConfigGroupController',
         'welcome/' => 'PhabricatorConfigWelcomeController',
+        'database/(?:(?P<database>[^/]+)/(?:(?P<table>[^/]+)/)?)?'
+          => 'PhabricatorConfigDatabaseController',
         '(?P<verb>ignore|unignore)/(?P<key>[^/]+)/'
           => 'PhabricatorConfigIgnoreController',
         'issue/' => array(

src/applications/config/controller/PhabricatorConfigController.php

@@ -15,6 +15,7 @@ abstract class PhabricatorConfigController extends PhabricatorController {
     $nav->addFilter('/', pht('Option Groups'));
     $nav->addFilter('all/', pht('All Settings'));
     $nav->addFilter('issue/', pht('Setup Issues'));
+    $nav->addFilter('database/', pht('Database Status'));
     $nav->addFilter('welcome/', pht('Welcome Screen'));
 
     return $nav;

src/applications/config/controller/PhabricatorConfigDatabaseController.php

@@ -0,0 +1,258 @@
+<?php
+
+final class PhabricatorConfigDatabaseController
+  extends PhabricatorConfigController {
+
+  private $database;
+  private $table;
+
+  public function willProcessRequest(array $data) {
+    $this->database = idx($data, 'database');
+    $this->table = idx($data, 'table');
+  }
+
+  public function processRequest() {
+    $request = $this->getRequest();
+    $viewer = $request->getUser();
+
+    $conf = PhabricatorEnv::newObjectFromConfig(
+      'mysql.configuration-provider',
+      array($dao = null, 'w'));
+
+    $api = id(new PhabricatorStorageManagementAPI())
+      ->setUser($conf->getUser())
+      ->setHost($conf->getHost())
+      ->setPort($conf->getPort())
+      ->setNamespace(PhabricatorLiskDAO::getDefaultStorageNamespace())
+      ->setPassword($conf->getPassword());
+
+    $query = id(new PhabricatorConfigSchemaQuery())
+      ->setAPI($api);
+
+    $actual = $query->loadActualSchema();
+    $expect = $query->loadExpectedSchema();
+
+    if ($this->table) {
+      return $this->renderTable(
+        $actual,
+        $expect,
+        $this->database,
+        $this->table);
+    } else if ($this->database) {
+      return $this->renderDatabase(
+        $actual,
+        $expect,
+        $this->database);
+    } else {
+      return $this->renderServer(
+        $actual,
+        $expect);
+    }
+  }
+
+  private function buildResponse($title, $body) {
+    $nav = $this->buildSideNavView();
+    $nav->selectFilter('database/');
+
+    $crumbs = $this->buildApplicationCrumbs();
+    if ($this->database) {
+      $crumbs->addTextCrumb(
+        pht('Database Status'),
+        $this->getApplicationURI('database/'));
+      if ($this->table) {
+        $crumbs->addTextCrumb(
+          $this->database,
+          $this->getApplicationURI('database/'.$this->database.'/'));
+        $crumbs->addTextCrumb($this->table);
+      } else {
+        $crumbs->addTextCrumb($this->database);
+      }
+    } else {
+      $crumbs->addTextCrumb(pht('Database Status'));
+    }
+
+    $nav->setCrumbs($crumbs);
+    $nav->appendChild($body);
+
+    return $this->buildApplicationPage(
+      $nav,
+      array(
+        'title' => $title,
+      ));
+  }
+
+
+  private function renderServer(
+    PhabricatorConfigServerSchema $schema,
+    PhabricatorConfigServerSchema $expect) {
+
+    $icon_ok = id(new PHUIIconView())
+      ->setIconFont('fa-check-circle green');
+
+    $icon_warn = id(new PHUIIconView())
+      ->setIconFont('fa-exclamation-circle yellow');
+
+    $rows = array();
+    foreach ($schema->getDatabases() as $database_name => $database) {
+
+      $expect_database = $expect->getDatabase($database_name);
+      if ($expect_database) {
+        $expect_set = $expect_database->getCharacterSet();
+        $expect_collation = $expect_database->getCollation();
+
+        if ($database->isSameSchema($expect_database)) {
+          $icon = $icon_ok;
+        } else {
+          $icon = $icon_warn;
+        }
+      } else {
+        $expect_set = null;
+        $expect_collation = null;
+        $icon = $icon_warn;
+      }
+
+      $actual_set = $database->getCharacterSet();
+      $actual_collation = $database->getCollation();
+
+
+
+      $rows[] = array(
+        $icon,
+        phutil_tag(
+          'a',
+          array(
+            'href' => $this->getApplicationURI(
+              '/database/'.$database_name.'/'),
+          ),
+          $database_name),
+        $actual_set,
+        $expect_set,
+        $actual_collation,
+        $expect_collation,
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setHeaders(
+        array(
+          null,
+          pht('Database'),
+          pht('Charset'),
+          pht('Expected Charset'),
+          pht('Collation'),
+          pht('Expected Collation'),
+        ))
+      ->setColumnClasses(
+        array(
+          '',
+          'wide pri',
+          null,
+          null,
+        ));
+
+    $title = pht('Database Status');
+
+    $box = id(new PHUIObjectBoxView())
+      ->setHeaderText($title)
+      ->appendChild($table);
+
+    return $this->buildResponse($title, $box);
+  }
+
+  private function renderDatabase(
+    PhabricatorConfigServerSchema $schema,
+    PhabricatorConfigServerSchema $expect,
+    $database_name) {
+
+    $database = $schema->getDatabase($database_name);
+    if (!$database) {
+      return new Aphront404Response();
+    }
+
+    $rows = array();
+    foreach ($database->getTables() as $table_name => $table) {
+      $rows[] = array(
+        phutil_tag(
+          'a',
+          array(
+            'href' => $this->getApplicationURI(
+              '/database/'.$database_name.'/'.$table_name.'/'),
+          ),
+          $table_name),
+        $table->getCollation(),
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setHeaders(
+        array(
+          pht('Table'),
+          pht('Collation'),
+        ))
+      ->setColumnClasses(
+        array(
+          'wide pri',
+          null,
+        ));
+
+    $title = pht('Database Status: %s', $database_name);
+
+    $box = id(new PHUIObjectBoxView())
+      ->setHeaderText($title)
+      ->appendChild($table);
+
+    return $this->buildResponse($title, $box);
+  }
+
+  private function renderTable(
+    PhabricatorConfigServerSchema $schema,
+    PhabricatorConfigServerSchema $expect,
+    $database_name,
+    $table_name) {
+
+    $database = $schema->getDatabase($database_name);
+    if (!$database) {
+      return new Aphront404Response();
+    }
+
+    $table = $database->getTable($table_name);
+    if (!$table) {
+      return new Aphront404Response();
+    }
+
+    $rows = array();
+    foreach ($table->getColumns() as $column_name => $column) {
+      $rows[] = array(
+        $column_name,
+        $column->getColumnType(),
+        $column->getCharacterSet(),
+        $column->getCollation(),
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setHeaders(
+        array(
+          pht('Table'),
+          pht('Column Type'),
+          pht('Character Set'),
+          pht('Collation'),
+        ))
+      ->setColumnClasses(
+        array(
+          'wide pri',
+          null,
+          null,
+          null
+        ));
+
+    $title = pht('Database Status: %s.%s', $database_name, $table_name);
+
+    $box = id(new PHUIObjectBoxView())
+      ->setHeaderText($title)
+      ->appendChild($table);
+
+    return $this->buildResponse($title, $box);
+  }
+
+}

src/applications/config/schema/PhabricatorConfigColumnSchema.php

@@ -0,0 +1,46 @@
+<?php
+
+final class PhabricatorConfigColumnSchema extends Phobject {
+
+  private $name;
+  private $characterSet;
+  private $collation;
+  private $columnType;
+
+  public function setColumnType($column_type) {
+    $this->columnType = $column_type;
+    return $this;
+  }
+
+  public function getColumnType() {
+    return $this->columnType;
+  }
+
+  public function setCollation($collation) {
+    $this->collation = $collation;
+    return $this;
+  }
+
+  public function getCollation() {
+    return $this->collation;
+  }
+
+  public function setCharacterSet($character_set) {
+    $this->characterSet = $character_set;
+    return $this;
+  }
+
+  public function getCharacterSet() {
+    return $this->characterSet;
+  }
+
+  public function setName($name) {
+    $this->name = $name;
+    return $this;
+  }
+
+  public function getName() {
+    return $this->name;
+  }
+
+}

src/applications/config/schema/PhabricatorConfigDatabaseSchema.php

@@ -0,0 +1,67 @@
+<?php
+
+final class PhabricatorConfigDatabaseSchema extends Phobject {
+
+  private $name;
+  private $characterSet;
+  private $collation;
+  private $tables = array();
+
+  public function addTable(PhabricatorConfigTableSchema $table) {
+    $key = $table->getName();
+    if (isset($this->tables[$key])) {
+      throw new Exception(
+        pht('Trying to add duplicate table "%s"!', $key));
+    }
+    $this->tables[$key] = $table;
+    return $this;
+  }
+
+  public function getTables() {
+    return $this->tables;
+  }
+
+  public function getTable($key) {
+    return idx($this->tables, $key);
+  }
+
+  public function isSameSchema(PhabricatorConfigDatabaseSchema $expect) {
+    return ($this->toDictionary() === $expect->toDictionary());
+  }
+
+  public function toDictionary() {
+    return array(
+      'name' => $this->getName(),
+      'characterSet' => $this->getCharacterSet(),
+      'collation' => $this->getCollation(),
+    );
+  }
+
+  public function setCollation($collation) {
+    $this->collation = $collation;
+    return $this;
+  }
+
+  public function getCollation() {
+    return $this->collation;
+  }
+
+  public function setCharacterSet($character_set) {
+    $this->characterSet = $character_set;
+    return $this;
+  }
+
+  public function getCharacterSet() {
+    return $this->characterSet;
+  }
+
+  public function setName($name) {
+    $this->name = $name;
+    return $this;
+  }
+
+  public function getName() {
+    return $this->name;
+  }
+
+}

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -0,0 +1,135 @@
+<?php
+
+final class PhabricatorConfigSchemaQuery extends Phobject {
+
+  private $api;
+
+  public function setAPI(PhabricatorStorageManagementAPI $api) {
+    $this->api = $api;
+    return $this;
+  }
+
+  protected function getAPI() {
+    if (!$this->api) {
+      throw new Exception(pht('Call setAPI() before issuing a query!'));
+    }
+    return $this->api;
+  }
+
+  protected function getConn() {
+    return $this->getAPI()->getConn(null);
+  }
+
+  private function getDatabaseNames() {
+    $api = $this->getAPI();
+    $patches = PhabricatorSQLPatchList::buildAllPatches();
+    return $api->getDatabaseList(
+      $patches,
+      $only_living = true);
+  }
+
+  public function loadActualSchema() {
+    $databases = $this->getDatabaseNames();
+
+    $conn = $this->getConn();
+    $tables = queryfx_all(
+      $conn,
+      'SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_COLLATION
+        FROM INFORMATION_SCHEMA.TABLES
+        WHERE TABLE_SCHEMA IN (%Ls)',
+      $databases);
+
+    $database_info = queryfx_all(
+      $conn,
+      'SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME
+        FROM INFORMATION_SCHEMA.SCHEMATA
+        WHERE SCHEMA_NAME IN (%Ls)',
+      $databases);
+    $database_info = ipull($database_info, null, 'SCHEMA_NAME');
+
+    $server_schema = new PhabricatorConfigServerSchema();
+
+    $tables = igroup($tables, 'TABLE_SCHEMA');
+    foreach ($tables as $database_name => $database_tables) {
+      $info = $database_info[$database_name];
+
+      $database_schema = id(new PhabricatorConfigDatabaseSchema())
+        ->setName($database_name)
+        ->setCharacterSet($info['DEFAULT_CHARACTER_SET_NAME'])
+        ->setCollation($info['DEFAULT_COLLATION_NAME']);
+
+      foreach ($database_tables as $table) {
+        $table_name = $table['TABLE_NAME'];
+
+        $table_schema = id(new PhabricatorConfigTableSchema())
+          ->setName($table_name)
+          ->setCollation($table['TABLE_COLLATION']);
+
+        $columns = queryfx_all(
+          $conn,
+          'SELECT COLUMN_NAME, CHARACTER_SET_NAME, COLLATION_NAME, COLUMN_TYPE
+            FROM INFORMATION_SCHEMA.COLUMNS
+            WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s',
+          $database_name,
+          $table_name);
+
+        foreach ($columns as $column) {
+          $column_schema = id(new PhabricatorConfigColumnSchema())
+            ->setName($column['COLUMN_NAME'])
+            ->setCharacterSet($column['CHARACTER_SET_NAME'])
+            ->setCollation($column['COLLATION_NAME'])
+            ->setColumnType($column['COLUMN_TYPE']);
+
+          $table_schema->addColumn($column_schema);
+        }
+
+        $database_schema->addTable($table_schema);
+      }
+
+      $server_schema->addDatabase($database_schema);
+    }
+
+    return $server_schema;
+  }
+
+
+  public function loadExpectedSchema() {
+    $databases = $this->getDatabaseNames();
+
+    $api = $this->getAPI();
+
+    if ($api->isCharacterSetAvailable('utf8mb4')) {
+      // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
+      // collation. This is most correct, and will sort properly.
+
+      $utf8_charset = 'utf8mb4';
+      $utf8_collate = 'utf8mb4_unicode_ci';
+    } else {
+      // If utf8mb4 is not available, we use binary. This allows us to store
+      // 4-byte unicode characters. This has some tradeoffs:
+      //
+      // Unicode characters won't sort correctly. There's nothing we can do
+      // about this while still supporting 4-byte characters.
+      //
+      // It's possible that strings will be truncated in the middle of a
+      // character on insert. We encourage users to set STRICT_ALL_TABLES
+      // to prevent this.
+
+      $utf8_charset = 'binary';
+      $utf8_collate = 'binary';
+    }
+
+    $server_schema = new PhabricatorConfigServerSchema();
+    foreach ($databases as $database_name) {
+      $database_schema = id(new PhabricatorConfigDatabaseSchema())
+        ->setName($database_name)
+        ->setCharacterSet($utf8_charset)
+        ->setCollation($utf8_collate);
+
+      $server_schema->addDatabase($database_schema);
+    }
+
+    return $server_schema;
+  }
+
+}

src/applications/config/schema/PhabricatorConfigServerSchema.php

@@ -0,0 +1,25 @@
+<?php
+
+final class PhabricatorConfigServerSchema extends Phobject {
+
+  private $databases = array();
+
+  public function addDatabase(PhabricatorConfigDatabaseSchema $database) {
+    $key = $database->getName();
+    if (isset($this->databases[$key])) {
+      throw new Exception(
+        pht('Trying to add duplicate database "%s"!', $key));
+    }
+    $this->databases[$key] = $database;
+    return $this;
+  }
+
+  public function getDatabases() {
+    return $this->databases;
+  }
+
+  public function getDatabase($key) {
+    return idx($this->getDatabases(), $key);
+  }
+
+}

src/applications/config/schema/PhabricatorConfigTableSchema.php

@@ -0,0 +1,41 @@
+<?php
+
+final class PhabricatorConfigTableSchema extends Phobject {
+
+  private $name;
+  private $collation;
+  private $columns = array();
+
+  public function addColumn(PhabricatorConfigColumnSchema $column) {
+    $key = $column->getName();
+    if (isset($this->columns[$key])) {
+      throw new Exception(
+        pht('Trying to add duplicate column "%s"!', $key));
+    }
+    $this->columns[$key] = $column;
+    return $this;
+  }
+
+  public function getColumns() {
+    return $this->columns;
+  }
+
+  public function setCollation($collation) {
+    $this->collation = $collation;
+    return $this;
+  }
+
+  public function getCollation() {
+    return $this->collation;
+  }
+
+  public function setName($name) {
+    $this->name = $name;
+    return $this;
+  }
+
+  public function getName() {
+    return $this->name;
+  }
+
+}

src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php

@@ -196,4 +196,16 @@ final class PhabricatorStorageManagementAPI {
     require_once $script;
   }
 
+  public function isCharacterSetAvailable($character_set) {
+    $conn = $this->getConn(null);
+
+    $result = queryfx_one(
+      $conn,
+      'SELECT CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.CHARACTER_SETS
+        WHERE CHARACTER_SET_NAME = %s',
+      $character_set);
+
+    return (bool)$result;
+  }
+
 }