Add a "Database Cluster Status" console in Config

Summary: Ref T4571. The configuration option still doesn't do anything, but add a status panel for basic setup monitoring.

Test Plan:
Here's what a good version looks like:

{F1212291}

Also faked most of the errors it can detect and got helpful diagnostic messages like this:

{F1212292}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T4571

Differential Revision: https://secure.phabricator.com/D15667

src/__phutil_library_map__.php

@@ -2029,6 +2029,7 @@ phutil_register_library_map(array(
     'PhabricatorConfigAllController' => 'applications/config/controller/PhabricatorConfigAllController.php',
     'PhabricatorConfigApplication' => 'applications/config/application/PhabricatorConfigApplication.php',
     'PhabricatorConfigCacheController' => 'applications/config/controller/PhabricatorConfigCacheController.php',
+    'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
     'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
     'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
     'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
@@ -2235,6 +2236,7 @@ phutil_register_library_map(array(
     'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
     'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
     'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
+    'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
     'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
     'PhabricatorDatasourceEditField' => 'applications/transactions/editfield/PhabricatorDatasourceEditField.php',
     'PhabricatorDatasourceEditType' => 'applications/transactions/edittype/PhabricatorDatasourceEditType.php',
@@ -6441,6 +6443,7 @@ phutil_register_library_map(array(
     'PhabricatorConfigAllController' => 'PhabricatorConfigController',
     'PhabricatorConfigApplication' => 'PhabricatorApplication',
     'PhabricatorConfigCacheController' => 'PhabricatorConfigController',
+    'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
     'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
     'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
     'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
@@ -6683,6 +6686,7 @@ phutil_register_library_map(array(
     'PhabricatorDashboardViewController' => 'PhabricatorDashboardController',
     'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
     'PhabricatorDataNotAttachedException' => 'Exception',
+    'PhabricatorDatabaseRef' => 'Phobject',
     'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorDatasourceEditField' => 'PhabricatorTokenizerEditField',
     'PhabricatorDatasourceEditType' => 'PhabricatorPHIDListEditType',

src/applications/config/application/PhabricatorConfigApplication.php

@@ -62,6 +62,9 @@ final class PhabricatorConfigApplication extends PhabricatorApplication {
         'module/' => array(
           '(?P<module>[^/]+)/' => 'PhabricatorConfigModuleController',
         ),
+        'cluster/' => array(
+          'databases/' => 'PhabricatorConfigClusterDatabasesController',
+        ),
       ),
     );
   }

src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php

@@ -0,0 +1,183 @@
+<?php
+
+final class PhabricatorConfigClusterDatabasesController
+  extends PhabricatorConfigController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $nav = $this->buildSideNavView();
+    $nav->selectFilter('cluster/databases/');
+
+    $title = pht('Cluster Databases');
+
+    $crumbs = $this
+      ->buildApplicationCrumbs($nav)
+      ->addTextCrumb(pht('Cluster Databases'));
+
+    $database_status = $this->buildClusterDatabaseStatus();
+
+    $view = id(new PHUITwoColumnView())
+      ->setNavigation($nav)
+      ->setMainColumn($database_status);
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs($crumbs)
+      ->appendChild($view);
+  }
+
+  private function buildClusterDatabaseStatus() {
+    $viewer = $this->getViewer();
+
+    $databases = PhabricatorDatabaseRef::queryAll();
+    $connection_map = PhabricatorDatabaseRef::getConnectionStatusMap();
+    $replica_map = PhabricatorDatabaseRef::getReplicaStatusMap();
+    Javelin::initBehavior('phabricator-tooltips');
+
+    $rows = array();
+    foreach ($databases as $database) {
+      if ($database->getIsMaster()) {
+        $role_icon = id(new PHUIIconView())
+          ->setIcon('fa-database sky')
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => pht('Master'),
+            ));
+      } else {
+        $role_icon = id(new PHUIIconView())
+          ->setIcon('fa-download')
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => pht('Replica'),
+            ));
+      }
+
+      if ($database->getDisabled()) {
+        $conn_icon = 'fa-times';
+        $conn_color = 'grey';
+        $conn_label = pht('Disabled');
+      } else {
+        $status = $database->getConnectionStatus();
+
+        $info = idx($connection_map, $status, array());
+        $conn_icon = idx($info, 'icon');
+        $conn_color = idx($info, 'color');
+        $conn_label = idx($info, 'label');
+
+        if ($status === PhabricatorDatabaseRef::STATUS_OKAY) {
+          $latency = $database->getConnectionLatency();
+          $latency = (int)(1000000 * $latency);
+          $conn_label = pht('%s us', new PhutilNumber($latency));
+        }
+      }
+
+      $connection = array(
+        id(new PHUIIconView())->setIcon("{$conn_icon} {$conn_color}"),
+        ' ',
+        $conn_label,
+      );
+
+      if ($database->getDisabled()) {
+        $replica_icon = 'fa-times';
+        $replica_color = 'grey';
+        $replica_label = pht('Disabled');
+      } else {
+        $status = $database->getReplicaStatus();
+
+        $info = idx($replica_map, $status, array());
+        $replica_icon = idx($info, 'icon');
+        $replica_color = idx($info, 'color');
+        $replica_label = idx($info, 'label');
+
+        if ($database->getIsMaster()) {
+          if ($status === PhabricatorDatabaseRef::REPLICATION_OKAY) {
+            $replica_icon = 'fa-database';
+          }
+        } else {
+          switch ($status) {
+            case PhabricatorDatabaseRef::REPLICATION_OKAY:
+            case PhabricatorDatabaseRef::REPLICATION_SLOW:
+              $delay = $database->getReplicaDelay();
+              if ($delay) {
+                $replica_label = pht('%ss Behind', new PhutilNumber($delay));
+              } else {
+                $replica_label = pht('Up to Date');
+              }
+              break;
+          }
+        }
+      }
+
+      $replication = array(
+        id(new PHUIIconView())->setIcon("{$replica_icon} {$replica_color}"),
+        ' ',
+        $replica_label,
+      );
+
+      $messages = array();
+
+      $conn_message = $database->getConnectionMessage();
+      if ($conn_message) {
+        $messages[] = $conn_message;
+      }
+
+      $replica_message = $database->getReplicaMessage();
+      if ($replica_message) {
+        $messages[] = $replica_message;
+      }
+
+      $messages = phutil_implode_html(phutil_tag('br'), $messages);
+
+      $rows[] = array(
+        $role_icon,
+        $database->getHost(),
+        $database->getPort(),
+        $database->getUser(),
+        $connection,
+        $replication,
+        $messages,
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setNoDataString(
+        pht('Phabricator is not configured in cluster mode.'))
+      ->setHeaders(
+        array(
+          null,
+          pht('Host'),
+          pht('Port'),
+          pht('User'),
+          pht('Connection'),
+          pht('Replication'),
+          pht('Messages'),
+        ))
+      ->setColumnClasses(
+        array(
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          'wide',
+        ));
+
+    $doc_href = PhabricatorEnv::getDoclink('Cluster: Databases');
+
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Cluster Database Status'))
+      ->addActionLink(
+        id(new PHUIButtonView())
+          ->setIcon('fa-book')
+          ->setHref($doc_href)
+          ->setTag('a')
+          ->setText(pht('Database Clustering Documentation')));
+
+    return id(new PHUIObjectBoxView())
+      ->setHeader($header)
+      ->setTable($table);
+  }
+
+}

src/applications/config/controller/PhabricatorConfigController.php

@@ -22,6 +22,8 @@ abstract class PhabricatorConfigController extends PhabricatorController {
     $nav->addFilter('dbissue/', pht('Database Issues'));
     $nav->addLabel(pht('Cache'));
     $nav->addFilter('cache/', pht('Cache Status'));
+    $nav->addLabel(pht('Cluster'));
+    $nav->addFilter('cluster/databases/', pht('Cluster Databases'));
     $nav->addLabel(pht('Welcome'));
     $nav->addFilter('welcome/', pht('Welcome Screen'));
     $nav->addLabel(pht('Modules'));

src/docs/user/cluster/cluster_databases.diviner

@@ -65,7 +65,11 @@ effect, then continue to "Monitoring and Testing" to verify the configuration.
 Monitoring and Testing
 ======================
 
-TODO: Write this part.
+You can monitor replicas in {nav Config > Cluster Databases}. This interface
+shows you a quick overview of replicas and their health, and can detect some
+common issues with replication.
+
+TODO: Write more stuff here.
 
 Degradation to Read-Only Mode
 =============================

src/infrastructure/cluster/PhabricatorDatabaseRef.php

@@ -0,0 +1,321 @@
+<?php
+
+final class PhabricatorDatabaseRef
+  extends Phobject {
+
+  const STATUS_OKAY = 'okay';
+  const STATUS_FAIL = 'fail';
+  const STATUS_AUTH = 'auth';
+  const STATUS_REPLICATION_CLIENT = 'replication-client';
+
+  const REPLICATION_OKAY = 'okay';
+  const REPLICATION_MASTER_REPLICA = 'master-replica';
+  const REPLICATION_REPLICA_NONE = 'replica-none';
+  const REPLICATION_SLOW = 'replica-slow';
+
+  private $host;
+  private $port;
+  private $user;
+  private $pass;
+  private $disabled;
+  private $isMaster;
+
+  private $connectionLatency;
+  private $connectionStatus;
+  private $connectionMessage;
+
+  private $replicaStatus;
+  private $replicaMessage;
+  private $replicaDelay;
+
+  public function setHost($host) {
+    $this->host = $host;
+    return $this;
+  }
+
+  public function getHost() {
+    return $this->host;
+  }
+
+  public function setPort($port) {
+    $this->port = $port;
+    return $this;
+  }
+
+  public function getPort() {
+    return $this->port;
+  }
+
+  public function setUser($user) {
+    $this->user = $user;
+    return $this;
+  }
+
+  public function getUser() {
+    return $this->user;
+  }
+
+  public function setPass(PhutilOpaqueEnvelope $pass) {
+    $this->pass = $pass;
+    return $this;
+  }
+
+  public function getPass() {
+    return $this->pass;
+  }
+
+  public function setIsMaster($is_master) {
+    $this->isMaster = $is_master;
+    return $this;
+  }
+
+  public function getIsMaster() {
+    return $this->isMaster;
+  }
+
+  public function setDisabled($disabled) {
+    $this->disabled = $disabled;
+    return $this;
+  }
+
+  public function getDisabled() {
+    return $this->disabled;
+  }
+
+  public function setConnectionLatency($connection_latency) {
+    $this->connectionLatency = $connection_latency;
+    return $this;
+  }
+
+  public function getConnectionLatency() {
+    return $this->connectionLatency;
+  }
+
+  public function setConnectionStatus($connection_status) {
+    $this->connectionStatus = $connection_status;
+    return $this;
+  }
+
+  public function getConnectionStatus() {
+    if ($this->connectionStatus === null) {
+      throw new PhutilInvalidStateException('queryAll');
+    }
+
+    return $this->connectionStatus;
+  }
+
+  public function setConnectionMessage($connection_message) {
+    $this->connectionMessage = $connection_message;
+    return $this;
+  }
+
+  public function getConnectionMessage() {
+    return $this->connectionMessage;
+  }
+
+  public function setReplicaStatus($replica_status) {
+    $this->replicaStatus = $replica_status;
+    return $this;
+  }
+
+  public function getReplicaStatus() {
+    return $this->replicaStatus;
+  }
+
+  public function setReplicaMessage($replica_message) {
+    $this->replicaMessage = $replica_message;
+    return $this;
+  }
+
+  public function getReplicaMessage() {
+    return $this->replicaMessage;
+  }
+
+  public function setReplicaDelay($replica_delay) {
+    $this->replicaDelay = $replica_delay;
+    return $this;
+  }
+
+  public function getReplicaDelay() {
+    return $this->replicaDelay;
+  }
+
+  public static function getConnectionStatusMap() {
+    return array(
+      self::STATUS_OKAY => array(
+        'icon' => 'fa-exchange',
+        'color' => 'green',
+        'label' => pht('Okay'),
+      ),
+      self::STATUS_FAIL => array(
+        'icon' => 'fa-times',
+        'color' => 'red',
+        'label' => pht('Failed'),
+      ),
+      self::STATUS_AUTH => array(
+        'icon' => 'fa-key',
+        'color' => 'red',
+        'label' => pht('Invalid Credentials'),
+      ),
+      self::STATUS_REPLICATION_CLIENT => array(
+        'icon' => 'fa-eye-slash',
+        'color' => 'yellow',
+        'label' => pht('Missing Permission'),
+      ),
+    );
+  }
+
+  public static function getReplicaStatusMap() {
+    return array(
+      self::REPLICATION_OKAY => array(
+        'icon' => 'fa-download',
+        'color' => 'green',
+        'label' => pht('Okay'),
+      ),
+      self::REPLICATION_MASTER_REPLICA => array(
+        'icon' => 'fa-database',
+        'color' => 'red',
+        'label' => pht('Replicating Master'),
+      ),
+      self::REPLICATION_REPLICA_NONE => array(
+        'icon' => 'fa-download',
+        'color' => 'red',
+        'label' => pht('Not Replicating'),
+      ),
+      self::REPLICATION_SLOW => array(
+        'icon' => 'fa-hourglass',
+        'color' => 'red',
+        'label' => pht('Slow Replication'),
+      ),
+    );
+  }
+
+  public static function loadAll() {
+    $refs = array();
+
+    $default_port = PhabricatorEnv::getEnvConfig('mysql.port');
+    $default_port = nonempty($default_port, 3306);
+
+    $default_user = PhabricatorEnv::getEnvConfig('mysql.user');
+
+    $default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
+    $default_pass = new PhutilOpaqueEnvelope($default_pass);
+
+    $config = PhabricatorEnv::getEnvConfig('cluster.databases');
+    foreach ($config as $server) {
+      $host = $server['host'];
+      $port = idx($server, 'port', $default_port);
+      $user = idx($server, 'user', $default_user);
+      $disabled = idx($server, 'disabled', false);
+
+      $pass = idx($server, 'pass');
+      if ($pass) {
+        $pass = new PhutilOpaqueEnvelope($pass);
+      } else {
+        $pass = clone $default_pass;
+      }
+
+      $role = $server['role'];
+
+      $ref = id(new self())
+        ->setHost($host)
+        ->setPort($port)
+        ->setUser($user)
+        ->setPass($pass)
+        ->setDisabled($disabled)
+        ->setIsMaster(($role == 'master'));
+
+      $refs[] = $ref;
+    }
+
+    return $refs;
+  }
+
+  public static function queryAll() {
+    $refs = self::loadAll();
+
+    foreach ($refs as $ref) {
+      if ($ref->getDisabled()) {
+        continue;
+      }
+
+      $conn = $ref->newConnection();
+
+      $t_start = microtime(true);
+      try {
+        $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
+        $ref->setConnectionStatus(self::STATUS_OKAY);
+      } catch (AphrontAccessDeniedQueryException $ex) {
+        $ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
+        $ref->setConnectionMessage(
+          pht(
+            'No permission to run "SHOW SLAVE STATUS". Grant this user '.
+            '"REPLICATION CLIENT" permission to allow Phabricator to '.
+            'monitor replica health.'));
+      } catch (AphrontInvalidCredentialsQueryException $ex) {
+        $ref->setConnectionStatus(self::STATUS_AUTH);
+        $ref->setConnectionMessage($ex->getMessage());
+      } catch (AphrontQueryException $ex) {
+        $ref->setConnectionStatus(self::STATUS_FAIL);
+
+        $class = get_class($ex);
+        $message = $ex->getMessage();
+        $ref->setConnectionMessage(
+          pht(
+            '%s: %s',
+            get_class($ex),
+            $ex->getMessage()));
+      }
+      $t_end = microtime(true);
+      $ref->setConnectionLatency($t_end - $t_start);
+
+      $is_replica = (bool)$replica_status;
+      if ($ref->getIsMaster() && $is_replica) {
+        $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
+        $ref->setReplicaMessage(
+          pht(
+            'This host has a "master" role, but is replicating data from '.
+            'another host ("%s")!',
+            idx($replica_status, 'Master_Host')));
+      } else if (!$ref->getIsMaster() && !$is_replica) {
+        $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
+        $ref->setReplicaMessage(
+          pht(
+            'This host has a "replica" role, but is not replicating data '.
+            'from a master (no output from "SHOW SLAVE STATUS").'));
+      } else {
+        $ref->setReplicaStatus(self::REPLICATION_OKAY);
+      }
+
+      if ($is_replica) {
+        $latency = (int)idx($replica_status, 'Seconds_Behind_Master');
+        $ref->setReplicaDelay($latency);
+        if ($latency > 30) {
+          $ref->setReplicaStatus(self::REPLICATION_SLOW);
+          $ref->setReplicaMessage(
+            pht(
+              'This replica is lagging far behind the master. Data is at '.
+              'risk!'));
+        }
+      }
+    }
+
+    return $refs;
+  }
+
+  protected function newConnection() {
+    return PhabricatorEnv::newObjectFromConfig(
+      'mysql.implementation',
+      array(
+        array(
+          'user'      => $this->getUser(),
+          'pass'      => $this->getPass(),
+          'host'      => $this->getHost(),
+          'port'      => $this->getPort(),
+          'database'  => null,
+          'retries'   => 0,
+        ),
+      ));
+  }
+
+}