From 0439645d5bb062c38e8df664ab51d706caff3d55 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 9 Apr 2016 15:21:29 -0700 Subject: [PATCH] Add a "Database Cluster Status" console in Config Summary: Ref T4571. The configuration option still doesn't do anything, but add a status panel for basic setup monitoring. Test Plan: Here's what a good version looks like: {F1212291} Also faked most of the errors it can detect and got helpful diagnostic messages like this: {F1212292} Reviewers: chad Reviewed By: chad Maniphest Tasks: T4571 Differential Revision: https://secure.phabricator.com/D15667 --- src/__phutil_library_map__.php | 4 + .../PhabricatorConfigApplication.php | 3 + ...icatorConfigClusterDatabasesController.php | 183 ++++++++++ .../PhabricatorConfigController.php | 2 + .../user/cluster/cluster_databases.diviner | 6 +- .../cluster/PhabricatorDatabaseRef.php | 321 ++++++++++++++++++ 6 files changed, 518 insertions(+), 1 deletion(-) create mode 100644 src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php create mode 100644 src/infrastructure/cluster/PhabricatorDatabaseRef.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 5e6ffbe0af..7c6537f3b5 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2029,6 +2029,7 @@ phutil_register_library_map(array( 'PhabricatorConfigAllController' => 'applications/config/controller/PhabricatorConfigAllController.php', 'PhabricatorConfigApplication' => 'applications/config/application/PhabricatorConfigApplication.php', 'PhabricatorConfigCacheController' => 'applications/config/controller/PhabricatorConfigCacheController.php', + 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -2235,6 +2236,7 @@ phutil_register_library_map(array( 'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php', 'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php', 'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php', + 'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php', 'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php', 'PhabricatorDatasourceEditField' => 'applications/transactions/editfield/PhabricatorDatasourceEditField.php', 'PhabricatorDatasourceEditType' => 'applications/transactions/edittype/PhabricatorDatasourceEditType.php', @@ -6441,6 +6443,7 @@ phutil_register_library_map(array( 'PhabricatorConfigAllController' => 'PhabricatorConfigController', 'PhabricatorConfigApplication' => 'PhabricatorApplication', 'PhabricatorConfigCacheController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', @@ -6683,6 +6686,7 @@ phutil_register_library_map(array( 'PhabricatorDashboardViewController' => 'PhabricatorDashboardController', 'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec', 'PhabricatorDataNotAttachedException' => 'Exception', + 'PhabricatorDatabaseRef' => 'Phobject', 'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorDatasourceEditField' => 'PhabricatorTokenizerEditField', 'PhabricatorDatasourceEditType' => 'PhabricatorPHIDListEditType', diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php index 8570484e31..65e1486ce1 100644 --- a/src/applications/config/application/PhabricatorConfigApplication.php +++ b/src/applications/config/application/PhabricatorConfigApplication.php @@ -62,6 +62,9 @@ final class PhabricatorConfigApplication extends PhabricatorApplication { 'module/' => array( '(?P[^/]+)/' => 'PhabricatorConfigModuleController', ), + 'cluster/' => array( + 'databases/' => 'PhabricatorConfigClusterDatabasesController', + ), ), ); } diff --git a/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php new file mode 100644 index 0000000000..e4633ac727 --- /dev/null +++ b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php @@ -0,0 +1,183 @@ +buildSideNavView(); + $nav->selectFilter('cluster/databases/'); + + $title = pht('Cluster Databases'); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb(pht('Cluster Databases')); + + $database_status = $this->buildClusterDatabaseStatus(); + + $view = id(new PHUITwoColumnView()) + ->setNavigation($nav) + ->setMainColumn($database_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->appendChild($view); + } + + private function buildClusterDatabaseStatus() { + $viewer = $this->getViewer(); + + $databases = PhabricatorDatabaseRef::queryAll(); + $connection_map = PhabricatorDatabaseRef::getConnectionStatusMap(); + $replica_map = PhabricatorDatabaseRef::getReplicaStatusMap(); + Javelin::initBehavior('phabricator-tooltips'); + + $rows = array(); + foreach ($databases as $database) { + if ($database->getIsMaster()) { + $role_icon = id(new PHUIIconView()) + ->setIcon('fa-database sky') + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => pht('Master'), + )); + } else { + $role_icon = id(new PHUIIconView()) + ->setIcon('fa-download') + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => pht('Replica'), + )); + } + + if ($database->getDisabled()) { + $conn_icon = 'fa-times'; + $conn_color = 'grey'; + $conn_label = pht('Disabled'); + } else { + $status = $database->getConnectionStatus(); + + $info = idx($connection_map, $status, array()); + $conn_icon = idx($info, 'icon'); + $conn_color = idx($info, 'color'); + $conn_label = idx($info, 'label'); + + if ($status === PhabricatorDatabaseRef::STATUS_OKAY) { + $latency = $database->getConnectionLatency(); + $latency = (int)(1000000 * $latency); + $conn_label = pht('%s us', new PhutilNumber($latency)); + } + } + + $connection = array( + id(new PHUIIconView())->setIcon("{$conn_icon} {$conn_color}"), + ' ', + $conn_label, + ); + + if ($database->getDisabled()) { + $replica_icon = 'fa-times'; + $replica_color = 'grey'; + $replica_label = pht('Disabled'); + } else { + $status = $database->getReplicaStatus(); + + $info = idx($replica_map, $status, array()); + $replica_icon = idx($info, 'icon'); + $replica_color = idx($info, 'color'); + $replica_label = idx($info, 'label'); + + if ($database->getIsMaster()) { + if ($status === PhabricatorDatabaseRef::REPLICATION_OKAY) { + $replica_icon = 'fa-database'; + } + } else { + switch ($status) { + case PhabricatorDatabaseRef::REPLICATION_OKAY: + case PhabricatorDatabaseRef::REPLICATION_SLOW: + $delay = $database->getReplicaDelay(); + if ($delay) { + $replica_label = pht('%ss Behind', new PhutilNumber($delay)); + } else { + $replica_label = pht('Up to Date'); + } + break; + } + } + } + + $replication = array( + id(new PHUIIconView())->setIcon("{$replica_icon} {$replica_color}"), + ' ', + $replica_label, + ); + + $messages = array(); + + $conn_message = $database->getConnectionMessage(); + if ($conn_message) { + $messages[] = $conn_message; + } + + $replica_message = $database->getReplicaMessage(); + if ($replica_message) { + $messages[] = $replica_message; + } + + $messages = phutil_implode_html(phutil_tag('br'), $messages); + + $rows[] = array( + $role_icon, + $database->getHost(), + $database->getPort(), + $database->getUser(), + $connection, + $replication, + $messages, + ); + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString( + pht('Phabricator is not configured in cluster mode.')) + ->setHeaders( + array( + null, + pht('Host'), + pht('Port'), + pht('User'), + pht('Connection'), + pht('Replication'), + pht('Messages'), + )) + ->setColumnClasses( + array( + null, + null, + null, + null, + null, + null, + 'wide', + )); + + $doc_href = PhabricatorEnv::getDoclink('Cluster: Databases'); + + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Cluster Database Status')) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Database Clustering Documentation'))); + + return id(new PHUIObjectBoxView()) + ->setHeader($header) + ->setTable($table); + } + +} diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php index a752c2754c..6efe8923f5 100644 --- a/src/applications/config/controller/PhabricatorConfigController.php +++ b/src/applications/config/controller/PhabricatorConfigController.php @@ -22,6 +22,8 @@ abstract class PhabricatorConfigController extends PhabricatorController { $nav->addFilter('dbissue/', pht('Database Issues')); $nav->addLabel(pht('Cache')); $nav->addFilter('cache/', pht('Cache Status')); + $nav->addLabel(pht('Cluster')); + $nav->addFilter('cluster/databases/', pht('Cluster Databases')); $nav->addLabel(pht('Welcome')); $nav->addFilter('welcome/', pht('Welcome Screen')); $nav->addLabel(pht('Modules')); diff --git a/src/docs/user/cluster/cluster_databases.diviner b/src/docs/user/cluster/cluster_databases.diviner index 176fb718a8..96ee3ad44c 100644 --- a/src/docs/user/cluster/cluster_databases.diviner +++ b/src/docs/user/cluster/cluster_databases.diviner @@ -65,7 +65,11 @@ effect, then continue to "Monitoring and Testing" to verify the configuration. Monitoring and Testing ====================== -TODO: Write this part. +You can monitor replicas in {nav Config > Cluster Databases}. This interface +shows you a quick overview of replicas and their health, and can detect some +common issues with replication. + +TODO: Write more stuff here. Degradation to Read-Only Mode ============================= diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php new file mode 100644 index 0000000000..b8180bda33 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -0,0 +1,321 @@ +host = $host; + return $this; + } + + public function getHost() { + return $this->host; + } + + public function setPort($port) { + $this->port = $port; + return $this; + } + + public function getPort() { + return $this->port; + } + + public function setUser($user) { + $this->user = $user; + return $this; + } + + public function getUser() { + return $this->user; + } + + public function setPass(PhutilOpaqueEnvelope $pass) { + $this->pass = $pass; + return $this; + } + + public function getPass() { + return $this->pass; + } + + public function setIsMaster($is_master) { + $this->isMaster = $is_master; + return $this; + } + + public function getIsMaster() { + return $this->isMaster; + } + + public function setDisabled($disabled) { + $this->disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public function setConnectionLatency($connection_latency) { + $this->connectionLatency = $connection_latency; + return $this; + } + + public function getConnectionLatency() { + return $this->connectionLatency; + } + + public function setConnectionStatus($connection_status) { + $this->connectionStatus = $connection_status; + return $this; + } + + public function getConnectionStatus() { + if ($this->connectionStatus === null) { + throw new PhutilInvalidStateException('queryAll'); + } + + return $this->connectionStatus; + } + + public function setConnectionMessage($connection_message) { + $this->connectionMessage = $connection_message; + return $this; + } + + public function getConnectionMessage() { + return $this->connectionMessage; + } + + public function setReplicaStatus($replica_status) { + $this->replicaStatus = $replica_status; + return $this; + } + + public function getReplicaStatus() { + return $this->replicaStatus; + } + + public function setReplicaMessage($replica_message) { + $this->replicaMessage = $replica_message; + return $this; + } + + public function getReplicaMessage() { + return $this->replicaMessage; + } + + public function setReplicaDelay($replica_delay) { + $this->replicaDelay = $replica_delay; + return $this; + } + + public function getReplicaDelay() { + return $this->replicaDelay; + } + + public static function getConnectionStatusMap() { + return array( + self::STATUS_OKAY => array( + 'icon' => 'fa-exchange', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::STATUS_FAIL => array( + 'icon' => 'fa-times', + 'color' => 'red', + 'label' => pht('Failed'), + ), + self::STATUS_AUTH => array( + 'icon' => 'fa-key', + 'color' => 'red', + 'label' => pht('Invalid Credentials'), + ), + self::STATUS_REPLICATION_CLIENT => array( + 'icon' => 'fa-eye-slash', + 'color' => 'yellow', + 'label' => pht('Missing Permission'), + ), + ); + } + + public static function getReplicaStatusMap() { + return array( + self::REPLICATION_OKAY => array( + 'icon' => 'fa-download', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::REPLICATION_MASTER_REPLICA => array( + 'icon' => 'fa-database', + 'color' => 'red', + 'label' => pht('Replicating Master'), + ), + self::REPLICATION_REPLICA_NONE => array( + 'icon' => 'fa-download', + 'color' => 'red', + 'label' => pht('Not Replicating'), + ), + self::REPLICATION_SLOW => array( + 'icon' => 'fa-hourglass', + 'color' => 'red', + 'label' => pht('Slow Replication'), + ), + ); + } + + public static function loadAll() { + $refs = array(); + + $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); + $default_port = nonempty($default_port, 3306); + + $default_user = PhabricatorEnv::getEnvConfig('mysql.user'); + + $default_pass = PhabricatorEnv::getEnvConfig('mysql.pass'); + $default_pass = new PhutilOpaqueEnvelope($default_pass); + + $config = PhabricatorEnv::getEnvConfig('cluster.databases'); + foreach ($config as $server) { + $host = $server['host']; + $port = idx($server, 'port', $default_port); + $user = idx($server, 'user', $default_user); + $disabled = idx($server, 'disabled', false); + + $pass = idx($server, 'pass'); + if ($pass) { + $pass = new PhutilOpaqueEnvelope($pass); + } else { + $pass = clone $default_pass; + } + + $role = $server['role']; + + $ref = id(new self()) + ->setHost($host) + ->setPort($port) + ->setUser($user) + ->setPass($pass) + ->setDisabled($disabled) + ->setIsMaster(($role == 'master')); + + $refs[] = $ref; + } + + return $refs; + } + + public static function queryAll() { + $refs = self::loadAll(); + + foreach ($refs as $ref) { + if ($ref->getDisabled()) { + continue; + } + + $conn = $ref->newConnection(); + + $t_start = microtime(true); + try { + $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS'); + $ref->setConnectionStatus(self::STATUS_OKAY); + } catch (AphrontAccessDeniedQueryException $ex) { + $ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT); + $ref->setConnectionMessage( + pht( + 'No permission to run "SHOW SLAVE STATUS". Grant this user '. + '"REPLICATION CLIENT" permission to allow Phabricator to '. + 'monitor replica health.')); + } catch (AphrontInvalidCredentialsQueryException $ex) { + $ref->setConnectionStatus(self::STATUS_AUTH); + $ref->setConnectionMessage($ex->getMessage()); + } catch (AphrontQueryException $ex) { + $ref->setConnectionStatus(self::STATUS_FAIL); + + $class = get_class($ex); + $message = $ex->getMessage(); + $ref->setConnectionMessage( + pht( + '%s: %s', + get_class($ex), + $ex->getMessage())); + } + $t_end = microtime(true); + $ref->setConnectionLatency($t_end - $t_start); + + $is_replica = (bool)$replica_status; + if ($ref->getIsMaster() && $is_replica) { + $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA); + $ref->setReplicaMessage( + pht( + 'This host has a "master" role, but is replicating data from '. + 'another host ("%s")!', + idx($replica_status, 'Master_Host'))); + } else if (!$ref->getIsMaster() && !$is_replica) { + $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE); + $ref->setReplicaMessage( + pht( + 'This host has a "replica" role, but is not replicating data '. + 'from a master (no output from "SHOW SLAVE STATUS").')); + } else { + $ref->setReplicaStatus(self::REPLICATION_OKAY); + } + + if ($is_replica) { + $latency = (int)idx($replica_status, 'Seconds_Behind_Master'); + $ref->setReplicaDelay($latency); + if ($latency > 30) { + $ref->setReplicaStatus(self::REPLICATION_SLOW); + $ref->setReplicaMessage( + pht( + 'This replica is lagging far behind the master. Data is at '. + 'risk!')); + } + } + } + + return $refs; + } + + protected function newConnection() { + return PhabricatorEnv::newObjectFromConfig( + 'mysql.implementation', + array( + array( + 'user' => $this->getUser(), + 'pass' => $this->getPass(), + 'host' => $this->getHost(), + 'port' => $this->getPort(), + 'database' => null, + 'retries' => 0, + ), + )); + } + +}