Lay cluster.databases configuration groundwork for database clustering

Summary:
Ref T4571. This adds a new option which allows you to upgrade your one-host configuration to a multi-host configuration by configuring it.

Doing this currently does nothing. I wrote a lot of words about what it is //supposed// to do in the future, though.

Test Plan:
  - Tried to configure the option in all the possible bad ways, got errors.
  - Read documentation.

Reviewers: chad

Reviewed By: chad

Subscribers: eadler

Maniphest Tasks: T4571

Differential Revision: https://secure.phabricator.com/D15663

resources/celerity/map.php

@@ -7,8 +7,8 @@
  */
 return array(
   'names' => array(
-    'core.pkg.css' => '82cefddc',
-    'core.pkg.js' => 'e5484f37',
+    'core.pkg.css' => '35e4a99a',
+    'core.pkg.js' => '8a616602',
     'darkconsole.pkg.js' => 'e7393ebb',
     'differential.pkg.css' => '7ba78475',
     'differential.pkg.js' => 'd0cd0df6',
@@ -22,7 +22,7 @@ return array(
     'rsrc/css/aphront/lightbox-attachment.css' => '7acac05d',
     'rsrc/css/aphront/list-filter-view.css' => '5d6f0526',
     'rsrc/css/aphront/multi-column.css' => 'fd18389d',
-    'rsrc/css/aphront/notification.css' => '7f684b62',
+    'rsrc/css/aphront/notification.css' => '3f6c89c9',
     'rsrc/css/aphront/panel-view.css' => '8427b78d',
     'rsrc/css/aphront/phabricator-nav-view.css' => 'ac79a758',
     'rsrc/css/aphront/table-view.css' => '9258e19f',
@@ -494,6 +494,7 @@ return array(
     'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
     'rsrc/js/core/behavior-phabricator-nav.js' => '56a1ca03',
     'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => '340c8eff',
+    'rsrc/js/core/behavior-read-only-warning.js' => 'f8ea359c',
     'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b',
     'rsrc/js/core/behavior-remarkup-preview.js' => '4b700e9e',
     'rsrc/js/core/behavior-reorder-applications.js' => '76b9fc3e',
@@ -666,6 +667,7 @@ return array(
     'javelin-behavior-project-boards' => '14a1faae',
     'javelin-behavior-project-create' => '065227cc',
     'javelin-behavior-quicksand-blacklist' => '7927a7d3',
+    'javelin-behavior-read-only-warning' => 'f8ea359c',
     'javelin-behavior-recurring-edit' => '5f1c4d5f',
     'javelin-behavior-refresh-csrf' => 'ab2f381b',
     'javelin-behavior-releeph-preview-branch' => 'b2b4fbaf',
@@ -766,7 +768,7 @@ return array(
     'phabricator-main-menu-view' => 'd00a795a',
     'phabricator-nav-view-css' => 'ac79a758',
     'phabricator-notification' => 'ccf1cbf8',
-    'phabricator-notification-css' => '7f684b62',
+    'phabricator-notification-css' => '3f6c89c9',
     'phabricator-notification-menu-css' => 'f31c0bde',
     'phabricator-object-selector-css' => '85ee8ce6',
     'phabricator-phtize' => 'd254d646',
@@ -2109,6 +2111,11 @@ return array(
       'javelin-util',
       'phabricator-busy',
     ),
+    'f8ea359c' => array(
+      'javelin-behavior',
+      'javelin-uri',
+      'phabricator-notification',
+    ),
     'fa0f4fc2' => array(
       'javelin-behavior',
       'javelin-dom',
@@ -2284,6 +2291,7 @@ return array(
       'javelin-quicksand',
       'javelin-behavior-quicksand-blacklist',
       'javelin-behavior-high-security-warning',
+      'javelin-behavior-read-only-warning',
       'javelin-scrollbar',
       'javelin-behavior-scrollbar',
       'javelin-behavior-durable-column',

src/__phutil_library_map__.php

@@ -1986,6 +1986,7 @@ phutil_register_library_map(array(
     'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
     'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
     'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
+    'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
     'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
     'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
     'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@@ -6392,6 +6393,7 @@ phutil_register_library_map(array(
     'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
+    'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType',
     'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
     'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
     'PhabricatorCommentEditField' => 'PhabricatorEditField',

src/applications/config/check/PhabricatorMySQLSetupCheck.php

@@ -20,6 +20,12 @@ final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
   }
 
   protected function executeChecks() {
+    // TODO: These checks should be executed against every reachable replica?
+    // See T10759.
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
 
     // This primarily supports setting the filesize limit for MySQL to 8MB,

src/applications/config/option/PhabricatorClusterConfigOptions.php

@@ -20,6 +20,20 @@ final class PhabricatorClusterConfigOptions
   }
 
   public function getOptions() {
+    $databases_type = 'custom:PhabricatorClusterDatabasesConfigOptionType';
+    $databases_help = $this->deformat(pht(<<<EOTEXT
+WARNING: This is a prototype option and the description below is currently pure
+fantasy.
+
+This option allows you to make Phabricator aware of database read replicas so
+it can monitor database health, spread load, and degrade gracefully to
+read-only mode in the event of a failure on the primary host. For help with
+configuring cluster databases, see **[[ %s | %s ]]** in the documentation.
+EOTEXT
+      ,
+      PhabricatorEnv::getDoclink('Cluster: Databases'),
+      pht('Cluster: Databases')));
+
     return array(
       $this->newOption('cluster.addresses', 'list<string>', array())
         ->setLocked(true)
@@ -88,7 +102,11 @@ final class PhabricatorClusterConfigOptions
             'into this mode automatically when it detects that the database '.
             'master is unreachable, but you can activate it manually in '.
             'order to perform maintenance or test configuration.')),
-
+      $this->newOption('cluster.databases', $databases_type, array())
+        ->setHidden(true)
+        ->setSummary(
+          pht('Configure database read replicas.'))
+        ->setDescription($databases_help),
     );
   }
 

src/docs/book/user.book

@@ -32,6 +32,9 @@
     "conduit": {
       "name": "API Documentation"
     },
+    "cluster": {
+      "name": "Cluster Configuration"
+    },
     "fieldmanual": {
       "name": "Field Manuals"
     },

src/docs/user/cluster/cluster.diviner

@@ -0,0 +1,40 @@
+@title Clustering Introduction
+@group cluster
+
+Guide to configuring Phabricator across multiple hosts for availability and
+performance.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+Phabricator can be configured to run on mulitple hosts with redundant services
+to improve its availability and scalability, and make disaster recovery much
+easier.
+
+Clustering is more complex to setup and maintain than running everything on a
+single host, but greatly reduces the cost of recovering from hardware and
+network failures.
+
+Each Phabricator service has an array of clustering options that can be
+configured independently. Configuring a cluster is inherently complex, and this
+is an advanced feature aimed at installs with large userbases and experienced
+operations personnel who need this high degree of flexibility.
+
+The remainder of this document summarizes how to add redundancy to each
+service and where your efforts are likely to have the greatest impact.
+
+Cluster: Databases
+=================
+
+Configuring multiple database hosts is moderately complex, but normally has the
+highest impact on availability and resistance to data loss. This is usually the
+most important service to make redundant if your focus is on availability and
+disaster recovery.
+
+Configuring replicas allows Phabricator to run in read-only mode if you lose
+the master, and to quickly promote the replica as a replacement.
+
+For details, see @{article:Cluster: Databases}.

src/docs/user/cluster/cluster_databases.diviner

@@ -0,0 +1,161 @@
+@title Cluster: Databases
+@group intro
+
+Configuring Phabricator to use multiple database hosts.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can deploy Phabricator with multiple database hosts, configured as a master
+and a set of replicas. The advantages of doing this are:
+
+  - faster recovery from disasters by promoting a replica;
+  - graceful degradation if the master fails;
+  - reduced load on the master; and
+  - some tools to help monitor and manage replica health.
+
+This configuration is complex, and many installs do not need to pursue it.
+
+Phabricator can not currently be configured into a multi-master mode, nor can
+it be configured to automatically promote a replica to become the new master.
+
+
+Setting up MySQL Replication
+============================
+
+TODO: Write this section.
+
+
+Configuring Replicas
+====================
+
+Once your replicas are in working order, tell Phabricator about them by
+configuring the `cluster.database` option. This option must be configured from
+the command line or in configuration files because Phabricator needs to read
+it //before// it can connect to databases.
+
+This option value will list all of the database hosts that you want Phabricator
+to interact with: your master and all your replicas. Each entry in the list
+should have these keys:
+
+  - `host`: //Required string.// The database host name.
+  - `role`: //Required string.// The cluster role of this host, one of
+    `master` or `replica`.
+  - `port`: //Optional int.// The port to connect to. If omitted, the default
+    port from `mysql.port` will be used.
+  - `user`: //Optional string.// The MySQL username to use to connect to this
+    host. If omitted, the default from `mysql.user` will be used.
+  - `pass`: //Optional string.// The password to use to connect to this host.
+    If omitted, the default from `mysql.pass` will be used.
+  - `disabled`: //Optional bool.// If set to `true`, Phabricator will not
+    connect to this host. You can use this to temporarily take a host out
+    of service.
+
+When `cluster.databases` is configured the `mysql.host` option is not used.
+The other MySQL connection configuration options (`mysql.port`, `mysql.user`,
+`mysql.pass`) are used only to provide defaults.
+
+Once you've configured this option, restart Phabricator for the changes to take
+effect, then continue to "Monitoring and Testing" to verify the configuration.
+
+
+Monitoring and Testing
+======================
+
+TODO: Write this part.
+
+Degradation to Read-Only Mode
+=============================
+
+Phabricator will degrade to read-only mode when any of these conditions occur:
+
+  - you turn it on explicitly;
+  - you configure cluster mode, but don't set up any masters;
+  - the master is misconfigured and unsafe to write to; or
+  - the master is unreachable.
+
+When Phabricator is running in read-only mode, users can still read data and
+browse and clone repositories, but they can not edit, update, or push new
+changes. For example, users can still read disaster recovery information on
+the wiki or emergency contact information on user profiles.
+
+You can enable this mode explicitly by configuring `cluster.read-only`. Some
+reasons you might want to do this include:
+
+  - to test that the mode works like you expect it to;
+  - to make sure that information you need will be available;
+  - to prevent new writes while performing database maintenance; or
+  - to permanently archive a Phabricator install.
+
+You can also enable this mode implicitly by configuring `cluster.databases`
+but disabling the master, or by not specifying any host as a master. This may
+be more convenient than turning it on explicitly during the course of
+operations work.
+
+Before writing to a master, Phabricator will verify that the host is not
+configured as a replica. This is a safety feature to prevent data loss if your
+MySQL and Phabricator configurations disagree about replica configuration. If
+your `master` is currently replicating from another host, Phabricator will
+treat it as a `replica` instead and implicitly degrade into read-only mode.
+
+Finally, if Phabricator is unable to reach the master, it will degrade into
+read-only mode. For details on how Phabricator determines that a master is
+unreachable, see "Unreachable Masters" below.
+
+If a master becomes unreachable, this normally corresponds to loss of the
+master host, a severed network link, or some other sort of disaster.
+Phabricator will degrade and continue operating in read-only mode until the
+master recovers or operations personnel can assess the situation and intervene.
+
+If you end up in a situation where you have lost the master and can not get it
+back online (or can not restore it quickly) you can promote a replica to become
+the new master. See the next section, "Promoting a Replica", for details.
+
+
+Promoting a Replica
+===================
+
+TODO: Write this, too.
+
+
+Unreachable Masters
+===================
+
+This section describes how Phabricator determines that a master has been lost,
+marks it unreachable, and degrades into read-only mode.
+
+TODO: For now, it doesn't.
+
+
+Backups
+======
+
+Even if you configure replication, you should still retain separate backup
+snapshots. Replicas protect you from data loss if you lose a host, but they do
+not let you recover from data mutation mistakes.
+
+If something issues `DELETE` or `UPDATE` statements and destroys data on the
+master, the mutation will propagate to the replicas almost immediately and the
+data will be gone forever. Normally, the only way to recover this data is from
+backup snapshots.
+
+Although you should still have a backup process, your backup process can
+safely pull dumps from a replica instead of the master. This operation can
+be slow, so offloading it to a replica can make the perforance of the master
+more consistent.
+
+To dump from a replica, wait for this TODO to be resolved and then do whatever
+it says to do:
+
+TODO: Make `bin/storage dump` replica-aware. See T10758.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - returning to @{article:Clustering Introduction}.

src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php

@@ -0,0 +1,98 @@
+<?php
+
+final class PhabricatorClusterDatabasesConfigOptionType
+  extends PhabricatorConfigJSONOptionType {
+
+  public function validateOption(PhabricatorConfigOption $option, $value) {
+    if (!is_array($value)) {
+      throw new Exception(
+        pht(
+          'Database cluster configuration is not valid: value must be a '.
+          'list of database hosts.'));
+    }
+
+    foreach ($value as $index => $spec) {
+      if (!is_array($spec)) {
+        throw new Exception(
+          pht(
+            'Database cluster configuration is not valid: each entry in the '.
+            'list must be a dictionary describing a database host, but '.
+            'the value with index "%s" is not a dictionary.',
+            $index));
+      }
+    }
+
+    $masters = array();
+    $map = array();
+    foreach ($value as $index => $spec) {
+      try {
+        PhutilTypeSpec::checkMap(
+          $spec,
+          array(
+            'host' => 'string',
+            'role' => 'string',
+            'port' => 'optional int',
+            'user' => 'optional string',
+            'pass' => 'optional string',
+            'disabled' => 'optional bool',
+          ));
+      } catch (Exception $ex) {
+        throw new Exception(
+          pht(
+            'Database cluster configuration has an invalid host '.
+            'specification (at index "%s"): %s.',
+            $index,
+            $ex->getMessage()));
+      }
+
+      $role = $spec['role'];
+      $host = $spec['host'];
+      $port = idx($spec, 'port');
+
+      switch ($role) {
+        case 'master':
+        case 'replica':
+          break;
+        default:
+          throw new Exception(
+            pht(
+              'Database cluster configuration describes an invalid '.
+              'host ("%s", at index "%s") with an unrecognized role ("%s"). '.
+              'Valid roles are "%s" or "%s".',
+              $spec['host'],
+              $index,
+              $spec['role'],
+              'master',
+              'replica'));
+      }
+
+      if ($role === 'master') {
+        $masters[] = $host;
+      }
+
+      // We can't guarantee that you didn't just give the same host two
+      // different names in DNS, but this check can catch silly copy/paste
+      // mistakes.
+      $key = "{$host}:{$port}";
+      if (isset($map[$key])) {
+        throw new Exception(
+          pht(
+            'Database cluster configuration is invalid: it describes the '.
+            'same host ("%s") multiple times. Each host should appear only '.
+            'once in the list.',
+            $host));
+      }
+      $map[$key] = true;
+    }
+
+    if (count($masters) > 1) {
+      throw new Exception(
+        pht(
+          'Database cluster configuration is invalid: it describes multiple '.
+          'masters. No more than one host may be a master. Hosts currently '.
+          'configured as masters: %s.',
+          implode(', ', $masters)));
+    }
+  }
+
+}

src/infrastructure/markup/PhabricatorMarkupEngine.php

@@ -254,6 +254,8 @@ final class PhabricatorMarkupEngine extends Phobject {
       }
     }
 
+    $is_readonly = PhabricatorEnv::isReadOnly();
+
     foreach ($objects as $key => $info) {
       // False check in case MySQL doesn't support unicode characters
       // in the string (T1191), resulting in unserialize returning false.
@@ -279,7 +281,7 @@ final class PhabricatorMarkupEngine extends Phobject {
         ->setCacheData($data)
         ->setMetadata($metadata);
 
-      if (isset($use_cache[$key])) {
+      if (isset($use_cache[$key]) && !$is_readonly) {
         // This is just filling a cache and always safe, even on a read pathway.
         $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
           $blocks[$key]->replace();

src/view/page/PhabricatorStandardPageView.php

@@ -276,7 +276,7 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
       Javelin::initBehavior(
         'read-only-warning',
         array(
-          'message' => pht('This install is currently in read-only mode.'),
+          'message' => pht('Phabricator is currently in read-only mode.'),
         ));
     }