When no master database is configured, automatically degrade to read-only mode

Summary: Ref T4571. If `cluster.databases` is configured but only has replicas, implicitly drop to read-only mode and send writes to a replica.

Test Plan:
  - Disabled the `master`, saw Phabricator automatically degrade into read-only mode against replicas.
  - (Also tested: explicit read-only mode, non-cluster mode, properly configured cluster mode).

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T4571

Differential Revision: https://secure.phabricator.com/D15672

src/__phutil_library_map__.php

@@ -1987,6 +1987,9 @@ phutil_register_library_map(array(
     'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
     'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
     'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
+    'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php',
+    'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
+    'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
     'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
     'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
     'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@@ -6397,6 +6400,9 @@ phutil_register_library_map(array(
     'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType',
+    'PhabricatorClusterException' => 'Exception',
+    'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
+    'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
     'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
     'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
     'PhabricatorCommentEditField' => 'PhabricatorEditField',

src/applications/system/controller/PhabricatorSystemReadOnlyController.php

@@ -25,6 +25,27 @@ final class PhabricatorSystemReadOnlyController
           'has been turned on by rolling your chair away from your desk and '.
           'yelling "Hey! Why is Phabricator in read-only mode??!" using '.
           'your very loudest outside voice.');
+        $body[] = pht(
+          'This mode is active because it is enabled in the configuration '.
+          'option "%s".',
+          phutil_tag('tt', array(), 'cluster.read-only'));
+        $button = pht('Wait Patiently');
+        break;
+      case PhabricatorEnv::READONLY_MASTERLESS:
+        $title = pht('No Writable Database');
+        $body[] = pht(
+          'Phabricator is currently configured with no writable ("master") '.
+          'database, so it can not write new information anywhere. '.
+          'Phabricator will run in read-only mode until an administrator '.
+          'reconfigures it with a writable database.');
+        $body[] = pht(
+          'This usually occurs when an administrator is actively working on '.
+          'fixing a temporary configuration or deployment problem.');
+        $body[] = pht(
+          'This mode is active because no database has a "%s" role in '.
+          'the configuration option "%s".',
+          phutil_tag('tt', array(), 'master'),
+          phutil_tag('tt', array(), 'cluster.databases'));
         $button = pht('Wait Patiently');
         break;
       default:
@@ -33,8 +54,8 @@ final class PhabricatorSystemReadOnlyController
 
     $body[] = pht(
       'In read-only mode you can read existing information, but you will not '.
-      'be able to edit information or create new information until this mode '.
+      'be able to edit objects or create new objects until this mode is '.
-      'is disabled.');
+      'disabled.');
 
     $dialog = $this->newDialog()
       ->setTitle($title)

src/infrastructure/cluster/PhabricatorClusterException.php

@@ -0,0 +1,8 @@
+<?php
+
+abstract class PhabricatorClusterException
+  extends Exception {
+
+  abstract public function getExceptionTitle();
+
+}

src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php

@@ -0,0 +1,35 @@
+<?php
+
+final class PhabricatorClusterExceptionHandler
+  extends PhabricatorRequestExceptionHandler {
+
+  public function getRequestExceptionHandlerPriority() {
+    return 300000;
+  }
+
+  public function getRequestExceptionHandlerDescription() {
+    return pht('Handles runtime problems with cluster configuration.');
+  }
+
+  public function canHandleRequestException(
+    AphrontRequest $request,
+    Exception $ex) {
+    return ($ex instanceof PhabricatorClusterException);
+  }
+
+  public function handleRequestException(
+    AphrontRequest $request,
+    Exception $ex) {
+
+    $viewer = $this->getViewer($request);
+
+    $title = $ex->getExceptionTitle();
+
+    return id(new AphrontDialogView())
+      ->setTitle($title)
+      ->setUser($viewer)
+      ->appendParagraph($ex->getMessage())
+      ->addCancelButton('/', pht('Proceed With Caution'));
+  }
+
+}

src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php

@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorClusterImproperWriteException
+  extends PhabricatorClusterException {
+
+  public function getExceptionTitle() {
+    return pht('Improper Cluster Write');
+  }
+
+}

src/infrastructure/cluster/PhabricatorDatabaseRef.php

@@ -347,6 +347,31 @@ final class PhabricatorDatabaseRef
     return null;
   }
 
+  public static function getReplicaDatabaseRef() {
+    $refs = self::loadAll();
+
+    if (!$refs) {
+      return null;
+    }
+
+    // TODO: We may have multiple replicas to choose from, and could make
+    // more of an effort to pick the "best" one here instead of always
+    // picking the first one. Once we've picked one, we should try to use
+    // the same replica for the rest of the request, though.
+
+    foreach ($refs as $ref) {
+      if ($ref->getDisabled()) {
+        continue;
+      }
+      if ($ref->getIsMaster()) {
+        continue;
+      }
+      return $ref;
+    }
+
+    return null;
+  }
+
   private function newConnection(array $options) {
     $spec = $options + array(
       'user' => $this->getUser(),

src/infrastructure/env/PhabricatorEnv.php

@@ -60,6 +60,7 @@ final class PhabricatorEnv extends Phobject {
   private static $readOnlyReason;
 
   const READONLY_CONFIG = 'config';
+  const READONLY_MASTERLESS = 'masterless';
 
   /**
    * @phutil-external-symbol class PhabricatorStartup
@@ -213,6 +214,11 @@ final class PhabricatorEnv extends Phobject {
       $stack->pushSource($site_source);
     }
 
+    $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
+    if (!$master) {
+      self::setReadOnly(true, self::READONLY_MASTERLESS);
+    }
+
     try {
       $stack->pushSource(
         id(new PhabricatorConfigDatabaseSource('default'))
@@ -456,7 +462,15 @@ final class PhabricatorEnv extends Phobject {
   }
 
   public static function getReadOnlyMessage() {
-    return pht('Phabricator is currently in read-only mode.');
+    $reason = self::getReadOnlyReason();
+    switch ($reason) {
+      case self::READONLY_MASTERLESS:
+        return pht(
+          'Phabricator is in read-only mode (no writable database '.
+          'is configured).');
+    }
+
+    return pht('Phabricator is in read-only mode.');
   }
 
   public static function getReadOnlyURI() {

src/infrastructure/storage/lisk/PhabricatorLiskDAO.php

@@ -57,16 +57,12 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
     $is_readonly = PhabricatorEnv::isReadOnly();
 
     if ($is_readonly && ($mode != 'r')) {
-      throw new Exception(
+      $this->raiseImproperWrite($database);
-        pht(
-          'Attempting to establish write-mode connection from a read-only '.
-          'page (to database "%s").',
-          $database));
     }
 
     $refs = PhabricatorDatabaseRef::loadAll();
     if ($refs) {
-      $connection = $this->newClusterConnection($database);
+      $connection = $this->newClusterConnection($database, $mode);
     } else {
       $connection = $this->newBasicConnection($database, $mode, $namespace);
     }
@@ -101,15 +97,31 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
       ));
   }
 
-  private function newClusterConnection($database) {
+  private function newClusterConnection($database, $mode) {
     $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
-
+    if ($master) {
-    if (!$master) {
+      return $master->newApplicationConnection($database);
-      // TODO: Implicitly degrade to read-only mode.
-      throw new Exception(pht('No master in database cluster config!'));
     }
 
-    return $master->newApplicationConnection($database);
+    $replica = PhabricatorDatabaseRef::getReplicaDatabaseRef();
+    if (!$replica) {
+      throw new Exception(
+        pht('No valid databases are configured!'));
+    }
+
+    $connection = $replica->newApplicationConnection($database);
+    $connection->setReadOnly(true);
+
+    return $connection;
+  }
+
+  private function raiseImproperWrite($database) {
+    throw new PhabricatorClusterImproperWriteException(
+      pht(
+        'Unable to establish a write-mode connection (to application '.
+        'database "%s") because Phabricator is in read-only mode. Whatever '.
+        'you are trying to do does not function correctly in read-only mode.',
+        $database));
   }