Add a cluster.read-only option

Summary:
Ref T4571. There will be a very long path beyond this, but add a basic read-only mode. You can explicitly enable this to put Phabricator in a sort of "maintenance" mode today if you're swapping databases or something.

In the long term, we'll automatically degrade into this mode if the master database is down.

Test Plan:
  - Enabled read-only mode.
  - Browsed around.
  - Didn't immediately see anything that was totally 100% broken.

Most stuff is 80-90% broken right now. For example:

  - Stuff like submitting comments doesn't work, and gives you a confusing, unhelpful error.
  - None of the UI really knows that it's read-only. EditEngine stuff should all hide itself and say "you can't add new comments while an install is in read-only mode", for example, but currently does not.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T4571

Differential Revision: https://secure.phabricator.com/D15662

resources/celerity/packages.php

@@ -76,6 +76,7 @@ return array(
     'javelin-quicksand',
     'javelin-behavior-quicksand-blacklist',
     'javelin-behavior-high-security-warning',
+    'javelin-behavior-read-only-warning',
     'javelin-scrollbar',
     'javelin-behavior-scrollbar',
     'javelin-behavior-durable-column',

src/applications/cache/PhabricatorKeyValueDatabaseCache.php

@@ -7,6 +7,10 @@ final class PhabricatorKeyValueDatabaseCache
   const CACHE_FORMAT_DEFLATE    = 'deflate';
 
   public function setKeys(array $keys, $ttl = null) {
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     if ($keys) {
       $map = $this->digestKeys(array_keys($keys));
       $conn_w = $this->establishConnection('w');
@@ -30,19 +34,19 @@ final class PhabricatorKeyValueDatabaseCache
 
       $guard = AphrontWriteGuard::beginScopedUnguardedWrites();
         foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
-            queryfx(
-              $conn_w,
-              'INSERT INTO %T
-                (cacheKeyHash, cacheKey, cacheFormat, cacheData,
-                  cacheCreated, cacheExpires) VALUES %Q
-                ON DUPLICATE KEY UPDATE
-                  cacheKey = VALUES(cacheKey),
-                  cacheFormat = VALUES(cacheFormat),
-                  cacheData = VALUES(cacheData),
-                  cacheCreated = VALUES(cacheCreated),
-                  cacheExpires = VALUES(cacheExpires)',
-              $this->getTableName(),
-              $chunk);
+          queryfx(
+            $conn_w,
+            'INSERT INTO %T
+              (cacheKeyHash, cacheKey, cacheFormat, cacheData,
+                cacheCreated, cacheExpires) VALUES %Q
+              ON DUPLICATE KEY UPDATE
+                cacheKey = VALUES(cacheKey),
+                cacheFormat = VALUES(cacheFormat),
+                cacheData = VALUES(cacheData),
+                cacheCreated = VALUES(cacheCreated),
+                cacheExpires = VALUES(cacheExpires)',
+            $this->getTableName(),
+            $chunk);
         }
       unset($guard);
     }

src/applications/config/option/PhabricatorClusterConfigOptions.php

@@ -73,6 +73,22 @@ final class PhabricatorClusterConfigOptions
             'subprocesses and commit hooks in the `%s` environmental variable.',
             'PhabricatorConfigSiteSource',
             'PHABRICATOR_INSTANCE')),
+      $this->newOption('cluster.read-only', 'bool', false)
+        ->setLocked(true)
+        ->setSummary(
+          pht(
+            'Activate read-only mode for maintenance or disaster recovery.'))
+        ->setDescription(
+          pht(
+            'WARNING: This is a prototype option and the description below '.
+            'is currently pure fantasy.'.
+            "\n\n".
+            'Switch Phabricator to read-only mode. In this mode, users will '.
+            'be unable to write new data. Normally, the cluster degrades '.
+            'into this mode automatically when it detects that the database '.
+            'master is unreachable, but you can activate it manually in '.
+            'order to perform maintenance or test configuration.')),
+
     );
   }
 

src/applications/multimeter/data/MultimeterControl.php

@@ -124,6 +124,10 @@ final class MultimeterControl extends Phobject {
   }
 
   private function writeEvents() {
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $events = $this->events;
 
     $random = Filesystem::readRandomBytes(32);

src/applications/notification/storage/PhabricatorFeedStoryNotification.php

@@ -39,6 +39,10 @@ final class PhabricatorFeedStoryNotification extends PhabricatorFeedDAO {
     PhabricatorUser $user,
     $object_phid) {
 
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
 
     $notification_table = new PhabricatorFeedStoryNotification();

src/infrastructure/env/PhabricatorEnv.php

@@ -56,6 +56,7 @@ final class PhabricatorEnv extends Phobject {
   private static $requestBaseURI;
   private static $cache;
   private static $localeCode;
+  private static $readOnly;
 
   /**
    * @phutil-external-symbol class PhabricatorStartup
@@ -439,6 +440,18 @@ final class PhabricatorEnv extends Phobject {
     self::$requestBaseURI = $uri;
   }
 
+  public static function isReadOnly() {
+    if (self::$readOnly !== null) {
+      return self::$readOnly;
+    }
+    return self::getEnvConfig('cluster.read-only');
+  }
+
+  public static function setReadOnly($read_only) {
+    self::$readOnly = $read_only;
+  }
+
+
 /* -(  Unit Test Support  )-------------------------------------------------- */
 
 

src/infrastructure/storage/lisk/PhabricatorLiskDAO.php

@@ -57,7 +57,16 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
       'mysql.configuration-provider',
       array($this, $mode, $namespace));
 
-    return PhabricatorEnv::newObjectFromConfig(
+    $is_readonly = PhabricatorEnv::isReadOnly();
+    if ($is_readonly && ($mode != 'r')) {
+      throw new Exception(
+        pht(
+          'Attempting to establish write-mode connection from a read-only '.
+          'page (to database "%s").',
+          $conf->getDatabase()));
+    }
+
+    $connection = PhabricatorEnv::newObjectFromConfig(
       'mysql.implementation',
       array(
         array(
@@ -69,6 +78,16 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
           'retries'   => 3,
         ),
       ));
+
+    // TODO: This should be testing if the mode is "r", but that would proably
+    // break a lot of things. Perform a more narrow test for readonly mode
+    // until we have greater certainty that this works correctly most of the
+    // time.
+    if ($is_readonly) {
+      $connection->setReadOnly(true);
+    }
+
+    return $connection;
   }
 
   /**

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php

@@ -50,6 +50,17 @@ abstract class PhabricatorStorageManagementWorkflow
     $this->setDryRun($args->getArg('dryrun'));
     $this->setForce($args->getArg('force'));
 
+    if (PhabricatorEnv::isReadOnly()) {
+      if ($this->isForce()) {
+        PhabricatorEnv::setReadOnly(false);
+      } else {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Phabricator is currently in read-only mode. Use --force to '.
+            'override this mode.'));
+      }
+    }
+
     $this->didExecute($args);
   }
 

src/infrastructure/testing/PhabricatorTestCase.php

@@ -126,6 +126,8 @@ abstract class PhabricatorTestCase extends PhutilTestCase {
 
     // Tests do their own stubbing/voiding for events.
     $this->env->overrideEnvConfig('phabricator.silent', false);
+
+    $this->env->overrideEnvConfig('cluster.read-only', false);
   }
 
   protected function didRunTests() {

src/view/page/PhabricatorStandardPageView.php

@@ -272,6 +272,14 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
       'high-security-warning',
       $this->getHighSecurityWarningConfig());
 
+    if (PhabricatorEnv::isReadOnly()) {
+      Javelin::initBehavior(
+        'read-only-warning',
+        array(
+          'message' => pht('This install is currently in read-only mode.'),
+        ));
+    }
+
     if ($console) {
       require_celerity_resource('aphront-dark-console-css');
 

webroot/rsrc/css/aphront/notification.css

@@ -52,6 +52,11 @@
   border: 1px solid {$violet};
 }
 
+.jx-notification-read-only {
+  background: {$greybackground};
+  border: 1px solid {$darkgreyborder};
+}
+
 .jx-notification-container .phabricator-notification {
   padding: 0;
 }

webroot/rsrc/js/core/behavior-read-only-warning.js

@@ -0,0 +1,16 @@
+/**
+ * @provides javelin-behavior-read-only-warning
+ * @requires javelin-behavior
+ *           javelin-uri
+ *           phabricator-notification
+ */
+
+JX.behavior('read-only-warning', function(config) {
+
+  new JX.Notification()
+    .setContent(config.message)
+    .setDuration(0)
+    .alterClassName('jx-notification-read-only', true)
+    .show();
+
+});