(stable) Promote 2016 Week 16

.gitignore

@@ -16,6 +16,7 @@
 /conf/keys/device.pub
 /conf/keys/device.key
 /conf/keys/device.id
+/conf/aphlict/aphlict.custom.json
 
 # Impact Font
 /resources/font/impact.ttf

conf/aphlict/README

@@ -0,0 +1,16 @@
+To customize this configuration, you have two options: create a custom
+configuration file in this directory, or specify a path to a configuration file
+explicitly when starting Aphlict.
+
+To create a custom configuration file, copy `aphlict.default.json` in this
+directory and rename it `aphlict.custom.json`. If this file exists, it will
+be read by default.
+
+To specify a path when starting Aphlict, use the `--config` flag:
+
+  phabricator/ $ ./bin/aphlict start --config path/to/config.json
+
+Specifying a configuration file explicitly overrides default configuration.
+
+For more information about configuring notifications, see the article
+"Notifications User Guide: Setup and Configuration" in the documentation.

conf/aphlict/aphlict.default.json

@@ -0,0 +1,26 @@
+{
+  "servers": [
+    {
+      "type": "client",
+      "port": 22280,
+      "listen": "0.0.0.0",
+      "ssl.key": null,
+      "ssl.cert": null,
+      "ssl.chain": null
+    },
+    {
+      "type": "admin",
+      "port": 22281,
+      "listen": "127.0.0.1",
+      "ssl.key": null,
+      "ssl.cert": null,
+      "ssl.chain": null
+    }
+  ],
+  "logs": [
+    {
+      "path": "/var/log/aphlict.log"
+    }
+  ],
+  "pidfile": "/var/tmp/aphlict/pid/aphlict.pid"
+}

resources/celerity/map.php

@@ -7,8 +7,8 @@
  */
 return array(
   'names' => array(
-    'core.pkg.css' => '82cefddc',
+    'core.pkg.css' => 'ce06b6f6',
-    'core.pkg.js' => 'e5484f37',
+    'core.pkg.js' => '37344f3c',
     'darkconsole.pkg.js' => 'e7393ebb',
     'differential.pkg.css' => '7ba78475',
     'differential.pkg.js' => 'd0cd0df6',
@@ -22,7 +22,7 @@ return array(
     'rsrc/css/aphront/lightbox-attachment.css' => '7acac05d',
     'rsrc/css/aphront/list-filter-view.css' => '5d6f0526',
     'rsrc/css/aphront/multi-column.css' => 'fd18389d',
-    'rsrc/css/aphront/notification.css' => '7f684b62',
+    'rsrc/css/aphront/notification.css' => '3f6c89c9',
     'rsrc/css/aphront/panel-view.css' => '8427b78d',
     'rsrc/css/aphront/phabricator-nav-view.css' => 'ac79a758',
     'rsrc/css/aphront/table-view.css' => '9258e19f',
@@ -109,7 +109,7 @@ return array(
     'rsrc/css/core/z-index.css' => '5b6fcf3f',
     'rsrc/css/diviner/diviner-shared.css' => 'aa3656aa',
     'rsrc/css/font/font-aleo.css' => '8bdb2835',
-    'rsrc/css/font/font-awesome.css' => 'c43323c5',
+    'rsrc/css/font/font-awesome.css' => '2b7ebbcc',
     'rsrc/css/font/font-lato.css' => 'c7ccd872',
     'rsrc/css/font/phui-font-icon-base.css' => '6449bce8',
     'rsrc/css/layout/phabricator-filetree-view.css' => 'fccf9f82',
@@ -121,7 +121,7 @@ return array(
     'rsrc/css/phui/calendar/phui-calendar.css' => 'ccabe893',
     'rsrc/css/phui/phui-action-list.css' => 'c5eba19d',
     'rsrc/css/phui/phui-action-panel.css' => '91c7b835',
-    'rsrc/css/phui/phui-badge.css' => 'f25c3476',
+    'rsrc/css/phui/phui-badge.css' => '3baef8db',
     'rsrc/css/phui/phui-big-info-view.css' => 'bd903741',
     'rsrc/css/phui/phui-box.css' => 'd909ea3d',
     'rsrc/css/phui/phui-button.css' => 'a64a8de6',
@@ -148,7 +148,7 @@ return array(
     'rsrc/css/phui/phui-object-item-list-view.css' => '8d99e42b',
     'rsrc/css/phui/phui-pager.css' => 'bea33d23',
     'rsrc/css/phui/phui-pinboard-view.css' => '2495140e',
-    'rsrc/css/phui/phui-profile-menu.css' => '7e92a89a',
+    'rsrc/css/phui/phui-profile-menu.css' => 'c8557f33',
     'rsrc/css/phui/phui-property-list-view.css' => '1d42ee7c',
     'rsrc/css/phui/phui-remarkup-preview.css' => '1a8f2591',
     'rsrc/css/phui/phui-segment-bar-view.css' => '46342871',
@@ -175,10 +175,10 @@ return array(
     'rsrc/externals/font/aleo/aleo-regular.ttf' => '751e7479',
     'rsrc/externals/font/aleo/aleo-regular.woff' => 'c3744be9',
     'rsrc/externals/font/aleo/aleo-regular.woff2' => '851aa0ee',
-    'rsrc/externals/font/fontawesome/fontawesome-webfont.eot' => '346fbcc5',
+    'rsrc/externals/font/fontawesome/fontawesome-webfont.eot' => '59b3076c',
-    'rsrc/externals/font/fontawesome/fontawesome-webfont.ttf' => '510fccb2',
+    'rsrc/externals/font/fontawesome/fontawesome-webfont.ttf' => '45ad7e57',
-    'rsrc/externals/font/fontawesome/fontawesome-webfont.woff' => '0334f580',
+    'rsrc/externals/font/fontawesome/fontawesome-webfont.woff' => 'f861e2a8',
-    'rsrc/externals/font/fontawesome/fontawesome-webfont.woff2' => '45dca585',
+    'rsrc/externals/font/fontawesome/fontawesome-webfont.woff2' => '0ee0f078',
     'rsrc/externals/font/lato/lato-bold.eot' => '99fbcf8c',
     'rsrc/externals/font/lato/lato-bold.svg' => '2aa83045',
     'rsrc/externals/font/lato/lato-bold.ttf' => '0a7141f7',
@@ -232,7 +232,7 @@ return array(
     'rsrc/externals/javelin/lib/DOM.js' => '805b806a',
     'rsrc/externals/javelin/lib/History.js' => 'd4505101',
     'rsrc/externals/javelin/lib/JSON.js' => '69adf288',
-    'rsrc/externals/javelin/lib/Leader.js' => '331b1611',
+    'rsrc/externals/javelin/lib/Leader.js' => 'b4ba945c',
     'rsrc/externals/javelin/lib/Mask.js' => '8a41885b',
     'rsrc/externals/javelin/lib/Quicksand.js' => '6b8ef10b',
     'rsrc/externals/javelin/lib/Request.js' => '94b750d2',
@@ -393,6 +393,7 @@ return array(
     'rsrc/js/application/diffusion/behavior-audit-preview.js' => 'd835b03a',
     'rsrc/js/application/diffusion/behavior-commit-branches.js' => 'bdaf4d04',
     'rsrc/js/application/diffusion/behavior-commit-graph.js' => '5a0b1a64',
+    'rsrc/js/application/diffusion/behavior-diffusion-browse-file.js' => '054a0f0b',
     'rsrc/js/application/diffusion/behavior-jump-to.js' => '73d09eef',
     'rsrc/js/application/diffusion/behavior-load-blame.js' => '42126667',
     'rsrc/js/application/diffusion/behavior-locate-file.js' => '6d3e1947',
@@ -471,6 +472,7 @@ return array(
     'rsrc/js/core/behavior-active-nav.js' => 'e379b58e',
     'rsrc/js/core/behavior-audio-source.js' => '59b251eb',
     'rsrc/js/core/behavior-autofocus.js' => '7319e029',
+    'rsrc/js/core/behavior-badge-view.js' => '8ff5e24c',
     'rsrc/js/core/behavior-choose-control.js' => '327a00d1',
     'rsrc/js/core/behavior-crop.js' => 'fa0f4fc2',
     'rsrc/js/core/behavior-dark-console.js' => 'f411b6ae',
@@ -494,6 +496,7 @@ return array(
     'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
     'rsrc/js/core/behavior-phabricator-nav.js' => '56a1ca03',
     'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => '340c8eff',
+    'rsrc/js/core/behavior-read-only-warning.js' => 'ba158207',
     'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b',
     'rsrc/js/core/behavior-remarkup-preview.js' => '4b700e9e',
     'rsrc/js/core/behavior-reorder-applications.js' => '76b9fc3e',
@@ -501,8 +504,8 @@ return array(
     'rsrc/js/core/behavior-scrollbar.js' => '834a1173',
     'rsrc/js/core/behavior-search-typeahead.js' => '06c32383',
     'rsrc/js/core/behavior-select-on-click.js' => '4e3e79a6',
-    'rsrc/js/core/behavior-time-typeahead.js' => 'f80d6bf0',
+    'rsrc/js/core/behavior-time-typeahead.js' => '522431f7',
-    'rsrc/js/core/behavior-toggle-class.js' => '5d7c9f33',
+    'rsrc/js/core/behavior-toggle-class.js' => '92b9ec77',
     'rsrc/js/core/behavior-tokenizer.js' => 'b3a4b884',
     'rsrc/js/core/behavior-tooltip.js' => '42fcb747',
     'rsrc/js/core/behavior-watch-anchor.js' => '9f36c42d',
@@ -558,7 +561,7 @@ return array(
     'diffusion-source-css' => '68b30fd3',
     'diviner-shared-css' => 'aa3656aa',
     'font-aleo' => '8bdb2835',
-    'font-fontawesome' => 'c43323c5',
+    'font-fontawesome' => '2b7ebbcc',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => '5c1b47c2',
     'harbormaster-css' => 'f491c9f4',
@@ -578,6 +581,7 @@ return array(
     'javelin-behavior-aphront-more' => 'a80d0378',
     'javelin-behavior-audio-source' => '59b251eb',
     'javelin-behavior-audit-preview' => 'd835b03a',
+    'javelin-behavior-badge-view' => '8ff5e24c',
     'javelin-behavior-bulk-job-reload' => 'edf8a145',
     'javelin-behavior-choose-control' => '327a00d1',
     'javelin-behavior-comment-actions' => '06460e71',
@@ -605,6 +609,7 @@ return array(
     'javelin-behavior-differential-populate' => '8694b1df',
     'javelin-behavior-differential-toggle-files' => 'ca3f91eb',
     'javelin-behavior-differential-user-select' => 'a8d8459d',
+    'javelin-behavior-diffusion-browse-file' => '054a0f0b',
     'javelin-behavior-diffusion-commit-branches' => 'bdaf4d04',
     'javelin-behavior-diffusion-commit-graph' => '5a0b1a64',
     'javelin-behavior-diffusion-jump-to' => '73d09eef',
@@ -666,6 +671,7 @@ return array(
     'javelin-behavior-project-boards' => '14a1faae',
     'javelin-behavior-project-create' => '065227cc',
     'javelin-behavior-quicksand-blacklist' => '7927a7d3',
+    'javelin-behavior-read-only-warning' => 'ba158207',
     'javelin-behavior-recurring-edit' => '5f1c4d5f',
     'javelin-behavior-refresh-csrf' => 'ab2f381b',
     'javelin-behavior-releeph-preview-branch' => 'b2b4fbaf',
@@ -682,8 +688,8 @@ return array(
     'javelin-behavior-slowvote-embed' => '887ad43f',
     'javelin-behavior-stripe-payment-form' => '3f5d6dbf',
     'javelin-behavior-test-payment-form' => 'fc91ab6c',
-    'javelin-behavior-time-typeahead' => 'f80d6bf0',
+    'javelin-behavior-time-typeahead' => '522431f7',
-    'javelin-behavior-toggle-class' => '5d7c9f33',
+    'javelin-behavior-toggle-class' => '92b9ec77',
     'javelin-behavior-typeahead-browse' => '635de1ec',
     'javelin-behavior-typeahead-search' => '93d0c9e3',
     'javelin-behavior-view-placeholder' => '47830651',
@@ -698,7 +704,7 @@ return array(
     'javelin-history' => 'd4505101',
     'javelin-install' => '05270951',
     'javelin-json' => '69adf288',
-    'javelin-leader' => '331b1611',
+    'javelin-leader' => 'b4ba945c',
     'javelin-magical-init' => '3010e992',
     'javelin-mask' => '8a41885b',
     'javelin-quicksand' => '6b8ef10b',
@@ -766,7 +772,7 @@ return array(
     'phabricator-main-menu-view' => 'd00a795a',
     'phabricator-nav-view-css' => 'ac79a758',
     'phabricator-notification' => 'ccf1cbf8',
-    'phabricator-notification-css' => '7f684b62',
+    'phabricator-notification-css' => '3f6c89c9',
     'phabricator-notification-menu-css' => 'f31c0bde',
     'phabricator-object-selector-css' => '85ee8ce6',
     'phabricator-phtize' => 'd254d646',
@@ -803,7 +809,7 @@ return array(
     'phrequent-css' => 'ffc185ad',
     'phriction-document-css' => 'd1861e06',
     'phui-action-panel-css' => '91c7b835',
-    'phui-badge-view-css' => 'f25c3476',
+    'phui-badge-view-css' => '3baef8db',
     'phui-big-info-view-css' => 'bd903741',
     'phui-box-css' => 'd909ea3d',
     'phui-button-css' => 'a64a8de6',
@@ -837,7 +843,7 @@ return array(
     'phui-object-item-list-view-css' => '8d99e42b',
     'phui-pager-css' => 'bea33d23',
     'phui-pinboard-view-css' => '2495140e',
-    'phui-profile-menu-css' => '7e92a89a',
+    'phui-profile-menu-css' => 'c8557f33',
     'phui-property-list-view-css' => '1d42ee7c',
     'phui-remarkup-preview-css' => '1a8f2591',
     'phui-segment-bar-view-css' => '46342871',
@@ -916,6 +922,12 @@ return array(
       'javelin-util',
       'javelin-magical-init',
     ),
+    '054a0f0b' => array(
+      'javelin-behavior',
+      'javelin-dom',
+      'javelin-util',
+      'phabricator-tooltip',
+    ),
     '056da01b' => array(
       'aphront-typeahead-control-css',
       'phui-tag-view-css',
@@ -1106,9 +1118,6 @@ return array(
       'javelin-dom',
       'javelin-workflow',
     ),
-    '331b1611' => array(
-      'javelin-install',
-    ),
     '340c8eff' => array(
       'javelin-behavior',
       'javelin-stratcom',
@@ -1235,6 +1244,14 @@ return array(
       'javelin-dom',
       'javelin-reactor-dom',
     ),
+    '522431f7' => array(
+      'javelin-behavior',
+      'javelin-util',
+      'javelin-dom',
+      'javelin-stratcom',
+      'javelin-vector',
+      'javelin-typeahead-static-source',
+    ),
     52291776 => array(
       'javelin-install',
       'javelin-dom',
@@ -1328,11 +1345,6 @@ return array(
       'javelin-stratcom',
       'javelin-dom',
     ),
-    '5d7c9f33' => array(
-      'javelin-behavior',
-      'javelin-stratcom',
-      'javelin-dom',
-    ),
     '5e9f347c' => array(
       'javelin-behavior',
       'multirow-row-manager',
@@ -1568,6 +1580,11 @@ return array(
       'javelin-stratcom',
       'javelin-install',
     ),
+    '8ff5e24c' => array(
+      'javelin-behavior',
+      'javelin-stratcom',
+      'javelin-dom',
+    ),
     '901935ef' => array(
       'javelin-behavior',
       'javelin-dom',
@@ -1582,6 +1599,11 @@ return array(
     92197373 => array(
       'phui-workcard-view-css',
     ),
+    '92b9ec77' => array(
+      'javelin-behavior',
+      'javelin-stratcom',
+      'javelin-dom',
+    ),
     '93d0c9e3' => array(
       'javelin-behavior',
       'javelin-stratcom',
@@ -1750,6 +1772,9 @@ return array(
       'javelin-typeahead-preloaded-source',
       'javelin-util',
     ),
+    'b4ba945c' => array(
+      'javelin-install',
+    ),
     'b59e1e96' => array(
       'javelin-behavior',
       'javelin-stratcom',
@@ -1781,6 +1806,11 @@ return array(
       'javelin-json',
       'phabricator-draggable-list',
     ),
+    'ba158207' => array(
+      'javelin-behavior',
+      'javelin-uri',
+      'phabricator-notification',
+    ),
     'bae58312' => array(
       'javelin-install',
       'javelin-workboard-card',
@@ -2088,14 +2118,6 @@ return array(
       'javelin-typeahead-ondemand-source',
       'javelin-util',
     ),
-    'f80d6bf0' => array(
-      'javelin-behavior',
-      'javelin-util',
-      'javelin-dom',
-      'javelin-stratcom',
-      'javelin-vector',
-      'javelin-typeahead-static-source',
-    ),
     'f829edb3' => array(
       'javelin-view',
       'javelin-install',
@@ -2284,6 +2306,7 @@ return array(
       'javelin-quicksand',
       'javelin-behavior-quicksand-blacklist',
       'javelin-behavior-high-security-warning',
+      'javelin-behavior-read-only-warning',
       'javelin-scrollbar',
       'javelin-behavior-scrollbar',
       'javelin-behavior-durable-column',

resources/celerity/packages.php

@@ -76,6 +76,7 @@ return array(
     'javelin-quicksand',
     'javelin-behavior-quicksand-blacklist',
     'javelin-behavior-high-security-warning',
+    'javelin-behavior-read-only-warning',
     'javelin-scrollbar',
     'javelin-behavior-scrollbar',
     'javelin-behavior-durable-column',

resources/sql/autopatches/20160411.repo.1.version.sql

@@ -0,0 +1,8 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_workingcopyversion (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  repositoryPHID VARBINARY(64) NOT NULL,
+  devicePHID VARBINARY(64) NOT NULL,
+  repositoryVersion INT UNSIGNED NOT NULL,
+  isWriting BOOL NOT NULL,
+  UNIQUE KEY `key_workingcopy` (repositoryPHID, devicePHID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

scripts/sql/manage_storage.php

@@ -19,13 +19,6 @@ EOHELP
 );
 $args->parseStandardArguments();
 
-$conf = PhabricatorEnv::newObjectFromConfig(
-  'mysql.configuration-provider',
-  array($dao = null, 'w'));
-
-$default_user       = $conf->getUser();
-$default_host       = $conf->getHost();
-$default_port       = $conf->getPort();
 $default_namespace  = PhabricatorLiskDAO::getDefaultStorageNamespace();
 
 try {
@@ -37,14 +30,18 @@ try {
         'help'    => pht(
           'Do not prompt before performing dangerous operations.'),
       ),
+      array(
+        'name' => 'host',
+        'param' => 'hostname',
+        'help' => pht(
+          'Connect to __host__ instead of the default host.'),
+      ),
       array(
         'name'    => 'user',
         'short'   => 'u',
         'param'   => 'username',
-        'default' => $default_user,
         'help'    => pht(
-          "Connect with __username__ instead of the configured default ('%s').",
+          'Connect with __username__ instead of the configured default.'),
-          $default_user),
       ),
       array(
         'name'    => 'password',
@@ -84,11 +81,48 @@ try {
 // First, test that the Phabricator configuration is set up correctly. After
 // we know this works we'll test any administrative credentials specifically.
 
+$host = $args->getArg('host');
+if (strlen($host)) {
+  $ref = null;
+
+  $refs = PhabricatorDatabaseRef::getLiveRefs();
+
+  // Include the master in case the user is just specifying a redundant
+  // "--host" flag for no reason and does not actually have a database
+  // cluster configured.
+  $refs[] = PhabricatorDatabaseRef::getMasterDatabaseRef();
+
+  foreach ($refs as $possible_ref) {
+    if ($possible_ref->getHost() == $host) {
+      $ref = $possible_ref;
+      break;
+    }
+  }
+
+  if (!$ref) {
+    throw new PhutilArgumentUsageException(
+      pht(
+        'There is no configured database on host "%s". This command can '.
+        'only interact with configured databases.',
+        $host));
+  }
+} else {
+  $ref = PhabricatorDatabaseRef::getMasterDatabaseRef();
+  if (!$ref) {
+    throw new Exception(
+      pht('No database master is configured.'));
+  }
+}
+
+$default_user = $ref->getUser();
+$default_host = $ref->getHost();
+$default_port = $ref->getPort();
+
 $test_api = id(new PhabricatorStorageManagementAPI())
   ->setUser($default_user)
   ->setHost($default_host)
   ->setPort($default_port)
-  ->setPassword($conf->getPassword())
+  ->setPassword($ref->getPass())
   ->setNamespace($args->getArg('namespace'));
 
 try {
@@ -120,15 +154,20 @@ try {
 
 if ($args->getArg('password') === null) {
   // This is already a PhutilOpaqueEnvelope.
-  $password = $conf->getPassword();
+  $password = $ref->getPass();
 } else {
   // Put this in a PhutilOpaqueEnvelope.
   $password = new PhutilOpaqueEnvelope($args->getArg('password'));
   PhabricatorEnv::overrideConfig('mysql.pass', $args->getArg('password'));
 }
 
+$selected_user = $args->getArg('user');
+if ($selected_user === null) {
+  $selected_user = $default_user;
+}
+
 $api = id(new PhabricatorStorageManagementAPI())
-  ->setUser($args->getArg('user'))
+  ->setUser($selected_user)
   ->setHost($default_host)
   ->setPort($default_port)
   ->setPassword($password)

src/__phutil_library_map__.php

@@ -650,6 +650,7 @@ phutil_register_library_map(array(
     'DiffusionHovercardEngineExtension' => 'applications/diffusion/engineextension/DiffusionHovercardEngineExtension.php',
     'DiffusionInlineCommentController' => 'applications/diffusion/controller/DiffusionInlineCommentController.php',
     'DiffusionInlineCommentPreviewController' => 'applications/diffusion/controller/DiffusionInlineCommentPreviewController.php',
+    'DiffusionInternalGitRawDiffQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionInternalGitRawDiffQueryConduitAPIMethod.php',
     'DiffusionLastModifiedController' => 'applications/diffusion/controller/DiffusionLastModifiedController.php',
     'DiffusionLastModifiedQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php',
     'DiffusionLintController' => 'applications/diffusion/controller/DiffusionLintController.php',
@@ -739,6 +740,7 @@ phutil_register_library_map(array(
     'DiffusionRefsQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRefsQueryConduitAPIMethod.php',
     'DiffusionRenameHistoryQuery' => 'applications/diffusion/query/DiffusionRenameHistoryQuery.php',
     'DiffusionRepositoryByIDRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryByIDRemarkupRule.php',
+    'DiffusionRepositoryClusterManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php',
     'DiffusionRepositoryController' => 'applications/diffusion/controller/DiffusionRepositoryController.php',
     'DiffusionRepositoryCreateController' => 'applications/diffusion/controller/DiffusionRepositoryCreateController.php',
     'DiffusionRepositoryDatasource' => 'applications/diffusion/typeahead/DiffusionRepositoryDatasource.php',
@@ -759,6 +761,8 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php',
     'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php',
     'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php',
+    'DiffusionRepositoryManageController' => 'applications/diffusion/controller/DiffusionRepositoryManageController.php',
+    'DiffusionRepositoryManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryManagementPanel.php',
     'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php',
     'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php',
     'DiffusionRepositoryRef' => 'applications/diffusion/data/DiffusionRepositoryRef.php',
@@ -1880,6 +1884,7 @@ phutil_register_library_map(array(
     'PhabricatorBadgesController' => 'applications/badges/controller/PhabricatorBadgesController.php',
     'PhabricatorBadgesCreateCapability' => 'applications/badges/capability/PhabricatorBadgesCreateCapability.php',
     'PhabricatorBadgesDAO' => 'applications/badges/storage/PhabricatorBadgesDAO.php',
+    'PhabricatorBadgesDatasource' => 'applications/badges/typeahead/PhabricatorBadgesDatasource.php',
     'PhabricatorBadgesDefaultEditCapability' => 'applications/badges/capability/PhabricatorBadgesDefaultEditCapability.php',
     'PhabricatorBadgesEditConduitAPIMethod' => 'applications/badges/conduit/PhabricatorBadgesEditConduitAPIMethod.php',
     'PhabricatorBadgesEditController' => 'applications/badges/controller/PhabricatorBadgesEditController.php',
@@ -1986,6 +1991,12 @@ phutil_register_library_map(array(
     'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
     'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
     'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
+    'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
+    'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php',
+    'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
+    'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php',
+    'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
+    'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php',
     'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
     'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
     'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@@ -2028,6 +2039,8 @@ phutil_register_library_map(array(
     'PhabricatorConfigAllController' => 'applications/config/controller/PhabricatorConfigAllController.php',
     'PhabricatorConfigApplication' => 'applications/config/application/PhabricatorConfigApplication.php',
     'PhabricatorConfigCacheController' => 'applications/config/controller/PhabricatorConfigCacheController.php',
+    'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
+    'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php',
     'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
     'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
     'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
@@ -2147,6 +2160,7 @@ phutil_register_library_map(array(
     'PhabricatorCustomFieldStringIndexStorage' => 'infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php',
     'PhabricatorCustomHeaderConfigType' => 'applications/config/custom/PhabricatorCustomHeaderConfigType.php',
     'PhabricatorDaemon' => 'infrastructure/daemon/PhabricatorDaemon.php',
+    'PhabricatorDaemonBulkJobController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobController.php',
     'PhabricatorDaemonBulkJobListController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobListController.php',
     'PhabricatorDaemonBulkJobMonitorController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php',
     'PhabricatorDaemonBulkJobViewController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php',
@@ -2234,6 +2248,8 @@ phutil_register_library_map(array(
     'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
     'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
     'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
+    'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php',
+    'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
     'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
     'PhabricatorDatasourceEditField' => 'applications/transactions/editfield/PhabricatorDatasourceEditField.php',
     'PhabricatorDatasourceEditType' => 'applications/transactions/edittype/PhabricatorDatasourceEditType.php',
@@ -2697,7 +2713,8 @@ phutil_register_library_map(array(
     'PhabricatorNotificationPanelController' => 'applications/notification/controller/PhabricatorNotificationPanelController.php',
     'PhabricatorNotificationQuery' => 'applications/notification/query/PhabricatorNotificationQuery.php',
     'PhabricatorNotificationSearchEngine' => 'applications/notification/query/PhabricatorNotificationSearchEngine.php',
-    'PhabricatorNotificationStatusController' => 'applications/notification/controller/PhabricatorNotificationStatusController.php',
+    'PhabricatorNotificationServerRef' => 'applications/notification/client/PhabricatorNotificationServerRef.php',
+    'PhabricatorNotificationServersConfigOptionType' => 'applications/notification/config/PhabricatorNotificationServersConfigOptionType.php',
     'PhabricatorNotificationStatusView' => 'applications/notification/view/PhabricatorNotificationStatusView.php',
     'PhabricatorNotificationTestController' => 'applications/notification/controller/PhabricatorNotificationTestController.php',
     'PhabricatorNotificationTestFeedStory' => 'applications/notification/feed/PhabricatorNotificationTestFeedStory.php',
@@ -3197,6 +3214,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryURITestCase' => 'applications/repository/storage/__tests__/PhabricatorRepositoryURITestCase.php',
     'PhabricatorRepositoryVCSPassword' => 'applications/repository/storage/PhabricatorRepositoryVCSPassword.php',
     'PhabricatorRepositoryVersion' => 'applications/repository/constants/PhabricatorRepositoryVersion.php',
+    'PhabricatorRepositoryWorkingCopyVersion' => 'applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php',
     'PhabricatorRequestExceptionHandler' => 'aphront/handler/PhabricatorRequestExceptionHandler.php',
     'PhabricatorResourceSite' => 'aphront/site/PhabricatorResourceSite.php',
     'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
@@ -3422,6 +3440,7 @@ phutil_register_library_map(array(
     'PhabricatorSystemDAO' => 'applications/system/storage/PhabricatorSystemDAO.php',
     'PhabricatorSystemDestructionGarbageCollector' => 'applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php',
     'PhabricatorSystemDestructionLog' => 'applications/system/storage/PhabricatorSystemDestructionLog.php',
+    'PhabricatorSystemReadOnlyController' => 'applications/system/controller/PhabricatorSystemReadOnlyController.php',
     'PhabricatorSystemRemoveDestroyWorkflow' => 'applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php',
     'PhabricatorSystemRemoveLogWorkflow' => 'applications/system/management/PhabricatorSystemRemoveLogWorkflow.php',
     'PhabricatorSystemRemoveWorkflow' => 'applications/system/management/PhabricatorSystemRemoveWorkflow.php',
@@ -4821,6 +4840,7 @@ phutil_register_library_map(array(
     'DiffusionHovercardEngineExtension' => 'PhabricatorHovercardEngineExtension',
     'DiffusionInlineCommentController' => 'PhabricatorInlineCommentController',
     'DiffusionInlineCommentPreviewController' => 'PhabricatorInlineCommentPreviewController',
+    'DiffusionInternalGitRawDiffQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
     'DiffusionLastModifiedController' => 'DiffusionController',
     'DiffusionLastModifiedQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
     'DiffusionLintController' => 'DiffusionController',
@@ -4910,6 +4930,7 @@ phutil_register_library_map(array(
     'DiffusionRefsQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
     'DiffusionRenameHistoryQuery' => 'Phobject',
     'DiffusionRepositoryByIDRemarkupRule' => 'PhabricatorObjectRemarkupRule',
+    'DiffusionRepositoryClusterManagementPanel' => 'DiffusionRepositoryManagementPanel',
     'DiffusionRepositoryController' => 'DiffusionController',
     'DiffusionRepositoryCreateController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryDatasource' => 'PhabricatorTypeaheadDatasource',
@@ -4930,6 +4951,8 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryListController' => 'DiffusionController',
+    'DiffusionRepositoryManageController' => 'DiffusionController',
+    'DiffusionRepositoryManagementPanel' => 'Phobject',
     'DiffusionRepositoryNewController' => 'DiffusionController',
     'DiffusionRepositoryPath' => 'Phobject',
     'DiffusionRepositoryRef' => 'Phobject',
@@ -6264,6 +6287,7 @@ phutil_register_library_map(array(
     'PhabricatorBadgesController' => 'PhabricatorController',
     'PhabricatorBadgesCreateCapability' => 'PhabricatorPolicyCapability',
     'PhabricatorBadgesDAO' => 'PhabricatorLiskDAO',
+    'PhabricatorBadgesDatasource' => 'PhabricatorTypeaheadDatasource',
     'PhabricatorBadgesDefaultEditCapability' => 'PhabricatorPolicyCapability',
     'PhabricatorBadgesEditConduitAPIMethod' => 'PhabricatorEditEngineAPIMethod',
     'PhabricatorBadgesEditController' => 'PhabricatorBadgesController',
@@ -6392,6 +6416,12 @@ phutil_register_library_map(array(
     'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
+    'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType',
+    'PhabricatorClusterException' => 'Exception',
+    'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
+    'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException',
+    'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
+    'PhabricatorClusterStrandedException' => 'PhabricatorClusterException',
     'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
     'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
     'PhabricatorCommentEditField' => 'PhabricatorEditField',
@@ -6439,6 +6469,8 @@ phutil_register_library_map(array(
     'PhabricatorConfigAllController' => 'PhabricatorConfigController',
     'PhabricatorConfigApplication' => 'PhabricatorApplication',
     'PhabricatorConfigCacheController' => 'PhabricatorConfigController',
+    'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
+    'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController',
     'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
     'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
     'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
@@ -6573,9 +6605,10 @@ phutil_register_library_map(array(
     'PhabricatorCustomFieldStringIndexStorage' => 'PhabricatorCustomFieldIndexStorage',
     'PhabricatorCustomHeaderConfigType' => 'PhabricatorConfigOptionType',
     'PhabricatorDaemon' => 'PhutilDaemon',
-    'PhabricatorDaemonBulkJobListController' => 'PhabricatorDaemonController',
+    'PhabricatorDaemonBulkJobController' => 'PhabricatorDaemonController',
-    'PhabricatorDaemonBulkJobMonitorController' => 'PhabricatorDaemonController',
+    'PhabricatorDaemonBulkJobListController' => 'PhabricatorDaemonBulkJobController',
-    'PhabricatorDaemonBulkJobViewController' => 'PhabricatorDaemonController',
+    'PhabricatorDaemonBulkJobMonitorController' => 'PhabricatorDaemonBulkJobController',
+    'PhabricatorDaemonBulkJobViewController' => 'PhabricatorDaemonBulkJobController',
     'PhabricatorDaemonConsoleController' => 'PhabricatorDaemonController',
     'PhabricatorDaemonContentSource' => 'PhabricatorContentSource',
     'PhabricatorDaemonController' => 'PhabricatorController',
@@ -6681,6 +6714,8 @@ phutil_register_library_map(array(
     'PhabricatorDashboardViewController' => 'PhabricatorDashboardController',
     'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
     'PhabricatorDataNotAttachedException' => 'Exception',
+    'PhabricatorDatabaseHealthRecord' => 'Phobject',
+    'PhabricatorDatabaseRef' => 'Phobject',
     'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorDatasourceEditField' => 'PhabricatorTokenizerEditField',
     'PhabricatorDatasourceEditType' => 'PhabricatorPHIDListEditType',
@@ -7197,7 +7232,8 @@ phutil_register_library_map(array(
     'PhabricatorNotificationPanelController' => 'PhabricatorNotificationController',
     'PhabricatorNotificationQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorNotificationSearchEngine' => 'PhabricatorApplicationSearchEngine',
-    'PhabricatorNotificationStatusController' => 'PhabricatorNotificationController',
+    'PhabricatorNotificationServerRef' => 'Phobject',
+    'PhabricatorNotificationServersConfigOptionType' => 'PhabricatorConfigJSONOptionType',
     'PhabricatorNotificationStatusView' => 'AphrontTagView',
     'PhabricatorNotificationTestController' => 'PhabricatorNotificationController',
     'PhabricatorNotificationTestFeedStory' => 'PhabricatorFeedStory',
@@ -7827,6 +7863,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryURITestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryVCSPassword' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryVersion' => 'Phobject',
+    'PhabricatorRepositoryWorkingCopyVersion' => 'PhabricatorRepositoryDAO',
     'PhabricatorRequestExceptionHandler' => 'AphrontRequestExceptionHandler',
     'PhabricatorResourceSite' => 'PhabricatorSite',
     'PhabricatorRobotsController' => 'PhabricatorController',
@@ -8070,6 +8107,7 @@ phutil_register_library_map(array(
     'PhabricatorSystemDAO' => 'PhabricatorLiskDAO',
     'PhabricatorSystemDestructionGarbageCollector' => 'PhabricatorGarbageCollector',
     'PhabricatorSystemDestructionLog' => 'PhabricatorSystemDAO',
+    'PhabricatorSystemReadOnlyController' => 'PhabricatorController',
     'PhabricatorSystemRemoveDestroyWorkflow' => 'PhabricatorSystemRemoveWorkflow',
     'PhabricatorSystemRemoveLogWorkflow' => 'PhabricatorSystemRemoveWorkflow',
     'PhabricatorSystemRemoveWorkflow' => 'PhabricatorManagementWorkflow',

src/aphront/configuration/AphrontApplicationConfiguration.php

@@ -345,6 +345,18 @@ abstract class AphrontApplicationConfiguration extends Phobject {
 
     if ($site->shouldRequireHTTPS()) {
       if (!$request->isHTTPS()) {
+
+        // Don't redirect intracluster requests: doing so drops headers and
+        // parameters, imposes a performance penalty, and indicates a
+        // misconfiguration.
+        if ($request->isProxiedClusterRequest()) {
+          throw new AphrontMalformedRequestException(
+            pht('HTTPS Required'),
+            pht(
+              'This request reached a site which requires HTTPS, but the '.
+              'request is not marked as HTTPS.'));
+        }
+
         $https_uri = $request->getRequestURI();
         $https_uri->setDomain($request->getHost());
         $https_uri->setProtocol('https');

src/aphront/site/PhabricatorSite.php

@@ -3,6 +3,15 @@
 abstract class PhabricatorSite extends AphrontSite {
 
   public function shouldRequireHTTPS() {
+    // If this is an intracluster request, it's okay for it to use HTTP even
+    // if the site otherwise requires HTTPS. It is common to terminate SSL at
+    // a load balancer and use plain HTTP from then on, and administrators are
+    // usually not concerned about attackers observing traffic within a
+    // datacenter.
+    if (PhabricatorEnv::isClusterRemoteAddress()) {
+      return false;
+    }
+
     return PhabricatorEnv::getEnvConfig('security.require-https');
   }
 

src/applications/almanac/application/PhabricatorAlmanacApplication.php

@@ -83,8 +83,7 @@ final class PhabricatorAlmanacApplication extends PhabricatorApplication {
       phutil_tag(
         'a',
         array(
-          'href' => PhabricatorEnv::getDoclink(
+          'href' => PhabricatorEnv::getDoclink('Clustering Introduction'),
-            'User Guide: Phabricator Clusters'),
           'target' => '_blank',
         ),
         pht('Learn More')));

src/applications/almanac/controller/AlmanacController.php

@@ -178,7 +178,7 @@ abstract class AlmanacController
       'a',
       array(
         'href' => PhabricatorEnv::getDoclink(
-          'User Guide: Phabricator Clusters'),
+          'Clustering Introduction'),
         'target' => '_blank',
       ),
       pht('Learn More'));

src/applications/almanac/controller/AlmanacDeviceViewController.php

@@ -117,7 +117,7 @@ final class AlmanacDeviceViewController
       ->setCanEdit($can_edit);
 
     $header = id(new PHUIHeaderView())
-      ->setHeader(pht('DEVICE INTERFACES'))
+      ->setHeader(pht('Device Interfaces'))
       ->addActionLink(
         id(new PHUIButtonView())
           ->setTag('a')
@@ -167,7 +167,7 @@ final class AlmanacDeviceViewController
     $upload_uri = '/auth/sshkey/upload/?objectPHID='.$device_phid;
 
     $header = id(new PHUIHeaderView())
-      ->setHeader(pht('SSH PUBLIC KEYS'))
+      ->setHeader(pht('SSH Public Keys'))
       ->addActionLink(
         id(new PHUIButtonView())
           ->setTag('a')
@@ -238,7 +238,7 @@ final class AlmanacDeviceViewController
         ));
 
     return id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('BOUND SERVICES'))
+      ->setHeaderText(pht('Bound Services'))
       ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setTable($table);
   }

src/applications/almanac/util/AlmanacKeys.php

@@ -19,4 +19,33 @@ final class AlmanacKeys extends Phobject {
     return null;
   }
 
+  public static function getLiveDevice() {
+    $device_id = self::getDeviceID();
+    if (!$device_id) {
+      return null;
+    }
+
+    $cache = PhabricatorCaches::getRequestCache();
+    $cache_key = 'almanac.device.self';
+
+    $device = $cache->getKey($cache_key);
+    if (!$device) {
+      $viewer = PhabricatorUser::getOmnipotentUser();
+      $device = id(new AlmanacDeviceQuery())
+        ->setViewer($viewer)
+        ->withNames(array($device_id))
+        ->executeOne();
+      if (!$device) {
+        throw new Exception(
+          pht(
+            'This host has device ID "%s", but there is no corresponding '.
+            'device record in Almanac.',
+            $device_id));
+      }
+      $cache->setKey($cache_key, $device);
+    }
+
+    return $device;
+  }
+
 }

src/applications/aphlict/management/PhabricatorAphlictManagementDebugWorkflow.php

@@ -4,17 +4,18 @@ final class PhabricatorAphlictManagementDebugWorkflow
   extends PhabricatorAphlictManagementWorkflow {
 
   protected function didConstruct() {
-    parent::didConstruct();
     $this
       ->setName('debug')
       ->setSynopsis(
         pht(
           'Start the notifications server in the foreground and print large '.
-          'volumes of diagnostic information to the console.'));
+          'volumes of diagnostic information to the console.'))
+      ->setArguments($this->getLaunchArguments());
   }
 
   public function execute(PhutilArgumentParser $args) {
-    parent::execute($args);
+    $this->parseLaunchArguments($args);
+
     $this->setDebug(true);
 
     $this->willLaunch();

src/applications/aphlict/management/PhabricatorAphlictManagementRestartWorkflow.php

@@ -4,19 +4,20 @@ final class PhabricatorAphlictManagementRestartWorkflow
   extends PhabricatorAphlictManagementWorkflow {
 
   protected function didConstruct() {
-    parent::didConstruct();
     $this
       ->setName('restart')
-      ->setSynopsis(pht('Stop, then start the notifications server.'));
+      ->setSynopsis(pht('Stop, then start the notification server.'))
+      ->setArguments($this->getLaunchArguments());
   }
 
   public function execute(PhutilArgumentParser $args) {
-    parent::execute($args);
+    $this->parseLaunchArguments($args);
 
     $err = $this->executeStopCommand();
     if ($err) {
       return $err;
     }
+
     return $this->executeStartCommand();
   }
 

src/applications/aphlict/management/PhabricatorAphlictManagementStartWorkflow.php

@@ -4,14 +4,14 @@ final class PhabricatorAphlictManagementStartWorkflow
   extends PhabricatorAphlictManagementWorkflow {
 
   protected function didConstruct() {
-    parent::didConstruct();
     $this
       ->setName('start')
-      ->setSynopsis(pht('Start the notifications server.'));
+      ->setSynopsis(pht('Start the notifications server.'))
+      ->setArguments($this->getLaunchArguments());
   }
 
   public function execute(PhutilArgumentParser $args) {
-    parent::execute($args);
+    $this->parseLaunchArguments($args);
     return $this->executeStartCommand();
   }
 

src/applications/aphlict/management/PhabricatorAphlictManagementStatusWorkflow.php

@@ -6,7 +6,7 @@ final class PhabricatorAphlictManagementStatusWorkflow
   protected function didConstruct() {
     $this
       ->setName('status')
-      ->setSynopsis(pht('Show the status of the notifications server.'))
+      ->setSynopsis(pht('Show the status of the notification server.'))
       ->setArguments(array());
   }
 

src/applications/aphlict/management/PhabricatorAphlictManagementStopWorkflow.php

@@ -6,11 +6,12 @@ final class PhabricatorAphlictManagementStopWorkflow
   protected function didConstruct() {
     $this
       ->setName('stop')
-      ->setSynopsis(pht('Stop the notifications server.'))
+      ->setSynopsis(pht('Stop the notification server.'))
-      ->setArguments(array());
+      ->setArguments($this->getLaunchArguments());
   }
 
   public function execute(PhutilArgumentParser $args) {
+    $this->parseLaunchArguments($args);
     return $this->executeStopCommand();
   }
 

src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php

@@ -4,66 +4,293 @@ abstract class PhabricatorAphlictManagementWorkflow
   extends PhabricatorManagementWorkflow {
 
   private $debug = false;
-  private $clientHost;
+  private $configData;
-  private $clientPort;
+  private $configPath;
 
-  protected function didConstruct() {
+  final protected function setDebug($debug) {
-    $this
+    $this->debug = $debug;
-      ->setArguments(
+    return $this;
-        array(
-          array(
-            'name'  => 'client-host',
-            'param' => 'hostname',
-            'help'  => pht('Hostname to bind to for the client server.'),
-          ),
-          array(
-            'name'  => 'client-port',
-            'param' => 'port',
-            'help'  => pht('Port to bind to for the client server.'),
-          ),
-        ));
   }
 
-  public function execute(PhutilArgumentParser $args) {
+  protected function getLaunchArguments() {
-    $this->clientHost = $args->getArg('client-host');
+    return array(
-    $this->clientPort = $args->getArg('client-port');
+      array(
-    return 0;
+        'name' => 'config',
+        'param' => 'file',
+        'help' => pht(
+          'Use a specific configuration file instead of the default '.
+          'configuration.'),
+      ),
+    );
+  }
+
+  protected function parseLaunchArguments(PhutilArgumentParser $args) {
+    $config_file = $args->getArg('config');
+    if ($config_file) {
+      $full_path = Filesystem::resolvePath($config_file);
+      $show_path = $full_path;
+    } else {
+      $root = dirname(dirname(phutil_get_library_root('phabricator')));
+
+      $try = array(
+        'phabricator/conf/aphlict/aphlict.custom.json',
+        'phabricator/conf/aphlict/aphlict.default.json',
+      );
+
+      foreach ($try as $config) {
+        $full_path = $root.'/'.$config;
+        $show_path = $config;
+        if (Filesystem::pathExists($full_path)) {
+          break;
+        }
+      }
+    }
+
+    echo tsprintf(
+      "%s\n",
+      pht(
+        'Reading configuration from: %s',
+        $show_path));
+
+    try {
+      $data = Filesystem::readFile($full_path);
+    } catch (Exception $ex) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Failed to read configuration file. %s',
+          $ex->getMessage()));
+    }
+
+    try {
+      $data = phutil_json_decode($data);
+    } catch (Exception $ex) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Configuration file is not properly formatted JSON. %s',
+          $ex->getMessage()));
+    }
+
+    try {
+      PhutilTypeSpec::checkMap(
+        $data,
+        array(
+          'servers' => 'list<wild>',
+          'logs' => 'optional list<wild>',
+          'cluster' => 'optional list<wild>',
+          'pidfile' => 'string',
+          'memory.hint' => 'optional int',
+        ));
+    } catch (Exception $ex) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Configuration file has improper configuration keys at top '.
+          'level. %s',
+          $ex->getMessage()));
+    }
+
+    $servers = $data['servers'];
+    $has_client = false;
+    $has_admin = false;
+    $port_map = array();
+    foreach ($servers as $index => $server) {
+      PhutilTypeSpec::checkMap(
+        $server,
+        array(
+          'type' => 'string',
+          'port' => 'int',
+          'listen' => 'optional string|null',
+          'ssl.key' => 'optional string|null',
+          'ssl.cert' => 'optional string|null',
+          'ssl.chain' => 'optional string|null',
+        ));
+
+      $port = $server['port'];
+      if (!isset($port_map[$port])) {
+        $port_map[$port] = $index;
+      } else {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Two servers (at indexes "%s" and "%s") both bind to the same '.
+            'port ("%s"). Each server must bind to a unique port.',
+            $port_map[$port],
+            $index,
+            $port));
+      }
+
+      $type = $server['type'];
+      switch ($type) {
+        case 'admin':
+          $has_admin = true;
+          break;
+        case 'client':
+          $has_client = true;
+          break;
+        default:
+          throw new PhutilArgumentUsageException(
+            pht(
+              'A specified server (at index "%s", on port "%s") has an '.
+              'invalid type ("%s"). Valid types are: admin, client.',
+              $index,
+              $port,
+              $type));
+      }
+
+      $ssl_key = idx($server, 'ssl.key');
+      $ssl_cert = idx($server, 'ssl.cert');
+      if (($ssl_key && !$ssl_cert) || ($ssl_cert && !$ssl_key)) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'A specified server (at index "%s", on port "%s") specifies '.
+            'only one of "%s" and "%s". Each server must specify neither '.
+            '(to disable SSL) or specify both (to enable it).',
+            $index,
+            $port,
+            'ssl.key',
+            'ssl.cert'));
+      }
+
+      $ssl_chain = idx($server, 'ssl.chain');
+      if ($ssl_chain && (!$ssl_key && !$ssl_cert)) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'A specified server (at index "%s", on port "%s") specifies '.
+            'a value for "%s", but no value for "%s" or "%s". Servers '.
+            'should only provide an SSL chain if they also provide an SSL '.
+            'key and SSL certificate.',
+            $index,
+            $port,
+            'ssl.chain',
+            'ssl.key',
+            'ssl.cert'));
+      }
+    }
+
+    if (!$servers) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Configuration file does not specify any servers. This service '.
+          'will not be able to interact with the outside world if it does '.
+          'not listen on any ports. You must specify at least one "%s" '.
+          'server and at least one "%s" server.',
+          'admin',
+          'client'));
+    }
+
+    if (!$has_client) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Configuration file does not specify any client servers. This '.
+          'service will be unable to transmit any notifications without a '.
+          'client server. You must specify at least one server with '.
+          'type "%s".',
+          'client'));
+    }
+
+    if (!$has_admin) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Configuration file does not specify any administrative '.
+          'servers. This service will be unable to receive messages. '.
+          'You must specify at least one server with type "%s".',
+          'admin'));
+    }
+
+    $logs = idx($data, 'logs', array());
+    foreach ($logs as $index => $log) {
+      PhutilTypeSpec::checkMap(
+        $log,
+        array(
+          'path' => 'string',
+        ));
+
+      $path = $log['path'];
+
+      try {
+        $dir = dirname($path);
+        if (!Filesystem::pathExists($dir)) {
+          Filesystem::createDirectory($dir, 0755, true);
+        }
+      } catch (FilesystemException $ex) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Failed to create directory "%s" for specified log file (with '.
+            'index "%s"). You should manually create this directory or '.
+            'choose a different logfile location. %s',
+            $dir,
+            $ex->getMessage()));
+      }
+    }
+
+    $peer_map = array();
+
+    $cluster = idx($data, 'cluster', array());
+    foreach ($cluster as $index => $peer) {
+      PhutilTypeSpec::checkMap(
+        $peer,
+        array(
+          'host' => 'string',
+          'port' => 'int',
+          'protocol' => 'string',
+        ));
+
+      $host = $peer['host'];
+      $port = $peer['port'];
+      $protocol = $peer['protocol'];
+
+      switch ($protocol) {
+        case 'http':
+        case 'https':
+          break;
+        default:
+          throw new PhutilArgumentUsageException(
+            pht(
+              'Configuration file specifies cluster peer ("%s", at index '.
+              '"%s") with an invalid protocol, "%s". Valid protocols are '.
+              '"%s" or "%s".',
+              $host,
+              $index,
+              $protocol,
+              'http',
+              'https'));
+      }
+
+      $peer_key = "{$host}:{$port}";
+      if (!isset($peer_map[$peer_key])) {
+        $peer_map[$peer_key] = $index;
+      } else {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Configuration file specifies cluster peer "%s" more than '.
+            'once (at indexes "%s" and "%s"). Each peer must have a '.
+            'unique host and port combination.',
+            $peer_key,
+            $peer_map[$peer_key],
+            $index));
+      }
+    }
+
+    $this->configData = $data;
+    $this->configPath = $full_path;
+
+    $pid_path = $this->getPIDPath();
+    try {
+      $dir = dirname($path);
+      if (!Filesystem::pathExists($dir)) {
+        Filesystem::createDirectory($dir, 0755, true);
+      }
+    } catch (FilesystemException $ex) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Failed to create directory "%s" for specified PID file. You '.
+          'should manually create this directory or choose a different '.
+          'PID file location. %s',
+          $dir,
+          $ex->getMessage()));
+    }
   }
 
   final public function getPIDPath() {
-    $path = PhabricatorEnv::getEnvConfig('notification.pidfile');
+    return $this->configData['pidfile'];
-
-    try {
-      $dir = dirname($path);
-      if (!Filesystem::pathExists($dir)) {
-        Filesystem::createDirectory($dir, 0755, true);
-      }
-    } catch (FilesystemException $ex) {
-      throw new Exception(
-        pht(
-          "Failed to create '%s'. You should manually create this directory.",
-          $dir));
-    }
-
-    return $path;
-  }
-
-  final public function getLogPath() {
-    $path = PhabricatorEnv::getEnvConfig('notification.log');
-
-    try {
-      $dir = dirname($path);
-      if (!Filesystem::pathExists($dir)) {
-        Filesystem::createDirectory($dir, 0755, true);
-      }
-    } catch (FilesystemException $ex) {
-      throw new Exception(
-        pht(
-          "Failed to create '%s'. You should manually create this directory.",
-          $dir));
-    }
-
-    return $path;
   }
 
   final public function getPID() {
@@ -86,11 +313,6 @@ abstract class PhabricatorAphlictManagementWorkflow
     exit(1);
   }
 
-  final protected function setDebug($debug) {
-    $this->debug = $debug;
-    return $this;
-  }
-
   public static function requireExtensions() {
     self::mustHaveExtension('pcntl');
     self::mustHaveExtension('posix');
@@ -146,54 +368,16 @@ abstract class PhabricatorAphlictManagementWorkflow
     $test_argv = $this->getServerArgv();
     $test_argv[] = '--test=true';
 
-    execx(
+
-      '%s %s %Ls',
+    execx('%C', $this->getStartCommand($test_argv));
-      $this->getNodeBinary(),
-      $this->getAphlictScriptPath(),
-      $test_argv);
   }
 
   private function getServerArgv() {
-    $ssl_key = PhabricatorEnv::getEnvConfig('notification.ssl-key');
-    $ssl_cert = PhabricatorEnv::getEnvConfig('notification.ssl-cert');
-
-    $server_uri = PhabricatorEnv::getEnvConfig('notification.server-uri');
-    $server_uri = new PhutilURI($server_uri);
-
-    $client_uri = PhabricatorEnv::getEnvConfig('notification.client-uri');
-    $client_uri = new PhutilURI($client_uri);
-
-    $log = $this->getLogPath();
-
     $server_argv = array();
-    $server_argv[] = '--client-port='.coalesce(
+    $server_argv[] = '--config='.$this->configPath;
-      $this->clientPort,
-      $client_uri->getPort());
-    $server_argv[] = '--admin-port='.$server_uri->getPort();
-    $server_argv[] = '--admin-host='.$server_uri->getDomain();
-
-    if ($ssl_key) {
-      $server_argv[] = '--ssl-key='.$ssl_key;
-    }
-
-    if ($ssl_cert) {
-      $server_argv[] = '--ssl-cert='.$ssl_cert;
-    }
-
-    $server_argv[] = '--log='.$log;
-
-    if ($this->clientHost) {
-      $server_argv[] = '--client-host='.$this->clientHost;
-    }
-
     return $server_argv;
   }
 
-  private function getAphlictScriptPath() {
-    $root = dirname(phutil_get_library_root('phabricator'));
-    return $root.'/support/aphlict/server/aphlict_server.js';
-  }
-
   final protected function launch() {
     $console = PhutilConsole::getConsole();
 
@@ -205,11 +389,7 @@ abstract class PhabricatorAphlictManagementWorkflow
       Filesystem::writeFile($this->getPIDPath(), getmypid());
     }
 
-    $command = csprintf(
+    $command = $this->getStartCommand($this->getServerArgv());
-      '%s %s %Ls',
-      $this->getNodeBinary(),
-      $this->getAphlictScriptPath(),
-      $this->getServerArgv());
 
     if (!$this->debug) {
       declare(ticks = 1);
@@ -267,7 +447,6 @@ abstract class PhabricatorAphlictManagementWorkflow
     fclose(STDOUT);
     fclose(STDERR);
 
-
     $this->launch();
     return 0;
   }
@@ -325,4 +504,29 @@ abstract class PhabricatorAphlictManagementWorkflow
         '$PATH'));
   }
 
+  private function getAphlictScriptPath() {
+    $root = dirname(phutil_get_library_root('phabricator'));
+    return $root.'/support/aphlict/server/aphlict_server.js';
+  }
+
+  private function getNodeArgv() {
+    $argv = array();
+
+    $hint = idx($this->configData, 'memory.hint');
+    $hint = nonempty($hint, 256);
+
+    $argv[] = sprintf('--max-old-space-size=%d', $hint);
+
+    return $argv;
+  }
+
+  private function getStartCommand(array $server_argv) {
+    return csprintf(
+      '%R %Ls -- %s %Ls',
+      $this->getNodeBinary(),
+      $this->getNodeArgv(),
+      $this->getAphlictScriptPath(),
+      $server_argv);
+  }
+
 }

src/applications/badges/controller/PhabricatorBadgesAwardController.php

@@ -18,23 +18,24 @@ final class PhabricatorBadgesAwardController
     $view_uri = '/p/'.$user->getUsername();
 
     if ($request->isFormPost()) {
-      $xactions = array();
+      $badge_phids = $request->getArr('badgePHIDs');
-      $badge_phid = $request->getStr('badgePHID');
+      $badges = id(new PhabricatorBadgesQuery())
-      $badge = id(new PhabricatorBadgesQuery())
         ->setViewer($viewer)
-        ->withPHIDs(array($badge_phid))
+        ->withPHIDs($badge_phids)
         ->needRecipients(true)
         ->requireCapabilities(
           array(
             PhabricatorPolicyCapability::CAN_EDIT,
             PhabricatorPolicyCapability::CAN_VIEW,
           ))
-        ->executeOne();
+        ->execute();
-      if (!$badge) {
+      if (!$badges) {
         return new Aphront404Response();
       }
       $award_phids = array($user->getPHID());
 
+      foreach ($badges as $badge) {
+        $xactions = array();
         $xactions[] = id(new PhabricatorBadgesTransaction())
           ->setTransactionType(PhabricatorBadgesTransaction::TYPE_AWARD)
           ->setNewValue($award_phids);
@@ -45,33 +46,24 @@ final class PhabricatorBadgesAwardController
           ->setContinueOnNoEffect(true)
           ->setContinueOnMissingFields(true)
           ->applyTransactions($badge, $xactions);
+      }
 
       return id(new AphrontRedirectResponse())
         ->setURI($view_uri);
     }
 
-    $badges = id(new PhabricatorBadgesQuery())
-      ->setViewer($viewer)
-      ->withStatuses(array(
-        PhabricatorBadgesBadge::STATUS_ACTIVE,
-      ))
-      ->requireCapabilities(
-        array(
-          PhabricatorPolicyCapability::CAN_VIEW,
-          PhabricatorPolicyCapability::CAN_EDIT,
-        ))
-      ->execute();
-
-    $options = mpull($badges, 'getName', 'getPHID');
-    asort($options);
-
     $form = id(new AphrontFormView())
       ->setUser($viewer)
-      ->appendChild(
+      ->appendControl(
-        id(new AphrontFormSelectControl())
+        id(new AphrontFormTokenizerControl())
           ->setLabel(pht('Badge'))
-          ->setName('badgePHID')
+          ->setName('badgePHIDs')
-          ->setOptions($options));
+          ->setDatasource(
+            id(new PhabricatorBadgesDatasource())
+              ->setParameters(
+                array(
+                  'recipientPHID' => $user->getPHID(),
+                  ))));
 
     $dialog = $this->newDialog()
       ->setTitle(pht('Grant Badge'))

src/applications/badges/typeahead/PhabricatorBadgesDatasource.php

@@ -0,0 +1,69 @@
+<?php
+
+final class PhabricatorBadgesDatasource
+  extends PhabricatorTypeaheadDatasource {
+
+  public function getBrowseTitle() {
+    return pht('Browse Badges');
+  }
+
+  public function getPlaceholderText() {
+    return pht('Type a badge name...');
+  }
+
+  public function getDatasourceApplicationClass() {
+    return 'PhabricatorBadgesApplication';
+  }
+
+  public function loadResults() {
+    $viewer = $this->getViewer();
+    $raw_query = $this->getRawQuery();
+
+    $params = $this->getParameters();
+    $recipient_phid = $params['recipientPHID'];
+
+    $badges = id(new PhabricatorBadgesQuery())
+      ->setViewer($viewer)
+      ->requireCapabilities(
+        array(
+          PhabricatorPolicyCapability::CAN_VIEW,
+          PhabricatorPolicyCapability::CAN_EDIT,
+        ))
+      ->execute();
+
+    $awards = id(new PhabricatorBadgesAwardQuery())
+      ->setViewer($viewer)
+      ->withAwarderPHIDs(array($viewer->getPHID()))
+      ->withRecipientPHIDs(array($recipient_phid))
+      ->execute();
+    $awards = mpull($awards, null, 'getBadgePHID');
+
+    $results = array();
+    foreach ($badges as $badge) {
+      $closed = null;
+
+      $badge_awards = idx($awards, $badge->getPHID(), null);
+      if ($badge_awards) {
+        $closed = pht('Already awarded');
+      }
+
+      $status = $badge->getStatus();
+      if ($status === PhabricatorBadgesBadge::STATUS_ARCHIVED) {
+        $closed = pht('Archived');
+      }
+
+      $results[] = id(new PhabricatorTypeaheadResult())
+        ->setName($badge->getName())
+        ->setIcon($badge->getIcon())
+        ->setColor(
+          PhabricatorBadgesQuality::getQualityColor($badge->getQuality()))
+        ->setClosed($closed)
+        ->setPHID($badge->getPHID());
+    }
+
+    $results = $this->filterResultsAgainstTokens($results);
+
+    return $results;
+  }
+
+}

src/applications/badges/view/PhabricatorBadgesRecipientsListView.php

@@ -62,7 +62,7 @@ final class PhabricatorBadgesRecipientsListView extends AphrontView {
     }
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('RECIPIENTS'))
+      ->setHeaderText(pht('Recipients'))
       ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setObjectList($list);
 

src/applications/cache/PhabricatorCaches.php

@@ -174,6 +174,11 @@ final class PhabricatorCaches extends Phobject {
    * @task setup
    */
   private static function buildSetupCaches() {
+    // If this is the CLI, just build a setup cache.
+    if (php_sapi_name() == 'cli') {
+      return array();
+    }
+
     // In most cases, we should have APC. This is an ideal cache for our
     // purposes -- it's fast and empties on server restart.
     $apc = new PhutilAPCKeyValueCache();

src/applications/cache/PhabricatorKeyValueDatabaseCache.php

@@ -7,6 +7,10 @@ final class PhabricatorKeyValueDatabaseCache
   const CACHE_FORMAT_DEFLATE    = 'deflate';
 
   public function setKeys(array $keys, $ttl = null) {
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     if ($keys) {
       $map = $this->digestKeys(array_keys($keys));
       $conn_w = $this->establishConnection('w');

src/applications/conduit/controller/PhabricatorConduitAPIController.php

@@ -402,6 +402,23 @@ final class PhabricatorConduitAPIController
         $user);
     }
 
+
+    // For intracluster requests, use a public user if no authentication
+    // information is provided. We could do this safely for any request,
+    // but making the API fully public means there's no way to disable badly
+    // behaved clients.
+    if (PhabricatorEnv::isClusterRemoteAddress()) {
+      if (PhabricatorEnv::getEnvConfig('policy.allow-public')) {
+        $api_request->setIsClusterRequest(true);
+
+        $user = new PhabricatorUser();
+        return $this->validateAuthenticatedUser(
+          $api_request,
+          $user);
+      }
+    }
+
+
     // Handle sessionless auth.
     // TODO: This is super messy.
     // TODO: Remove this in favor of token-based auth.

src/applications/conduit/method/ConduitAPIMethod.php

@@ -52,6 +52,9 @@ abstract class ConduitAPIMethod
 
   abstract protected function execute(ConduitAPIRequest $request);
 
+  public function isInternalAPI() {
+    return false;
+  }
 
   public function getParamTypes() {
     $types = $this->defineParamTypes();

src/applications/conduit/query/PhabricatorConduitMethodQuery.php

@@ -9,6 +9,7 @@ final class PhabricatorConduitMethodQuery
   private $applicationNames;
   private $nameContains;
   private $methods;
+  private $isInternal;
 
   public function withMethods(array $methods) {
     $this->methods = $methods;
@@ -40,6 +41,11 @@ final class PhabricatorConduitMethodQuery
     return $this;
   }
 
+  public function withIsInternal($is_internal) {
+    $this->isInternal = $is_internal;
+    return $this;
+  }
+
   protected function loadPage() {
     $methods = $this->getAllMethods();
     $methods = $this->filterMethods($methods);
@@ -112,6 +118,14 @@ final class PhabricatorConduitMethodQuery
       }
     }
 
+    if ($this->isInternal !== null) {
+      foreach ($methods as $key => $method) {
+        if ($method->isInternalAPI() !== $this->isInternal) {
+          unset($methods[$key]);
+        }
+      }
+    }
+
     return $methods;
   }
 

src/applications/conduit/query/PhabricatorConduitSearchEngine.php

@@ -37,6 +37,7 @@ final class PhabricatorConduitSearchEngine
     $query->withIsStable($saved->getParameter('isStable'));
     $query->withIsUnstable($saved->getParameter('isUnstable'));
     $query->withIsDeprecated($saved->getParameter('isDeprecated'));
+    $query->withIsInternal(false);
 
     $names = $saved->getParameter('applicationNames', array());
     if ($names) {

src/applications/config/application/PhabricatorConfigApplication.php

@@ -62,6 +62,10 @@ final class PhabricatorConfigApplication extends PhabricatorApplication {
         'module/' => array(
           '(?P<module>[^/]+)/' => 'PhabricatorConfigModuleController',
         ),
+        'cluster/' => array(
+          'databases/' => 'PhabricatorConfigClusterDatabasesController',
+          'notifications/' => 'PhabricatorConfigClusterNotificationsController',
+        ),
       ),
     );
   }

src/applications/config/check/PhabricatorDaemonsSetupCheck.php

@@ -46,49 +46,40 @@ final class PhabricatorDaemonsSetupCheck extends PhabricatorSetupCheck {
         ->addCommand('phabricator/ $ ./bin/phd start');
     }
 
-    $phd_user = PhabricatorEnv::getEnvConfig('phd.user');
+    $expect_user = PhabricatorEnv::getEnvConfig('phd.user');
+    if (strlen($expect_user)) {
       $all_daemons = id(new PhabricatorDaemonLogQuery())
         ->setViewer(PhabricatorUser::getOmnipotentUser())
         ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)
         ->execute();
       foreach ($all_daemons as $daemon) {
-
+        $actual_user = $daemon->getRunningAsUser();
-      if ($phd_user) {
+        if ($actual_user == $expect_user) {
-        if ($daemon->getRunningAsUser() != $phd_user) {
+          continue;
-          $doc_href = PhabricatorEnv::getDocLink('Managing Daemons with phd');
+        }
 
         $summary = pht(
-            'At least one daemon is currently running as a different '.
+          'At least one daemon is currently running as the wrong user.');
-            'user than configured in the Phabricator %s setting',
-            'phd.user');
 
         $message = pht(
-            'A daemon is running as user %s while the Phabricator config '.
+          'A daemon is running as user %s, but daemons should be '.
-            'specifies %s to be %s.'.
+          'running as %s.'.
           "\n\n".
-            'Either adjust %s to match %s or start '.
+          'Either adjust the configuration setting %s or restart the '.
-            'the daemons as the correct user. '.
+          'daemons. Daemons should attempt to run as the proper user when '.
-            "\n\n".
+          'restarted.',
-            '%s Daemons will try to use %s to start as the configured user. '.
+          phutil_tag('tt', array(), $actual_user),
-            'Make sure that the user who starts %s has the correct '.
+          phutil_tag('tt', array(), $expect_user),
-            'sudo permissions to start %s daemons as %s',
+          phutil_tag('tt', array(), 'phd.user'));
-            'phd.user',
-            'phd.user',
-            'phd',
-            'sudo',
-            'phd',
-            'phd',
-            phutil_tag('tt', array(), $daemon->getRunningAsUser()),
-            phutil_tag('tt', array(), $phd_user),
-            phutil_tag('tt', array(), $daemon->getRunningAsUser()),
-            phutil_tag('tt', array(), $phd_user));
 
         $this->newIssue('daemons.run-as-different-user')
-            ->setName(pht('Daemons are running as the wrong user'))
+          ->setName(pht('Daemon Running as Wrong User'))
           ->setSummary($summary)
           ->setMessage($message)
+          ->addPhabricatorConfig('phd.user')
           ->addCommand('phabricator/ $ ./bin/phd restart');
-        }
+
+        break;
       }
     }
   }

src/applications/config/check/PhabricatorDatabaseSetupCheck.php

@@ -12,25 +12,14 @@ final class PhabricatorDatabaseSetupCheck extends PhabricatorSetupCheck {
   }
 
   protected function executeChecks() {
-    $conf = PhabricatorEnv::newObjectFromConfig('mysql.configuration-provider');
+    $master = PhabricatorDatabaseRef::getMasterDatabaseRef();
-    $conn_user = $conf->getUser();
+    if (!$master) {
-    $conn_pass = $conf->getPassword();
+      // If we're implicitly in read-only mode during disaster recovery,
-    $conn_host = $conf->getHost();
+      // don't bother with these setup checks.
-    $conn_port = $conf->getPort();
+      return;
+    }
 
-    ini_set('mysql.connect_timeout', 2);
+    $conn_raw = $master->newManagementConnection();
-
-    $config = array(
-      'user'      => $conn_user,
-      'pass'      => $conn_pass,
-      'host'      => $conn_host,
-      'port'      => $conn_port,
-      'database'  => null,
-    );
-
-    $conn_raw = PhabricatorEnv::newObjectFromConfig(
-      'mysql.implementation',
-      array($config));
 
     try {
       queryfx($conn_raw, 'SELECT 1');
@@ -88,11 +77,8 @@ final class PhabricatorDatabaseSetupCheck extends PhabricatorSetupCheck {
         ->setIsFatal(true)
         ->addCommand(hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
     } else {
-
+      $conn_meta = $master->newApplicationConnection(
-      $config['database'] = $namespace.'_meta_data';
+        $namespace.'_meta_data');
-      $conn_meta = PhabricatorEnv::newObjectFromConfig(
-        'mysql.implementation',
-        array($config));
 
       $applied = queryfx_all($conn_meta, 'SELECT patch FROM patch_status');
       $applied = ipull($applied, 'patch', 'patch');
@@ -113,7 +99,6 @@ final class PhabricatorDatabaseSetupCheck extends PhabricatorSetupCheck {
       }
     }
 
-
     $host = PhabricatorEnv::getEnvConfig('mysql.host');
     $matches = null;
     if (preg_match('/^([^:]+):(\d+)$/', $host, $matches)) {

src/applications/config/check/PhabricatorExtraConfigSetupCheck.php

@@ -182,6 +182,10 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
       'Garbage collectors are now configured with "%s".',
       'bin/garbage set-policy');
 
+    $aphlict_reason = pht(
+      'Configuration of the notification server has changed substantially. '.
+      'For discussion, see T10794.');
+
     $ancient_config += array(
       'phid.external-loaders' =>
         pht(
@@ -298,6 +302,14 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
       'phd.variant-config' => pht(
         'This configuration is no longer relevant because daemons '.
         'restart automatically on configuration changes.'),
+
+      'notification.ssl-cert' => $aphlict_reason,
+      'notification.ssl-key' => $aphlict_reason,
+      'notification.pidfile' => $aphlict_reason,
+      'notification.log' => $aphlict_reason,
+      'notification.enabled' => $aphlict_reason,
+      'notification.client-uri' => $aphlict_reason,
+      'notification.server-uri' => $aphlict_reason,
     );
 
     return $ancient_config;

src/applications/config/check/PhabricatorMySQLSetupCheck.php

@@ -20,6 +20,12 @@ final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
   }
 
   protected function executeChecks() {
+    // TODO: These checks should be executed against every reachable replica?
+    // See T10759.
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
 
     // This primarily supports setting the filesize limit for MySQL to 8MB,

src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php

@@ -0,0 +1,213 @@
+<?php
+
+final class PhabricatorConfigClusterDatabasesController
+  extends PhabricatorConfigController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $nav = $this->buildSideNavView();
+    $nav->selectFilter('cluster/databases/');
+
+    $title = pht('Database Servers');
+
+    $crumbs = $this
+      ->buildApplicationCrumbs($nav)
+      ->addTextCrumb(pht('Database Servers'));
+
+    $database_status = $this->buildClusterDatabaseStatus();
+
+    $view = id(new PHUITwoColumnView())
+      ->setNavigation($nav)
+      ->setMainColumn($database_status);
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs($crumbs)
+      ->appendChild($view);
+  }
+
+  private function buildClusterDatabaseStatus() {
+    $viewer = $this->getViewer();
+
+    $databases = PhabricatorDatabaseRef::queryAll();
+    $connection_map = PhabricatorDatabaseRef::getConnectionStatusMap();
+    $replica_map = PhabricatorDatabaseRef::getReplicaStatusMap();
+    Javelin::initBehavior('phabricator-tooltips');
+
+    $rows = array();
+    foreach ($databases as $database) {
+      $messages = array();
+
+      if ($database->getIsMaster()) {
+        $role_icon = id(new PHUIIconView())
+          ->setIcon('fa-database sky')
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => pht('Master'),
+            ));
+      } else {
+        $role_icon = id(new PHUIIconView())
+          ->setIcon('fa-download')
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => pht('Replica'),
+            ));
+      }
+
+      if ($database->getDisabled()) {
+        $conn_icon = 'fa-times';
+        $conn_color = 'grey';
+        $conn_label = pht('Disabled');
+      } else {
+        $status = $database->getConnectionStatus();
+
+        $info = idx($connection_map, $status, array());
+        $conn_icon = idx($info, 'icon');
+        $conn_color = idx($info, 'color');
+        $conn_label = idx($info, 'label');
+
+        if ($status === PhabricatorDatabaseRef::STATUS_OKAY) {
+          $latency = $database->getConnectionLatency();
+          $latency = (int)(1000000 * $latency);
+          $conn_label = pht('%s us', new PhutilNumber($latency));
+        }
+      }
+
+      $connection = array(
+        id(new PHUIIconView())->setIcon("{$conn_icon} {$conn_color}"),
+        ' ',
+        $conn_label,
+      );
+
+      if ($database->getDisabled()) {
+        $replica_icon = 'fa-times';
+        $replica_color = 'grey';
+        $replica_label = pht('Disabled');
+      } else {
+        $status = $database->getReplicaStatus();
+
+        $info = idx($replica_map, $status, array());
+        $replica_icon = idx($info, 'icon');
+        $replica_color = idx($info, 'color');
+        $replica_label = idx($info, 'label');
+
+        if ($database->getIsMaster()) {
+          if ($status === PhabricatorDatabaseRef::REPLICATION_OKAY) {
+            $replica_icon = 'fa-database';
+          }
+        } else {
+          switch ($status) {
+            case PhabricatorDatabaseRef::REPLICATION_OKAY:
+            case PhabricatorDatabaseRef::REPLICATION_SLOW:
+              $delay = $database->getReplicaDelay();
+              if ($delay) {
+                $replica_label = pht('%ss Behind', new PhutilNumber($delay));
+              } else {
+                $replica_label = pht('Up to Date');
+              }
+              break;
+          }
+        }
+      }
+
+      $replication = array(
+        id(new PHUIIconView())->setIcon("{$replica_icon} {$replica_color}"),
+        ' ',
+        $replica_label,
+      );
+
+      $health = $database->getHealthRecord();
+      $health_up = $health->getUpEventCount();
+      $health_down = $health->getDownEventCount();
+
+      if ($health->getIsHealthy()) {
+        $health_icon = id(new PHUIIconView())
+          ->setIcon('fa-plus green');
+      } else {
+        $health_icon = id(new PHUIIconView())
+          ->setIcon('fa-times red');
+        $messages[] = pht(
+          'UNHEALTHY: This database has failed recent health checks. Traffic '.
+          'will not be sent to it until it recovers.');
+      }
+
+      $health_count = pht(
+        '%s / %s',
+        new PhutilNumber($health_up),
+        new PhutilNumber($health_up + $health_down));
+
+      $health_status = array(
+        $health_icon,
+        ' ',
+        $health_count,
+      );
+
+      $conn_message = $database->getConnectionMessage();
+      if ($conn_message) {
+        $messages[] = $conn_message;
+      }
+
+      $replica_message = $database->getReplicaMessage();
+      if ($replica_message) {
+        $messages[] = $replica_message;
+      }
+
+      $messages = phutil_implode_html(phutil_tag('br'), $messages);
+
+      $rows[] = array(
+        $role_icon,
+        $database->getHost(),
+        $database->getPort(),
+        $database->getUser(),
+        $connection,
+        $replication,
+        $health_status,
+        $messages,
+      );
+    }
+
+
+    $table = id(new AphrontTableView($rows))
+      ->setNoDataString(
+        pht('Phabricator is not configured in cluster mode.'))
+      ->setHeaders(
+        array(
+          null,
+          pht('Host'),
+          pht('Port'),
+          pht('User'),
+          pht('Connection'),
+          pht('Replication'),
+          pht('Health'),
+          pht('Messages'),
+        ))
+      ->setColumnClasses(
+        array(
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          'wide',
+        ));
+
+    $doc_href = PhabricatorEnv::getDoclink('Cluster: Databases');
+
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Cluster Database Status'))
+      ->addActionLink(
+        id(new PHUIButtonView())
+          ->setIcon('fa-book')
+          ->setHref($doc_href)
+          ->setTag('a')
+          ->setText(pht('Documentation')));
+
+    return id(new PHUIObjectBoxView())
+      ->setHeader($header)
+      ->setTable($table);
+  }
+
+}

src/applications/config/controller/PhabricatorConfigClusterNotificationsController.php

@@ -0,0 +1,163 @@
+<?php
+
+final class PhabricatorConfigClusterNotificationsController
+  extends PhabricatorConfigController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $nav = $this->buildSideNavView();
+    $nav->selectFilter('cluster/notifications/');
+
+    $title = pht('Cluster Notifications');
+
+    $crumbs = $this
+      ->buildApplicationCrumbs($nav)
+      ->addTextCrumb(pht('Cluster Notifications'));
+
+    $notification_status = $this->buildClusterNotificationStatus();
+
+    $view = id(new PHUITwoColumnView())
+      ->setNavigation($nav)
+      ->setMainColumn($notification_status);
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs($crumbs)
+      ->appendChild($view);
+  }
+
+  private function buildClusterNotificationStatus() {
+    $viewer = $this->getViewer();
+
+    $servers = PhabricatorNotificationServerRef::newRefs();
+    Javelin::initBehavior('phabricator-tooltips');
+
+    $rows = array();
+    foreach ($servers as $server) {
+      if ($server->isAdminServer()) {
+        $type_icon = 'fa-database sky';
+        $type_tip = pht('Admin Server');
+      } else {
+        $type_icon = 'fa-bell sky';
+        $type_tip = pht('Client Server');
+      }
+
+      $type_icon = id(new PHUIIconView())
+        ->setIcon($type_icon)
+        ->addSigil('has-tooltip')
+        ->setMetadata(
+          array(
+            'tip' => $type_tip,
+          ));
+
+      $messages = array();
+
+      $details = array();
+      if ($server->isAdminServer()) {
+        try {
+          $details = $server->loadServerStatus();
+          $status_icon = 'fa-exchange green';
+          $status_label = pht('Version %s', idx($details, 'version'));
+        } catch (Exception $ex) {
+          $status_icon = 'fa-times red';
+          $status_label = pht('Connection Error');
+          $messages[] = $ex->getMessage();
+        }
+      } else {
+        try {
+          $server->testClient();
+          $status_icon = 'fa-exchange green';
+          $status_label = pht('Connected');
+        } catch (Exception $ex) {
+          $status_icon = 'fa-times red';
+          $status_label = pht('Connection Error');
+          $messages[] = $ex->getMessage();
+        }
+      }
+
+      if ($details) {
+        $uptime = idx($details, 'uptime');
+        $uptime = $uptime / 1000;
+        $uptime = phutil_format_relative_time_detailed($uptime);
+
+        $clients = pht(
+          '%s Active / %s Total',
+          new PhutilNumber(idx($details, 'clients.active')),
+          new PhutilNumber(idx($details, 'clients.total')));
+
+        $stats = pht(
+          '%s In / %s Out',
+          new PhutilNumber(idx($details, 'messages.in')),
+          new PhutilNumber(idx($details, 'messages.out')));
+
+      } else {
+        $uptime = null;
+        $clients = null;
+        $stats = null;
+      }
+
+      $status_view = array(
+        id(new PHUIIconView())->setIcon($status_icon),
+        ' ',
+        $status_label,
+      );
+
+      $messages = phutil_implode_html(phutil_tag('br'), $messages);
+
+      $rows[] = array(
+        $type_icon,
+        $server->getProtocol(),
+        $server->getHost(),
+        $server->getPort(),
+        $status_view,
+        $uptime,
+        $clients,
+        $stats,
+        $messages,
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setNoDataString(
+        pht('No notification servers are configured.'))
+      ->setHeaders(
+        array(
+          null,
+          pht('Proto'),
+          pht('Host'),
+          pht('Port'),
+          pht('Status'),
+          pht('Uptime'),
+          pht('Clients'),
+          pht('Messages'),
+          null,
+        ))
+      ->setColumnClasses(
+        array(
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          null,
+          'wide',
+        ));
+
+    $doc_href = PhabricatorEnv::getDoclink('Cluster: Notifications');
+
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Cluster Notification Status'))
+      ->addActionLink(
+        id(new PHUIButtonView())
+          ->setIcon('fa-book')
+          ->setHref($doc_href)
+          ->setTag('a')
+          ->setText(pht('Documentation')));
+
+    return id(new PHUIObjectBoxView())
+      ->setHeader($header)
+      ->setTable($table);
+  }
+
+}

src/applications/config/controller/PhabricatorConfigController.php

@@ -22,6 +22,9 @@ abstract class PhabricatorConfigController extends PhabricatorController {
     $nav->addFilter('dbissue/', pht('Database Issues'));
     $nav->addLabel(pht('Cache'));
     $nav->addFilter('cache/', pht('Cache Status'));
+    $nav->addLabel(pht('Cluster'));
+    $nav->addFilter('cluster/databases/', pht('Database Servers'));
+    $nav->addFilter('cluster/notifications/', pht('Notification Servers'));
     $nav->addLabel(pht('Welcome'));
     $nav->addFilter('welcome/', pht('Welcome Screen'));
     $nav->addLabel(pht('Modules'));

src/applications/config/option/PhabricatorClusterConfigOptions.php

@@ -20,25 +20,46 @@ final class PhabricatorClusterConfigOptions
   }
 
   public function getOptions() {
+    $databases_type = 'custom:PhabricatorClusterDatabasesConfigOptionType';
+    $databases_help = $this->deformat(pht(<<<EOTEXT
+WARNING: This is a prototype option and the description below is currently pure
+fantasy.
+
+This option allows you to make Phabricator aware of database read replicas so
+it can monitor database health, spread load, and degrade gracefully to
+read-only mode in the event of a failure on the primary host. For help with
+configuring cluster databases, see **[[ %s | %s ]]** in the documentation.
+EOTEXT
+      ,
+      PhabricatorEnv::getDoclink('Cluster: Databases'),
+      pht('Cluster: Databases')));
+
+
+    $intro_href = PhabricatorEnv::getDoclink('Clustering Introduction');
+    $intro_name = pht('Clustering Introduction');
+
     return array(
       $this->newOption('cluster.addresses', 'list<string>', array())
         ->setLocked(true)
         ->setSummary(pht('Address ranges of cluster hosts.'))
         ->setDescription(
           pht(
-            'To allow Phabricator nodes to communicate with other nodes '.
+            'Define a Phabricator cluster by providing a whitelist of host '.
-            'in the cluster, provide an address whitelist of hosts that '.
+            'addresses that are part of the cluster.'.
-            'are part of the cluster.'.
             "\n\n".
-            'Hosts on this whitelist are permitted to use special cluster '.
+            'Hosts on this whitelist have special powers. These hosts are '.
-            'mechanisms to authenticate requests. By default, these '.
+            'permitted to bend security rules, and misconfiguring this list '.
-            'mechanisms are disabled.'.
+            'can make your install less secure. For more information, '.
+            'see **[[ %s | %s ]]**.'.
             "\n\n".
             'Define a list of CIDR blocks which whitelist all hosts in the '.
-            'cluster. See the examples below for details.',
+            'cluster and no additional hosts. See the examples below for '.
+            'details.'.
             "\n\n".
             'When cluster addresses are defined, Phabricator hosts will also '.
-            'reject requests to interfaces which are not whitelisted.'))
+            'reject requests to interfaces which are not whitelisted.',
+            $intro_href,
+            $intro_name))
         ->addExample(
           array(
             '23.24.25.80/32',
@@ -73,6 +94,26 @@ final class PhabricatorClusterConfigOptions
             'subprocesses and commit hooks in the `%s` environmental variable.',
             'PhabricatorConfigSiteSource',
             'PHABRICATOR_INSTANCE')),
+      $this->newOption('cluster.read-only', 'bool', false)
+        ->setLocked(true)
+        ->setSummary(
+          pht(
+            'Activate read-only mode for maintenance or disaster recovery.'))
+        ->setDescription(
+          pht(
+            'WARNING: This is a prototype option and the description below '.
+            'is currently pure fantasy.'.
+            "\n\n".
+            'Switch Phabricator to read-only mode. In this mode, users will '.
+            'be unable to write new data. Normally, the cluster degrades '.
+            'into this mode automatically when it detects that the database '.
+            'master is unreachable, but you can activate it manually in '.
+            'order to perform maintenance or test configuration.')),
+      $this->newOption('cluster.databases', $databases_type, array())
+        ->setHidden(true)
+        ->setSummary(
+          pht('Configure database read replicas.'))
+        ->setDescription($databases_help),
     );
   }
 

src/applications/config/option/PhabricatorNotificationConfigOptions.php

@@ -20,45 +20,43 @@ final class PhabricatorNotificationConfigOptions
   }
 
   public function getOptions() {
-    return array(
+    $servers_type = 'custom:PhabricatorNotificationServersConfigOptionType';
-      $this->newOption('notification.enabled', 'bool', false)
+    $servers_help = $this->deformat(pht(<<<EOTEXT
-        ->setBoolOptions(
+Provide a list of notification servers to enable real-time notifications.
+
+For help setting up notification servers, see **[[ %s | %s ]]** in the
+documentation.
+EOTEXT
+      ,
+      PhabricatorEnv::getDoclink(
+        'Notifications User Guide: Setup and Configuration'),
+      pht('Notifications User Guide: Setup and Configuration')));
+
+    $servers_example1 = array(
       array(
-            pht('Enable Real-Time Notifications'),
+        'type' => 'client',
-            pht('Disable Real-Time Notifications'),
+        'host' => 'phabricator.mycompany.com',
-          ))
+        'port' => 22280,
-        ->setSummary(pht('Enable real-time notifications.'))
+        'protocol' => 'https',
-        ->setDescription(
+      ),
-          pht(
+      array(
-            "Enable real-time notifications. You must also run a Node.js ".
+        'type' => 'admin',
-            "based notification server for this to work. Consult the ".
+        'host' => '127.0.0.1',
-            "documentation in 'Notifications User Guide: Setup and ".
+        'port' => 22281,
-            "Configuration' for instructions.")),
+        'protocol' => 'http',
-      $this->newOption(
+      ),
-        'notification.client-uri',
+    );
-        'string',
+
-        'http://localhost:22280/')
+    $servers_example1 = id(new PhutilJSON())->encodeAsList(
-        ->setDescription(pht('Location of the client server.')),
+      $servers_example1);
-      $this->newOption(
+
-        'notification.server-uri',
+    return array(
-        'string',
+      $this->newOption('notification.servers', $servers_type, array())
-        'http://localhost:22281/')
+        ->setSummary(pht('Configure real-time notifications.'))
-        ->setDescription(pht('Location of the notification receiver server.')),
+        ->setDescription($servers_help)
-      $this->newOption('notification.log', 'string', '/var/log/aphlict.log')
+        ->addExample(
-        ->setDescription(pht('Location of the server log file.')),
+          $servers_example1,
-      $this->newOption('notification.ssl-key', 'string', null)
+          pht('Simple Example')),
-        ->setLocked(true)
-        ->setDescription(
-          pht('Path to SSL key to use for secure WebSockets.')),
-      $this->newOption('notification.ssl-cert', 'string', null)
-        ->setLocked(true)
-        ->setDescription(
-          pht('Path to SSL certificate to use for secure WebSockets.')),
-      $this->newOption(
-        'notification.pidfile',
-        'string',
-        '/var/tmp/aphlict/pid/aphlict.pid')
-        ->setDescription(pht('Location of the server PID file.')),
     );
   }
 

src/applications/config/option/PhabricatorSecurityConfigOptions.php

@@ -80,22 +80,25 @@ final class PhabricatorSecurityConfigOptions
           pht(
             "If the web server responds to both HTTP and HTTPS requests but ".
             "you want users to connect with only HTTPS, you can set this ".
-            "to true to make Phabricator redirect HTTP requests to HTTPS.\n\n".
+            "to `true` to make Phabricator redirect HTTP requests to HTTPS.".
-
+            "\n\n".
             "Normally, you should just configure your server not to accept ".
             "HTTP traffic, but this setting may be useful if you originally ".
             "used HTTP and have now switched to HTTPS but don't want to ".
             "break old links, or if your webserver sits behind a load ".
             "balancer which terminates HTTPS connections and you can not ".
-            "reasonably configure more granular behavior there.\n\n".
+            "reasonably configure more granular behavior there.".
-
+            "\n\n".
             "IMPORTANT: Phabricator determines if a request is HTTPS or not ".
             "by examining the PHP `%s` variable. If you run ".
             "Apache/mod_php this will probably be set correctly for you ".
             "automatically, but if you run Phabricator as CGI/FCGI (e.g., ".
             "through nginx or lighttpd), you need to configure your web ".
             "server so that it passes the value correctly based on the ".
-            "connection type.",
+            "connection type.".
+            "\n\n".
+            "If you configure Phabricator in cluster mode, note that this ".
+            "setting is ignored by intracluster requests.",
             "\$_SERVER['HTTPS']"))
         ->setBoolOptions(
           array(

src/applications/conpherence/controller/ConpherenceUpdateController.php

@@ -111,7 +111,7 @@ final class ConpherenceUpdateController
             break;
           }
           $person_phid = $request->getStr('remove_person');
-          if ($person_phid && $person_phid == $user->getPHID()) {
+          if ($person_phid) {
             $xactions[] = id(new ConpherenceTransaction())
               ->setTransactionType(
                 ConpherenceTransaction::TYPE_PARTICIPANTS)
@@ -321,38 +321,83 @@ final class ConpherenceUpdateController
     ConpherenceThread $conpherence) {
 
     $request = $this->getRequest();
-    $user = $request->getUser();
+    $viewer = $request->getUser();
     $remove_person = $request->getStr('remove_person');
     $participants = $conpherence->getParticipants();
 
-    $message = pht('Are you sure you want to leave this room?');
+    $removed_user = id(new PhabricatorPeopleQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($remove_person))
+      ->executeOne();
+    if (!$removed_user) {
+      return new Aphront404Response();
+    }
+
+    $is_self = ($viewer->getPHID() == $removed_user->getPHID());
+    $is_last = (count($participants) == 1);
+
     $test_conpherence = clone $conpherence;
     $test_conpherence->attachParticipants(array());
-    if (!PhabricatorPolicyFilter::hasCapability(
+    $still_visible = PhabricatorPolicyFilter::hasCapability(
-      $user,
+      $removed_user,
       $test_conpherence,
-      PhabricatorPolicyCapability::CAN_VIEW)) {
+      PhabricatorPolicyCapability::CAN_VIEW);
-      if (count($participants) == 1) {
+
-        $message .= ' '.pht('The room will be inaccessible forever and ever.');
+    $body = array();
+
+    if ($is_self) {
+      $title = pht('Leave Room');
+      $body[] = pht(
+        'Are you sure you want to leave this room?');
     } else {
-        $message .= ' '.pht('Someone else in the room can add you back later.');
+      $title = pht('Banish User');
+      $body[] = pht(
+        'Banish %s from the realm?',
+        phutil_tag('strong', array(), $removed_user->getUsername()));
+    }
+
+    if ($still_visible) {
+      if ($is_self) {
+        $body[] = pht(
+          'You will be able to rejoin the room later.');
+      } else {
+        $body[] = pht(
+          'This user will be able to rejoin the room later.');
+      }
+    } else {
+      if ($is_self) {
+        if ($is_last) {
+          $body[] = pht(
+            'You are the last member, so you will never be able to rejoin '.
+            'the room.');
+        } else {
+          $body[] = pht(
+            'You will not be able to rejoin the room on your own, but '.
+            'someone else can invite you later.');
+        }
+      } else {
+        $body[] = pht(
+          'This user will not be able to rejoin the room unless invited '.
+          'again.');
       }
     }
-    $body = phutil_tag(
-      'p',
-      array(),
-      $message);
 
     require_celerity_resource('conpherence-update-css');
-    return id(new AphrontDialogView())
+
-      ->setTitle(pht('Leave Room'))
+    $dialog = id(new AphrontDialogView())
+      ->setTitle($title)
       ->addHiddenInput('action', 'remove_person')
       ->addHiddenInput('remove_person', $remove_person)
       ->addHiddenInput(
         'latest_transaction_id',
         $request->getInt('latest_transaction_id'))
-      ->addHiddenInput('__continue__', true)
+      ->addHiddenInput('__continue__', true);
-      ->appendChild($body);
+
+    foreach ($body as $paragraph) {
+      $dialog->appendParagraph($paragraph);
+    }
+
+    return $dialog;
   }
 
   private function renderMetadataDialogue(

src/applications/conpherence/view/ConpherencePeopleWidgetView.php

@@ -5,11 +5,11 @@ final class ConpherencePeopleWidgetView extends ConpherenceWidgetView {
   public function render() {
     $conpherence = $this->getConpherence();
     $widget_data = $conpherence->getWidgetData();
-    $user = $this->getUser();
+    $viewer = $this->getUser();
-    $conpherence = $this->getConpherence();
+
     $participants = $conpherence->getParticipants();
     $handles = $conpherence->getHandles();
-    $head_handles = array_select_keys($handles, array($user->getPHID()));
+    $head_handles = array_select_keys($handles, array($viewer->getPHID()));
     $handle_list = mpull($handles, 'getName');
     natcasesort($handle_list);
     $handles = mpull($handles, null, 'getName');
@@ -17,11 +17,16 @@ final class ConpherencePeopleWidgetView extends ConpherenceWidgetView {
     $head_handles = mpull($head_handles, null, 'getName');
     $handles = $head_handles + $handles;
 
+    $can_edit = PhabricatorPolicyFilter::hasCapability(
+      $viewer,
+      $conpherence,
+      PhabricatorPolicyCapability::CAN_EDIT);
+
     $body = array();
     foreach ($handles as $handle) {
       $user_phid = $handle->getPHID();
-      $remove_html = '';
+
-      if ($user_phid == $user->getPHID()) {
+      if (($user_phid == $viewer->getPHID()) || $can_edit) {
         $icon = id(new PHUIIconView())
           ->setIcon('fa-times lightbluetext');
         $remove_html = javelin_tag(
@@ -35,7 +40,10 @@ final class ConpherencePeopleWidgetView extends ConpherenceWidgetView {
             ),
           ),
           $icon);
+      } else {
+        $remove_html = null;
       }
+
       $body[] = phutil_tag(
         'div',
         array(

src/applications/daemon/controller/PhabricatorDaemonBulkJobController.php

@@ -0,0 +1,25 @@
+<?php
+
+abstract class PhabricatorDaemonBulkJobController
+  extends PhabricatorDaemonController {
+
+  public function shouldRequireAdmin() {
+    return false;
+  }
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function buildApplicationMenu() {
+    return $this->newApplicationMenu()
+      ->setSearchEngine(new PhabricatorWorkerBulkJobSearchEngine());
+  }
+
+  protected function buildApplicationCrumbs() {
+    $crumbs = parent::buildApplicationCrumbs();
+    $crumbs->addTextCrumb(pht('Bulk Jobs'), '/daemon/bulk/');
+    return $crumbs;
+  }
+
+}

src/applications/daemon/controller/PhabricatorDaemonBulkJobListController.php

@@ -1,31 +1,12 @@
 <?php
 
 final class PhabricatorDaemonBulkJobListController
-  extends PhabricatorDaemonController {
+  extends PhabricatorDaemonBulkJobController {
-
-  public function shouldAllowPublic() {
-    return true;
-  }
 
   public function handleRequest(AphrontRequest $request) {
-    $controller = id(new PhabricatorApplicationSearchController())
+    return id(new PhabricatorWorkerBulkJobSearchEngine())
-      ->setQueryKey($request->getURIData('queryKey'))
+      ->setController($this)
-      ->setSearchEngine(new PhabricatorWorkerBulkJobSearchEngine())
+      ->buildResponse();
-      ->setNavigation($this->buildSideNavView());
-    return $this->delegateToController($controller);
   }
 
-  protected function buildSideNavView($for_app = false) {
-    $user = $this->getRequest()->getUser();
-
-    $nav = new AphrontSideNavFilterView();
-    $nav->setBaseURI(new PhutilURI($this->getApplicationURI()));
-
-    id(new PhabricatorWorkerBulkJobSearchEngine())
-      ->setViewer($user)
-      ->addNavigationItems($nav->getMenu());
-    $nav->selectFilter(null);
-
-    return $nav;
-  }
 }

src/applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php

@@ -1,11 +1,7 @@
 <?php
 
 final class PhabricatorDaemonBulkJobMonitorController
-  extends PhabricatorDaemonController {
+  extends PhabricatorDaemonBulkJobController {
-
-  public function shouldAllowPublic() {
-    return true;
-  }
 
   public function handleRequest(AphrontRequest $request) {
     $viewer = $this->getViewer();

src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php

@@ -1,11 +1,7 @@
 <?php
 
 final class PhabricatorDaemonBulkJobViewController
-  extends PhabricatorDaemonController {
+  extends PhabricatorDaemonBulkJobController {
-
-  public function shouldAllowPublic() {
-    return true;
-  }
 
   public function handleRequest(AphrontRequest $request) {
     $viewer = $this->getViewer();
@@ -21,7 +17,6 @@ final class PhabricatorDaemonBulkJobViewController
     $title = pht('Bulk Job %d', $job->getID());
 
     $crumbs = $this->buildApplicationCrumbs();
-    $crumbs->addTextCrumb(pht('Bulk Jobs'), '/daemon/bulk/');
     $crumbs->addTextCrumb($title);
     $crumbs->setBorder(true);
 

src/applications/daemon/controller/PhabricatorDaemonConsoleController.php

@@ -121,14 +121,13 @@ final class PhabricatorDaemonConsoleController
       ->setHeaderText(pht('Recently Completed Tasks (Last 15m)'))
       ->setTable($completed_table);
 
-    $daemon_table = new PhabricatorDaemonLogListView();
+    $daemon_table = id(new PhabricatorDaemonLogListView())
-    $daemon_table->setUser($viewer);
+      ->setUser($viewer)
-    $daemon_table->setDaemonLogs($logs);
+      ->setDaemonLogs($logs);
-
-    $daemon_panel = id(new PHUIObjectBoxView());
-    $daemon_panel->setHeaderText(pht('Active Daemons'));
-    $daemon_panel->setObjectList($daemon_table);
 
+    $daemon_panel = id(new PHUIObjectBoxView())
+      ->setHeaderText(pht('Active Daemons'))
+      ->setTable($daemon_table);
 
     $tasks = id(new PhabricatorWorkerLeaseQuery())
       ->setSkipLease(true)

src/applications/daemon/controller/PhabricatorDaemonController.php

@@ -1,6 +1,11 @@
 <?php
 
-abstract class PhabricatorDaemonController extends PhabricatorController {
+abstract class PhabricatorDaemonController
+  extends PhabricatorController {
+
+  public function shouldRequireAdmin() {
+    return true;
+  }
 
   protected function buildSideNavView() {
     $nav = new AphrontSideNavFilterView();

src/applications/daemon/controller/PhabricatorDaemonLogListController.php

@@ -4,7 +4,7 @@ final class PhabricatorDaemonLogListController
   extends PhabricatorDaemonController {
 
   public function handleRequest(AphrontRequest $request) {
-    $viewer = $request->getViewer();
+    $viewer = $this->getViewer();
 
     $pager = new AphrontCursorPagerView();
     $pager->readFromRequest($request);
@@ -14,13 +14,13 @@ final class PhabricatorDaemonLogListController
       ->setAllowStatusWrites(true)
       ->executeWithCursorPager($pager);
 
-    $daemon_table = new PhabricatorDaemonLogListView();
+    $daemon_table = id(new PhabricatorDaemonLogListView())
-    $daemon_table->setUser($request->getUser());
+      ->setViewer($viewer)
-    $daemon_table->setDaemonLogs($logs);
+      ->setDaemonLogs($logs);
 
     $box = id(new PHUIObjectBoxView())
       ->setHeaderText(pht('All Daemons'))
-      ->appendChild($daemon_table);
+      ->setTable($daemon_table);
 
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('All Daemons'));

src/applications/daemon/controller/PhabricatorDaemonLogViewController.php

@@ -16,10 +16,6 @@ final class PhabricatorDaemonLogViewController
       return new Aphront404Response();
     }
 
-    $events = id(new PhabricatorDaemonLogEvent())->loadAllWhere(
-      'logID = %d ORDER BY id DESC LIMIT 1000',
-      $log->getID());
-
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('Daemon %s', $log->getID()));
     $crumbs->setBorder(true);
@@ -69,23 +65,15 @@ final class PhabricatorDaemonLogViewController
 
     $properties = $this->buildPropertyListView($log);
 
-    $event_view = id(new PhabricatorDaemonLogEventsView())
-      ->setUser($viewer)
-      ->setEvents($events);
-
-    $event_panel = id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('Events'))
-      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
-      ->appendChild($event_view);
-
     $object_box = id(new PHUIObjectBoxView())
+      ->setHeaderText(pht('Daemon Details'))
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->addPropertyList($properties);
 
     $view = id(new PHUITwoColumnView())
       ->setHeader($header)
       ->setFooter(array(
         $object_box,
-        $event_panel,
       ));
 
     return $this->newPage()

src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php

@@ -224,17 +224,18 @@ abstract class PhabricatorDaemonManagementWorkflow
           $daemon_script_dir,
           $config,
           $this->runDaemonsAsUser);
-      } catch (Exception $e) {
+      } catch (Exception $ex) {
-        // Retry without sudo
+        throw new PhutilArgumentUsageException(
-        $console->writeOut(
-          "%s\n",
           pht(
-            '%s command failed. Starting daemon as current user.',
+            'Daemons are configured to run as user "%s" in configuration '.
-            'sudo'));
+            'option `%s`, but the current user is "%s" and `phd` was unable '.
-        $this->executeDaemonLaunchCommand(
+            'to switch to the correct user with `sudo`. Command output:'.
-          $command,
+            "\n\n".
-          $daemon_script_dir,
+            '%s',
-          $config);
+            $phd_user,
+            'phd.user',
+            $current_user,
+            $ex->getMessage()));
       }
     }
   }

src/applications/daemon/view/PhabricatorDaemonLogListView.php

@@ -14,65 +14,107 @@ final class PhabricatorDaemonLogListView extends AphrontView {
     $viewer = $this->getViewer();
 
     $rows = array();
+    $daemons = $this->daemonLogs;
 
-    $list = new PHUIObjectItemListView();
+    foreach ($daemons as $daemon) {
-    $list->setFlush(true);
+      $id = $daemon->getID();
-    foreach ($this->daemonLogs as $log) {
+      $host = $daemon->getHost();
-      $id = $log->getID();
+      $pid = $daemon->getPID();
-      $epoch = $log->getDateCreated();
+      $name = phutil_tag(
+        'a',
+        array(
+          'href' => "/daemon/log/{$id}/",
+        ),
+        $daemon->getDaemon());
 
-      $item = id(new PHUIObjectItemView())
+      $status = $daemon->getStatus();
-        ->setObjectName(pht('Daemon %s', $id))
-        ->setHeader($log->getDaemon())
-        ->setHref("/daemon/log/{$id}/")
-        ->addIcon('none', phabricator_datetime($epoch, $viewer));
-
-      $status = $log->getStatus();
       switch ($status) {
         case PhabricatorDaemonLog::STATUS_RUNNING:
-          $item->setStatusIcon('fa-rocket green');
+          $status_icon = 'fa-rocket green';
-          $item->addAttribute(pht('This daemon is running.'));
+          $status_label = pht('Running');
+          $status_tip = pht('This daemon is running.');
           break;
         case PhabricatorDaemonLog::STATUS_DEAD:
-          $item->setStatusIcon('fa-warning red');
+          $status_icon = 'fa-warning red';
-          $item->addAttribute(
+          $status_label = pht('Dead');
-            pht(
+          $status_tip = pht(
-              'This daemon is lost or exited uncleanly, and is presumed '.
+            'This daemon has been lost or exited uncleanly, and is '.
-              'dead.'));
+            'presumed dead.');
-          $item->addIcon('fa-times grey', pht('Dead'));
           break;
         case PhabricatorDaemonLog::STATUS_EXITING:
-          $item->addAttribute(pht('This daemon is exiting.'));
+          $status_icon = 'fa-check';
-          $item->addIcon('fa-check', pht('Exiting'));
+          $status_label = pht('Shutting Down');
+          $status_tip = pht('This daemon is shutting down.');
           break;
         case PhabricatorDaemonLog::STATUS_EXITED:
-          $item->setDisabled(true);
+          $status_icon = 'fa-check grey';
-          $item->addAttribute(pht('This daemon exited cleanly.'));
+          $status_label = pht('Exited');
-          $item->addIcon('fa-check grey', pht('Exited'));
+          $status_tip = pht('This daemon exited cleanly.');
           break;
         case PhabricatorDaemonLog::STATUS_WAIT:
-          $item->setStatusIcon('fa-clock-o blue');
+          $status_icon = 'fa-clock-o blue';
-          $item->addAttribute(
+          $status_label = pht('Waiting');
-            pht(
+          $status_tip = pht(
             'This daemon encountered an error recently and is waiting a '.
-              'moment to restart.'));
+            'moment to restart.');
-          $item->addIcon('fa-clock-o grey', pht('Waiting'));
           break;
         case PhabricatorDaemonLog::STATUS_UNKNOWN:
         default:
-          $item->setStatusIcon('fa-warning orange');
+          $status_icon = 'fa-warning orange';
-          $item->addAttribute(
+          $status_label = pht('Unknown');
-            pht(
+          $status_tip = pht(
             'This daemon has not reported its status recently. It may '.
-              'have exited uncleanly.'));
+            'have exited uncleanly.');
-          $item->addIcon('fa-warning', pht('Unknown'));
           break;
       }
 
-      $list->addItem($item);
+      $status = phutil_tag(
+        'span',
+        array(
+          'sigil' => 'has-tooltip',
+          'meta' => array(
+            'tip' => $status_tip,
+          ),
+        ),
+        array(
+          id(new PHUIIconView())->setIcon($status_icon),
+          ' ',
+          $status_label,
+        ));
+
+      $launched = phabricator_datetime($daemon->getDateCreated(), $viewer);
+
+      $rows[] = array(
+        $id,
+        $host,
+        $pid,
+        $name,
+        $status,
+        $launched,
+      );
     }
 
-    return $list;
+    $table = id(new AphrontTableView($rows))
+      ->setHeaders(
+        array(
+          pht('ID'),
+          pht('Host'),
+          pht('PPID'),
+          pht('Daemon'),
+          pht('Status'),
+          pht('Launched'),
+        ))
+      ->setColumnClasses(
+        array(
+          null,
+          null,
+          null,
+          'pri',
+          'wide',
+          'right date',
+        ));
+
+    return $table;
   }
 
 }

src/applications/differential/storage/DifferentialDiff.php

@@ -249,7 +249,6 @@ final class DifferentialDiff
       'unitStatus' => $this->getUnitStatus(),
       'lintStatus' => $this->getLintStatus(),
       'changes' => array(),
-      'properties' => array(),
     );
 
     $dict['changes'] = $this->buildChangesList();
@@ -258,7 +257,7 @@ final class DifferentialDiff
   }
 
   public function getDiffAuthorshipDict() {
-    $dict = array();
+    $dict = array('properties' => array());
 
     $properties = id(new DifferentialDiffProperty())->loadAllWhere(
       'diffID = %d',

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

@@ -87,6 +87,8 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
             => 'DiffusionCommitTagsController',
           'commit/(?P<commit>[a-z0-9]+)/edit/'
             => 'DiffusionCommitEditController',
+          'manage/(?:(?P<panel>[^/]+)/)?'
+            => 'DiffusionRepositoryManageController',
           'edit/' => array(
             '' => 'DiffusionRepositoryEditMainController',
             'basic/' => 'DiffusionRepositoryEditBasicController',

src/applications/diffusion/conduit/DiffusionInternalGitRawDiffQueryConduitAPIMethod.php

@@ -0,0 +1,74 @@
+<?php
+
+final class DiffusionInternalGitRawDiffQueryConduitAPIMethod
+  extends DiffusionQueryConduitAPIMethod {
+
+  public function isInternalAPI() {
+    return true;
+  }
+
+  public function getAPIMethodName() {
+    return 'diffusion.internal.gitrawdiffquery';
+  }
+
+  public function getMethodDescription() {
+    return pht('Internal method for getting raw diff information.');
+  }
+
+  protected function defineReturnType() {
+    return 'string';
+  }
+
+  protected function defineCustomParamTypes() {
+    return array(
+      'commit' => 'required string',
+    );
+  }
+
+  protected function getResult(ConduitAPIRequest $request) {
+    $drequest = $this->getDiffusionRequest();
+    $repository = $drequest->getRepository();
+
+    if (!$repository->isGit()) {
+      throw new Exception(
+        pht(
+          'This API method can only be called on Git repositories.'));
+    }
+
+    // Check if the commit has parents. We're testing to see whether it is the
+    // first commit in history (in which case we must use "git log") or some
+    // other commit (in which case we can use "git diff"). We'd rather use
+    // "git diff" because it has the right behavior for merge commits, but
+    // it requires the commit to have a parent that we can diff against. The
+    // first commit doesn't, so "commit^" is not a valid ref.
+    list($parents) = $repository->execxLocalCommand(
+      'log -n1 --format=%s %s',
+      '%P',
+      $request->getValue('commit'));
+
+    $use_log = !strlen(trim($parents));
+    if ($use_log) {
+      // This is the first commit so we need to use "log". We know it's not a
+      // merge commit because it couldn't be merging anything, so this is safe.
+
+      // NOTE: "--pretty=format: " is to disable diff output, we only want the
+      // part we get from "--raw".
+      list($raw) = $repository->execxLocalCommand(
+        'log -n1 -M -C -B --find-copies-harder --raw -t '.
+          '--pretty=format: --abbrev=40 %s',
+        $request->getValue('commit'));
+    } else {
+      // Otherwise, we can use "diff", which will give us output for merges.
+      // We diff against the first parent, as this is generally the expectation
+      // and results in sensible behavior.
+      list($raw) = $repository->execxLocalCommand(
+        'diff -n1 -M -C -B --find-copies-harder --raw -t '.
+          '--abbrev=40 %s^1 %s',
+        $request->getValue('commit'),
+        $request->getValue('commit'));
+    }
+
+    return $raw;
+  }
+
+}

src/applications/diffusion/controller/DiffusionBrowseController.php

@@ -1187,6 +1187,19 @@ final class DiffusionBrowseController extends DiffusionController {
     $commit_links = $this->renderCommitLinks($blame_commits, $handles);
     $revision_links = $this->renderRevisionLinks($revisions, $handles);
 
+    if ($this->coverage) {
+      require_celerity_resource('differential-changeset-view-css');
+      Javelin::initBehavior(
+        'diffusion-browse-file',
+        array(
+          'labels' => array(
+            'cov-C' => pht('Covered'),
+            'cov-N' => pht('Not Covered'),
+            'cov-U' => pht('Not Executable'),
+          ),
+        ));
+    }
+
     $skip_text = pht('Skip Past This Commit');
     foreach ($display as $line_index => $line) {
       $row = array();
@@ -1304,7 +1317,6 @@ final class DiffusionBrowseController extends DiffusionController {
         ));
 
       if ($this->coverage) {
-        require_celerity_resource('differential-changeset-view-css');
         $cov_index = $line_index;
 
         if (isset($this->coverage[$cov_index])) {

src/applications/diffusion/controller/DiffusionRepositoryManageController.php

@@ -0,0 +1,99 @@
+<?php
+
+final class DiffusionRepositoryManageController
+  extends DiffusionController {
+
+  private $navigation;
+
+  public function buildApplicationMenu() {
+    // TODO: This is messy for now; the mobile menu should be set automatically
+    // when the body content is a two-column view with navigation.
+    if ($this->navigation) {
+      return $this->navigation->getMenu();
+    }
+    return parent::buildApplicationMenu();
+  }
+
+
+  public function handleRequest(AphrontRequest $request) {
+    $response = $this->loadDiffusionContext();
+    if ($response) {
+      return $response;
+    }
+
+    $viewer = $this->getViewer();
+    $drequest = $this->getDiffusionRequest();
+    $repository = $drequest->getRepository();
+
+    $panels = DiffusionRepositoryManagementPanel::getAllPanels();
+
+    foreach ($panels as $panel) {
+      $panel
+        ->setViewer($viewer)
+        ->setRepository($repository);
+    }
+
+    $selected = $request->getURIData('panel');
+    if (!strlen($selected)) {
+      $selected = head_key($panels);
+    }
+
+    if (empty($panels[$selected])) {
+      return new Aphront404Response();
+    }
+
+    $nav = $this->renderSideNav($repository, $panels, $selected);
+    $this->navigation = $nav;
+
+    $panel = $panels[$selected];
+
+    $content = $panel->buildManagementPanelContent();
+
+    $title = array(
+      $panel->getManagementPanelLabel(),
+      $repository->getDisplayName(),
+    );
+
+    $crumbs = $this->buildApplicationCrumbs();
+    $crumbs->addTextCrumb(
+      $repository->getDisplayName(),
+      $repository->getURI());
+    $crumbs->addTextCrumb(
+      pht('Manage'),
+      $repository->getPathURI('manage/'));
+    $crumbs->addTextCrumb($panel->getManagementPanelLabel());
+
+    $view = id(new PHUITwoColumnView())
+      ->setNavigation($nav)
+      ->setMainColumn($content);
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs($crumbs)
+      ->appendChild($view);
+  }
+
+  private function renderSideNav(
+    PhabricatorRepository $repository,
+    array $panels,
+    $selected) {
+
+    $base_uri = $repository->getPathURI('manage/');
+    $base_uri = new PhutilURI($base_uri);
+
+    $nav = id(new AphrontSideNavFilterView())
+      ->setBaseURI($base_uri);
+
+    foreach ($panels as $panel) {
+      $nav->addFilter(
+        $panel->getManagementPanelKey(),
+        $panel->getManagementPanelLabel());
+    }
+
+    $nav->selectFilter($selected);
+
+    return $nav;
+  }
+
+
+}

src/applications/diffusion/controller/DiffusionServeController.php

@@ -164,7 +164,14 @@ final class DiffusionServeController extends DiffusionController {
 
     // If authentication credentials have been provided, try to find a user
     // that actually matches those credentials.
-    if (isset($_SERVER['PHP_AUTH_USER']) && isset($_SERVER['PHP_AUTH_PW'])) {
+
+    // We require both the username and password to be nonempty, because Git
+    // won't prompt users who provide a username but no password otherwise.
+    // See T10797 for discussion.
+
+    $have_user = strlen(idx($_SERVER, 'PHP_AUTH_USER'));
+    $have_pass = strlen(idx($_SERVER, 'PHP_AUTH_PW'));
+    if ($have_user && $have_pass) {
       $username = $_SERVER['PHP_AUTH_USER'];
       $password = new PhutilOpaqueEnvelope($_SERVER['PHP_AUTH_PW']);
 

src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php

@@ -0,0 +1,166 @@
+<?php
+
+final class DiffusionRepositoryClusterManagementPanel
+  extends DiffusionRepositoryManagementPanel {
+
+  const PANELKEY = 'cluster';
+
+  public function getManagementPanelLabel() {
+    return pht('Cluster Configuration');
+  }
+
+  public function getManagementPanelOrder() {
+    return 12345;
+  }
+
+  public function buildManagementPanelContent() {
+    $repository = $this->getRepository();
+    $viewer = $this->getViewer();
+
+    $service_phid = $repository->getAlmanacServicePHID();
+    if ($service_phid) {
+      $service = id(new AlmanacServiceQuery())
+        ->setViewer($viewer)
+        ->withServiceTypes(
+          array(
+            AlmanacClusterRepositoryServiceType::SERVICETYPE,
+          ))
+        ->withPHIDs(array($service_phid))
+        ->needBindings(true)
+        ->executeOne();
+      if (!$service) {
+        // TODO: Viewer may not have permission to see the service, or it may
+        // be invalid? Raise some more useful error here?
+        throw new Exception(pht('Unable to load cluster service.'));
+      }
+    } else {
+      $service = null;
+    }
+
+    Javelin::initBehavior('phabricator-tooltips');
+
+    $rows = array();
+    if ($service) {
+      $bindings = $service->getBindings();
+      $bindings = mgroup($bindings, 'getDevicePHID');
+
+      // This is an unusual read which always comes from the master.
+      if (PhabricatorEnv::isReadOnly()) {
+        $versions = array();
+      } else {
+        $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
+          $repository->getPHID());
+      }
+
+      $versions = mpull($versions, null, 'getDevicePHID');
+
+      foreach ($bindings as $binding_group) {
+        $all_disabled = true;
+        foreach ($binding_group as $binding) {
+          if (!$binding->getIsDisabled()) {
+            $all_disabled = false;
+            break;
+          }
+        }
+
+        $any_binding = head($binding_group);
+
+        if ($all_disabled) {
+          $binding_icon = 'fa-times grey';
+          $binding_tip = pht('Disabled');
+        } else {
+          $binding_icon = 'fa-folder-open green';
+          $binding_tip = pht('Active');
+        }
+
+        $binding_icon = id(new PHUIIconView())
+          ->setIcon($binding_icon)
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => $binding_tip,
+            ));
+
+        $device = $any_binding->getDevice();
+
+        $version = idx($versions, $device->getPHID());
+        if ($version) {
+          $version_number = $version->getRepositoryVersion();
+          $version_number = phutil_tag(
+            'a',
+            array(
+              'href' => "/diffusion/pushlog/view/{$version_number}/",
+            ),
+            $version_number);
+        } else {
+          $version_number = '-';
+        }
+
+        if ($version && $version->getIsWriting()) {
+          $is_writing = id(new PHUIIconView())
+            ->setIcon('fa-pencil green');
+        } else {
+          $is_writing = id(new PHUIIconView())
+            ->setIcon('fa-pencil grey');
+        }
+
+        $rows[] = array(
+          $binding_icon,
+          phutil_tag(
+            'a',
+            array(
+              'href' => $device->getURI(),
+            ),
+            $device->getName()),
+          $version_number,
+          $is_writing,
+        );
+      }
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setNoDataString(pht('This is not a cluster repository.'))
+      ->setHeaders(
+        array(
+          null,
+          pht('Device'),
+          pht('Version'),
+          pht('Writing'),
+        ))
+      ->setColumnClasses(
+        array(
+          null,
+          null,
+          null,
+          'right wide',
+        ));
+
+    $doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories');
+
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Cluster Status'))
+      ->addActionLink(
+        id(new PHUIButtonView())
+          ->setIcon('fa-book')
+          ->setHref($doc_href)
+          ->setTag('a')
+          ->setText(pht('Documentation')));
+
+    if ($service) {
+      $header->setSubheader(
+        pht(
+          'This repository is hosted on %s.',
+          phutil_tag(
+            'a',
+            array(
+              'href' => $service->getURI(),
+            ),
+            $service->getName())));
+    }
+
+    return id(new PHUIObjectBoxView())
+      ->setHeader($header)
+      ->setTable($table);
+  }
+
+}

src/applications/diffusion/management/DiffusionRepositoryManagementPanel.php

@@ -0,0 +1,43 @@
+<?php
+
+abstract class DiffusionRepositoryManagementPanel
+  extends Phobject {
+
+  private $viewer;
+  private $repository;
+
+  final public function setViewer(PhabricatorUser $viewer) {
+    $this->viewer = $viewer;
+    return $this;
+  }
+
+  final public function getViewer() {
+    return $this->viewer;
+  }
+
+  final public function setRepository(PhabricatorRepository $repository) {
+    $this->repository = $repository;
+    return $this;
+  }
+
+  final public function getRepository() {
+    return $this->repository;
+  }
+
+  final public function getManagementPanelKey() {
+    return $this->getPhobjectClassConstant('PANELKEY');
+  }
+
+  abstract public function getManagementPanelLabel();
+  abstract public function getManagementPanelOrder();
+  abstract public function buildManagementPanelContent();
+
+  public static function getAllPanels() {
+    return id(new PhutilClassMapQuery())
+      ->setAncestorClass(__CLASS__)
+      ->setUniqueMethod('getManagementPanelKey')
+      ->setSortMethod('getManagementPanelOrder')
+      ->execute();
+  }
+
+}

src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php

@@ -21,8 +21,12 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow {
 
     if ($this->shouldProxy()) {
       $command = $this->getProxyCommand();
+      $is_proxy = true;
     } else {
       $command = csprintf('git-receive-pack %s', $repository->getLocalPath());
+      $is_proxy = false;
+
+      $repository->synchronizeWorkingCopyBeforeWrite();
     }
     $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
 
@@ -41,6 +45,10 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow {
       $this->waitForGitClient();
     }
 
+    if (!$is_proxy) {
+      $repository->synchronizeWorkingCopyAfterWrite();
+    }
+
     return $err;
   }
 

src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php

@@ -20,6 +20,7 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
       $command = $this->getProxyCommand();
     } else {
       $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
+      $repository->synchronizeWorkingCopyBeforeRead();
     }
     $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
 

src/applications/diffusion/ssh/DiffusionSubversionServeSSHWorkflow.php

@@ -21,6 +21,7 @@ final class DiffusionSubversionServeSSHWorkflow
   private $externalBaseURI;
   private $peekBuffer;
   private $command;
+  private $isProxying;
 
   private function getCommand() {
     return $this->command;
@@ -146,6 +147,7 @@ final class DiffusionSubversionServeSSHWorkflow
 
     if ($this->shouldProxy()) {
       $command = $this->getProxyCommand();
+      $this->isProxying = true;
     } else {
       $command = csprintf(
         'svnserve -t --tunnel-user=%s',
@@ -372,6 +374,10 @@ final class DiffusionSubversionServeSSHWorkflow
   }
 
   private function makeInternalURI($uri_string) {
+    if ($this->isProxying) {
+      return $uri_string;
+    }
+
     $uri = new PhutilURI($uri_string);
 
     $repository = $this->getRepository();
@@ -409,6 +415,10 @@ final class DiffusionSubversionServeSSHWorkflow
   }
 
   private function makeExternalURI($uri) {
+    if ($this->isProxying) {
+      return $uri;
+    }
+
     $internal = $this->internalBaseURI;
     $external = $this->externalBaseURI;
 

src/applications/multimeter/data/MultimeterControl.php

@@ -124,6 +124,10 @@ final class MultimeterControl extends Phobject {
   }
 
   private function writeEvents() {
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $events = $this->events;
 
     $random = Filesystem::readRandomBytes(32);

src/applications/notification/application/PhabricatorNotificationsApplication.php

@@ -25,7 +25,6 @@ final class PhabricatorNotificationsApplication extends PhabricatorApplication {
           => 'PhabricatorNotificationListController',
         'panel/' => 'PhabricatorNotificationPanelController',
         'individual/' => 'PhabricatorNotificationIndividualController',
-        'status/' => 'PhabricatorNotificationStatusController',
         'clear/' => 'PhabricatorNotificationClearController',
         'test/' => 'PhabricatorNotificationTestController',
       ),

src/applications/notification/client/PhabricatorNotificationClient.php

@@ -2,62 +2,34 @@
 
 final class PhabricatorNotificationClient extends Phobject {
 
-  const EXPECT_VERSION = 7;
+  public static function tryAnyConnection() {
+    $servers = PhabricatorNotificationServerRef::getEnabledAdminServers();
 
-  public static function getServerStatus() {
+    if (!$servers) {
-    $uri = PhabricatorEnv::getEnvConfig('notification.server-uri');
-    $uri = id(new PhutilURI($uri))
-      ->setPath('/status/')
-      ->setQueryParam('instance', self::getInstance());
-
-    // We always use HTTP to connect to the server itself: it's simpler and
-    // there's no meaningful security benefit to securing this link today.
-    // Force the protocol to HTTP in case users have set it to something else.
-    $uri->setProtocol('http');
-
-    list($body) = id(new HTTPSFuture($uri))
-      ->setTimeout(3)
-      ->resolvex();
-
-    $status = phutil_json_decode($body);
-    if (!is_array($status)) {
-      throw new Exception(
-        pht(
-          'Expected JSON response from notification server, received: %s',
-          $body));
-    }
-
-    return $status;
-  }
-
-  public static function tryToPostMessage(array $data) {
-    if (!PhabricatorEnv::getEnvConfig('notification.enabled')) {
       return;
     }
 
+    foreach ($servers as $server) {
+      $server->loadServerStatus();
+      return;
+    }
+
+    return;
+  }
+
+  public static function tryToPostMessage(array $data) {
+    $servers = PhabricatorNotificationServerRef::getEnabledAdminServers();
+
+    shuffle($servers);
+
+    foreach ($servers as $server) {
       try {
-      self::postMessage($data);
+        $server->postMessage($data);
+        return;
       } catch (Exception $ex) {
         // Just ignore any issues here.
-      phlog($ex);
+      }
     }
   }
 
-  private static function postMessage(array $data) {
-    $server_uri = PhabricatorEnv::getEnvConfig('notification.server-uri');
-    $server_uri = id(new PhutilURI($server_uri))
-      ->setPath('/')
-      ->setQueryParam('instance', self::getInstance());
-
-    id(new HTTPSFuture($server_uri, json_encode($data)))
-      ->setMethod('POST')
-      ->setTimeout(1)
-      ->resolvex();
-  }
-
-  private static function getInstance() {
-    $client_uri = PhabricatorEnv::getEnvConfig('notification.client-uri');
-    return id(new PhutilURI($client_uri))->getPath();
-  }
-
 }

src/applications/notification/client/PhabricatorNotificationServerRef.php

@@ -0,0 +1,234 @@
+<?php
+
+final class PhabricatorNotificationServerRef
+  extends Phobject {
+
+  private $type;
+  private $host;
+  private $port;
+  private $protocol;
+  private $path;
+  private $isDisabled;
+
+  const KEY_REFS = 'notification.refs';
+
+  public function setType($type) {
+    $this->type = $type;
+    return $this;
+  }
+
+  public function getType() {
+    return $this->type;
+  }
+
+  public function setHost($host) {
+    $this->host = $host;
+    return $this;
+  }
+
+  public function getHost() {
+    return $this->host;
+  }
+
+  public function setPort($port) {
+    $this->port = $port;
+    return $this;
+  }
+
+  public function getPort() {
+    return $this->port;
+  }
+
+  public function setProtocol($protocol) {
+    $this->protocol = $protocol;
+    return $this;
+  }
+
+  public function getProtocol() {
+    return $this->protocol;
+  }
+
+  public function setPath($path) {
+    $this->path = $path;
+    return $this;
+  }
+
+  public function getPath() {
+    return $this->path;
+  }
+
+  public function setIsDisabled($is_disabled) {
+    $this->isDisabled = $is_disabled;
+    return $this;
+  }
+
+  public function getIsDisabled() {
+    return $this->isDisabled;
+  }
+
+  public static function getLiveServers() {
+    $cache = PhabricatorCaches::getRequestCache();
+
+    $refs = $cache->getKey(self::KEY_REFS);
+    if (!$refs) {
+      $refs = self::newRefs();
+      $cache->setKey(self::KEY_REFS, $refs);
+    }
+
+    return $refs;
+  }
+
+  public static function newRefs() {
+    $configs = PhabricatorEnv::getEnvConfig('notification.servers');
+
+    $refs = array();
+    foreach ($configs as $config) {
+      $ref = id(new self())
+        ->setType($config['type'])
+        ->setHost($config['host'])
+        ->setPort($config['port'])
+        ->setProtocol($config['protocol'])
+        ->setPath(idx($config, 'path'))
+        ->setIsDisabled(idx($config, 'disabled', false));
+      $refs[] = $ref;
+    }
+
+    return $refs;
+  }
+
+  public static function getEnabledServers() {
+    $servers = self::getLiveServers();
+
+    foreach ($servers as $key => $server) {
+      if ($server->getIsDisabled()) {
+        unset($servers[$key]);
+      }
+    }
+
+    return array_values($servers);
+  }
+
+  public static function getEnabledAdminServers() {
+    $servers = self::getEnabledServers();
+
+    foreach ($servers as $key => $server) {
+      if (!$server->isAdminServer()) {
+        unset($servers[$key]);
+      }
+    }
+
+    return array_values($servers);
+  }
+
+  public static function getEnabledClientServers($with_protocol) {
+    $servers = self::getEnabledServers();
+
+    foreach ($servers as $key => $server) {
+      if ($server->isAdminServer()) {
+        unset($servers[$key]);
+        continue;
+      }
+
+      $protocol = $server->getProtocol();
+      if ($protocol != $with_protocol) {
+        unset($servers[$key]);
+        continue;
+      }
+    }
+
+    return array_values($servers);
+  }
+
+  public function isAdminServer() {
+    return ($this->type == 'admin');
+  }
+
+  public function getURI($to_path = null) {
+    $full_path = rtrim($this->getPath(), '/').'/'.ltrim($to_path, '/');
+
+    $uri = id(new PhutilURI('http://'.$this->getHost()))
+      ->setProtocol($this->getProtocol())
+      ->setPort($this->getPort())
+      ->setPath($full_path);
+
+    $instance = PhabricatorEnv::getEnvConfig('cluster.instance');
+    if (strlen($instance)) {
+      $uri->setQueryParam('instance', $instance);
+    }
+
+    return $uri;
+  }
+
+  public function getWebsocketURI($to_path = null) {
+    $instance = PhabricatorEnv::getEnvConfig('cluster.instance');
+    if (strlen($instance)) {
+      $to_path = $to_path.'~'.$instance.'/';
+    }
+
+    $uri = $this->getURI($to_path);
+
+    if ($this->getProtocol() == 'https') {
+      $uri->setProtocol('wss');
+    } else {
+      $uri->setProtocol('ws');
+    }
+
+    return $uri;
+  }
+
+  public function testClient() {
+    if ($this->isAdminServer()) {
+      throw new Exception(
+        pht('Unable to test client on an admin server!'));
+    }
+
+    $server_uri = $this->getURI();
+
+    try {
+      id(new HTTPSFuture($server_uri))
+        ->setTimeout(2)
+        ->resolvex();
+    } catch (HTTPFutureHTTPResponseStatus $ex) {
+      // This is what we expect when things are working correctly.
+      if ($ex->getStatusCode() == 501) {
+        return true;
+      }
+      throw $ex;
+    }
+
+    throw new Exception(
+      pht('Got HTTP 200, but expected HTTP 501 (WebSocket Upgrade)!'));
+  }
+
+  public function loadServerStatus() {
+    if (!$this->isAdminServer()) {
+      throw new Exception(
+        pht(
+          'Unable to load server status: this is not an admin server!'));
+    }
+
+    $server_uri = $this->getURI('/status/');
+
+    list($body) = id(new HTTPSFuture($server_uri))
+      ->setTimeout(2)
+      ->resolvex();
+
+    return phutil_json_decode($body);
+  }
+
+  public function postMessage(array $data) {
+    if (!$this->isAdminServer()) {
+      throw new Exception(
+        pht('Unable to post message: this is not an admin server!'));
+    }
+
+    $server_uri = $this->getURI('/');
+    $payload = phutil_json_encode($data);
+
+    id(new HTTPSFuture($server_uri, $payload))
+      ->setMethod('POST')
+      ->setTimeout(2)
+      ->resolvex();
+  }
+
+}

src/applications/notification/config/PhabricatorNotificationServersConfigOptionType.php

@@ -0,0 +1,140 @@
+<?php
+
+final class PhabricatorNotificationServersConfigOptionType
+  extends PhabricatorConfigJSONOptionType {
+
+  public function validateOption(PhabricatorConfigOption $option, $value) {
+    if (!is_array($value)) {
+      throw new Exception(
+        pht(
+          'Notification server configuration is not valid: value must be a '.
+          'list of servers'));
+    }
+
+    foreach ($value as $index => $spec) {
+      if (!is_array($spec)) {
+        throw new Exception(
+          pht(
+            'Notification server configuration is not valid: each entry in '.
+            'the list must be a dictionary describing a service, but '.
+            'the value with index "%s" is not a dictionary.',
+            $index));
+      }
+    }
+
+    $has_admin = false;
+    $has_client = false;
+    $map = array();
+    foreach ($value as $index => $spec) {
+      try {
+        PhutilTypeSpec::checkMap(
+          $spec,
+          array(
+            'type' => 'string',
+            'host' => 'string',
+            'port' => 'int',
+            'protocol' => 'string',
+            'path' => 'optional string',
+            'disabled' => 'optional bool',
+          ));
+      } catch (Exception $ex) {
+        throw new Exception(
+          pht(
+            'Notification server configuration has an invalid service '.
+            'specification (at index "%s"): %s.',
+            $index,
+            $ex->getMessage()));
+      }
+
+      $type = $spec['type'];
+      $host = $spec['host'];
+      $port = $spec['port'];
+      $protocol = $spec['protocol'];
+      $disabled = idx($spec, 'disabled');
+
+      switch ($type) {
+        case 'admin':
+          if (!$disabled) {
+            $has_admin = true;
+          }
+          break;
+        case 'client':
+          if (!$disabled) {
+            $has_client = true;
+          }
+          break;
+        default:
+          throw new Exception(
+            pht(
+              'Notification server configuration describes an invalid '.
+              'host ("%s", at index "%s") with an unrecognized type ("%s"). '.
+              'Valid types are "%s" or "%s".',
+              $host,
+              $index,
+              $type,
+              'admin',
+              'client'));
+      }
+
+      switch ($protocol) {
+        case 'http':
+        case 'https':
+          break;
+        default:
+          throw new Exception(
+            pht(
+              'Notification server configuration describes an invalid '.
+              'host ("%s", at index "%s") with an invalid protocol ("%s"). '.
+              'Valid protocols are "%s" or "%s".',
+              $host,
+              $index,
+              $protocol,
+              'http',
+              'https'));
+      }
+
+      $path = idx($spec, 'path');
+      if ($type == 'admin' && strlen($path)) {
+        throw new Exception(
+          pht(
+            'Notification server configuration describes an invalid host '.
+            '("%s", at index "%s"). This is an "admin" service but it has a '.
+            '"path" property. This property is only valid for "client" '.
+            'services.'));
+      }
+
+      // We can't guarantee that you didn't just give the same host two
+      // different names in DNS, but this check can catch silly copy/paste
+      // mistakes.
+      $key = "{$host}:{$port}";
+      if (isset($map[$key])) {
+        throw new Exception(
+          pht(
+            'Notification server configuration is invalid: it describes the '.
+            'same host and port ("%s") multiple times. Each host and port '.
+            'combination should appear only once in the list.',
+            $key));
+      }
+      $map[$key] = true;
+    }
+
+    if ($value) {
+      if (!$has_admin) {
+        throw new Exception(
+          pht(
+            'Notification server configuration is invalid: it does not '.
+            'specify any enabled servers with type "admin". Notifications '.
+            'require at least one active "admin" server.'));
+      }
+
+      if (!$has_client) {
+        throw new Exception(
+          pht(
+            'Notification server configuration is invalid: it does not '.
+            'specify any enabled servers with type "client". Notifications '.
+            'require at least one active "client" server.'));
+      }
+    }
+  }
+
+}

src/applications/notification/controller/PhabricatorNotificationPanelController.php

@@ -44,17 +44,8 @@ final class PhabricatorNotificationPanelController
       ),
       pht('Notifications'));
 
-    if (PhabricatorEnv::getEnvConfig('notification.enabled')) {
     $connection_status = new PhabricatorNotificationStatusView();
-    } else {
+
-      $connection_status = phutil_tag(
-        'a',
-        array(
-          'href' => PhabricatorEnv::getDoclink(
-            'Notifications User Guide: Setup and Configuration'),
-        ),
-        pht('Notification Server not enabled.'));
-    }
     $connection_ui = phutil_tag(
       'div',
       array(

src/applications/notification/controller/PhabricatorNotificationStatusController.php

@@ -1,82 +0,0 @@
-<?php
-
-final class PhabricatorNotificationStatusController
-  extends PhabricatorNotificationController {
-
-  public function handleRequest(AphrontRequest $request) {
-
-    try {
-      $status = PhabricatorNotificationClient::getServerStatus();
-      $status = $this->renderServerStatus($status);
-    } catch (Exception $ex) {
-      $status = new PHUIInfoView();
-      $status->setTitle(pht('Notification Server Issue'));
-      $status->appendChild(hsprintf(
-        '%s<br /><br />'.
-        '<strong>%s</strong> %s',
-        pht(
-          'Unable to determine server status. This probably means the server '.
-          'is not in great shape. The specific issue encountered was:'),
-        get_class($ex),
-        phutil_escape_html_newlines($ex->getMessage())));
-    }
-
-    $crumbs = $this->buildApplicationCrumbs();
-    $crumbs->addTextCrumb(pht('Status'));
-
-    $title = pht('Notification Server Status');
-
-    return $this->newPage()
-      ->setTitle($title)
-      ->setCrumbs($crumbs)
-      ->appendChild($status);
-  }
-
-  private function renderServerStatus(array $status) {
-
-    $rows = array();
-    foreach ($status as $key => $value) {
-      switch ($key) {
-        case 'uptime':
-          $value /= 1000;
-          $value = phutil_format_relative_time_detailed($value);
-          break;
-        case 'log':
-        case 'instance':
-          break;
-        default:
-          $value = number_format($value);
-          break;
-      }
-
-      $rows[] = array($key, $value);
-    }
-
-    $table = new AphrontTableView($rows);
-    $table->setColumnClasses(
-      array(
-        'header',
-        'wide',
-      ));
-
-    $test_icon = id(new PHUIIconView())
-      ->setIcon('fa-exclamation-triangle');
-
-    $test_button = id(new PHUIButtonView())
-        ->setTag('a')
-        ->setWorkflow(true)
-        ->setText(pht('Send Test Notification'))
-        ->setHref($this->getApplicationURI('test/'))
-        ->setIcon($test_icon);
-
-    $header = id(new PHUIHeaderView())
-      ->setHeader(pht('Notification Server Status'))
-      ->addActionLink($test_button);
-
-    $box = id(new PHUIObjectBoxView())
-      ->setHeader($header)
-      ->appendChild($table);
-
-    return $box;
-  }
-}

src/applications/notification/setup/PhabricatorAphlictSetupCheck.php

@@ -3,14 +3,8 @@
 final class PhabricatorAphlictSetupCheck extends PhabricatorSetupCheck {
 
   protected function executeChecks() {
-    $enabled = PhabricatorEnv::getEnvConfig('notification.enabled');
-    if (!$enabled) {
-      // Notifications aren't set up, so just ignore all of these checks.
-      return;
-    }
-
     try {
-      $status = PhabricatorNotificationClient::getServerStatus();
+      PhabricatorNotificationClient::tryAnyConnection();
     } catch (Exception $ex) {
       $message = pht(
         "Phabricator is configured to use a notification server, but is ".
@@ -31,8 +25,7 @@ final class PhabricatorAphlictSetupCheck extends PhabricatorSetupCheck {
         ->setShortName(pht('Notification Server Down'))
         ->setName(pht('Unable to Connect to Notification Server'))
         ->setMessage($message)
-        ->addRelatedPhabricatorConfig('notification.enabled')
+        ->addRelatedPhabricatorConfig('notification.servers')
-        ->addRelatedPhabricatorConfig('notification.server-uri')
         ->addCommand(
           pht(
             "(To start the server, run this command.)\n%s",
@@ -40,23 +33,5 @@ final class PhabricatorAphlictSetupCheck extends PhabricatorSetupCheck {
 
       return;
     }
-
-    $expect_version = PhabricatorNotificationClient::EXPECT_VERSION;
-    $have_version = idx($status, 'version', 1);
-    if ($have_version != $expect_version) {
-      $message = pht(
-        'The notification server is out of date. You are running server '.
-        'version %d, but Phabricator expects version %d. Restart the server '.
-        'to update it, using the command below:',
-        $have_version,
-        $expect_version);
-
-      $this->newIssue('aphlict.version')
-        ->setShortName(pht('Notification Server Version'))
-        ->setName(pht('Notification Server Out of Date'))
-        ->setMessage($message)
-        ->addCommand('phabricator/ $ ./bin/aphlict restart');
-    }
-
   }
 }

src/applications/notification/storage/PhabricatorFeedStoryNotification.php

@@ -39,6 +39,10 @@ final class PhabricatorFeedStoryNotification extends PhabricatorFeedDAO {
     PhabricatorUser $user,
     $object_phid) {
 
+    if (PhabricatorEnv::isReadOnly()) {
+      return;
+    }
+
     $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
 
     $notification_table = new PhabricatorFeedStoryNotification();

src/applications/passphrase/controller/PassphraseCredentialRevealController.php

@@ -51,6 +51,7 @@ final class PassphraseCredentialRevealController
             id(new AphrontFormTextAreaControl())
               ->setLabel(pht('Plaintext'))
               ->setReadOnly(true)
+              ->setCustomClass('PhabricatorMonospaced')
               ->setHeight(AphrontFormTextAreaControl::HEIGHT_VERY_TALL)
               ->setValue($secret->openEnvelope()));
       }

src/applications/people/profilepanel/PhabricatorPeopleManageProfilePanel.php

@@ -6,7 +6,7 @@ final class PhabricatorPeopleManageProfilePanel
   const PANELKEY = 'people.manage';
 
   public function getPanelTypeName() {
-    return pht('Mangage User');
+    return pht('Manage User');
   }
 
   private function getDefaultName() {

src/applications/people/storage/PhabricatorUser.php

@@ -133,6 +133,19 @@ final class PhabricatorUser
   }
 
   public function canEstablishAPISessions() {
+    if ($this->getIsDisabled()) {
+      return false;
+    }
+
+    // Intracluster requests are permitted even if the user is logged out:
+    // in particular, public users are allowed to issue intracluster requests
+    // when browsing Diffusion.
+    if (PhabricatorEnv::isClusterRemoteAddress()) {
+      if (!$this->isLoggedIn()) {
+        return true;
+      }
+    }
+
     if (!$this->isUserActivated()) {
       return false;
     }

src/applications/phurl/controller/PhabricatorPhurlURLViewController.php

@@ -114,7 +114,7 @@ final class PhabricatorPhurlURLViewController
     $curtain
       ->addAction(
         id(new PhabricatorActionView())
-          ->setName(pht('Edit'))
+          ->setName(pht('Edit Phurl'))
           ->setIcon('fa-pencil')
           ->setHref($this->getApplicationURI("url/edit/{$id}/"))
           ->setDisabled(!$can_edit)

src/applications/phurl/query/PhabricatorPhurlURLSearchEngine.php

@@ -44,8 +44,8 @@ final class PhabricatorPhurlURLSearchEngine
 
   protected function getBuiltinQueryNames() {
     $names = array(
-      'authored' => pht('Authored'),
       'all' => pht('All URLs'),
+      'authored' => pht('Authored'),
     );
 
     return $names;
@@ -77,10 +77,16 @@ final class PhabricatorPhurlURLSearchEngine
     $handles = $viewer->loadHandles(mpull($urls, 'getAuthorPHID'));
 
     foreach ($urls as $url) {
+      $name = $url->getName();
+
       $item = id(new PHUIObjectItemView())
         ->setUser($viewer)
         ->setObject($url)
-        ->setHeader($viewer->renderHandle($url->getPHID()));
+        ->setObjectName('U'.$url->getID())
+        ->setHeader($name)
+        ->setHref('/U'.$url->getID())
+        ->addAttribute($url->getAlias())
+        ->addAttribute($url->getLongURL());
 
       $list->addItem($item);
     }

src/applications/project/controller/PhabricatorProjectMembersViewController.php

@@ -22,21 +22,23 @@ final class PhabricatorProjectMembersViewController
     $title = pht('Members and Watchers');
 
     $properties = $this->buildProperties($project);
-    $actions = $this->buildActions($project);
+    $curtain = $this->buildCurtainView($project);
-    $properties->setActionList($actions);
 
     $object_box = id(new PHUIObjectBoxView())
-      ->setHeaderText($title)
+      ->setHeaderText(pht('Details'))
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->addPropertyList($properties);
 
     $member_list = id(new PhabricatorProjectMemberListView())
       ->setUser($viewer)
       ->setProject($project)
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setUserPHIDs($project->getMemberPHIDs());
 
     $watcher_list = id(new PhabricatorProjectWatcherListView())
       ->setUser($viewer)
       ->setProject($project)
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setUserPHIDs($project->getWatcherPHIDs());
 
     $nav = $this->getProfileMenu();
@@ -44,17 +46,27 @@ final class PhabricatorProjectMembersViewController
 
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('Members'));
+    $crumbs->setBorder(true);
 
-    return $this->newPage()
+    $header = id(new PHUIHeaderView())
-      ->setNavigation($nav)
+      ->setHeader($title)
-      ->setCrumbs($crumbs)
+      ->setHeaderIcon('fa-group');
-      ->setTitle(array($project->getDisplayName(), $title))
+
-      ->appendChild(
+    $view = id(new PHUITwoColumnView())
-        array(
+      ->setHeader($header)
+      ->setCurtain($curtain)
+      ->setMainColumn(array(
         $object_box,
         $member_list,
         $watcher_list,
       ));
+
+
+    return $this->newPage()
+      ->setNavigation($nav)
+      ->setCrumbs($crumbs)
+      ->setTitle(array($project->getName(), $title))
+      ->appendChild($view);
   }
 
   private function buildProperties(PhabricatorProject $project) {
@@ -156,12 +168,11 @@ final class PhabricatorProjectMembersViewController
     return $view;
   }
 
-  private function buildActions(PhabricatorProject $project) {
+  private function buildCurtainView(PhabricatorProject $project) {
     $viewer = $this->getViewer();
     $id = $project->getID();
 
-    $view = id(new PhabricatorActionListView())
+    $curtain = $this->newCurtainView($project);
-      ->setUser($viewer);
 
     $is_locked = $project->getIsMembershipLocked();
 
@@ -182,7 +193,7 @@ final class PhabricatorProjectMembersViewController
     $viewer_phid = $viewer->getPHID();
 
     if (!$project->isUserMember($viewer_phid)) {
-      $view->addAction(
+      $curtain->addAction(
         id(new PhabricatorActionView())
           ->setHref('/project/update/'.$project->getID().'/join/')
           ->setIcon('fa-plus')
@@ -190,7 +201,7 @@ final class PhabricatorProjectMembersViewController
           ->setWorkflow(true)
           ->setName(pht('Join Project')));
     } else {
-      $view->addAction(
+      $curtain->addAction(
         id(new PhabricatorActionView())
           ->setHref('/project/update/'.$project->getID().'/leave/')
           ->setIcon('fa-times')
@@ -200,14 +211,14 @@ final class PhabricatorProjectMembersViewController
     }
 
     if (!$project->isUserWatcher($viewer->getPHID())) {
-      $view->addAction(
+      $curtain->addAction(
         id(new PhabricatorActionView())
           ->setWorkflow(true)
           ->setHref('/project/watch/'.$project->getID().'/')
           ->setIcon('fa-eye')
           ->setName(pht('Watch Project')));
     } else {
-      $view->addAction(
+      $curtain->addAction(
         id(new PhabricatorActionView())
           ->setWorkflow(true)
           ->setHref('/project/unwatch/'.$project->getID().'/')
@@ -224,7 +235,7 @@ final class PhabricatorProjectMembersViewController
       $silence_text = pht('Disable Mail');
     }
 
-    $view->addAction(
+    $curtain->addAction(
       id(new PhabricatorActionView())
         ->setName($silence_text)
         ->setIcon('fa-envelope-o')
@@ -234,7 +245,7 @@ final class PhabricatorProjectMembersViewController
 
     $can_add = $can_edit && $supports_edit;
 
-    $view->addAction(
+    $curtain->addAction(
       id(new PhabricatorActionView())
         ->setName(pht('Add Members'))
         ->setIcon('fa-user-plus')
@@ -253,7 +264,7 @@ final class PhabricatorProjectMembersViewController
       $lock_icon = 'fa-lock';
     }
 
-    $view->addAction(
+    $curtain->addAction(
       id(new PhabricatorActionView())
         ->setName($lock_name)
         ->setIcon($lock_icon)
@@ -261,7 +272,7 @@ final class PhabricatorProjectMembersViewController
         ->setDisabled(!$can_lock)
         ->setWorkflow(true));
 
-    return $view;
+    return $curtain;
   }
 
   private function isProjectSilenced(PhabricatorProject $project) {

src/applications/project/controller/PhabricatorProjectSubprojectsController.php

@@ -52,6 +52,7 @@ final class PhabricatorProjectSubprojectsController
     if ($milestones) {
       $milestone_list = id(new PHUIObjectBoxView())
         ->setHeaderText(pht('Milestones'))
+        ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
         ->setObjectList(
           id(new PhabricatorProjectListView())
             ->setUser($viewer)
@@ -64,6 +65,7 @@ final class PhabricatorProjectSubprojectsController
     if ($subprojects) {
       $subproject_list = id(new PHUIObjectBoxView())
         ->setHeaderText(pht('Subprojects'))
+        ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
         ->setObjectList(
           id(new PhabricatorProjectListView())
             ->setUser($viewer)
@@ -78,15 +80,15 @@ final class PhabricatorProjectSubprojectsController
       $milestones,
       $subprojects);
 
-    $action_list = $this->buildActionList(
+    $curtain = $this->buildCurtainView(
       $project,
       $milestones,
       $subprojects);
 
-    $property_list->setActionList($action_list);
 
-    $header_box = id(new PHUIObjectBoxView())
+    $details = id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('Subprojects and Milestones'))
+      ->setHeaderText(pht('Details'))
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->addPropertyList($property_list);
 
     $nav = $this->getProfileMenu();
@@ -94,17 +96,26 @@ final class PhabricatorProjectSubprojectsController
 
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('Subprojects'));
+    $crumbs->setBorder(true);
+
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Subprojects and Milestones'))
+      ->setHeaderIcon('fa-sitemap');
+
+    $view = id(new PHUITwoColumnView())
+      ->setHeader($header)
+      ->setCurtain($curtain)
+      ->setMainColumn(array(
+          $details,
+          $milestone_list,
+          $subproject_list,
+      ));
 
     return $this->newPage()
       ->setNavigation($nav)
       ->setCrumbs($crumbs)
       ->setTitle(array($project->getName(), pht('Subprojects')))
-      ->appendChild(
+      ->appendChild($view);
-        array(
-          $header_box,
-          $milestone_list,
-          $subproject_list,
-        ));
   }
 
   private function buildPropertyList(
@@ -174,7 +185,7 @@ final class PhabricatorProjectSubprojectsController
     return $view;
   }
 
-  private function buildActionList(
+  private function buildCurtainView(
     PhabricatorProject $project,
     array $milestones,
     array $subprojects) {
@@ -192,8 +203,7 @@ final class PhabricatorProjectSubprojectsController
     $allows_milestones = $project->supportsMilestones();
     $allows_subprojects = $project->supportsSubprojects();
 
-    $view = id(new PhabricatorActionListView())
+    $curtain = $this->newCurtainView($project);
-      ->setUser($viewer);
 
     if ($allows_milestones && $milestones) {
       $milestone_text = pht('Create Next Milestone');
@@ -204,7 +214,7 @@ final class PhabricatorProjectSubprojectsController
     $can_milestone = ($can_create && $can_edit && $allows_milestones);
     $milestone_href = "/project/edit/?milestone={$id}";
 
-    $view->addAction(
+    $curtain->addAction(
       id(new PhabricatorActionView())
         ->setName($milestone_text)
         ->setIcon('fa-plus')
@@ -226,7 +236,7 @@ final class PhabricatorProjectSubprojectsController
       $subproject_workflow = !$can_subproject;
     }
 
-    $view->addAction(
+    $curtain->addAction(
       id(new PhabricatorActionView())
         ->setName(pht('Create Subproject'))
         ->setIcon('fa-plus')
@@ -234,7 +244,7 @@ final class PhabricatorProjectSubprojectsController
         ->setDisabled($subproject_disabled)
         ->setWorkflow($subproject_workflow));
 
-    return $view;
+    return $curtain;
   }
 
   private function renderStatus($icon, $target, $note) {

src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php

@@ -74,7 +74,9 @@ final class PhabricatorRepositoryPullLocalDaemon
 
     while (!$this->shouldExit()) {
       PhabricatorCaches::destroyRequestCache();
-      $pullable = $this->loadPullableRepositories($include, $exclude);
+      $device = AlmanacKeys::getLiveDevice();
+
+      $pullable = $this->loadPullableRepositories($include, $exclude, $device);
 
       // If any repositories have the NEEDS_UPDATE flag set, pull them
       // as soon as possible.
@@ -297,7 +299,11 @@ final class PhabricatorRepositoryPullLocalDaemon
   /**
    * @task pull
    */
-  private function loadPullableRepositories(array $include, array $exclude) {
+  private function loadPullableRepositories(
+    array $include,
+    array $exclude,
+    AlmanacDevice $device = null) {
+
     $query = id(new PhabricatorRepositoryQuery())
       ->setViewer($this->getViewer());
 
@@ -348,6 +354,119 @@ final class PhabricatorRepositoryPullLocalDaemon
       }
     }
 
+    $service_phids = array();
+    foreach ($repositories as $key => $repository) {
+      $service_phid = $repository->getAlmanacServicePHID();
+
+      // If the repository is bound to a service but this host is not a
+      // recognized device, or vice versa, don't pull the repository.
+      $is_cluster_repo = (bool)$service_phid;
+      $is_cluster_device = (bool)$device;
+      if ($is_cluster_repo != $is_cluster_device) {
+        if ($is_cluster_device) {
+          $this->log(
+            pht(
+              'Repository "%s" is not a cluster repository, but the current '.
+              'host is a cluster device ("%s"), so the repository will not '.
+              'be updated on this host.',
+              $repository->getDisplayName(),
+              $device->getName()));
+        } else {
+          $this->log(
+            pht(
+              'Repository "%s" is a cluster repository, but the current '.
+              'host is not a cluster device (it has no device ID), so the '.
+              'repository will not be updated on this host.',
+              $repository->getDisplayName()));
+        }
+        unset($repositories[$key]);
+        continue;
+      }
+
+      if ($service_phid) {
+        $service_phids[] = $service_phid;
+      }
+    }
+
+    if ($device) {
+      $device_phid = $device->getPHID();
+
+      if ($service_phids) {
+        // We could include `withDevicePHIDs()` here to pull a smaller result
+        // set, but we can provide more helpful diagnostic messages below if
+        // we fetch a little more data.
+        $services = id(new AlmanacServiceQuery())
+          ->setViewer($this->getViewer())
+          ->withPHIDs($service_phids)
+          ->withServiceTypes(
+            array(
+              AlmanacClusterRepositoryServiceType::SERVICETYPE,
+            ))
+          ->needBindings(true)
+          ->execute();
+        $services = mpull($services, null, 'getPHID');
+      } else {
+        $services = array();
+      }
+
+      foreach ($repositories as $key => $repository) {
+        $service_phid = $repository->getAlmanacServicePHID();
+
+        $service = idx($services, $service_phid);
+        if (!$service) {
+          $this->log(
+            pht(
+              'Repository "%s" is on cluster service "%s", but that service '.
+              'could not be loaded, so the repository will not be updated '.
+              'on this host.',
+              $repository->getDisplayName(),
+              $service_phid));
+          unset($repositories[$key]);
+          continue;
+        }
+
+        $bindings = $service->getBindings();
+        $bindings = mgroup($bindings, 'getDevicePHID');
+        $bindings = idx($bindings, $device_phid);
+        if (!$bindings) {
+          $this->log(
+            pht(
+              'Repository "%s" is on cluster service "%s", but that service '.
+              'is not bound to this device ("%s"), so the repository will '.
+              'not be updated on this host.',
+              $repository->getDisplayName(),
+              $service->getName(),
+              $device->getName()));
+          unset($repositories[$key]);
+          continue;
+        }
+
+        $all_disabled = true;
+        foreach ($bindings as $binding) {
+          if (!$binding->getIsDisabled()) {
+            $all_disabled = false;
+            break;
+          }
+        }
+
+        if ($all_disabled) {
+          $this->log(
+            pht(
+              'Repository "%s" is on cluster service "%s", but the binding '.
+              'between that service and this device ("%s") is disabled, so '.
+              'the not be updated on this host.',
+              $repository->getDisplayName(),
+              $service->getName(),
+              $device->getName()));
+          unset($repositories[$key]);
+          continue;
+        }
+
+        // We have a valid service that is actively bound to the current host
+        // device, so we're good to go.
+      }
+    }
+
     // Shuffle the repositories, then re-key the array since shuffle()
     // discards keys. This is mostly for startup, we'll use soft priorities
     // later.

src/applications/repository/storage/PhabricatorRepository.php

@@ -3,6 +3,7 @@
 /**
  * @task uri        Repository URI Management
  * @task autoclose  Autoclose
+ * @task sync       Cluster Synchronization
  */
 final class PhabricatorRepository extends PhabricatorRepositoryDAO
   implements
@@ -62,6 +63,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
   private $mostRecentCommit = self::ATTACHABLE;
   private $projectPHIDs = self::ATTACHABLE;
 
+  private $clusterWriteLock;
+  private $clusterWriteVersion;
+
   public static function initializeNewRepository(PhabricatorUser $actor) {
     $app = id(new PhabricatorApplicationQuery())
       ->setViewer($actor)
@@ -2262,6 +2266,174 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
   }
 
 
+/* -(  Cluster Synchronization  )-------------------------------------------- */
+
+
+  private function shouldEnableSynchronization() {
+    // TODO: This mostly works, but isn't stable enough for production yet.
+    return false;
+
+    $device = AlmanacKeys::getLiveDevice();
+    if (!$device) {
+      return false;
+    }
+
+    return true;
+  }
+
+
+  /**
+   * @task sync
+   */
+  public function synchronizeWorkingCopyBeforeRead() {
+    if (!$this->shouldEnableSynchronization()) {
+      return;
+    }
+
+    $repository_phid = $this->getPHID();
+
+    $device = AlmanacKeys::getLiveDevice();
+    $device_phid = $device->getPHID();
+
+    $read_lock = PhabricatorRepositoryWorkingCopyVersion::getReadLock(
+      $repository_phid,
+      $device_phid);
+
+    // TODO: Raise a more useful exception if we fail to grab this lock.
+    $read_lock->lock(phutil_units('2 minutes in seconds'));
+
+    $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
+      $repository_phid);
+    $versions = mpull($versions, null, 'getDevicePHID');
+
+    $this_version = idx($versions, $device_phid);
+    if ($this_version) {
+      $this_version = (int)$this_version->getRepositoryVersion();
+    } else {
+      $this_version = 0;
+    }
+
+    if ($versions) {
+      $max_version = (int)max(mpull($versions, 'getRepositoryVersion'));
+    } else {
+      $max_version = 0;
+    }
+
+    if ($max_version > $this_version) {
+      $fetchable = array();
+      foreach ($versions as $version) {
+        if ($version->getRepositoryVersion() == $max_version) {
+          $fetchable[] = $version->getDevicePHID();
+        }
+      }
+
+      // TODO: Actualy fetch the newer version from one of the nodes which has
+      // it.
+
+      PhabricatorRepositoryWorkingCopyVersion::updateVersion(
+        $repository_phid,
+        $device_phid,
+        $max_version);
+    }
+
+    $read_lock->unlock();
+
+    return $max_version;
+  }
+
+
+  /**
+   * @task sync
+   */
+  public function synchronizeWorkingCopyBeforeWrite() {
+    if (!$this->shouldEnableSynchronization()) {
+      return;
+    }
+
+    $repository_phid = $this->getPHID();
+
+    $device = AlmanacKeys::getLiveDevice();
+    $device_phid = $device->getPHID();
+
+    $write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock(
+      $repository_phid);
+
+    // TODO: Raise a more useful exception if we fail to grab this lock.
+    $write_lock->lock(phutil_units('2 minutes in seconds'));
+
+    $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
+      $repository_phid);
+    foreach ($versions as $version) {
+      if (!$version->getIsWriting()) {
+        continue;
+      }
+
+      // TODO: This should provide more help so users can resolve the issue.
+      throw new Exception(
+        pht(
+          'An incomplete write was previously performed to this repository; '.
+          'refusing new writes.'));
+    }
+
+    $max_version = $this->synchronizeWorkingCopyBeforeRead();
+
+    PhabricatorRepositoryWorkingCopyVersion::willWrite(
+      $repository_phid,
+      $device_phid);
+
+    $this->clusterWriteVersion = $max_version;
+    $this->clusterWriteLock = $write_lock;
+  }
+
+
+  /**
+   * @task sync
+   */
+  public function synchronizeWorkingCopyAfterWrite() {
+    if (!$this->shouldEnableSynchronization()) {
+      return;
+    }
+
+    if (!$this->clusterWriteLock) {
+      throw new Exception(
+        pht(
+          'Trying to synchronize after write, but not holding a write '.
+          'lock!'));
+    }
+
+    $repository_phid = $this->getPHID();
+
+    $device = AlmanacKeys::getLiveDevice();
+    $device_phid = $device->getPHID();
+
+    // NOTE: This means we're still bumping the version when pushes fail. We
+    // could select only un-rejected events instead to bump a little less
+    // often.
+
+    $new_log = id(new PhabricatorRepositoryPushEventQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withRepositoryPHIDs(array($repository_phid))
+      ->setLimit(1)
+      ->executeOne();
+
+    $old_version = $this->clusterWriteVersion;
+    if ($new_log) {
+      $new_version = $new_log->getID();
+    } else {
+      $new_version = $old_version;
+    }
+
+    PhabricatorRepositoryWorkingCopyVersion::didWrite(
+      $repository_phid,
+      $device_phid,
+      $this->clusterWriteVersion,
+      $new_log->getID());
+
+    $this->clusterWriteLock->unlock();
+    $this->clusterWriteLock = null;
+  }
+
+
 /* -(  Symbols  )-------------------------------------------------------------*/
 
   public function getSymbolSources() {

src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php

@@ -0,0 +1,145 @@
+<?php
+
+final class PhabricatorRepositoryWorkingCopyVersion
+  extends PhabricatorRepositoryDAO {
+
+  protected $repositoryPHID;
+  protected $devicePHID;
+  protected $repositoryVersion;
+  protected $isWriting;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'repositoryVersion' => 'uint32',
+        'isWriting' => 'bool',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_workingcopy' => array(
+          'columns' => array('repositoryPHID', 'devicePHID'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public static function loadVersions($repository_phid) {
+    $version = new self();
+    $conn_w = $version->establishConnection('w');
+    $table = $version->getTableName();
+
+    // This is a normal read, but force it to come from the master.
+    $rows = queryfx_all(
+      $conn_w,
+      'SELECT * FROM %T WHERE repositoryPHID = %s',
+      $table,
+      $repository_phid);
+
+    return $version->loadAllFromArray($rows);
+  }
+
+  public static function getReadLock($repository_phid, $device_phid) {
+    $repository_hash = PhabricatorHash::digestForIndex($repository_phid);
+    $device_hash = PhabricatorHash::digestForIndex($device_phid);
+    $lock_key = "repo.read({$repository_hash}, {$device_hash})";
+
+    return PhabricatorGlobalLock::newLock($lock_key);
+  }
+
+  public static function getWriteLock($repository_phid) {
+    $repository_hash = PhabricatorHash::digestForIndex($repository_phid);
+    $lock_key = "repo.write({$repository_hash})";
+
+    return PhabricatorGlobalLock::newLock($lock_key);
+  }
+
+
+  /**
+   * Before a write, set the "isWriting" flag.
+   *
+   * This allows us to detect when we lose a node partway through a write and
+   * may have committed and acknowledged a write on a node that lost the lock
+   * partway through the write and is no longer reachable.
+   *
+   * In particular, if a node loses its connection to the datbase the global
+   * lock is released by default. This is a durable lock which stays locked
+   * by default.
+   */
+  public static function willWrite($repository_phid, $device_phid) {
+    $version = new self();
+    $conn_w = $version->establishConnection('w');
+    $table = $version->getTableName();
+
+    queryfx(
+      $conn_w,
+      'INSERT INTO %T
+        (repositoryPHID, devicePHID, repositoryVersion, isWriting)
+        VALUES
+        (%s, %s, %d, %d)
+        ON DUPLICATE KEY UPDATE
+          isWriting = VALUES(isWriting)',
+      $table,
+      $repository_phid,
+      $device_phid,
+      1,
+      1);
+  }
+
+
+  /**
+   * After a write, update the version and release the "isWriting" lock.
+   */
+  public static function didWrite(
+    $repository_phid,
+    $device_phid,
+    $old_version,
+    $new_version) {
+    $version = new self();
+    $conn_w = $version->establishConnection('w');
+    $table = $version->getTableName();
+
+    queryfx(
+      $conn_w,
+      'UPDATE %T SET repositoryVersion = %d, isWriting = 0
+        WHERE
+          repositoryPHID = %s AND
+          devicePHID = %s AND
+          repositoryVersion = %d AND
+          isWriting = 1',
+      $table,
+      $new_version,
+      $repository_phid,
+      $device_phid,
+      $old_version);
+  }
+
+
+  /**
+   * After a fetch, set the local version to the fetched version.
+   */
+  public static function updateVersion(
+    $repository_phid,
+    $device_phid,
+    $new_version) {
+    $version = new self();
+    $conn_w = $version->establishConnection('w');
+    $table = $version->getTableName();
+
+    queryfx(
+      $conn_w,
+      'INSERT INTO %T
+        (repositoryPHID, devicePHID, repositoryVersion, isWriting)
+        VALUES
+        (%s, %s, %d, %d)
+        ON DUPLICATE KEY UPDATE
+          repositoryVersion = VALUES(repositoryVersion)',
+      $table,
+      $repository_phid,
+      $device_phid,
+      $new_version,
+      0);
+  }
+
+
+}

src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php

@@ -7,38 +7,18 @@ final class PhabricatorRepositoryGitCommitChangeParserWorker
     PhabricatorRepository $repository,
     PhabricatorRepositoryCommit $commit) {
 
-    // Check if the commit has parents. We're testing to see whether it is the
+    $viewer = PhabricatorUser::getOmnipotentUser();
-    // first commit in history (in which case we must use "git log") or some
+    $raw = DiffusionQuery::callConduitWithDiffusionRequest(
-    // other commit (in which case we can use "git diff"). We'd rather use
+      $viewer,
-    // "git diff" because it has the right behavior for merge commits, but
+      DiffusionRequest::newFromDictionary(
-    // it requires the commit to have a parent that we can diff against. The
+        array(
-    // first commit doesn't, so "commit^" is not a valid ref.
+          'repository' => $repository,
-    list($parents) = $repository->execxLocalCommand(
+          'user' => $viewer,
-      'log -n1 --format=%s %s',
+        )),
-      '%P',
+      'diffusion.internal.gitrawdiffquery',
-      $commit->getCommitIdentifier());
+      array(
-
+        'commit' => $commit->getCommitIdentifier(),
-    $use_log = !strlen(trim($parents));
+      ));
-    if ($use_log) {
-      // This is the first commit so we need to use "log". We know it's not a
-      // merge commit because it couldn't be merging anything, so this is safe.
-
-      // NOTE: "--pretty=format: " is to disable diff output, we only want the
-      // part we get from "--raw".
-      list($raw) = $repository->execxLocalCommand(
-        'log -n1 -M -C -B --find-copies-harder --raw -t '.
-          '--pretty=format: --abbrev=40 %s',
-        $commit->getCommitIdentifier());
-    } else {
-      // Otherwise, we can use "diff", which will give us output for merges.
-      // We diff against the first parent, as this is generally the expectation
-      // and results in sensible behavior.
-      list($raw) = $repository->execxLocalCommand(
-        'diff -n1 -M -C -B --find-copies-harder --raw -t '.
-          '--abbrev=40 %s^1 %s',
-        $commit->getCommitIdentifier(),
-        $commit->getCommitIdentifier());
-    }
 
     $changes = array();
     $move_away = array();

src/applications/settings/panel/PhabricatorDesktopNotificationsSettingsPanel.php

@@ -4,8 +4,12 @@ final class PhabricatorDesktopNotificationsSettingsPanel
   extends PhabricatorSettingsPanel {
 
   public function isEnabled() {
-    return PhabricatorEnv::getEnvConfig('notification.enabled') &&
+    $servers = PhabricatorNotificationServerRef::getEnabledAdminServers();
-      PhabricatorApplication::isClassInstalled(
+    if (!$servers) {
+      return false;
+    }
+
+    return PhabricatorApplication::isClassInstalled(
       'PhabricatorNotificationsApplication');
   }
 

src/applications/system/application/PhabricatorSystemApplication.php

@@ -23,6 +23,9 @@ final class PhabricatorSystemApplication extends PhabricatorApplication {
         'encoding/' => 'PhabricatorSystemSelectEncodingController',
         'highlight/' => 'PhabricatorSystemSelectHighlightController',
       ),
+      '/readonly/' => array(
+        '(?P<reason>[^/]+)/' => 'PhabricatorSystemReadOnlyController',
+      ),
     );
   }
 

src/applications/system/controller/PhabricatorSystemReadOnlyController.php

@@ -0,0 +1,134 @@
+<?php
+
+final class PhabricatorSystemReadOnlyController
+  extends PhabricatorController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  public function handleRequest(AphrontRequest $request) {
+    $viewer = $this->getViewer();
+    $reason = $request->getURIData('reason');
+
+    $body = array();
+    switch ($reason) {
+      case PhabricatorEnv::READONLY_CONFIG:
+        $title = pht('Administrative Read-Only Mode');
+        $body[] = pht(
+          'An administrator has placed Phabricator into read-only mode.');
+        $body[] = pht(
+          'This mode may be used to perform temporary maintenance, test '.
+          'configuration, or archive an installation permanently.');
+        $body[] = pht(
+          'Read-only mode was enabled by the explicit action of a human '.
+          'administrator, so you can get more information about why it '.
+          'has been turned on by rolling your chair away from your desk and '.
+          'yelling "Hey! Why is Phabricator in read-only mode??!" using '.
+          'your very loudest outside voice.');
+        $body[] = pht(
+          'This mode is active because it is enabled in the configuration '.
+          'option "%s".',
+          phutil_tag('tt', array(), 'cluster.read-only'));
+        $button = pht('Wait Patiently');
+        break;
+      case PhabricatorEnv::READONLY_MASTERLESS:
+        $title = pht('No Writable Database');
+        $body[] = pht(
+          'Phabricator is currently configured with no writable ("master") '.
+          'database, so it can not write new information anywhere. '.
+          'Phabricator will run in read-only mode until an administrator '.
+          'reconfigures it with a writable database.');
+        $body[] = pht(
+          'This usually occurs when an administrator is actively working on '.
+          'fixing a temporary configuration or deployment problem.');
+        $body[] = pht(
+          'This mode is active because no database has a "%s" role in '.
+          'the configuration option "%s".',
+          phutil_tag('tt', array(), 'master'),
+          phutil_tag('tt', array(), 'cluster.databases'));
+        $button = pht('Wait Patiently');
+        break;
+      case PhabricatorEnv::READONLY_UNREACHABLE:
+        $title = pht('Unable to Reach Master');
+        $body[] = pht(
+          'Phabricator was unable to connect to the writable ("master") '.
+          'database while handling this request, and automatically degraded '.
+          'into read-only mode.');
+        $body[] = pht(
+          'This may happen if there is a temporary network anomaly on the '.
+          'server side, like cosmic radiation or spooky ghosts. If this '.
+          'failure was caused by a transient service interruption, '.
+          'Phabricator will recover momentarily.');
+        $body[] = pht(
+          'This may also indicate that a more serious failure has occurred. '.
+          'If this interruption does not resolve on its own, Phabricator '.
+          'will soon detect the persistent disruption and degrade into '.
+          'read-only mode until the issue is resolved.');
+        $button = pht('Quite Unsettling');
+        break;
+      case PhabricatorEnv::READONLY_SEVERED:
+        $title = pht('Severed From Master');
+        $body[] = pht(
+          'Phabricator has consistently been unable to reach the writable '.
+          '("master") database while processing recent requests.');
+        $body[] = pht(
+          'This likely indicates a severe misconfiguration or major service '.
+          'interruption.');
+        $body[] = pht(
+          'Phabricator will periodically retry the connection and recover '.
+          'once service is restored. Most causes of persistent service '.
+          'interruption will require administrative intervention in order '.
+          'to restore service.');
+        $body[] = pht(
+          'Although this may be the result of a misconfiguration or '.
+          'operational error, this is also the state you reach if a '.
+          'meteor recently obliterated a datacenter.');
+        $button = pht('Panic!');
+        break;
+      default:
+        return new Aphront404Response();
+    }
+
+    switch ($reason) {
+      case PhabricatorEnv::READONLY_UNREACHABLE:
+      case PhabricatorEnv::READONLY_SEVERED:
+        $body[] = pht(
+          'This request was served from a replica database. Replica '.
+          'databases may lag behind the master, so very recent activity '.
+          'may not be reflected in the UI. This data will be restored if '.
+          'the master database is restored, but may have been lost if the '.
+          'master database has been reduced to a pile of ash.');
+        break;
+    }
+
+    $body[] = pht(
+      'In read-only mode you can read existing information, but you will not '.
+      'be able to edit objects or create new objects until this mode is '.
+      'disabled.');
+
+    if ($viewer->getIsAdmin()) {
+      $body[] = pht(
+        'As an administrator, you can review status information from the '.
+        '%s control panel. This may provide more information about the '.
+        'current state of affairs.',
+        phutil_tag(
+          'a',
+          array(
+            'href' => '/config/cluster/databases/',
+          ),
+          pht('Cluster Database Status')));
+    }
+
+    $dialog = $this->newDialog()
+      ->setTitle($title)
+      ->setWidth(AphrontDialogView::WIDTH_FORM)
+      ->addCancelButton('/', $button);
+
+    foreach ($body as $paragraph) {
+      $dialog->appendParagraph($paragraph);
+    }
+
+    return $dialog;
+  }
+}

src/applications/uiexample/examples/PHUIBadgeExample.php

@@ -54,7 +54,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-user')
       ->setHeader(pht('Phabricator User'))
       ->setSubhead(pht('Confirmed your account.'))
-      ->setQuality(PHUIBadgeView::POOR)
+      ->setQuality(PhabricatorBadgesQuality::POOR)
       ->setSource(pht('People (automatic)'))
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('212 Issued (100%)');
@@ -63,7 +63,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-code')
       ->setHeader(pht('Code Contributor'))
       ->setSubhead(pht('Wrote code that was acceptable'))
-      ->setQuality(PHUIBadgeView::COMMON)
+      ->setQuality(PhabricatorBadgesQuality::COMMON)
       ->setSource('Diffusion (automatic)')
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('200 Awarded (98%)');
@@ -72,7 +72,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-bug')
       ->setHeader(pht('Task Master'))
       ->setSubhead(pht('Closed over 100 tasks'))
-      ->setQuality(PHUIBadgeView::UNCOMMON)
+      ->setQuality(PhabricatorBadgesQuality::UNCOMMON)
       ->setSource('Maniphest (automatic)')
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('56 Awarded (43%)');
@@ -81,7 +81,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-star')
       ->setHeader(pht('Code Weaver'))
       ->setSubhead(pht('Landed 1,000 Commits'))
-      ->setQuality(PHUIBadgeView::RARE)
+      ->setQuality(PhabricatorBadgesQuality::RARE)
       ->setSource('Diffusion (automatic)')
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('42 Awarded (20%)');
@@ -90,7 +90,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-users')
       ->setHeader(pht('Security Team'))
       ->setSubhead(pht('<script>alert(1);</script>'))
-      ->setQuality(PHUIBadgeView::EPIC)
+      ->setQuality(PhabricatorBadgesQuality::EPIC)
       ->setSource('Projects (automatic)')
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('21 Awarded (10%)');
@@ -99,7 +99,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-user')
       ->setHeader(pht('Adminstrator'))
       ->setSubhead(pht('Drew the short stick'))
-      ->setQuality(PHUIBadgeView::LEGENDARY)
+      ->setQuality(PhabricatorBadgesQuality::LEGENDARY)
       ->setSource(pht('People (automatic)'))
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('3 Awarded (1.4%)');
@@ -108,7 +108,7 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
       ->setIcon('fa-compass')
       ->setHeader(pht('Lead Developer'))
       ->setSubhead(pht('Lead Developer of Phabricator'))
-      ->setQuality(PHUIBadgeView::HEIRLOOM)
+      ->setQuality(PhabricatorBadgesQuality::HEIRLOOM)
       ->setSource(pht('Direct Award (epriestley)'))
       ->addByline(pht('Dec 31, 1969'))
       ->addByline('1 Awarded (0.4%)');
@@ -129,17 +129,17 @@ final class PHUIBadgeExample extends PhabricatorUIExample {
     $badges3[] = id(new PHUIBadgeMiniView())
       ->setIcon('fa-heart')
       ->setHeader(pht('Funder'))
-      ->setQuality(PHUIBadgeView::UNCOMMON);
+      ->setQuality(PhabricatorBadgesQuality::UNCOMMON);
 
     $badges3[] = id(new PHUIBadgeMiniView())
       ->setIcon('fa-user')
       ->setHeader(pht('Administrator'))
-      ->setQuality(PHUIBadgeView::RARE);
+      ->setQuality(PhabricatorBadgesQuality::RARE);
 
     $badges3[] = id(new PHUIBadgeMiniView())
       ->setIcon('fa-camera-retro')
       ->setHeader(pht('Designer'))
-      ->setQuality(PHUIBadgeView::EPIC);
+      ->setQuality(PhabricatorBadgesQuality::EPIC);
 
     $flex1 = new PHUIBadgeBoxView();
     $flex1->addItems($badges1);

src/applications/uiexample/examples/PHUIBoxExample.php

@@ -75,7 +75,7 @@ final class PHUIBoxExample extends PhabricatorUIExample {
     $badge2 = id(new PHUIBadgeMiniView())
       ->setIcon('fa-heart')
       ->setHeader(pht('Funder'))
-      ->setQuality(PHUIBadgeView::UNCOMMON);
+      ->setQuality(PhabricatorBadgesQuality::UNCOMMON);
 
     $header = id(new PHUIHeaderView())
       ->setHeader(pht('Fancy Box'))

src/applications/uiexample/examples/PHUITimelineExample.php

@@ -1,5 +1,4 @@
 <?php
-
 final class PHUITimelineExample extends PhabricatorUIExample {
 
   public function getName() {
@@ -24,12 +23,12 @@ final class PHUITimelineExample extends PhabricatorUIExample {
     $designer = id(new PHUIBadgeMiniView())
       ->setIcon('fa-camera-retro')
       ->setHeader(pht('Designer'))
-      ->setQuality(PHUIBadgeView::EPIC);
+      ->setQuality(PhabricatorBadgesQuality::EPIC);
 
     $admin = id(new PHUIBadgeMiniView())
       ->setIcon('fa-user')
       ->setHeader(pht('Administrator'))
-      ->setQuality(PHUIBadgeView::RARE);
+      ->setQuality(PhabricatorBadgesQuality::RARE);
 
     $events = array();
 

src/docs/book/user.book

@@ -32,6 +32,9 @@
     "conduit": {
       "name": "API Documentation"
     },
+    "cluster": {
+      "name": "Cluster Configuration"
+    },
     "fieldmanual": {
       "name": "Field Manuals"
     },

src/docs/user/cluster/cluster.diviner

@@ -0,0 +1,289 @@
+@title Clustering Introduction
+@group cluster
+
+Guide to configuring Phabricator across multiple hosts for availability and
+performance.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+Phabricator can be configured to run on multiple hosts with redundant services
+to improve its availability and scalability, and make disaster recovery much
+easier.
+
+Clustering is more complex to setup and maintain than running everything on a
+single host, but greatly reduces the cost of recovering from hardware and
+network failures.
+
+Each Phabricator service has an array of clustering options that can be
+configured independently. Configuring a cluster is inherently complex, and this
+is an advanced feature aimed at installs with large userbases and experienced
+operations personnel who need this high degree of flexibility.
+
+The remainder of this document summarizes how to add redundancy to each
+service and where your efforts are likely to have the greatest impact.
+
+For additional guidance on setting up a cluster, see "Overlaying Services"
+and "Cluster Recipes" at the bottom of this document.
+
+
+Preparing for Clustering
+========================
+
+To begin deploying Phabricator in cluster mode, set up `cluster.addresses`
+in your configuration.
+
+This option should contain a list of network address blocks which are considered
+to be part of the cluster. Hosts in this list are allowed to bend (or even
+break) some of the security and policy rules when they make requests to other
+hosts in the cluster, so this list should be as small as possible. See "Cluster
+Whitelist Security" below for discussion.
+
+If you are deploying hardware in EC2, a reasonable approach is to launch a
+dedicated Phabricator VPC, whitelist the whole VPC as a Phabricator cluster,
+and then deploy only Phabricator services into that VPC.
+
+If you have additional auxiliary hosts which run builds and tests via Drydock,
+you should //not// include them in the cluster address definition. For more
+detailed discussion of the Drydock security model, see @{Drydock User Guide:
+Security}.
+
+Most other clustering features will not work until you define a cluster by
+configuring `cluster.addresses`.
+
+
+Cluster Whitelist Security
+========================
+
+When you configure `cluster.addresses`, you should keep the list of trusted
+cluster hosts as small as possible. Hosts on this list gain additional
+capabilities, including these:
+
+**Trusted HTTP Headers**: Normally, Phabricator distrusts the load balancer
+HTTP headers `X-Forwarded-For` and `X-Forwarded-Proto` because they may be
+client-controlled and can be set to arbitrary values by an attacker if no load
+balancer is deployed. In particular, clients can set `X-Forwarded-For` to any
+value and spoof traffic from arbitrary remotes.
+
+These headers are trusted when they are received from a host on the cluster
+address whitelist. This allows requests from cluster loadbalancers to be
+interpreted correctly by default without requiring additional custom code or
+configuration.
+
+**Intracluster HTTP**: Requests from cluster hosts are not required to use
+HTTPS, even if `security.require-https` is enabled, because it is common to
+terminate HTTPS on load balancers and use plain HTTP for requests within a
+cluster.
+
+**Special Authentication Mechanisms**: Cluster hosts are allowed to connect to
+other cluster hosts with "root credentials", and to impersonate any user
+account.
+
+The use of root credentials is required because the daemons must be able to
+bypass policies in order to function properly: they need to send mail about
+private conversations and import commits in private repositories.
+
+The ability to impersonate users is required because SSH nodes must receive,
+interpret, modify, and forward SSH traffic. They can not use the original
+credentials to do this because SSH authentication is asymmetric and they do not
+have the user's private key. Instead, they use root credentials and impersonate
+the user within the cluster.
+
+These mechanisms are still authenticated (and use asymmetric keys, like SSH
+does), so access to a host in the cluster address block does not mean that an
+attacker can immediately compromise the cluster. However, an over-broad cluster
+address whitelist may give an attacker who gains some access additional tools
+to escalate access.
+
+Note that if an attacker gains access to an actual cluster host, these extra
+powers are largely moot. Most cluster hosts must be able to connect to the
+master database to function properly, so the attacker will just do that and
+freely read or modify whatever data they want.
+
+
+Cluster: Databases
+=================
+
+Configuring multiple database hosts is moderately complex, but normally has the
+highest impact on availability and resistance to data loss. This is usually the
+most important service to make redundant if your focus is on availability and
+disaster recovery.
+
+Configuring replicas allows Phabricator to run in read-only mode if you lose
+the master and to quickly promote the replica as a replacement.
+
+For details, see @{article:Cluster: Databases}.
+
+
+Cluster: Repositories
+=====================
+
+Configuring multiple repository hosts is complex, but is required before you
+can add multiple daemon or web hosts.
+
+Repository replicas are important for availability if you host repositories
+on Phabricator, but less important if you host repositories elsewhere
+(instead, you should focus on making that service more available).
+
+The distributed nature of Git and Mercurial tend to mean that they are
+naturally somewhat resistant to data loss: every clone of a repository includes
+the entire history.
+
+Repositories may become a scalability bottleneck, although this is rare unless
+your install has an unusually heavy repository read volume. Slow clones/fetches
+may hint at a repository capacity problem. Adding more repository hosts will
+provide an approximately linear increase in capacity.
+
+For details, see @{article:Cluster: Repositories}.
+
+
+Cluster: Daemons
+================
+
+Configuring multiple daemon hosts is straightforward, but you must configure
+repositories first.
+
+With daemons running on multiple hosts, you can transparently survive the loss
+of any subset of hosts without an interruption to daemon services, as long as
+at least one host remains alive. Daemons are stateless, so spreading daemons
+across multiple hosts provides no resistance to data loss.
+
+Daemons can become a bottleneck, particularly if your install sees a large
+volume of write traffic to repositories. If the daemon task queue has a
+backlog, that hints at a capacity problem. If existing hosts have unused
+resources, increase `phd.taskmasters` until they are fully utilized. From
+there, adding more daemon hosts will provide an approximately linear increase
+in capacity.
+
+For details, see @{article:Cluster: Daemons}.
+
+
+Cluster: Web Servers
+====================
+
+Configuring multiple web hosts is straightforward, but you must configure
+repositories first.
+
+With multiple web hosts, you can transparently survive the loss of any subset
+of hosts as long as at least one host remains alive. Web hosts are stateless,
+so putting multiple hosts in service provides no resistance to data loss
+because no data is at risk.
+
+Web hosts can become a bottleneck, particularly if you have a workload that is
+heavily focused on reads from the web UI (like a public install with many
+anonymous users). Slow responses to web requests may hint at a web capacity
+problem. Adding more hosts will provide an approximately linear increase in
+capacity.
+
+For details, see @{article:Cluster: Web Servers}.
+
+
+Cluster: Notifications
+======================
+
+Configuring multiple notification hosts is simple and has no pre-requisites.
+
+With multiple notification hosts, you can survive the loss of any subset of
+hosts as long as at least one host remains alive. Service may be breifly
+disrupted directly after the incident which destroys the other hosts.
+
+Notifications are noncritical, so this normally has little practical impact
+on service availability. Notifications are also stateless, so clustering this
+service provides no resistance to data loss because no data is at risk.
+
+Notification delivery normally requires very few resources, so adding more
+hosts is unlikely to have much impact on scalability.
+
+For details, see @{article:Cluster: Notifications}.
+
+
+Overlaying Services
+===================
+
+Although hosts can run a single dedicated service type, certain groups of
+services work well together. Phabricator clusters usually do not need to be
+very large, so deploying a small number of hosts with multiple services is a
+good place to start.
+
+In planning a cluster, consider these blended host types:
+
+**Everything**: Run HTTP, SSH, MySQL, notifications, repositories and daemons
+on a single host. This is the starting point for single-node setups, and
+usually also the best configuration when adding the second node.
+
+**Everything Except Databases**: Run HTTP, SSH, notifications, repositories and
+daemons on one host, and MySQL on a different host. MySQL uses many of the same
+resources that other services use. It's also simpler to separate than other
+services, and tends to benefit the most from dedicated hardware.
+
+**Repositories and Daemons**: Run repositories and daemons on the same host.
+Repository hosts //must// run daemons, and it normally makes sense to
+completely overlay repositories and daemons. These services tend to use
+different resources (repositories are heavier on I/O and lighter on CPU/RAM;
+daemons are heavier on CPU/RAM and lighter on I/O).
+
+Repositories and daemons are also both less latency sensitive than other
+service types, so there's a wider margin of error for under provisioning them
+before performance is noticeably affected.
+
+These nodes tend to use system resources in a balanced way. Individual nodes
+in this class do not need to be particularly powerful.
+
+**Frontend Servers**: Run HTTP and SSH on the same host. These are easy to set
+up, stateless, and you can scale the pool up or down easily to meet demand.
+Routing both types of ingress traffic through the same initial tier can
+simplify load balancing.
+
+These nodes tend to need relatively little RAM.
+
+
+Cluster Recipes
+===============
+
+This section provides some guidance on reasonable ways to scale up a cluster.
+
+The smallest possible cluster is **two hosts**. Run everything (web, ssh,
+database, notifications, repositories, and daemons) on each host. One host will
+serve as the master; the other will serve as a replica.
+
+Ideally, you should physically separate these hosts to reduce the chance that a
+natural disaster or infrastructure disruption could disable or destroy both
+hosts at the same time.
+
+From here, you can choose how you expand the cluster.
+
+To improve **scalability and performance**, separate loaded services onto
+dedicated hosts and then add more hosts of that type to increase capacity. If
+you have a two-node cluster, the best way to improve scalability by adding one
+host is likely to separate the master database onto its own host.
+
+Note that increasing scale may //decrease// availability by leaving you with
+too little capacity after a failure. If you have three hosts handling traffic
+and one datacenter fails, too much traffic may be sent to the single remaining
+host in the surviving datacenter. You can hedge against this by mirroring new
+hosts in other datacenters (for example, also separate the replica database
+onto its own host).
+
+After separating databases, separating repository + daemon nodes is likely
+the next step to consider.
+
+To improve **availability**, add another copy of everything you run in one
+datacenter to a new datacenter. For example, if you have a two-node cluster,
+the best way to improve availability is to run everything on a third host in a
+third datacenter. If you have a 6-node cluster with a web node, a database node
+and a repo + daemon node in two datacenters, add 3 more nodes to create a copy
+of each node in a third datacenter.
+
+You can continue adding hosts until you run out of hosts.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - learning how Phacility configures and operates a large, multi-tenant
+    production cluster in ((cluster)).

src/docs/user/cluster/cluster_daemons.diviner

@@ -0,0 +1,59 @@
+@title Cluster: Daemons
+@group intro
+
+Configuring Phabricator to use multiple daemon hosts.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can run daemons on multiple hosts. The advantages of doing this are:
+
+  - you can completely survive the loss of multiple daemon hosts; and
+  - worker queue throughput may improve.
+
+This configuration is simple, but you must configure repositories first. For
+details, see @{article:Cluster: Repositories}.
+
+Since repository hosts must run daemons anyway, you usually do not need to do
+any additional work and can skip this entirely.
+
+
+Adding Daemon Hosts
+===================
+
+After configuring repositories for clustering, launch daemons on every
+repository host according to the documentation in
+@{article:Cluster: Repositories}. These daemons are necessary: repositories
+will not fetch, update, or synchronize properly without them.
+
+If your repository clustering is redundant (you have at least two repository
+hosts), these daemons are also likely to be sufficient in most cases. If you
+want to launch additional hosts anyway (for example, to increase queue capacity
+for unusual workloads), see "Dedicated Daemon Hosts" below.
+
+
+Dedicated Daemon Hosts
+======================
+
+You can launch additional daemon hosts without any special configuration.
+Daemon hosts must be able to reach other hosts on the network, but do not need
+to run any services (like HTTP or SSH). Simply deploy the Phabricator software
+and configuration and start the daemons.
+
+Normally, there is little reason to deploy dedicated daemon hosts. They can
+improve queue capacity, but generally do not improve availability or increase
+resistance to data loss on their own. Instead, consider deploying more
+repository hosts: repository hosts run daemons, so this will increase queue
+capacity but also improve repository availability and cluster resistance.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - returning to @{article:Clustering Introduction}; or
+  - configuring repositories first with @{article:Cluster: Repositories}.

src/docs/user/cluster/cluster_databases.diviner

@@ -0,0 +1,327 @@
+@title Cluster: Databases
+@group intro
+
+Configuring Phabricator to use multiple database hosts.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can deploy Phabricator with multiple database hosts, configured as a master
+and a set of replicas. The advantages of doing this are:
+
+  - faster recovery from disasters by promoting a replica;
+  - graceful degradation if the master fails;
+  - reduced load on the master; and
+  - some tools to help monitor and manage replica health.
+
+This configuration is complex, and many installs do not need to pursue it.
+
+Phabricator can not currently be configured into a multi-master mode, nor can
+it be configured to automatically promote a replica to become the new master.
+
+If you lose the master, Phabricator can degrade automatically into read-only
+mode and remain available, but can not fully recover without operational
+intervention unless the master recovers on its own.
+
+
+Setting up MySQL Replication
+============================
+
+TODO: Write this section.
+
+
+Configuring Replicas
+====================
+
+Once your replicas are in working order, tell Phabricator about them by
+configuring the `cluster.databases` option. This option must be configured from
+the command line or in configuration files because Phabricator needs to read
+it //before// it can connect to databases.
+
+This option value will list all of the database hosts that you want Phabricator
+to interact with: your master and all your replicas. Each entry in the list
+should have these keys:
+
+  - `host`: //Required string.// The database host name.
+  - `role`: //Required string.// The cluster role of this host, one of
+    `master` or `replica`.
+  - `port`: //Optional int.// The port to connect to. If omitted, the default
+    port from `mysql.port` will be used.
+  - `user`: //Optional string.// The MySQL username to use to connect to this
+    host. If omitted, the default from `mysql.user` will be used.
+  - `pass`: //Optional string.// The password to use to connect to this host.
+    If omitted, the default from `mysql.pass` will be used.
+  - `disabled`: //Optional bool.// If set to `true`, Phabricator will not
+    connect to this host. You can use this to temporarily take a host out
+    of service.
+
+When `cluster.databases` is configured the `mysql.host` option is not used.
+The other MySQL connection configuration options (`mysql.port`, `mysql.user`,
+`mysql.pass`) are used only to provide defaults.
+
+Once you've configured this option, restart Phabricator for the changes to take
+effect, then continue to "Monitoring Replicas" to verify the configuration.
+
+
+Monitoring Replicas
+===================
+
+You can monitor replicas in {nav Config > Database Servers}. This interface
+shows you a quick overview of replicas and their health, and can detect some
+common issues with replication.
+
+The table on this page shows each database and current status.
+
+NOTE: This page runs its diagnostics //from the web server that is serving the
+request//. If you are recovering from a disaster, the view this page shows
+may be partial or misleading, and two requests served by different servers may
+see different views of the cluster.
+
+**Connection**: Phabricator tries to connect to each configured database, then
+shows the result in this column. If it fails, a brief diagnostic message with
+details about the error is shown. If it succeeds, the column shows a rough
+measurement of latency from the current webserver to the database.
+
+**Replication**: This is a summary of replication status on the database. If
+things are properly configured and stable, the replicas should be actively
+replicating and no more than a few seconds behind master, and the master
+should //not// be replicating from another database.
+
+To report this status, the user Phabricator is connecting as must have the
+`REPLICATION CLIENT` privilege (or the `SUPER` privilege) so it can run the
+`SHOW SLAVE STATUS` command. The `REPLICATION CLIENT` privilege only enables
+the user to run diagnostic commands so it should be reasonable to grant it in
+most cases, but it is not required. If you choose not to grant it, this page
+can not show any useful diagnostic information about replication status but
+everything else will still work.
+
+If a replica is more than a second behind master, this page will show the
+current replication delay. If the replication delay is more than 30 seconds,
+it will report "Slow Replication" with a warning icon.
+
+If replication is delayed, data is at risk: if you lose the master and can not
+later recover it (for example, because a meteor has obliterated the datacenter
+housing the physical host), data which did not make it to the replica will be
+lost forever.
+
+Beyond the risk of data loss, any read-only traffic sent to the replica will
+see an older view of the world which could be confusing for users: it may
+appear that their data has been lost, even if it is safe and just hasn't
+replicated yet.
+
+Phabricator will attempt to prevent clients from seeing out-of-date views, but
+sometimes sending traffic to a delayed replica is the best available option
+(for example, if the master can not be reached).
+
+**Health**: This column shows the result of recent health checks against the
+server. After several checks in a row fail, Phabricator will mark the server
+as unhealthy and stop sending traffic to it until several checks in a row
+later succeed.
+
+Note that each web server tracks database health independently, so if you have
+several servers they may have different views of database health. This is
+normal and not problematic.
+
+For more information on health checks, see "Unreachable Masters" below.
+
+**Messages**: This column has additional details about any errors shown in the
+other columns. These messages can help you understand or resolve problems.
+
+
+Testing Replicas
+================
+
+To test that your configuration can survive a disaster, turn off the master
+database. Do this with great ceremony, making a cool explosion sound as you
+run the `mysqld stop` command.
+
+If things have been set up properly, Phabricator should degrade to a temporary
+read-only mode immediately. After a brief period of unresponsiveness, it will
+degrade further into a longer-term read-only mode. For details on how this
+works internally, see "Unreachable Masters" below.
+
+Once satisfied, turn the master back on. After a brief delay, Phabricator
+should recognize that the master is healthy again and recover fully.
+
+Throughout this process, the {nav Database Servers} console will show a
+current view of the world from the perspective of the web server handling the
+request. You can use it to monitor state.
+
+You can perform a more narrow test by enabling `cluster.read-only` in
+configuration. This will put Phabricator into read-only mode immediately
+without turning off any databases.
+
+You can use this mode to understand which capabilities will and will not be
+available in read-only mode, and make sure any information you want to remain
+accessible in a disaster (like wiki pages or contact information) is really
+accessible.
+
+See the next section, "Degradation to Read Only Mode", for more details about
+when, why, and how Phabricator degrades.
+
+If you run custom code or extensions, they may not accommodate read-only mode
+properly. You should specifically test that they function correctly in
+read-only mode and do not prevent you from accessing important information.
+
+
+Degradation to Read-Only Mode
+=============================
+
+Phabricator will degrade to read-only mode when any of these conditions occur:
+
+  - you turn it on explicitly;
+  - you configure cluster mode, but don't set up any masters;
+  - the master can not be reached while handling a request; or
+  - recent attempts to connect to the master have consistently failed.
+
+When Phabricator is running in read-only mode, users can still read data and
+browse and clone repositories, but they can not edit, update, or push new
+changes. For example, users can still read disaster recovery information on
+the wiki or emergency contact information on user profiles.
+
+You can enable this mode explicitly by configuring `cluster.read-only`. Some
+reasons you might want to do this include:
+
+  - to test that the mode works like you expect it to;
+  - to make sure that information you need will be available;
+  - to prevent new writes while performing database maintenance; or
+  - to permanently archive a Phabricator install.
+
+You can also enable this mode implicitly by configuring `cluster.databases`
+but disabling the master, or by not specifying any host as a master. This may
+be more convenient than turning it on explicitly during the course of
+operations work.
+
+If Phabricator is unable to reach the master database, it will degrade into
+read-only mode automatically. See "Unreachable Masters" below for details on
+how this process works.
+
+If you end up in a situation where you have lost the master and can not get it
+back online (or can not restore it quickly) you can promote a replica to become
+the new master. See the next section, "Promoting a Replica", for details.
+
+
+Promoting a Replica
+===================
+
+TODO: Write this section.
+
+
+Unreachable Masters
+===================
+
+This section describes how Phabricator determines that a master has been lost,
+marks it unreachable, and degrades into read-only mode.
+
+Phabricator degrades into read-only mode automatically in two ways: very
+briefly in response to a single connection failure, or more permanently in
+response to a series of connection failures.
+
+In the first case, if a request needs to connect to the master but is not able
+to, Phabricator will temporarily degrade into read-only mode for the remainder
+of that request. The alternative is to fail abruptly, but Phabricator can
+sometimes degrade successfully and still respond to the user's request, so it
+makes an effort to finish serving the request from replicas.
+
+If the request was a write (like posting a comment) it will fail anyway, but
+if it was a read that did not actually need to use the master it may succeed.
+
+This temporary mode is intended to recover as gracefully as possible from brief
+interruptions in service (a few seconds), like a server being restarted, a
+network link becoming temporarily unavailable, or brief periods of load-related
+disruption. If the anomaly is temporary, Phabricator should recover immediately
+(on the next request once service is restored).
+
+This mode can be slow for users (they need to wait on connection attempts to
+the master which fail) and does not reduce load on the master (requests still
+attempt to connect to it).
+
+The second way Phabricator degrades is by running periodic health checks
+against databases, and marking them unhealthy if they fail over a longer period
+of time. This mechanism is very similar to the health checks that most HTTP
+load balancers perform against web servers.
+
+If a database fails several health checks in a row, Phabricator will mark it as
+unhealthy and stop sending all traffic (except for more health checks) to it.
+This improves performance during a service interruption and reduces load on the
+master, which may help it recover from load problems.
+
+You can monitor the status of health checks in the {nav Database Servers}
+console. The "Health" column shows how many checks have run recently and
+how many have succeeded.
+
+Health checks run every 3 seconds, and 5 checks in a row must fail or succeed
+before Phabricator marks the database as healthy or unhealthy, so it will
+generally take about 15 seconds for a database to change state after it goes
+down or comes up.
+
+If all of the recent checks fail, Phabricator will mark the database as
+unhealthy and stop sending traffic to it. If the master was the database that
+was marked as unhealthy, Phabricator will actively degrade into read-only mode
+until it recovers.
+
+This mode only attempts to connect to the unhealthy database once every few
+seconds to see if it is recovering, so performance will be better on average
+(users rarely need to wait for bad connections to fail or time out) and the
+database will receive less load.
+
+Once all of the recent checks succeed, Phabricator will mark the database as
+healthy again and continue sending traffic to it.
+
+Health checks are tracked individually for each web server, so some web servers
+may see a host as healthy while others see it as unhealthy. This is normal, and
+can accurately reflect the state of the world: for example, the link between
+datacenters may have been lost, so hosts in one datacenter can no longer see
+the master, while hosts in the other datacenter still have a healthy link to
+it.
+
+
+Backups
+======
+
+Even if you configure replication, you should still retain separate backup
+snapshots. Replicas protect you from data loss if you lose a host, but they do
+not let you recover from data mutation mistakes.
+
+If something issues `DELETE` or `UPDATE` statements and destroys data on the
+master, the mutation will propagate to the replicas almost immediately and the
+data will be gone forever. Normally, the only way to recover this data is from
+backup snapshots.
+
+Although you should still have a backup process, your backup process can
+safely pull dumps from a replica instead of the master. This operation can
+be slow, so offloading it to a replica can make the performance of the master
+more consistent.
+
+To dump from a replica, you can use `bin/storage dump --host <host>` to
+control which host the command connects to. (You may still want to execute
+this command //from// that host, to avoid sending the whole dump over the
+network).
+
+With the `--for-replica` flag, the `bin/storage dump` command creates dumps
+with `--master-data`, which includes a `CHANGE MASTER` statement in the output.
+This may be helpful when initially setting up new replicas, as it can make it
+easier to change the binlog coordinates to the correct position for the dump.
+
+With recent versions of MySQL, it is also possible to configure a //delayed//
+replica which intentionally lags behind the master (say, by 12 hours). In the
+event of a bad mutation, this could give you a larger window of time to
+recognize the issue and recover the lost data from the delayed replica (which
+might be quick) without needing to restore backups (which might be very slow).
+
+Delayed replication is outside the scope of this document, but may be worth
+considering as an additional data security step on top of backup snapshots
+depending on your resources and needs. If you configure a delayed replica, do
+not add it to the `cluster.databases` configuration: Phabricator should never
+send traffic to it, and does not need to know about it.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - returning to @{article:Clustering Introduction}.

src/docs/user/cluster/cluster_notifications.diviner

@@ -0,0 +1,174 @@
+@title Cluster: Notifications
+@group intro
+
+Configuring Phabricator to use multiple notification servers.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can run multiple notification servers. The advantages of doing this
+are:
+
+  - you can completely survive the loss of any subset so long as one
+    remains standing; and
+  - performance and capacity may improve.
+
+This configuration is relatively simple, but has a small impact on availability
+and does nothing to increase resitance to data loss.
+
+
+Clustering Design Goals
+=======================
+
+Notification clustering aims to restore service automatically after the loss
+of some nodes. It does **not** attempt to guarantee that every message is
+delivered.
+
+Notification messages provide timely information about events, but they are
+never authoritative and never the only way for users to learn about events.
+For example, if a notification about a task update is not delivered, the next
+page you load will still show the notification in your notification menu.
+
+Generally, Phabricator works fine without notifications configured at all, so
+clustering assumes that losing some messages during a disruption is acceptable.
+
+
+How Clustering Works
+====================
+
+Notification clustering is very simple: notification servers relay every
+message they receive to a list of peers.
+
+When you configure clustering, you'll run multiple servers and tell them that
+the other servers exist. When any server receives a message, it retransmits it
+to all the severs it knows about.
+
+When a server is lost, clients will automatically reconnect after a brief
+delay. They may lose some notifications while their client is reconnecting,
+but normally this should only last for a few seconds.
+
+
+Configuring Aphlict
+===================
+
+To configure clustering on the server side, add a `cluster` key to your
+Aphlict configuration file. For more details about configuring Aphlict,
+see @{article:Notifications User Guide: Setup and Configuration}.
+
+The `cluster` key should contain a list of `"admin"` server locations. Every
+message the server receives will be retransmitted to all nodes in the list.
+
+The server is smart enough to avoid sending messages in a cycle, and to avoid
+sending messages to itself. You can safely list every server you run in the
+configuration file, including the current server.
+
+You do not need to configure servers in an acyclic graph or only list //other//
+servers: just list everything on every server and Aphlict will figure things
+out from there.
+
+A simple example with two servers might look like this:
+
+```lang=json, name="aphlict.json (Cluster)"
+{
+  ...
+  "cluster": [
+    {
+      "host": "notify001.mycompany.com",
+      "port": 22281,
+      "protocol": "http"
+    },
+    {
+      "host": "notify002.mycompany.com",
+      "port": 22281,
+      "protocol": "http"
+    }
+  ]
+  ...
+}
+```
+
+
+Configuring Phabricator
+=======================
+
+To configure clustering on the client side, add every service you run to
+`notification.servers`. Generally, this will be twice as many entries as
+you run actual servers, since each server runs a `"client"` service and an
+`"admin"` service.
+
+A simple example with the two servers above (providing four total services)
+might look like this:
+
+```lang=json, name="notification.servers (Cluster)"
+[
+  {
+    "type": "client",
+    "host": "notify001.mycompany.com",
+    "port": 22280,
+    "protocol": "https"
+  },
+  {
+    "type": "client",
+    "host": "notify002.mycompany.com",
+    "port": 22280,
+    "protocol": "https"
+  },
+  {
+    "type": "admin",
+    "host": "notify001.mycompany.com",
+    "port": 22281,
+    "protocol": "http"
+  },
+  {
+    "type": "admin",
+    "host": "notify002.mycompany.com",
+    "port": 22281,
+    "protocol": "http"
+  }
+]
+```
+
+If you put all of the `"client"` servers behind a load balancer, you would
+just list the load balancer and let it handle pulling nodes in and out of
+service.
+
+```lang=json, name="notification.servers (Cluster + Load Balancer)"
+[
+  {
+    "type": "client",
+    "host": "notify-lb.mycompany.com",
+    "port": 22280,
+    "protocol": "https"
+  },
+  {
+    "type": "admin",
+    "host": "notify001.mycompany.com",
+    "port": 22281,
+    "protocol": "http"
+  },
+  {
+    "type": "admin",
+    "host": "notify002.mycompany.com",
+    "port": 22281,
+    "protocol": "http"
+  }
+]
+```
+
+Notification hosts do not need to run any additional services, although they
+are free to do so. The notification server generally consumes few resources
+and is resistant to most other loads on the machine, so it's reasonable to
+overlay these on top of other services wherever it is convenient.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - reviewing notification configuration with
+    @{article:Notifications User Guide: Setup and Configuration}; or
+  - returning to @{article:Clustering Introduction}.

src/docs/user/cluster/cluster_repositories.diviner

@@ -0,0 +1,112 @@
+@title Cluster: Repositories
+@group intro
+
+Configuring Phabricator to use multiple repository hosts.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+If you use Git or Mercurial, you can deploy Phabricator with multiple
+repository hosts, configured so that each host is readable and writable. The
+advantages of doing this are:
+
+  - you can completely survive the loss of repository hosts;
+  - reads and writes can scale across multiple machines; and
+  - read and write performance across multiple geographic regions may improve.
+
+This configuration is complex, and many installs do not need to pursue it.
+
+This configuration is not currently supported with Subversion.
+
+
+Repository Hosts
+================
+
+Repository hosts must run a complete, fully configured copy of Phabricator,
+including a webserver. If you make repositories available over SSH, they must
+also run a properly configured `sshd`.
+
+Generally, these hosts will run the same set of services and configuration that
+web hosts run. If you prefer, you can overlay these services and put web and
+repository services on the same hosts.
+
+When a user requests information about a repository that can only be satisfied
+by examining a repository working copy, the webserver receiving the request
+will make an HTTP service call to a repository server which hosts the
+repository to retrieve the data it needs. It will use the result of this query
+to respond to the user.
+
+
+How Reads and Writes Work
+=========================
+
+Phabricator repository replicas are multi-master: every node is readable and
+writable, and a cluster of nodes can (almost always) survive the loss of any
+arbitrary subset of nodes so long as at least one node is still alive.
+
+Phabricator maintains an internal version for each repository, and increments
+it when the repository is mutated.
+
+Before responding to a read, replicas make sure their version of the repository
+is up to date (no node in the cluster has a newer version of the repository).
+If it isn't, they block the read until they can complete a fetch.
+
+Before responding to a write, replicas obtain a global lock, perform the same
+version check and fetch if necessary, then allow the write to continue.
+
+
+HTTP vs HTTPS
+=============
+
+Intracluster requests (from the daemons to repository servers, or from
+webservers to repository servers) are permitted to use HTTP, even if you have
+set `security.require-https` in your configuration.
+
+It is common to terminate SSL at a load balancer and use plain HTTP beyond
+that, and the `security.require-https` feature is primarily focused on making
+client browser behavior more convenient for users, so it does not apply to
+intracluster traffic.
+
+Using HTTP within the cluster leaves you vulnerable to attackers who can
+observe traffic within a datacenter, or observe traffic between datacenters.
+This is normally very difficult, but within reach for state-level adversaries
+like the NSA.
+
+If you are concerned about these attackers, you can terminate HTTPS on
+repository hosts and bind to them with the "https" protocol. Just be aware that
+the `security.require-https` setting won't prevent you from making
+configuration mistakes, as it doesn't cover intracluster traffic.
+
+Other mitigations are possible, but securing a network against the NSA and
+similar agents of other rogue nations is beyond the scope of this document.
+
+
+Backups
+======
+
+Even if you configure clustering, you should still consider retaining separate
+backup snapshots. Replicas protect you from data loss if you lose a host, but
+they do not let you rewind time to recover from data mutation mistakes.
+
+If something issues a `--force` push that destroys branch heads, the mutation
+will propagate to the replicas.
+
+You may be able to manually restore the branches by using tools like the
+Phabricator push log or the Git reflog so it is less important to retain
+repository snapshots than database snapshots, but it is still possible for
+data to be lost permanently, especially if you don't notice the problem for
+some time.
+
+Retaining separate backup snapshots will improve your ability to recover more
+data more easily in a wider range of disaster situations.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - returning to @{article:Clustering Introduction}.

src/docs/user/cluster/cluster_webservers.diviner

@@ -0,0 +1,42 @@
+@title Cluster: Web Servers
+@group intro
+
+Configuring Phabricator to use multiple web servers.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can run Phabricator on multiple web servers. The advantages of doing this
+are:
+
+  - you can completely survive the loss of multiple web hosts; and
+  - performance and capacity may improve.
+
+This configuration is simple, but you must configure repositories first. For
+details, see @{article:Cluster: Repositories}.
+
+
+Adding Web Hosts
+================
+
+After configuring repositories in cluster mode, you can add more web hosts
+at any time: simply deploy the Phabricator software and configuration to a
+host, start the web server, and then add the host to the load balancer pool.
+
+Phabricator web servers are stateless, so you can pull them in and out of
+production freely.
+
+You may also want to run SSH services on these hosts, since the service is very
+similar to HTTP, also stateless, and it may be simpler to load balance the
+services together.
+
+
+Next Steps
+==========
+
+Continue by:
+
+  - returning to @{article:Clustering Introduction}.

src/docs/user/configuration/cluster.diviner

@@ -1,50 +0,0 @@
-@title User Guide: Phabricator Clusters
-@group config
-
-Guide on scaling Phabricator across multiple machines.
-
-Overview
-========
-
-IMPORTANT: Phabricator clustering is in its infancy and does not work at all
-yet. This document is mostly a placeholder.
-
-IMPORTANT: DO NOT CONFIGURE CLUSTER SERVICES UNLESS YOU HAVE **TWENTY YEARS OF
-EXPERIENCE WITH PHABRICATOR** AND **A MINIMUM OF 17 PHABRICATOR PHDs**. YOU
-WILL BREAK YOUR INSTALL AND BE UNABLE TO REPAIR IT.
-
-See also @{article:Almanac User Guide}.
-
-
-Managing Cluster Configuration
-==============================
-
-Cluster configuration is managed primarily from the **Almanac** application.
-
-To define cluster services and create or edit cluster configuration, you must
-have the **Can Manage Cluster Services** application permission in Almanac. If
-you do not have this permission, all cluster services and all connected devices
-will be locked and not editable.
-
-The **Can Manage Cluster Services** permission is stronger than service and
-device policies, and overrides them. You can never edit a cluster service if
-you don't have this permission, even if the **Can Edit** policy on the service
-itself is very permissive.
-
-
-Locking Cluster Configuration
-=============================
-
-IMPORTANT: Managing cluster services is **dangerous** and **fragile**.
-
-If you make a mistake, you can break your install. Because the install is
-broken, you will be unable to load the web interface in order to repair it.
-
-IMPORTANT: Currently, broken clusters must be repaired by manually fixing them
-in the database. There are no instructions available on how to do this, and no
-tools to help you. Do not configure cluster services.
-
-If an attacker gains access to an account with permission to manage cluster
-services, they can add devices they control as database servers. These servers
-will then receive sensitive data and traffic, and allow the attacker to
-escalate their access and completely compromise an install.

src/docs/user/configuration/managing_daemons.diviner

@@ -113,25 +113,16 @@ This daemon will daemonize and run normally.
   - See @{article:Diffusion User Guide} for details about tuning the repository
     daemon.
 
-== Multiple Machines ==
 
-If you have multiple machines, you should use `phd launch` to tweak which
+Multiple Hosts
-daemons launch, and split daemons across machines like this:
+==============
 
-  - `PhabricatorRepositoryPullLocalDaemon`: Run one copy on any machine.
+For information about running daemons on multiple hosts, see
-    On each web frontend which is not running a normal copy, run a copy
+@{article:Cluster: Daemons}.
-    with the `--no-discovery` flag.
-  - `PhabricatorTriggerDaemon`: Run one copy on any machine.
-  - `PhabricatorTaskmasterDaemon`: Run as many copies as you need to keep
-    tasks from backing up. You can run them all on one machine or split them
-    across machines.
 
-A gratuitously wasteful install might have a dedicated daemon machine which
-runs `phd start` with a large pool of taskmasters set in the config, and then
-runs `phd launch PhabricatorRepositoryPullLocalDaemon -- --no-discovery` on each
-web server. This is grossly excessive in normal cases.
 
-= Next Steps =
+Next Steps
+==========
 
 Continue by:
 

src/docs/user/configuration/notifications.diviner

@@ -11,13 +11,13 @@ tasks or commenting on code reviews) through email and in-application
 notifications.
 
 Phabricator can also be configured to deliver notifications in real time, by
-popping up a message in any open browser windows if something has
+popping up a message in any open browser windows if something has happened or
-happened or an object has been updated.
+an object has been updated.
 
 To enable real-time notifications:
 
-  - Set `notification.enabled` in your configuration to true.
+  - Configure and start the notification server, as described below.
-  - Run the notification server, as described below.
+  - Adjust `notification.servers` to point at it.
 
 This document describes the process in detail.
 
@@ -59,40 +59,104 @@ After installing Node.js, you can control the notification server with the
 
   phabricator/ $ bin/aphlict start
 
-The server must be able to listen on port **22280** for Aphlict to work. In
+By default, the server must be able to listen on port `22280`. If you're using
-particular, if you're running in EC2, you need to unblock this port in the
+a host firewall (like a security group in EC2), make sure traffic can reach the
-server's security group configuration. You can change this port in the
+server.
-`notification.client-uri` config.
 
-You may need to adjust these settings:
+The server configuration is controlled by a configuration file, which is
+separate from Phabricator's configuration settings. The default file can
+be found at `phabricator/conf/aphlict/aphlict.default.json`.
 
-  - `notification.ssl-cert` Point this at an SSL certificate for secure
+To make adjustments to the default configuration, either copy this file to
-    WebSockets.
+create `aphlict.custom.json` in the same directory (this file will be used if
-  - `notification.ssl-key` Point this at an SSL keyfile for secure WebSockets.
+it exists) or specify a configuration file explicitly with the `--config` flag:
 
-In particular, if your server uses HTTPS, you **must** configure these options.
+  phabricator/ $ bin/aphlict start --config path/to/config.json
-Browsers will not allow you to use non-SSL websockets from an SSL web page.
 
-You may also want to adjust these settings:
+The configuration file has these settings:
 
-  - `notification.client-uri` Externally-facing host and port that browsers will
+  - `servers`: //Required list.// A list of servers to start.
-    connect to in order to listen for notifications.
+  - `logs`: //Optional list.// A list of logs to write to.
-  - `notification.server-uri` Internally-facing host and port that Phabricator
+  - `cluster`: //Optional list.// A list of cluster peers. This is an advanced
-    will connect to in order to publish notifications.
+    feature.
-  - `notification.log` Log file location for the server.
+  - `pidfile`: //Required string.// Path to a PID file.
-  - `notification.pidfile` Pidfile location used to stop any running server when
+  - `memory.hint`: //Optional int.// Suggestion to `node` about how much
-    aphlict is restarted.
+    memory to use, via `--max-old-stack-size`. In most cases, this can be
+    left unspecified.
+
+Each server in the `servers` list should be an object with these keys:
+
+  - `type`: //Required string.// The type of server to start. Options are
+    `admin` or `client`. Normally, you should run one of each.
+  - `port`: //Required int.// The port this server should listen on.
+  - `listen`: //Optional string.// Which interface to bind to. By default,
+    the `admin` server is bound to `127.0.0.1` (so only other services on the
+    local machine can connect to it), while the `client` server is bound
+    to `0.0.0.0` (so any client can connect).
+  - `ssl.key`: //Optional string.// If you want to use SSL on this port,
+    the path to an SSL key.
+  - `ssl.cert`: //Optional string.// If you want to use SSL on this port,
+    the path to an SSL certificate.
+  - `ssl.chain`: //Optional string.// If you have configured SSL on this
+    port, an optional path to a certificate chain file.
+
+Each log in the `logs` list should be an object with these keys:
+
+  - `path`: //Required string.// Path to the log file.
+
+Each peer in the `cluster` list should be an object with these keys:
+
+  - `host`: //Required string.// The peer host address.
+  - `port`: //Required int.// The peer port.
+  - `protocol`: //Required string.// The protocol to connect with, one of
+    `"http"` or `"https"`.
+
+Cluster configuration is an advanced topic and can be omitted for most
+installs. For more information on how to configure a cluster, see
+@{article:Clustering Introduction} and @{article:Cluster: Notifications}.
+
+The defaults are appropriate for simple cases, but you may need to adjust them
+if you are running a more complex configuration.
+
+Configuring Phabricator
+=======================
+
+After starting the server, configure Phabricator to connect to it by adjusting
+`notification.servers`. This configuration option should have a list of servers
+that Phabricator should interact with.
+
+Normally, you'll list one client server and one admin server, like this:
+
+```lang=json
+[
+  {
+    "type": "client",
+    "host": "phabricator.mycompany.com",
+    "port": 22280,
+    "protocol": "https"
+  },
+  {
+    "type": "admin",
+    "host": "127.0.0.1",
+    "port": 22281,
+    "protocol": "http"
+  }
+]
+```
+
+This definition defines which services the user's browser will attempt to
+connect to. Most of the time, it will be very similar to the services defined
+in the Aphlict configuration. However, if you are sending traffic through a
+load balancer or terminating SSL somewhere before traffic reaches Aphlict,
+the services the browser connects to may need to have different hosts, ports
+or protocols than the underlying server listens on.
 
 
 Verifying Server Status
 =======================
 
-Access `/notification/status/` to verify the server is operational. You should
+After configuring `notification.servers`, navigate to
-see a table showing stats like "uptime" and connection/message counts if the
+{nav Config > Notification Servers} to verify that things are operational.
-server is working. If it isn't working, you should see an error.
-
-You can also send a test notification by clicking the button in the upper right
-corner of this screen.
 
 
 Troubleshooting
@@ -106,31 +170,61 @@ Because the notification server uses WebSockets, your browser error console
 may also have information that is useful in figuring out what's wrong.
 
 The server also generates a log, by default in `/var/log/aphlict.log`. You can
-change this location by changing `notification.log` in your configuration. The
+change this location by adjusting configuration. The log may contain
-log may contain information useful in resolving issues.
+information that is useful in resolving issues.
 
 
-Advanced Usage
+SSL and HTTPS
-==============
+=============
 
-It is possible to route the WebSockets traffic for Aphlict through a reverse
+If you serve Phabricator over HTTPS, you must also serve websockets over HTTPS.
-proxy such as `nginx` (see @{article:Configuration Guide} for instructions on
+Browsers will refuse to connect to `ws://` websockets from HTTPS pages.
-configuring `nginx`). In order to do this with `nginx`, you will require at
-least version 1.3. You can read some more information about using `nginx` with
-WebSockets at http://nginx.com/blog/websocket-nginx/.
 
-There are a few benefits of this approach:
+If a client connects to Phabricator over HTTPS, Phabricator will automatically
+select an appropriate HTTPS service from `notification.servers` and instruct
+the browser to open a websocket connection with `wss://`.
 
-  - SSL is terminated at the `nginx` layer and consequently there is no need to
+The simplest way to do this is configure Aphlict with an SSL key and
-    configure `notificaton.ssl-cert` and `notification.ssl-key` (in fact, with
+certificate and let it terminate SSL directly.
-    this approach you should //not// configure these options because otherwise
-    the Aphlict server will not accept HTTP traffic).
-  - You don't have to open up a separate port on the server.
-  - Clients don't need to be able to connect to Aphlict over a non-standard
-    port which may be blocked by a firewall or anti-virus software.
 
-The following files show an example `nginx` configuration. Note that this is an
+If you prefer not to do this, two other options are:
-example only and you may need to adjust this to suit your own setup.
+
+  - run the websocket through a websocket-capable loadbalancer and terminate
+    SSL there; or
+  - run the websocket through `nginx` over the same socket as the rest of
+    your web traffic.
+
+See the next sections for more detail.
+
+
+Terminating SSL with a Load Balancer
+====================================
+
+If you want to terminate SSL in front of the notification server with a
+traditional load balancer or a similar device, do this:
+
+  - Point `notification.servers` at your load balancer or reverse proxy,
+    specifying that the protocol is `https`.
+  - On the load balancer or proxy, terminate SSL and forward traffic to the
+    Aphlict server.
+  - In the Aphlict configuration, listen on the target port with `http`.
+
+
+Terminating SSL with Nginx
+==========================
+
+If you use `nginx`, you can send websocket traffic to the same port as normal
+HTTP traffic and have `nginx` proxy it selectively based on the request path.
+
+This requires `nginx` 1.3 or greater. See the `nginx` documentation for
+details:
+
+> http://nginx.com/blog/websocket-nginx/
+
+This is very complex, but allows you to support notifications without opening
+additional ports.
+
+An example `nginx` configuration might look something like this:
 
 ```lang=nginx, name=/etc/nginx/conf.d/connection_upgrade.conf
 map $http_upgrade $connection_upgrade {
@@ -163,8 +257,23 @@ server {
 }
 ```
 
-With this approach, you should set `notification.client-uri` to
+With this approach, you should make these additional adjustments:
-`http://localhost/ws/`. Additionally, there is no need for the Aphlict server
+
-to bind to `0.0.0.0` anymore (which is the default behavior), so you could
+**Phabricator Configuration**: The entry in `notification.servers` with type
-start the Aphlict server with `./bin/aphlict start --client-host=localhost`
+`"client"` should have these adjustments made:
-instead.
+
+  - Set `host` to the Phabricator host.
+  - Set `port` to the standard HTTPS port (usually `443`).
+  - Set `protocol` to `"https"`.
+  - Set `path` to `/ws/`, so it matches the special `location` in your
+    `nginx` config.
+
+You do not need to adjust the `"admin"` server.
+
+**Aphlict**: Your Aphlict configuration should make these adjustments to
+the `"client"` server:
+
+  - The `protocol` should be `"http"`: `nginx` will send plain HTTP traffic
+    to Aphlict.
+  - Optionally, you can `listen` on `127.0.0.1` instead of `0.0.0.0`, because
+    the server will no longer receive external traffic.

src/docs/user/userguide/arcanist_commit_ranges.diviner

@@ -193,12 +193,14 @@ or when it reaches the merge-base commit.
 
 This rule works well for trees that look like this:
 
+```
   |   *  Commit B1, on branch "subfeature" (HEAD)
   |  /
   | *    Commit A1, on branch "feature"
   |/
   *      Commit M1, on branch "master"
   |
+```
 
 This tree represents using feature branches to develop one feature ("feature"),
 and then creating a sub-branch to develop a dependent feature ("subfeature").
@@ -218,6 +220,7 @@ The rule will also do the right thing when run from "feature" in this case.
 However, this rule will select the wrong commit range in some cases. For
 instance, it will do the wrong thing in this tree:
 
+```
   |
   | *    Commit A2, on branch "feature" (HEAD)
   | |
@@ -227,6 +230,7 @@ instance, it will do the wrong thing in this tree:
   |/
   *      Commit M1, on branch "master"
   |
+```
 
 This tree represents making another commit (`A2`) on "feature", on top of `A1`.
 
@@ -240,6 +244,7 @@ commits, or by rebasing "subfeature" before running `arc diff`.
 
 This rule will also select the wrong commit range in a tree like this:
 
+```
   |
   | *    Commit A1', on branch "feature", created by amending A1
   | |
@@ -249,6 +254,7 @@ This rule will also select the wrong commit range in a tree like this:
   |/
   *      Commit M1, on branch "master"
   |
+```
 
 This tree represents amending `A1` without rebasing "subfeature", so that `A1`
 is no longer on "feature" (replaced with `A1'`) but still on "subfeature". In
@@ -269,6 +275,7 @@ This rule operates like `arc:outgoing`, but then walks the commits between
 `.` and the selected base commit. It stops when it encounters a bookmark. For
 example, if you have a tree like this:
 
+```
   |
   | *  C4 (outgoing, bookmark: stripes)
   | |
@@ -278,6 +285,7 @@ example, if you have a tree like this:
   |/
   *    C1 (pushed, no bookmark)
   |
+```
 
 When run from `C4`, this rule will select just `C4`, stopping on `C3` because
 it has a different bookmark. When run from `C3`, it will select `C2` and `C3`.