Update bin/storage workflows to accommodate multiple masters

Summary: Depends on D16847. Ref T11044. This updates the remaining storage-related workflows from the CLI to accommodate multiple masters.

Test Plan:
  - Configured multiple masters.
  - Ran all `bin/storage` workflows.
  - Ran `arc unit --everything`.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11044

Differential Revision: https://secure.phabricator.com/D16848

scripts/sql/manage_storage.php

@@ -34,7 +34,13 @@ try {
         'name' => 'host',
         'param' => 'hostname',
         'help' => pht(
-          'Connect to __host__ instead of the default host.'),
+          'Operate on the database server identified by __hostname__.'),
+      ),
+      array(
+        'name' => 'ref',
+        'param' => 'ref',
+        'help' => pht(
+          'Operate on the database identified by __ref__.'),
       ),
       array(
         'name'    => 'user',
@@ -81,120 +87,147 @@ try {
 // First, test that the Phabricator configuration is set up correctly. After
 // we know this works we'll test any administrative credentials specifically.
 
+$refs = PhabricatorDatabaseRef::getActiveDatabaseRefs();
+if (!$refs) {
+  throw new PhutilArgumentUsageException(
+    pht('No databases are configured.'));
+}
+
 $host = $args->getArg('host');
-if (strlen($host)) {
-  $ref = null;
-
-  $refs = PhabricatorDatabaseRef::getLiveRefs();
-
-  // Include the master in case the user is just specifying a redundant
-  // "--host" flag for no reason and does not actually have a database
-  // cluster configured.
-  foreach (PhabricatorDatabaseRef::getMasterDatabaseRefs() as $master_ref) {
-    $refs[] = $master_ref;
+$ref_key = $args->getArg('ref');
+if (strlen($host) || strlen($ref_key)) {
+  if ($host && $ref_key) {
+    throw new PhutilArgumentUsageException(
+      pht(
+        'Use "--host" or "--ref" to select a database, but not both.'));
   }
 
+  $refs = PhabricatorDatabaseRef::getActiveDatabaseRefs();
+
+  $possible_refs = array();
   foreach ($refs as $possible_ref) {
-    if ($possible_ref->getHost() == $host) {
-      $ref = $possible_ref;
+    if ($host && ($possible_ref->getHost() == $host)) {
+      $possible_refs[] = $possible_ref;
+      break;
+    }
+    if ($ref_key && ($possible_ref->getRefKey() == $ref_key)) {
+      $possible_refs[] = $possible_ref;
       break;
     }
   }
 
-  if (!$ref) {
+  if (!$possible_refs) {
+    if ($host) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'There is no configured database on host "%s". This command can '.
+          'only interact with configured databases.',
+          $host));
+    } else {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'There is no configured database with ref "%s". This command can '.
+          'only interact with configured databases.',
+          $ref_key));
+    }
+  }
+
+  if (count($possible_refs) > 1) {
     throw new PhutilArgumentUsageException(
       pht(
-        'There is no configured database on host "%s". This command can '.
-        'only interact with configured databases.',
+        'Host "%s" identifies more than one database. Use "--ref" to select '.
+        'a specific database.',
         $host));
   }
-} else {
-  $ref = PhabricatorDatabaseRef::getMasterDatabaseRef();
-  if (!$ref) {
-    throw new Exception(
-      pht('No database master is configured.'));
+
+  $refs = $possible_refs;
+}
+
+$apis = array();
+foreach ($refs as $ref) {
+  $default_user = $ref->getUser();
+  $default_host = $ref->getHost();
+  $default_port = $ref->getPort();
+
+  $test_api = id(new PhabricatorStorageManagementAPI())
+    ->setUser($default_user)
+    ->setHost($default_host)
+    ->setPort($default_port)
+    ->setPassword($ref->getPass())
+    ->setNamespace($args->getArg('namespace'));
+
+  try {
+    queryfx(
+      $test_api->getConn(null),
+      'SELECT 1');
+  } catch (AphrontQueryException $ex) {
+    $message = phutil_console_format(
+      "**%s**\n\n%s\n\n%s\n\n%s\n\n**%s**: %s\n",
+      pht('MySQL Credentials Not Configured'),
+      pht(
+        'Unable to connect to MySQL using the configured credentials. '.
+        'You must configure standard credentials before you can upgrade '.
+        'storage. Run these commands to set up credentials:'),
+      "  phabricator/ $ ./bin/config set mysql.host __host__\n".
+      "  phabricator/ $ ./bin/config set mysql.user __username__\n".
+      "  phabricator/ $ ./bin/config set mysql.pass __password__",
+      pht(
+        'These standard credentials are separate from any administrative '.
+        'credentials provided to this command with __%s__ or '.
+        '__%s__, and must be configured correctly before you can proceed.',
+        '--user',
+        '--password'),
+      pht('Raw MySQL Error'),
+      $ex->getMessage());
+    echo phutil_console_wrap($message);
+    exit(1);
   }
-}
 
-$default_user = $ref->getUser();
-$default_host = $ref->getHost();
-$default_port = $ref->getPort();
+  if ($args->getArg('password') === null) {
+    // This is already a PhutilOpaqueEnvelope.
+    $password = $ref->getPass();
+  } else {
+    // Put this in a PhutilOpaqueEnvelope.
+    $password = new PhutilOpaqueEnvelope($args->getArg('password'));
+    PhabricatorEnv::overrideConfig('mysql.pass', $args->getArg('password'));
+  }
 
-$test_api = id(new PhabricatorStorageManagementAPI())
-  ->setUser($default_user)
-  ->setHost($default_host)
-  ->setPort($default_port)
-  ->setPassword($ref->getPass())
-  ->setNamespace($args->getArg('namespace'));
+  $selected_user = $args->getArg('user');
+  if ($selected_user === null) {
+    $selected_user = $default_user;
+  }
 
-try {
-  queryfx(
-    $test_api->getConn(null),
-    'SELECT 1');
-} catch (AphrontQueryException $ex) {
-  $message = phutil_console_format(
-    "**%s**\n\n%s\n\n%s\n\n%s\n\n**%s**: %s\n",
-    pht('MySQL Credentials Not Configured'),
-    pht(
-      'Unable to connect to MySQL using the configured credentials. '.
-      'You must configure standard credentials before you can upgrade '.
-      'storage. Run these commands to set up credentials:'),
-    "  phabricator/ $ ./bin/config set mysql.host __host__\n".
-    "  phabricator/ $ ./bin/config set mysql.user __username__\n".
-    "  phabricator/ $ ./bin/config set mysql.pass __password__",
-    pht(
-      'These standard credentials are separate from any administrative '.
-      'credentials provided to this command with __%s__ or '.
-      '__%s__, and must be configured correctly before you can proceed.',
-      '--user',
-      '--password'),
-    pht('Raw MySQL Error'),
-    $ex->getMessage());
-  echo phutil_console_wrap($message);
-  exit(1);
-}
+  $api = id(new PhabricatorStorageManagementAPI())
+    ->setUser($selected_user)
+    ->setHost($default_host)
+    ->setPort($default_port)
+    ->setPassword($password)
+    ->setNamespace($args->getArg('namespace'))
+    ->setDisableUTF8MB4($args->getArg('disable-utf8mb4'));
+  PhabricatorEnv::overrideConfig('mysql.user', $api->getUser());
 
-if ($args->getArg('password') === null) {
-  // This is already a PhutilOpaqueEnvelope.
-  $password = $ref->getPass();
-} else {
-  // Put this in a PhutilOpaqueEnvelope.
-  $password = new PhutilOpaqueEnvelope($args->getArg('password'));
-  PhabricatorEnv::overrideConfig('mysql.pass', $args->getArg('password'));
-}
+  try {
+    queryfx(
+      $api->getConn(null),
+      'SELECT 1');
+  } catch (AphrontQueryException $ex) {
+    $message = phutil_console_format(
+      "**%s**\n\n%s\n\n**%s**: %s\n",
+      pht('Bad Administrative Credentials'),
+      pht(
+        'Unable to connect to MySQL using the administrative credentials '.
+        'provided with the __%s__ and __%s__ flags. Check that '.
+        'you have entered them correctly.',
+        '--user',
+        '--password'),
+      pht('Raw MySQL Error'),
+      $ex->getMessage());
+    echo phutil_console_wrap($message);
+    exit(1);
+  }
 
-$selected_user = $args->getArg('user');
-if ($selected_user === null) {
-  $selected_user = $default_user;
-}
-
-$api = id(new PhabricatorStorageManagementAPI())
-  ->setUser($selected_user)
-  ->setHost($default_host)
-  ->setPort($default_port)
-  ->setPassword($password)
-  ->setNamespace($args->getArg('namespace'))
-  ->setDisableUTF8MB4($args->getArg('disable-utf8mb4'));
-PhabricatorEnv::overrideConfig('mysql.user', $api->getUser());
-
-try {
-  queryfx(
-    $api->getConn(null),
-    'SELECT 1');
-} catch (AphrontQueryException $ex) {
-  $message = phutil_console_format(
-    "**%s**\n\n%s\n\n**%s**: %s\n",
-    pht('Bad Administrative Credentials'),
-    pht(
-      'Unable to connect to MySQL using the administrative credentials '.
-      'provided with the __%s__ and __%s__ flags. Check that '.
-      'you have entered them correctly.',
-      '--user',
-      '--password'),
-    pht('Raw MySQL Error'),
-    $ex->getMessage());
-  echo phutil_console_wrap($message);
-  exit(1);
+  $api->setRef($ref);
+  $apis[] = $api;
 }
 
 $workflows = id(new PhutilClassMapQuery())
@@ -204,7 +237,7 @@ $workflows = id(new PhutilClassMapQuery())
 $patches = PhabricatorSQLPatchList::buildAllPatches();
 
 foreach ($workflows as $workflow) {
-  $workflow->setAPI($api);
+  $workflow->setAPIs($apis);
   $workflow->setPatches($patches);
 }
 

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -2,6 +2,30 @@
 
 final class PhabricatorConfigSchemaQuery extends Phobject {
 
+  private $refs;
+  private $apis;
+
+  public function setRefs(array $refs) {
+    $this->refs = $refs;
+    return $this;
+  }
+
+  public function getRefs() {
+    if (!$this->refs) {
+      return PhabricatorDatabaseRef::getMasterDatabaseRefs();
+    }
+    return $this->refs;
+  }
+
+  public function setAPIs(array $apis) {
+    $map = array();
+    foreach ($apis as $api) {
+      $map[$api->getRef()->getRefKey()] = $api;
+    }
+    $this->apis = $map;
+    return $this;
+  }
+
   private function getDatabaseNames(PhabricatorDatabaseRef $ref) {
     $api = $this->getAPI($ref);
     $patches = PhabricatorSQLPatchList::buildAllPatches();
@@ -11,6 +35,12 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
   }
 
   private function getAPI(PhabricatorDatabaseRef $ref) {
+    $key = $ref->getRefKey();
+
+    if (isset($this->apis[$key])) {
+      return $this->apis[$key];
+    }
+
     return id(new PhabricatorStorageManagementAPI())
       ->setUser($ref->getUser())
       ->setHost($ref->getHost())
@@ -20,7 +50,7 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
   }
 
   public function loadActualSchemata() {
-    $refs = PhabricatorDatabaseRef::getMasterDatabaseRefs();
+    $refs = $this->getRefs();
 
     $schemata = array();
     foreach ($refs as $ref) {
@@ -183,7 +213,7 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
   }
 
   public function loadExpectedSchemata() {
-    $refs = PhabricatorDatabaseRef::getMasterDatabaseRefs();
+    $refs = $this->getRefs();
 
     $schemata = array();
     foreach ($refs as $ref) {

src/infrastructure/cluster/PhabricatorDatabaseRef.php

@@ -223,7 +223,7 @@ final class PhabricatorDatabaseRef
     );
   }
 
-  public static function getLiveRefs() {
+  public static function getClusterRefs() {
     $cache = PhabricatorCaches::getRequestCache();
 
     $refs = $cache->getKey(self::KEY_REFS);
@@ -457,8 +457,22 @@ final class PhabricatorDatabaseRef
     return $this->healthRecord;
   }
 
+  public static function getActiveDatabaseRefs() {
+    $refs = array();
+
+    foreach (self::getMasterDatabaseRefs() as $ref) {
+      $refs[] = $ref;
+    }
+
+    foreach (self::getReplicaDatabaseRefs() as $ref) {
+      $refs[] = $ref;
+    }
+
+    return $refs;
+  }
+
   public static function getMasterDatabaseRefs() {
-    $refs = self::getLiveRefs();
+    $refs = self::getClusterRefs();
 
     if (!$refs) {
       return array(self::getLiveIndividualRef());
@@ -477,13 +491,6 @@ final class PhabricatorDatabaseRef
     return $masters;
   }
 
-  public static function getMasterDatabaseRef() {
-    // TODO: Remove this method; it no longer makes sense with application
-    // partitioning.
-
-    return head(self::getMasterDatabaseRefs());
-  }
-
   public static function getMasterDatabaseRefForDatabase($database) {
     $masters = self::getMasterDatabaseRefs();
 
@@ -507,7 +514,7 @@ final class PhabricatorDatabaseRef
   }
 
   public static function getReplicaDatabaseRefs() {
-    $refs = self::getLiveRefs();
+    $refs = self::getClusterRefs();
 
     if (!$refs) {
       return array();

src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php

@@ -2,6 +2,7 @@
 
 final class PhabricatorStorageManagementAPI extends Phobject {
 
+  private $ref;
   private $host;
   private $user;
   private $port;
@@ -74,6 +75,15 @@ final class PhabricatorStorageManagementAPI extends Phobject {
     return $this->port;
   }
 
+  public function setRef(PhabricatorDatabaseRef $ref) {
+    $this->ref = $ref;
+    return $this;
+  }
+
+  public function getRef() {
+    return $this->ref;
+  }
+
   public function getDatabaseName($fragment) {
     return $this->namespace.'_'.$fragment;
   }

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php

@@ -27,12 +27,19 @@ final class PhabricatorStorageManagementAdjustWorkflow
   public function didExecute(PhutilArgumentParser $args) {
     $unsafe = $args->getArg('unsafe');
 
-    $this->requireAllPatchesApplied();
-    return $this->adjustSchemata($unsafe);
+    foreach ($this->getMasterAPIs() as $api) {
+      $this->requireAllPatchesApplied($api);
+      $err = $this->adjustSchemata($api, $unsafe);
+      if ($err) {
+        return $err;
+      }
+    }
+
+    return 0;
   }
 
-  private function requireAllPatchesApplied() {
-    $api = $this->getAPI();
+  private function requireAllPatchesApplied(
+    PhabricatorStorageManagementAPI $api) {
     $applied = $api->getAppliedPatches();
 
     if ($applied === null) {

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php

@@ -15,7 +15,8 @@ final class PhabricatorStorageManagementDatabasesWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
-    $api     = $this->getAPI();
+    $api = $this->getAnyAPI();
+
     $patches = $this->getPatches();
 
     $databases = $api->getDatabaseList($patches, true);

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php

@@ -23,6 +23,8 @@ final class PhabricatorStorageManagementDestroyWorkflow
   public function didExecute(PhutilArgumentParser $args) {
     $console = PhutilConsole::getConsole();
 
+    $api = $this->getSingleAPI();
+
     if (!$this->isDryRun() && !$this->isForce()) {
       $console->writeOut(
         phutil_console_wrap(
@@ -42,7 +44,6 @@ final class PhabricatorStorageManagementDestroyWorkflow
       }
     }
 
-    $api     = $this->getAPI();
     $patches = $this->getPatches();
 
     if ($args->getArg('unittest-fixtures')) {

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php

@@ -44,7 +44,7 @@ final class PhabricatorStorageManagementDumpWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
-    $api = $this->getAPI();
+    $api = $this->getSingleAPI();
     $patches = $this->getPatches();
 
     $console = PhutilConsole::getConsole();

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php

@@ -15,13 +15,14 @@ final class PhabricatorStorageManagementProbeWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
+    $api = $this->getSingleAPI();
+
     $console = PhutilConsole::getConsole();
     $console->writeErr(
       "%s\n",
       pht('Analyzing table sizes (this may take a moment)...'));
 
-    $api       = $this->getAPI();
-    $patches   = $this->getPatches();
+    $patches = $this->getPatches();
     $databases = $api->getDatabaseList($patches, true);
 
     $conn_r = $api->getConn(null);

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php

@@ -36,7 +36,14 @@ final class PhabricatorStorageManagementQuickstartWorkflow
 
     $bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage';
 
-    if (!$this->getAPI()->isCharacterSetAvailable('utf8mb4')) {
+    // We don't care which database we're using to generate a quickstart file,
+    // since all of the schemata should be identical.
+    $api = $this->getAnyAPI();
+
+    $ref = $api->getRef();
+    $ref_key = $ref->getRefKey();
+
+    if (!$api->isCharacterSetAvailable('utf8mb4')) {
       throw new PhutilArgumentUsageException(
         pht(
           'You can only generate a new quickstart file if MySQL supports '.
@@ -47,35 +54,39 @@ final class PhabricatorStorageManagementQuickstartWorkflow
     }
 
     $err = phutil_passthru(
-      '%s upgrade --force --no-quickstart --namespace %s',
+      '%s upgrade --force --no-quickstart --namespace %s --ref %s',
       $bin,
-      $namespace);
+      $namespace,
+      $ref_key);
     if ($err) {
       return $err;
     }
 
     $err = phutil_passthru(
-      '%s adjust --force --namespace %s',
+      '%s adjust --force --namespace %s --ref %s',
       $bin,
-      $namespace);
+      $namespace,
+      $ref_key);
     if ($err) {
       return $err;
     }
 
     $tmp = new TempFile();
     $err = phutil_passthru(
-      '%s dump --namespace %s > %s',
+      '%s dump --namespace %s --ref %s > %s',
       $bin,
       $namespace,
+      $ref_key,
       $tmp);
     if ($err) {
       return $err;
     }
 
     $err = phutil_passthru(
-      '%s destroy --force --namespace %s',
+      '%s destroy --force --namespace %s --ref %s',
       $bin,
-      $namespace);
+      $namespace,
+      $ref_key);
     if ($err) {
       return $err;
     }

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php

@@ -62,7 +62,7 @@ final class PhabricatorStorageManagementRenamespaceWorkflow
     if (!strlen($input) && !$is_live) {
       throw new PhutilArgumentUsageException(
         pht(
-          'Specify the dumpfile to read with "--in", or use "--live" to '.
+          'Specify the dumpfile to read with "--input", or use "--live" to '.
           'generate one automatically.'));
     }
 
@@ -108,11 +108,15 @@ final class PhabricatorStorageManagementRenamespaceWorkflow
     }
 
     if ($is_live) {
+      $api = $this->getSingleAPI();
+      $ref_key = $api->getRef()->getRefKey();
+
       $root = dirname(phutil_get_library_root('phabricator'));
 
       $future = new ExecFuture(
-        '%R dump',
-        $root.'/bin/storage');
+        '%R dump --ref %s',
+        $root.'/bin/storage',
+        $ref_key);
 
       $lines = new LinesOfALargeExecFuture($future);
     } else {

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementShellWorkflow.php

@@ -15,7 +15,7 @@ final class PhabricatorStorageManagementShellWorkflow
   }
 
   public function execute(PhutilArgumentParser $args) {
-    $api = $this->getAPI();
+    $api = $this->getSingleAPI();
     list($host, $port) = $this->getBareHostAndPort($api->getHost());
 
     $flag_port = $port

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php

@@ -15,50 +15,54 @@ final class PhabricatorStorageManagementStatusWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
-    $api     = $this->getAPI();
-    $patches = $this->getPatches();
+    foreach ($this->getAPIs() as $api) {
+      $patches = $this->getPatches();
+      $applied = $api->getAppliedPatches();
 
-    $applied = $api->getAppliedPatches();
+      if ($applied === null) {
+        echo phutil_console_format(
+          "**%s**: %s\n",
+          pht('Database Not Initialized'),
+          pht('Run **%s** to initialize.', './bin/storage upgrade'));
 
-    if ($applied === null) {
-      echo phutil_console_format(
-        "**%s**: %s\n",
-        pht('Database Not Initialized'),
-        pht('Run **%s** to initialize.', './bin/storage upgrade'));
-
-      return 1;
-    }
-
-    $table = id(new PhutilConsoleTable())
-      ->setShowHeader(false)
-      ->addColumn('id',       array('title' => pht('ID')))
-      ->addColumn('status',   array('title' => pht('Status')))
-      ->addColumn('duration', array('title' => pht('Duration')))
-      ->addColumn('type',     array('title' => pht('Type')))
-      ->addColumn('name',     array('title' => pht('Name')));
-
-    $durations = $api->getPatchDurations();
-
-    foreach ($patches as $patch) {
-      $duration = idx($durations, $patch->getFullKey());
-      if ($duration === null) {
-        $duration = '-';
-      } else {
-        $duration = pht('%s us', new PhutilNumber($duration));
+        return 1;
       }
 
-      $table->addRow(array(
-        'id' => $patch->getFullKey(),
-        'status' => in_array($patch->getFullKey(), $applied)
-          ? pht('Applied')
-          : pht('Not Applied'),
-        'duration' => $duration,
-        'type' => $patch->getType(),
-        'name' => $patch->getName(),
-      ));
-    }
+      $ref = $api->getRef();
 
-    $table->draw();
+      $table = id(new PhutilConsoleTable())
+        ->setShowHeader(false)
+        ->addColumn('id', array('title' => pht('ID')))
+        ->addColumn('host', array('title' => pht('Host')))
+        ->addColumn('status', array('title' => pht('Status')))
+        ->addColumn('duration', array('title' => pht('Duration')))
+        ->addColumn('type', array('title' => pht('Type')))
+        ->addColumn('name', array('title' => pht('Name')));
+
+      $durations = $api->getPatchDurations();
+
+      foreach ($patches as $patch) {
+        $duration = idx($durations, $patch->getFullKey());
+        if ($duration === null) {
+          $duration = '-';
+        } else {
+          $duration = pht('%s us', new PhutilNumber($duration));
+        }
+
+        $table->addRow(array(
+          'id' => $patch->getFullKey(),
+          'host' => $ref->getRefKey(),
+          'status' => in_array($patch->getFullKey(), $applied)
+            ? pht('Applied')
+            : pht('Not Applied'),
+          'duration' => $duration,
+          'type' => $patch->getType(),
+          'name' => $patch->getName(),
+        ));
+      }
+
+      $table->draw();
+    }
     return 0;
   }
 

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php

@@ -73,16 +73,24 @@ final class PhabricatorStorageManagementUpgradeWorkflow
     $init_only     = $args->getArg('init-only');
     $no_adjust     = $args->getArg('no-adjust');
 
-    $this->upgradeSchemata($apply_only, $no_quickstart, $init_only);
+    $apis = $this->getMasterAPIs();
 
-    if ($no_adjust || $init_only || $apply_only) {
-      $console->writeOut(
-        "%s\n",
-        pht('Declining to apply storage adjustments.'));
-      return 0;
-    } else {
-      return $this->adjustSchemata(false);
+    foreach ($apis as $api) {
+      $this->upgradeSchemata($api, $apply_only, $no_quickstart, $init_only);
+
+      if ($no_adjust || $init_only || $apply_only) {
+        $console->writeOut(
+          "%s\n",
+          pht('Declining to apply storage adjustments.'));
+      } else {
+        $err = $this->adjustSchemata($api, false);
+        if ($err) {
+          return $err;
+        }
+      }
     }
+
+    return 0;
   }
 
 }

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php

@@ -3,20 +3,56 @@
 abstract class PhabricatorStorageManagementWorkflow
   extends PhabricatorManagementWorkflow {
 
-  private $api;
+  private $apis = array();
   private $dryRun;
   private $force;
   private $patches;
 
   private $didInitialize;
 
-  final public function getAPI() {
-    return $this->api;
+  final public function setAPIs(array $apis) {
+    $this->apis = $apis;
+    return $this;
   }
 
-  final public function setAPI(PhabricatorStorageManagementAPI $api) {
-    $this->api = $api;
-    return $this;
+  final public function getAnyAPI() {
+    return head($this->getAPIs());
+  }
+
+  final public function getMasterAPIs() {
+    $apis = $this->getAPIs();
+
+    $results = array();
+    foreach ($apis as $api) {
+      if ($api->getRef()->getIsMaster()) {
+        $results[] = $api;
+      }
+    }
+
+    if (!$results) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'This command only operates on database masters, but the selected '.
+          'database hosts do not include any masters.'));
+    }
+
+    return $results;
+  }
+
+  final public function getSingleAPI() {
+    $apis = $this->getAPIs();
+    if (count($apis) == 1) {
+      return head($apis);
+    }
+
+    throw new PhutilArgumentUsageException(
+      pht(
+        'Phabricator is configured in cluster mode, with multiple database '.
+        'hosts. Use "--host" to specify which host you want to operate on.'));
+  }
+
+  final public function getAPIs() {
+    return $this->apis;
   }
 
   final protected function isDryRun() {
@@ -73,22 +109,34 @@ abstract class PhabricatorStorageManagementWorkflow
 
   public function didExecute(PhutilArgumentParser $args) {}
 
-  private function loadSchemata() {
-    $query = id(new PhabricatorConfigSchemaQuery())
-      ->setAPI($this->getAPI());
+  private function loadSchemata(PhabricatorStorageManagementAPI $api) {
+    $query = id(new PhabricatorConfigSchemaQuery());
 
-    $actual = $query->loadActualSchema();
-    $expect = $query->loadExpectedSchema();
-    $comp = $query->buildComparisonSchema($expect, $actual);
+    $ref = $api->getRef();
+    $ref_key = $ref->getRefKey();
 
-    return array($comp, $expect, $actual);
+    $query->setAPIs(array($api));
+    $query->setRefs(array($ref));
+
+    $actual = $query->loadActualSchemata();
+    $expect = $query->loadExpectedSchemata();
+    $comp = $query->buildComparisonSchemata($expect, $actual);
+
+    return array(
+      $comp[$ref_key],
+      $expect[$ref_key],
+      $actual[$ref_key],
+    );
   }
 
-  final protected function adjustSchemata($unsafe) {
-    $lock = $this->lock();
+  final protected function adjustSchemata(
+    PhabricatorStorageManagementAPI $api,
+    $unsafe) {
+
+    $lock = $this->lock($api);
 
     try {
-      $err = $this->doAdjustSchemata($unsafe);
+      $err = $this->doAdjustSchemata($api, $unsafe);
     } catch (Exception $ex) {
       $lock->unlock();
       throw $ex;
@@ -99,15 +147,19 @@ abstract class PhabricatorStorageManagementWorkflow
     return $err;
   }
 
-  final private function doAdjustSchemata($unsafe) {
+  final private function doAdjustSchemata(
+    PhabricatorStorageManagementAPI $api,
+    $unsafe) {
+
     $console = PhutilConsole::getConsole();
 
     $console->writeOut(
       "%s\n",
-      pht('Verifying database schemata...'));
+      pht(
+        'Verifying database schemata on "%s"...',
+        $api->getRef()->getRefKey()));
 
-    list($adjustments, $errors) = $this->findAdjustments();
-    $api = $this->getAPI();
+    list($adjustments, $errors) = $this->findAdjustments($api);
 
     if (!$adjustments) {
       $console->writeOut(
@@ -415,8 +467,9 @@ abstract class PhabricatorStorageManagementWorkflow
     return $this->printErrors($errors, $err);
   }
 
-  private function findAdjustments() {
-    list($comp, $expect, $actual) = $this->loadSchemata();
+  private function findAdjustments(
+    PhabricatorStorageManagementAPI $api) {
+    list($comp, $expect, $actual) = $this->loadSchemata($api);
 
     $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
     $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
@@ -766,14 +819,15 @@ abstract class PhabricatorStorageManagementWorkflow
   }
 
   final protected function upgradeSchemata(
+    PhabricatorStorageManagementAPI $api,
     $apply_only = null,
     $no_quickstart = false,
     $init_only = false) {
 
-    $lock = $this->lock();
+    $lock = $this->lock($api);
 
     try {
-      $this->doUpgradeSchemata($apply_only, $no_quickstart, $init_only);
+      $this->doUpgradeSchemata($api, $apply_only, $no_quickstart, $init_only);
     } catch (Exception $ex) {
       $lock->unlock();
       throw $ex;
@@ -783,13 +837,12 @@ abstract class PhabricatorStorageManagementWorkflow
   }
 
   final private function doUpgradeSchemata(
+    PhabricatorStorageManagementAPI $api,
     $apply_only,
     $no_quickstart,
     $init_only) {
 
-    $api = $this->getAPI();
-
-    $applied = $this->getApi()->getAppliedPatches();
+    $applied = $api->getAppliedPatches();
     if ($applied === null) {
       if ($this->dryRun) {
         echo pht(
@@ -923,11 +976,13 @@ abstract class PhabricatorStorageManagementWorkflow
         if (count($this->patches)) {
           throw new Exception(
             pht(
-              'Some patches could not be applied: %s',
+              'Some patches could not be applied to "%s": %s',
+              $api->getRef()->getRefKey(),
               implode(', ', array_keys($this->patches))));
         } else if (!$this->dryRun && !$apply_only) {
           echo pht(
-            "Storage is up to date. Use '%s' for details.",
+            'Storage is up to date on "%s". Use "%s" for details.',
+            $api->getRef()->getRefKey(),
             'storage status')."\n";
         }
         break;
@@ -955,9 +1010,9 @@ abstract class PhabricatorStorageManagementWorkflow
    *
    * @return PhabricatorGlobalLock
    */
-  final protected function lock() {
+  final protected function lock(PhabricatorStorageManagementAPI $api) {
     return PhabricatorGlobalLock::newLock(__CLASS__)
-      ->useSpecificConnection($this->getApi()->getConn(null))
+      ->useSpecificConnection($api->getConn(null))
       ->lock();
   }