Provide an ad-hoc maintenance lock for clustered repositories

Summary: Ref T13614. Provide "bin/repository lock" to temporarily lock repositories for manual maintenance.

Test Plan:
  - Read instructions.
  - Used `bin/repository lock` according to the instructions.
  - Saw Storage tab in Diffusion report lock held during maintenance, released after it completes.
  - Saw "maintenance" push log generated and repository version bump.
  - Tried to lock some invalid repositories.

Maniphest Tasks: T13614

Differential Revision: https://secure.phabricator.com/D21671

src/__phutil_library_map__.php

@@ -4613,6 +4613,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementImportingWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementImportingWorkflow.php',
     'PhabricatorRepositoryManagementListPathsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListPathsWorkflow.php',
     'PhabricatorRepositoryManagementListWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListWorkflow.php',
+    'PhabricatorRepositoryManagementLockWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php',
     'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMaintenanceWorkflow.php',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php',
     'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkReachableWorkflow.php',
@@ -11382,6 +11383,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementImportingWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementListPathsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementListWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
+    'PhabricatorRepositoryManagementLockWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'PhabricatorRepositoryManagementWorkflow',

src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php

@@ -0,0 +1,139 @@
+<?php
+
+final class PhabricatorRepositoryManagementLockWorkflow
+  extends PhabricatorRepositoryManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('lock')
+      ->setExamples('**lock** [options] __repository__ ...')
+      ->setSynopsis(
+        pht(
+          'Temporarily lock clustered repositories to perform maintenance.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'repositories',
+            'wildcard' => true,
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $repositories = $this->loadRepositories($args, 'repositories');
+    if (!$repositories) {
+      throw new PhutilArgumentUsageException(
+        pht('Specify one or more repositories to lock.'));
+    }
+
+    foreach ($repositories as $repository) {
+      $display_name = $repository->getDisplayName();
+
+      if (!$repository->isHosted()) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Unable to lock repository "%s": only hosted repositories may be '.
+            'locked.',
+            $display_name));
+      }
+
+      if (!$repository->supportsSynchronization()) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Unable to lock repository "%s": only repositories that support '.
+            'clustering may be locked.',
+            $display_name));
+      }
+
+      if (!$repository->getAlmanacServicePHID()) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Unable to lock repository "%s": only clustered repositories '.
+            'may be locked.',
+            $display_name));
+      }
+    }
+
+    $diffusion_phid = id(new PhabricatorDiffusionApplication())
+      ->getPHID();
+
+    $locks = array();
+    foreach ($repositories as $repository) {
+      $engine = id(new DiffusionRepositoryClusterEngine())
+        ->setViewer($viewer)
+        ->setActingAsPHID($diffusion_phid)
+        ->setRepository($repository);
+
+      $event = $engine->newMaintenanceEvent();
+
+      $logs = array();
+      $logs[] = $engine->newMaintenanceLog();
+
+      $locks[] = array(
+        'repository' => $repository,
+        'engine' => $engine,
+        'event' => $event,
+        'logs' => $logs,
+      );
+    }
+
+    $display_list = new PhutilConsoleList();
+    foreach ($repositories as $repository) {
+      $display_list->addItem(
+        pht(
+          '%s %s',
+          $repository->getMonogram(),
+          $repository->getName()));
+    }
+
+    echo tsprintf(
+      "%s\n\n%B\n",
+      pht('These repositories will be locked:'),
+      $display_list->drawConsoleString());
+
+    echo tsprintf(
+      "%s\n",
+      pht(
+        'While the lock is held: users will be unable to write to this '.
+        'repository, and you may safely perform working copy maintenance '.
+        'on this node in another terminal window.'));
+
+    $query = pht('Lock repositories and begin maintenance?');
+    if (!phutil_console_confirm($query)) {
+      throw new ArcanistUserAbortException();
+    }
+
+    foreach ($locks as $key => $lock) {
+      $engine = $lock['engine'];
+      $engine->synchronizeWorkingCopyBeforeWrite();
+    }
+
+    echo tsprintf(
+      "%s\n",
+      pht(
+        'Repositories are now locked. You may begin maintenance in '.
+        'another terminal window. Keep this process running until '.
+        'you complete the maintenance, then confirm that you are ready to '.
+        'release the locks.'));
+
+    while (!phutil_console_confirm('Ready to release the locks?')) {
+      // Wait for the user to confirm that they're ready.
+    }
+
+    foreach ($locks as $key => $lock) {
+      $lock['event']->saveWithLogs($lock['logs']);
+
+      $engine = $lock['engine'];
+      $engine->synchronizeWorkingCopyAfterWrite();
+    }
+
+    echo tsprintf(
+      "%s\n",
+      pht('Done.'));
+
+    return 0;
+  }
+
+}

src/docs/user/cluster/cluster_repositories.diviner

@@ -523,6 +523,87 @@ Retaining separate backup snapshots will improve your ability to recover more
 data more easily in a wider range of disaster situations.
 
 
+Ad-Hoc Maintenance Locks
+========================
+
+Occasionally, you may want to perform maintenance to a clustered repository
+which requires you modify the actual content of the repository.
+
+For example: you might want to delete a large number of old or temporary
+branches; or you might want to merge a very large number of commits from
+another source.
+
+These operations may be prohibitively slow or complex to perform using normal
+pushes. In cases where you would prefer to directly modify a working copy, you
+can use a maintenance lock to safely make a working copy mutable.
+
+If you simply perform this kind of content-modifying maintenance by directly
+modifying the repository on disk with commands like `git update-ref`, your
+changes may either encounter conflicts or encounter problems with change
+propagation.
+
+You can encounter conflicts because directly modifying the working copy on disk
+won't prevent users or Phabricator itself from performing writes to the same
+working copy at the same time. Phabricator does not compromise the lower-level
+locks provided by the VCS so this is theoretically safe -- and this rarely
+causes any significant problems in practice -- but doesn't make things any
+simpler or easier.
+
+Your changes may fail to propagate because writing directly to the repository
+doesn't turn it into the new cluster leader after your writes complete. If
+another node accepts the next push, it will become the new leader -- without
+your changes -- and all other nodes will synchronize from it.
+
+Note that some maintenance operations (like `git gc`, `git prune`, or
+`git  repack`) do not modify repository content. In theory, these operations do
+not require a maintenance lock: lower-level Git locks should protect
+them from conflicts, and they can not be affected by propagation issues because
+they do not propagate. In practice, these operations are not conflict-free in
+all circumstances. Using a maintenance lock may be overkill, but it's probably
+still a good idea.
+
+To use a maintenance lock:
+
+  - Open two terminal windows. You'll use one window to hold the lock and a
+    second window to perform maintenance.
+  - Run `bin/repository lock <repository> ...` in one terminal.
+  - When the process reports that repositories are locked, switch to the second
+    terminal and perform maintenance. The `repository lock` process should
+    still be running in your first terminal.
+  - After maintenance completes, switch back to the first terminal and answer
+    the prompt to confirm maintenance is complete.
+
+The workflow looks something like this:
+
+```
+$ ./bin/repository lock R2
+
+These repositories will be locked:
+
+      - R2 Git Test Repository
+
+While the lock is held: users will be unable to write to this repository,
+and you may safely perform working copy maintenance on this node in another
+terminal window.
+
+    Lock repositories and begin maintenance? [y/N] y
+
+Repositories are now locked. You may begin maintenance in another terminal
+window. Keep this process running until you complete the maintenance, then
+confirm that you are ready to release the locks.
+
+    Ready to release the locks? [y/N] y
+
+Done.
+```
+
+As maintenance completes, the push log for the repository will be updated to
+reflect that you performed maintenance.
+
+If the lock is interrupted, you may encounter a "Write Interruptions" condition
+described earlier in this document. See that section for details. In most
+cases, you can resolve this issue by demoting the node you are working on.
+
 Next Steps
 ==========