From 51cb7a3db9e8e510d564f6b37297111f7a690c91 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 1 Jun 2021 06:34:49 -0700 Subject: [PATCH] Provide an ad-hoc maintenance lock for clustered repositories Summary: Ref T13614. Provide "bin/repository lock" to temporarily lock repositories for manual maintenance. Test Plan: - Read instructions. - Used `bin/repository lock` according to the instructions. - Saw Storage tab in Diffusion report lock held during maintenance, released after it completes. - Saw "maintenance" push log generated and repository version bump. - Tried to lock some invalid repositories. Maniphest Tasks: T13614 Differential Revision: https://secure.phabricator.com/D21671 --- src/__phutil_library_map__.php | 2 + ...icatorRepositoryManagementLockWorkflow.php | 139 ++++++++++++++++++ .../user/cluster/cluster_repositories.diviner | 81 ++++++++++ 3 files changed, 222 insertions(+) create mode 100644 src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index e811ddbc92..3eedd1605a 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4613,6 +4613,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryManagementImportingWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementImportingWorkflow.php', 'PhabricatorRepositoryManagementListPathsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListPathsWorkflow.php', 'PhabricatorRepositoryManagementListWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListWorkflow.php', + 'PhabricatorRepositoryManagementLockWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php', 'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMaintenanceWorkflow.php', 'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php', 'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkReachableWorkflow.php', @@ -11382,6 +11383,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryManagementImportingWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementListPathsWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementListWorkflow' => 'PhabricatorRepositoryManagementWorkflow', + 'PhabricatorRepositoryManagementLockWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'PhabricatorRepositoryManagementWorkflow', diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php new file mode 100644 index 0000000000..289e8d73ee --- /dev/null +++ b/src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php @@ -0,0 +1,139 @@ +setName('lock') + ->setExamples('**lock** [options] __repository__ ...') + ->setSynopsis( + pht( + 'Temporarily lock clustered repositories to perform maintenance.')) + ->setArguments( + array( + array( + 'name' => 'repositories', + 'wildcard' => true, + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $repositories = $this->loadRepositories($args, 'repositories'); + if (!$repositories) { + throw new PhutilArgumentUsageException( + pht('Specify one or more repositories to lock.')); + } + + foreach ($repositories as $repository) { + $display_name = $repository->getDisplayName(); + + if (!$repository->isHosted()) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to lock repository "%s": only hosted repositories may be '. + 'locked.', + $display_name)); + } + + if (!$repository->supportsSynchronization()) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to lock repository "%s": only repositories that support '. + 'clustering may be locked.', + $display_name)); + } + + if (!$repository->getAlmanacServicePHID()) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to lock repository "%s": only clustered repositories '. + 'may be locked.', + $display_name)); + } + } + + $diffusion_phid = id(new PhabricatorDiffusionApplication()) + ->getPHID(); + + $locks = array(); + foreach ($repositories as $repository) { + $engine = id(new DiffusionRepositoryClusterEngine()) + ->setViewer($viewer) + ->setActingAsPHID($diffusion_phid) + ->setRepository($repository); + + $event = $engine->newMaintenanceEvent(); + + $logs = array(); + $logs[] = $engine->newMaintenanceLog(); + + $locks[] = array( + 'repository' => $repository, + 'engine' => $engine, + 'event' => $event, + 'logs' => $logs, + ); + } + + $display_list = new PhutilConsoleList(); + foreach ($repositories as $repository) { + $display_list->addItem( + pht( + '%s %s', + $repository->getMonogram(), + $repository->getName())); + } + + echo tsprintf( + "%s\n\n%B\n", + pht('These repositories will be locked:'), + $display_list->drawConsoleString()); + + echo tsprintf( + "%s\n", + pht( + 'While the lock is held: users will be unable to write to this '. + 'repository, and you may safely perform working copy maintenance '. + 'on this node in another terminal window.')); + + $query = pht('Lock repositories and begin maintenance?'); + if (!phutil_console_confirm($query)) { + throw new ArcanistUserAbortException(); + } + + foreach ($locks as $key => $lock) { + $engine = $lock['engine']; + $engine->synchronizeWorkingCopyBeforeWrite(); + } + + echo tsprintf( + "%s\n", + pht( + 'Repositories are now locked. You may begin maintenance in '. + 'another terminal window. Keep this process running until '. + 'you complete the maintenance, then confirm that you are ready to '. + 'release the locks.')); + + while (!phutil_console_confirm('Ready to release the locks?')) { + // Wait for the user to confirm that they're ready. + } + + foreach ($locks as $key => $lock) { + $lock['event']->saveWithLogs($lock['logs']); + + $engine = $lock['engine']; + $engine->synchronizeWorkingCopyAfterWrite(); + } + + echo tsprintf( + "%s\n", + pht('Done.')); + + return 0; + } + +} diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index 9e4b324a58..a2e5fd5b68 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -523,6 +523,87 @@ Retaining separate backup snapshots will improve your ability to recover more data more easily in a wider range of disaster situations. +Ad-Hoc Maintenance Locks +======================== + +Occasionally, you may want to perform maintenance to a clustered repository +which requires you modify the actual content of the repository. + +For example: you might want to delete a large number of old or temporary +branches; or you might want to merge a very large number of commits from +another source. + +These operations may be prohibitively slow or complex to perform using normal +pushes. In cases where you would prefer to directly modify a working copy, you +can use a maintenance lock to safely make a working copy mutable. + +If you simply perform this kind of content-modifying maintenance by directly +modifying the repository on disk with commands like `git update-ref`, your +changes may either encounter conflicts or encounter problems with change +propagation. + +You can encounter conflicts because directly modifying the working copy on disk +won't prevent users or Phabricator itself from performing writes to the same +working copy at the same time. Phabricator does not compromise the lower-level +locks provided by the VCS so this is theoretically safe -- and this rarely +causes any significant problems in practice -- but doesn't make things any +simpler or easier. + +Your changes may fail to propagate because writing directly to the repository +doesn't turn it into the new cluster leader after your writes complete. If +another node accepts the next push, it will become the new leader -- without +your changes -- and all other nodes will synchronize from it. + +Note that some maintenance operations (like `git gc`, `git prune`, or +`git repack`) do not modify repository content. In theory, these operations do +not require a maintenance lock: lower-level Git locks should protect +them from conflicts, and they can not be affected by propagation issues because +they do not propagate. In practice, these operations are not conflict-free in +all circumstances. Using a maintenance lock may be overkill, but it's probably +still a good idea. + +To use a maintenance lock: + + - Open two terminal windows. You'll use one window to hold the lock and a + second window to perform maintenance. + - Run `bin/repository lock ...` in one terminal. + - When the process reports that repositories are locked, switch to the second + terminal and perform maintenance. The `repository lock` process should + still be running in your first terminal. + - After maintenance completes, switch back to the first terminal and answer + the prompt to confirm maintenance is complete. + +The workflow looks something like this: + +``` +$ ./bin/repository lock R2 + +These repositories will be locked: + + - R2 Git Test Repository + +While the lock is held: users will be unable to write to this repository, +and you may safely perform working copy maintenance on this node in another +terminal window. + + Lock repositories and begin maintenance? [y/N] y + +Repositories are now locked. You may begin maintenance in another terminal +window. Keep this process running until you complete the maintenance, then +confirm that you are ready to release the locks. + + Ready to release the locks? [y/N] y + +Done. +``` + +As maintenance completes, the push log for the repository will be updated to +reflect that you performed maintenance. + +If the lock is interrupted, you may encounter a "Write Interruptions" condition +described earlier in this document. See that section for details. In most +cases, you can resolve this issue by demoting the node you are working on. + Next Steps ==========