revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-25 00:02:41 +01:00
Code Issues Releases Wiki Activity

Provide an ad-hoc maintenance lock for clustered repositories

Browse source 
Summary: Ref T13614. Provide "bin/repository lock" to temporarily lock repositories for manual maintenance.

Test Plan:
  - Read instructions.
  - Used `bin/repository lock` according to the instructions.
  - Saw Storage tab in Diffusion report lock held during maintenance, released after it completes.
  - Saw "maintenance" push log generated and repository version bump.
  - Tried to lock some invalid repositories.

Maniphest Tasks: T13614

Differential Revision: https://secure.phabricator.com/D21671
This commit is contained in:
epriestley 2021-06-01 06:34:49 -07:00
parent 12a5eb4062
commit 51cb7a3db9
3 changed files with 222 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/__phutil_library_map__.php
View file

@ -4613,6 +4613,7 @@ phutil_register_library_map(array(
'PhabricatorRepositoryManagementImportingWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementImportingWorkflow.php',
'PhabricatorRepositoryManagementListPathsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListPathsWorkflow.php',
'PhabricatorRepositoryManagementListWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementListWorkflow.php',
'PhabricatorRepositoryManagementLockWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php',
'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMaintenanceWorkflow.php',
'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php',
'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkReachableWorkflow.php',
@ -11382,6 +11383,7 @@ phutil_register_library_map(array(
'PhabricatorRepositoryManagementImportingWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementListPathsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementListWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementLockWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementMaintenanceWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
'PhabricatorRepositoryManagementMarkReachableWorkflow' => 'PhabricatorRepositoryManagementWorkflow',

139
src/applications/repository/management/PhabricatorRepositoryManagementLockWorkflow.php Normal file
View file

@ -0,0 +1,139 @@
<?php
final class PhabricatorRepositoryManagementLockWorkflow
extends PhabricatorRepositoryManagementWorkflow {
protected function didConstruct() {
$this
->setName('lock')
->setExamples('**lock** [options] __repository__ ...')
->setSynopsis(
pht(
'Temporarily lock clustered repositories to perform maintenance.'))
->setArguments(
array(
array(
'name' => 'repositories',
'wildcard' => true,
),
));
}
public function execute(PhutilArgumentParser $args) {
$viewer = $this->getViewer();
$repositories = $this->loadRepositories($args, 'repositories');
if (!$repositories) {
throw new PhutilArgumentUsageException(
pht('Specify one or more repositories to lock.'));
}
foreach ($repositories as $repository) {
$display_name = $repository->getDisplayName();
if (!$repository->isHosted()) {
throw new PhutilArgumentUsageException(
pht(
'Unable to lock repository "%s": only hosted repositories may be '.
'locked.',
$display_name));
}
if (!$repository->supportsSynchronization()) {
throw new PhutilArgumentUsageException(
pht(
'Unable to lock repository "%s": only repositories that support '.
'clustering may be locked.',
$display_name));
}
if (!$repository->getAlmanacServicePHID()) {
throw new PhutilArgumentUsageException(
pht(
'Unable to lock repository "%s": only clustered repositories '.
'may be locked.',
$display_name));
}
}
$diffusion_phid = id(new PhabricatorDiffusionApplication())
->getPHID();
$locks = array();
foreach ($repositories as $repository) {
$engine = id(new DiffusionRepositoryClusterEngine())
->setViewer($viewer)
->setActingAsPHID($diffusion_phid)
->setRepository($repository);
$event = $engine->newMaintenanceEvent();
$logs = array();
$logs[] = $engine->newMaintenanceLog();
$locks[] = array(
'repository' => $repository,
'engine' => $engine,
'event' => $event,
'logs' => $logs,
);
}
$display_list = new PhutilConsoleList();
foreach ($repositories as $repository) {
$display_list->addItem(
pht(
'%s %s',
$repository->getMonogram(),
$repository->getName()));
}
echo tsprintf(
"%s\n\n%B\n",
pht('These repositories will be locked:'),
$display_list->drawConsoleString());
echo tsprintf(
"%s\n",
pht(
'While the lock is held: users will be unable to write to this '.
'repository, and you may safely perform working copy maintenance '.
'on this node in another terminal window.'));
$query = pht('Lock repositories and begin maintenance?');
if (!phutil_console_confirm($query)) {
throw new ArcanistUserAbortException();
}
foreach ($locks as $key => $lock) {
$engine = $lock['engine'];
$engine->synchronizeWorkingCopyBeforeWrite();
}
echo tsprintf(
"%s\n",
pht(
'Repositories are now locked. You may begin maintenance in '.
'another terminal window. Keep this process running until '.
'you complete the maintenance, then confirm that you are ready to '.
'release the locks.'));
while (!phutil_console_confirm('Ready to release the locks?')) {
// Wait for the user to confirm that they're ready.
}
foreach ($locks as $key => $lock) {
$lock['event']->saveWithLogs($lock['logs']);
$engine = $lock['engine'];
$engine->synchronizeWorkingCopyAfterWrite();
}
echo tsprintf(
"%s\n",
pht('Done.'));
return 0;
}
}

81
src/docs/user/cluster/cluster_repositories.diviner
View file

@ -523,6 +523,87 @@ Retaining separate backup snapshots will improve your ability to recover more
data more easily in a wider range of disaster situations.
Ad-Hoc Maintenance Locks
========================
Occasionally, you may want to perform maintenance to a clustered repository
which requires you modify the actual content of the repository.
For example: you might want to delete a large number of old or temporary
branches; or you might want to merge a very large number of commits from
another source.
These operations may be prohibitively slow or complex to perform using normal
pushes. In cases where you would prefer to directly modify a working copy, you
can use a maintenance lock to safely make a working copy mutable.
If you simply perform this kind of content-modifying maintenance by directly
modifying the repository on disk with commands like `git update-ref`, your
changes may either encounter conflicts or encounter problems with change
propagation.
You can encounter conflicts because directly modifying the working copy on disk
won't prevent users or Phabricator itself from performing writes to the same
working copy at the same time. Phabricator does not compromise the lower-level
locks provided by the VCS so this is theoretically safe -- and this rarely
causes any significant problems in practice -- but doesn't make things any
simpler or easier.
Your changes may fail to propagate because writing directly to the repository
doesn't turn it into the new cluster leader after your writes complete. If
another node accepts the next push, it will become the new leader -- without
your changes -- and all other nodes will synchronize from it.
Note that some maintenance operations (like `git gc`, `git prune`, or
`git repack`) do not modify repository content. In theory, these operations do
not require a maintenance lock: lower-level Git locks should protect
them from conflicts, and they can not be affected by propagation issues because
they do not propagate. In practice, these operations are not conflict-free in
all circumstances. Using a maintenance lock may be overkill, but it's probably
still a good idea.
To use a maintenance lock:
- Open two terminal windows. You'll use one window to hold the lock and a
second window to perform maintenance.
- Run `bin/repository lock <repository> ...` in one terminal.
- When the process reports that repositories are locked, switch to the second
terminal and perform maintenance. The `repository lock` process should
still be running in your first terminal.
- After maintenance completes, switch back to the first terminal and answer
the prompt to confirm maintenance is complete.
The workflow looks something like this:
```
$ ./bin/repository lock R2
These repositories will be locked:
- R2 Git Test Repository
While the lock is held: users will be unable to write to this repository,
and you may safely perform working copy maintenance on this node in another
terminal window.
Lock repositories and begin maintenance? [y/N] y
Repositories are now locked. You may begin maintenance in another terminal
window. Keep this process running until you complete the maintenance, then
confirm that you are ready to release the locks.
Ready to release the locks? [y/N] y
Done.
```
As maintenance completes, the push log for the repository will be updated to
reflect that you performed maintenance.
If the lock is interrupted, you may encounter a "Write Interruptions" condition
described earlier in this document. See that section for details. In most
cases, you can resolve this issue by demoting the node you are working on.
Next Steps
==========