mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-27 01:02:42 +01:00
Retain repository update cooldowns across daemon restarts
Summary: Ref T11665. Fixes T7865. When we restart the daemons, the repository pull daemon currently resets the cooldowns on all of its pulls. This can generate a burst of initial load when restarting a lot of instance daemons (as in the Phacility cluster), described in T7865. This smooths things out so that recent pulls are considered, and any repositories which were waiting keep waiting. Somewhat counterintuitively, hosted repositories write `TYPE_FETCH` status messages, so this should work equally well for hosted and observed repositories. This also paves the way for better backoff behavior on repository errors, described in T11665. The error backoff now uses the same logic that the standard backoff does. The next change will make backoff computation consider recent errors. (This is technically too large for repositories which have encountered one error and have a low commit rate, but I'll fix that in the following change; this is just a checkpoint on the way there.) Test Plan: Ran `bin/phd debug pull`, saw the daemon compute reasonable windows based on previous pull activity. Reviewers: chad Maniphest Tasks: T7865, T11665 Differential Revision: https://secure.phabricator.com/D16574
This commit is contained in:
parent
adf9d5ffdd
commit
e41a64607e
2 changed files with 74 additions and 29 deletions
|
@ -102,43 +102,58 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
$retry_after,
|
||||
array_keys($pullable));
|
||||
|
||||
|
||||
// Figure out which repositories we need to queue for an update.
|
||||
foreach ($pullable as $id => $repository) {
|
||||
$monogram = $repository->getMonogram();
|
||||
$now = PhabricatorTime::getNow();
|
||||
$display_name = $repository->getDisplayName();
|
||||
|
||||
if (isset($futures[$id])) {
|
||||
$this->log(pht('Repository "%s" is currently updating.', $monogram));
|
||||
$this->log(
|
||||
pht(
|
||||
'Repository "%s" is currently updating.',
|
||||
$display_name));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isset($queue[$id])) {
|
||||
$this->log(pht('Repository "%s" is already queued.', $monogram));
|
||||
$this->log(
|
||||
pht(
|
||||
'Repository "%s" is already queued.',
|
||||
$display_name));
|
||||
continue;
|
||||
}
|
||||
|
||||
$after = idx($retry_after, $id, 0);
|
||||
$after = idx($retry_after, $id);
|
||||
if (!$after) {
|
||||
$smart_wait = $repository->loadUpdateInterval($min_sleep);
|
||||
$last_update = $this->loadLastUpdate($repository);
|
||||
|
||||
$after = $last_update + $smart_wait;
|
||||
$retry_after[$id] = $after;
|
||||
|
||||
$this->log(
|
||||
pht(
|
||||
'Scheduling repository "%s" with an update window of %s '.
|
||||
'second(s). Last update was %s second(s) ago.',
|
||||
$display_name,
|
||||
new PhutilNumber($smart_wait),
|
||||
new PhutilNumber($now - $last_update)));
|
||||
}
|
||||
|
||||
if ($after > time()) {
|
||||
$this->log(
|
||||
pht(
|
||||
'Repository "%s" is not due for an update for %s second(s).',
|
||||
$monogram,
|
||||
new PhutilNumber($after - time())));
|
||||
$display_name,
|
||||
new PhutilNumber($after - $now)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!$after) {
|
||||
$this->log(
|
||||
pht(
|
||||
'Scheduling repository "%s" for an initial update.',
|
||||
$monogram));
|
||||
} else {
|
||||
$this->log(
|
||||
pht(
|
||||
'Scheduling repository "%s" for an update (%s seconds overdue).',
|
||||
$monogram,
|
||||
new PhutilNumber(time() - $after)));
|
||||
}
|
||||
$this->log(
|
||||
pht(
|
||||
'Scheduling repository "%s" for an update (%s seconds overdue).',
|
||||
$display_name,
|
||||
new PhutilNumber($now - $after)));
|
||||
|
||||
$queue[$id] = $after;
|
||||
}
|
||||
|
@ -157,8 +172,11 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
continue;
|
||||
}
|
||||
|
||||
$monogram = $repository->getMonogram();
|
||||
$this->log(pht('Starting update for repository "%s".', $monogram));
|
||||
$display_name = $repository->getDisplayName();
|
||||
$this->log(
|
||||
pht(
|
||||
'Starting update for repository "%s".',
|
||||
$display_name));
|
||||
|
||||
unset($queue[$id]);
|
||||
$futures[$id] = $this->buildUpdateFuture(
|
||||
|
@ -296,6 +314,32 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task pull
|
||||
*/
|
||||
private function loadLastUpdate(PhabricatorRepository $repository) {
|
||||
$table = new PhabricatorRepositoryStatusMessage();
|
||||
$conn = $table->establishConnection('r');
|
||||
|
||||
$epoch = queryfx_one(
|
||||
$conn,
|
||||
'SELECT MAX(epoch) last_update FROM %T
|
||||
WHERE repositoryID = %d
|
||||
AND statusType IN (%Ls)',
|
||||
$table->getTableName(),
|
||||
$repository->getID(),
|
||||
array(
|
||||
PhabricatorRepositoryStatusMessage::TYPE_INIT,
|
||||
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
|
||||
));
|
||||
|
||||
if ($epoch) {
|
||||
return (int)$epoch['last_update'];
|
||||
}
|
||||
|
||||
return PhabricatorTime::getNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* @task pull
|
||||
*/
|
||||
|
@ -385,9 +429,9 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
ExecFuture $future,
|
||||
$min_sleep) {
|
||||
|
||||
$monogram = $repository->getMonogram();
|
||||
$display_name = $repository->getDisplayName();
|
||||
|
||||
$this->log(pht('Resolving update for "%s".', $monogram));
|
||||
$this->log(pht('Resolving update for "%s".', $display_name));
|
||||
|
||||
try {
|
||||
list($stdout, $stderr) = $future->resolvex();
|
||||
|
@ -395,17 +439,18 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
$proxy = new PhutilProxyException(
|
||||
pht(
|
||||
'Error while updating the "%s" repository.',
|
||||
$repository->getMonogram()),
|
||||
$display_name),
|
||||
$ex);
|
||||
phlog($proxy);
|
||||
|
||||
return time() + $min_sleep;
|
||||
$smart_wait = $repository->loadUpdateInterval($min_sleep);
|
||||
return PhabricatorTime::getNow() + $smart_wait;
|
||||
}
|
||||
|
||||
if (strlen($stderr)) {
|
||||
$stderr_msg = pht(
|
||||
'Unexpected output while updating repository "%s": %s',
|
||||
$monogram,
|
||||
$display_name,
|
||||
$stderr);
|
||||
phlog($stderr_msg);
|
||||
}
|
||||
|
@ -416,10 +461,10 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
pht(
|
||||
'Based on activity in repository "%s", considering a wait of %s '.
|
||||
'seconds before update.',
|
||||
$repository->getMonogram(),
|
||||
$display_name,
|
||||
new PhutilNumber($smart_wait)));
|
||||
|
||||
return time() + $smart_wait;
|
||||
return PhabricatorTime::getNow() + $smart_wait;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -1783,7 +1783,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
|
|||
$smart_wait = $minimum;
|
||||
}
|
||||
|
||||
return $smart_wait;
|
||||
return (int)$smart_wait;
|
||||
}
|
||||
|
||||
|
||||
|
|
Loading…
Reference in a new issue