revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-24 06:20:56 +01:00
Code Issues Releases Wiki Activity

Make task queue more robust against long-running tasks

Browse source 
Summary:
See discussion in D8773. Three small adjustments which should help prevent this kind of issue:

  - When queueing followup tasks, hold them on the worker until we finish the task, then queue them only if the work was successful.
  - Increase the default lease time from 60 seconds to 2 hours. Although most tasks finish in far fewer than 60 seconds, the daemons are generally stable nowadays and these short leases don't serve much of a purpose. I think they also date from an era where lease expiry and failure were less clearly distinguished.
  - Increase the default wait-after-failure from 60 seconds to 5 minutes. This largely dates from the MetaMTA era, where Facebook ran services with high failure rates and it was appropriate to repeatedly hammer them until things went through. In modern infrastructure, such failures are rare.

Test Plan:
  - Verified that tasks queued properly after the main task was updated.
  - Verified that leases default to 7200 seconds.
  - Intentionally failed a task and verified default 300 second wait before retry.
  - Removed all default leases shorter than 7200 seconds (there was only one).
  - Checked all the wait before retry implementations for anything much shorter than 5 minutes (they all seem reasonable).

Reviewers: btrahan, sowedance

Reviewed By: sowedance

Subscribers: epriestley

Differential Revision: https://secure.phabricator.com/D8774
This commit is contained in:
epriestley 2014-04-15 08:42:02 -07:00
parent 6a4f126000
commit cb545856a9
11 changed files with 59 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/applications/feed/worker/FeedPublisherWorker.php
View file

@ -7,7 +7,7 @@ final class FeedPublisherWorker extends FeedPushWorker {
$uris = PhabricatorEnv::getEnvConfig('feed.http-hooks');
foreach ($uris as $uri) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'FeedPublisherHTTPWorker',
array(
'key' => $story->getChronologicalKey(),
@ -27,7 +27,7 @@ final class FeedPublisherWorker extends FeedPushWorker {
if (!$worker->isEnabled()) {
continue;
}
PhabricatorWorker::scheduleTask(
$this->queueTask(
get_class($worker),
array(
'key' => $story->getChronologicalKey(),

2
src/applications/harbormaster/worker/HarbormasterTargetWorker.php
View file

@ -8,7 +8,7 @@ final class HarbormasterTargetWorker extends HarbormasterWorker {
public function getRequiredLeaseTime() {
// This worker performs actual build work, which may involve a long wait
// on external systems.
return 60 * 60 * 24;
return phutil_units('24 hours in seconds');
}
private function loadBuildTarget() {

2
src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php
View file

@ -5,7 +5,7 @@ final class PhabricatorRepositoryCommitHeraldWorker
public function getRequiredLeaseTime() {
// Herald rules may take a long time to process.
return 4 * 60 * 60;
return phutil_units('4 hours in seconds');
}
public function parseCommit(

7
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
View file

@ -3,11 +3,6 @@
final class PhabricatorRepositoryCommitOwnersWorker
extends PhabricatorRepositoryCommitParserWorker {
public function getRequiredLeaseTime() {
// Commit owner parser may take a while to process for huge commits.
return 60 * 60;
}
protected function parseCommit(
PhabricatorRepository $repository,
PhabricatorRepositoryCommit $commit) {
@ -18,7 +13,7 @@ final class PhabricatorRepositoryCommitOwnersWorker
PhabricatorRepositoryCommit::IMPORTED_OWNERS);
if ($this->shouldQueueFollowupTasks()) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'PhabricatorRepositoryCommitHeraldWorker',
array(
'commitID' => $commit->getID(),

4
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
View file

@ -6,7 +6,7 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
public function getRequiredLeaseTime() {
// It can take a very long time to parse commits; some commits in the
// Facebook repository affect many millions of paths. Acquire 24h leases.
return 60 * 60 * 24;
return phutil_units('24 hours in seconds');
}
abstract protected function parseCommitChanges(
@ -98,7 +98,7 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
PhabricatorOwnersPackagePathValidator::updateOwnersPackagePaths($commit);
if ($this->shouldQueueFollowupTasks()) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'PhabricatorRepositoryCommitOwnersWorker',
array(
'commitID' => $commit->getID(),

2
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
View file

@ -15,7 +15,7 @@ final class PhabricatorRepositoryGitCommitMessageParserWorker
$this->updateCommitData($ref);
if ($this->shouldQueueFollowupTasks()) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'PhabricatorRepositoryGitCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),

2
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
View file

@ -15,7 +15,7 @@ final class PhabricatorRepositoryMercurialCommitMessageParserWorker
$this->updateCommitData($ref);
if ($this->shouldQueueFollowupTasks()) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'PhabricatorRepositoryMercurialCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),

2
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
View file

@ -15,7 +15,7 @@ final class PhabricatorRepositorySvnCommitMessageParserWorker
$this->updateCommitData($ref);
if ($this->shouldQueueFollowupTasks()) {
PhabricatorWorker::scheduleTask(
$this->queueTask(
'PhabricatorRepositorySvnCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),

30
src/infrastructure/daemon/workers/PhabricatorWorker.php
View file

@ -9,6 +9,7 @@ abstract class PhabricatorWorker {
private $data;
private static $runAllTasksInProcess = false;
private $queuedTasks = array();
/* -( Configuring Retries and Failures )----------------------------------- */
@ -17,13 +18,13 @@ abstract class PhabricatorWorker {
/**
* Return the number of seconds this worker needs hold a lease on the task for
* while it performs work. For most tasks you can leave this at `null`, which
* will give you a short default lease (currently 60 seconds).
* will give you a default lease (currently 2 hours).
*
* For tasks which may take a very long time to complete, you should return
* an upper bound on the amount of time the task may require.
*
* @return int|null Number of seconds this task needs to remain leased for,
* or null for a default (currently 60 second) lease.
* or null for a default lease.
*
* @task config
*/
@ -194,4 +195,29 @@ abstract class PhabricatorWorker {
return $this;
}
/**
* Queue a task to be executed after this one suceeds.
*
* The followup task will be queued only if this task completes cleanly.
*
* @param string Task class to queue.
* @param array Data for the followup task.
* @return this
*/
protected function queueTask($class, array $data) {
$this->queuedTasks[] = array($class, $data);
return $this;
}
/**
* Get tasks queued as followups by @{method:queueTask}.
*
* @return list<pair<string, wild>> Queued task specifications.
*/
public function getQueuedTasks() {
return $this->queuedTasks;
}
}

12
src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php
View file

@ -10,11 +10,17 @@ final class PhabricatorWorkerLeaseQuery extends PhabricatorQuery {
const PHASE_UNLEASED = 'unleased';
const PHASE_EXPIRED = 'expired';
const DEFAULT_LEASE_DURATION = 60; // Seconds
private $ids;
private $limit;
public static function getDefaultWaitBeforeRetry() {
return phutil_units('5 minutes in seconds');
}
public static function getDefaultLeaseDuration() {
return phutil_units('2 hours in seconds');
}
public function withIDs(array $ids) {
$this->ids = $ids;
return $this;
@ -78,7 +84,7 @@ final class PhabricatorWorkerLeaseQuery extends PhabricatorQuery {
%Q',
$task_table->getTableName(),
$lease_ownership_name,
self::DEFAULT_LEASE_DURATION,
self::getDefaultLeaseDuration(),
$this->buildUpdateWhereClause($conn_w, $phase, $rows));
$leased += $conn_w->getAffectedRows();

13
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
View file

@ -100,6 +100,7 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
// to release the lease otherwise.
$this->checkLease();
$did_succeed = false;
try {
$worker = $this->getWorkerInstance();
@ -126,6 +127,7 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
$result = $this->archiveTask(
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
$duration);
$did_succeed = true;
} catch (PhabricatorWorkerPermanentFailureException $ex) {
$result = $this->archiveTask(
PhabricatorWorkerArchiveTask::RESULT_FAILURE,
@ -139,7 +141,7 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
$retry = $worker->getWaitBeforeRetry($this);
$retry = coalesce(
$retry,
PhabricatorWorkerLeaseQuery::DEFAULT_LEASE_DURATION);
PhabricatorWorkerLeaseQuery::getDefaultWaitBeforeRetry());
// NOTE: As a side effect, this saves the object.
$this->setLeaseDuration($retry);
@ -147,6 +149,15 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
$result = $this;
}
// NOTE: If this throws, we don't want it to cause the task to fail again,
// so execute it out here and just let the exception escape.
if ($did_succeed) {
foreach ($worker->getQueuedTasks() as $task) {
list($class, $data) = $task;
PhabricatorWorker::scheduleTask($class, $data);
}
}
return $result;
}