mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-29 02:02:41 +01:00
Move completed tasks to an "archive" table and delete them in the GC
Summary: Currently, when taskmasters complete a task it is immediately deleted. This prevents us from doing some general things, like: - Supporting the idea of permanent failure (e.g., after N failures just stop trying). - Showing the user how fast taskmasters are completing tasks. - Showing the user how long tasks took to complete. Having better visibility into this is important to Drydock, which builds on the task system. Also, generally buff debug output for task execution. Test Plan: Ran `bin/phd debug taskmaster`. Ran `bin/phd debug garbage`. Queued some tasks via various systems. Reviewers: btrahan Reviewed By: btrahan CC: aran Maniphest Tasks: T2015 Differential Revision: https://secure.phabricator.com/D3852
This commit is contained in:
parent
4edf8ae2fc
commit
5903ed650c
24 changed files with 292 additions and 102 deletions
|
@ -1126,6 +1126,7 @@ return array(
|
||||||
'gcdaemon.ttl.daemon-logs' => 7 * (24 * 60 * 60),
|
'gcdaemon.ttl.daemon-logs' => 7 * (24 * 60 * 60),
|
||||||
'gcdaemon.ttl.differential-parse-cache' => 14 * (24 * 60 * 60),
|
'gcdaemon.ttl.differential-parse-cache' => 14 * (24 * 60 * 60),
|
||||||
'gcdaemon.ttl.markup-cache' => 30 * (24 * 60 * 60),
|
'gcdaemon.ttl.markup-cache' => 30 * (24 * 60 * 60),
|
||||||
|
'gcdaemon.ttl.task-archive' => 14 * (24 * 60 * 60),
|
||||||
|
|
||||||
|
|
||||||
// -- Feed ------------------------------------------------------------------ //
|
// -- Feed ------------------------------------------------------------------ //
|
||||||
|
|
13
resources/sql/patches/daemontaskarchive.sql
Normal file
13
resources/sql/patches/daemontaskarchive.sql
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
CREATE TABLE {$NAMESPACE}_worker.worker_archivetask (
|
||||||
|
id INT UNSIGNED PRIMARY KEY,
|
||||||
|
taskClass VARCHAR(255) NOT NULL COLLATE utf8_bin,
|
||||||
|
leaseOwner VARCHAR(255) COLLATE utf8_bin,
|
||||||
|
leaseExpires INT UNSIGNED,
|
||||||
|
failureCount INT UNSIGNED NOT NULL,
|
||||||
|
dataID INT UNSIGNED NOT NULL,
|
||||||
|
result INT UNSIGNED NOT NULL,
|
||||||
|
duration BIGINT UNSIGNED NOT NULL,
|
||||||
|
dateCreated INT UNSIGNED NOT NULL,
|
||||||
|
dateModified INT UNSIGNED NOT NULL,
|
||||||
|
key(dateCreated)
|
||||||
|
) ENGINE=InnoDB, COLLATE utf8_general_ci;
|
|
@ -33,10 +33,9 @@ $messages = id(new PhabricatorMetaMTAMail())->loadAllWhere(
|
||||||
|
|
||||||
foreach ($messages as $message) {
|
foreach ($messages as $message) {
|
||||||
if (!$message->getWorkerTaskID()) {
|
if (!$message->getWorkerTaskID()) {
|
||||||
$mailer_task = new PhabricatorWorkerTask();
|
$mailer_task = PhabricatorWorker::scheduleTask(
|
||||||
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
|
'PhabricatorMetaMTAWorker',
|
||||||
$mailer_task->setData($message->getID());
|
$message->getID());
|
||||||
$mailer_task->save();
|
|
||||||
|
|
||||||
$message->setWorkerTaskID($mailer_task->getID());
|
$message->setWorkerTaskID($mailer_task->getID());
|
||||||
$message->save();
|
$message->save();
|
||||||
|
|
|
@ -224,10 +224,7 @@ foreach ($commits as $commit) {
|
||||||
|
|
||||||
if ($all_from_repo && !$force_local) {
|
if ($all_from_repo && !$force_local) {
|
||||||
foreach ($classes as $class) {
|
foreach ($classes as $class) {
|
||||||
$task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask($class, $spec);
|
||||||
$task->setTaskClass($class);
|
|
||||||
$task->setData($spec);
|
|
||||||
$task->save();
|
|
||||||
|
|
||||||
$commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
|
$commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
|
||||||
echo " Queued '{$class}' for commit '{$commit_name}'.\n";
|
echo " Queued '{$class}' for commit '{$commit_name}'.\n";
|
||||||
|
|
|
@ -1140,6 +1140,8 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorUserStatusOverlapException' => 'applications/people/exception/PhabricatorUserStatusOverlapException.php',
|
'PhabricatorUserStatusOverlapException' => 'applications/people/exception/PhabricatorUserStatusOverlapException.php',
|
||||||
'PhabricatorUserTestCase' => 'applications/people/storage/__tests__/PhabricatorUserTestCase.php',
|
'PhabricatorUserTestCase' => 'applications/people/storage/__tests__/PhabricatorUserTestCase.php',
|
||||||
'PhabricatorWorker' => 'infrastructure/daemon/workers/PhabricatorWorker.php',
|
'PhabricatorWorker' => 'infrastructure/daemon/workers/PhabricatorWorker.php',
|
||||||
|
'PhabricatorWorkerActiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php',
|
||||||
|
'PhabricatorWorkerArchiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php',
|
||||||
'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php',
|
'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php',
|
||||||
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
|
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
|
||||||
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
|
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
|
||||||
|
@ -2300,6 +2302,8 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorUserStatusInvalidEpochException' => 'Exception',
|
'PhabricatorUserStatusInvalidEpochException' => 'Exception',
|
||||||
'PhabricatorUserStatusOverlapException' => 'Exception',
|
'PhabricatorUserStatusOverlapException' => 'Exception',
|
||||||
'PhabricatorUserTestCase' => 'PhabricatorTestCase',
|
'PhabricatorUserTestCase' => 'PhabricatorTestCase',
|
||||||
|
'PhabricatorWorkerActiveTask' => 'PhabricatorWorkerTask',
|
||||||
|
'PhabricatorWorkerArchiveTask' => 'PhabricatorWorkerTask',
|
||||||
'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO',
|
'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO',
|
||||||
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
|
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
|
||||||
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
|
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
|
||||||
|
|
|
@ -34,7 +34,7 @@ final class PhabricatorDaemonConsoleController
|
||||||
$daemon_panel->setHeader('Recently Launched Daemons');
|
$daemon_panel->setHeader('Recently Launched Daemons');
|
||||||
$daemon_panel->appendChild($daemon_table);
|
$daemon_panel->appendChild($daemon_table);
|
||||||
|
|
||||||
$tasks = id(new PhabricatorWorkerTask())->loadAllWhere(
|
$tasks = id(new PhabricatorWorkerActiveTask())->loadAllWhere(
|
||||||
'leaseOwner IS NOT NULL');
|
'leaseOwner IS NOT NULL');
|
||||||
|
|
||||||
$rows = array();
|
$rows = array();
|
||||||
|
@ -80,7 +80,7 @@ final class PhabricatorDaemonConsoleController
|
||||||
$leased_panel->setHeader('Leased Tasks');
|
$leased_panel->setHeader('Leased Tasks');
|
||||||
$leased_panel->appendChild($leased_table);
|
$leased_panel->appendChild($leased_table);
|
||||||
|
|
||||||
$task_table = new PhabricatorWorkerTask();
|
$task_table = new PhabricatorWorkerActiveTask();
|
||||||
$queued = queryfx_all(
|
$queued = queryfx_all(
|
||||||
$task_table->establishConnection('r'),
|
$task_table->establishConnection('r'),
|
||||||
'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass
|
'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass
|
||||||
|
|
|
@ -29,7 +29,7 @@ final class PhabricatorWorkerTaskDetailController
|
||||||
$request = $this->getRequest();
|
$request = $this->getRequest();
|
||||||
$user = $request->getUser();
|
$user = $request->getUser();
|
||||||
|
|
||||||
$task = id(new PhabricatorWorkerTask())->load($this->id);
|
$task = id(new PhabricatorWorkerActiveTask())->load($this->id);
|
||||||
if (!$task) {
|
if (!$task) {
|
||||||
$error_view = new AphrontErrorView();
|
$error_view = new AphrontErrorView();
|
||||||
$error_view->setTitle('No Such Task');
|
$error_view->setTitle('No Such Task');
|
||||||
|
|
|
@ -31,7 +31,7 @@ final class PhabricatorWorkerTaskUpdateController
|
||||||
$request = $this->getRequest();
|
$request = $this->getRequest();
|
||||||
$user = $request->getUser();
|
$user = $request->getUser();
|
||||||
|
|
||||||
$task = id(new PhabricatorWorkerTask())->load($this->id);
|
$task = id(new PhabricatorWorkerActiveTask())->load($this->id);
|
||||||
if (!$task) {
|
if (!$task) {
|
||||||
return new Aphront404Response();
|
return new Aphront404Response();
|
||||||
}
|
}
|
||||||
|
|
|
@ -59,10 +59,7 @@ final class DrydockAllocator {
|
||||||
$worker = new DrydockAllocatorWorker($data);
|
$worker = new DrydockAllocatorWorker($data);
|
||||||
$worker->executeTask();
|
$worker->executeTask();
|
||||||
} else {
|
} else {
|
||||||
$task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask('DrydockAllocatorWorker', $data);
|
||||||
$task->setTaskClass('DrydockAllocatorWorker');
|
|
||||||
$task->setData($data);
|
|
||||||
$task->save();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return $lease;
|
return $lease;
|
||||||
|
|
|
@ -298,10 +298,10 @@ final class PhabricatorMetaMTAMail extends PhabricatorMetaMTADAO {
|
||||||
parent::didWriteData();
|
parent::didWriteData();
|
||||||
|
|
||||||
if (!$this->getWorkerTaskID()) {
|
if (!$this->getWorkerTaskID()) {
|
||||||
$mailer_task = new PhabricatorWorkerTask();
|
$mailer_task = PhabricatorWorker::scheduleTask(
|
||||||
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
|
'PhabricatorMetaMTAWorker',
|
||||||
$mailer_task->setData($this->getID());
|
$this->getID());
|
||||||
$mailer_task->save();
|
|
||||||
$this->setWorkerTaskID($mailer_task->getID());
|
$this->setWorkerTaskID($mailer_task->getID());
|
||||||
$this->save();
|
$this->save();
|
||||||
}
|
}
|
||||||
|
|
|
@ -450,12 +450,9 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
throw new Exception("Unknown repository type '{$vcs}'!");
|
throw new Exception("Unknown repository type '{$vcs}'!");
|
||||||
}
|
}
|
||||||
|
|
||||||
$task = new PhabricatorWorkerTask();
|
|
||||||
$task->setTaskClass($class);
|
|
||||||
$data['commitID'] = $commit->getID();
|
$data['commitID'] = $commit->getID();
|
||||||
|
|
||||||
$task->setData($data);
|
PhabricatorWorker::scheduleTask($class, $data);
|
||||||
$task->save();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -73,13 +73,11 @@ final class PhabricatorRepositoryCommitOwnersWorker
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($this->shouldQueueFollowupTasks()) {
|
if ($this->shouldQueueFollowupTasks()) {
|
||||||
$herald_task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask(
|
||||||
$herald_task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker');
|
'PhabricatorRepositoryCommitHeraldWorker',
|
||||||
$herald_task->setData(
|
|
||||||
array(
|
array(
|
||||||
'commitID' => $commit->getID(),
|
'commitID' => $commit->getID(),
|
||||||
));
|
));
|
||||||
$herald_task->save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -76,13 +76,11 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
|
||||||
PhabricatorSearchCommitIndexer::indexCommit($commit);
|
PhabricatorSearchCommitIndexer::indexCommit($commit);
|
||||||
|
|
||||||
if ($this->shouldQueueFollowupTasks()) {
|
if ($this->shouldQueueFollowupTasks()) {
|
||||||
$owner_task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask(
|
||||||
$owner_task->setTaskClass('PhabricatorRepositoryCommitOwnersWorker');
|
'PhabricatorRepositoryCommitOwnersWorker',
|
||||||
$owner_task->setData(
|
|
||||||
array(
|
array(
|
||||||
'commitID' => $commit->getID(),
|
'commitID' => $commit->getID(),
|
||||||
));
|
));
|
||||||
$owner_task->save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -69,13 +69,11 @@ final class PhabricatorRepositoryGitCommitMessageParserWorker
|
||||||
$this->updateCommitData($author, $message, $committer);
|
$this->updateCommitData($author, $message, $committer);
|
||||||
|
|
||||||
if ($this->shouldQueueFollowupTasks()) {
|
if ($this->shouldQueueFollowupTasks()) {
|
||||||
$task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask(
|
||||||
$task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
|
'PhabricatorRepositoryGitCommitChangeParserWorker',
|
||||||
$task->setData(
|
|
||||||
array(
|
array(
|
||||||
'commitID' => $commit->getID(),
|
'commitID' => $commit->getID(),
|
||||||
));
|
));
|
||||||
$task->save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,14 +37,11 @@ final class PhabricatorRepositoryMercurialCommitMessageParserWorker
|
||||||
$this->updateCommitData($author, $message);
|
$this->updateCommitData($author, $message);
|
||||||
|
|
||||||
if ($this->shouldQueueFollowupTasks()) {
|
if ($this->shouldQueueFollowupTasks()) {
|
||||||
$task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask(
|
||||||
$task->setTaskClass(
|
'PhabricatorRepositoryMercurialCommitChangeParserWorker',
|
||||||
'PhabricatorRepositoryMercurialCommitChangeParserWorker');
|
|
||||||
$task->setData(
|
|
||||||
array(
|
array(
|
||||||
'commitID' => $commit->getID(),
|
'commitID' => $commit->getID(),
|
||||||
));
|
));
|
||||||
$task->save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -38,13 +38,11 @@ final class PhabricatorRepositorySvnCommitMessageParserWorker
|
||||||
$this->updateCommitData($author, $message);
|
$this->updateCommitData($author, $message);
|
||||||
|
|
||||||
if ($this->shouldQueueFollowupTasks()) {
|
if ($this->shouldQueueFollowupTasks()) {
|
||||||
$task = new PhabricatorWorkerTask();
|
PhabricatorWorker::scheduleTask(
|
||||||
$task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker');
|
'PhabricatorRepositorySvnCommitChangeParserWorker',
|
||||||
$task->setData(
|
|
||||||
array(
|
array(
|
||||||
'commitID' => $commit->getID(),
|
'commitID' => $commit->getID(),
|
||||||
));
|
));
|
||||||
$task->save();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -64,12 +64,14 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
|
||||||
$n_daemon = $this->collectDaemonLogs();
|
$n_daemon = $this->collectDaemonLogs();
|
||||||
$n_parse = $this->collectParseCaches();
|
$n_parse = $this->collectParseCaches();
|
||||||
$n_markup = $this->collectMarkupCaches();
|
$n_markup = $this->collectMarkupCaches();
|
||||||
|
$n_tasks = $this->collectArchivedTasks();
|
||||||
|
|
||||||
$collected = array(
|
$collected = array(
|
||||||
'Herald Transcript' => $n_herald,
|
'Herald Transcript' => $n_herald,
|
||||||
'Daemon Log' => $n_daemon,
|
'Daemon Log' => $n_daemon,
|
||||||
'Differential Parse Cache' => $n_parse,
|
'Differential Parse Cache' => $n_parse,
|
||||||
'Markup Cache' => $n_markup,
|
'Markup Cache' => $n_markup,
|
||||||
|
'Archived Tasks' => $n_tasks,
|
||||||
);
|
);
|
||||||
$collected = array_filter($collected);
|
$collected = array_filter($collected);
|
||||||
|
|
||||||
|
@ -172,4 +174,46 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
|
||||||
return $conn_w->getAffectedRows();
|
return $conn_w->getAffectedRows();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function collectArchivedTasks() {
|
||||||
|
$key = 'gcdaemon.ttl.task-archive';
|
||||||
|
$ttl = PhabricatorEnv::getEnvConfig($key);
|
||||||
|
if ($ttl <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
$table = new PhabricatorWorkerArchiveTask();
|
||||||
|
$data_table = new PhabricatorWorkerTaskData();
|
||||||
|
$conn_w = $table->establishConnection('w');
|
||||||
|
|
||||||
|
$rows = queryfx_all(
|
||||||
|
$conn_w,
|
||||||
|
'SELECT id, dataID FROM %T WHERE dateCreated < %d LIMIT 100',
|
||||||
|
$table->getTableName(),
|
||||||
|
time() - $ttl);
|
||||||
|
|
||||||
|
if (!$rows) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
$data_ids = array_filter(ipull($rows, 'dataID'));
|
||||||
|
$task_ids = ipull($rows, 'id');
|
||||||
|
|
||||||
|
$table->openTransaction();
|
||||||
|
if ($data_ids) {
|
||||||
|
queryfx(
|
||||||
|
$conn_w,
|
||||||
|
'DELETE FROM %T WHERE id IN (%Ld)',
|
||||||
|
$data_table->getTableName(),
|
||||||
|
$data_ids);
|
||||||
|
}
|
||||||
|
queryfx(
|
||||||
|
$conn_w,
|
||||||
|
'DELETE FROM %T WHERE id IN (%Ld)',
|
||||||
|
$table->getTableName(),
|
||||||
|
$task_ids);
|
||||||
|
$table->saveTransaction();
|
||||||
|
|
||||||
|
return count($task_ids);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,11 +21,13 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
public function run() {
|
public function run() {
|
||||||
$lease_ownership_name = $this->getLeaseOwnershipName();
|
$lease_ownership_name = $this->getLeaseOwnershipName();
|
||||||
|
|
||||||
$task_table = new PhabricatorWorkerTask();
|
$task_table = new PhabricatorWorkerActiveTask();
|
||||||
$taskdata_table = new PhabricatorWorkerTaskData();
|
$taskdata_table = new PhabricatorWorkerTaskData();
|
||||||
|
|
||||||
$sleep = 0;
|
$sleep = 0;
|
||||||
do {
|
do {
|
||||||
|
$this->log('Dequeuing a task...');
|
||||||
|
|
||||||
$conn_w = $task_table->establishConnection('w');
|
$conn_w = $task_table->establishConnection('w');
|
||||||
queryfx(
|
queryfx(
|
||||||
$conn_w,
|
$conn_w,
|
||||||
|
@ -36,6 +38,7 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
$rows = $conn_w->getAffectedRows();
|
$rows = $conn_w->getAffectedRows();
|
||||||
|
|
||||||
if (!$rows) {
|
if (!$rows) {
|
||||||
|
$this->log('No unleased tasks. Dequeuing an expired lease...');
|
||||||
queryfx(
|
queryfx(
|
||||||
$conn_w,
|
$conn_w,
|
||||||
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
|
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
|
||||||
|
@ -70,6 +73,11 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach ($tasks as $task) {
|
foreach ($tasks as $task) {
|
||||||
|
$id = $task->getID();
|
||||||
|
$class = $task->getTaskClass();
|
||||||
|
|
||||||
|
$this->log("Working on task {$id} ({$class})...");
|
||||||
|
|
||||||
// TODO: We should detect if we acquired a task with an expired lease
|
// TODO: We should detect if we acquired a task with an expired lease
|
||||||
// and log about it / bump up failure count.
|
// and log about it / bump up failure count.
|
||||||
|
|
||||||
|
@ -77,7 +85,6 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
// failure count and fail it permanently.
|
// failure count and fail it permanently.
|
||||||
|
|
||||||
$data = idx($task_data, $task->getID());
|
$data = idx($task_data, $task->getID());
|
||||||
$class = $task->getTaskClass();
|
|
||||||
try {
|
try {
|
||||||
if (!class_exists($class) ||
|
if (!class_exists($class) ||
|
||||||
!is_subclass_of($class, 'PhabricatorWorker')) {
|
!is_subclass_of($class, 'PhabricatorWorker')) {
|
||||||
|
@ -91,19 +98,19 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
$task->setLeaseDuration($lease);
|
$task->setLeaseDuration($lease);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$t_start = microtime(true);
|
||||||
$worker->executeTask();
|
$worker->executeTask();
|
||||||
|
$t_end = microtime(true);
|
||||||
|
|
||||||
$task->delete();
|
$task->archiveTask(
|
||||||
if ($data !== null) {
|
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
|
||||||
queryfx(
|
(int)(1000000 * ($t_end - $t_start)));
|
||||||
$conn_w,
|
$this->log("Task {$id} complete! Moved to archive.");
|
||||||
'DELETE FROM %T WHERE id = %d',
|
|
||||||
$taskdata_table->getTableName(),
|
|
||||||
$task->getDataID());
|
|
||||||
}
|
|
||||||
} catch (Exception $ex) {
|
} catch (Exception $ex) {
|
||||||
$task->setFailureCount($task->getFailureCount() + 1);
|
$task->setFailureCount($task->getFailureCount() + 1);
|
||||||
$task->save();
|
$task->save();
|
||||||
|
|
||||||
|
$this->log("Task {$id} failed!");
|
||||||
throw $ex;
|
throw $ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright 2011 Facebook, Inc.
|
* Copyright 2012 Facebook, Inc.
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -37,4 +37,12 @@ abstract class PhabricatorWorker {
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract protected function doWork();
|
abstract protected function doWork();
|
||||||
|
|
||||||
|
final public static function scheduleTask($task_class, $data) {
|
||||||
|
return id(new PhabricatorWorkerActiveTask())
|
||||||
|
->setTaskClass($task_class)
|
||||||
|
->setData($data)
|
||||||
|
->save();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright 2012 Facebook, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
|
||||||
|
|
||||||
|
private $serverTime;
|
||||||
|
private $localTime;
|
||||||
|
|
||||||
|
public function getTableName() {
|
||||||
|
return 'worker_task';
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getConfiguration() {
|
||||||
|
return array(
|
||||||
|
self::CONFIG_TIMESTAMPS => false,
|
||||||
|
) + parent::getConfiguration();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setServerTime($server_time) {
|
||||||
|
$this->serverTime = $server_time;
|
||||||
|
$this->localTime = time();
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setLeaseDuration($lease_duration) {
|
||||||
|
$server_lease_expires = $this->serverTime + $lease_duration;
|
||||||
|
$this->setLeaseExpires($server_lease_expires);
|
||||||
|
return $this->save();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function save() {
|
||||||
|
$this->checkLease();
|
||||||
|
|
||||||
|
$is_new = !$this->getID();
|
||||||
|
if ($is_new) {
|
||||||
|
$this->failureCount = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($is_new && $this->getData()) {
|
||||||
|
$data = new PhabricatorWorkerTaskData();
|
||||||
|
$data->setData($this->getData());
|
||||||
|
$data->save();
|
||||||
|
|
||||||
|
$this->setDataID($data->getID());
|
||||||
|
}
|
||||||
|
|
||||||
|
return parent::save();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function checkLease() {
|
||||||
|
if ($this->leaseOwner) {
|
||||||
|
$current_server_time = $this->serverTime + (time() - $this->localTime);
|
||||||
|
if ($current_server_time >= $this->leaseExpires) {
|
||||||
|
throw new Exception("Trying to update task after lease expiration!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function delete() {
|
||||||
|
throw new Exception(
|
||||||
|
"Active tasks can not be deleted directly. ".
|
||||||
|
"Use archiveTask() to move tasks to the archive.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public function archiveTask($result, $duration) {
|
||||||
|
if (!$this->getID()) {
|
||||||
|
throw new Exception(
|
||||||
|
"Attempting to archive a task which hasn't been save()d!");
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->checkLease();
|
||||||
|
|
||||||
|
$archive = id(new PhabricatorWorkerArchiveTask())
|
||||||
|
->setID($this->getID())
|
||||||
|
->setTaskClass($this->getTaskClass())
|
||||||
|
->setLeaseOwner($this->getLeaseOwner())
|
||||||
|
->setLeaseExpires($this->getLeaseExpires())
|
||||||
|
->setFailureCount($this->getFailureCount())
|
||||||
|
->setDataID($this->getDataID())
|
||||||
|
->setResult($result)
|
||||||
|
->setDuration($duration);
|
||||||
|
|
||||||
|
// NOTE: This deletes the active task (this object)!
|
||||||
|
$archive->save();
|
||||||
|
|
||||||
|
return $archive;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,66 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright 2012 Facebook, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
final class PhabricatorWorkerArchiveTask extends PhabricatorWorkerTask {
|
||||||
|
|
||||||
|
const RESULT_SUCCESS = 0;
|
||||||
|
const RESULT_FAILURE = 1;
|
||||||
|
|
||||||
|
protected $duration;
|
||||||
|
protected $result;
|
||||||
|
|
||||||
|
public function save() {
|
||||||
|
if (!$this->getID()) {
|
||||||
|
throw new Exception(
|
||||||
|
"Trying to archive a task with no ID.");
|
||||||
|
}
|
||||||
|
|
||||||
|
$other = new PhabricatorWorkerActiveTask();
|
||||||
|
$conn_w = $this->establishConnection('w');
|
||||||
|
|
||||||
|
$this->openTransaction();
|
||||||
|
queryfx(
|
||||||
|
$conn_w,
|
||||||
|
'DELETE FROM %T WHERE id = %d',
|
||||||
|
$other->getTableName(),
|
||||||
|
$this->getID());
|
||||||
|
$result = parent::insert();
|
||||||
|
$this->saveTransaction();
|
||||||
|
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function delete() {
|
||||||
|
$this->openTransaction();
|
||||||
|
if ($this->getDataID()) {
|
||||||
|
$conn_w = $this->establishConnection('w');
|
||||||
|
$data_table = new PhabricatorWorkerTaskData();
|
||||||
|
|
||||||
|
queryfx(
|
||||||
|
$conn_w,
|
||||||
|
'DELETE FROM %T WHERE id = %d',
|
||||||
|
$data_table->getTableName(),
|
||||||
|
$this->getDataID());
|
||||||
|
}
|
||||||
|
|
||||||
|
$result = parent::delete();
|
||||||
|
$this->saveTransaction();
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright 2011 Facebook, Inc.
|
* Copyright 2012 Facebook, Inc.
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -17,7 +17,9 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
abstract class PhabricatorWorkerDAO extends PhabricatorLiskDAO {
|
abstract class PhabricatorWorkerDAO extends PhabricatorLiskDAO {
|
||||||
|
|
||||||
public function getApplicationName() {
|
public function getApplicationName() {
|
||||||
return 'worker';
|
return 'worker';
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,60 +16,18 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
final class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
|
abstract class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
|
||||||
|
|
||||||
|
// NOTE: If you provide additional fields here, make sure they are handled
|
||||||
|
// correctly in the archiving process.
|
||||||
protected $taskClass;
|
protected $taskClass;
|
||||||
protected $leaseOwner;
|
protected $leaseOwner;
|
||||||
protected $leaseExpires;
|
protected $leaseExpires;
|
||||||
protected $failureCount;
|
protected $failureCount;
|
||||||
protected $dataID;
|
protected $dataID;
|
||||||
|
|
||||||
private $serverTime;
|
|
||||||
private $localTime;
|
|
||||||
private $data;
|
private $data;
|
||||||
|
|
||||||
public function getConfiguration() {
|
|
||||||
return array(
|
|
||||||
self::CONFIG_TIMESTAMPS => false,
|
|
||||||
) + parent::getConfiguration();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setServerTime($server_time) {
|
|
||||||
$this->serverTime = $server_time;
|
|
||||||
$this->localTime = time();
|
|
||||||
return $this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setLeaseDuration($lease_duration) {
|
|
||||||
$server_lease_expires = $this->serverTime + $lease_duration;
|
|
||||||
$this->setLeaseExpires($server_lease_expires);
|
|
||||||
return $this->save();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function save() {
|
|
||||||
if ($this->leaseOwner) {
|
|
||||||
$current_server_time = $this->serverTime + (time() - $this->localTime);
|
|
||||||
if ($current_server_time >= $this->leaseExpires) {
|
|
||||||
throw new Exception("Trying to update task after lease expiration!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$is_new = !$this->getID();
|
|
||||||
if ($is_new) {
|
|
||||||
$this->failureCount = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($is_new && $this->data) {
|
|
||||||
$data = new PhabricatorWorkerTaskData();
|
|
||||||
$data->setData($this->data);
|
|
||||||
$data->save();
|
|
||||||
|
|
||||||
$this->setDataID($data->getID());
|
|
||||||
}
|
|
||||||
|
|
||||||
return parent::save();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setData($data) {
|
public function setData($data) {
|
||||||
$this->data = $data;
|
$this->data = $data;
|
||||||
return $this;
|
return $this;
|
||||||
|
|
|
@ -1016,6 +1016,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
|
||||||
'type' => 'sql',
|
'type' => 'sql',
|
||||||
'name' => $this->getPatchPath('statustxt.sql'),
|
'name' => $this->getPatchPath('statustxt.sql'),
|
||||||
),
|
),
|
||||||
|
'daemontaskarchive.sql' => array(
|
||||||
|
'type' => 'sql',
|
||||||
|
'name' => $this->getPatchPath('daemontaskarchive.sql'),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue