revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-21 22:32:41 +01:00
Code Issues Releases Wiki Activity

Move completed tasks to an "archive" table and delete them in the GC

Browse source 
Summary:
Currently, when taskmasters complete a task it is immediately deleted. This prevents us from doing some general things, like:

  - Supporting the idea of permanent failure (e.g., after N failures just stop trying).
  - Showing the user how fast taskmasters are completing tasks.
  - Showing the user how long tasks took to complete.

Having better visibility into this is important to Drydock, which builds on the task system. Also, generally buff debug output for task execution.

Test Plan: Ran `bin/phd debug taskmaster`. Ran `bin/phd debug garbage`. Queued some tasks via various systems.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2015

Differential Revision: https://secure.phabricator.com/D3852
This commit is contained in:
epriestley 2012-10-31 15:22:16 -07:00
parent 4edf8ae2fc
commit 5903ed650c
24 changed files with 292 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
conf/default.conf.php
View file

@ -1126,6 +1126,7 @@ return array(
'gcdaemon.ttl.daemon-logs' => 7 * (24 * 60 * 60),
'gcdaemon.ttl.differential-parse-cache' => 14 * (24 * 60 * 60),
'gcdaemon.ttl.markup-cache' => 30 * (24 * 60 * 60),
'gcdaemon.ttl.task-archive' => 14 * (24 * 60 * 60),
// -- Feed ------------------------------------------------------------------ //

13
resources/sql/patches/daemontaskarchive.sql Normal file
View file

@ -0,0 +1,13 @@
CREATE TABLE {$NAMESPACE}_worker.worker_archivetask (
id INT UNSIGNED PRIMARY KEY,
taskClass VARCHAR(255) NOT NULL COLLATE utf8_bin,
leaseOwner VARCHAR(255) COLLATE utf8_bin,
leaseExpires INT UNSIGNED,
failureCount INT UNSIGNED NOT NULL,
dataID INT UNSIGNED NOT NULL,
result INT UNSIGNED NOT NULL,
duration BIGINT UNSIGNED NOT NULL,
dateCreated INT UNSIGNED NOT NULL,
dateModified INT UNSIGNED NOT NULL,
key(dateCreated)
) ENGINE=InnoDB, COLLATE utf8_general_ci;

7
scripts/mail/create_worker_tasks.php
View file

@ -33,10 +33,9 @@ $messages = id(new PhabricatorMetaMTAMail())->loadAllWhere(
foreach ($messages as $message) {
if (!$message->getWorkerTaskID()) {
$mailer_task = new PhabricatorWorkerTask();
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
$mailer_task->setData($message->getID());
$mailer_task->save();
$mailer_task = PhabricatorWorker::scheduleTask(
'PhabricatorMetaMTAWorker',
$message->getID());
$message->setWorkerTaskID($mailer_task->getID());
$message->save();

5
scripts/repository/reparse.php
View file

@ -224,10 +224,7 @@ foreach ($commits as $commit) {
if ($all_from_repo && !$force_local) {
foreach ($classes as $class) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$task->setData($spec);
$task->save();
PhabricatorWorker::scheduleTask($class, $spec);
$commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
echo " Queued '{$class}' for commit '{$commit_name}'.\n";

4
src/__phutil_library_map__.php
View file

@ -1140,6 +1140,8 @@ phutil_register_library_map(array(
'PhabricatorUserStatusOverlapException' => 'applications/people/exception/PhabricatorUserStatusOverlapException.php',
'PhabricatorUserTestCase' => 'applications/people/storage/__tests__/PhabricatorUserTestCase.php',
'PhabricatorWorker' => 'infrastructure/daemon/workers/PhabricatorWorker.php',
'PhabricatorWorkerActiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php',
'PhabricatorWorkerArchiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php',
'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php',
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
@ -2300,6 +2302,8 @@ phutil_register_library_map(array(
'PhabricatorUserStatusInvalidEpochException' => 'Exception',
'PhabricatorUserStatusOverlapException' => 'Exception',
'PhabricatorUserTestCase' => 'PhabricatorTestCase',
'PhabricatorWorkerActiveTask' => 'PhabricatorWorkerTask',
'PhabricatorWorkerArchiveTask' => 'PhabricatorWorkerTask',
'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO',
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',

4
src/applications/daemon/controller/PhabricatorDaemonConsoleController.php
View file

@ -34,7 +34,7 @@ final class PhabricatorDaemonConsoleController
$daemon_panel->setHeader('Recently Launched Daemons');
$daemon_panel->appendChild($daemon_table);
$tasks = id(new PhabricatorWorkerTask())->loadAllWhere(
$tasks = id(new PhabricatorWorkerActiveTask())->loadAllWhere(
'leaseOwner IS NOT NULL');
$rows = array();
@ -80,7 +80,7 @@ final class PhabricatorDaemonConsoleController
$leased_panel->setHeader('Leased Tasks');
$leased_panel->appendChild($leased_table);
$task_table = new PhabricatorWorkerTask();
$task_table = new PhabricatorWorkerActiveTask();
$queued = queryfx_all(
$task_table->establishConnection('r'),
'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass

2
src/applications/daemon/controller/PhabricatorWorkerTaskDetailController.php
View file

@ -29,7 +29,7 @@ final class PhabricatorWorkerTaskDetailController
$request = $this->getRequest();
$user = $request->getUser();
$task = id(new PhabricatorWorkerTask())->load($this->id);
$task = id(new PhabricatorWorkerActiveTask())->load($this->id);
if (!$task) {
$error_view = new AphrontErrorView();
$error_view->setTitle('No Such Task');

2
src/applications/daemon/controller/PhabricatorWorkerTaskUpdateController.php
View file

@ -31,7 +31,7 @@ final class PhabricatorWorkerTaskUpdateController
$request = $this->getRequest();
$user = $request->getUser();
$task = id(new PhabricatorWorkerTask())->load($this->id);
$task = id(new PhabricatorWorkerActiveTask())->load($this->id);
if (!$task) {
return new Aphront404Response();
}

5
src/applications/drydock/allocator/DrydockAllocator.php
View file

@ -59,10 +59,7 @@ final class DrydockAllocator {
$worker = new DrydockAllocatorWorker($data);
$worker->executeTask();
} else {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('DrydockAllocatorWorker');
$task->setData($data);
$task->save();
PhabricatorWorker::scheduleTask('DrydockAllocatorWorker', $data);
}
return $lease;

8
src/applications/metamta/storage/PhabricatorMetaMTAMail.php
View file

@ -298,10 +298,10 @@ final class PhabricatorMetaMTAMail extends PhabricatorMetaMTADAO {
parent::didWriteData();
if (!$this->getWorkerTaskID()) {
$mailer_task = new PhabricatorWorkerTask();
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
$mailer_task->setData($this->getID());
$mailer_task->save();
$mailer_task = PhabricatorWorker::scheduleTask(
'PhabricatorMetaMTAWorker',
$this->getID());
$this->setWorkerTaskID($mailer_task->getID());
$this->save();
}

5
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
View file

@ -450,12 +450,9 @@ final class PhabricatorRepositoryPullLocalDaemon
throw new Exception("Unknown repository type '{$vcs}'!");
}
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$data['commitID'] = $commit->getID();
$task->setData($data);
$task->save();
PhabricatorWorker::scheduleTask($class, $data);
}

6
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
View file

@ -73,13 +73,11 @@ final class PhabricatorRepositoryCommitOwnersWorker
}
if ($this->shouldQueueFollowupTasks()) {
$herald_task = new PhabricatorWorkerTask();
$herald_task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker');
$herald_task->setData(
PhabricatorWorker::scheduleTask(
'PhabricatorRepositoryCommitHeraldWorker',
array(
'commitID' => $commit->getID(),
));
$herald_task->save();
}
}

6
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
View file

@ -76,13 +76,11 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
PhabricatorSearchCommitIndexer::indexCommit($commit);
if ($this->shouldQueueFollowupTasks()) {
$owner_task = new PhabricatorWorkerTask();
$owner_task->setTaskClass('PhabricatorRepositoryCommitOwnersWorker');
$owner_task->setData(
PhabricatorWorker::scheduleTask(
'PhabricatorRepositoryCommitOwnersWorker',
array(
'commitID' => $commit->getID(),
));
$owner_task->save();
}
}

6
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
View file

@ -69,13 +69,11 @@ final class PhabricatorRepositoryGitCommitMessageParserWorker
$this->updateCommitData($author, $message, $committer);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
$task->setData(
PhabricatorWorker::scheduleTask(
'PhabricatorRepositoryGitCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),
));
$task->save();
}
}

7
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
View file

@ -37,14 +37,11 @@ final class PhabricatorRepositoryMercurialCommitMessageParserWorker
$this->updateCommitData($author, $message);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass(
'PhabricatorRepositoryMercurialCommitChangeParserWorker');
$task->setData(
PhabricatorWorker::scheduleTask(
'PhabricatorRepositoryMercurialCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),
));
$task->save();
}
}

6
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
View file

@ -38,13 +38,11 @@ final class PhabricatorRepositorySvnCommitMessageParserWorker
$this->updateCommitData($author, $message);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker');
$task->setData(
PhabricatorWorker::scheduleTask(
'PhabricatorRepositorySvnCommitChangeParserWorker',
array(
'commitID' => $commit->getID(),
));
$task->save();
}
}

44
src/infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php
View file

@ -64,12 +64,14 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
$n_daemon = $this->collectDaemonLogs();
$n_parse = $this->collectParseCaches();
$n_markup = $this->collectMarkupCaches();
$n_tasks = $this->collectArchivedTasks();
$collected = array(
'Herald Transcript' => $n_herald,
'Daemon Log' => $n_daemon,
'Differential Parse Cache' => $n_parse,
'Markup Cache' => $n_markup,
'Archived Tasks' => $n_tasks,
);
$collected = array_filter($collected);
@ -172,4 +174,46 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
return $conn_w->getAffectedRows();
}
private function collectArchivedTasks() {
$key = 'gcdaemon.ttl.task-archive';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorWorkerArchiveTask();
$data_table = new PhabricatorWorkerTaskData();
$conn_w = $table->establishConnection('w');
$rows = queryfx_all(
$conn_w,
'SELECT id, dataID FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
if (!$rows) {
return 0;
}
$data_ids = array_filter(ipull($rows, 'dataID'));
$task_ids = ipull($rows, 'id');
$table->openTransaction();
if ($data_ids) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$data_table->getTableName(),
$data_ids);
}
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$table->getTableName(),
$task_ids);
$table->saveTransaction();
return count($task_ids);
}
}

27
src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
View file

@ -21,11 +21,13 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
public function run() {
$lease_ownership_name = $this->getLeaseOwnershipName();
$task_table = new PhabricatorWorkerTask();
$task_table = new PhabricatorWorkerActiveTask();
$taskdata_table = new PhabricatorWorkerTaskData();
$sleep = 0;
do {
$this->log('Dequeuing a task...');
$conn_w = $task_table->establishConnection('w');
queryfx(
$conn_w,
@ -36,6 +38,7 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
$rows = $conn_w->getAffectedRows();
if (!$rows) {
$this->log('No unleased tasks. Dequeuing an expired lease...');
queryfx(
$conn_w,
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
@ -70,6 +73,11 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
}
foreach ($tasks as $task) {
$id = $task->getID();
$class = $task->getTaskClass();
$this->log("Working on task {$id} ({$class})...");
// TODO: We should detect if we acquired a task with an expired lease
// and log about it / bump up failure count.
@ -77,7 +85,6 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
// failure count and fail it permanently.
$data = idx($task_data, $task->getID());
$class = $task->getTaskClass();
try {
if (!class_exists($class) ||
!is_subclass_of($class, 'PhabricatorWorker')) {
@ -91,19 +98,19 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
$task->setLeaseDuration($lease);
}
$t_start = microtime(true);
$worker->executeTask();
$t_end = microtime(true);
$task->delete();
if ($data !== null) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE id = %d',
$taskdata_table->getTableName(),
$task->getDataID());
}
$task->archiveTask(
PhabricatorWorkerArchiveTask::RESULT_SUCCESS,
(int)(1000000 * ($t_end - $t_start)));
$this->log("Task {$id} complete! Moved to archive.");
} catch (Exception $ex) {
$task->setFailureCount($task->getFailureCount() + 1);
$task->save();
$this->log("Task {$id} failed!");
throw $ex;
}
}

10
src/infrastructure/daemon/workers/PhabricatorWorker.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -37,4 +37,12 @@ abstract class PhabricatorWorker {
}
abstract protected function doWork();
final public static function scheduleTask($task_class, $data) {
return id(new PhabricatorWorkerActiveTask())
->setTaskClass($task_class)
->setData($data)
->save();
}
}

104
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php Normal file
View file

@ -0,0 +1,104 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
private $serverTime;
private $localTime;
public function getTableName() {
return 'worker_task';
}
public function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
) + parent::getConfiguration();
}
public function setServerTime($server_time) {
$this->serverTime = $server_time;
$this->localTime = time();
return $this;
}
public function setLeaseDuration($lease_duration) {
$server_lease_expires = $this->serverTime + $lease_duration;
$this->setLeaseExpires($server_lease_expires);
return $this->save();
}
public function save() {
$this->checkLease();
$is_new = !$this->getID();
if ($is_new) {
$this->failureCount = 0;
}
if ($is_new && $this->getData()) {
$data = new PhabricatorWorkerTaskData();
$data->setData($this->getData());
$data->save();
$this->setDataID($data->getID());
}
return parent::save();
}
protected function checkLease() {
if ($this->leaseOwner) {
$current_server_time = $this->serverTime + (time() - $this->localTime);
if ($current_server_time >= $this->leaseExpires) {
throw new Exception("Trying to update task after lease expiration!");
}
}
}
public function delete() {
throw new Exception(
"Active tasks can not be deleted directly. ".
"Use archiveTask() to move tasks to the archive.");
}
public function archiveTask($result, $duration) {
if (!$this->getID()) {
throw new Exception(
"Attempting to archive a task which hasn't been save()d!");
}
$this->checkLease();
$archive = id(new PhabricatorWorkerArchiveTask())
->setID($this->getID())
->setTaskClass($this->getTaskClass())
->setLeaseOwner($this->getLeaseOwner())
->setLeaseExpires($this->getLeaseExpires())
->setFailureCount($this->getFailureCount())
->setDataID($this->getDataID())
->setResult($result)
->setDuration($duration);
// NOTE: This deletes the active task (this object)!
$archive->save();
return $archive;
}
}

66
src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php Normal file
View file

@ -0,0 +1,66 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorWorkerArchiveTask extends PhabricatorWorkerTask {
const RESULT_SUCCESS = 0;
const RESULT_FAILURE = 1;
protected $duration;
protected $result;
public function save() {
if (!$this->getID()) {
throw new Exception(
"Trying to archive a task with no ID.");
}
$other = new PhabricatorWorkerActiveTask();
$conn_w = $this->establishConnection('w');
$this->openTransaction();
queryfx(
$conn_w,
'DELETE FROM %T WHERE id = %d',
$other->getTableName(),
$this->getID());
$result = parent::insert();
$this->saveTransaction();
return $result;
}
public function delete() {
$this->openTransaction();
if ($this->getDataID()) {
$conn_w = $this->establishConnection('w');
$data_table = new PhabricatorWorkerTaskData();
queryfx(
$conn_w,
'DELETE FROM %T WHERE id = %d',
$data_table->getTableName(),
$this->getDataID());
}
$result = parent::delete();
$this->saveTransaction();
return $result;
}
}

4
src/infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,7 +17,9 @@
*/
abstract class PhabricatorWorkerDAO extends PhabricatorLiskDAO {
public function getApplicationName() {
return 'worker';
}
}

48
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
View file

@ -16,60 +16,18 @@
* limitations under the License.
*/
final class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
abstract class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
// NOTE: If you provide additional fields here, make sure they are handled
// correctly in the archiving process.
protected $taskClass;
protected $leaseOwner;
protected $leaseExpires;
protected $failureCount;
protected $dataID;
private $serverTime;
private $localTime;
private $data;
public function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
) + parent::getConfiguration();
}
public function setServerTime($server_time) {
$this->serverTime = $server_time;
$this->localTime = time();
return $this;
}
public function setLeaseDuration($lease_duration) {
$server_lease_expires = $this->serverTime + $lease_duration;
$this->setLeaseExpires($server_lease_expires);
return $this->save();
}
public function save() {
if ($this->leaseOwner) {
$current_server_time = $this->serverTime + (time() - $this->localTime);
if ($current_server_time >= $this->leaseExpires) {
throw new Exception("Trying to update task after lease expiration!");
}
}
$is_new = !$this->getID();
if ($is_new) {
$this->failureCount = 0;
}
if ($is_new && $this->data) {
$data = new PhabricatorWorkerTaskData();
$data->setData($this->data);
$data->save();
$this->setDataID($data->getID());
}
return parent::save();
}
public function setData($data) {
$this->data = $data;
return $this;

4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
View file

@ -1016,6 +1016,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
'type' => 'sql',
'name' => $this->getPatchPath('statustxt.sql'),
),
'daemontaskarchive.sql' => array(
'type' => 'sql',
'name' => $this->getPatchPath('daemontaskarchive.sql'),
),
);
}