1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00

Rough cut of Workers

Summary: workers do work, provided I have written them correctly. perhaps it
is so.

Test Plan:

Reviewers:

CC:
This commit is contained in:
epriestley 2011-03-10 13:48:29 -08:00
parent c82cab35e2
commit ec084ca419
31 changed files with 590 additions and 23 deletions

View file

@ -0,0 +1,20 @@
create database phabricator_worker;
create table phabricator_worker.worker_task (
id int unsigned not null auto_increment primary key,
taskClass varchar(255) not null,
leaseOwner varchar(255),
leaseExpires int unsigned,
priority bigint unsigned not null,
failureCount int unsigned not null,
key(taskClass),
key(leaseOwner),
key(leaseExpires)
);
create table phabricator_worker.worker_taskdata (
id int unsigned not null auto_increment primary key,
taskID int unsigned not null,
data longblob not null,
unique key (taskID)
);

View file

@ -190,6 +190,8 @@ phutil_register_library_map(array(
'PhabricatorConduitMethodCallLog' => 'applications/conduit/storage/methodcalllog',
'PhabricatorController' => 'applications/base/controller/base',
'PhabricatorDaemon' => 'infrastructure/daemon/base',
'PhabricatorDaemonConsoleController' => 'applications/daemon/controller/console',
'PhabricatorDaemonController' => 'applications/daemon/controller/base',
'PhabricatorDirectoryCategory' => 'applications/directory/storage/category',
'PhabricatorDirectoryCategoryDeleteController' => 'applications/directory/controller/categorydelete',
'PhabricatorDirectoryCategoryEditController' => 'applications/directory/controller/categoryedit',
@ -214,6 +216,7 @@ phutil_register_library_map(array(
'PhabricatorFileURI' => 'applications/files/uri',
'PhabricatorFileUploadController' => 'applications/files/controller/upload',
'PhabricatorFileViewController' => 'applications/files/controller/view',
'PhabricatorGoodForNothingWorker' => 'infrastructure/daemon/workers/worker/goodfornothing',
'PhabricatorHandleObjectSelectorDataView' => 'applications/phid/handle/view/selector',
'PhabricatorLiskDAO' => 'applications/base/storage/lisk',
'PhabricatorLoginController' => 'applications/auth/controller/login',
@ -300,11 +303,12 @@ phutil_register_library_map(array(
'PhabricatorSearchQuery' => 'applications/search/storage/query',
'PhabricatorSearchRelationship' => 'applications/search/constants/relationship',
'PhabricatorStandardPageView' => 'view/page/standard',
'PhabricatorTimelineCursor' => 'applications/timeline/storage/cursor',
'PhabricatorTimelineDAO' => 'applications/timeline/storage/base',
'PhabricatorTimelineEvent' => 'applications/timeline/storage/event',
'PhabricatorTimelineEventData' => 'applications/timeline/storage/eventdata',
'PhabricatorTimelineIterator' => 'applications/timeline/cursor/iterator',
'PhabricatorTaskmasterDaemon' => 'infrastructure/daemon/workers/taskmaster',
'PhabricatorTimelineCursor' => 'infrastructure/daemon/timeline/storage/cursor',
'PhabricatorTimelineDAO' => 'infrastructure/daemon/timeline/storage/base',
'PhabricatorTimelineEvent' => 'infrastructure/daemon/timeline/storage/event',
'PhabricatorTimelineEventData' => 'infrastructure/daemon/timeline/storage/eventdata',
'PhabricatorTimelineIterator' => 'infrastructure/daemon/timeline/cursor/iterator',
'PhabricatorTypeaheadCommonDatasourceController' => 'applications/typeahead/controller/common',
'PhabricatorTypeaheadDatasourceController' => 'applications/typeahead/controller/base',
'PhabricatorUser' => 'applications/people/storage/user',
@ -312,6 +316,10 @@ phutil_register_library_map(array(
'PhabricatorUserOAuthInfo' => 'applications/people/storage/useroauthinfo',
'PhabricatorUserProfile' => 'applications/people/storage/profile',
'PhabricatorUserSettingsController' => 'applications/people/controller/settings',
'PhabricatorWorker' => 'infrastructure/daemon/workers/worker',
'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/base',
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/task',
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/taskdata',
'PhabricatorXHProfController' => 'applications/xhprof/controller/base',
'PhabricatorXHProfProfileController' => 'applications/xhprof/controller/profile',
'PhabricatorXHProfProfileSymbolView' => 'applications/xhprof/view/symbol',
@ -473,6 +481,8 @@ phutil_register_library_map(array(
'PhabricatorConduitMethodCallLog' => 'PhabricatorConduitDAO',
'PhabricatorController' => 'AphrontController',
'PhabricatorDaemon' => 'PhutilDaemon',
'PhabricatorDaemonConsoleController' => 'PhabricatorDaemonController',
'PhabricatorDaemonController' => 'PhabricatorController',
'PhabricatorDirectoryCategory' => 'PhabricatorDirectoryDAO',
'PhabricatorDirectoryCategoryDeleteController' => 'PhabricatorDirectoryController',
'PhabricatorDirectoryCategoryEditController' => 'PhabricatorDirectoryController',
@ -495,6 +505,7 @@ phutil_register_library_map(array(
'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO',
'PhabricatorFileUploadController' => 'PhabricatorFileController',
'PhabricatorFileViewController' => 'PhabricatorFileController',
'PhabricatorGoodForNothingWorker' => 'PhabricatorWorker',
'PhabricatorLiskDAO' => 'LiskDAO',
'PhabricatorLoginController' => 'PhabricatorAuthController',
'PhabricatorLogoutController' => 'PhabricatorAuthController',
@ -567,6 +578,7 @@ phutil_register_library_map(array(
'PhabricatorSearchMySQLExecutor' => 'PhabricatorSearchExecutor',
'PhabricatorSearchQuery' => 'PhabricatorSearchDAO',
'PhabricatorStandardPageView' => 'AphrontPageView',
'PhabricatorTaskmasterDaemon' => 'PhabricatorDaemon',
'PhabricatorTimelineCursor' => 'PhabricatorTimelineDAO',
'PhabricatorTimelineDAO' => 'PhabricatorLiskDAO',
'PhabricatorTimelineEvent' => 'PhabricatorTimelineDAO',
@ -578,6 +590,9 @@ phutil_register_library_map(array(
'PhabricatorUserOAuthInfo' => 'PhabricatorUserDAO',
'PhabricatorUserProfile' => 'PhabricatorUserDAO',
'PhabricatorUserSettingsController' => 'PhabricatorPeopleController',
'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO',
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
'PhabricatorXHProfController' => 'PhabricatorController',
'PhabricatorXHProfProfileController' => 'PhabricatorXHProfController',
'PhabricatorXHProfProfileSymbolView' => 'AphrontView',

View file

@ -200,6 +200,10 @@ class AphrontDefaultApplicationConfiguration
),
),
'/daemon/' => array(
'$' => 'PhabricatorDaemonConsoleController',
),
);
}

View file

@ -0,0 +1,42 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorDaemonController extends PhabricatorController {
public function buildStandardPageResponse($view, array $data) {
$page = $this->buildStandardPageView();
$page->setApplicationName('Daemon Console');
$page->setBaseURI('/');
$page->setTitle(idx($data, 'title'));
$page->setTabs(
array(
'console' => array(
'href' => '/daemon/',
'name' => 'Console',
),
),
idx($data, 'tab'));
$page->setGlyph("\xE2\x98\xAF");
$page->appendChild($view);
$response = new AphrontWebpageResponse();
return $response->setContent($page->render());
}
}

View file

@ -0,0 +1,15 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'aphront/response/webpage');
phutil_require_module('phabricator', 'applications/base/controller/base');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorDaemonController.php');

View file

@ -0,0 +1,70 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
class PhabricatorDaemonConsoleController extends PhabricatorDaemonController {
public function processRequest() {
$request = $this->getRequest();
if ($request->getStr('new')) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorGoodForNothingWorker');
$task->setPriority(4);
$task->setFailureCount(0);
$task->save();
}
$tasks = id(new PhabricatorWorkerTask())
->loadAll();
$rows = array();
foreach ($tasks as $task) {
$rows[] = array(
$task->getID(),
$task->getTaskClass(),
$task->getLeaseOwner(),
$task->getLeaseExpires(),
$task->getPriority(),
$task->getFailureCount(),
);
}
$table = new AphrontTableView($rows);
$table->setHeaders(
array(
'ID',
'Class',
'Owner',
'Expries',
'Priority',
'Count',
));
$panel = new AphrontPanelView();
$panel->setHeader('Tasks');
$panel->appendChild($table);
return $this->buildStandardPageResponse(
$panel,
array(
'title' => 'Console',
'tab' => 'console',
));
}
}

View file

@ -0,0 +1,17 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/daemon/controller/base');
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
phutil_require_module('phabricator', 'view/control/table');
phutil_require_module('phabricator', 'view/layout/panel');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorDaemonConsoleController.php');

View file

@ -1,13 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/timeline/storage/base');
phutil_require_module('phabricator', 'applications/timeline/storage/eventdata');
phutil_require_source('PhabricatorTimelineEvent.php');

View file

@ -6,9 +6,9 @@
phutil_require_module('phabricator', 'applications/timeline/storage/cursor');
phutil_require_module('phabricator', 'applications/timeline/storage/event');
phutil_require_module('phabricator', 'applications/timeline/storage/eventdata');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/cursor');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/eventdata');
phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_module('phutil', 'utils');

View file

@ -6,7 +6,7 @@
phutil_require_module('phabricator', 'applications/timeline/storage/base');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/base');
phutil_require_source('PhabricatorTimelineCursor.php');

View file

@ -0,0 +1,13 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/base');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/eventdata');
phutil_require_source('PhabricatorTimelineEvent.php');

View file

@ -6,7 +6,7 @@
phutil_require_module('phabricator', 'applications/timeline/storage/base');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/base');
phutil_require_source('PhabricatorTimelineEventData.php');

View file

@ -0,0 +1,23 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorWorkerDAO extends PhabricatorLiskDAO {
public function getApplicationName() {
return 'worker';
}
}

View file

@ -0,0 +1,12 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/base/storage/lisk');
phutil_require_source('PhabricatorWorkerDAO.php');

View file

@ -0,0 +1,60 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
protected $taskClass;
protected $leaseOwner;
protected $leaseExpires;
protected $priority;
protected $failureCount;
private $serverTime;
private $localTime;
public function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
) + parent::getConfiguration();
}
public function setServerTime($server_time) {
$this->serverTime = $server_time;
$this->localTime = time();
return $this;
}
public function setLeaseDuration($lease_duration) {
$server_lease_expires = $this->serverTime + $lease_duration;
$this->setLeaseExpires($server_lease_expires);
return $this->save();
}
public function save() {
if ($this->leaseOwner) {
$current_server_time = $this->serverTime + (time() - $this->localTime);
if ($current_server_time >= $this->leaseExpires) {
throw new Exception("Trying to update task after lease expiration!");
}
}
return parent::save();
}
}

View file

@ -0,0 +1,12 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/base');
phutil_require_source('PhabricatorWorkerTask.php');

View file

@ -0,0 +1,33 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
class PhabricatorWorkerTaskData extends PhabricatorWorkerDAO {
protected $taskID;
protected $data;
public function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_SERIALIZATION => array(
'data' => self::SERIALIZATION_JSON,
),
) + parent::getConfiguration();
}
}

View file

@ -0,0 +1,12 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/base');
phutil_require_source('PhabricatorWorkerTaskData.php');

View file

@ -0,0 +1,121 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
public function run() {
$lease_ownership_name = $this->getLeaseOwnershipName();
$task_table = new PhabricatorWorkerTask();
$taskdata_table = new PhabricatorWorkerTaskData();
$sleep = 0;
do {
$conn_w = $task_table->establishConnection('w');
queryfx(
$conn_w,
'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
WHERE leaseOwner IS NULL
OR leaseExpires < UNIX_TIMESTAMP()
ORDER BY leaseOwner IS NULL, failureCount, priority
LIMIT 1',
$task_table->getTableName(),
$lease_ownership_name);
$rows = $conn_w->getAffectedRows();
if ($rows) {
$data = queryfx_all(
$conn_w,
'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime
FROM %T task LEFT JOIN %T taskdata
ON taskdata.taskID = task.id
WHERE leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP()
LIMIT 1',
$task_table->getTableName(),
$taskdata_table->getTableName(),
$lease_ownership_name);
$tasks = $task_table->loadAllFromArray($data);
$tasks = mpull($tasks, null, 'getID');
$task_data = array();
foreach ($data as $row) {
$tasks[$row['id']]->setServerTime($row['_serverTime']);
if ($row['_taskData']) {
$task_data[$row['id']] = json_decode($row['_taskData'], true);
} else {
$task_data[$row['id']] = null;
}
}
foreach ($tasks as $task) {
// TODO: We should detect if we acquired a task with an expired lease
// and log about it / bump up failure count.
// TODO: We should detect if we acquired a task with an excessive
// failure count and fail it permanently.
$data = idx($task_data, $task->getID());
$class = $task->getTaskClass();
try {
PhutilSymbolLoader::loadClass($class);
if (!is_subclass_of($class, 'PhabricatorWorker')) {
throw new Exception(
"Task class '{$class}' does not extend PhabricatorWorker.");
}
$worker = newv($class, array($data));
$lease = $worker->getRequiredLeaseTime();
if ($lease !== null) {
$task->setLeaseDuration($lease);
}
$worker->executeTask();
$task->delete();
if ($data !== null) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE taskID = %d',
$taskdata_table,
$task->getID());
}
} catch (Exception $ex) {
$task->setFailureCount($task->getFailureCount() + 1);
$task->save();
throw $ex;
}
}
$sleep = 0;
} else {
$sleep = min($sleep + 1, 30);
}
$this->sleep($sleep);
} while (true);
}
private function getLeaseOwnershipName() {
static $name = null;
if ($name === null) {
$name = getmypid().':'.time().':'.php_uname('n');
}
return $name;
}
}

View file

@ -0,0 +1,18 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/daemon/base');
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/taskdata');
phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_module('phutil', 'symbols');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorTaskmasterDaemon.php');

View file

@ -0,0 +1,40 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorWorker {
private $data;
final public function __construct($data) {
$this->data = $data;
}
final protected function getTaskData() {
return $this->data;
}
final public function executeTask() {
$this->doWork();
}
public function getRequiredLeaseTime() {
return null;
}
abstract protected function doWork();
}

View file

@ -0,0 +1,10 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_source('PhabricatorWorker.php');

View file

@ -0,0 +1,28 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Trivial example worker; processes tasks which require no work very slowly.
*
* @group worker
*/
class PhabricatorGoodForNothingWorker extends PhabricatorWorker {
protected function doWork() {
sleep(10);
}
}

View file

@ -0,0 +1,12 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/daemon/workers/worker');
phutil_require_source('PhabricatorGoodForNothingWorker.php');

View file

@ -442,6 +442,9 @@ abstract class LiskDAO {
* @task load
*/
public function loadFromArray(array $row) {
// TODO: We should load only valid properties.
$map = array();
foreach ($row as $k => $v) {
$map[$k] = $v;