revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00
Code Issues Releases Wiki Activity

Make various Drydock CLI/Allocator improvements

Browse source 
Summary:
  - Remove EC2, RemoteHost, Application, etc., blueprints for now. They're very proof-of-concept and Blueprints are getting API changes I don't want to bother propagating for now. Leave the abstract base class and the LocalHost blueprint. I'll restore the more complicated ones once better foundations are in place.
  - Remove the Allocate controller from the web UI. The original vision here was that you'd manually allocate resources in some cases, but it no longer makes sense to do so as all allocations come from leases now. This simplifies allocations and makes the rule for when we can clean up resources clear-cut (if a resource has no more active leases, it can be cleaned up). Instead, we'll build resources like the localhost and remote hosts lazily, when leases come in for them.
  - Add some configuration to manage the localhost blueprint.
  - Refactor `canAllocateResources()` into `isEnabled()` (for config checks) and `canAllocateMoreResources()` (for quota checks, e.g. too many resources are allocated already).
  - Juggle some signatures to align better with a world where blueprints generally do allocate.
  - Add some more logging and error handling.
  - Fix an issue with log ordering.

Test Plan: Allocated some localhost leases.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2015

Differential Revision: https://secure.phabricator.com/D3902
This commit is contained in:
epriestley 2012-11-06 15:30:11 -08:00
parent 0e774dac93
commit 7e0ce08154
16 changed files with 220 additions and 517 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
conf/default.conf.php
View file

@ -1136,6 +1136,22 @@ return array(
// -- Drydock --------------------------------------------------------------- //
// Drydock is used to allocate various software resources. For example, it
// allocates working copies so continuous integration tests can be executed.
// It needs at least one host to allocate these resources on.
//
// Set this option to true to let Drydock use the localhost for allocations.
// This is the simplest configuration, but the least scalable. You MUST
// disable this if you run daemons on more than one machine -- if you do not,
// a daemon on machine A may allocate a resource locally, and then a daemon
// on machine B may try to access it.
'drydock.localhost.enabled' => true,
// If the localhost is available to Drydock, specify the path on disk where
// Drydock should write files. You should create this directory and make sure
// the user that the daemons run as has permission to write to it.
'drydock.localhost.path' => '/var/drydock/',
// If you want to use Drydock's builtin EC2 Blueprints, configure your AWS
// EC2 credentials here.
'amazon-ec2.access-key' => null,

10
src/__phutil_library_map__.php
View file

@ -412,14 +412,12 @@ phutil_register_library_map(array(
'DiffusionView' => 'applications/diffusion/view/DiffusionView.php',
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
'DrydockApacheWebrootBlueprint' => 'applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php',
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
'DrydockBlueprint' => 'applications/drydock/blueprint/DrydockBlueprint.php',
'DrydockCommandInterface' => 'applications/drydock/interface/command/DrydockCommandInterface.php',
'DrydockConstants' => 'applications/drydock/constants/DrydockConstants.php',
'DrydockController' => 'applications/drydock/controller/DrydockController.php',
'DrydockDAO' => 'applications/drydock/storage/DrydockDAO.php',
'DrydockEC2HostBlueprint' => 'applications/drydock/blueprint/DrydockEC2HostBlueprint.php',
'DrydockInterface' => 'applications/drydock/interface/DrydockInterface.php',
'DrydockLease' => 'applications/drydock/storage/DrydockLease.php',
'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
@ -433,10 +431,7 @@ phutil_register_library_map(array(
'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php',
'DrydockManagementWaitForLeaseWorkflow' => 'applications/drydock/management/DrydockManagementWaitForLeaseWorkflow.php',
'DrydockManagementWorkflow' => 'applications/drydock/management/DrydockManagementWorkflow.php',
'DrydockPhabricatorApplicationBlueprint' => 'applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php',
'DrydockRemoteHostBlueprint' => 'applications/drydock/blueprint/DrydockRemoteHostBlueprint.php',
'DrydockResource' => 'applications/drydock/storage/DrydockResource.php',
'DrydockResourceAllocateController' => 'applications/drydock/controller/DrydockResourceAllocateController.php',
'DrydockResourceListController' => 'applications/drydock/controller/DrydockResourceListController.php',
'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php',
'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php',
@ -1649,12 +1644,10 @@ phutil_register_library_map(array(
'DiffusionView' => 'AphrontView',
'DivinerListController' => 'PhabricatorController',
'DrydockAllocatorWorker' => 'PhabricatorWorker',
'DrydockApacheWebrootBlueprint' => 'DrydockBlueprint',
'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
'DrydockCommandInterface' => 'DrydockInterface',
'DrydockController' => 'PhabricatorController',
'DrydockDAO' => 'PhabricatorLiskDAO',
'DrydockEC2HostBlueprint' => 'DrydockRemoteHostBlueprint',
'DrydockLease' => 'DrydockDAO',
'DrydockLeaseListController' => 'DrydockController',
'DrydockLeaseStatus' => 'DrydockConstants',
@ -1667,10 +1660,7 @@ phutil_register_library_map(array(
'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow',
'DrydockManagementWaitForLeaseWorkflow' => 'DrydockManagementWorkflow',
'DrydockManagementWorkflow' => 'PhutilArgumentWorkflow',
'DrydockPhabricatorApplicationBlueprint' => 'DrydockBlueprint',
'DrydockRemoteHostBlueprint' => 'DrydockBlueprint',
'DrydockResource' => 'DrydockDAO',
'DrydockResourceAllocateController' => 'DrydockController',
'DrydockResourceListController' => 'DrydockController',
'DrydockResourceStatus' => 'DrydockConstants',
'DrydockSSHCommandInterface' => 'DrydockCommandInterface',

1
src/applications/drydock/application/PhabricatorApplicationDrydock.php
View file

@ -31,7 +31,6 @@ final class PhabricatorApplicationDrydock extends PhabricatorApplication {
'/drydock/' => array(
'' => 'DrydockResourceListController',
'resource/' => 'DrydockResourceListController',
'resource/allocate/' => 'DrydockResourceAllocateController',
'lease/' => array(
'' => 'DrydockLeaseListController',
'(?P<id>[1-9]\d*)/' => 'DrydockLeaseViewController',

65
src/applications/drydock/blueprint/DrydockBlueprint.php
View file

@ -11,12 +11,22 @@ abstract class DrydockBlueprint {
DrydockLease $lease,
$type);
protected function executeAcquireLease(
DrydockResource $resource,
DrydockLease $lease) {
return;
abstract public function isEnabled();
public function getBlueprintClass() {
return get_class($this);
}
public function canAllocateMoreResources(array $pool) {
return true;
}
abstract protected function executeAllocateResource(DrydockLease $lease);
abstract protected function executeAcquireLease(
DrydockResource $resource,
DrydockLease $lease);
final public function acquireLease(
DrydockResource $resource,
DrydockLease $lease) {
@ -74,22 +84,19 @@ abstract class DrydockBlueprint {
$log->save();
}
public function canAllocateResources() {
return false;
}
protected function executeAllocateResource(DrydockLease $lease) {
throw new Exception("This blueprint can not allocate resources!");
}
final public function allocateResource(DrydockLease $lease) {
$this->activeLease = $lease;
$this->activeResource = null;
$this->log('Allocating Resource');
$this->log(
pht(
"Blueprint '%s': Allocating Resource for '%s'",
$this->getBlueprintClass(),
$lease->getLeaseName()));
try {
$resource = $this->executeAllocateResource($lease);
$this->validateAllocatedResource($resource);
} catch (Exception $ex) {
$this->logException($ex);
$this->activeResource = null;
@ -128,17 +135,45 @@ abstract class DrydockBlueprint {
protected function newResourceTemplate($name) {
$resource = new DrydockResource();
$resource->setBlueprintClass(get_class($this));
$resource->setBlueprintClass($this->getBlueprintClass());
$resource->setType($this->getType());
$resource->setStatus(DrydockResourceStatus::STATUS_PENDING);
$resource->setName($name);
$resource->save();
$this->activeResource = $resource;
$this->log('New Template');
$this->log(
pht(
"Blueprint '%s': Created New Template",
$this->getBlueprintClass()));
return $resource;
}
/**
* Sanity checks that the blueprint is implemented properly.
*/
private function validateAllocatedResource($resource) {
$blueprint = $this->getBlueprintClass();
if (!($resource instanceof DrydockResource)) {
throw new Exception(
"Blueprint '{$blueprint}' is not properly implemented: ".
"executeAllocateResource() must return an object of type ".
"DrydockResource or throw, but returned something else.");
}
$current_status = $resource->getStatus();
$req_status = DrydockResourceStatus::STATUS_OPEN;
if ($current_status != $req_status) {
$current_name = DrydockResourceStatus::getNameForStatus($current_status);
$req_name = DrydockResourceStatus::getNameForStatus($req_status);
throw new Exception(
"Blueprint '{$blueprint}' is not properly implemented: ".
"executeAllocateResource() must return a DrydockResource with ".
"status '{$req_name}', but returned one with status ".
"'{$current_name}'.");
}
}
}

147
src/applications/drydock/blueprint/DrydockEC2HostBlueprint.php
View file

@ -1,147 +0,0 @@
<?php
final class DrydockEC2HostBlueprint extends DrydockRemoteHostBlueprint {
public function canAllocateResources() {
return true;
}
public function executeAllocateResource(DrydockLease $lease) {
$resource = $this->newResourceTemplate('EC2 Host');
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
$resource->save();
$xml = $this->executeEC2Query(
'RunInstances',
array(
'ImageId' => 'ami-c7c99982',
'MinCount' => 1,
'MaxCount' => 1,
'KeyName' => 'ec2wc',
'SecurityGroupId.1' => 'sg-6bffff2e',
'InstanceType' => 't1.micro',
));
$instance_id = (string)$xml->instancesSet[0]->item[0]->instanceId[0];
$this->log("Started Instance: {$instance_id}");
$resource->setAttribute('instance.id', $instance_id);
$resource->save();
$n = 1;
do {
$xml = $this->executeEC2Query(
'DescribeInstances',
array(
'InstanceId.1' => $instance_id,
));
$instance = $xml->reservationSet[0]->item[0]->instancesSet[0]->item[0];
$state = (string)$instance->instanceState[0]->name;
if ($state == 'pending') {
sleep(min($n++, 15));
} else if ($state == 'running') {
break;
} else {
$this->log("EC2 host reported in unknown state '{$state}'.");
$resource->setStatus(DrydockResourceStatus::STATUS_BROKEN);
$resource->save();
}
} while (true);
$this->log('Waiting for Init');
$n = 1;
do {
$xml = $this->executeEC2Query(
'DescribeInstanceStatus',
array(
'InstanceId' => $instance_id,
));
$item = $xml->instanceStatusSet[0]->item[0];
$system_status = (string)$item->systemStatus->status[0];
$instance_status = (string)$item->instanceStatus->status[0];
if (($system_status == 'initializing') ||
($instance_status == 'initializing')) {
sleep(min($n++, 15));
} else if (($system_status == 'ok') &&
($instance_status == 'ok')) {
break;
} else {
$this->log(
"EC2 system and instance status in bad states: ".
"'{$system_status}', '{$instance_status}'.");
$resource->setStatus(DrydockResourceStatus::STATUS_BROKEN);
$resource->save();
}
} while (true);
$resource->setAttributes(
array(
'host' => (string)$instance->dnsName,
'user' => 'ec2-user',
'ssh-keyfile' => '/Users/epriestley/.ssh/id_ec2w',
));
$resource->setName($resource->getName().' ('.$instance->dnsName.')');
$resource->save();
$this->log('Waiting for SSH');
// SSH isn't immediately responsive, so wait for it to actually come up.
$cmd = $this->getInterface($resource, new DrydockLease(), 'command');
$n = 1;
do {
list($err) = $cmd->exec('true');
if ($err) {
sleep(min($n++, 15));
} else {
break;
}
} while (true);
$this->log('SSH OK');
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
$resource->save();
return $resource;
}
public function getInterface(
DrydockResource $resource,
DrydockLease $lease,
$type) {
switch ($type) {
case 'command':
$ssh = new DrydockSSHCommandInterface();
$ssh->setConfiguration(
array(
'host' => $resource->getAttribute('host'),
'user' => $resource->getAttribute('user'),
'ssh-keyfile' => $resource->getAttribute('ssh-keyfile'),
));
return $ssh;
}
throw new Exception("No interface of type '{$type}'.");
}
private function executeEC2Query($action, array $params) {
$future = new PhutilAWSEC2Future();
$future->setAWSKeys(
PhabricatorEnv::getEnvConfig('amazon-ec2.access-key'),
PhabricatorEnv::getEnvConfig('amazon-ec2.secret-key'));
$future->setRawAWSQuery($action, $params);
return $future->resolve();
}
}

39
src/applications/drydock/blueprint/DrydockLocalHostBlueprint.php
View file

@ -2,6 +2,45 @@
final class DrydockLocalHostBlueprint extends DrydockBlueprint {
public function isEnabled() {
return PhabricatorEnv::getEnvConfig('drydock.localhost.enabled');
}
public function canAllocateMoreResources(array $pool) {
assert_instances_of($pool, 'DrydockResource');
// The localhost can be allocated only once.
foreach ($pool as $resource) {
if ($resource->getBlueprintClass() == $this->getBlueprintClass()) {
return false;
}
}
return true;
}
protected function executeAllocateResource(DrydockLease $lease) {
$path = PhabricatorEnv::getEnvConfig('drydock.localhost.path');
if (!Filesystem::pathExists($path)) {
throw new Exception(
"Path '{$path}' does not exist!");
}
Filesystem::assertIsDirectory($path);
Filesystem::assertWritable($path);
$resource = $this->newResourceTemplate('localhost');
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
$resource->save();
return $resource;
}
protected function executeAcquireLease(
DrydockResource $resource,
DrydockLease $lease) {
return;
}
public function getType() {
return 'host';
}

32
src/applications/drydock/blueprint/DrydockRemoteHostBlueprint.php
View file

@ -1,32 +0,0 @@
<?php
/**
* TODO: Is this concrete-extensible?
*/
class DrydockRemoteHostBlueprint extends DrydockBlueprint {
public function getType() {
return 'host';
}
public function getInterface(
DrydockResource $resource,
DrydockLease $lease,
$type) {
switch ($type) {
case 'command':
$ssh = new DrydockSSHCommandInterface();
$ssh->setConfiguration(
array(
'host' => 'secure.phabricator.com',
'user' => 'ec2-user',
'ssh-keyfile' => '/Users/epriestley/.ssh/id_ec2w',
));
return $ssh;
}
throw new Exception("No interface of type '{$type}'.");
}
}

52
src/applications/drydock/blueprint/application/DrydockPhabricatorApplicationBlueprint.php
View file

@ -1,52 +0,0 @@
<?php
abstract class DrydockPhabricatorApplicationBlueprint
extends DrydockBlueprint {
public function getType() {
return 'application';
}
public function canAllocateResources() {
return true;
}
public function executeAllocateResource(DrydockLease $lease) {
$resource = $this->newResourceTemplate('Phabricator');
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
$resource->save();
$host = id(new DrydockLease())
->setResourceType('host')
->queueForActivation();
$cmd = $host->waitUntilActive()->getInterface('command');
$cmd->execx(<<<EOINSTALL
yum install git &&
mkdir -p /opt/drydock &&
cd /opt/drydock &&
git clone git://github.com/facebook/libphutil.git &&
git clone git://github.com/facebook/arcanist.git &&
git clone git://github.com/facebook/phabricator.git
EOINSTALL
);
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
$resource->save();
return $resource;
}
public function getInterface(
DrydockResource $resource,
DrydockLease $lease,
$type) {
throw new Exception("No interface of type '{$type}'.");
}
}

113
src/applications/drydock/blueprint/webroot/DrydockApacheWebrootBlueprint.php
View file

@ -1,113 +0,0 @@
<?php
final class DrydockApacheWebrootBlueprint
extends DrydockBlueprint {
public function getType() {
return 'webroot';
}
public function canAllocateResources() {
return true;
}
public function executeAcquireLease(
DrydockResource $resource,
DrydockLease $lease) {
$key = Filesystem::readRandomCharacters(12);
$ports = $resource->getAttribute('ports', array());
for ($ii = 2000; ; $ii++) {
if (empty($ports[$ii])) {
$ports[$ii] = $lease->getID();
$port = $ii;
break;
}
}
$resource->setAttribute('ports', $ports);
$resource->save();
$host = $resource->getAttribute('host');
$lease->setAttribute('port', $port);
$lease->setAttribute('key', $key);
$lease->save();
$config = <<<EOCONFIG
Listen *:{$port}
<VirtualHost *:{$port}>
DocumentRoot /opt/drydock/webroot/{$key}/
ServerName {$host}
</VirtualHost>
EOCONFIG;
$cmd = $this->getInterface($resource, $lease, 'command');
$cmd->execx(<<<EOSETUP
sudo mkdir -p %s &&
sudo sh -c %s &&
sudo /etc/init.d/httpd restart
EOSETUP
,
"/opt/drydock/webroot/{$key}/",
csprintf(
'echo %s > %s',
$config,
"/etc/httpd/conf.d/drydock-{$key}.conf"));
$lease->setAttribute('uri', "http://{$host}:{$port}/");
$lease->save();
}
public function executeAllocateResource(DrydockLease $lease) {
$resource = $this->newResourceTemplate('Apache');
$resource->setStatus(DrydockResourceStatus::STATUS_ALLOCATING);
$resource->save();
$allocator = $this->getAllocator('host');
$host = $allocator->allocate();
$cmd = $host->waitUntilActive()->getInterface('command');
$cmd->execx(<<<EOINSTALL
(yes | sudo yum install httpd) && sudo mkdir -p /opt/drydock/webroot/
EOINSTALL
);
$resource->setAttribute('lease.host', $host->getID());
$resource->setAttribute('host', $host->getResource()->getAttribute('host'));
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
$resource->save();
return $resource;
}
public function getInterface(
DrydockResource $resource,
DrydockLease $lease,
$type) {
switch ($type) {
case 'webroot':
$iface = new DrydockApacheWebrootInterface();
$iface->setConfiguration(
array(
'uri' => $lease->getAttribute('uri'),
));
return $iface;
case 'command':
$host_lease_id = $resource->getAttribute('lease.host');
$host_lease = id(new DrydockLease())->load($host_lease_id);
$host_lease->attachResource($host_lease->loadResource());
return $host_lease->getInterface($type);
}
throw new Exception("No interface of type '{$type}'.");
}
}

10
src/applications/drydock/constants/DrydockResourceStatus.php
View file

@ -3,16 +3,14 @@
final class DrydockResourceStatus extends DrydockConstants {
const STATUS_PENDING = 0;
const STATUS_ALLOCATING = 1;
const STATUS_OPEN = 2;
const STATUS_CLOSED = 3;
const STATUS_BROKEN = 4;
const STATUS_DESTROYED = 5;
const STATUS_OPEN = 1;
const STATUS_CLOSED = 2;
const STATUS_BROKEN = 3;
const STATUS_DESTROYED = 4;
public static function getNameForStatus($status) {
static $map = array(
self::STATUS_PENDING => 'Pending',
self::STATUS_ALLOCATING => 'Pending',
self::STATUS_OPEN => 'Open',
self::STATUS_CLOSED => 'Closed',
self::STATUS_BROKEN => 'Broken',

119
src/applications/drydock/controller/DrydockResourceAllocateController.php
View file

@ -1,119 +0,0 @@
<?php
final class DrydockResourceAllocateController extends DrydockController {
public function processRequest() {
$request = $this->getRequest();
$user = $request->getUser();
$resource = new DrydockResource();
$json = new PhutilJSON();
$err_attributes = true;
$err_capabilities = true;
$json_attributes = $json->encodeFormatted($resource->getAttributes());
$json_capabilities = $json->encodeFormatted($resource->getCapabilities());
$errors = array();
if ($request->isFormPost()) {
$raw_attributes = $request->getStr('attributes');
$attributes = json_decode($raw_attributes, true);
if (!is_array($attributes)) {
$err_attributes = 'Invalid';
$errors[] = 'Enter attributes as a valid JSON object.';
$json_attributes = $raw_attributes;
} else {
$resource->setAttributes($attributes);
$json_attributes = $json->encodeFormatted($attributes);
$err_attributes = null;
}
$raw_capabilities = $request->getStr('capabilities');
$capabilities = json_decode($raw_capabilities, true);
if (!is_array($capabilities)) {
$err_capabilities = 'Invalid';
$errors[] = 'Enter capabilities as a valid JSON object.';
$json_capabilities = $raw_capabilities;
} else {
$resource->setCapabilities($capabilities);
$json_capabilities = $json->encodeFormatted($capabilities);
$err_capabilities = null;
}
$resource->setBlueprintClass($request->getStr('blueprint'));
$resource->setType($resource->getBlueprint()->getType());
$resource->setOwnerPHID($user->getPHID());
$resource->setName($request->getStr('name'));
if (!$errors) {
$resource->save();
return id(new AphrontRedirectResponse())
->setURI('/drydock/resource/');
}
}
$error_view = null;
if ($errors) {
$error_view = new AphrontErrorView();
$error_view->setTitle('Form Errors');
$error_view->setErrors($errors);
}
$blueprints = id(new PhutilSymbolLoader())
->setType('class')
->setAncestorClass('DrydockBlueprint')
->selectAndLoadSymbols();
$blueprints = ipull($blueprints, 'name', 'name');
$panel = new AphrontPanelView();
$panel->setWidth(AphrontPanelView::WIDTH_FORM);
$panel->setHeader('Allocate Drydock Resource');
$form = id(new AphrontFormView())
->setUser($request->getUser())
->appendChild(
id(new AphrontFormTextControl())
->setLabel('Name')
->setName('name')
->setValue($resource->getName()))
->appendChild(
id(new AphrontFormSelectControl())
->setLabel('Blueprint')
->setOptions($blueprints)
->setName('blueprint')
->setValue($resource->getBlueprintClass()))
->appendChild(
id(new AphrontFormTextAreaControl())
->setLabel('Attributes')
->setName('attributes')
->setValue($json_attributes)
->setError($err_attributes)
->setCaption('Specify attributes in JSON.'))
->appendChild(
id(new AphrontFormTextAreaControl())
->setLabel('Capabilities')
->setName('capabilities')
->setValue($json_capabilities)
->setError($err_capabilities)
->setCaption('Specify capabilities in JSON.'))
->appendChild(
id(new AphrontFormSubmitControl())
->setValue('Allocate Resource'));
$panel->appendChild($form);
return $this->buildStandardPageResponse(
array(
$error_view,
$panel,
),
array(
'title' => 'Allocate Resource',
));
}
}

9
src/applications/drydock/controller/DrydockResourceListController.php
View file

@ -57,15 +57,6 @@ final class DrydockResourceListController extends DrydockController {
$panel = new AphrontPanelView();
$panel->setHeader('Drydock Resources');
$panel->addButton(
phutil_render_tag(
'a',
array(
'href' => '/drydock/resource/allocate/',
'class' => 'green button',
),
'Allocate Resource'));
$panel->appendChild($table);
$panel->appendChild($pager);

2
src/applications/drydock/query/DrydockLogQuery.php
View file

@ -75,7 +75,7 @@ final class DrydockLogQuery extends PhabricatorOffsetPagedQuery {
private function buildOrderClause(AphrontDatabaseConnection $conn_r) {
switch ($this->order) {
case self::ORDER_EPOCH:
return 'ORDER BY log.epoch DESC';
return 'ORDER BY log.epoch DESC, log.id DESC';
case self::ORDER_ID:
return 'ORDER BY id ASC';
default:

10
src/applications/drydock/storage/DrydockLease.php
View file

@ -2,7 +2,6 @@
final class DrydockLease extends DrydockDAO {
protected $phid;
protected $resourceID;
protected $resourceType;
protected $until;
@ -13,6 +12,10 @@ final class DrydockLease extends DrydockDAO {
private $resource;
public function getLeaseName() {
return pht('Lease %d', $this->getID());
}
public function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
@ -112,6 +115,7 @@ final class DrydockLease extends DrydockDAO {
assert_instances_of($leases, 'DrydockLease');
$task_ids = array_filter(mpull($leases, 'getTaskID'));
PhabricatorWorker::waitForTasks($task_ids);
$unresolved = $leases;
@ -123,9 +127,11 @@ final class DrydockLease extends DrydockDAO {
unset($unresolved[$key]);
break;
case DrydockLeaseStatus::STATUS_RELEASED:
throw new Exception("Lease has already been released!");
case DrydockLeaseStatus::STATUS_EXPIRED:
throw new Exception("Lease has already expired!");
case DrydockLeaseStatus::STATUS_BROKEN:
throw new Exception("Lease will never become active!");
throw new Exception("Lease has been broken!");
case DrydockLeaseStatus::STATUS_PENDING:
break;
}

97
src/applications/drydock/worker/DrydockAllocatorWorker.php
View file

@ -2,40 +2,117 @@
final class DrydockAllocatorWorker extends PhabricatorWorker {
protected function doWork() {
$lease_id = $this->getTaskData();
private $lease;
$lease = id(new DrydockLease())->load($lease_id);
if (!$lease) {
return;
public function getMaximumRetryCount() {
// TODO: Allow Drydock allocations to retry. For now, every failure is
// permanent and most of them are because I am bad at programming, so fail
// fast rather than ending up in limbo.
return 0;
}
private function loadLease() {
if (empty($this->lease)) {
$lease = id(new DrydockLease())->load($this->getTaskData());
if (!$lease) {
throw new PhabricatorWorkerPermanentFailureException(
"No such lease!");
}
$this->lease = $lease;
}
return $this->lease;
}
private function log($message) {
DrydockBlueprint::writeLog(
null,
$this->loadLease(),
$message);
}
protected function doWork() {
$lease = $this->loadLease();
$this->log('Allocating Lease');
try {
$this->allocateLease($lease);
} catch (Exception $ex) {
// TODO: We should really do this when archiving the task, if we've
// suffered a permanent failure. But we don't have hooks for that yet
// and always fail after the first retry right now, so this is
// functionally equivalent.
$lease->reload();
if ($lease->getStatus() == DrydockLeaseStatus::STATUS_PENDING) {
$lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
$lease->save();
}
throw $ex;
}
}
private function allocateLease(DrydockLease $lease) {
$type = $lease->getResourceType();
$candidates = id(new DrydockResource())->loadAllWhere(
$pool = id(new DrydockResource())->loadAllWhere(
'type = %s AND status = %s',
$lease->getResourceType(),
DrydockResourceStatus::STATUS_OPEN);
$this->log(
pht('Found %d Open Resource(s)', count($pool)));
$candidates = array();
foreach ($pool as $key => $candidate) {
try {
$candidate->getBlueprint();
} catch (Exception $ex) {
unset($pool[$key]);
}
// TODO: Filter candidates according to ability to satisfy the lease.
$candidates[] = $candidate;
}
$this->log(
pht('%d Open Resource(s) Remain', count($candidates)));
if ($candidates) {
shuffle($candidates);
$resource = head($candidates);
} else {
$blueprints = DrydockBlueprint::getAllBlueprintsForResource($type);
$this->log(
pht('Found %d Blueprints', count($blueprints)));
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canAllocateResources()) {
if (!$blueprint->isEnabled()) {
unset($blueprints[$key]);
continue;
}
}
$this->log(
pht('%d Blueprints Enabled', count($blueprints)));
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canAllocateMoreResources($pool)) {
unset($blueprints[$key]);
continue;
}
}
$this->log(
pht('%d Blueprints Can Allocate', count($blueprints)));
if (!$blueprints) {
$lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
$lease->save();
DrydockBlueprint::writeLog(
null,
$lease,
$this->log(
"There are no resources of type '{$type}' available, and no ".
"blueprints which can allocate new ones.");

15
src/infrastructure/daemon/workers/PhabricatorWorker.php
View file

@ -131,6 +131,21 @@ abstract class PhabricatorWorker {
}
$task = head($tasks)->executeTask();
$ex = $task->getExecutionException();
if ($ex) {
throw $ex;
}
}
$tasks = id(new PhabricatorWorkerArchiveTask())->loadAllWhere(
'id IN (%Ld)',
$task_ids);
foreach ($tasks as $task) {
if ($task->getResult() != PhabricatorWorkerArchiveTask::RESULT_SUCCESS) {
throw new Exception("Task ".$task->getID()." failed!");
}
}
}