revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00
Code Issues Releases Wiki Activity

Merge the DrydockLease workers into a single worker

Browse source 
Summary:
Ref T9252. This is the same as D14201, but for lease stuff instead of resource stuff.

This one is a little heavier but still feels pretty reasonable to me at the end of the day (worker is <1K lines and has a ton of comment stuff).

Also fixes a few random bugs I hit in the task queue.

Test Plan:
  - Restarted some Harbormaster builds, saw them go through cleanly.
  - Released pre-activation resources/leases.
  - Probably still kinda buggy but I'll iron the details out over time.

Logs are starting to look somewhat plausible:

{F855747}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9252

Differential Revision: https://secure.phabricator.com/D14202
This commit is contained in:
epriestley 2015-10-01 08:11:02 -07:00
parent 91e5ca0ee2
commit 4ac82be5ed
9 changed files with 615 additions and 628 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/__phutil_library_map__.php
View file

@ -796,7 +796,6 @@ phutil_register_library_map(array(
'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php',
'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php',
'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php',
'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
'DrydockAlmanacServiceHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php',
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php',
@ -834,7 +833,6 @@ phutil_register_library_map(array(
'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php',
'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php',
'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php',
'DrydockLeaseDestroyWorker' => 'applications/drydock/worker/DrydockLeaseDestroyWorker.php',
'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php',
'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
'DrydockLeaseListView' => 'applications/drydock/view/DrydockLeaseListView.php',
@ -847,7 +845,6 @@ phutil_register_library_map(array(
'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php',
'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php',
'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php',
'DrydockLeaseWorker' => 'applications/drydock/worker/DrydockLeaseWorker.php',
'DrydockLog' => 'applications/drydock/storage/DrydockLog.php',
'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php',
'DrydockLogGarbageCollector' => 'applications/drydock/garbagecollector/DrydockLogGarbageCollector.php',
@ -4525,7 +4522,6 @@ phutil_register_library_map(array(
'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec',
'DoorkeeperTagView' => 'AphrontView',
'DoorkeeperTagsController' => 'PhabricatorController',
'DrydockAllocatorWorker' => 'DrydockWorker',
'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation',
'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
'DrydockBlueprint' => array(
@ -4577,7 +4573,6 @@ phutil_register_library_map(array(
'DrydockLeaseActivatedLogType' => 'DrydockLogType',
'DrydockLeaseController' => 'DrydockController',
'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource',
'DrydockLeaseDestroyWorker' => 'DrydockWorker',
'DrydockLeaseDestroyedLogType' => 'DrydockLogType',
'DrydockLeaseListController' => 'DrydockLeaseController',
'DrydockLeaseListView' => 'AphrontView',
@ -4590,7 +4585,6 @@ phutil_register_library_map(array(
'DrydockLeaseStatus' => 'DrydockConstants',
'DrydockLeaseUpdateWorker' => 'DrydockWorker',
'DrydockLeaseViewController' => 'DrydockLeaseController',
'DrydockLeaseWorker' => 'DrydockWorker',
'DrydockLog' => array(
'DrydockDAO',
'PhabricatorPolicyInterface',

18
src/applications/drydock/storage/DrydockLease.php
View file

@ -135,14 +135,7 @@ final class DrydockLease extends DrydockDAO
->setStatus(DrydockLeaseStatus::STATUS_PENDING)
->save();
$task = PhabricatorWorker::scheduleTask(
'DrydockAllocatorWorker',
array(
'leasePHID' => $this->getPHID(),
),
array(
'objectPHID' => $this->getPHID(),
));
$this->scheduleUpdate();
$this->logEvent(DrydockLeaseQueuedLogType::LOGCONST);
@ -321,12 +314,13 @@ final class DrydockLease extends DrydockDAO
}
}
public function canUpdate() {
public function canReceiveCommands() {
switch ($this->getStatus()) {
case DrydockLeaseStatus::STATUS_ACTIVE:
return true;
default:
case DrydockLeaseStatus::STATUS_RELEASED:
case DrydockLeaseStatus::STATUS_DESTROYED:
return false;
default:
return true;
}
}

479
src/applications/drydock/worker/DrydockAllocatorWorker.php
View file

@ -1,479 +0,0 @@
<?php
/**
* @task allocate Allocator
* @task resource Managing Resources
* @task lease Managing Leases
*/
final class DrydockAllocatorWorker extends DrydockWorker {
protected function doWork() {
$lease_phid = $this->getTaskDataValue('leasePHID');
$lease = $this->loadLease($lease_phid);
$this->allocateAndAcquireLease($lease);
}
/* -( Allocator )---------------------------------------------------------- */
/**
* Find or build a resource which can satisfy a given lease request, then
* acquire the lease.
*
* @param DrydockLease Requested lease.
* @return void
* @task allocator
*/
private function allocateAndAcquireLease(DrydockLease $lease) {
$blueprints = $this->loadBlueprintsForAllocatingLease($lease);
// If we get nothing back, that means no blueprint is defined which can
// ever build the requested resource. This is a permanent failure, since
// we don't expect to succeed no matter how many times we try.
if (!$blueprints) {
$lease
->setStatus(DrydockLeaseStatus::STATUS_BROKEN)
->save();
throw new PhabricatorWorkerPermanentFailureException(
pht(
'No active Drydock blueprint exists which can ever allocate a '.
'resource for lease "%s".',
$lease->getPHID()));
}
// First, try to find a suitable open resource which we can acquire a new
// lease on.
$resources = $this->loadResourcesForAllocatingLease($blueprints, $lease);
// If no resources exist yet, see if we can build one.
if (!$resources) {
$usable_blueprints = $this->removeOverallocatedBlueprints(
$blueprints,
$lease);
// If we get nothing back here, some blueprint claims it can eventually
// satisfy the lease, just not right now. This is a temporary failure,
// and we expect allocation to succeed eventually.
if (!$blueprints) {
// TODO: More formal temporary failure here. We should retry this
// "soon" but not "immediately".
throw new Exception(
pht('No blueprints have space to allocate a resource right now.'));
}
$usable_blueprints = $this->rankBlueprints($blueprints, $lease);
$exceptions = array();
foreach ($usable_blueprints as $blueprint) {
try {
$resources[] = $this->allocateResource($blueprint, $lease);
// Bail after allocating one resource, we don't need any more than
// this.
break;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
if (!$resources) {
// TODO: We should distinguish between temporary and permament failures
// here. If any blueprint failed temporarily, retry "soon". If none
// of these failures were temporary, maybe this should be a permanent
// failure?
throw new PhutilAggregateException(
pht(
'All blueprints failed to allocate a suitable new resource when '.
'trying to allocate lease "%s".',
$lease->getPHID()),
$exceptions);
}
// NOTE: We have not acquired the lease yet, so it is possible that the
// resource we just built will be snatched up by some other lease before
// we can. This is not problematic: we'll retry a little later and should
// suceed eventually.
}
$resources = $this->rankResources($resources, $lease);
$exceptions = array();
$allocated = false;
foreach ($resources as $resource) {
try {
$this->acquireLease($resource, $lease);
$allocated = true;
break;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
if (!$allocated) {
// TODO: We should distinguish between temporary and permanent failures
// here. If any failures were temporary (specifically, failed to acquire
// locks)
throw new PhutilAggregateException(
pht(
'Unable to acquire lease "%s" on any resouce.',
$lease->getPHID()),
$exceptions);
}
}
/**
* Get all the @{class:DrydockBlueprintImplementation}s which can possibly
* build a resource to satisfy a lease.
*
* This method returns blueprints which might, at some time, be able to
* build a resource which can satisfy the lease. They may not be able to
* build that resource right now.
*
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprintImplementation> List of qualifying blueprint
* implementations.
* @task allocator
*/
private function loadBlueprintImplementationsForAllocatingLease(
DrydockLease $lease) {
$impls = DrydockBlueprintImplementation::getAllBlueprintImplementations();
$keep = array();
foreach ($impls as $key => $impl) {
// Don't use disabled blueprint types.
if (!$impl->isEnabled()) {
continue;
}
// Don't use blueprint types which can't allocate the correct kind of
// resource.
if ($impl->getType() != $lease->getResourceType()) {
continue;
}
if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $impl;
}
return $keep;
}
/**
* Get all the concrete @{class:DrydockBlueprint}s which can possibly
* build a resource to satisfy a lease.
*
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprint> List of qualifying blueprints.
* @task allocator
*/
private function loadBlueprintsForAllocatingLease(
DrydockLease $lease) {
$viewer = $this->getViewer();
$impls = $this->loadBlueprintImplementationsForAllocatingLease($lease);
if (!$impls) {
return array();
}
$blueprints = id(new DrydockBlueprintQuery())
->setViewer($viewer)
->withBlueprintClasses(array_keys($impls))
->withDisabled(false)
->execute();
$keep = array();
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canEverAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $blueprint;
}
return $keep;
}
/**
* Load a list of all resources which a given lease can possibly be
* allocated against.
*
* @param list<DrydockBlueprint> Blueprints which may produce suitable
* resources.
* @param DrydockLease Requested lease.
* @return list<DrydockResource> Resources which may be able to allocate
* the lease.
* @task allocator
*/
private function loadResourcesForAllocatingLease(
array $blueprints,
DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
$viewer = $this->getViewer();
$resources = id(new DrydockResourceQuery())
->setViewer($viewer)
->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
->withTypes(array($lease->getResourceType()))
->withStatuses(
array(
DrydockResourceStatus::STATUS_PENDING,
DrydockResourceStatus::STATUS_ACTIVE,
))
->execute();
$keep = array();
foreach ($resources as $key => $resource) {
$blueprint = $resource->getBlueprint();
if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
continue;
}
$keep[$key] = $resource;
}
return $keep;
}
/**
* Remove blueprints which are too heavily allocated to build a resource for
* a lease from a list of blueprints.
*
* @param list<DrydockBlueprint> List of blueprints.
* @return list<DrydockBlueprint> List with blueprints that can not allocate
* a resource for the lease right now removed.
* @task allocator
*/
private function removeOverallocatedBlueprints(
array $blueprints,
DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
$keep = array();
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $blueprint;
}
return $keep;
}
/**
* Rank blueprints by suitability for building a new resource for a
* particular lease.
*
* @param list<DrydockBlueprint> List of blueprints.
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprint> Ranked list of blueprints.
* @task allocator
*/
private function rankBlueprints(array $blueprints, DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
// TODO: Implement improvements to this ranking algorithm if they become
// available.
shuffle($blueprints);
return $blueprints;
}
/**
* Rank resources by suitability for allocating a particular lease.
*
* @param list<DrydockResource> List of resources.
* @param DrydockLease Requested lease.
* @return list<DrydockResource> Ranked list of resources.
* @task allocator
*/
private function rankResources(array $resources, DrydockLease $lease) {
assert_instances_of($resources, 'DrydockResource');
// TODO: Implement improvements to this ranking algorithm if they become
// available.
shuffle($resources);
return $resources;
}
/* -( Managing Resources )------------------------------------------------- */
/**
* Perform an actual resource allocation with a particular blueprint.
*
* @param DrydockBlueprint The blueprint to allocate a resource from.
* @param DrydockLease Requested lease.
* @return DrydockResource Allocated resource.
* @task resource
*/
private function allocateResource(
DrydockBlueprint $blueprint,
DrydockLease $lease) {
$resource = $blueprint->allocateResource($lease);
$this->validateAllocatedResource($blueprint, $resource, $lease);
// If this resource was allocated as a pending resource, queue a task to
// activate it.
if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {
PhabricatorWorker::scheduleTask(
'DrydockResourceUpdateWorker',
array(
'resourcePHID' => $resource->getPHID(),
),
array(
'objectPHID' => $resource->getPHID(),
));
}
return $resource;
}
/**
* Check that the resource a blueprint allocated is roughly the sort of
* object we expect.
*
* @param DrydockBlueprint Blueprint which built the resource.
* @param wild Thing which the blueprint claims is a valid resource.
* @param DrydockLease Lease the resource was allocated for.
* @return void
* @task resource
*/
private function validateAllocatedResource(
DrydockBlueprint $blueprint,
$resource,
DrydockLease $lease) {
if (!($resource instanceof DrydockResource)) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
'return an object of type %s or throw, but returned something else.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'allocateResource()',
'DrydockResource'));
}
if (!$resource->isAllocatedResource()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
'must actually allocate the resource it returns.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'allocateResource()'));
}
$resource_type = $resource->getType();
$lease_type = $lease->getResourceType();
if ($resource_type !== $lease_type) {
// TODO: Destroy the resource here?
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'built a resource of type "%s" to satisfy a lease requesting a '.
'resource of type "%s".',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
$resource_type,
$lease_type));
}
}
/* -( Managing Leases )---------------------------------------------------- */
/**
* Perform an actual lease acquisition on a particular resource.
*
* @param DrydockResource Resource to acquire a lease on.
* @param DrydockLease Lease to acquire.
* @return void
* @task lease
*/
private function acquireLease(
DrydockResource $resource,
DrydockLease $lease) {
$blueprint = $resource->getBlueprint();
$blueprint->acquireLease($resource, $lease);
$this->validateAcquiredLease($blueprint, $resource, $lease);
// If this lease has been acquired but not activated, queue a task to
// activate it.
if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
PhabricatorWorker::scheduleTask(
'DrydockLeaseWorker',
array(
'leasePHID' => $lease->getPHID(),
),
array(
'objectPHID' => $lease->getPHID(),
));
}
}
/**
* Make sure that a lease was really acquired properly.
*
* @param DrydockBlueprint Blueprint which created the resource.
* @param DrydockResource Resource which was acquired.
* @param DrydockLease The lease which was supposedly acquired.
* @return void
* @task lease
*/
private function validateAcquiredLease(
DrydockBlueprint $blueprint,
DrydockResource $resource,
DrydockLease $lease) {
if (!$lease->isAcquiredLease()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" without acquiring a lease.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
$lease_phid = $lease->getResourcePHID();
$resource_phid = $resource->getPHID();
if ($lease_phid !== $resource_phid) {
// TODO: Destroy the lease?
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" with a lease acquired on the wrong resource.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
}
}

39
src/applications/drydock/worker/DrydockLeaseDestroyWorker.php
View file

@ -1,39 +0,0 @@
<?php
final class DrydockLeaseDestroyWorker extends DrydockWorker {
protected function doWork() {
$lease_phid = $this->getTaskDataValue('leasePHID');
$lease = $this->loadLease($lease_phid);
$this->destroyLease($lease);
}
private function destroyLease(DrydockLease $lease) {
$status = $lease->getStatus();
switch ($status) {
case DrydockLeaseStatus::STATUS_RELEASED:
case DrydockLeaseStatus::STATUS_BROKEN:
break;
default:
throw new PhabricatorWorkerPermanentFailureException(
pht(
'Unable to destroy lease ("%s"), lease has the wrong '.
'status ("%s").',
$lease->getPHID(),
$status));
}
$resource = $lease->getResource();
$blueprint = $resource->getBlueprint();
$blueprint->destroyLease($resource, $lease);
$lease
->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
->save();
$lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
}
}

617
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
View file

@ -1,5 +1,14 @@
<?php
/**
* @task update Updating Leases
* @task command Processing Commands
* @task allocator Drydock Allocator
* @task acquire Acquiring Leases
* @task activate Activating Leases
* @task release Releasing Leases
* @task destroy Destroying Leases
*/
final class DrydockLeaseUpdateWorker extends DrydockWorker {
protected function doWork() {
@ -22,8 +31,47 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
$lock->unlock();
}
/* -( Updating Leases )---------------------------------------------------- */
/**
* @task update
*/
private function updateLease(DrydockLease $lease) {
if (!$lease->canUpdate()) {
$this->processLeaseCommands($lease);
$lease_status = $lease->getStatus();
switch ($lease_status) {
case DrydockLeaseStatus::STATUS_PENDING:
$this->executeAllocator($lease);
break;
case DrydockLeaseStatus::STATUS_ACQUIRED:
$this->activateLease($lease);
break;
case DrydockLeaseStatus::STATUS_ACTIVE:
// Nothing to do.
break;
case DrydockLeaseStatus::STATUS_RELEASED:
case DrydockLeaseStatus::STATUS_BROKEN:
$this->destroyLease($lease);
break;
case DrydockLeaseStatus::STATUS_DESTROYED:
break;
}
$this->yieldIfExpiringLease($lease);
}
/* -( Processing Commands )------------------------------------------------ */
/**
* @task command
*/
private function processLeaseCommands(DrydockLease $lease) {
if (!$lease->canReceiveCommands()) {
return;
}
@ -31,21 +79,23 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
$commands = $this->loadCommands($lease->getPHID());
foreach ($commands as $command) {
if (!$lease->canUpdate()) {
if (!$lease->canReceiveCommands()) {
break;
}
$this->processCommand($lease, $command);
$this->processLeaseCommand($lease, $command);
$command
->setIsConsumed(true)
->save();
}
$this->yieldIfExpiringLease($lease);
}
private function processCommand(
/**
* @task command
*/
private function processLeaseCommand(
DrydockLease $lease,
DrydockCommand $command) {
switch ($command->getCommand()) {
@ -55,6 +105,530 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
}
}
/* -( Drydock Allocator )-------------------------------------------------- */
/**
* Find or build a resource which can satisfy a given lease request, then
* acquire the lease.
*
* @param DrydockLease Requested lease.
* @return void
* @task allocator
*/
private function executeAllocator(DrydockLease $lease) {
$blueprints = $this->loadBlueprintsForAllocatingLease($lease);
// If we get nothing back, that means no blueprint is defined which can
// ever build the requested resource. This is a permanent failure, since
// we don't expect to succeed no matter how many times we try.
if (!$blueprints) {
throw new PhabricatorWorkerPermanentFailureException(
pht(
'No active Drydock blueprint exists which can ever allocate a '.
'resource for lease "%s".',
$lease->getPHID()));
}
// First, try to find a suitable open resource which we can acquire a new
// lease on.
$resources = $this->loadResourcesForAllocatingLease($blueprints, $lease);
// If no resources exist yet, see if we can build one.
if (!$resources) {
$usable_blueprints = $this->removeOverallocatedBlueprints(
$blueprints,
$lease);
// If we get nothing back here, some blueprint claims it can eventually
// satisfy the lease, just not right now. This is a temporary failure,
// and we expect allocation to succeed eventually.
if (!$usable_blueprints) {
// TODO: More formal temporary failure here. We should retry this
// "soon" but not "immediately".
throw new Exception(
pht('No blueprints have space to allocate a resource right now.'));
}
$usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);
$exceptions = array();
foreach ($usable_blueprints as $blueprint) {
try {
$resources[] = $this->allocateResource($blueprint, $lease);
// Bail after allocating one resource, we don't need any more than
// this.
break;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
if (!$resources) {
// TODO: We should distinguish between temporary and permament failures
// here. If any blueprint failed temporarily, retry "soon". If none
// of these failures were temporary, maybe this should be a permanent
// failure?
throw new PhutilAggregateException(
pht(
'All blueprints failed to allocate a suitable new resource when '.
'trying to allocate lease "%s".',
$lease->getPHID()),
$exceptions);
}
// NOTE: We have not acquired the lease yet, so it is possible that the
// resource we just built will be snatched up by some other lease before
// we can. This is not problematic: we'll retry a little later and should
// suceed eventually.
}
$resources = $this->rankResources($resources, $lease);
$exceptions = array();
$allocated = false;
foreach ($resources as $resource) {
try {
$this->acquireLease($resource, $lease);
$allocated = true;
break;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
if (!$allocated) {
// TODO: We should distinguish between temporary and permanent failures
// here. If any failures were temporary (specifically, failed to acquire
// locks)
throw new PhutilAggregateException(
pht(
'Unable to acquire lease "%s" on any resouce.',
$lease->getPHID()),
$exceptions);
}
}
/**
* Get all the @{class:DrydockBlueprintImplementation}s which can possibly
* build a resource to satisfy a lease.
*
* This method returns blueprints which might, at some time, be able to
* build a resource which can satisfy the lease. They may not be able to
* build that resource right now.
*
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprintImplementation> List of qualifying blueprint
* implementations.
* @task allocator
*/
private function loadBlueprintImplementationsForAllocatingLease(
DrydockLease $lease) {
$impls = DrydockBlueprintImplementation::getAllBlueprintImplementations();
$keep = array();
foreach ($impls as $key => $impl) {
// Don't use disabled blueprint types.
if (!$impl->isEnabled()) {
continue;
}
// Don't use blueprint types which can't allocate the correct kind of
// resource.
if ($impl->getType() != $lease->getResourceType()) {
continue;
}
if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $impl;
}
return $keep;
}
/**
* Get all the concrete @{class:DrydockBlueprint}s which can possibly
* build a resource to satisfy a lease.
*
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprint> List of qualifying blueprints.
* @task allocator
*/
private function loadBlueprintsForAllocatingLease(
DrydockLease $lease) {
$viewer = $this->getViewer();
$impls = $this->loadBlueprintImplementationsForAllocatingLease($lease);
if (!$impls) {
return array();
}
$blueprints = id(new DrydockBlueprintQuery())
->setViewer($viewer)
->withBlueprintClasses(array_keys($impls))
->withDisabled(false)
->execute();
$keep = array();
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canEverAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $blueprint;
}
return $keep;
}
/**
* Load a list of all resources which a given lease can possibly be
* allocated against.
*
* @param list<DrydockBlueprint> Blueprints which may produce suitable
* resources.
* @param DrydockLease Requested lease.
* @return list<DrydockResource> Resources which may be able to allocate
* the lease.
* @task allocator
*/
private function loadResourcesForAllocatingLease(
array $blueprints,
DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
$viewer = $this->getViewer();
$resources = id(new DrydockResourceQuery())
->setViewer($viewer)
->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
->withTypes(array($lease->getResourceType()))
->withStatuses(
array(
DrydockResourceStatus::STATUS_PENDING,
DrydockResourceStatus::STATUS_ACTIVE,
))
->execute();
$keep = array();
foreach ($resources as $key => $resource) {
$blueprint = $resource->getBlueprint();
if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
continue;
}
$keep[$key] = $resource;
}
return $keep;
}
/**
* Remove blueprints which are too heavily allocated to build a resource for
* a lease from a list of blueprints.
*
* @param list<DrydockBlueprint> List of blueprints.
* @return list<DrydockBlueprint> List with blueprints that can not allocate
* a resource for the lease right now removed.
* @task allocator
*/
private function removeOverallocatedBlueprints(
array $blueprints,
DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
$keep = array();
foreach ($blueprints as $key => $blueprint) {
if (!$blueprint->canAllocateResourceForLease($lease)) {
continue;
}
$keep[$key] = $blueprint;
}
return $keep;
}
/**
* Rank blueprints by suitability for building a new resource for a
* particular lease.
*
* @param list<DrydockBlueprint> List of blueprints.
* @param DrydockLease Requested lease.
* @return list<DrydockBlueprint> Ranked list of blueprints.
* @task allocator
*/
private function rankBlueprints(array $blueprints, DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');
// TODO: Implement improvements to this ranking algorithm if they become
// available.
shuffle($blueprints);
return $blueprints;
}
/**
* Rank resources by suitability for allocating a particular lease.
*
* @param list<DrydockResource> List of resources.
* @param DrydockLease Requested lease.
* @return list<DrydockResource> Ranked list of resources.
* @task allocator
*/
private function rankResources(array $resources, DrydockLease $lease) {
assert_instances_of($resources, 'DrydockResource');
// TODO: Implement improvements to this ranking algorithm if they become
// available.
shuffle($resources);
return $resources;
}
/**
* Perform an actual resource allocation with a particular blueprint.
*
* @param DrydockBlueprint The blueprint to allocate a resource from.
* @param DrydockLease Requested lease.
* @return DrydockResource Allocated resource.
* @task allocator
*/
private function allocateResource(
DrydockBlueprint $blueprint,
DrydockLease $lease) {
$resource = $blueprint->allocateResource($lease);
$this->validateAllocatedResource($blueprint, $resource, $lease);
// If this resource was allocated as a pending resource, queue a task to
// activate it.
if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {
PhabricatorWorker::scheduleTask(
'DrydockResourceUpdateWorker',
array(
'resourcePHID' => $resource->getPHID(),
),
array(
'objectPHID' => $resource->getPHID(),
));
}
return $resource;
}
/**
* Check that the resource a blueprint allocated is roughly the sort of
* object we expect.
*
* @param DrydockBlueprint Blueprint which built the resource.
* @param wild Thing which the blueprint claims is a valid resource.
* @param DrydockLease Lease the resource was allocated for.
* @return void
* @task allocator
*/
private function validateAllocatedResource(
DrydockBlueprint $blueprint,
$resource,
DrydockLease $lease) {
if (!($resource instanceof DrydockResource)) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
'return an object of type %s or throw, but returned something else.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'allocateResource()',
'DrydockResource'));
}
if (!$resource->isAllocatedResource()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
'must actually allocate the resource it returns.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'allocateResource()'));
}
$resource_type = $resource->getType();
$lease_type = $lease->getResourceType();
if ($resource_type !== $lease_type) {
// TODO: Destroy the resource here?
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'built a resource of type "%s" to satisfy a lease requesting a '.
'resource of type "%s".',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
$resource_type,
$lease_type));
}
}
/* -( Acquiring Leases )--------------------------------------------------- */
/**
* Perform an actual lease acquisition on a particular resource.
*
* @param DrydockResource Resource to acquire a lease on.
* @param DrydockLease Lease to acquire.
* @return void
* @task acquire
*/
private function acquireLease(
DrydockResource $resource,
DrydockLease $lease) {
$blueprint = $resource->getBlueprint();
$blueprint->acquireLease($resource, $lease);
$this->validateAcquiredLease($blueprint, $resource, $lease);
// If this lease has been acquired but not activated, queue a task to
// activate it.
if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
PhabricatorWorker::scheduleTask(
__CLASS__,
array(
'leasePHID' => $lease->getPHID(),
),
array(
'objectPHID' => $lease->getPHID(),
));
}
}
/**
* Make sure that a lease was really acquired properly.
*
* @param DrydockBlueprint Blueprint which created the resource.
* @param DrydockResource Resource which was acquired.
* @param DrydockLease The lease which was supposedly acquired.
* @return void
* @task acquire
*/
private function validateAcquiredLease(
DrydockBlueprint $blueprint,
DrydockResource $resource,
DrydockLease $lease) {
if (!$lease->isAcquiredLease()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" without acquiring a lease.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
$lease_phid = $lease->getResourcePHID();
$resource_phid = $resource->getPHID();
if ($lease_phid !== $resource_phid) {
// TODO: Destroy the lease?
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" with a lease acquired on the wrong resource.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
}
/* -( Activating Leases )-------------------------------------------------- */
/**
* @task activate
*/
private function activateLease(DrydockLease $lease) {
$resource = $lease->getResource();
if (!$resource) {
throw new PhabricatorWorkerPermanentFailureException(
pht('Trying to activate lease with no resource.'));
}
$resource_status = $resource->getStatus();
if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
// TODO: This is explicitly a temporary failure -- we are waiting for
// the resource to come up.
throw new Exception(pht('Resource still activating.'));
}
if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
throw new PhabricatorWorkerPermanentFailureException(
pht(
'Trying to activate lease on a dead resource (in status "%s").',
$resource_status));
}
// NOTE: We can race resource destruction here. Between the time we
// performed the read above and now, the resource might have closed, so
// we may activate leases on dead resources. At least for now, this seems
// fine: a resource dying right before we activate a lease on it should not
// be distinguisahble from a resource dying right after we activate a lease
// on it. We end up with an active lease on a dead resource either way, and
// can not prevent resources dying from lightning strikes.
$blueprint = $resource->getBlueprint();
$blueprint->activateLease($resource, $lease);
$this->validateActivatedLease($blueprint, $resource, $lease);
}
/**
* @task activate
*/
private function validateActivatedLease(
DrydockBlueprint $blueprint,
DrydockResource $resource,
DrydockLease $lease) {
if (!$lease->isActivatedLease()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" without activating a lease.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
}
/* -( Releasing Leases )--------------------------------------------------- */
/**
* @task release
*/
private function releaseLease(DrydockLease $lease) {
$lease->openTransaction();
$lease
@ -65,21 +639,34 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
DrydockSlotLock::releaseLocks($lease->getPHID());
$lease->saveTransaction();
PhabricatorWorker::scheduleTask(
'DrydockLeaseDestroyWorker',
array(
'leasePHID' => $lease->getPHID(),
),
array(
'objectPHID' => $lease->getPHID(),
));
$lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST);
$resource = $lease->getResource();
$blueprint = $resource->getBlueprint();
$blueprint->didReleaseLease($resource, $lease);
$this->destroyLease($lease);
}
/* -( Destroying Leases )-------------------------------------------------- */
/**
* @task destroy
*/
private function destroyLease(DrydockLease $lease) {
$resource = $lease->getResource();
$blueprint = $resource->getBlueprint();
$blueprint->destroyLease($resource, $lease);
$lease
->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
->save();
$lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
}
}

74
src/applications/drydock/worker/DrydockLeaseWorker.php
View file

@ -1,74 +0,0 @@
<?php
final class DrydockLeaseWorker extends DrydockWorker {
protected function doWork() {
$lease_phid = $this->getTaskDataValue('leasePHID');
$lease = $this->loadLease($lease_phid);
$this->activateLease($lease);
}
private function activateLease(DrydockLease $lease) {
$actual_status = $lease->getStatus();
if ($actual_status != DrydockLeaseStatus::STATUS_ACQUIRED) {
throw new PhabricatorWorkerPermanentFailureException(
pht(
'Trying to activate lease from wrong status ("%s").',
$actual_status));
}
$resource = $lease->getResource();
if (!$resource) {
throw new PhabricatorWorkerPermanentFailureException(
pht('Trying to activate lease with no resource.'));
}
$resource_status = $resource->getStatus();
if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
// TODO: This is explicitly a temporary failure -- we are waiting for
// the resource to come up.
throw new Exception(pht('Resource still activating.'));
}
if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
throw new PhabricatorWorkerPermanentFailureException(
pht(
'Trying to activate lease on a dead resource (in status "%s").',
$resource_status));
}
// NOTE: We can race resource destruction here. Between the time we
// performed the read above and now, the resource might have closed, so
// we may activate leases on dead resources. At least for now, this seems
// fine: a resource dying right before we activate a lease on it should not
// be distinguisahble from a resource dying right after we activate a lease
// on it. We end up with an active lease on a dead resource either way, and
// can not prevent resources dying from lightning strikes.
$blueprint = $resource->getBlueprint();
$blueprint->activateLease($resource, $lease);
$this->validateActivatedLease($blueprint, $resource, $lease);
}
private function validateActivatedLease(
DrydockBlueprint $blueprint,
DrydockResource $resource,
DrydockLease $lease) {
if (!$lease->isActivatedLease()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" without activating a lease.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'acquireLease()'));
}
}
}

2
src/applications/drydock/worker/DrydockWorker.php
View file

@ -86,7 +86,7 @@ abstract class DrydockWorker extends PhabricatorWorker {
}
protected function yieldIfExpiringLease(DrydockLease $lease) {
if (!$lease->canUpdate()) {
if (!$lease->canReceiveCommands()) {
return;
}

3
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
View file

@ -154,7 +154,8 @@ final class HarbormasterBuildLog extends HarbormasterDAO
public function finalize($start = 0) {
if (!$this->getLive()) {
throw new Exception(pht('Start logging before finalizing it.'));
// TODO: Clean up this API.
return;
}
// TODO: Encode the log contents in a gzipped format.

5
src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php
View file

@ -60,7 +60,10 @@ abstract class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
$id = $this->getID();
$class = $this->getTaskClass();
if (!class_exists($class)) {
try {
// NOTE: If the class does not exist, libphutil will throw an exception.
class_exists($class);
} catch (PhutilMissingSymbolException $ex) {
throw new PhabricatorWorkerPermanentFailureException(
pht(
"Task class '%s' does not exist!",