diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index d4bcc2b6a0..d00b105260 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -796,7 +796,6 @@ phutil_register_library_map(array( 'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php', 'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php', 'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php', - 'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php', 'DrydockAlmanacServiceHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php', 'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php', 'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php', @@ -834,7 +833,6 @@ phutil_register_library_map(array( 'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php', 'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php', 'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php', - 'DrydockLeaseDestroyWorker' => 'applications/drydock/worker/DrydockLeaseDestroyWorker.php', 'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php', 'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php', 'DrydockLeaseListView' => 'applications/drydock/view/DrydockLeaseListView.php', @@ -847,7 +845,6 @@ phutil_register_library_map(array( 'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php', 'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php', 'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php', - 'DrydockLeaseWorker' => 'applications/drydock/worker/DrydockLeaseWorker.php', 'DrydockLog' => 'applications/drydock/storage/DrydockLog.php', 'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php', 'DrydockLogGarbageCollector' => 'applications/drydock/garbagecollector/DrydockLogGarbageCollector.php', @@ -4525,7 +4522,6 @@ phutil_register_library_map(array( 'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'DoorkeeperTagView' => 'AphrontView', 'DoorkeeperTagsController' => 'PhabricatorController', - 'DrydockAllocatorWorker' => 'DrydockWorker', 'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation', 'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface', 'DrydockBlueprint' => array( @@ -4577,7 +4573,6 @@ phutil_register_library_map(array( 'DrydockLeaseActivatedLogType' => 'DrydockLogType', 'DrydockLeaseController' => 'DrydockController', 'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource', - 'DrydockLeaseDestroyWorker' => 'DrydockWorker', 'DrydockLeaseDestroyedLogType' => 'DrydockLogType', 'DrydockLeaseListController' => 'DrydockLeaseController', 'DrydockLeaseListView' => 'AphrontView', @@ -4590,7 +4585,6 @@ phutil_register_library_map(array( 'DrydockLeaseStatus' => 'DrydockConstants', 'DrydockLeaseUpdateWorker' => 'DrydockWorker', 'DrydockLeaseViewController' => 'DrydockLeaseController', - 'DrydockLeaseWorker' => 'DrydockWorker', 'DrydockLog' => array( 'DrydockDAO', 'PhabricatorPolicyInterface', diff --git a/src/applications/drydock/storage/DrydockLease.php b/src/applications/drydock/storage/DrydockLease.php index 1dc9f0e180..c5b8b827c0 100644 --- a/src/applications/drydock/storage/DrydockLease.php +++ b/src/applications/drydock/storage/DrydockLease.php @@ -135,14 +135,7 @@ final class DrydockLease extends DrydockDAO ->setStatus(DrydockLeaseStatus::STATUS_PENDING) ->save(); - $task = PhabricatorWorker::scheduleTask( - 'DrydockAllocatorWorker', - array( - 'leasePHID' => $this->getPHID(), - ), - array( - 'objectPHID' => $this->getPHID(), - )); + $this->scheduleUpdate(); $this->logEvent(DrydockLeaseQueuedLogType::LOGCONST); @@ -321,12 +314,13 @@ final class DrydockLease extends DrydockDAO } } - public function canUpdate() { + public function canReceiveCommands() { switch ($this->getStatus()) { - case DrydockLeaseStatus::STATUS_ACTIVE: - return true; - default: + case DrydockLeaseStatus::STATUS_RELEASED: + case DrydockLeaseStatus::STATUS_DESTROYED: return false; + default: + return true; } } diff --git a/src/applications/drydock/worker/DrydockAllocatorWorker.php b/src/applications/drydock/worker/DrydockAllocatorWorker.php deleted file mode 100644 index 08a702b589..0000000000 --- a/src/applications/drydock/worker/DrydockAllocatorWorker.php +++ /dev/null @@ -1,479 +0,0 @@ -getTaskDataValue('leasePHID'); - $lease = $this->loadLease($lease_phid); - - $this->allocateAndAcquireLease($lease); - } - - -/* -( Allocator )---------------------------------------------------------- */ - - - /** - * Find or build a resource which can satisfy a given lease request, then - * acquire the lease. - * - * @param DrydockLease Requested lease. - * @return void - * @task allocator - */ - private function allocateAndAcquireLease(DrydockLease $lease) { - $blueprints = $this->loadBlueprintsForAllocatingLease($lease); - - // If we get nothing back, that means no blueprint is defined which can - // ever build the requested resource. This is a permanent failure, since - // we don't expect to succeed no matter how many times we try. - if (!$blueprints) { - $lease - ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) - ->save(); - throw new PhabricatorWorkerPermanentFailureException( - pht( - 'No active Drydock blueprint exists which can ever allocate a '. - 'resource for lease "%s".', - $lease->getPHID())); - } - - // First, try to find a suitable open resource which we can acquire a new - // lease on. - $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); - - // If no resources exist yet, see if we can build one. - if (!$resources) { - $usable_blueprints = $this->removeOverallocatedBlueprints( - $blueprints, - $lease); - - // If we get nothing back here, some blueprint claims it can eventually - // satisfy the lease, just not right now. This is a temporary failure, - // and we expect allocation to succeed eventually. - if (!$blueprints) { - // TODO: More formal temporary failure here. We should retry this - // "soon" but not "immediately". - throw new Exception( - pht('No blueprints have space to allocate a resource right now.')); - } - - $usable_blueprints = $this->rankBlueprints($blueprints, $lease); - - $exceptions = array(); - foreach ($usable_blueprints as $blueprint) { - try { - $resources[] = $this->allocateResource($blueprint, $lease); - - // Bail after allocating one resource, we don't need any more than - // this. - break; - } catch (Exception $ex) { - $exceptions[] = $ex; - } - } - - if (!$resources) { - // TODO: We should distinguish between temporary and permament failures - // here. If any blueprint failed temporarily, retry "soon". If none - // of these failures were temporary, maybe this should be a permanent - // failure? - throw new PhutilAggregateException( - pht( - 'All blueprints failed to allocate a suitable new resource when '. - 'trying to allocate lease "%s".', - $lease->getPHID()), - $exceptions); - } - - // NOTE: We have not acquired the lease yet, so it is possible that the - // resource we just built will be snatched up by some other lease before - // we can. This is not problematic: we'll retry a little later and should - // suceed eventually. - } - - $resources = $this->rankResources($resources, $lease); - - $exceptions = array(); - $allocated = false; - foreach ($resources as $resource) { - try { - $this->acquireLease($resource, $lease); - $allocated = true; - break; - } catch (Exception $ex) { - $exceptions[] = $ex; - } - } - - if (!$allocated) { - // TODO: We should distinguish between temporary and permanent failures - // here. If any failures were temporary (specifically, failed to acquire - // locks) - - throw new PhutilAggregateException( - pht( - 'Unable to acquire lease "%s" on any resouce.', - $lease->getPHID()), - $exceptions); - } - } - - - /** - * Get all the @{class:DrydockBlueprintImplementation}s which can possibly - * build a resource to satisfy a lease. - * - * This method returns blueprints which might, at some time, be able to - * build a resource which can satisfy the lease. They may not be able to - * build that resource right now. - * - * @param DrydockLease Requested lease. - * @return list List of qualifying blueprint - * implementations. - * @task allocator - */ - private function loadBlueprintImplementationsForAllocatingLease( - DrydockLease $lease) { - - $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); - - $keep = array(); - foreach ($impls as $key => $impl) { - // Don't use disabled blueprint types. - if (!$impl->isEnabled()) { - continue; - } - - // Don't use blueprint types which can't allocate the correct kind of - // resource. - if ($impl->getType() != $lease->getResourceType()) { - continue; - } - - if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { - continue; - } - - $keep[$key] = $impl; - } - - return $keep; - } - - - /** - * Get all the concrete @{class:DrydockBlueprint}s which can possibly - * build a resource to satisfy a lease. - * - * @param DrydockLease Requested lease. - * @return list List of qualifying blueprints. - * @task allocator - */ - private function loadBlueprintsForAllocatingLease( - DrydockLease $lease) { - $viewer = $this->getViewer(); - - $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); - if (!$impls) { - return array(); - } - - $blueprints = id(new DrydockBlueprintQuery()) - ->setViewer($viewer) - ->withBlueprintClasses(array_keys($impls)) - ->withDisabled(false) - ->execute(); - - $keep = array(); - foreach ($blueprints as $key => $blueprint) { - if (!$blueprint->canEverAllocateResourceForLease($lease)) { - continue; - } - - $keep[$key] = $blueprint; - } - - return $keep; - } - - - /** - * Load a list of all resources which a given lease can possibly be - * allocated against. - * - * @param list Blueprints which may produce suitable - * resources. - * @param DrydockLease Requested lease. - * @return list Resources which may be able to allocate - * the lease. - * @task allocator - */ - private function loadResourcesForAllocatingLease( - array $blueprints, - DrydockLease $lease) { - assert_instances_of($blueprints, 'DrydockBlueprint'); - $viewer = $this->getViewer(); - - $resources = id(new DrydockResourceQuery()) - ->setViewer($viewer) - ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) - ->withTypes(array($lease->getResourceType())) - ->withStatuses( - array( - DrydockResourceStatus::STATUS_PENDING, - DrydockResourceStatus::STATUS_ACTIVE, - )) - ->execute(); - - $keep = array(); - foreach ($resources as $key => $resource) { - $blueprint = $resource->getBlueprint(); - - if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { - continue; - } - - $keep[$key] = $resource; - } - - return $keep; - } - - - /** - * Remove blueprints which are too heavily allocated to build a resource for - * a lease from a list of blueprints. - * - * @param list List of blueprints. - * @return list List with blueprints that can not allocate - * a resource for the lease right now removed. - * @task allocator - */ - private function removeOverallocatedBlueprints( - array $blueprints, - DrydockLease $lease) { - assert_instances_of($blueprints, 'DrydockBlueprint'); - - $keep = array(); - foreach ($blueprints as $key => $blueprint) { - if (!$blueprint->canAllocateResourceForLease($lease)) { - continue; - } - - $keep[$key] = $blueprint; - } - - return $keep; - } - - - /** - * Rank blueprints by suitability for building a new resource for a - * particular lease. - * - * @param list List of blueprints. - * @param DrydockLease Requested lease. - * @return list Ranked list of blueprints. - * @task allocator - */ - private function rankBlueprints(array $blueprints, DrydockLease $lease) { - assert_instances_of($blueprints, 'DrydockBlueprint'); - - // TODO: Implement improvements to this ranking algorithm if they become - // available. - shuffle($blueprints); - - return $blueprints; - } - - - /** - * Rank resources by suitability for allocating a particular lease. - * - * @param list List of resources. - * @param DrydockLease Requested lease. - * @return list Ranked list of resources. - * @task allocator - */ - private function rankResources(array $resources, DrydockLease $lease) { - assert_instances_of($resources, 'DrydockResource'); - - // TODO: Implement improvements to this ranking algorithm if they become - // available. - shuffle($resources); - - return $resources; - } - - -/* -( Managing Resources )------------------------------------------------- */ - - - /** - * Perform an actual resource allocation with a particular blueprint. - * - * @param DrydockBlueprint The blueprint to allocate a resource from. - * @param DrydockLease Requested lease. - * @return DrydockResource Allocated resource. - * @task resource - */ - private function allocateResource( - DrydockBlueprint $blueprint, - DrydockLease $lease) { - $resource = $blueprint->allocateResource($lease); - $this->validateAllocatedResource($blueprint, $resource, $lease); - - // If this resource was allocated as a pending resource, queue a task to - // activate it. - if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { - PhabricatorWorker::scheduleTask( - 'DrydockResourceUpdateWorker', - array( - 'resourcePHID' => $resource->getPHID(), - ), - array( - 'objectPHID' => $resource->getPHID(), - )); - } - - return $resource; - } - - - /** - * Check that the resource a blueprint allocated is roughly the sort of - * object we expect. - * - * @param DrydockBlueprint Blueprint which built the resource. - * @param wild Thing which the blueprint claims is a valid resource. - * @param DrydockLease Lease the resource was allocated for. - * @return void - * @task resource - */ - private function validateAllocatedResource( - DrydockBlueprint $blueprint, - $resource, - DrydockLease $lease) { - - if (!($resource instanceof DrydockResource)) { - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. - 'return an object of type %s or throw, but returned something else.', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - 'allocateResource()', - 'DrydockResource')); - } - - if (!$resource->isAllocatedResource()) { - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. - 'must actually allocate the resource it returns.', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - 'allocateResource()')); - } - - $resource_type = $resource->getType(); - $lease_type = $lease->getResourceType(); - - if ($resource_type !== $lease_type) { - // TODO: Destroy the resource here? - - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. - 'built a resource of type "%s" to satisfy a lease requesting a '. - 'resource of type "%s".', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - $resource_type, - $lease_type)); - } - } - - -/* -( Managing Leases )---------------------------------------------------- */ - - - /** - * Perform an actual lease acquisition on a particular resource. - * - * @param DrydockResource Resource to acquire a lease on. - * @param DrydockLease Lease to acquire. - * @return void - * @task lease - */ - private function acquireLease( - DrydockResource $resource, - DrydockLease $lease) { - - $blueprint = $resource->getBlueprint(); - $blueprint->acquireLease($resource, $lease); - - $this->validateAcquiredLease($blueprint, $resource, $lease); - - // If this lease has been acquired but not activated, queue a task to - // activate it. - if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { - PhabricatorWorker::scheduleTask( - 'DrydockLeaseWorker', - array( - 'leasePHID' => $lease->getPHID(), - ), - array( - 'objectPHID' => $lease->getPHID(), - )); - } - } - - - /** - * Make sure that a lease was really acquired properly. - * - * @param DrydockBlueprint Blueprint which created the resource. - * @param DrydockResource Resource which was acquired. - * @param DrydockLease The lease which was supposedly acquired. - * @return void - * @task lease - */ - private function validateAcquiredLease( - DrydockBlueprint $blueprint, - DrydockResource $resource, - DrydockLease $lease) { - - if (!$lease->isAcquiredLease()) { - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. - 'returned from "%s" without acquiring a lease.', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - 'acquireLease()')); - } - - $lease_phid = $lease->getResourcePHID(); - $resource_phid = $resource->getPHID(); - - if ($lease_phid !== $resource_phid) { - // TODO: Destroy the lease? - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. - 'returned from "%s" with a lease acquired on the wrong resource.', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - 'acquireLease()')); - } - } - - -} diff --git a/src/applications/drydock/worker/DrydockLeaseDestroyWorker.php b/src/applications/drydock/worker/DrydockLeaseDestroyWorker.php deleted file mode 100644 index 12b9aa5a35..0000000000 --- a/src/applications/drydock/worker/DrydockLeaseDestroyWorker.php +++ /dev/null @@ -1,39 +0,0 @@ -getTaskDataValue('leasePHID'); - $lease = $this->loadLease($lease_phid); - $this->destroyLease($lease); - } - - private function destroyLease(DrydockLease $lease) { - $status = $lease->getStatus(); - - switch ($status) { - case DrydockLeaseStatus::STATUS_RELEASED: - case DrydockLeaseStatus::STATUS_BROKEN: - break; - default: - throw new PhabricatorWorkerPermanentFailureException( - pht( - 'Unable to destroy lease ("%s"), lease has the wrong '. - 'status ("%s").', - $lease->getPHID(), - $status)); - } - - $resource = $lease->getResource(); - $blueprint = $resource->getBlueprint(); - - $blueprint->destroyLease($resource, $lease); - - $lease - ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) - ->save(); - - $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); - } - -} diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 2003372b76..942db6d7c0 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,5 +1,14 @@ unlock(); } + +/* -( Updating Leases )---------------------------------------------------- */ + + + /** + * @task update + */ private function updateLease(DrydockLease $lease) { - if (!$lease->canUpdate()) { + $this->processLeaseCommands($lease); + + $lease_status = $lease->getStatus(); + switch ($lease_status) { + case DrydockLeaseStatus::STATUS_PENDING: + $this->executeAllocator($lease); + break; + case DrydockLeaseStatus::STATUS_ACQUIRED: + $this->activateLease($lease); + break; + case DrydockLeaseStatus::STATUS_ACTIVE: + // Nothing to do. + break; + case DrydockLeaseStatus::STATUS_RELEASED: + case DrydockLeaseStatus::STATUS_BROKEN: + $this->destroyLease($lease); + break; + case DrydockLeaseStatus::STATUS_DESTROYED: + break; + } + + $this->yieldIfExpiringLease($lease); + } + + +/* -( Processing Commands )------------------------------------------------ */ + + + /** + * @task command + */ + private function processLeaseCommands(DrydockLease $lease) { + if (!$lease->canReceiveCommands()) { return; } @@ -31,21 +79,23 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { - if (!$lease->canUpdate()) { + if (!$lease->canReceiveCommands()) { break; } - $this->processCommand($lease, $command); + $this->processLeaseCommand($lease, $command); $command ->setIsConsumed(true) ->save(); } - - $this->yieldIfExpiringLease($lease); } - private function processCommand( + + /** + * @task command + */ + private function processLeaseCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { @@ -55,6 +105,530 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { } } + +/* -( Drydock Allocator )-------------------------------------------------- */ + + + /** + * Find or build a resource which can satisfy a given lease request, then + * acquire the lease. + * + * @param DrydockLease Requested lease. + * @return void + * @task allocator + */ + private function executeAllocator(DrydockLease $lease) { + $blueprints = $this->loadBlueprintsForAllocatingLease($lease); + + // If we get nothing back, that means no blueprint is defined which can + // ever build the requested resource. This is a permanent failure, since + // we don't expect to succeed no matter how many times we try. + if (!$blueprints) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'No active Drydock blueprint exists which can ever allocate a '. + 'resource for lease "%s".', + $lease->getPHID())); + } + + // First, try to find a suitable open resource which we can acquire a new + // lease on. + $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); + + // If no resources exist yet, see if we can build one. + if (!$resources) { + $usable_blueprints = $this->removeOverallocatedBlueprints( + $blueprints, + $lease); + + // If we get nothing back here, some blueprint claims it can eventually + // satisfy the lease, just not right now. This is a temporary failure, + // and we expect allocation to succeed eventually. + if (!$usable_blueprints) { + // TODO: More formal temporary failure here. We should retry this + // "soon" but not "immediately". + throw new Exception( + pht('No blueprints have space to allocate a resource right now.')); + } + + $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease); + + $exceptions = array(); + foreach ($usable_blueprints as $blueprint) { + try { + $resources[] = $this->allocateResource($blueprint, $lease); + + // Bail after allocating one resource, we don't need any more than + // this. + break; + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + + if (!$resources) { + // TODO: We should distinguish between temporary and permament failures + // here. If any blueprint failed temporarily, retry "soon". If none + // of these failures were temporary, maybe this should be a permanent + // failure? + throw new PhutilAggregateException( + pht( + 'All blueprints failed to allocate a suitable new resource when '. + 'trying to allocate lease "%s".', + $lease->getPHID()), + $exceptions); + } + + // NOTE: We have not acquired the lease yet, so it is possible that the + // resource we just built will be snatched up by some other lease before + // we can. This is not problematic: we'll retry a little later and should + // suceed eventually. + } + + $resources = $this->rankResources($resources, $lease); + + $exceptions = array(); + $allocated = false; + foreach ($resources as $resource) { + try { + $this->acquireLease($resource, $lease); + $allocated = true; + break; + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + + if (!$allocated) { + // TODO: We should distinguish between temporary and permanent failures + // here. If any failures were temporary (specifically, failed to acquire + // locks) + + throw new PhutilAggregateException( + pht( + 'Unable to acquire lease "%s" on any resouce.', + $lease->getPHID()), + $exceptions); + } + } + + + /** + * Get all the @{class:DrydockBlueprintImplementation}s which can possibly + * build a resource to satisfy a lease. + * + * This method returns blueprints which might, at some time, be able to + * build a resource which can satisfy the lease. They may not be able to + * build that resource right now. + * + * @param DrydockLease Requested lease. + * @return list List of qualifying blueprint + * implementations. + * @task allocator + */ + private function loadBlueprintImplementationsForAllocatingLease( + DrydockLease $lease) { + + $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); + + $keep = array(); + foreach ($impls as $key => $impl) { + // Don't use disabled blueprint types. + if (!$impl->isEnabled()) { + continue; + } + + // Don't use blueprint types which can't allocate the correct kind of + // resource. + if ($impl->getType() != $lease->getResourceType()) { + continue; + } + + if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { + continue; + } + + $keep[$key] = $impl; + } + + return $keep; + } + + + /** + * Get all the concrete @{class:DrydockBlueprint}s which can possibly + * build a resource to satisfy a lease. + * + * @param DrydockLease Requested lease. + * @return list List of qualifying blueprints. + * @task allocator + */ + private function loadBlueprintsForAllocatingLease( + DrydockLease $lease) { + $viewer = $this->getViewer(); + + $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); + if (!$impls) { + return array(); + } + + $blueprints = id(new DrydockBlueprintQuery()) + ->setViewer($viewer) + ->withBlueprintClasses(array_keys($impls)) + ->withDisabled(false) + ->execute(); + + $keep = array(); + foreach ($blueprints as $key => $blueprint) { + if (!$blueprint->canEverAllocateResourceForLease($lease)) { + continue; + } + + $keep[$key] = $blueprint; + } + + return $keep; + } + + + /** + * Load a list of all resources which a given lease can possibly be + * allocated against. + * + * @param list Blueprints which may produce suitable + * resources. + * @param DrydockLease Requested lease. + * @return list Resources which may be able to allocate + * the lease. + * @task allocator + */ + private function loadResourcesForAllocatingLease( + array $blueprints, + DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); + $viewer = $this->getViewer(); + + $resources = id(new DrydockResourceQuery()) + ->setViewer($viewer) + ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) + ->withTypes(array($lease->getResourceType())) + ->withStatuses( + array( + DrydockResourceStatus::STATUS_PENDING, + DrydockResourceStatus::STATUS_ACTIVE, + )) + ->execute(); + + $keep = array(); + foreach ($resources as $key => $resource) { + $blueprint = $resource->getBlueprint(); + + if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { + continue; + } + + $keep[$key] = $resource; + } + + return $keep; + } + + + /** + * Remove blueprints which are too heavily allocated to build a resource for + * a lease from a list of blueprints. + * + * @param list List of blueprints. + * @return list List with blueprints that can not allocate + * a resource for the lease right now removed. + * @task allocator + */ + private function removeOverallocatedBlueprints( + array $blueprints, + DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); + + $keep = array(); + foreach ($blueprints as $key => $blueprint) { + if (!$blueprint->canAllocateResourceForLease($lease)) { + continue; + } + + $keep[$key] = $blueprint; + } + + return $keep; + } + + + /** + * Rank blueprints by suitability for building a new resource for a + * particular lease. + * + * @param list List of blueprints. + * @param DrydockLease Requested lease. + * @return list Ranked list of blueprints. + * @task allocator + */ + private function rankBlueprints(array $blueprints, DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); + + // TODO: Implement improvements to this ranking algorithm if they become + // available. + shuffle($blueprints); + + return $blueprints; + } + + + /** + * Rank resources by suitability for allocating a particular lease. + * + * @param list List of resources. + * @param DrydockLease Requested lease. + * @return list Ranked list of resources. + * @task allocator + */ + private function rankResources(array $resources, DrydockLease $lease) { + assert_instances_of($resources, 'DrydockResource'); + + // TODO: Implement improvements to this ranking algorithm if they become + // available. + shuffle($resources); + + return $resources; + } + + + /** + * Perform an actual resource allocation with a particular blueprint. + * + * @param DrydockBlueprint The blueprint to allocate a resource from. + * @param DrydockLease Requested lease. + * @return DrydockResource Allocated resource. + * @task allocator + */ + private function allocateResource( + DrydockBlueprint $blueprint, + DrydockLease $lease) { + $resource = $blueprint->allocateResource($lease); + $this->validateAllocatedResource($blueprint, $resource, $lease); + + // If this resource was allocated as a pending resource, queue a task to + // activate it. + if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { + PhabricatorWorker::scheduleTask( + 'DrydockResourceUpdateWorker', + array( + 'resourcePHID' => $resource->getPHID(), + ), + array( + 'objectPHID' => $resource->getPHID(), + )); + } + + return $resource; + } + + + /** + * Check that the resource a blueprint allocated is roughly the sort of + * object we expect. + * + * @param DrydockBlueprint Blueprint which built the resource. + * @param wild Thing which the blueprint claims is a valid resource. + * @param DrydockLease Lease the resource was allocated for. + * @return void + * @task allocator + */ + private function validateAllocatedResource( + DrydockBlueprint $blueprint, + $resource, + DrydockLease $lease) { + + if (!($resource instanceof DrydockResource)) { + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. + 'return an object of type %s or throw, but returned something else.', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + 'allocateResource()', + 'DrydockResource')); + } + + if (!$resource->isAllocatedResource()) { + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. + 'must actually allocate the resource it returns.', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + 'allocateResource()')); + } + + $resource_type = $resource->getType(); + $lease_type = $lease->getResourceType(); + + if ($resource_type !== $lease_type) { + // TODO: Destroy the resource here? + + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. + 'built a resource of type "%s" to satisfy a lease requesting a '. + 'resource of type "%s".', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + $resource_type, + $lease_type)); + } + } + + +/* -( Acquiring Leases )--------------------------------------------------- */ + + + /** + * Perform an actual lease acquisition on a particular resource. + * + * @param DrydockResource Resource to acquire a lease on. + * @param DrydockLease Lease to acquire. + * @return void + * @task acquire + */ + private function acquireLease( + DrydockResource $resource, + DrydockLease $lease) { + + $blueprint = $resource->getBlueprint(); + $blueprint->acquireLease($resource, $lease); + + $this->validateAcquiredLease($blueprint, $resource, $lease); + + // If this lease has been acquired but not activated, queue a task to + // activate it. + if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { + PhabricatorWorker::scheduleTask( + __CLASS__, + array( + 'leasePHID' => $lease->getPHID(), + ), + array( + 'objectPHID' => $lease->getPHID(), + )); + } + } + + + /** + * Make sure that a lease was really acquired properly. + * + * @param DrydockBlueprint Blueprint which created the resource. + * @param DrydockResource Resource which was acquired. + * @param DrydockLease The lease which was supposedly acquired. + * @return void + * @task acquire + */ + private function validateAcquiredLease( + DrydockBlueprint $blueprint, + DrydockResource $resource, + DrydockLease $lease) { + + if (!$lease->isAcquiredLease()) { + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. + 'returned from "%s" without acquiring a lease.', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + 'acquireLease()')); + } + + $lease_phid = $lease->getResourcePHID(); + $resource_phid = $resource->getPHID(); + + if ($lease_phid !== $resource_phid) { + // TODO: Destroy the lease? + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. + 'returned from "%s" with a lease acquired on the wrong resource.', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + 'acquireLease()')); + } + } + + +/* -( Activating Leases )-------------------------------------------------- */ + + + /** + * @task activate + */ + private function activateLease(DrydockLease $lease) { + $resource = $lease->getResource(); + if (!$resource) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Trying to activate lease with no resource.')); + } + + $resource_status = $resource->getStatus(); + + if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { + // TODO: This is explicitly a temporary failure -- we are waiting for + // the resource to come up. + throw new Exception(pht('Resource still activating.')); + } + + if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Trying to activate lease on a dead resource (in status "%s").', + $resource_status)); + } + + // NOTE: We can race resource destruction here. Between the time we + // performed the read above and now, the resource might have closed, so + // we may activate leases on dead resources. At least for now, this seems + // fine: a resource dying right before we activate a lease on it should not + // be distinguisahble from a resource dying right after we activate a lease + // on it. We end up with an active lease on a dead resource either way, and + // can not prevent resources dying from lightning strikes. + + $blueprint = $resource->getBlueprint(); + $blueprint->activateLease($resource, $lease); + $this->validateActivatedLease($blueprint, $resource, $lease); + } + + /** + * @task activate + */ + private function validateActivatedLease( + DrydockBlueprint $blueprint, + DrydockResource $resource, + DrydockLease $lease) { + + if (!$lease->isActivatedLease()) { + throw new Exception( + pht( + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. + 'returned from "%s" without activating a lease.', + $blueprint->getBlueprintName(), + $blueprint->getClassName(), + 'acquireLease()')); + } + + } + + +/* -( Releasing Leases )--------------------------------------------------- */ + + + /** + * @task release + */ private function releaseLease(DrydockLease $lease) { $lease->openTransaction(); $lease @@ -65,21 +639,34 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { DrydockSlotLock::releaseLocks($lease->getPHID()); $lease->saveTransaction(); - PhabricatorWorker::scheduleTask( - 'DrydockLeaseDestroyWorker', - array( - 'leasePHID' => $lease->getPHID(), - ), - array( - 'objectPHID' => $lease->getPHID(), - )); - $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST); $resource = $lease->getResource(); $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); + + $this->destroyLease($lease); + } + + +/* -( Destroying Leases )-------------------------------------------------- */ + + + /** + * @task destroy + */ + private function destroyLease(DrydockLease $lease) { + $resource = $lease->getResource(); + $blueprint = $resource->getBlueprint(); + + $blueprint->destroyLease($resource, $lease); + + $lease + ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) + ->save(); + + $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); } } diff --git a/src/applications/drydock/worker/DrydockLeaseWorker.php b/src/applications/drydock/worker/DrydockLeaseWorker.php deleted file mode 100644 index 5919c59613..0000000000 --- a/src/applications/drydock/worker/DrydockLeaseWorker.php +++ /dev/null @@ -1,74 +0,0 @@ -getTaskDataValue('leasePHID'); - $lease = $this->loadLease($lease_phid); - - $this->activateLease($lease); - } - - - private function activateLease(DrydockLease $lease) { - $actual_status = $lease->getStatus(); - - if ($actual_status != DrydockLeaseStatus::STATUS_ACQUIRED) { - throw new PhabricatorWorkerPermanentFailureException( - pht( - 'Trying to activate lease from wrong status ("%s").', - $actual_status)); - } - - $resource = $lease->getResource(); - if (!$resource) { - throw new PhabricatorWorkerPermanentFailureException( - pht('Trying to activate lease with no resource.')); - } - - $resource_status = $resource->getStatus(); - - if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { - // TODO: This is explicitly a temporary failure -- we are waiting for - // the resource to come up. - throw new Exception(pht('Resource still activating.')); - } - - if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { - throw new PhabricatorWorkerPermanentFailureException( - pht( - 'Trying to activate lease on a dead resource (in status "%s").', - $resource_status)); - } - - // NOTE: We can race resource destruction here. Between the time we - // performed the read above and now, the resource might have closed, so - // we may activate leases on dead resources. At least for now, this seems - // fine: a resource dying right before we activate a lease on it should not - // be distinguisahble from a resource dying right after we activate a lease - // on it. We end up with an active lease on a dead resource either way, and - // can not prevent resources dying from lightning strikes. - - $blueprint = $resource->getBlueprint(); - $blueprint->activateLease($resource, $lease); - $this->validateActivatedLease($blueprint, $resource, $lease); - } - - private function validateActivatedLease( - DrydockBlueprint $blueprint, - DrydockResource $resource, - DrydockLease $lease) { - - if (!$lease->isActivatedLease()) { - throw new Exception( - pht( - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. - 'returned from "%s" without activating a lease.', - $blueprint->getBlueprintName(), - $blueprint->getClassName(), - 'acquireLease()')); - } - - } - -} diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php index a7273b2f9d..fb2666f735 100644 --- a/src/applications/drydock/worker/DrydockWorker.php +++ b/src/applications/drydock/worker/DrydockWorker.php @@ -86,7 +86,7 @@ abstract class DrydockWorker extends PhabricatorWorker { } protected function yieldIfExpiringLease(DrydockLease $lease) { - if (!$lease->canUpdate()) { + if (!$lease->canReceiveCommands()) { return; } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php index 761f445e4e..85e7ae2411 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -154,7 +154,8 @@ final class HarbormasterBuildLog extends HarbormasterDAO public function finalize($start = 0) { if (!$this->getLive()) { - throw new Exception(pht('Start logging before finalizing it.')); + // TODO: Clean up this API. + return; } // TODO: Encode the log contents in a gzipped format. diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php index 3d8c6887c6..b0937c4e6d 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php @@ -60,7 +60,10 @@ abstract class PhabricatorWorkerTask extends PhabricatorWorkerDAO { $id = $this->getID(); $class = $this->getTaskClass(); - if (!class_exists($class)) { + try { + // NOTE: If the class does not exist, libphutil will throw an exception. + class_exists($class); + } catch (PhutilMissingSymbolException $ex) { throw new PhabricatorWorkerPermanentFailureException( pht( "Task class '%s' does not exist!",