diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index a006b7fcdf..dadc3b74ac 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -883,6 +883,7 @@ phutil_register_library_map(array( 'DrydockLeasePHIDType' => 'applications/drydock/phid/DrydockLeasePHIDType.php', 'DrydockLeaseQuery' => 'applications/drydock/query/DrydockLeaseQuery.php', 'DrydockLeaseQueuedLogType' => 'applications/drydock/logtype/DrydockLeaseQueuedLogType.php', + 'DrydockLeaseReclaimLogType' => 'applications/drydock/logtype/DrydockLeaseReclaimLogType.php', 'DrydockLeaseReleaseController' => 'applications/drydock/controller/DrydockLeaseReleaseController.php', 'DrydockLeaseReleasedLogType' => 'applications/drydock/logtype/DrydockLeaseReleasedLogType.php', 'DrydockLeaseSearchEngine' => 'applications/drydock/query/DrydockLeaseSearchEngine.php', @@ -900,6 +901,7 @@ phutil_register_library_map(array( 'DrydockLogType' => 'applications/drydock/logtype/DrydockLogType.php', 'DrydockManagementCommandWorkflow' => 'applications/drydock/management/DrydockManagementCommandWorkflow.php', 'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php', + 'DrydockManagementReclaimWorkflow' => 'applications/drydock/management/DrydockManagementReclaimWorkflow.php', 'DrydockManagementReleaseLeaseWorkflow' => 'applications/drydock/management/DrydockManagementReleaseLeaseWorkflow.php', 'DrydockManagementReleaseResourceWorkflow' => 'applications/drydock/management/DrydockManagementReleaseResourceWorkflow.php', 'DrydockManagementUpdateLeaseWorkflow' => 'applications/drydock/management/DrydockManagementUpdateLeaseWorkflow.php', @@ -926,6 +928,7 @@ phutil_register_library_map(array( 'DrydockResourceListView' => 'applications/drydock/view/DrydockResourceListView.php', 'DrydockResourcePHIDType' => 'applications/drydock/phid/DrydockResourcePHIDType.php', 'DrydockResourceQuery' => 'applications/drydock/query/DrydockResourceQuery.php', + 'DrydockResourceReclaimLogType' => 'applications/drydock/logtype/DrydockResourceReclaimLogType.php', 'DrydockResourceReleaseController' => 'applications/drydock/controller/DrydockResourceReleaseController.php', 'DrydockResourceSearchEngine' => 'applications/drydock/query/DrydockResourceSearchEngine.php', 'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php', @@ -4819,6 +4822,7 @@ phutil_register_library_map(array( 'DrydockLeasePHIDType' => 'PhabricatorPHIDType', 'DrydockLeaseQuery' => 'DrydockQuery', 'DrydockLeaseQueuedLogType' => 'DrydockLogType', + 'DrydockLeaseReclaimLogType' => 'DrydockLogType', 'DrydockLeaseReleaseController' => 'DrydockLeaseController', 'DrydockLeaseReleasedLogType' => 'DrydockLogType', 'DrydockLeaseSearchEngine' => 'PhabricatorApplicationSearchEngine', @@ -4839,6 +4843,7 @@ phutil_register_library_map(array( 'DrydockLogType' => 'Phobject', 'DrydockManagementCommandWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow', + 'DrydockManagementReclaimWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementReleaseLeaseWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementReleaseResourceWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementUpdateLeaseWorkflow' => 'DrydockManagementWorkflow', @@ -4871,6 +4876,7 @@ phutil_register_library_map(array( 'DrydockResourceListView' => 'AphrontView', 'DrydockResourcePHIDType' => 'PhabricatorPHIDType', 'DrydockResourceQuery' => 'DrydockQuery', + 'DrydockResourceReclaimLogType' => 'DrydockLogType', 'DrydockResourceReleaseController' => 'DrydockResourceController', 'DrydockResourceSearchEngine' => 'PhabricatorApplicationSearchEngine', 'DrydockResourceStatus' => 'DrydockConstants', diff --git a/src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php b/src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php new file mode 100644 index 0000000000..8dbc13d9d4 --- /dev/null +++ b/src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php @@ -0,0 +1,25 @@ +getViewer(); + + $resource_phids = idx($data, 'resourcePHIDs', array()); + + return pht( + 'Reclaimed resource %s.', + $viewer->renderHandleList($resource_phids)->render()); + } + +} diff --git a/src/applications/drydock/logtype/DrydockResourceReclaimLogType.php b/src/applications/drydock/logtype/DrydockResourceReclaimLogType.php new file mode 100644 index 0000000000..9e9d5fdef3 --- /dev/null +++ b/src/applications/drydock/logtype/DrydockResourceReclaimLogType.php @@ -0,0 +1,24 @@ +getViewer(); + $reclaimer_phid = idx($data, 'reclaimerPHID'); + + return pht( + 'Resource reclaimed by %s.', + $viewer->renderHandle($reclaimer_phid)->render()); + } + +} diff --git a/src/applications/drydock/management/DrydockManagementReclaimWorkflow.php b/src/applications/drydock/management/DrydockManagementReclaimWorkflow.php new file mode 100644 index 0000000000..a312f109f2 --- /dev/null +++ b/src/applications/drydock/management/DrydockManagementReclaimWorkflow.php @@ -0,0 +1,63 @@ +setName('reclaim') + ->setSynopsis(pht('Reclaim unused resources.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); + + PhabricatorWorker::setRunAllTasksInProcess(true); + + $resources = id(new DrydockResourceQuery()) + ->setViewer($viewer) + ->withStatuses( + array( + DrydockResourceStatus::STATUS_ACTIVE, + )) + ->execute(); + foreach ($resources as $resource) { + $command = DrydockCommand::initializeNewCommand($viewer) + ->setTargetPHID($resource->getPHID()) + ->setAuthorPHID($drydock_phid) + ->setCommand(DrydockCommand::COMMAND_RECLAIM) + ->save(); + + $resource->scheduleUpdate(); + + $resource = $resource->reload(); + + $name = pht( + 'Resource %d: %s', + $resource->getID(), + $resource->getResourceName()); + + switch ($resource->getStatus()) { + case DrydockResourceStatus::STATUS_RELEASED: + case DrydockResourceStatus::STATUS_DESTROYED: + echo tsprintf( + "%s\n", + pht( + 'Resource "%s" was reclaimed.', + $name)); + break; + default: + echo tsprintf( + "%s\n", + pht( + 'Resource "%s" could not be reclaimed.', + $name)); + break; + } + } + + } + +} diff --git a/src/applications/drydock/storage/DrydockCommand.php b/src/applications/drydock/storage/DrydockCommand.php index 60cb363ecb..e7d003bdd6 100644 --- a/src/applications/drydock/storage/DrydockCommand.php +++ b/src/applications/drydock/storage/DrydockCommand.php @@ -5,6 +5,7 @@ final class DrydockCommand implements PhabricatorPolicyInterface { const COMMAND_RELEASE = 'release'; + const COMMAND_RECLAIM = 'reclaim'; protected $authorPHID; protected $targetPHID; diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 80b2a6f30c..a41e57fbf8 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -179,6 +179,23 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { // satisfy the lease, just not right now. This is a temporary failure, // and we expect allocation to succeed eventually. if (!$usable_blueprints) { + $blueprints = $this->rankBlueprints($blueprints, $lease); + + // Try to actively reclaim unused resources. If we succeed, jump back + // into the queue in an effort to claim it. + foreach ($blueprints as $blueprint) { + $reclaimed = $this->reclaimResources($blueprint, $lease); + if ($reclaimed) { + $lease->logEvent( + DrydockLeaseReclaimLogType::LOGCONST, + array( + 'resourcePHIDs' => array($reclaimed->getPHID()), + )); + + throw new PhabricatorWorkerYieldException(15); + } + } + $lease->logEvent( DrydockLeaseWaitingForResourcesLogType::LOGCONST, array( @@ -439,6 +456,7 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { assert_instances_of($blueprints, 'DrydockBlueprint'); $keep = array(); + foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canAllocateResourceForLease($lease)) { continue; @@ -573,6 +591,35 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { } } + private function reclaimResources( + DrydockBlueprint $blueprint, + DrydockLease $lease) { + $viewer = $this->getViewer(); + + $resources = id(new DrydockResourceQuery()) + ->setViewer($viewer) + ->withBlueprintPHIDs(array($blueprint->getPHID())) + ->withStatuses( + array( + DrydockResourceStatus::STATUS_ACTIVE, + )) + ->execute(); + + // TODO: We could be much smarter about this and try to release long-unused + // resources, resources with many similar copies, old resources, resources + // that are cheap to rebuild, etc. + shuffle($resources); + + foreach ($resources as $resource) { + if ($this->canReclaimResource($resource)) { + $this->reclaimResource($resource, $lease); + return $resource; + } + } + + return null; + } + /* -( Acquiring Leases )--------------------------------------------------- */ diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php index 5d27e79d5c..baa80f90d8 100644 --- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php @@ -143,7 +143,11 @@ final class DrydockResourceUpdateWorker extends DrydockWorker { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: - $this->releaseResource($resource); + $this->releaseResource($resource, null); + break; + case DrydockCommand::COMMAND_RECLAIM: + $reclaimer_phid = $command->getAuthorPHID(); + $this->releaseResource($resource, $reclaimer_phid); break; } } @@ -188,7 +192,22 @@ final class DrydockResourceUpdateWorker extends DrydockWorker { /** * @task release */ - private function releaseResource(DrydockResource $resource) { + private function releaseResource( + DrydockResource $resource, + $reclaimer_phid) { + + if ($reclaimer_phid) { + if (!$this->canReclaimResource($resource)) { + return; + } + + $resource->logEvent( + DrydockResourceReclaimLogType::LOGCONST, + array( + 'reclaimerPHID' => $reclaimer_phid, + )); + } + $viewer = $this->getViewer(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php index 08c32be869..f167fef5d0 100644 --- a/src/applications/drydock/worker/DrydockWorker.php +++ b/src/applications/drydock/worker/DrydockWorker.php @@ -195,5 +195,61 @@ abstract class DrydockWorker extends PhabricatorWorker { return $this; } + protected function canReclaimResource(DrydockResource $resource) { + $viewer = $this->getViewer(); + + // Don't reclaim a resource if it has been updated recently. If two + // leases are fighting, we don't want them to keep reclaming resources + // from one another forever without making progress, so make resources + // immune to reclamation for a little while after they activate or update. + + // TODO: It would be nice to use a more narrow time here, like "last + // activation or lease release", but we don't currently store that + // anywhere. + + $updated = $resource->getDateModified(); + $now = PhabricatorTime::getNow(); + $ago = ($now - $updated); + if ($ago < phutil_units('3 minutes in seconds')) { + return false; + } + + $statuses = array( + DrydockLeaseStatus::STATUS_PENDING, + DrydockLeaseStatus::STATUS_ACQUIRED, + DrydockLeaseStatus::STATUS_ACTIVE, + DrydockLeaseStatus::STATUS_RELEASED, + DrydockLeaseStatus::STATUS_BROKEN, + ); + + // Don't reclaim resources that have any active leases. + $leases = id(new DrydockLeaseQuery()) + ->setViewer($viewer) + ->withResourcePHIDs(array($resource->getPHID())) + ->withStatuses($statuses) + ->setLimit(1) + ->execute(); + if ($leases) { + return false; + } + + return true; + } + + protected function reclaimResource( + DrydockResource $resource, + DrydockLease $lease) { + $viewer = $this->getViewer(); + + $command = DrydockCommand::initializeNewCommand($viewer) + ->setTargetPHID($resource->getPHID()) + ->setAuthorPHID($lease->getPHID()) + ->setCommand(DrydockCommand::COMMAND_RECLAIM) + ->save(); + + $resource->scheduleUpdate(); + + return $this; + } }