Make Drydock reclaim unused resources when it reaches a resource limit

Summary:
Fixes T9994. Currently, when Drydock can't allocate a new resource because some limit has been reached, it waits patiently for a resource to become available.

It is possible that no resource will ever become available. Particularly with "Working Copy" resources, the new lease may want a copy of `rB`, but the resource may already be maxed out on `rA`.

Right now, no process exists to automatically reclaim the unused `rA`.

When we encounter this situation, try to reclaim one of the other resources if it is just sitting there unused.

Specifically:

  - Add a "reclaim" command which means "release this resource //if// it is completely unused".
  - Add a `bin/drydock reclaim` to send this command to every active resource.
  - When we try to acquire a resource and can't, but only because of some kind of limit / utilization problem, try to release an unused resource to free up some room.

Test Plan:
  - Set "Working Copy" resource limit to 1.
  - Ran "Test Configuration" in `rA`, which worked.
  - Ran "Test Configuration" in `rB`, which hung forever.
  - Applied patch.
  - Ran "Test Configuration" in `rB`, saw it reclaim the `rA` resource, use the slot, then succeed.
  - Ran "Test Configuration" in `rA` again, saw it grab the slot back.
  - Ran `bin/drydock reclaim` and saw it reclaim a bunch of old orphaned resources.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9994

Differential Revision: https://secure.phabricator.com/D14819

src/__phutil_library_map__.php

@@ -883,6 +883,7 @@ phutil_register_library_map(array(
     'DrydockLeasePHIDType' => 'applications/drydock/phid/DrydockLeasePHIDType.php',
     'DrydockLeaseQuery' => 'applications/drydock/query/DrydockLeaseQuery.php',
     'DrydockLeaseQueuedLogType' => 'applications/drydock/logtype/DrydockLeaseQueuedLogType.php',
+    'DrydockLeaseReclaimLogType' => 'applications/drydock/logtype/DrydockLeaseReclaimLogType.php',
     'DrydockLeaseReleaseController' => 'applications/drydock/controller/DrydockLeaseReleaseController.php',
     'DrydockLeaseReleasedLogType' => 'applications/drydock/logtype/DrydockLeaseReleasedLogType.php',
     'DrydockLeaseSearchEngine' => 'applications/drydock/query/DrydockLeaseSearchEngine.php',
@@ -900,6 +901,7 @@ phutil_register_library_map(array(
     'DrydockLogType' => 'applications/drydock/logtype/DrydockLogType.php',
     'DrydockManagementCommandWorkflow' => 'applications/drydock/management/DrydockManagementCommandWorkflow.php',
     'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php',
+    'DrydockManagementReclaimWorkflow' => 'applications/drydock/management/DrydockManagementReclaimWorkflow.php',
     'DrydockManagementReleaseLeaseWorkflow' => 'applications/drydock/management/DrydockManagementReleaseLeaseWorkflow.php',
     'DrydockManagementReleaseResourceWorkflow' => 'applications/drydock/management/DrydockManagementReleaseResourceWorkflow.php',
     'DrydockManagementUpdateLeaseWorkflow' => 'applications/drydock/management/DrydockManagementUpdateLeaseWorkflow.php',
@@ -926,6 +928,7 @@ phutil_register_library_map(array(
     'DrydockResourceListView' => 'applications/drydock/view/DrydockResourceListView.php',
     'DrydockResourcePHIDType' => 'applications/drydock/phid/DrydockResourcePHIDType.php',
     'DrydockResourceQuery' => 'applications/drydock/query/DrydockResourceQuery.php',
+    'DrydockResourceReclaimLogType' => 'applications/drydock/logtype/DrydockResourceReclaimLogType.php',
     'DrydockResourceReleaseController' => 'applications/drydock/controller/DrydockResourceReleaseController.php',
     'DrydockResourceSearchEngine' => 'applications/drydock/query/DrydockResourceSearchEngine.php',
     'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php',
@@ -4819,6 +4822,7 @@ phutil_register_library_map(array(
     'DrydockLeasePHIDType' => 'PhabricatorPHIDType',
     'DrydockLeaseQuery' => 'DrydockQuery',
     'DrydockLeaseQueuedLogType' => 'DrydockLogType',
+    'DrydockLeaseReclaimLogType' => 'DrydockLogType',
     'DrydockLeaseReleaseController' => 'DrydockLeaseController',
     'DrydockLeaseReleasedLogType' => 'DrydockLogType',
     'DrydockLeaseSearchEngine' => 'PhabricatorApplicationSearchEngine',
@@ -4839,6 +4843,7 @@ phutil_register_library_map(array(
     'DrydockLogType' => 'Phobject',
     'DrydockManagementCommandWorkflow' => 'DrydockManagementWorkflow',
     'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow',
+    'DrydockManagementReclaimWorkflow' => 'DrydockManagementWorkflow',
     'DrydockManagementReleaseLeaseWorkflow' => 'DrydockManagementWorkflow',
     'DrydockManagementReleaseResourceWorkflow' => 'DrydockManagementWorkflow',
     'DrydockManagementUpdateLeaseWorkflow' => 'DrydockManagementWorkflow',
@@ -4871,6 +4876,7 @@ phutil_register_library_map(array(
     'DrydockResourceListView' => 'AphrontView',
     'DrydockResourcePHIDType' => 'PhabricatorPHIDType',
     'DrydockResourceQuery' => 'DrydockQuery',
+    'DrydockResourceReclaimLogType' => 'DrydockLogType',
     'DrydockResourceReleaseController' => 'DrydockResourceController',
     'DrydockResourceSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'DrydockResourceStatus' => 'DrydockConstants',

src/applications/drydock/logtype/DrydockLeaseReclaimLogType.php

@@ -0,0 +1,25 @@
+<?php
+
+final class DrydockLeaseReclaimLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.lease.reclaim';
+
+  public function getLogTypeName() {
+    return pht('Reclaimed Resources');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-refresh yellow';
+  }
+
+  public function renderLog(array $data) {
+    $viewer = $this->getViewer();
+
+    $resource_phids = idx($data, 'resourcePHIDs', array());
+
+    return pht(
+      'Reclaimed resource %s.',
+      $viewer->renderHandleList($resource_phids)->render());
+  }
+
+}

src/applications/drydock/logtype/DrydockResourceReclaimLogType.php

@@ -0,0 +1,24 @@
+<?php
+
+final class DrydockResourceReclaimLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.resource.reclaim';
+
+  public function getLogTypeName() {
+    return pht('Reclaimed');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-refresh red';
+  }
+
+  public function renderLog(array $data) {
+    $viewer = $this->getViewer();
+    $reclaimer_phid = idx($data, 'reclaimerPHID');
+
+    return pht(
+      'Resource reclaimed by %s.',
+      $viewer->renderHandle($reclaimer_phid)->render());
+  }
+
+}

src/applications/drydock/management/DrydockManagementReclaimWorkflow.php

@@ -0,0 +1,63 @@
+<?php
+
+final class DrydockManagementReclaimWorkflow
+  extends DrydockManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('reclaim')
+      ->setSynopsis(pht('Reclaim unused resources.'))
+      ->setArguments(array());
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+    $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
+
+    PhabricatorWorker::setRunAllTasksInProcess(true);
+
+    $resources = id(new DrydockResourceQuery())
+      ->setViewer($viewer)
+      ->withStatuses(
+        array(
+          DrydockResourceStatus::STATUS_ACTIVE,
+        ))
+      ->execute();
+    foreach ($resources as $resource) {
+      $command = DrydockCommand::initializeNewCommand($viewer)
+        ->setTargetPHID($resource->getPHID())
+        ->setAuthorPHID($drydock_phid)
+        ->setCommand(DrydockCommand::COMMAND_RECLAIM)
+        ->save();
+
+      $resource->scheduleUpdate();
+
+      $resource = $resource->reload();
+
+      $name = pht(
+        'Resource %d: %s',
+        $resource->getID(),
+        $resource->getResourceName());
+
+      switch ($resource->getStatus()) {
+        case DrydockResourceStatus::STATUS_RELEASED:
+        case DrydockResourceStatus::STATUS_DESTROYED:
+          echo tsprintf(
+            "%s\n",
+            pht(
+              'Resource "%s" was reclaimed.',
+              $name));
+          break;
+        default:
+          echo tsprintf(
+            "%s\n",
+            pht(
+              'Resource "%s" could not be reclaimed.',
+              $name));
+          break;
+      }
+    }
+
+  }
+
+}

src/applications/drydock/storage/DrydockCommand.php

@@ -5,6 +5,7 @@ final class DrydockCommand
   implements PhabricatorPolicyInterface {
 
   const COMMAND_RELEASE = 'release';
+  const COMMAND_RECLAIM = 'reclaim';
 
   protected $authorPHID;
   protected $targetPHID;

src/applications/drydock/worker/DrydockLeaseUpdateWorker.php

@@ -179,6 +179,23 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
       // satisfy the lease, just not right now. This is a temporary failure,
       // and we expect allocation to succeed eventually.
       if (!$usable_blueprints) {
+        $blueprints = $this->rankBlueprints($blueprints, $lease);
+
+        // Try to actively reclaim unused resources. If we succeed, jump back
+        // into the queue in an effort to claim it.
+        foreach ($blueprints as $blueprint) {
+          $reclaimed = $this->reclaimResources($blueprint, $lease);
+          if ($reclaimed) {
+            $lease->logEvent(
+              DrydockLeaseReclaimLogType::LOGCONST,
+              array(
+                'resourcePHIDs' => array($reclaimed->getPHID()),
+              ));
+
+            throw new PhabricatorWorkerYieldException(15);
+          }
+        }
+
         $lease->logEvent(
           DrydockLeaseWaitingForResourcesLogType::LOGCONST,
           array(
@@ -439,6 +456,7 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
     assert_instances_of($blueprints, 'DrydockBlueprint');
 
     $keep = array();
+
     foreach ($blueprints as $key => $blueprint) {
       if (!$blueprint->canAllocateResourceForLease($lease)) {
         continue;
@@ -573,6 +591,35 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
     }
   }
 
+  private function reclaimResources(
+    DrydockBlueprint $blueprint,
+    DrydockLease $lease) {
+    $viewer = $this->getViewer();
+
+    $resources = id(new DrydockResourceQuery())
+      ->setViewer($viewer)
+      ->withBlueprintPHIDs(array($blueprint->getPHID()))
+      ->withStatuses(
+        array(
+          DrydockResourceStatus::STATUS_ACTIVE,
+        ))
+      ->execute();
+
+    // TODO: We could be much smarter about this and try to release long-unused
+    // resources, resources with many similar copies, old resources, resources
+    // that are cheap to rebuild, etc.
+    shuffle($resources);
+
+    foreach ($resources as $resource) {
+      if ($this->canReclaimResource($resource)) {
+        $this->reclaimResource($resource, $lease);
+        return $resource;
+      }
+    }
+
+    return null;
+  }
+
 
 /* -(  Acquiring Leases  )--------------------------------------------------- */
 

src/applications/drydock/worker/DrydockResourceUpdateWorker.php

@@ -143,7 +143,11 @@ final class DrydockResourceUpdateWorker extends DrydockWorker {
 
     switch ($command->getCommand()) {
       case DrydockCommand::COMMAND_RELEASE:
-        $this->releaseResource($resource);
+        $this->releaseResource($resource, null);
+        break;
+      case DrydockCommand::COMMAND_RECLAIM:
+        $reclaimer_phid = $command->getAuthorPHID();
+        $this->releaseResource($resource, $reclaimer_phid);
         break;
     }
   }
@@ -188,7 +192,22 @@ final class DrydockResourceUpdateWorker extends DrydockWorker {
   /**
    * @task release
    */
-  private function releaseResource(DrydockResource $resource) {
+  private function releaseResource(
+    DrydockResource $resource,
+    $reclaimer_phid) {
+
+    if ($reclaimer_phid) {
+      if (!$this->canReclaimResource($resource)) {
+        return;
+      }
+
+      $resource->logEvent(
+        DrydockResourceReclaimLogType::LOGCONST,
+        array(
+          'reclaimerPHID' => $reclaimer_phid,
+        ));
+    }
+
     $viewer = $this->getViewer();
     $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
 

src/applications/drydock/worker/DrydockWorker.php

@@ -195,5 +195,61 @@ abstract class DrydockWorker extends PhabricatorWorker {
     return $this;
   }
 
+  protected function canReclaimResource(DrydockResource $resource) {
+    $viewer = $this->getViewer();
+
+    // Don't reclaim a resource if it has been updated recently. If two
+    // leases are fighting, we don't want them to keep reclaming resources
+    // from one another forever without making progress, so make resources
+    // immune to reclamation for a little while after they activate or update.
+
+    // TODO: It would be nice to use a more narrow time here, like "last
+    // activation or lease release", but we don't currently store that
+    // anywhere.
+
+    $updated = $resource->getDateModified();
+    $now = PhabricatorTime::getNow();
+    $ago = ($now - $updated);
+    if ($ago < phutil_units('3 minutes in seconds')) {
+      return false;
+    }
+
+    $statuses = array(
+      DrydockLeaseStatus::STATUS_PENDING,
+      DrydockLeaseStatus::STATUS_ACQUIRED,
+      DrydockLeaseStatus::STATUS_ACTIVE,
+      DrydockLeaseStatus::STATUS_RELEASED,
+      DrydockLeaseStatus::STATUS_BROKEN,
+    );
+
+    // Don't reclaim resources that have any active leases.
+    $leases = id(new DrydockLeaseQuery())
+      ->setViewer($viewer)
+      ->withResourcePHIDs(array($resource->getPHID()))
+      ->withStatuses($statuses)
+      ->setLimit(1)
+      ->execute();
+    if ($leases) {
+      return false;
+    }
+
+    return true;
+  }
+
+  protected function reclaimResource(
+    DrydockResource $resource,
+    DrydockLease $lease) {
+    $viewer = $this->getViewer();
+
+    $command = DrydockCommand::initializeNewCommand($viewer)
+      ->setTargetPHID($resource->getPHID())
+      ->setAuthorPHID($lease->getPHID())
+      ->setCommand(DrydockCommand::COMMAND_RECLAIM)
+      ->save();
+
+    $resource->scheduleUpdate();
+
+    return $this;
+  }
 
 }