Merge the DrydockLease workers into a single worker

Summary:
Ref T9252. This is the same as D14201, but for lease stuff instead of resource stuff.

This one is a little heavier but still feels pretty reasonable to me at the end of the day (worker is <1K lines and has a ton of comment stuff).

Also fixes a few random bugs I hit in the task queue.

Test Plan:
  - Restarted some Harbormaster builds, saw them go through cleanly.
  - Released pre-activation resources/leases.
  - Probably still kinda buggy but I'll iron the details out over time.

Logs are starting to look somewhat plausible:

{F855747}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9252

Differential Revision: https://secure.phabricator.com/D14202

src/__phutil_library_map__.php

@@ -796,7 +796,6 @@ phutil_register_library_map(array(
     'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php',
     'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php',
     'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php',
-    'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
     'DrydockAlmanacServiceHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php',
     'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
     'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php',
@@ -834,7 +833,6 @@ phutil_register_library_map(array(
     'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php',
     'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php',
     'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php',
-    'DrydockLeaseDestroyWorker' => 'applications/drydock/worker/DrydockLeaseDestroyWorker.php',
     'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php',
     'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
     'DrydockLeaseListView' => 'applications/drydock/view/DrydockLeaseListView.php',
@@ -847,7 +845,6 @@ phutil_register_library_map(array(
     'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php',
     'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php',
     'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php',
-    'DrydockLeaseWorker' => 'applications/drydock/worker/DrydockLeaseWorker.php',
     'DrydockLog' => 'applications/drydock/storage/DrydockLog.php',
     'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php',
     'DrydockLogGarbageCollector' => 'applications/drydock/garbagecollector/DrydockLogGarbageCollector.php',
@@ -4525,7 +4522,6 @@ phutil_register_library_map(array(
     'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'DoorkeeperTagView' => 'AphrontView',
     'DoorkeeperTagsController' => 'PhabricatorController',
-    'DrydockAllocatorWorker' => 'DrydockWorker',
     'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation',
     'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
     'DrydockBlueprint' => array(
@@ -4577,7 +4573,6 @@ phutil_register_library_map(array(
     'DrydockLeaseActivatedLogType' => 'DrydockLogType',
     'DrydockLeaseController' => 'DrydockController',
     'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource',
-    'DrydockLeaseDestroyWorker' => 'DrydockWorker',
     'DrydockLeaseDestroyedLogType' => 'DrydockLogType',
     'DrydockLeaseListController' => 'DrydockLeaseController',
     'DrydockLeaseListView' => 'AphrontView',
@@ -4590,7 +4585,6 @@ phutil_register_library_map(array(
     'DrydockLeaseStatus' => 'DrydockConstants',
     'DrydockLeaseUpdateWorker' => 'DrydockWorker',
     'DrydockLeaseViewController' => 'DrydockLeaseController',
-    'DrydockLeaseWorker' => 'DrydockWorker',
     'DrydockLog' => array(
       'DrydockDAO',
       'PhabricatorPolicyInterface',

src/applications/drydock/storage/DrydockLease.php

@@ -135,14 +135,7 @@ final class DrydockLease extends DrydockDAO
       ->setStatus(DrydockLeaseStatus::STATUS_PENDING)
       ->save();
 
-    $task = PhabricatorWorker::scheduleTask(
+    $this->scheduleUpdate();
-      'DrydockAllocatorWorker',
-      array(
-        'leasePHID' => $this->getPHID(),
-      ),
-      array(
-        'objectPHID' => $this->getPHID(),
-      ));
 
     $this->logEvent(DrydockLeaseQueuedLogType::LOGCONST);
 
@@ -321,12 +314,13 @@ final class DrydockLease extends DrydockDAO
     }
   }
 
-  public function canUpdate() {
+  public function canReceiveCommands() {
     switch ($this->getStatus()) {
-      case DrydockLeaseStatus::STATUS_ACTIVE:
+      case DrydockLeaseStatus::STATUS_RELEASED:
-        return true;
+      case DrydockLeaseStatus::STATUS_DESTROYED:
-      default:
         return false;
+      default:
+        return true;
     }
   }
 

src/applications/drydock/worker/DrydockAllocatorWorker.php

@@ -1,479 +0,0 @@
-<?php
-
-/**
- * @task allocate Allocator
- * @task resource Managing Resources
- * @task lease Managing Leases
- */
-final class DrydockAllocatorWorker extends DrydockWorker {
-
-  protected function doWork() {
-    $lease_phid = $this->getTaskDataValue('leasePHID');
-    $lease = $this->loadLease($lease_phid);
-
-    $this->allocateAndAcquireLease($lease);
-  }
-
-
-/* -(  Allocator  )---------------------------------------------------------- */
-
-
-  /**
-   * Find or build a resource which can satisfy a given lease request, then
-   * acquire the lease.
-   *
-   * @param DrydockLease Requested lease.
-   * @return void
-   * @task allocator
-   */
-  private function allocateAndAcquireLease(DrydockLease $lease) {
-    $blueprints = $this->loadBlueprintsForAllocatingLease($lease);
-
-    // If we get nothing back, that means no blueprint is defined which can
-    // ever build the requested resource. This is a permanent failure, since
-    // we don't expect to succeed no matter how many times we try.
-    if (!$blueprints) {
-      $lease
-        ->setStatus(DrydockLeaseStatus::STATUS_BROKEN)
-        ->save();
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht(
-          'No active Drydock blueprint exists which can ever allocate a '.
-          'resource for lease "%s".',
-          $lease->getPHID()));
-    }
-
-    // First, try to find a suitable open resource which we can acquire a new
-    // lease on.
-    $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease);
-
-    // If no resources exist yet, see if we can build one.
-    if (!$resources) {
-      $usable_blueprints = $this->removeOverallocatedBlueprints(
-        $blueprints,
-        $lease);
-
-      // If we get nothing back here, some blueprint claims it can eventually
-      // satisfy the lease, just not right now. This is a temporary failure,
-      // and we expect allocation to succeed eventually.
-      if (!$blueprints) {
-        // TODO: More formal temporary failure here. We should retry this
-        // "soon" but not "immediately".
-        throw new Exception(
-          pht('No blueprints have space to allocate a resource right now.'));
-      }
-
-      $usable_blueprints = $this->rankBlueprints($blueprints, $lease);
-
-      $exceptions = array();
-      foreach ($usable_blueprints as $blueprint) {
-        try {
-          $resources[] = $this->allocateResource($blueprint, $lease);
-
-          // Bail after allocating one resource, we don't need any more than
-          // this.
-          break;
-        } catch (Exception $ex) {
-          $exceptions[] = $ex;
-        }
-      }
-
-      if (!$resources) {
-        // TODO: We should distinguish between temporary and permament failures
-        // here. If any blueprint failed temporarily, retry "soon". If none
-        // of these failures were temporary, maybe this should be a permanent
-        // failure?
-        throw new PhutilAggregateException(
-          pht(
-            'All blueprints failed to allocate a suitable new resource when '.
-            'trying to allocate lease "%s".',
-            $lease->getPHID()),
-          $exceptions);
-      }
-
-      // NOTE: We have not acquired the lease yet, so it is possible that the
-      // resource we just built will be snatched up by some other lease before
-      // we can. This is not problematic: we'll retry a little later and should
-      // suceed eventually.
-    }
-
-    $resources = $this->rankResources($resources, $lease);
-
-    $exceptions = array();
-    $allocated = false;
-    foreach ($resources as $resource) {
-      try {
-        $this->acquireLease($resource, $lease);
-        $allocated = true;
-        break;
-      } catch (Exception $ex) {
-        $exceptions[] = $ex;
-      }
-    }
-
-    if (!$allocated) {
-      // TODO: We should distinguish between temporary and permanent failures
-      // here. If any failures were temporary (specifically, failed to acquire
-      // locks)
-
-      throw new PhutilAggregateException(
-        pht(
-          'Unable to acquire lease "%s" on any resouce.',
-          $lease->getPHID()),
-        $exceptions);
-    }
-  }
-
-
-  /**
-   * Get all the @{class:DrydockBlueprintImplementation}s which can possibly
-   * build a resource to satisfy a lease.
-   *
-   * This method returns blueprints which might, at some time, be able to
-   * build a resource which can satisfy the lease. They may not be able to
-   * build that resource right now.
-   *
-   * @param DrydockLease Requested lease.
-   * @return list<DrydockBlueprintImplementation> List of qualifying blueprint
-   *   implementations.
-   * @task allocator
-   */
-  private function loadBlueprintImplementationsForAllocatingLease(
-    DrydockLease $lease) {
-
-    $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations();
-
-    $keep = array();
-    foreach ($impls as $key => $impl) {
-      // Don't use disabled blueprint types.
-      if (!$impl->isEnabled()) {
-        continue;
-      }
-
-      // Don't use blueprint types which can't allocate the correct kind of
-      // resource.
-      if ($impl->getType() != $lease->getResourceType()) {
-        continue;
-      }
-
-      if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) {
-        continue;
-      }
-
-      $keep[$key] = $impl;
-    }
-
-    return $keep;
-  }
-
-
-  /**
-   * Get all the concrete @{class:DrydockBlueprint}s which can possibly
-   * build a resource to satisfy a lease.
-   *
-   * @param DrydockLease Requested lease.
-   * @return list<DrydockBlueprint> List of qualifying blueprints.
-   * @task allocator
-   */
-  private function loadBlueprintsForAllocatingLease(
-    DrydockLease $lease) {
-    $viewer = $this->getViewer();
-
-    $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease);
-    if (!$impls) {
-      return array();
-    }
-
-    $blueprints = id(new DrydockBlueprintQuery())
-      ->setViewer($viewer)
-      ->withBlueprintClasses(array_keys($impls))
-      ->withDisabled(false)
-      ->execute();
-
-    $keep = array();
-    foreach ($blueprints as $key => $blueprint) {
-      if (!$blueprint->canEverAllocateResourceForLease($lease)) {
-        continue;
-      }
-
-      $keep[$key] = $blueprint;
-    }
-
-    return $keep;
-  }
-
-
-  /**
-   * Load a list of all resources which a given lease can possibly be
-   * allocated against.
-   *
-   * @param list<DrydockBlueprint> Blueprints which may produce suitable
-   *   resources.
-   * @param DrydockLease Requested lease.
-   * @return list<DrydockResource> Resources which may be able to allocate
-   *   the lease.
-   * @task allocator
-   */
-  private function loadResourcesForAllocatingLease(
-    array $blueprints,
-    DrydockLease $lease) {
-    assert_instances_of($blueprints, 'DrydockBlueprint');
-    $viewer = $this->getViewer();
-
-    $resources = id(new DrydockResourceQuery())
-      ->setViewer($viewer)
-      ->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
-      ->withTypes(array($lease->getResourceType()))
-      ->withStatuses(
-        array(
-          DrydockResourceStatus::STATUS_PENDING,
-          DrydockResourceStatus::STATUS_ACTIVE,
-        ))
-      ->execute();
-
-    $keep = array();
-    foreach ($resources as $key => $resource) {
-      $blueprint = $resource->getBlueprint();
-
-      if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
-        continue;
-      }
-
-      $keep[$key] = $resource;
-    }
-
-    return $keep;
-  }
-
-
-  /**
-   * Remove blueprints which are too heavily allocated to build a resource for
-   * a lease from a list of blueprints.
-   *
-   * @param list<DrydockBlueprint> List of blueprints.
-   * @return list<DrydockBlueprint> List with blueprints that can not allocate
-   *   a resource for the lease right now removed.
-   * @task allocator
-   */
-  private function removeOverallocatedBlueprints(
-    array $blueprints,
-    DrydockLease $lease) {
-    assert_instances_of($blueprints, 'DrydockBlueprint');
-
-    $keep = array();
-    foreach ($blueprints as $key => $blueprint) {
-      if (!$blueprint->canAllocateResourceForLease($lease)) {
-        continue;
-      }
-
-      $keep[$key] = $blueprint;
-    }
-
-    return $keep;
-  }
-
-
-  /**
-   * Rank blueprints by suitability for building a new resource for a
-   * particular lease.
-   *
-   * @param list<DrydockBlueprint> List of blueprints.
-   * @param DrydockLease Requested lease.
-   * @return list<DrydockBlueprint> Ranked list of blueprints.
-   * @task allocator
-   */
-  private function rankBlueprints(array $blueprints, DrydockLease $lease) {
-    assert_instances_of($blueprints, 'DrydockBlueprint');
-
-    // TODO: Implement improvements to this ranking algorithm if they become
-    // available.
-    shuffle($blueprints);
-
-    return $blueprints;
-  }
-
-
-  /**
-   * Rank resources by suitability for allocating a particular lease.
-   *
-   * @param list<DrydockResource> List of resources.
-   * @param DrydockLease Requested lease.
-   * @return list<DrydockResource> Ranked list of resources.
-   * @task allocator
-   */
-  private function rankResources(array $resources, DrydockLease $lease) {
-    assert_instances_of($resources, 'DrydockResource');
-
-    // TODO: Implement improvements to this ranking algorithm if they become
-    // available.
-    shuffle($resources);
-
-    return $resources;
-  }
-
-
-/* -(  Managing Resources  )------------------------------------------------- */
-
-
-  /**
-   * Perform an actual resource allocation with a particular blueprint.
-   *
-   * @param DrydockBlueprint The blueprint to allocate a resource from.
-   * @param DrydockLease Requested lease.
-   * @return DrydockResource Allocated resource.
-   * @task resource
-   */
-  private function allocateResource(
-    DrydockBlueprint $blueprint,
-    DrydockLease $lease) {
-    $resource = $blueprint->allocateResource($lease);
-    $this->validateAllocatedResource($blueprint, $resource, $lease);
-
-    // If this resource was allocated as a pending resource, queue a task to
-    // activate it.
-    if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {
-      PhabricatorWorker::scheduleTask(
-        'DrydockResourceUpdateWorker',
-        array(
-          'resourcePHID' => $resource->getPHID(),
-        ),
-        array(
-          'objectPHID' => $resource->getPHID(),
-        ));
-    }
-
-    return $resource;
-  }
-
-
-  /**
-   * Check that the resource a blueprint allocated is roughly the sort of
-   * object we expect.
-   *
-   * @param DrydockBlueprint Blueprint which built the resource.
-   * @param wild Thing which the blueprint claims is a valid resource.
-   * @param DrydockLease Lease the resource was allocated for.
-   * @return void
-   * @task resource
-   */
-  private function validateAllocatedResource(
-    DrydockBlueprint $blueprint,
-    $resource,
-    DrydockLease $lease) {
-
-    if (!($resource instanceof DrydockResource)) {
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
-          'return an object of type %s or throw, but returned something else.',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          'allocateResource()',
-          'DrydockResource'));
-    }
-
-    if (!$resource->isAllocatedResource()) {
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
-          'must actually allocate the resource it returns.',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          'allocateResource()'));
-    }
-
-    $resource_type = $resource->getType();
-    $lease_type = $lease->getResourceType();
-
-    if ($resource_type !== $lease_type) {
-      // TODO: Destroy the resource here?
-
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
-          'built a resource of type "%s" to satisfy a lease requesting a '.
-          'resource of type "%s".',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          $resource_type,
-          $lease_type));
-    }
-  }
-
-
-/* -(  Managing Leases  )---------------------------------------------------- */
-
-
-  /**
-   * Perform an actual lease acquisition on a particular resource.
-   *
-   * @param DrydockResource Resource to acquire a lease on.
-   * @param DrydockLease Lease to acquire.
-   * @return void
-   * @task lease
-   */
-  private function acquireLease(
-    DrydockResource $resource,
-    DrydockLease $lease) {
-
-    $blueprint = $resource->getBlueprint();
-    $blueprint->acquireLease($resource, $lease);
-
-    $this->validateAcquiredLease($blueprint, $resource, $lease);
-
-    // If this lease has been acquired but not activated, queue a task to
-    // activate it.
-    if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
-      PhabricatorWorker::scheduleTask(
-        'DrydockLeaseWorker',
-        array(
-          'leasePHID' => $lease->getPHID(),
-        ),
-        array(
-          'objectPHID' => $lease->getPHID(),
-        ));
-    }
-  }
-
-
-  /**
-   * Make sure that a lease was really acquired properly.
-   *
-   * @param DrydockBlueprint Blueprint which created the resource.
-   * @param DrydockResource Resource which was acquired.
-   * @param DrydockLease The lease which was supposedly acquired.
-   * @return void
-   * @task lease
-   */
-  private function validateAcquiredLease(
-    DrydockBlueprint $blueprint,
-    DrydockResource $resource,
-    DrydockLease $lease) {
-
-    if (!$lease->isAcquiredLease()) {
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
-          'returned from "%s" without acquiring a lease.',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          'acquireLease()'));
-    }
-
-    $lease_phid = $lease->getResourcePHID();
-    $resource_phid = $resource->getPHID();
-
-    if ($lease_phid !== $resource_phid) {
-      // TODO: Destroy the lease?
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
-          'returned from "%s" with a lease acquired on the wrong resource.',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          'acquireLease()'));
-    }
-  }
-
-
-}

src/applications/drydock/worker/DrydockLeaseDestroyWorker.php

@@ -1,39 +0,0 @@
-<?php
-
-final class DrydockLeaseDestroyWorker extends DrydockWorker {
-
-  protected function doWork() {
-    $lease_phid = $this->getTaskDataValue('leasePHID');
-    $lease = $this->loadLease($lease_phid);
-    $this->destroyLease($lease);
-  }
-
-  private function destroyLease(DrydockLease $lease) {
-    $status = $lease->getStatus();
-
-    switch ($status) {
-      case DrydockLeaseStatus::STATUS_RELEASED:
-      case DrydockLeaseStatus::STATUS_BROKEN:
-        break;
-      default:
-        throw new PhabricatorWorkerPermanentFailureException(
-          pht(
-            'Unable to destroy lease ("%s"), lease has the wrong '.
-            'status ("%s").',
-            $lease->getPHID(),
-            $status));
-    }
-
-    $resource = $lease->getResource();
-    $blueprint = $resource->getBlueprint();
-
-    $blueprint->destroyLease($resource, $lease);
-
-    $lease
-      ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
-      ->save();
-
-    $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
-  }
-
-}

src/applications/drydock/worker/DrydockLeaseUpdateWorker.php

@@ -1,5 +1,14 @@
 <?php
 
+/**
+ * @task update Updating Leases
+ * @task command Processing Commands
+ * @task allocator Drydock Allocator
+ * @task acquire Acquiring Leases
+ * @task activate Activating Leases
+ * @task release Releasing Leases
+ * @task destroy Destroying Leases
+ */
 final class DrydockLeaseUpdateWorker extends DrydockWorker {
 
   protected function doWork() {
@@ -22,8 +31,47 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
     $lock->unlock();
   }
 
+
+/* -(  Updating Leases  )---------------------------------------------------- */
+
+
+  /**
+   * @task update
+   */
   private function updateLease(DrydockLease $lease) {
-    if (!$lease->canUpdate()) {
+    $this->processLeaseCommands($lease);
+
+    $lease_status = $lease->getStatus();
+    switch ($lease_status) {
+      case DrydockLeaseStatus::STATUS_PENDING:
+        $this->executeAllocator($lease);
+        break;
+      case DrydockLeaseStatus::STATUS_ACQUIRED:
+        $this->activateLease($lease);
+        break;
+      case DrydockLeaseStatus::STATUS_ACTIVE:
+        // Nothing to do.
+        break;
+      case DrydockLeaseStatus::STATUS_RELEASED:
+      case DrydockLeaseStatus::STATUS_BROKEN:
+        $this->destroyLease($lease);
+        break;
+      case DrydockLeaseStatus::STATUS_DESTROYED:
+        break;
+    }
+
+    $this->yieldIfExpiringLease($lease);
+  }
+
+
+/* -(  Processing Commands  )------------------------------------------------ */
+
+
+  /**
+   * @task command
+   */
+  private function processLeaseCommands(DrydockLease $lease) {
+    if (!$lease->canReceiveCommands()) {
       return;
     }
 
@@ -31,21 +79,23 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
 
     $commands = $this->loadCommands($lease->getPHID());
     foreach ($commands as $command) {
-      if (!$lease->canUpdate()) {
+      if (!$lease->canReceiveCommands()) {
         break;
       }
 
-      $this->processCommand($lease, $command);
+      $this->processLeaseCommand($lease, $command);
 
       $command
         ->setIsConsumed(true)
         ->save();
     }
-
-    $this->yieldIfExpiringLease($lease);
   }
 
-  private function processCommand(
+
+  /**
+   * @task command
+   */
+  private function processLeaseCommand(
     DrydockLease $lease,
     DrydockCommand $command) {
     switch ($command->getCommand()) {
@@ -55,6 +105,530 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
     }
   }
 
+
+/* -(  Drydock Allocator  )-------------------------------------------------- */
+
+
+  /**
+   * Find or build a resource which can satisfy a given lease request, then
+   * acquire the lease.
+   *
+   * @param DrydockLease Requested lease.
+   * @return void
+   * @task allocator
+   */
+  private function executeAllocator(DrydockLease $lease) {
+    $blueprints = $this->loadBlueprintsForAllocatingLease($lease);
+
+    // If we get nothing back, that means no blueprint is defined which can
+    // ever build the requested resource. This is a permanent failure, since
+    // we don't expect to succeed no matter how many times we try.
+    if (!$blueprints) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht(
+          'No active Drydock blueprint exists which can ever allocate a '.
+          'resource for lease "%s".',
+          $lease->getPHID()));
+    }
+
+    // First, try to find a suitable open resource which we can acquire a new
+    // lease on.
+    $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease);
+
+    // If no resources exist yet, see if we can build one.
+    if (!$resources) {
+      $usable_blueprints = $this->removeOverallocatedBlueprints(
+        $blueprints,
+        $lease);
+
+      // If we get nothing back here, some blueprint claims it can eventually
+      // satisfy the lease, just not right now. This is a temporary failure,
+      // and we expect allocation to succeed eventually.
+      if (!$usable_blueprints) {
+        // TODO: More formal temporary failure here. We should retry this
+        // "soon" but not "immediately".
+        throw new Exception(
+          pht('No blueprints have space to allocate a resource right now.'));
+      }
+
+      $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);
+
+      $exceptions = array();
+      foreach ($usable_blueprints as $blueprint) {
+        try {
+          $resources[] = $this->allocateResource($blueprint, $lease);
+
+          // Bail after allocating one resource, we don't need any more than
+          // this.
+          break;
+        } catch (Exception $ex) {
+          $exceptions[] = $ex;
+        }
+      }
+
+      if (!$resources) {
+        // TODO: We should distinguish between temporary and permament failures
+        // here. If any blueprint failed temporarily, retry "soon". If none
+        // of these failures were temporary, maybe this should be a permanent
+        // failure?
+        throw new PhutilAggregateException(
+          pht(
+            'All blueprints failed to allocate a suitable new resource when '.
+            'trying to allocate lease "%s".',
+            $lease->getPHID()),
+          $exceptions);
+      }
+
+      // NOTE: We have not acquired the lease yet, so it is possible that the
+      // resource we just built will be snatched up by some other lease before
+      // we can. This is not problematic: we'll retry a little later and should
+      // suceed eventually.
+    }
+
+    $resources = $this->rankResources($resources, $lease);
+
+    $exceptions = array();
+    $allocated = false;
+    foreach ($resources as $resource) {
+      try {
+        $this->acquireLease($resource, $lease);
+        $allocated = true;
+        break;
+      } catch (Exception $ex) {
+        $exceptions[] = $ex;
+      }
+    }
+
+    if (!$allocated) {
+      // TODO: We should distinguish between temporary and permanent failures
+      // here. If any failures were temporary (specifically, failed to acquire
+      // locks)
+
+      throw new PhutilAggregateException(
+        pht(
+          'Unable to acquire lease "%s" on any resouce.',
+          $lease->getPHID()),
+        $exceptions);
+    }
+  }
+
+
+  /**
+   * Get all the @{class:DrydockBlueprintImplementation}s which can possibly
+   * build a resource to satisfy a lease.
+   *
+   * This method returns blueprints which might, at some time, be able to
+   * build a resource which can satisfy the lease. They may not be able to
+   * build that resource right now.
+   *
+   * @param DrydockLease Requested lease.
+   * @return list<DrydockBlueprintImplementation> List of qualifying blueprint
+   *   implementations.
+   * @task allocator
+   */
+  private function loadBlueprintImplementationsForAllocatingLease(
+    DrydockLease $lease) {
+
+    $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations();
+
+    $keep = array();
+    foreach ($impls as $key => $impl) {
+      // Don't use disabled blueprint types.
+      if (!$impl->isEnabled()) {
+        continue;
+      }
+
+      // Don't use blueprint types which can't allocate the correct kind of
+      // resource.
+      if ($impl->getType() != $lease->getResourceType()) {
+        continue;
+      }
+
+      if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) {
+        continue;
+      }
+
+      $keep[$key] = $impl;
+    }
+
+    return $keep;
+  }
+
+
+  /**
+   * Get all the concrete @{class:DrydockBlueprint}s which can possibly
+   * build a resource to satisfy a lease.
+   *
+   * @param DrydockLease Requested lease.
+   * @return list<DrydockBlueprint> List of qualifying blueprints.
+   * @task allocator
+   */
+  private function loadBlueprintsForAllocatingLease(
+    DrydockLease $lease) {
+    $viewer = $this->getViewer();
+
+    $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease);
+    if (!$impls) {
+      return array();
+    }
+
+    $blueprints = id(new DrydockBlueprintQuery())
+      ->setViewer($viewer)
+      ->withBlueprintClasses(array_keys($impls))
+      ->withDisabled(false)
+      ->execute();
+
+    $keep = array();
+    foreach ($blueprints as $key => $blueprint) {
+      if (!$blueprint->canEverAllocateResourceForLease($lease)) {
+        continue;
+      }
+
+      $keep[$key] = $blueprint;
+    }
+
+    return $keep;
+  }
+
+
+  /**
+   * Load a list of all resources which a given lease can possibly be
+   * allocated against.
+   *
+   * @param list<DrydockBlueprint> Blueprints which may produce suitable
+   *   resources.
+   * @param DrydockLease Requested lease.
+   * @return list<DrydockResource> Resources which may be able to allocate
+   *   the lease.
+   * @task allocator
+   */
+  private function loadResourcesForAllocatingLease(
+    array $blueprints,
+    DrydockLease $lease) {
+    assert_instances_of($blueprints, 'DrydockBlueprint');
+    $viewer = $this->getViewer();
+
+    $resources = id(new DrydockResourceQuery())
+      ->setViewer($viewer)
+      ->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
+      ->withTypes(array($lease->getResourceType()))
+      ->withStatuses(
+        array(
+          DrydockResourceStatus::STATUS_PENDING,
+          DrydockResourceStatus::STATUS_ACTIVE,
+        ))
+      ->execute();
+
+    $keep = array();
+    foreach ($resources as $key => $resource) {
+      $blueprint = $resource->getBlueprint();
+
+      if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
+        continue;
+      }
+
+      $keep[$key] = $resource;
+    }
+
+    return $keep;
+  }
+
+
+  /**
+   * Remove blueprints which are too heavily allocated to build a resource for
+   * a lease from a list of blueprints.
+   *
+   * @param list<DrydockBlueprint> List of blueprints.
+   * @return list<DrydockBlueprint> List with blueprints that can not allocate
+   *   a resource for the lease right now removed.
+   * @task allocator
+   */
+  private function removeOverallocatedBlueprints(
+    array $blueprints,
+    DrydockLease $lease) {
+    assert_instances_of($blueprints, 'DrydockBlueprint');
+
+    $keep = array();
+    foreach ($blueprints as $key => $blueprint) {
+      if (!$blueprint->canAllocateResourceForLease($lease)) {
+        continue;
+      }
+
+      $keep[$key] = $blueprint;
+    }
+
+    return $keep;
+  }
+
+
+  /**
+   * Rank blueprints by suitability for building a new resource for a
+   * particular lease.
+   *
+   * @param list<DrydockBlueprint> List of blueprints.
+   * @param DrydockLease Requested lease.
+   * @return list<DrydockBlueprint> Ranked list of blueprints.
+   * @task allocator
+   */
+  private function rankBlueprints(array $blueprints, DrydockLease $lease) {
+    assert_instances_of($blueprints, 'DrydockBlueprint');
+
+    // TODO: Implement improvements to this ranking algorithm if they become
+    // available.
+    shuffle($blueprints);
+
+    return $blueprints;
+  }
+
+
+  /**
+   * Rank resources by suitability for allocating a particular lease.
+   *
+   * @param list<DrydockResource> List of resources.
+   * @param DrydockLease Requested lease.
+   * @return list<DrydockResource> Ranked list of resources.
+   * @task allocator
+   */
+  private function rankResources(array $resources, DrydockLease $lease) {
+    assert_instances_of($resources, 'DrydockResource');
+
+    // TODO: Implement improvements to this ranking algorithm if they become
+    // available.
+    shuffle($resources);
+
+    return $resources;
+  }
+
+
+  /**
+   * Perform an actual resource allocation with a particular blueprint.
+   *
+   * @param DrydockBlueprint The blueprint to allocate a resource from.
+   * @param DrydockLease Requested lease.
+   * @return DrydockResource Allocated resource.
+   * @task allocator
+   */
+  private function allocateResource(
+    DrydockBlueprint $blueprint,
+    DrydockLease $lease) {
+    $resource = $blueprint->allocateResource($lease);
+    $this->validateAllocatedResource($blueprint, $resource, $lease);
+
+    // If this resource was allocated as a pending resource, queue a task to
+    // activate it.
+    if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {
+      PhabricatorWorker::scheduleTask(
+        'DrydockResourceUpdateWorker',
+        array(
+          'resourcePHID' => $resource->getPHID(),
+        ),
+        array(
+          'objectPHID' => $resource->getPHID(),
+        ));
+    }
+
+    return $resource;
+  }
+
+
+  /**
+   * Check that the resource a blueprint allocated is roughly the sort of
+   * object we expect.
+   *
+   * @param DrydockBlueprint Blueprint which built the resource.
+   * @param wild Thing which the blueprint claims is a valid resource.
+   * @param DrydockLease Lease the resource was allocated for.
+   * @return void
+   * @task allocator
+   */
+  private function validateAllocatedResource(
+    DrydockBlueprint $blueprint,
+    $resource,
+    DrydockLease $lease) {
+
+    if (!($resource instanceof DrydockResource)) {
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
+          'return an object of type %s or throw, but returned something else.',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          'allocateResource()',
+          'DrydockResource'));
+    }
+
+    if (!$resource->isAllocatedResource()) {
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
+          'must actually allocate the resource it returns.',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          'allocateResource()'));
+    }
+
+    $resource_type = $resource->getType();
+    $lease_type = $lease->getResourceType();
+
+    if ($resource_type !== $lease_type) {
+      // TODO: Destroy the resource here?
+
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
+          'built a resource of type "%s" to satisfy a lease requesting a '.
+          'resource of type "%s".',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          $resource_type,
+          $lease_type));
+    }
+  }
+
+
+/* -(  Acquiring Leases  )--------------------------------------------------- */
+
+
+  /**
+   * Perform an actual lease acquisition on a particular resource.
+   *
+   * @param DrydockResource Resource to acquire a lease on.
+   * @param DrydockLease Lease to acquire.
+   * @return void
+   * @task acquire
+   */
+  private function acquireLease(
+    DrydockResource $resource,
+    DrydockLease $lease) {
+
+    $blueprint = $resource->getBlueprint();
+    $blueprint->acquireLease($resource, $lease);
+
+    $this->validateAcquiredLease($blueprint, $resource, $lease);
+
+    // If this lease has been acquired but not activated, queue a task to
+    // activate it.
+    if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
+      PhabricatorWorker::scheduleTask(
+        __CLASS__,
+        array(
+          'leasePHID' => $lease->getPHID(),
+        ),
+        array(
+          'objectPHID' => $lease->getPHID(),
+        ));
+    }
+  }
+
+
+  /**
+   * Make sure that a lease was really acquired properly.
+   *
+   * @param DrydockBlueprint Blueprint which created the resource.
+   * @param DrydockResource Resource which was acquired.
+   * @param DrydockLease The lease which was supposedly acquired.
+   * @return void
+   * @task acquire
+   */
+  private function validateAcquiredLease(
+    DrydockBlueprint $blueprint,
+    DrydockResource $resource,
+    DrydockLease $lease) {
+
+    if (!$lease->isAcquiredLease()) {
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
+          'returned from "%s" without acquiring a lease.',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          'acquireLease()'));
+    }
+
+    $lease_phid = $lease->getResourcePHID();
+    $resource_phid = $resource->getPHID();
+
+    if ($lease_phid !== $resource_phid) {
+      // TODO: Destroy the lease?
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
+          'returned from "%s" with a lease acquired on the wrong resource.',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          'acquireLease()'));
+    }
+  }
+
+
+/* -(  Activating Leases  )-------------------------------------------------- */
+
+
+  /**
+   * @task activate
+   */
+  private function activateLease(DrydockLease $lease) {
+    $resource = $lease->getResource();
+    if (!$resource) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht('Trying to activate lease with no resource.'));
+    }
+
+    $resource_status = $resource->getStatus();
+
+    if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
+      // TODO: This is explicitly a temporary failure -- we are waiting for
+      // the resource to come up.
+      throw new Exception(pht('Resource still activating.'));
+    }
+
+    if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht(
+          'Trying to activate lease on a dead resource (in status "%s").',
+          $resource_status));
+    }
+
+    // NOTE: We can race resource destruction here. Between the time we
+    // performed the read above and now, the resource might have closed, so
+    // we may activate leases on dead resources. At least for now, this seems
+    // fine: a resource dying right before we activate a lease on it should not
+    // be distinguisahble from a resource dying right after we activate a lease
+    // on it. We end up with an active lease on a dead resource either way, and
+    // can not prevent resources dying from lightning strikes.
+
+    $blueprint = $resource->getBlueprint();
+    $blueprint->activateLease($resource, $lease);
+    $this->validateActivatedLease($blueprint, $resource, $lease);
+  }
+
+  /**
+   * @task activate
+   */
+  private function validateActivatedLease(
+    DrydockBlueprint $blueprint,
+    DrydockResource $resource,
+    DrydockLease $lease) {
+
+    if (!$lease->isActivatedLease()) {
+      throw new Exception(
+        pht(
+          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
+          'returned from "%s" without activating a lease.',
+          $blueprint->getBlueprintName(),
+          $blueprint->getClassName(),
+          'acquireLease()'));
+    }
+
+  }
+
+
+/* -(  Releasing Leases  )--------------------------------------------------- */
+
+
+  /**
+   * @task release
+   */
   private function releaseLease(DrydockLease $lease) {
     $lease->openTransaction();
       $lease
@@ -65,21 +639,34 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker {
       DrydockSlotLock::releaseLocks($lease->getPHID());
     $lease->saveTransaction();
 
-    PhabricatorWorker::scheduleTask(
-      'DrydockLeaseDestroyWorker',
-      array(
-        'leasePHID' => $lease->getPHID(),
-      ),
-      array(
-        'objectPHID' => $lease->getPHID(),
-      ));
-
     $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST);
 
     $resource = $lease->getResource();
     $blueprint = $resource->getBlueprint();
 
     $blueprint->didReleaseLease($resource, $lease);
+
+    $this->destroyLease($lease);
+  }
+
+
+/* -(  Destroying Leases  )-------------------------------------------------- */
+
+
+  /**
+   * @task destroy
+   */
+  private function destroyLease(DrydockLease $lease) {
+    $resource = $lease->getResource();
+    $blueprint = $resource->getBlueprint();
+
+    $blueprint->destroyLease($resource, $lease);
+
+    $lease
+      ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
+      ->save();
+
+    $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
   }
 
 }

src/applications/drydock/worker/DrydockLeaseWorker.php

@@ -1,74 +0,0 @@
-<?php
-
-final class DrydockLeaseWorker extends DrydockWorker {
-
-  protected function doWork() {
-    $lease_phid = $this->getTaskDataValue('leasePHID');
-    $lease = $this->loadLease($lease_phid);
-
-    $this->activateLease($lease);
-  }
-
-
-  private function activateLease(DrydockLease $lease) {
-    $actual_status = $lease->getStatus();
-
-    if ($actual_status != DrydockLeaseStatus::STATUS_ACQUIRED) {
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht(
-          'Trying to activate lease from wrong status ("%s").',
-          $actual_status));
-    }
-
-    $resource = $lease->getResource();
-    if (!$resource) {
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht('Trying to activate lease with no resource.'));
-    }
-
-    $resource_status = $resource->getStatus();
-
-    if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
-      // TODO: This is explicitly a temporary failure -- we are waiting for
-      // the resource to come up.
-      throw new Exception(pht('Resource still activating.'));
-    }
-
-    if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht(
-          'Trying to activate lease on a dead resource (in status "%s").',
-          $resource_status));
-    }
-
-    // NOTE: We can race resource destruction here. Between the time we
-    // performed the read above and now, the resource might have closed, so
-    // we may activate leases on dead resources. At least for now, this seems
-    // fine: a resource dying right before we activate a lease on it should not
-    // be distinguisahble from a resource dying right after we activate a lease
-    // on it. We end up with an active lease on a dead resource either way, and
-    // can not prevent resources dying from lightning strikes.
-
-    $blueprint = $resource->getBlueprint();
-    $blueprint->activateLease($resource, $lease);
-    $this->validateActivatedLease($blueprint, $resource, $lease);
-  }
-
-  private function validateActivatedLease(
-    DrydockBlueprint $blueprint,
-    DrydockResource $resource,
-    DrydockLease $lease) {
-
-    if (!$lease->isActivatedLease()) {
-      throw new Exception(
-        pht(
-          'Blueprint "%s" (of type "%s") is not properly implemented: it '.
-          'returned from "%s" without activating a lease.',
-          $blueprint->getBlueprintName(),
-          $blueprint->getClassName(),
-          'acquireLease()'));
-    }
-
-  }
-
-}

src/applications/drydock/worker/DrydockWorker.php

@@ -86,7 +86,7 @@ abstract class DrydockWorker extends PhabricatorWorker {
   }
 
   protected function yieldIfExpiringLease(DrydockLease $lease) {
-    if (!$lease->canUpdate()) {
+    if (!$lease->canReceiveCommands()) {
       return;
     }
 

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

@@ -154,7 +154,8 @@ final class HarbormasterBuildLog extends HarbormasterDAO
 
   public function finalize($start = 0) {
     if (!$this->getLive()) {
-      throw new Exception(pht('Start logging before finalizing it.'));
+      // TODO: Clean up this API.
+      return;
     }
 
     // TODO: Encode the log contents in a gzipped format.

src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php

@@ -60,7 +60,10 @@ abstract class PhabricatorWorkerTask extends PhabricatorWorkerDAO {
     $id = $this->getID();
     $class = $this->getTaskClass();
 
-    if (!class_exists($class)) {
+    try {
+      // NOTE: If the class does not exist, libphutil will throw an exception.
+      class_exists($class);
+    } catch (PhutilMissingSymbolException $ex) {
       throw new PhabricatorWorkerPermanentFailureException(
         pht(
           "Task class '%s' does not exist!",