Add more Drydock log types and some additional logging

Summary: Ref T9252. Add a bit more logging and improve some behaviors.

Test Plan: Restarted a build, got a good result.

Reviewers: chad, hach-que

Reviewed By: hach-que

Maniphest Tasks: T9252

Differential Revision: https://secure.phabricator.com/D14210

src/__phutil_library_map__.php

@@ -831,6 +831,8 @@ phutil_register_library_map(array(
     'DrydockLease' => 'applications/drydock/storage/DrydockLease.php',
     'DrydockLeaseAcquiredLogType' => 'applications/drydock/logtype/DrydockLeaseAcquiredLogType.php',
     'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php',
+    'DrydockLeaseActivationFailureLogType' => 'applications/drydock/logtype/DrydockLeaseActivationFailureLogType.php',
+    'DrydockLeaseActivationYieldLogType' => 'applications/drydock/logtype/DrydockLeaseActivationYieldLogType.php',
     'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php',
     'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php',
     'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php',
@@ -845,6 +847,7 @@ phutil_register_library_map(array(
     'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php',
     'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php',
     'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php',
+    'DrydockLeaseWaitingForResourcesLogType' => 'applications/drydock/logtype/DrydockLeaseWaitingForResourcesLogType.php',
     'DrydockLog' => 'applications/drydock/storage/DrydockLog.php',
     'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php',
     'DrydockLogGarbageCollector' => 'applications/drydock/garbagecollector/DrydockLogGarbageCollector.php',
@@ -862,6 +865,8 @@ phutil_register_library_map(array(
     'DrydockManagementWorkflow' => 'applications/drydock/management/DrydockManagementWorkflow.php',
     'DrydockQuery' => 'applications/drydock/query/DrydockQuery.php',
     'DrydockResource' => 'applications/drydock/storage/DrydockResource.php',
+    'DrydockResourceActivationFailureLogType' => 'applications/drydock/logtype/DrydockResourceActivationFailureLogType.php',
+    'DrydockResourceActivationYieldLogType' => 'applications/drydock/logtype/DrydockResourceActivationYieldLogType.php',
     'DrydockResourceController' => 'applications/drydock/controller/DrydockResourceController.php',
     'DrydockResourceDatasource' => 'applications/drydock/typeahead/DrydockResourceDatasource.php',
     'DrydockResourceListController' => 'applications/drydock/controller/DrydockResourceListController.php',
@@ -877,6 +882,7 @@ phutil_register_library_map(array(
     'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php',
     'DrydockSlotLock' => 'applications/drydock/storage/DrydockSlotLock.php',
     'DrydockSlotLockException' => 'applications/drydock/exception/DrydockSlotLockException.php',
+    'DrydockSlotLockFailureLogType' => 'applications/drydock/logtype/DrydockSlotLockFailureLogType.php',
     'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php',
     'DrydockWorker' => 'applications/drydock/worker/DrydockWorker.php',
     'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php',
@@ -4571,6 +4577,8 @@ phutil_register_library_map(array(
     ),
     'DrydockLeaseAcquiredLogType' => 'DrydockLogType',
     'DrydockLeaseActivatedLogType' => 'DrydockLogType',
+    'DrydockLeaseActivationFailureLogType' => 'DrydockLogType',
+    'DrydockLeaseActivationYieldLogType' => 'DrydockLogType',
     'DrydockLeaseController' => 'DrydockController',
     'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource',
     'DrydockLeaseDestroyedLogType' => 'DrydockLogType',
@@ -4585,6 +4593,7 @@ phutil_register_library_map(array(
     'DrydockLeaseStatus' => 'DrydockConstants',
     'DrydockLeaseUpdateWorker' => 'DrydockWorker',
     'DrydockLeaseViewController' => 'DrydockLeaseController',
+    'DrydockLeaseWaitingForResourcesLogType' => 'DrydockLogType',
     'DrydockLog' => array(
       'DrydockDAO',
       'PhabricatorPolicyInterface',
@@ -4608,6 +4617,8 @@ phutil_register_library_map(array(
       'DrydockDAO',
       'PhabricatorPolicyInterface',
     ),
+    'DrydockResourceActivationFailureLogType' => 'DrydockLogType',
+    'DrydockResourceActivationYieldLogType' => 'DrydockLogType',
     'DrydockResourceController' => 'DrydockController',
     'DrydockResourceDatasource' => 'PhabricatorTypeaheadDatasource',
     'DrydockResourceListController' => 'DrydockResourceController',
@@ -4623,6 +4634,7 @@ phutil_register_library_map(array(
     'DrydockSSHCommandInterface' => 'DrydockCommandInterface',
     'DrydockSlotLock' => 'DrydockDAO',
     'DrydockSlotLockException' => 'Exception',
+    'DrydockSlotLockFailureLogType' => 'DrydockLogType',
     'DrydockWebrootInterface' => 'DrydockInterface',
     'DrydockWorker' => 'PhabricatorWorker',
     'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation',

src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php

@@ -172,14 +172,16 @@ final class DrydockWorkingCopyBlueprintImplementation
     // Destroy the lease on the host.
     $lease->releaseOnDestruction();
 
-    // Destroy the working copy on disk.
+    if ($lease->isActive()) {
-    $command_type = DrydockCommandInterface::INTERFACE_TYPE;
+      // Destroy the working copy on disk.
-    $interface = $lease->getInterface($command_type);
+      $command_type = DrydockCommandInterface::INTERFACE_TYPE;
+      $interface = $lease->getInterface($command_type);
 
-    $root_key = 'workingcopy.root';
+      $root_key = 'workingcopy.root';
-    $root = $resource->getAttribute($root_key);
+      $root = $resource->getAttribute($root_key);
-    if (strlen($root)) {
+      if (strlen($root)) {
-      $interface->execx('rm -rf -- %s', $root);
+        $interface->execx('rm -rf -- %s', $root);
+      }
     }
   }
 

src/applications/drydock/exception/DrydockSlotLockException.php

@@ -22,4 +22,8 @@ final class DrydockSlotLockException extends Exception {
     parent::__construct($message);
   }
 
+  public function getLockMap() {
+    return $this->lockMap;
+  }
+
 }

src/applications/drydock/logtype/DrydockLeaseActivationFailureLogType.php

@@ -0,0 +1,22 @@
+<?php
+
+final class DrydockLeaseActivationFailureLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.lease.activation-failure';
+
+  public function getLogTypeName() {
+    return pht('Activation Failed');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-times red';
+  }
+
+  public function renderLog(array $data) {
+    $class = idx($data, 'class');
+    $message = idx($data, 'message');
+
+    return pht('Lease activation failed: [%s] %s', $class, $message);
+  }
+
+}

src/applications/drydock/logtype/DrydockLeaseActivationYieldLogType.php

@@ -0,0 +1,23 @@
+<?php
+
+final class DrydockLeaseActivationYieldLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.lease.activation-yield';
+
+  public function getLogTypeName() {
+    return pht('Waiting for Activation');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-clock-o green';
+  }
+
+  public function renderLog(array $data) {
+    $duration = idx($data, 'duration');
+
+    return pht(
+      'Waiting %s second(s) for lease to activate.',
+      new PhutilNumber($duration));
+  }
+
+}

src/applications/drydock/logtype/DrydockLeaseWaitingForResourcesLogType.php

@@ -0,0 +1,25 @@
+<?php
+
+final class DrydockLeaseWaitingForResourcesLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.lease.waiting-for-resources';
+
+  public function getLogTypeName() {
+    return pht('Waiting For Resource');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-clock-o yellow';
+  }
+
+  public function renderLog(array $data) {
+    $viewer = $this->getViewer();
+
+    $blueprint_phids = idx($data, 'blueprintPHIDs', array());
+
+    return pht(
+      'Waiting for available resources from: %s.',
+      $viewer->renderHandleList($blueprint_phids));
+  }
+
+}

src/applications/drydock/logtype/DrydockResourceActivationFailureLogType.php

@@ -0,0 +1,22 @@
+<?php
+
+final class DrydockResourceActivationFailureLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.resource.activation-failure';
+
+  public function getLogTypeName() {
+    return pht('Activation Failed');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-times red';
+  }
+
+  public function renderLog(array $data) {
+    $class = idx($data, 'class');
+    $message = idx($data, 'message');
+
+    return pht('Resource activation failed: [%s] %s', $class, $message);
+  }
+
+}

src/applications/drydock/logtype/DrydockResourceActivationYieldLogType.php

@@ -0,0 +1,23 @@
+<?php
+
+final class DrydockResourceActivationYieldLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.resource.activation-yield';
+
+  public function getLogTypeName() {
+    return pht('Waiting for Activation');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-clock-o green';
+  }
+
+  public function renderLog(array $data) {
+    $duration = idx($data, 'duration');
+
+    return pht(
+      'Waiting %s second(s) for resource to activate.',
+      new PhutilNumber($duration));
+  }
+
+}

src/applications/drydock/logtype/DrydockSlotLockFailureLogType.php

@@ -0,0 +1,26 @@
+<?php
+
+final class DrydockSlotLockFailureLogType extends DrydockLogType {
+
+  const LOGCONST = 'core.resource.slot-lock.failure';
+
+  public function getLogTypeName() {
+    return pht('Slot Lock Failure');
+  }
+
+  public function getLogTypeIcon(array $data) {
+    return 'fa-lock yellow';
+  }
+
+  public function renderLog(array $data) {
+    $locks = idx($data, 'locks', array());
+    if ($locks) {
+      return pht(
+        'Failed to acquire slot locks: %s.',
+        implode(', ', array_keys($locks)));
+    } else {
+      return pht('Failed to acquire slot locks.');
+    }
+  }
+
+}

src/applications/drydock/storage/DrydockLease.php

@@ -232,16 +232,28 @@ final class DrydockLease extends DrydockDAO
     }
 
     $this->openTransaction();
+    try {
+      try {
+        DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
+        $this->slotLocks = array();
+      } catch (DrydockSlotLockException $ex) {
+        $this->logEvent(
+          DrydockSlotLockFailureLogType::LOGCONST,
+          array(
+            'locks' => $ex->getLockMap(),
+          ));
+        throw $ex;
+      }
 
       $this
         ->setResourcePHID($resource->getPHID())
         ->attachResource($resource)
         ->setStatus($new_status)
         ->save();
-
+    } catch (Exception $ex) {
-      DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
+      $this->killTransaction();
-      $this->slotLocks = array();
+      throw $ex;
-
+    }
     $this->saveTransaction();
 
     $this->isAcquired = true;

src/applications/drydock/storage/DrydockResource.php

@@ -135,15 +135,30 @@ final class DrydockResource extends DrydockDAO
       $new_status = DrydockResourceStatus::STATUS_PENDING;
     }
 
+    $phid = $this->generatePHID();
+
     $this->openTransaction();
+    try {
+      try {
+        DrydockSlotLock::acquireLocks($phid, $this->slotLocks);
+        $this->slotLocks = array();
+      } catch (DrydockSlotLockException $ex) {
+        $this->logEvent(
+          DrydockSlotLockFailureLogType::LOGCONST,
+          array(
+            'locks' => $ex->getLockMap(),
+          ));
+        throw $ex;
+      }
 
       $this
+        ->setPHID($phid)
         ->setStatus($new_status)
         ->save();
-
+    } catch (Exception $ex) {
-      DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
+      $this->killTransaction();
-      $this->slotLocks = array();
+      throw $ex;
-
+    }
     $this->saveTransaction();
 
     $this->isAllocated = true;
@@ -257,6 +272,18 @@ final class DrydockResource extends DrydockDAO
     }
   }
 
+  public function logEvent($type, array $data = array()) {
+    $log = id(new DrydockLog())
+      ->setEpoch(PhabricatorTime::getNow())
+      ->setType($type)
+      ->setData($data);
+
+    $log->setResourcePHID($this->getPHID());
+    $log->setBlueprintPHID($this->getBlueprintPHID());
+
+    return $log->save();
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */