diff --git a/resources/sql/autopatches/20181024.drydock.01.commandprops.sql b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql new file mode 100644 index 0000000000..e808146b02 --- /dev/null +++ b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_drydock.drydock_command + ADD properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql new file mode 100644 index 0000000000..2c336dc40e --- /dev/null +++ b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql @@ -0,0 +1,2 @@ +UPDATE {$NAMESPACE}_drydock.drydock_command + SET properties = '{}' WHERE properties = ''; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 8bcf3d6120..a472b66411 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1146,6 +1146,7 @@ phutil_register_library_map(array( 'DrydockLeaseReclaimLogType' => 'applications/drydock/logtype/DrydockLeaseReclaimLogType.php', 'DrydockLeaseReleaseController' => 'applications/drydock/controller/DrydockLeaseReleaseController.php', 'DrydockLeaseReleasedLogType' => 'applications/drydock/logtype/DrydockLeaseReleasedLogType.php', + 'DrydockLeaseSearchConduitAPIMethod' => 'applications/drydock/conduit/DrydockLeaseSearchConduitAPIMethod.php', 'DrydockLeaseSearchEngine' => 'applications/drydock/query/DrydockLeaseSearchEngine.php', 'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php', 'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php', @@ -6532,6 +6533,7 @@ phutil_register_library_map(array( 'DrydockLease' => array( 'DrydockDAO', 'PhabricatorPolicyInterface', + 'PhabricatorConduitResultInterface', ), 'DrydockLeaseAcquiredLogType' => 'DrydockLogType', 'DrydockLeaseActivatedLogType' => 'DrydockLogType', @@ -6552,6 +6554,7 @@ phutil_register_library_map(array( 'DrydockLeaseReclaimLogType' => 'DrydockLogType', 'DrydockLeaseReleaseController' => 'DrydockLeaseController', 'DrydockLeaseReleasedLogType' => 'DrydockLogType', + 'DrydockLeaseSearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', 'DrydockLeaseSearchEngine' => 'PhabricatorApplicationSearchEngine', 'DrydockLeaseStatus' => 'PhabricatorObjectStatus', 'DrydockLeaseUpdateWorker' => 'DrydockWorker', diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementLogWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementLogWorkflow.php index 394b4735a5..8fca624b44 100644 --- a/src/applications/daemon/management/PhabricatorDaemonManagementLogWorkflow.php +++ b/src/applications/daemon/management/PhabricatorDaemonManagementLogWorkflow.php @@ -40,15 +40,20 @@ final class PhabricatorDaemonManagementLogWorkflow $query->withIDs($ids); } $daemons = $query->execute(); + $daemons = mpull($daemons, null, 'getID'); - if (!$daemons) { - if ($ids) { - throw new PhutilArgumentUsageException( - pht('No daemon(s) with id(s) "%s" exist!', implode(', ', $ids))); - } else { - throw new PhutilArgumentUsageException( - pht('No daemons are running.')); + if ($ids) { + foreach ($ids as $id) { + if (!isset($daemons[$id])) { + throw new PhutilArgumentUsageException( + pht( + 'No log record exists for a daemon with ID "%s".', + $id)); + } } + } else if (!$daemons) { + throw new PhutilArgumentUsageException( + pht('No log records exist for any daemons.')); } $console = PhutilConsole::getConsole(); diff --git a/src/applications/diffusion/xaction/DiffusionCommitAcceptTransaction.php b/src/applications/diffusion/xaction/DiffusionCommitAcceptTransaction.php index 2e0dab2c35..5ade7f3513 100644 --- a/src/applications/diffusion/xaction/DiffusionCommitAcceptTransaction.php +++ b/src/applications/diffusion/xaction/DiffusionCommitAcceptTransaction.php @@ -7,7 +7,7 @@ final class DiffusionCommitAcceptTransaction const ACTIONKEY = 'accept'; protected function getCommitActionLabel() { - return pht("Accept Commit \xE2\x9C\x94"); + return pht('Accept Commit'); } protected function getCommitActionDescription() { @@ -70,4 +70,12 @@ final class DiffusionCommitAcceptTransaction $this->renderObject()); } + public function getTransactionTypeForConduit($xaction) { + return 'accept'; + } + + public function getFieldValuesForConduit($object, $data) { + return array(); + } + } diff --git a/src/applications/diffusion/xaction/DiffusionCommitConcernTransaction.php b/src/applications/diffusion/xaction/DiffusionCommitConcernTransaction.php index 1f6ae72621..ffd084412e 100644 --- a/src/applications/diffusion/xaction/DiffusionCommitConcernTransaction.php +++ b/src/applications/diffusion/xaction/DiffusionCommitConcernTransaction.php @@ -7,7 +7,7 @@ final class DiffusionCommitConcernTransaction const ACTIONKEY = 'concern'; protected function getCommitActionLabel() { - return pht("Raise Concern \xE2\x9C\x98"); + return pht('Raise Concern'); } protected function getCommitActionDescription() { @@ -76,4 +76,12 @@ final class DiffusionCommitConcernTransaction $this->renderObject()); } + public function getTransactionTypeForConduit($xaction) { + return 'concern'; + } + + public function getFieldValuesForConduit($object, $data) { + return array(); + } + } diff --git a/src/applications/diffusion/xaction/DiffusionCommitResignTransaction.php b/src/applications/diffusion/xaction/DiffusionCommitResignTransaction.php index 103d0fabfe..8adc8346dc 100644 --- a/src/applications/diffusion/xaction/DiffusionCommitResignTransaction.php +++ b/src/applications/diffusion/xaction/DiffusionCommitResignTransaction.php @@ -63,4 +63,12 @@ final class DiffusionCommitResignTransaction $this->renderObject()); } + public function getTransactionTypeForConduit($xaction) { + return 'resign'; + } + + public function getFieldValuesForConduit($object, $data) { + return array(); + } + } diff --git a/src/applications/diffusion/xaction/DiffusionCommitVerifyTransaction.php b/src/applications/diffusion/xaction/DiffusionCommitVerifyTransaction.php index f1ec5834b1..f9f19f4290 100644 --- a/src/applications/diffusion/xaction/DiffusionCommitVerifyTransaction.php +++ b/src/applications/diffusion/xaction/DiffusionCommitVerifyTransaction.php @@ -68,4 +68,12 @@ final class DiffusionCommitVerifyTransaction $this->renderObject()); } + public function getTransactionTypeForConduit($xaction) { + return 'request-verification'; + } + + public function getFieldValuesForConduit($object, $data) { + return array(); + } + } diff --git a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php index ef904e1019..4724a2c807 100644 --- a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php @@ -113,6 +113,14 @@ final class DrydockAlmanacServiceHostBlueprintImplementation DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { + + // Require the binding to a given host be active before we'll hand out more + // leases on the corresponding resource. + $binding = $this->loadBindingForResource($resource); + if ($binding->getIsDisabled()) { + return false; + } + return true; } @@ -154,24 +162,10 @@ final class DrydockAlmanacServiceHostBlueprintImplementation DrydockLease $lease, $type) { - $viewer = PhabricatorUser::getOmnipotentUser(); - switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $credential_phid = $blueprint->getFieldValue('credentialPHID'); - $binding_phid = $resource->getAttribute('almanacBindingPHID'); - - $binding = id(new AlmanacBindingQuery()) - ->setViewer($viewer) - ->withPHIDs(array($binding_phid)) - ->executeOne(); - if (!$binding) { - throw new Exception( - pht( - 'Unable to load binding "%s" to create command interface.', - $binding_phid)); - } - + $binding = $this->loadBindingForResource($resource); $interface = $binding->getInterface(); return id(new DrydockSSHCommandInterface()) @@ -213,7 +207,7 @@ final class DrydockAlmanacServiceHostBlueprintImplementation $blueprint->getBlueprintName())); } - $viewer = PhabricatorUser::getOmnipotentUser(); + $viewer = $this->getViewer(); $services = id(new AlmanacServiceQuery()) ->setViewer($viewer) ->withPHIDs($service_phids) @@ -246,7 +240,7 @@ final class DrydockAlmanacServiceHostBlueprintImplementation private function loadFreeBindings(DrydockBlueprint $blueprint) { if ($this->freeBindings === null) { - $viewer = PhabricatorUser::getOmnipotentUser(); + $viewer = $this->getViewer(); $pool = id(new DrydockResourceQuery()) ->setViewer($viewer) @@ -293,5 +287,31 @@ final class DrydockAlmanacServiceHostBlueprintImplementation ); } + private function loadBindingForResource(DrydockResource $resource) { + $binding_phid = $resource->getAttribute('almanacBindingPHID'); + if (!$binding_phid) { + throw new Exception( + pht( + 'Drydock resource ("%s") has no Almanac binding PHID, so its '. + 'binding can not be loaded.', + $resource->getPHID())); + } + + $viewer = $this->getViewer(); + + $binding = id(new AlmanacBindingQuery()) + ->setViewer($viewer) + ->withPHIDs(array($binding_phid)) + ->executeOne(); + if (!$binding) { + throw new Exception( + pht( + 'Unable to load Almanac binding ("%s") for resource ("%s").', + $binding_phid, + $resource->getPHID())); + } + + return $binding; + } } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php index f45c4f2adb..c1672f99fb 100644 --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -256,8 +256,9 @@ final class DrydockWorkingCopyBlueprintImplementation $ref = idx($spec, 'ref'); // Reset things first, in case previous builds left anything staged or - // dirty. - $cmd[] = 'git reset --hard HEAD'; + // dirty. Note that we don't reset to "HEAD" because that does not work + // in empty repositories. + $cmd[] = 'git reset --hard'; if ($commit !== null) { $cmd[] = 'git checkout %s --'; diff --git a/src/applications/drydock/conduit/DrydockLeaseSearchConduitAPIMethod.php b/src/applications/drydock/conduit/DrydockLeaseSearchConduitAPIMethod.php new file mode 100644 index 0000000000..6d75ed1718 --- /dev/null +++ b/src/applications/drydock/conduit/DrydockLeaseSearchConduitAPIMethod.php @@ -0,0 +1,18 @@ + pht('Set lease expiration time.'), ), array( - 'name' => 'attributes', - 'param' => 'name=value,...', - 'help' => pht('Resource specification.'), + 'name' => 'attributes', + 'param' => 'file', + 'help' => pht( + 'JSON file with lease attributes. Use "-" to read attributes '. + 'from stdin.'), ), )); } @@ -49,11 +51,20 @@ final class DrydockManagementLeaseWorkflow } } - $attributes = $args->getArg('attributes'); - if ($attributes) { - $options = new PhutilSimpleOptions(); - $options->setCaseSensitive(true); - $attributes = $options->parse($attributes); + $attributes_file = $args->getArg('attributes'); + if (strlen($attributes_file)) { + if ($attributes_file == '-') { + echo tsprintf( + "%s\n", + 'Reading JSON attributes from stdin...'); + $data = file_get_contents('php://stdin'); + } else { + $data = Filesystem::readFile($attributes_file); + } + + $attributes = phutil_json_decode($data); + } else { + $attributes = array(); } $lease = id(new DrydockLease()) diff --git a/src/applications/drydock/query/DrydockLeaseQuery.php b/src/applications/drydock/query/DrydockLeaseQuery.php index c5fb41ff56..e524c1f9c5 100644 --- a/src/applications/drydock/query/DrydockLeaseQuery.php +++ b/src/applications/drydock/query/DrydockLeaseQuery.php @@ -5,6 +5,7 @@ final class DrydockLeaseQuery extends DrydockQuery { private $ids; private $phids; private $resourcePHIDs; + private $ownerPHIDs; private $statuses; private $datasourceQuery; private $needUnconsumedCommands; @@ -24,6 +25,11 @@ final class DrydockLeaseQuery extends DrydockQuery { return $this; } + public function withOwnerPHIDs(array $phids) { + $this->ownerPHIDs = $phids; + return $this; + } + public function withStatuses(array $statuses) { $this->statuses = $statuses; return $this; @@ -105,6 +111,13 @@ final class DrydockLeaseQuery extends DrydockQuery { $this->resourcePHIDs); } + if ($this->ownerPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'ownerPHID IN (%Ls)', + $this->ownerPHIDs); + } + if ($this->ids !== null) { $where[] = qsprintf( $conn, diff --git a/src/applications/drydock/query/DrydockLeaseSearchEngine.php b/src/applications/drydock/query/DrydockLeaseSearchEngine.php index 3b551023b2..39c91e66bf 100644 --- a/src/applications/drydock/query/DrydockLeaseSearchEngine.php +++ b/src/applications/drydock/query/DrydockLeaseSearchEngine.php @@ -40,6 +40,10 @@ final class DrydockLeaseSearchEngine $query->withStatuses($map['statuses']); } + if ($map['ownerPHIDs']) { + $query->withOwnerPHIDs($map['ownerPHIDs']); + } + return $query; } @@ -49,6 +53,11 @@ final class DrydockLeaseSearchEngine ->setLabel(pht('Statuses')) ->setKey('statuses') ->setOptions(DrydockLeaseStatus::getStatusMap()), + id(new PhabricatorPHIDsSearchField()) + ->setLabel(pht('Owners')) + ->setKey('ownerPHIDs') + ->setAliases(array('owner', 'owners', 'ownerPHID')) + ->setDescription(pht('Search leases by owner.')), ); } diff --git a/src/applications/drydock/storage/DrydockCommand.php b/src/applications/drydock/storage/DrydockCommand.php index e7d003bdd6..0f7f253217 100644 --- a/src/applications/drydock/storage/DrydockCommand.php +++ b/src/applications/drydock/storage/DrydockCommand.php @@ -11,6 +11,7 @@ final class DrydockCommand protected $targetPHID; protected $command; protected $isConsumed; + protected $properties = array(); private $commandTarget = self::ATTACHABLE; @@ -22,6 +23,9 @@ final class DrydockCommand protected function getConfiguration() { return array( + self::CONFIG_SERIALIZATION => array( + 'properties' => self::SERIALIZATION_JSON, + ), self::CONFIG_COLUMN_SCHEMA => array( 'command' => 'text32', 'isConsumed' => 'bool', @@ -43,6 +47,14 @@ final class DrydockCommand return $this->assertAttached($this->commandTarget); } + public function setProperty($key, $value) { + $this->properties[$key] = $value; + return $this; + } + + public function getProperty($key, $default = null) { + return idx($this->properties, $key, $default); + } /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/drydock/storage/DrydockLease.php b/src/applications/drydock/storage/DrydockLease.php index 866bb21b37..2b77146495 100644 --- a/src/applications/drydock/storage/DrydockLease.php +++ b/src/applications/drydock/storage/DrydockLease.php @@ -1,7 +1,9 @@ array( 'columns' => array('status'), ), + 'key_owner' => array( + 'columns' => array('ownerPHID'), + ), ), ) + parent::getConfiguration(); } @@ -535,4 +540,66 @@ final class DrydockLease extends DrydockDAO return pht('Leases inherit policies from the resources they lease.'); } + +/* -( PhabricatorConduitResultInterface )---------------------------------- */ + + + public function getFieldSpecificationsForConduit() { + return array( + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('resourcePHID') + ->setType('phid?') + ->setDescription(pht('PHID of the leased resource, if any.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('resourceType') + ->setType('string') + ->setDescription(pht('Type of resource being leased.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('until') + ->setType('int?') + ->setDescription(pht('Epoch at which this lease expires, if any.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('ownerPHID') + ->setType('phid?') + ->setDescription(pht('The PHID of the object that owns this lease.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('authorizingPHID') + ->setType('phid') + ->setDescription(pht( + 'The PHID of the object that authorized this lease.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('status') + ->setType('map') + ->setDescription(pht( + "The string constant and name of this lease's status.")), + ); + } + + public function getFieldValuesForConduit() { + $status = $this->getStatus(); + + $until = $this->getUntil(); + if ($until) { + $until = (int)$until; + } else { + $until = null; + } + + return array( + 'resourcePHID' => $this->getResourcePHID(), + 'resourceType' => $this->getResourceType(), + 'until' => $until, + 'ownerPHID' => $this->getOwnerPHID(), + 'authorizingPHID' => $this->getAuthorizingPHID(), + 'status' => array( + 'value' => $status, + 'name' => DrydockLeaseStatus::getNameForStatus($status), + ), + ); + } + + public function getConduitSearchAttachments() { + return array(); + } + } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 4a0cba8de2..3ff9900239 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -599,6 +599,13 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $resource->getPHID(), + + // This task will generally yield while the resource activates, so + // wake it back up once the resource comes online. Most of the time, + // we'll be able to lease the newly activated resource. + 'awakenOnActivation' => array( + $this->getCurrentWorkerTaskID(), + ), ), array( 'objectPHID' => $resource->getPHID(), @@ -666,6 +673,26 @@ final class DrydockLeaseUpdateWorker extends DrydockWorker { DrydockLease $lease) { $viewer = $this->getViewer(); + // If this lease is marked as already in the process of reclaiming a + // resource, don't let it reclaim another one until the first reclaim + // completes. This stops one lease from reclaiming a large number of + // resources if the reclaims take a while to complete. + $reclaiming_phid = $lease->getAttribute('drydock.reclaimingPHID'); + if ($reclaiming_phid) { + $reclaiming_resource = id(new DrydockResourceQuery()) + ->setViewer($viewer) + ->withPHIDs(array($reclaiming_phid)) + ->withStatuses( + array( + DrydockResourceStatus::STATUS_ACTIVE, + DrydockResourceStatus::STATUS_RELEASED, + )) + ->executeOne(); + if ($reclaiming_resource) { + return null; + } + } + $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(array($blueprint->getPHID())) diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php index baa80f90d8..14ef8e4936 100644 --- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php @@ -150,6 +150,13 @@ final class DrydockResourceUpdateWorker extends DrydockWorker { $this->releaseResource($resource, $reclaimer_phid); break; } + + // If the command specifies that other worker tasks should be awakened + // after it executes, awaken them now. + $awaken_ids = $command->getProperty('awakenTaskIDs'); + if (is_array($awaken_ids) && $awaken_ids) { + PhabricatorWorker::awakenTaskIDs($awaken_ids); + } } @@ -163,6 +170,11 @@ final class DrydockResourceUpdateWorker extends DrydockWorker { $blueprint = $resource->getBlueprint(); $blueprint->activateResource($resource); $this->validateActivatedResource($blueprint, $resource); + + $awaken_ids = $this->getTaskDataValue('awakenOnActivation'); + if (is_array($awaken_ids) && $awaken_ids) { + PhabricatorWorker::awakenTaskIDs($awaken_ids); + } } diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php index 443780680d..fcb6876cfe 100644 --- a/src/applications/drydock/worker/DrydockWorker.php +++ b/src/applications/drydock/worker/DrydockWorker.php @@ -241,11 +241,25 @@ abstract class DrydockWorker extends PhabricatorWorker { DrydockLease $lease) { $viewer = $this->getViewer(); + // Mark the lease as reclaiming this resource. It won't be allowed to start + // another reclaim as long as this resource is still in the process of + // being reclaimed. + $lease->setAttribute('drydock.reclaimingPHID', $resource->getPHID()); + + // When the resource releases, we we want to reawaken this task since it + // should (usually) be able to start building a new resource right away. + $worker_task_id = $this->getCurrentWorkerTaskID(); + $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($resource->getPHID()) ->setAuthorPHID($lease->getPHID()) ->setCommand(DrydockCommand::COMMAND_RECLAIM) - ->save(); + ->setProperty('awakenTaskIDs', array($worker_task_id)); + + $lease->openTransaction(); + $lease->save(); + $command->save(); + $lease->saveTransaction(); $resource->scheduleUpdate(); diff --git a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php index c2241995e7..5169c08ffd 100644 --- a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php +++ b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php @@ -9,10 +9,6 @@ final class PhabricatorUSEnglishTranslation protected function getTranslations() { return array( - 'No daemon(s) with id(s) "%s" exist!' => array( - 'No daemon with id %s exists!', - 'No daemons with ids %s exist!', - ), 'These %d configuration value(s) are related:' => array( 'This configuration value is related:', 'These configuration values are related:',