Implement "Upload Artifact" build step

Summary: This implements a build step for uploading an artifact from a build machine to Phabricator.  It uses SFTP so that it will work on both UNIX and Windows build machines.

Test Plan: Ran an "Upload Artifact" build against a Windows machine (with FreeSSHD installed).  The artifact uploaded to Phabricator, appeared on the build view and the file contents could be viewed from Phabricator.

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley

CC: Korvin, epriestley, aran

Maniphest Tasks: T1049

Differential Revision: https://secure.phabricator.com/D7582

src/__phutil_library_map__.php

@@ -642,6 +642,7 @@ phutil_register_library_map(array(
     'DrydockConstants' => 'applications/drydock/constants/DrydockConstants.php',
     'DrydockController' => 'applications/drydock/controller/DrydockController.php',
     'DrydockDAO' => 'applications/drydock/storage/DrydockDAO.php',
+    'DrydockFilesystemInterface' => 'applications/drydock/interface/filesystem/DrydockFilesystemInterface.php',
     'DrydockInterface' => 'applications/drydock/interface/DrydockInterface.php',
     'DrydockLease' => 'applications/drydock/storage/DrydockLease.php',
     'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php',
@@ -668,6 +669,7 @@ phutil_register_library_map(array(
     'DrydockResourceQuery' => 'applications/drydock/query/DrydockResourceQuery.php',
     'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php',
     'DrydockResourceViewController' => 'applications/drydock/controller/DrydockResourceViewController.php',
+    'DrydockSFTPFilesystemInterface' => 'applications/drydock/interface/filesystem/DrydockSFTPFilesystemInterface.php',
     'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php',
     'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php',
     'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php',
@@ -2332,6 +2334,7 @@ phutil_register_library_map(array(
     'SleepBuildStepImplementation' => 'applications/harbormaster/step/SleepBuildStepImplementation.php',
     'SlowvoteEmbedView' => 'applications/slowvote/view/SlowvoteEmbedView.php',
     'SlowvoteRemarkupRule' => 'applications/slowvote/remarkup/SlowvoteRemarkupRule.php',
+    'UploadArtifactBuildStepImplementation' => 'applications/harbormaster/step/UploadArtifactBuildStepImplementation.php',
     'VariableBuildStepImplementation' => 'applications/harbormaster/step/VariableBuildStepImplementation.php',
   ),
   'function' =>
@@ -2987,6 +2990,7 @@ phutil_register_library_map(array(
     'DrydockCommandInterface' => 'DrydockInterface',
     'DrydockController' => 'PhabricatorController',
     'DrydockDAO' => 'PhabricatorLiskDAO',
+    'DrydockFilesystemInterface' => 'DrydockInterface',
     'DrydockLease' => 'DrydockDAO',
     'DrydockLeaseListController' => 'DrydockController',
     'DrydockLeaseQuery' => 'PhabricatorOffsetPagedQuery',
@@ -3016,6 +3020,7 @@ phutil_register_library_map(array(
     'DrydockResourceQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'DrydockResourceStatus' => 'DrydockConstants',
     'DrydockResourceViewController' => 'DrydockController',
+    'DrydockSFTPFilesystemInterface' => 'DrydockFilesystemInterface',
     'DrydockSSHCommandInterface' => 'DrydockCommandInterface',
     'DrydockWebrootInterface' => 'DrydockInterface',
     'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation',
@@ -4973,6 +4978,7 @@ phutil_register_library_map(array(
     'SleepBuildStepImplementation' => 'BuildStepImplementation',
     'SlowvoteEmbedView' => 'AphrontView',
     'SlowvoteRemarkupRule' => 'PhabricatorRemarkupRuleObject',
+    'UploadArtifactBuildStepImplementation' => 'VariableBuildStepImplementation',
     'VariableBuildStepImplementation' => 'BuildStepImplementation',
   ),
 ));

src/applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php

@@ -105,6 +105,12 @@ final class DrydockPreallocatedHostBlueprintImplementation
             'port' => $resource->getAttribute('port'),
             'credential' => $resource->getAttribute('credential'),
             'platform' => $resource->getAttribute('platform')));
+      case 'filesystem':
+        return id(new DrydockSFTPFilesystemInterface())
+          ->setConfiguration(array(
+            'host' => $resource->getAttribute('host'),
+            'port' => $resource->getAttribute('port'),
+            'credential' => $resource->getAttribute('credential')));
     }
 
     throw new Exception("No interface of type '{$type}'.");

src/applications/drydock/interface/filesystem/DrydockFilesystemInterface.php

@@ -0,0 +1,24 @@
+<?php
+
+abstract class DrydockFilesystemInterface extends DrydockInterface {
+
+  final public function getInterfaceType() {
+    return 'filesystem';
+  }
+
+  /**
+   * Reads a file on the Drydock resource and returns the contents of the file.
+   */
+  abstract public function readFile($path);
+
+  /**
+   * Reads a file on the Drydock resource and saves it as a PhabricatorFile.
+   */
+  abstract public function saveFile($path, $name);
+
+  /**
+   * Writes a file to the Drydock resource.
+   */
+  abstract public function writeFile($path, $data);
+
+}

src/applications/drydock/interface/filesystem/DrydockSFTPFilesystemInterface.php

@@ -0,0 +1,63 @@
+<?php
+
+final class DrydockSFTPFilesystemInterface extends DrydockFilesystemInterface {
+
+  private $passphraseSSHKey;
+
+  private function openCredentialsIfNotOpen() {
+    if ($this->passphraseSSHKey !== null) {
+      return;
+    }
+
+    $credential = id(new PassphraseCredentialQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withIDs(array($this->getConfig('credential')))
+      ->needSecrets(true)
+      ->executeOne();
+
+    if ($credential->getProvidesType() !==
+      PassphraseCredentialTypeSSHPrivateKey::PROVIDES_TYPE) {
+      throw new Exception("Only private key credentials are supported.");
+    }
+
+    $this->passphraseSSHKey = PassphraseSSHKey::loadFromPHID(
+      $credential->getPHID(),
+      PhabricatorUser::getOmnipotentUser());
+  }
+
+  private function getExecFuture($path) {
+    $this->openCredentialsIfNotOpen();
+
+    return new ExecFuture(
+      'sftp -o "StrictHostKeyChecking no" -P %s -i %P %P@%s',
+      $this->getConfig('port'),
+      $this->passphraseSSHKey->getKeyfileEnvelope(),
+      $this->passphraseSSHKey->getUsernameEnvelope(),
+      $this->getConfig('host'));
+  }
+
+  public function readFile($path) {
+    $target = new TempFile();
+    $future = $this->getExecFuture($path);
+    $future->write(csprintf("get %s %s", $path, $target));
+    $future->resolvex();
+    return Filesystem::readFile($target);
+  }
+
+  public function saveFile($path, $name) {
+    $data = $this->readFile($path);
+    $file = PhabricatorFile::newFromFileData($data);
+    $file->setName($name);
+    $file->save();
+    return $file;
+  }
+
+  public function writeFile($path, $data) {
+    $source = new TempFile();
+    Filesystem::writeFile($source, $data);
+    $future = $this->getExecFuture($path);
+    $future->write(csprintf("put %s %s", $source, $path));
+    $future->resolvex();
+  }
+
+}

src/applications/drydock/query/DrydockBlueprintQuery.php

@@ -56,7 +56,7 @@ final class DrydockBlueprintQuery
     if ($this->phids) {
       $where[] = qsprintf(
         $conn_r,
-        'phid IN (%Ld)',
+        'phid IN (%Ls)',
         $this->phids);
     }
 

src/applications/harbormaster/step/CommandBuildStepImplementation.php

@@ -32,30 +32,9 @@ final class CommandBuildStepImplementation
       $settings['command'],
       $variables);
 
-    $artifact = id(new HarbormasterBuildArtifactQuery())
+    $artifact = $build->loadArtifact($settings['hostartifact']);
-      ->setViewer(PhabricatorUser::getOmnipotentUser())
-      ->withArtifactKeys(
-        $build->getPHID(),
-        array($settings['hostartifact']))
-      ->executeOne();
-    if ($artifact === null) {
-      throw new Exception("Associated Drydock host artifact not found!");
-    }
 
-    $data = $artifact->getArtifactData();
+    $lease = $artifact->loadDrydockLease();
-
-    // FIXME: Is there a better way of doing this?
-    $lease = id(new DrydockLease())->load(
-      $data['drydock-lease']);
-    if ($lease === null) {
-      throw new Exception("Associated Drydock lease not found!");
-    }
-    $resource = id(new DrydockResource())->load(
-      $lease->getResourceID());
-    if ($resource === null) {
-      throw new Exception("Associated Drydock resource not found!");
-    }
-    $lease->attachResource($resource);
 
     $interface = $lease->getInterface('command');
 

src/applications/harbormaster/step/UploadArtifactBuildStepImplementation.php

@@ -0,0 +1,97 @@
+<?php
+
+final class UploadArtifactBuildStepImplementation
+  extends VariableBuildStepImplementation {
+
+  public function getName() {
+    return pht('Upload Artifact');
+  }
+
+  public function getGenericDescription() {
+    return pht('Upload an artifact from a Drydock host to Phabricator.');
+  }
+
+  public function getDescription() {
+    $settings = $this->getSettings();
+
+    return pht(
+      'Upload artifact located at \'%s\' on \'%s\'.',
+      $settings['path'],
+      $settings['hostartifact']);
+  }
+
+  public function execute(
+    HarbormasterBuild $build,
+    HarbormasterBuildTarget $build_target) {
+
+    $settings = $this->getSettings();
+    $variables = $build_target->getVariables();
+
+    $path = $this->mergeVariables(
+      'vsprintf',
+      $settings['path'],
+      $variables);
+
+    $artifact = $build->loadArtifact($settings['hostartifact']);
+
+    $lease = $artifact->loadDrydockLease();
+
+    $interface = $lease->getInterface('filesystem');
+
+    // TODO: Handle exceptions.
+    $file = $interface->saveFile($path, $settings['name']);
+
+    // Insert the artifact record.
+    $artifact = $build->createArtifact(
+      $build_target,
+      $settings['name'],
+      HarbormasterBuildArtifact::TYPE_FILE);
+    $artifact->setArtifactData(array(
+      'filePHID' => $file->getPHID()));
+    $artifact->save();
+  }
+
+  public function validateSettings() {
+    $settings = $this->getSettings();
+
+    if ($settings['path'] === null || !is_string($settings['path'])) {
+      return false;
+    }
+    if ($settings['name'] === null || !is_string($settings['name'])) {
+      return false;
+    }
+    if ($settings['hostartifact'] === null ||
+      !is_string($settings['hostartifact'])) {
+      return false;
+    }
+
+    // TODO: Check if the host artifact is provided by previous build steps.
+
+    return true;
+  }
+
+  public function getSettingDefinitions() {
+    return array(
+      'path' => array(
+        'name' => 'Path',
+        'description' =>
+          'The path of the file that should be retrieved.  Note that on '.
+          'Windows machines running FreeSSHD, this path will be relative '.
+          'to the SFTP root path (configured under the SFTP tab).  You can '.
+          'not specify an absolute path for Windows machines.',
+        'type' => BuildStepImplementation::SETTING_TYPE_STRING),
+      'name' => array(
+        'name' => 'Local Name',
+        'description' =>
+          'The name for the file when it is stored in Phabricator.',
+        'type' => BuildStepImplementation::SETTING_TYPE_STRING),
+      'hostartifact' => array(
+        'name' => 'Host Artifact',
+        'description' =>
+          'The host artifact that determines what machine the command '.
+          'will run on.',
+        'type' => BuildStepImplementation::SETTING_TYPE_ARTIFACT,
+        'artifact_type' => HarbormasterBuildArtifact::TYPE_HOST));
+  }
+
+}

src/applications/harbormaster/storage/build/HarbormasterBuild.php

@@ -119,6 +119,19 @@ final class HarbormasterBuild extends HarbormasterDAO
     return $artifact;
   }
 
+  public function loadArtifact($name) {
+    $artifact = id(new HarbormasterBuildArtifactQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withArtifactKeys(
+        $this->getPHID(),
+        array($name))
+      ->executeOne();
+    if ($artifact === null) {
+      throw new Exception("Artifact not found!");
+    }
+    return $artifact;
+  }
+
   /**
    * Checks for and handles build cancellation.  If this method returns
    * true, the caller should stop any current operations and return control

src/applications/harbormaster/storage/build/HarbormasterBuildArtifact.php

@@ -72,6 +72,30 @@ final class HarbormasterBuildArtifact extends HarbormasterDAO
     }
   }
 
+  public function loadDrydockLease() {
+    if ($this->getArtifactType() !== self::TYPE_HOST) {
+      throw new Exception(
+        "`loadDrydockLease` may only be called on host artifacts.");
+    }
+
+    $data = $this->getArtifactData();
+
+    // FIXME: Is there a better way of doing this?
+    $lease = id(new DrydockLease())->load(
+      $data['drydock-lease']);
+    if ($lease === null) {
+      throw new Exception("Associated Drydock lease not found!");
+    }
+    $resource = id(new DrydockResource())->load(
+      $lease->getResourceID());
+    if ($resource === null) {
+      throw new Exception("Associated Drydock resource not found!");
+    }
+    $lease->attachResource($resource);
+
+    return $lease;
+  }
+
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */