Add HarbormasterRunnerWorker, for running CI tests

Summary:
This is very preliminary and doesn't actually do anything useful. In theory, it uses Drydock to check out a working copy and run tests. In practice, it's not actually capable of running any of our tests (because of complicated interdependency stuff), but does check out a working copy and //try// to run tests there.

Adds various sorts of utility methods to various things as well.

Test Plan: Ran `reparse.php --harbormaster --trace <commit>`, observed attempt to run tests via Drydock.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2015, T1049

Differential Revision: https://secure.phabricator.com/D4215

scripts/repository/reparse.php

@@ -55,6 +55,10 @@ $args->parse(
                     'delete existing relationship entries between your '.
                     'package and some old commits!)',
     ),
+    array(
+      'name'     => 'harbormaster',
+      'help'     => 'EXPERIMENTAL. Execute Harbormaster.',
+    ),
     // misc options
     array(
       'name'     => 'force',
@@ -73,6 +77,7 @@ $reparse_message = $args->getArg('message');
 $reparse_change = $args->getArg('change');
 $reparse_herald = $args->getArg('herald');
 $reparse_owners = $args->getArg('owners');
+$reparse_harbormaster = $args->getArg('harbormaster');
 $reparse_what = $args->getArg('revision');
 $force = $args->getArg('force');
 $force_local = $args->getArg('force-local');
@@ -83,9 +88,9 @@ if (!$all_from_repo && !$reparse_what) {
 }
 
 if (!$reparse_message && !$reparse_change && !$reparse_herald &&
-    !$reparse_owners) {
+    !$reparse_owners && !$reparse_harbormaster) {
   usage("Specify what information to reparse with --message, --change,  ".
-        "--herald, and/or --owners");
+        "--herald, --harbormaster, and/or --owners");
 }
 if ($reparse_owners && !$force) {
   echo phutil_console_wrap(
@@ -202,6 +207,10 @@ foreach ($commits as $commit) {
     $classes[] = 'PhabricatorRepositoryCommitOwnersWorker';
   }
 
+  if ($reparse_harbormaster) {
+    $classes[] = 'HarbormasterRunnerWorker';
+  }
+
   $spec = array(
     'commitID'  => $commit->getID(),
     'only'      => true,
@@ -218,7 +227,7 @@ foreach ($commits as $commit) {
     foreach ($classes as $class) {
       $worker = newv($class, array($spec));
       echo "Running '{$class}'...\n";
-      $worker->doWork();
+      $worker->executeTask();
     }
   }
 }

src/__phutil_library_map__.php

@@ -456,6 +456,7 @@ phutil_register_library_map(array(
     'FeedPublisherWorker' => 'applications/feed/worker/FeedPublisherWorker.php',
     'HarbormasterDAO' => 'applications/harbormaster/storage/HarbormasterDAO.php',
     'HarbormasterObject' => 'applications/harbormaster/storage/HarbormasterObject.php',
+    'HarbormasterRunnerWorker' => 'applications/harbormaster/worker/HarbormasterRunnerWorker.php',
     'HarbormasterScratchTable' => 'applications/harbormaster/storage/HarbormasterScratchTable.php',
     'HeraldAction' => 'applications/herald/storage/HeraldAction.php',
     'HeraldActionConfig' => 'applications/herald/config/HeraldActionConfig.php',
@@ -1744,6 +1745,7 @@ phutil_register_library_map(array(
     'FeedPublisherWorker' => 'PhabricatorWorker',
     'HarbormasterDAO' => 'PhabricatorLiskDAO',
     'HarbormasterObject' => 'HarbormasterDAO',
+    'HarbormasterRunnerWorker' => 'PhabricatorWorker',
     'HarbormasterScratchTable' => 'HarbormasterDAO',
     'HeraldAction' => 'HeraldDAO',
     'HeraldApplyTranscript' => 'HeraldDAO',

src/applications/drydock/blueprint/DrydockBlueprint.php

@@ -22,6 +22,17 @@ abstract class DrydockBlueprint {
     return get_class($this);
   }
 
+  protected function loadLease($lease_id) {
+    $lease = id(new DrydockLease())->load($lease_id);
+    if (!$lease) {
+      throw new Exception("No such lease '{$lease_id}'!");
+    }
+
+    $resource = $lease->loadResource();
+    $lease->attachResource($resource);
+    return $lease;
+  }
+
 
 /* -(  Lease Acquisition  )-------------------------------------------------- */
 

src/applications/drydock/blueprint/DrydockWorkingCopyBlueprint.php

@@ -86,6 +86,13 @@ final class DrydockWorkingCopyBlueprint extends DrydockBlueprint {
     DrydockLease $lease,
     $type) {
 
+    switch ($type) {
+      case 'command':
+        return $this
+          ->loadLease($resource->getAttribute('lease.host'))
+          ->getInterface($type);
+    }
+
     throw new Exception("No interface of type '{$type}'.");
   }
 

src/applications/drydock/interface/command/DrydockCommandInterface.php

@@ -2,6 +2,17 @@
 
 abstract class DrydockCommandInterface extends DrydockInterface {
 
+  private $workingDirectory;
+
+  public function setWorkingDirectory($working_directory) {
+    $this->workingDirectory = $working_directory;
+    return $this;
+  }
+
+  public function getWorkingDirectory() {
+    return $this->workingDirectory;
+  }
+
   final public function getInterfaceType() {
     return 'command';
   }
@@ -24,4 +35,17 @@ abstract class DrydockCommandInterface extends DrydockInterface {
 
   abstract public function getExecFuture($command);
 
+  protected function applyWorkingDirectoryToArgv(array $argv) {
+    if ($this->getWorkingDirectory() !== null) {
+      $cmd = $argv[0];
+      $cmd = "(cd %s; {$cmd})";
+      $argv = array_merge(
+        array($cmd),
+        array($this->getWorkingDirectory()),
+        array_slice($argv, 1));
+    }
+
+    return $argv;
+  }
+
 }

src/applications/drydock/interface/command/DrydockLocalCommandInterface.php

@@ -4,6 +4,8 @@ final class DrydockLocalCommandInterface extends DrydockCommandInterface {
 
   public function getExecFuture($command) {
     $argv = func_get_args();
+    $argv = $this->applyWorkingDirectoryToArgv($argv);
+
     return newv('ExecFuture', $argv);
   }
 

src/applications/drydock/interface/command/DrydockSSHCommandInterface.php

@@ -4,6 +4,8 @@ final class DrydockSSHCommandInterface extends DrydockCommandInterface {
 
   public function getExecFuture($command) {
     $argv = func_get_args();
+    $argv = $this->applyWorkingDirectoryToArgv($argv);
+
     $full_command = call_user_func_array('csprintf', $argv);
 
     // NOTE: The "-t -t" is for psuedo-tty allocation so we can "sudo" on some

src/applications/drydock/storage/DrydockLease.php

@@ -11,6 +11,26 @@ final class DrydockLease extends DrydockDAO {
   protected $taskID;
 
   private $resource;
+  private $releaseOnDestruction;
+
+  /**
+   * Flag this lease to be released when its destructor is called. This is
+   * mostly useful if you have a script which acquires, uses, and then releases
+   * a lease, as you don't need to explicitly handle exceptions to properly
+   * release the lease.
+   */
+  public function releaseOnDestruction() {
+    $this->releaseOnDestruction = true;
+    return $this;
+  }
+
+  public function __destruct() {
+    if ($this->releaseOnDestruction) {
+      if ($this->isActive()) {
+        $this->release();
+      }
+    }
+  }
 
   public function getLeaseName() {
     return pht('Lease %d', $this->getID());
@@ -103,9 +123,17 @@ final class DrydockLease extends DrydockDAO {
     return $this;
   }
 
+  private function isActive() {
+    switch ($this->status) {
+      case DrydockLeaseStatus::STATUS_ACTIVE:
+      case DrydockLeaseStatus::STATUS_ACQUIRING:
+        return true;
+    }
+    return false;
+  }
+
   private function assertActive() {
-    if (($this->status != DrydockLeaseStatus::STATUS_ACTIVE) &&
+    if (!$this->isActive()) {
-        ($this->status != DrydockLeaseStatus::STATUS_ACQUIRING)) {
       throw new Exception(
         "Lease is not active! You can not interact with resources through ".
         "an inactive lease.");

src/applications/harbormaster/worker/HarbormasterRunnerWorker.php

@@ -0,0 +1,52 @@
+<?php
+
+final class HarbormasterRunnerWorker extends PhabricatorWorker {
+
+  public function getRequiredLeaseTime() {
+    return 60 * 60 * 24;
+  }
+
+  protected function doWork() {
+    $data = $this->getTaskData();
+    $id = idx($data, 'commitID');
+
+    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+      'id = %d',
+      $id);
+
+    if (!$commit) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        "Commit '{$id}' does not exist!");
+    }
+
+    $repository = id(new PhabricatorRepository())->loadOneWhere(
+      'id = %d',
+      $commit->getRepositoryID());
+
+    if (!$repository) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        "Unable to load repository for commit '{$id}'!");
+    }
+
+    $lease = id(new DrydockLease())
+      ->setResourceType('working-copy')
+      ->setAttributes(
+        array(
+          'repositoryID' => $repository->getID(),
+          'commit' => $commit->getCommitIdentifier(),
+        ))
+      ->releaseOnDestruction()
+      ->waitUntilActive();
+
+    $cmd = $lease->getInterface('command');
+    list($json) = $cmd
+      ->setWorkingDirectory($lease->getResource()->getAttribute('path'))
+      ->execx('arc unit --everything --json');
+    $lease->release();
+
+    // TODO: Do something actually useful with this. Requires Harbormaster
+    // buildout.
+    echo $json;
+  }
+
+}