Allow WorkingCopy resources to have multiple working copies

Summary:
Ref T9252. For building Phabricator itself, we need to have `libphutil/`, `arcanist/` and `phabricator/` next to one another on disk.

Expand the Drydock WorkingCopy resource so that it can have multiple repositories if the caller needs them.

I'm not sure if I'm going to put the actual config for this in Harbormaster or Drydock yet, but the WorkingCopy resource itself should work the same way in either case.

Test Plan: Restarted a Harbormaster build which leases a working copy, saw it build as expected.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9252

Differential Revision: https://secure.phabricator.com/D14180

src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php

@@ -37,10 +37,37 @@ final class DrydockWorkingCopyBlueprintImplementation
     DrydockResource $resource,
     DrydockLease $lease) {
 
-    $have_phid = $resource->getAttribute('repositoryPHID');
-    $need_phid = $lease->getAttribute('repositoryPHID');
+    $need_map = $lease->getAttribute('repositories.map');
+    if (!is_array($need_map)) {
+      return false;
+    }
 
-    if ($need_phid !== $have_phid) {
+    $have_map = $resource->getAttribute('repositories.map');
+    if (!is_array($have_map)) {
+      return false;
+    }
+
+    $have_as = ipull($have_map, 'phid');
+    $need_as = ipull($need_map, 'phid');
+
+    foreach ($need_as as $need_directory => $need_phid) {
+      if (empty($have_as[$need_directory])) {
+        // This resource is missing a required working copy.
+        return false;
+      }
+
+      if ($have_as[$need_directory] != $need_phid) {
+        // This resource has a required working copy, but it contains
+        // the wrong repository.
+        return false;
+      }
+
+      unset($have_as[$need_directory]);
+    }
+
+    if ($have_as && $lease->getAttribute('repositories.strict')) {
+      // This resource has extra repositories, but the lease is strict about
+      // which repositories are allowed to exist.
       return false;
     }
 
@@ -70,14 +97,9 @@ final class DrydockWorkingCopyBlueprintImplementation
     DrydockBlueprint $blueprint,
     DrydockLease $lease) {
 
-    $repository_phid = $lease->getAttribute('repositoryPHID');
-    $repository = $this->loadRepository($repository_phid);
-
     $resource = $this->newResourceTemplate(
       $blueprint,
-      pht(
-        'Working Copy (%s)',
-        $repository->getCallsign()));
+      pht('Working Copy'));
 
     $resource_phid = $resource->getPHID();
 
@@ -90,8 +112,17 @@ final class DrydockWorkingCopyBlueprintImplementation
     // TODO: Add some limits to the number of working copies we can have at
     // once?
 
+    $map = $lease->getAttribute('repositories.map');
+    foreach ($map as $key => $value) {
+      $map[$key] = array_select_keys(
+        $value,
+        array(
+          'phid',
+        ));
+    }
+
     return $resource
-      ->setAttribute('repositoryPHID', $repository->getPHID())
+      ->setAttribute('repositories.map', $map)
       ->setAttribute('host.leasePHID', $host_lease->getPHID())
       ->allocateResource();
   }
@@ -103,26 +134,32 @@ final class DrydockWorkingCopyBlueprintImplementation
     $lease = $this->loadHostLease($resource);
     $this->requireActiveLease($lease);
 
-    $repository_phid = $resource->getAttribute('repositoryPHID');
-    $repository = $this->loadRepository($repository_phid);
-    $repository_id = $repository->getID();
-
     $command_type = DrydockCommandInterface::INTERFACE_TYPE;
     $interface = $lease->getInterface($command_type);
 
     // TODO: Make this configurable.
     $resource_id = $resource->getID();
     $root = "/var/drydock/workingcopy-{$resource_id}";
-    $path = "{$root}/repo/{$repository_id}/";
 
+    $map = $resource->getAttribute('repositories.map');
+
+    $repositories = $this->loadRepositories(ipull($map, 'phid'));
+    foreach ($map as $directory => $spec) {
+      // TODO: Validate directory isn't goofy like "/etc" or "../../lol"
+      // somewhere?
+
+      $repository = $repositories[$spec['phid']];
+      $path = "{$root}/repo/{$directory}/";
+
+      // TODO: Run these in parallel?
       $interface->execx(
         'git clone -- %s %s',
         (string)$repository->getCloneURIObject(),
         $path);
+    }
 
     $resource
       ->setAttribute('workingcopy.root', $root)
-      ->setAttribute('workingcopy.path', $path)
       ->activateResource();
   }
 
@@ -151,18 +188,26 @@ final class DrydockWorkingCopyBlueprintImplementation
     DrydockResource $resource,
     DrydockLease $lease) {
 
+    $host_lease = $this->loadHostLease($resource);
     $command_type = DrydockCommandInterface::INTERFACE_TYPE;
-    $interface = $lease->getInterface($command_type);
+    $interface = $host_lease->getInterface($command_type);
 
+    $map = $lease->getAttribute('repositories.map');
+    $root = $resource->getAttribute('workingcopy.root');
+
+    $default = null;
+    foreach ($map as $directory => $spec) {
       $cmd = array();
       $arg = array();
 
+      $cmd[] = 'cd %s';
+      $arg[] = "{$root}/repo/{$directory}/";
+
       $cmd[] = 'git clean -d --force';
-    $cmd[] = 'git reset --hard HEAD';
       $cmd[] = 'git fetch';
 
-    $commit = $lease->getAttribute('commit');
-    $branch = $lease->getAttribute('branch');
+      $commit = idx($spec, 'commit');
+      $branch = idx($spec, 'branch');
 
       if ($commit !== null) {
         $cmd[] = 'git reset --hard %s';
@@ -170,6 +215,8 @@ final class DrydockWorkingCopyBlueprintImplementation
       } else if ($branch !== null) {
         $cmd[] = 'git reset --hard %s';
         $arg[] = $branch;
+      } else {
+        $cmd[] = 'git reset --hard HEAD';
       }
 
       $cmd = implode(' && ', $cmd);
@@ -179,6 +226,18 @@ final class DrydockWorkingCopyBlueprintImplementation
         array($interface, 'execx'),
         $argv);
 
+      if (idx($spec, 'default')) {
+        $default = $directory;
+      }
+    }
+
+    if ($default === null) {
+      $default = head_key($map);
+    }
+
+    // TODO: Use working storage?
+    $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/");
+
     $lease->activateOnResource($resource);
   }
 
@@ -217,26 +276,31 @@ final class DrydockWorkingCopyBlueprintImplementation
         $host_lease = $this->loadHostLease($resource);
         $command_interface = $host_lease->getInterface($type);
 
-        $path = $resource->getAttribute('workingcopy.path');
+        $path = $lease->getAttribute('workingcopy.default');
         $command_interface->setWorkingDirectory($path);
 
         return $command_interface;
     }
   }
 
-  private function loadRepository($repository_phid) {
-    $repository = id(new PhabricatorRepositoryQuery())
+  private function loadRepositories(array $phids) {
+    $repositories = id(new PhabricatorRepositoryQuery())
       ->setViewer(PhabricatorUser::getOmnipotentUser())
-      ->withPHIDs(array($repository_phid))
-      ->executeOne();
-    if (!$repository) {
+      ->withPHIDs($phids)
+      ->execute();
+    $repositories = mpull($repositories, null, 'getPHID');
+
+    foreach ($phids as $phid) {
+      if (empty($repositories[$phid])) {
         // TODO: Permanent failure.
         throw new Exception(
           pht(
             'Repository PHID "%s" does not exist.',
-          $repository_phid));
+            $phid));
+      }
     }
 
+    foreach ($repositories as $repository) {
       switch ($repository->getVersionControlSystem()) {
         case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
           break;
@@ -244,8 +308,9 @@ final class DrydockWorkingCopyBlueprintImplementation
           // TODO: Permanent failure.
           throw new Exception(pht('Unsupported VCS!'));
       }
+    }
 
-    return $repository;
+    return $repositories;
   }
 
   private function loadHostLease(DrydockResource $resource) {

src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php

@@ -45,14 +45,9 @@ final class HarbormasterLeaseWorkingCopyBuildStepImplementation
         ->setResourceType($working_copy_type)
         ->setOwnerPHID($build_target->getPHID());
 
-      $variables = $build_target->getVariables();
+      $map = $this->buildRepositoryMap($build_target);
 
-      $repository_phid = idx($variables, 'repository.phid');
-      $commit = idx($variables, 'repository.commit');
-
-      $lease
-        ->setAttribute('repositoryPHID', $repository_phid)
-        ->setAttribute('commit', $commit);
+      $lease->setAttribute('repositories.map', $map);
 
       $task_id = $this->getCurrentWorkerTaskID();
       if ($task_id) {
@@ -108,4 +103,35 @@ final class HarbormasterLeaseWorkingCopyBuildStepImplementation
     );
   }
 
+  private function buildRepositoryMap(HarbormasterBuildTarget $build_target) {
+    $viewer = PhabricatorUser::getOmnipotentUser();
+    $variables = $build_target->getVariables();
+
+    $repository_phid = idx($variables, 'repository.phid');
+
+    $repository = id(new PhabricatorRepositoryQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($repository_phid))
+      ->executeOne();
+    if (!$repository) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht(
+          'Unable to load repository with PHID "%s".',
+          $repository_phid));
+    }
+
+    $commit = idx($variables, 'repository.commit');
+
+    $map = array();
+
+    $directory = $repository->getCloneName();
+    $map[$directory] = array(
+      'phid' => $repository->getPHID(),
+      'commit' => $commit,
+      'default' => true,
+    );
+
+    return $map;
+  }
+
 }