Add a "Build with CircleCI" build step

Summary: Ref T9456. Some rough edges and we can't complete the build yet since I haven't written a webhook, but this mostly seems to be working.

Test Plan:
  - Ran this build on some stuff.
  - Ran a normal HTTP step build to make sure I didn't break that.

{F880301}

{F880302}

{F880303}

Reviewers: chad

Reviewed By: chad

Subscribers: JustinTulloss, joshma

Maniphest Tasks: T9456

Differential Revision: https://secure.phabricator.com/D14286

src/__phutil_library_map__.php

@@ -1106,6 +1106,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildableTransactionQuery' => 'applications/harbormaster/query/HarbormasterBuildableTransactionQuery.php',
     'HarbormasterBuildableViewController' => 'applications/harbormaster/controller/HarbormasterBuildableViewController.php',
     'HarbormasterBuiltinBuildStepGroup' => 'applications/harbormaster/stepgroup/HarbormasterBuiltinBuildStepGroup.php',
+    'HarbormasterCircleCIBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterCircleCIBuildStepImplementation.php',
+    'HarbormasterCircleCIBuildableInterface' => 'applications/harbormaster/interface/HarbormasterCircleCIBuildableInterface.php',
     'HarbormasterConduitAPIMethod' => 'applications/harbormaster/conduit/HarbormasterConduitAPIMethod.php',
     'HarbormasterController' => 'applications/harbormaster/controller/HarbormasterController.php',
     'HarbormasterCreateArtifactConduitAPIMethod' => 'applications/harbormaster/conduit/HarbormasterCreateArtifactConduitAPIMethod.php',
@@ -4517,6 +4519,7 @@ phutil_register_library_map(array(
       'DifferentialDAO',
       'PhabricatorPolicyInterface',
       'HarbormasterBuildableInterface',
+      'HarbormasterCircleCIBuildableInterface',
       'PhabricatorApplicationTransactionInterface',
       'PhabricatorDestructibleInterface',
     ),
@@ -5334,6 +5337,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildableTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
     'HarbormasterBuildableViewController' => 'HarbormasterController',
     'HarbormasterBuiltinBuildStepGroup' => 'HarbormasterBuildStepGroup',
+    'HarbormasterCircleCIBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterConduitAPIMethod' => 'ConduitAPIMethod',
     'HarbormasterController' => 'PhabricatorController',
     'HarbormasterCreateArtifactConduitAPIMethod' => 'HarbormasterConduitAPIMethod',
@@ -7652,6 +7656,7 @@ phutil_register_library_map(array(
       'PhabricatorSubscribableInterface',
       'PhabricatorMentionableInterface',
       'HarbormasterBuildableInterface',
+      'HarbormasterCircleCIBuildableInterface',
       'PhabricatorCustomFieldInterface',
       'PhabricatorApplicationTransactionInterface',
       'PhabricatorFulltextInterface',

src/applications/differential/storage/DifferentialDiff.php

@@ -5,6 +5,7 @@ final class DifferentialDiff
   implements
     PhabricatorPolicyInterface,
     HarbormasterBuildableInterface,
+    HarbormasterCircleCIBuildableInterface,
     PhabricatorApplicationTransactionInterface,
     PhabricatorDestructibleInterface {
 
@@ -524,6 +525,72 @@ final class DifferentialDiff
     );
   }
 
+
+/* -(  HarbormasterCircleCIBuildableInterface  )----------------------------- */
+
+
+  public function getCircleCIGitHubRepositoryURI() {
+    $diff_phid = $this->getPHID();
+    $repository_phid = $this->getRepositoryPHID();
+    if (!$repository_phid) {
+      throw new Exception(
+        pht(
+          'This diff ("%s") is not associated with a repository. A diff '.
+          'must belong to a tracked repository to be built by CircleCI.',
+          $diff_phid));
+    }
+
+    $repository = id(new PhabricatorRepositoryQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withPHIDs(array($repository_phid))
+      ->executeOne();
+    if (!$repository) {
+      throw new Exception(
+        pht(
+          'This diff ("%s") is associated with a repository ("%s") which '.
+          'could not be loaded.',
+          $diff_phid,
+          $repository_phid));
+    }
+
+    $staging_uri = $repository->getStagingURI();
+    if (!$staging_uri) {
+      throw new Exception(
+        pht(
+          'This diff ("%s") is associated with a repository ("%s") that '.
+          'does not have a Staging Area configured. You must configure a '.
+          'Staging Area to use CircleCI integration.',
+          $diff_phid,
+          $repository_phid));
+    }
+
+    $path = HarbormasterCircleCIBuildStepImplementation::getGitHubPath(
+      $staging_uri);
+    if (!$path) {
+      throw new Exception(
+        pht(
+          'This diff ("%s") is associated with a repository ("%s") that '.
+          'does not have a Staging Area ("%s") that is hosted on GitHub. '.
+          'CircleCI can only build from GitHub, so the Staging Area for '.
+          'the repository must be hosted there.',
+          $diff_phid,
+          $repository_phid,
+          $staging_uri));
+    }
+
+    return $staging_uri;
+  }
+
+  public function getCircleCIBuildIdentifierType() {
+    return 'tag';
+  }
+
+  public function getCircleCIBuildIdentifier() {
+    $ref = $this->getStagingRef();
+    $ref = preg_replace('(^refs/tags/)', '', $ref);
+    return $ref;
+  }
+
   public function getStagingRef() {
     // TODO: We're just hoping to get lucky. Instead, `arc` should store
     // where it sent changes and we should only provide staging details

src/applications/harbormaster/controller/HarbormasterStepEditController.php

@@ -136,13 +136,19 @@ final class HarbormasterStepEditController
     }
 
     $form = id(new AphrontFormView())
-      ->setUser($viewer)
-      ->appendChild(
-        id(new AphrontFormTextControl())
-          ->setName('name')
-          ->setLabel(pht('Name'))
-          ->setError($e_name)
-          ->setValue($v_name));
+      ->setUser($viewer);
+
+    $instructions = $implementation->getEditInstructions();
+    if (strlen($instructions)) {
+      $form->appendRemarkupInstructions($instructions);
+    }
+
+    $form->appendChild(
+      id(new AphrontFormTextControl())
+        ->setName('name')
+        ->setLabel(pht('Name'))
+        ->setError($e_name)
+        ->setValue($v_name));
 
     $form->appendChild(id(new AphrontFormDividerControl()));
 

src/applications/harbormaster/interface/HarbormasterCircleCIBuildableInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+/**
+ * Support for CircleCI.
+ */
+interface HarbormasterCircleCIBuildableInterface {
+
+  public function getCircleCIGitHubRepositoryURI();
+  public function getCircleCIBuildIdentifierType();
+  public function getCircleCIBuildIdentifier();
+
+}

src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php

@@ -69,6 +69,10 @@ abstract class HarbormasterBuildStepImplementation extends Phobject {
     return $this->getGenericDescription();
   }
 
+  public function getEditInstructions() {
+    return null;
+  }
+
   /**
    * Run the build target against the specified build.
    */
@@ -265,6 +269,37 @@ abstract class HarbormasterBuildStepImplementation extends Phobject {
 
   }
 
+  protected function logHTTPResponse(
+    HarbormasterBuild $build,
+    HarbormasterBuildTarget $build_target,
+    BaseHTTPFuture $future,
+    $label) {
+
+    list($status, $body, $headers) = $future->resolve();
+
+    $header_lines = array();
+
+    // TODO: We don't currently preserve the entire "HTTP" response header, but
+    // should. Once we do, reproduce it here faithfully.
+    $status_code = $status->getStatusCode();
+    $header_lines[] = "HTTP {$status_code}";
+
+    foreach ($headers as $header) {
+      list($head, $tail) = $header;
+      $header_lines[] = "{$head}: {$tail}";
+    }
+    $header_lines = implode("\n", $header_lines);
+
+    $build_target
+      ->newLog($label, 'http.head')
+      ->append($header_lines);
+
+    $build_target
+      ->newLog($label, 'http.body')
+      ->append($body);
+  }
+
+
 
 /* -(  Automatic Targets  )-------------------------------------------------- */
 

src/applications/harbormaster/step/HarbormasterCircleCIBuildStepImplementation.php

@@ -0,0 +1,246 @@
+<?php
+
+final class HarbormasterCircleCIBuildStepImplementation
+  extends HarbormasterBuildStepImplementation {
+
+  public function getName() {
+    return pht('Build with CircleCI');
+  }
+
+  public function getGenericDescription() {
+    return pht('Trigger a build in CircleCI.');
+  }
+
+  public function getBuildStepGroupKey() {
+    return HarbormasterExternalBuildStepGroup::GROUPKEY;
+  }
+
+  public function getDescription() {
+    return pht('Run a build in CircleCI.');
+  }
+
+  public function getEditInstructions() {
+    return pht(<<<EOTEXT
+WARNING: This build step is new and experimental!
+
+To build **revisions** with CircleCI, they must:
+
+  - belong to a tracked repository;
+  - the repository must have a Staging Area configured;
+  - the Staging Area must be hosted on GitHub; and
+  - you must configure the webhook described below.
+
+To build **commits** with CircleCI, they must:
+
+  - belong to a repository that is being imported from GitHub; and
+  - you must configure the webhook described below.
+
+Webhook Configuration
+=====================
+
+IMPORTANT: This has not been implemented yet.
+
+Environment
+===========
+
+These variables will be available in the build environment:
+
+| Variable | Description |
+|----------|-------------|
+| `HARBORMASTER_BUILD_TARGET_PHID` | PHID of the Build Target.
+
+EOTEXT
+    );
+  }
+
+  public static function getGitHubPath($uri) {
+    $uri_object = new PhutilURI($uri);
+    $domain = $uri_object->getDomain();
+
+    if (!strlen($domain)) {
+      $uri_object = new PhutilGitURI($uri);
+      $domain = $uri_object->getDomain();
+    }
+
+    $domain = phutil_utf8_strtolower($domain);
+    switch ($domain) {
+      case 'github.com':
+      case 'www.github.com':
+        return $uri_object->getPath();
+      default:
+        return null;
+    }
+  }
+
+  public function execute(
+    HarbormasterBuild $build,
+    HarbormasterBuildTarget $build_target) {
+    $viewer = PhabricatorUser::getOmnipotentUser();
+
+    $buildable = $build->getBuildable();
+
+    $object = $buildable->getBuildableObject();
+    $object_phid = $object->getPHID();
+    if (!($object instanceof HarbormasterCircleCIBuildableInterface)) {
+      throw new Exception(
+        pht(
+          'Object ("%s") does not implement interface "%s". Only objects '.
+          'which implement this interface can be built with CircleCI.',
+          $object_phid,
+          'HarbormasterCircleCIBuildableInterface'));
+    }
+
+    $github_uri = $object->getCircleCIGitHubRepositoryURI();
+    $build_type = $object->getCircleCIBuildIdentifierType();
+    $build_identifier = $object->getCircleCIBuildIdentifier();
+
+    $path = self::getGitHubPath($github_uri);
+    if ($path === null) {
+      throw new Exception(
+        pht(
+          'Object ("%s") claims "%s" is a GitHub repository URI, but the '.
+          'domain does not appear to be GitHub.',
+          $object_phid,
+          $github_uri));
+    }
+
+    $path_parts = trim($path, '/');
+    $path_parts = explode('/', $path_parts);
+    if (count($path_parts) < 2) {
+      throw new Exception(
+        pht(
+          'Object ("%s") claims "%s" is a GitHub repository URI, but the '.
+          'path ("%s") does not have enough components (expected at least '.
+          'two).',
+          $object_phid,
+          $github_uri,
+          $path));
+    }
+
+    list($github_namespace, $github_name) = $path_parts;
+    $github_name = preg_replace('(\\.git$)', '', $github_name);
+
+    $credential_phid = $this->getSetting('token');
+    $api_token = id(new PassphraseCredentialQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($credential_phid))
+      ->needSecrets(true)
+      ->executeOne();
+    if (!$api_token) {
+      throw new Exception(
+        pht(
+          'Unable to load API token ("%s")!',
+          $credential_phid));
+    }
+
+    // When we pass "revision", the branch is ignored (and does not even need
+    // to exist), and only shows up in the UI. Use a cute string which will
+    // certainly never break anything or cause any kind of problem.
+    $ship = "\xF0\x9F\x9A\xA2";
+    $branch = "{$ship}Harbormaster";
+
+    $token = $api_token->getSecret()->openEnvelope();
+    $parts = array(
+      'https://circleci.com/api/v1/project',
+      phutil_escape_uri($github_namespace),
+      phutil_escape_uri($github_name)."?circle-token={$token}",
+    );
+
+    $uri = implode('/', $parts);
+
+    $data_structure = array();
+    switch ($build_type) {
+      case 'tag':
+        $data_structure['tag'] = $build_identifier;
+        break;
+      case 'revision':
+        $data_structure['revision'] = $build_identifier;
+        break;
+      default:
+        throw new Exception(
+          pht(
+            'Unknown CircleCI build type "%s". Expected "%s" or "%s".',
+            $build_type,
+            'tag',
+            'revision'));
+    }
+
+    $data_structure['build_parameters'] = array(
+      'HARBORMASTER_BUILD_TARGET_PHID' => $build_target->getPHID(),
+    );
+
+    $json_data = phutil_json_encode($data_structure);
+
+    $future = id(new HTTPSFuture($uri, $json_data))
+      ->setMethod('POST')
+      ->addHeader('Content-Type', 'application/json')
+      ->addHeader('Accept', 'application/json')
+      ->setTimeout(60);
+
+    $this->resolveFutures(
+      $build,
+      $build_target,
+      array($future));
+
+    $this->logHTTPResponse($build, $build_target, $future, pht('CircleCI'));
+
+    list($status, $body) = $future->resolve();
+    if ($status->isError()) {
+      throw new HarbormasterBuildFailureException();
+    }
+
+    $response = phutil_json_decode($body);
+    $build_uri = idx($response, 'build_url');
+    if (!$build_uri) {
+      throw new Exception(
+        pht(
+          'CircleCI did not return a "%s"!',
+          'build_url'));
+    }
+
+    $target_phid = $build_target->getPHID();
+
+    // Write an artifact to create a link to the external build in CircleCI.
+
+    $api_method = 'harbormaster.createartifact';
+    $api_params = array(
+      'buildTargetPHID' => $target_phid,
+      'artifactType' => HarbormasterURIArtifact::ARTIFACTCONST,
+      'artifactKey' => 'circleci.uri',
+      'artifactData' => array(
+        'uri' => $build_uri,
+        'name' => pht('View in CircleCI'),
+        'ui.external' => true,
+      ),
+    );
+
+    id(new ConduitCall($api_method, $api_params))
+      ->setUser($viewer)
+      ->execute();
+  }
+
+  public function getFieldSpecifications() {
+    return array(
+      'token' => array(
+        'name' => pht('API Token'),
+        'type' => 'credential',
+        'credential.type'
+          => PassphraseTokenCredentialType::CREDENTIAL_TYPE,
+        'credential.provides'
+          => PassphraseTokenCredentialType::PROVIDES_TYPE,
+        'required' => true,
+      ),
+    );
+  }
+
+  public function supportsWaitForMessage() {
+    // NOTE: We always wait for a message, but don't need to show the UI
+    // control since "Wait" is the only valid choice.
+    return false;
+  }
+
+  public function shouldWaitForMessage(HarbormasterBuildTarget $target) {
+    return true;
+  }
+
+}

src/applications/harbormaster/step/HarbormasterHTTPRequestBuildStepImplementation.php

@@ -72,29 +72,9 @@ final class HarbormasterHTTPRequestBuildStepImplementation
       $build_target,
       array($future));
 
-    list($status, $body, $headers) = $future->resolve();
-
-    $header_lines = array();
-
-    // TODO: We don't currently preserve the entire "HTTP" response header, but
-    // should. Once we do, reproduce it here faithfully.
-    $status_code = $status->getStatusCode();
-    $header_lines[] = "HTTP {$status_code}";
-
-    foreach ($headers as $header) {
-      list($head, $tail) = $header;
-      $header_lines[] = "{$head}: {$tail}";
-    }
-    $header_lines = implode("\n", $header_lines);
-
-    $build_target
-      ->newLog($uri, 'http.head')
-      ->append($header_lines);
-
-    $build_target
-      ->newLog($uri, 'http.body')
-      ->append($body);
+    $this->logHTTPResponse($build, $build_target, $future, $uri);
 
+    list($status) = $future->resolve();
     if ($status->isError()) {
       throw new HarbormasterBuildFailureException();
     }

src/applications/repository/storage/PhabricatorRepositoryCommit.php

@@ -10,6 +10,7 @@ final class PhabricatorRepositoryCommit
     PhabricatorSubscribableInterface,
     PhabricatorMentionableInterface,
     HarbormasterBuildableInterface,
+    HarbormasterCircleCIBuildableInterface,
     PhabricatorCustomFieldInterface,
     PhabricatorApplicationTransactionInterface,
     PhabricatorFulltextInterface {
@@ -411,6 +412,52 @@ final class PhabricatorRepositoryCommit
   }
 
 
+/* -(  HarbormasterCircleCIBuildableInterface  )----------------------------- */
+
+
+  public function getCircleCIGitHubRepositoryURI() {
+    $repository = $this->getRepository();
+
+    $commit_phid = $this->getPHID();
+    $repository_phid = $repository->getPHID();
+
+    if ($repository->isHosted()) {
+      throw new Exception(
+        pht(
+          'This commit ("%s") is associated with a hosted repository '.
+          '("%s"). Repositories must be imported from GitHub to be built '.
+          'with CircleCI.',
+          $commit_phid,
+          $repository_phid));
+    }
+
+    $remote_uri = $repository->getRemoteURI();
+    $path = HarbormasterCircleCIBuildStepImplementation::getGitHubPath(
+      $remote_uri);
+    if (!$path) {
+      throw new Exception(
+        pht(
+          'This commit ("%s") is associated with a repository ("%s") that '.
+          'with a remote URI ("%s") that does not appear to be hosted on '.
+          'GitHub. Repositories must be hosted on GitHub to be built with '.
+          'CircleCI.',
+          $commit_phid,
+          $repository_phid,
+          $remote_uri));
+    }
+
+    return $remote_uri;
+  }
+
+  public function getCircleCIBuildIdentifierType() {
+    return 'revision';
+  }
+
+  public function getCircleCIBuildIdentifier() {
+    return $this->getCommitIdentifier();
+  }
+
+
 /* -(  PhabricatorCustomFieldInterface  )------------------------------------ */