Integrate Harbormaster with Buildkite

Summary: Ref T12173. This might need some additional work but the basics seem like they're in good shape.

Test Plan:
  - Buildkite is "bring your own hardware", so you need to launch a host to test anything.
  - Launched a host in AWS.
  - Configured Buildkite to use that host to run builds.
  - Added a Buildkite build step to a new Harbormaster build plan.
  - Used `bin/harbormaster build ...` to run the plan.
  - Saw buildkite execute builds and report status back to Harbormaster

{F2553076}

{F2553077}

Reviewers: chad

Reviewed By: chad

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T12173

Differential Revision: https://secure.phabricator.com/D17270

src/__phutil_library_map__.php

@@ -1210,6 +1210,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildableTransactionEditor' => 'applications/harbormaster/editor/HarbormasterBuildableTransactionEditor.php',
     'HarbormasterBuildableTransactionQuery' => 'applications/harbormaster/query/HarbormasterBuildableTransactionQuery.php',
     'HarbormasterBuildableViewController' => 'applications/harbormaster/controller/HarbormasterBuildableViewController.php',
+    'HarbormasterBuildkiteBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterBuildkiteBuildStepImplementation.php',
+    'HarbormasterBuildkiteHookController' => 'applications/harbormaster/controller/HarbormasterBuildkiteHookController.php',
     'HarbormasterBuiltinBuildStepGroup' => 'applications/harbormaster/stepgroup/HarbormasterBuiltinBuildStepGroup.php',
     'HarbormasterCircleCIBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterCircleCIBuildStepImplementation.php',
     'HarbormasterCircleCIBuildableInterface' => 'applications/harbormaster/interface/HarbormasterCircleCIBuildableInterface.php',
@@ -6017,6 +6019,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildableTransactionEditor' => 'PhabricatorApplicationTransactionEditor',
     'HarbormasterBuildableTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
     'HarbormasterBuildableViewController' => 'HarbormasterController',
+    'HarbormasterBuildkiteBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
+    'HarbormasterBuildkiteHookController' => 'HarbormasterController',
     'HarbormasterBuiltinBuildStepGroup' => 'HarbormasterBuildStepGroup',
     'HarbormasterCircleCIBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterCircleCIHookController' => 'HarbormasterController',

src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php

@@ -94,6 +94,7 @@ final class PhabricatorHarbormasterApplication extends PhabricatorApplication {
         ),
         'hook/' => array(
           'circleci/' => 'HarbormasterCircleCIHookController',
+          'buildkite/' => 'HarbormasterBuildkiteHookController',
         ),
       ),
     );

src/applications/harbormaster/controller/HarbormasterBuildkiteHookController.php

@@ -0,0 +1,111 @@
+<?php
+
+final class HarbormasterBuildkiteHookController
+  extends HarbormasterController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  /**
+   * @phutil-external-symbol class PhabricatorStartup
+   */
+  public function handleRequest(AphrontRequest $request) {
+    $raw_body = PhabricatorStartup::getRawInput();
+    $body = phutil_json_decode($raw_body);
+
+    $event = idx($body, 'event');
+    if ($event != 'build.finished') {
+      return $this->newHookResponse(pht('OK: Ignored event.'));
+    }
+
+    $build = idx($body, 'build');
+    if (!is_array($build)) {
+      throw new Exception(
+        pht(
+          'Expected "%s" property to contain a dictionary.',
+          'build'));
+    }
+
+    $meta_data = idx($build, 'meta_data');
+    if (!is_array($meta_data)) {
+      throw new Exception(
+        pht(
+          'Expected "%s" property to contain a dictionary.',
+          'build.meta_data'));
+    }
+
+    $target_phid = idx($meta_data, 'buildTargetPHID');
+    if (!$target_phid) {
+      return $this->newHookResponse(pht('OK: No Harbormaster target PHID.'));
+    }
+
+    $viewer = PhabricatorUser::getOmnipotentUser();
+    $target = id(new HarbormasterBuildTargetQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($target_phid))
+      ->needBuildSteps(true)
+      ->executeOne();
+    if (!$target) {
+      throw new Exception(
+        pht(
+          'Harbormaster build target "%s" does not exist.',
+          $target_phid));
+    }
+
+    $step = $target->getBuildStep();
+    $impl = $step->getStepImplementation();
+    if (!($impl instanceof HarbormasterBuildkiteBuildStepImplementation)) {
+      throw new Exception(
+        pht(
+          'Harbormaster build target "%s" is not a Buildkite build step. '.
+          'Only Buildkite steps may be updated via the Buildkite hook.',
+          $target_phid));
+    }
+
+    $webhook_token = $impl->getSetting('webhook.token');
+    $request_token = $request->getHTTPHeader('X-Buildkite-Token');
+
+    if (!phutil_hashes_are_identical($webhook_token, $request_token)) {
+      throw new Exception(
+        pht(
+          'Buildkite request to target "%s" had the wrong authentication '.
+          'token. The Buildkite pipeline and Harbormaster build step must '.
+          'be configured with the same token.',
+          $target_phid));
+    }
+
+    $state = idx($build, 'state');
+    switch ($state) {
+      case 'passed':
+        $message_type = HarbormasterMessageType::MESSAGE_PASS;
+        break;
+      default:
+        $message_type = HarbormasterMessageType::MESSAGE_FAIL;
+        break;
+    }
+
+    $api_method = 'harbormaster.sendmessage';
+    $api_params = array(
+      'buildTargetPHID' => $target_phid,
+      'type' => $message_type,
+    );
+
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+
+      id(new ConduitCall($api_method, $api_params))
+        ->setUser($viewer)
+        ->execute();
+
+    unset($unguarded);
+
+    return $this->newHookResponse(pht('OK: Processed event.'));
+  }
+
+  private function newHookResponse($message) {
+    $response = new AphrontWebpageResponse();
+    $response->setContent($message);
+    return $response;
+  }
+
+}

src/applications/harbormaster/step/HarbormasterBuildkiteBuildStepImplementation.php

@@ -0,0 +1,210 @@
+<?php
+
+final class HarbormasterBuildkiteBuildStepImplementation
+  extends HarbormasterBuildStepImplementation {
+
+  public function getName() {
+    return pht('Build with Buildkite');
+  }
+
+  public function getGenericDescription() {
+    return pht('Trigger a build in Buildkite.');
+  }
+
+  public function getBuildStepGroupKey() {
+    return HarbormasterExternalBuildStepGroup::GROUPKEY;
+  }
+
+  public function getDescription() {
+    return pht('Run a build in Buildkite.');
+  }
+
+  public function getEditInstructions() {
+    $hook_uri = '/harbormaster/hook/buildkite/';
+    $hook_uri = PhabricatorEnv::getProductionURI($hook_uri);
+
+    return pht(<<<EOTEXT
+WARNING: This build step is new and experimental!
+
+To build **revisions** with Buildkite, they must:
+
+  - belong to a tracked repository;
+  - the repository must have a Staging Area configured;
+  - you must configure a Buildkite pipeline for that Staging Area; and
+  - you must configure the webhook described below.
+
+To build **commits** with Buildkite, they must:
+
+  - belong to a tracked repository;
+  - you must configure a Buildkite pipeline for that repository; and
+  - you must configure the webhook described below.
+
+Webhook Configuration
+=====================
+
+In {nav Settings} for your Organization in Buildkite, under
+{nav Notification Services}, add a new **Webook Notification**.
+
+Use these settings:
+
+  - **Webhook URL**: %s
+  - **Token**: The "Webhook Token" field below and the "Token" field in
+    Buildkite should both be set to the same nonempty value (any random
+    secret). You can use copy/paste the value Buildkite generates into
+    this form.
+  - **Events**: Only **build.finish** needs to be active.
+
+Environment
+===========
+
+These variables will be available in the build environment:
+
+| Variable | Description |
+|----------|-------------|
+| `HARBORMASTER_BUILD_TARGET_PHID` | PHID of the Build Target.
+EOTEXT
+    ,
+    $hook_uri);
+  }
+
+  public function execute(
+    HarbormasterBuild $build,
+    HarbormasterBuildTarget $build_target) {
+    $viewer = PhabricatorUser::getOmnipotentUser();
+
+    $buildable = $build->getBuildable();
+
+    $object = $buildable->getBuildableObject();
+    if (!($object instanceof HarbormasterCircleCIBuildableInterface)) {
+      throw new Exception(
+        pht('This object does not support builds with Buildkite.'));
+    }
+
+    $organization = $this->getSetting('organization');
+    $pipeline = $this->getSetting('pipeline');
+
+    $uri = urisprintf(
+      'https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds',
+      $organization,
+      $pipeline);
+
+    $data_structure = array(
+      'commit' => $object->getCircleCIBuildIdentifier(),
+      'branch' => 'master',
+      'message' => pht(
+        'Harbormaster Build %s ("%s") for %s',
+        $build->getID(),
+        $build->getName(),
+        $buildable->getMonogram()),
+      'env' => array(
+        'HARBORMASTER_BUILD_TARGET_PHID' => $build_target->getPHID(),
+      ),
+      'meta_data' => array(
+        'buildTargetPHID' => $build_target->getPHID(),
+      ),
+    );
+
+    $json_data = phutil_json_encode($data_structure);
+
+    $credential_phid = $this->getSetting('token');
+    $api_token = id(new PassphraseCredentialQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($credential_phid))
+      ->needSecrets(true)
+      ->executeOne();
+    if (!$api_token) {
+      throw new Exception(
+        pht(
+          'Unable to load API token ("%s")!',
+          $credential_phid));
+    }
+
+    $token = $api_token->getSecret()->openEnvelope();
+
+    $future = id(new HTTPSFuture($uri, $json_data))
+      ->setMethod('POST')
+      ->addHeader('Content-Type', 'application/json')
+      ->addHeader('Accept', 'application/json')
+      ->addHeader('Authorization', "Bearer {$token}")
+      ->setTimeout(60);
+
+    $this->resolveFutures(
+      $build,
+      $build_target,
+      array($future));
+
+    $this->logHTTPResponse($build, $build_target, $future, pht('Buildkite'));
+
+    list($status, $body) = $future->resolve();
+    if ($status->isError()) {
+      throw new HarbormasterBuildFailureException();
+    }
+
+    $response = phutil_json_decode($body);
+
+    $uri_key = 'web_url';
+    $build_uri = idx($response, $uri_key);
+    if (!$build_uri) {
+      throw new Exception(
+        pht(
+          'Buildkite did not return a "%s"!',
+          $uri_key));
+    }
+
+    $target_phid = $build_target->getPHID();
+
+    $api_method = 'harbormaster.createartifact';
+    $api_params = array(
+      'buildTargetPHID' => $target_phid,
+      'artifactType' => HarbormasterURIArtifact::ARTIFACTCONST,
+      'artifactKey' => 'buildkite.uri',
+      'artifactData' => array(
+        'uri' => $build_uri,
+        'name' => pht('View in Buildkite'),
+        'ui.external' => true,
+      ),
+    );
+
+    id(new ConduitCall($api_method, $api_params))
+      ->setUser($viewer)
+      ->execute();
+  }
+
+  public function getFieldSpecifications() {
+    return array(
+      'token' => array(
+        'name' => pht('API Token'),
+        'type' => 'credential',
+        'credential.type'
+          => PassphraseTokenCredentialType::CREDENTIAL_TYPE,
+        'credential.provides'
+          => PassphraseTokenCredentialType::PROVIDES_TYPE,
+        'required' => true,
+      ),
+      'organization' => array(
+        'name' => pht('Organization Name'),
+        'type' => 'text',
+        'required' => true,
+      ),
+      'pipeline' => array(
+        'name' => pht('Pipeline Name'),
+        'type' => 'text',
+        'required' => true,
+      ),
+      'webhook.token' => array(
+        'name' => pht('Webhook Token'),
+        'type' => 'text',
+        'required' => true,
+      ),
+    );
+  }
+
+  public function supportsWaitForMessage() {
+    return false;
+  }
+
+  public function shouldWaitForMessage(HarbormasterBuildTarget $target) {
+    return true;
+  }
+
+}