Allow external systems to send messages to build targets

Summary:
Ref T1049. Allows external systems to send a message to a build target. The primary intended use case is:

  - You make an HTTP request to Jenkins.
  - The build goes into a "waiting" state.
  - Later, Jenkins calls `harbormaster.sendmessage` to report that the target passed or failed.
  - The build continues as appropriate.

This is deceptively complicated because:

  - There are a lot of race concerns. We might get a message back from an external system before it even responds to the request we made. We want to make sure we process these messages no matter when we receive them.
  - These messages need to be sent to a build target (vs a build or buildable) because we'll get into trouble with parallelization later on otherwise (Jenkins is told to do 3 builds; we can't tell which ones failed or what overall state is unless the message are sent to targets).
  - I initially thought about implementing this as a separate "Wait for a response from an external system" build step. This gets a lot more complicated for users once we do parallelization, though. Particularly, in the case where you've told Jenkins to do 3 builds, the three "wait" steps need to know which target they're waiting for (and jenkins needs to know some unique identifier for each target). So this pretty much boils down to a more complicated, more error-prone version of using target PHIDs.

This makes the already-muddy Build UI a bit worse, but it needs a general clarity pass anyway (it's showing way too much uninteresting data, and should show a better summary of results instead).

Test Plan:
  - This doesn't really do anything interesting yet.
  - Used Conduit to send messages to build plans.
  - Viewed the messages on the build screen.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1049

Differential Revision: https://secure.phabricator.com/D8604

resources/sql/autopatches/20140323.harbor.2.message.sql

@@ -0,0 +1,10 @@
+CREATE TABLE {$NAMESPACE}_harbormaster.harbormaster_buildmessage (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  authorPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  buildTargetPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  type VARCHAR(16) NOT NULL,
+  isConsumed BOOL NOT NULL,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  KEY `key_buildtarget` (buildTargetPHID)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;

src/__phutil_library_map__.php

@@ -189,6 +189,8 @@ phutil_register_library_map(array(
     'ConduitAPI_flag_delete_Method' => 'applications/flag/conduit/ConduitAPI_flag_delete_Method.php',
     'ConduitAPI_flag_edit_Method' => 'applications/flag/conduit/ConduitAPI_flag_edit_Method.php',
     'ConduitAPI_flag_query_Method' => 'applications/flag/conduit/ConduitAPI_flag_query_Method.php',
+    'ConduitAPI_harbormaster_Method' => 'applications/harbormaster/conduit/ConduitAPI_harbormaster_Method.php',
+    'ConduitAPI_harbormaster_sendmessage_Method' => 'applications/harbormaster/conduit/ConduitAPI_harbormaster_sendmessage_Method.php',
     'ConduitAPI_macro_Method' => 'applications/macro/conduit/ConduitAPI_macro_Method.php',
     'ConduitAPI_macro_creatememe_Method' => 'applications/macro/conduit/ConduitAPI_macro_creatememe_Method.php',
     'ConduitAPI_macro_query_Method' => 'applications/macro/conduit/ConduitAPI_macro_query_Method.php',
@@ -701,6 +703,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildItemQuery' => 'applications/harbormaster/query/HarbormasterBuildItemQuery.php',
     'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
+    'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php',
+    'HarbormasterBuildMessageQuery' => 'applications/harbormaster/query/HarbormasterBuildMessageQuery.php',
     'HarbormasterBuildPlan' => 'applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php',
     'HarbormasterBuildPlanEditor' => 'applications/harbormaster/editor/HarbormasterBuildPlanEditor.php',
     'HarbormasterBuildPlanQuery' => 'applications/harbormaster/query/HarbormasterBuildPlanQuery.php',
@@ -2743,6 +2747,8 @@ phutil_register_library_map(array(
     'ConduitAPI_flag_delete_Method' => 'ConduitAPI_flag_Method',
     'ConduitAPI_flag_edit_Method' => 'ConduitAPI_flag_Method',
     'ConduitAPI_flag_query_Method' => 'ConduitAPI_flag_Method',
+    'ConduitAPI_harbormaster_Method' => 'ConduitAPIMethod',
+    'ConduitAPI_harbormaster_sendmessage_Method' => 'ConduitAPI_harbormaster_Method',
     'ConduitAPI_macro_Method' => 'ConduitAPIMethod',
     'ConduitAPI_macro_creatememe_Method' => 'ConduitAPI_macro_Method',
     'ConduitAPI_macro_query_Method' => 'ConduitAPI_macro_Method',
@@ -3297,6 +3303,12 @@ phutil_register_library_map(array(
       1 => 'PhabricatorPolicyInterface',
     ),
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'HarbormasterBuildMessage' =>
+    array(
+      0 => 'HarbormasterDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'HarbormasterBuildMessageQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'HarbormasterBuildPlan' =>
     array(
       0 => 'HarbormasterDAO',

src/applications/harbormaster/conduit/ConduitAPI_harbormaster_Method.php

@@ -0,0 +1,18 @@
+<?php
+
+abstract class ConduitAPI_harbormaster_Method extends ConduitAPIMethod {
+
+  public function getApplication() {
+    return PhabricatorApplication::getByClass(
+      'PhabricatorApplicationHarbormaster');
+  }
+
+  public function getMethodStatus() {
+    return self::METHOD_STATUS_UNSTABLE;
+  }
+
+  public function getMethodStatusDescription() {
+    return pht('All Harbormaster APIs are new and subject to change.');
+  }
+
+}

src/applications/harbormaster/conduit/ConduitAPI_harbormaster_sendmessage_Method.php

@@ -0,0 +1,49 @@
+<?php
+
+final class ConduitAPI_harbormaster_sendmessage_Method
+  extends ConduitAPI_harbormaster_Method {
+
+  public function getMethodDescription() {
+    return pht(
+      'Send a message to a build target, notifying it of results in an '.
+      'external system.');
+  }
+
+  public function defineParamTypes() {
+    return array(
+      'buildTargetPHID' => 'phid',
+      'type'            => 'enum<pass, fail>',
+    );
+  }
+
+  public function defineReturnType() {
+    return 'void';
+  }
+
+  public function defineErrorTypes() {
+    return array();
+  }
+
+  protected function execute(ConduitAPIRequest $request) {
+    $viewer = $request->getUser();
+
+    $build_target_phid = $request->getValue('buildTargetPHID');
+    $message_type = $request->getValue('type');
+
+    $build_target = id(new HarbormasterBuildTargetQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($build_target_phid))
+      ->executeOne();
+    if (!$build_target) {
+      throw new Exception(pht('No such build target!'));
+    }
+
+    $message = HarbormasterBuildMessage::initializeNewMessage($viewer)
+      ->setBuildTargetPHID($build_target->getPHID())
+      ->setType($message_type)
+      ->save();
+
+    return null;
+  }
+
+}

src/applications/harbormaster/controller/HarbormasterBuildViewController.php

@@ -55,6 +55,17 @@ final class HarbormasterBuildViewController
       ->withBuildPHIDs(array($build->getPHID()))
       ->execute();
 
+
+    if ($build_targets) {
+      $messages = id(new HarbormasterBuildMessageQuery())
+        ->setViewer($viewer)
+        ->withBuildTargetPHIDs(mpull($build_targets, 'getPHID'))
+        ->execute();
+      $messages = mgroup($messages, 'getBuildTargetPHID');
+    } else {
+      $messages = array();
+    }
+
     $targets = array();
     foreach ($build_targets as $build_target) {
       $header = id(new PHUIHeaderView())
@@ -85,6 +96,9 @@ final class HarbormasterBuildViewController
         ->setHeader($header)
         ->addPropertyList($properties);
 
+      $build_messages = idx($messages, $build_target->getPHID(), array());
+      $targets[] = $this->buildMessages($build_messages);
+
       $targets[] = $this->buildArtifacts($build_target);
       $targets[] = $this->buildLog($build, $build_target);
     }
@@ -316,4 +330,55 @@ final class HarbormasterBuildViewController
     }
   }
 
+  private function buildMessages(array $messages) {
+    $viewer = $this->getRequest()->getUser();
+
+    if ($messages) {
+      $handles = id(new PhabricatorHandleQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(mpull($messages, 'getAuthorPHID'))
+        ->execute();
+    } else {
+      $handles = array();
+    }
+
+    $rows = array();
+    foreach ($messages as $message) {
+      $rows[] = array(
+        $message->getID(),
+        $handles[$message->getAuthorPHID()]->renderLink(),
+        $message->getType(),
+        $message->getIsConsumed() ? pht('Consumed') : null,
+        phabricator_datetime($message->getDateCreated(), $viewer),
+      );
+    }
+
+    $table = new AphrontTableView($rows);
+    $table->setNoDataString(pht('No messages for this build target.'));
+    $table->setHeaders(
+      array(
+        pht('ID'),
+        pht('From'),
+        pht('Type'),
+        pht('Consumed'),
+        pht('Received'),
+      ));
+    $table->setColumnClasses(
+      array(
+        '',
+        '',
+        'wide',
+        '',
+        'date',
+      ));
+
+    $box = id(new PHUIObjectBoxView())
+      ->setHeaderText(pht('Build Target Messages'))
+      ->appendChild($table);
+
+    return $box;
+  }
+
+
+
 }

src/applications/harbormaster/query/HarbormasterBuildMessageQuery.php

@@ -0,0 +1,98 @@
+<?php
+
+final class HarbormasterBuildMessageQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $ids;
+  private $buildTargetPHIDs;
+  private $consumed;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withBuildTargetPHIDs(array $phids) {
+    $this->buildTargetPHIDs = $phids;
+    return $this;
+  }
+
+  public function withConsumed($consumed) {
+    $this->consumed = $consumed;
+    return $this;
+  }
+
+  protected function loadPage() {
+    $table = new HarbormasterBuildMessage();
+    $conn_r = $table->establishConnection('r');
+
+    $data = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T %Q %Q %Q',
+      $table->getTableName(),
+      $this->buildWhereClause($conn_r),
+      $this->buildOrderClause($conn_r),
+      $this->buildLimitClause($conn_r));
+
+    return $table->loadAllFromArray($data);
+  }
+
+  protected function willFilterPage(array $page) {
+    $build_target_phids = array_filter(mpull($page, 'getBuildTargetPHID'));
+    if ($build_target_phids) {
+      $build_targets = id(new PhabricatorObjectQuery())
+        ->setViewer($this->getViewer())
+        ->withPHIDs($build_target_phids)
+        ->setParentQuery($this)
+        ->execute();
+      $build_targets = mpull($build_targets, null, 'getPHID');
+    } else {
+      $build_targets = array();
+    }
+
+    foreach ($page as $key => $message) {
+      $build_target_phid = $message->getBuildTargetPHID();
+      if (empty($build_targets[$build_target_phid])) {
+        unset($page[$key]);
+        continue;
+      }
+      $message->attachBuildTarget($build_targets[$build_target_phid]);
+    }
+
+    return $page;
+  }
+
+  private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
+    $where = array();
+
+    if ($this->ids) {
+      $where[] = qsprintf(
+        $conn_r,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->buildTargetPHIDs) {
+      $where[] = qsprintf(
+        $conn_r,
+        'buildTargetPHID IN (%Ls)',
+        $this->buildTargetPHIDs);
+    }
+
+    if ($this->consumed !== null) {
+      $where[] = qsprintf(
+        $conn_r,
+        'isConsumed = %d',
+        (int)$this->isConsumed);
+    }
+
+    $where[] = $this->buildPagingClause($conn_r);
+
+    return $this->formatWhereClause($where);
+  }
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorApplicationHarbormaster';
+  }
+
+}

src/applications/harbormaster/storage/HarbormasterBuildMessage.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * A message sent to an executing build target by an external system. We
+ * capture these messages and process them asynchronously to avoid race
+ * conditions where we receive a message before a build plan is ready to
+ * accept it.
+ */
+final class HarbormasterBuildMessage extends HarbormasterDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $authorPHID;
+  protected $buildTargetPHID;
+  protected $type;
+  protected $isConsumed;
+
+  private $buildTarget = self::ATTACHABLE;
+
+  public static function initializeNewMessage(PhabricatorUser $actor) {
+    return id(new HarbormasterBuildMessage())
+      ->setAuthorPHID($actor->getPHID())
+      ->setIsConsumed(0);
+  }
+
+  public function getBuildTarget() {
+    return $this->assertAttached($this->buildTarget);
+  }
+
+  public function attachBuildTarget(HarbormasterBuildTarget $target) {
+    $this->buildTarget = $target;
+    return $this;
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return $this->getBuildTarget()->getPolicy($capability);
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return $this->getBuildTarget()->hasAutomaticCapability(
+      $capability,
+      $viewer);
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return pht('Build messages have the same policies as their targets.');
+  }
+
+}

src/applications/harbormaster/storage/build/HarbormasterBuild.php

@@ -168,7 +168,8 @@ final class HarbormasterBuild extends HarbormasterDAO
       'repository.vcs' => null,
       'repository.uri' => null,
       'step.timestamp' => null,
-      'build.id' => null);
+      'build.id' => null,
+    );
 
     $buildable = $this->getBuildable();
     $object = $buildable->getBuildableObject();
@@ -210,7 +211,9 @@ final class HarbormasterBuild extends HarbormasterDAO
       'repository.uri' =>
         pht('The URI to clone or checkout the repository from.'),
       'step.timestamp' => pht('The current UNIX timestamp.'),
-      'build.id' => pht('The ID of the current build.'));
+      'build.id' => pht('The ID of the current build.'),
+      'target.phid' => pht('The PHID of the current build target.'),
+    );
   }
 
   public function isComplete() {

src/applications/harbormaster/storage/build/HarbormasterBuildTarget.php

@@ -73,6 +73,10 @@ final class HarbormasterBuildTarget extends HarbormasterDAO
     return $this;
   }
 
+  public function getVariables() {
+    return parent::getVariables() + $this->getBuildTargetVariables();
+  }
+
   public function getVariable($key, $default = null) {
     return idx($this->variables, $key, $default);
   }
@@ -93,6 +97,12 @@ final class HarbormasterBuildTarget extends HarbormasterDAO
     return $this->implementation;
   }
 
+  private function getBuildTargetVariables() {
+    return array(
+      'target.phid' => $this->getPHID(),
+    );
+  }
+
 
 /* -(  Status  )------------------------------------------------------------- */