Give Buildables a status, populate it, and return it over Conduit

Summary:
Ref T4809. Currently, buildables have a status field but nothing populates it. Populate it:

  - When builds change state, update the Buildable state.
  - Use the new Buildable state on the web UI.
  - Return the new Buildable state from Conduit.

To make it easier to debug/test this:

  - Provide `bin/harbormaster update Bxxx ...` to force foreground update of a Buildable.

Test Plan:
  - Used `bin/harbormaster update Bxxx --force --trace` to update buildables.
  - Looked at buidlable list, saw statuses reported properly.
  - Used Conduit to read statuses.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T4809

Differential Revision: https://secure.phabricator.com/D8799

src/__phutil_library_map__.php

@@ -744,6 +744,7 @@ phutil_register_library_map(array(
     'HarbormasterHTTPRequestBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterHTTPRequestBuildStepImplementation.php',
     'HarbormasterLeaseHostBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseHostBuildStepImplementation.php',
     'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
+    'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
     'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
     'HarbormasterObject' => 'applications/harbormaster/storage/HarbormasterObject.php',
     'HarbormasterPHIDTypeBuild' => 'applications/harbormaster/phid/HarbormasterPHIDTypeBuild.php',
@@ -3411,6 +3412,7 @@ phutil_register_library_map(array(
     'HarbormasterHTTPRequestBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterLeaseHostBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
+    'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'HarbormasterObject' => 'HarbormasterDAO',
     'HarbormasterPHIDTypeBuild' => 'PhabricatorPHIDType',

src/applications/harbormaster/conduit/ConduitAPI_harbormaster_querybuildables_Method.php

@@ -64,11 +64,16 @@ final class ConduitAPI_harbormaster_querybuildables_Method
     foreach ($buildables as $buildable) {
       $monogram = $buildable->getMonogram();
 
+      $status = $buildable->getBuildableStatus();
+      $status_name = HarbormasterBuildable::getBuildableStatusName($status);
+
       $data[] = array(
         'id' => $buildable->getID(),
         'phid' => $buildable->getPHID(),
         'monogram' => $monogram,
         'uri' => PhabricatorEnv::getProductionURI('/'.$monogram),
+        'buildableStatus' => $status,
+        'buildableStatusName' => $status_name,
         'buildablePHID' => $buildable->getBuildablePHID(),
         'containerPHID' => $buildable->getContainerPHID(),
         'isManualBuildable' => (bool)$buildable->getIsManualBuildable(),

src/applications/harbormaster/conduit/ConduitAPI_harbormaster_sendmessage_Method.php

@@ -45,10 +45,11 @@ final class ConduitAPI_harbormaster_sendmessage_Method
 
     // If the build has completely paused because all steps are blocked on
     // waiting targets, this will resume it.
-    id(new HarbormasterBuildEngine())
+    PhabricatorWorker::scheduleTask(
-      ->setViewer($viewer)
+      'HarbormasterBuildWorker',
-      ->setBuild($build_target->getBuild())
+      array(
-      ->continueBuild();
+        'buildID' => $build_target->getBuild()->getID(),
+      ));
 
     return null;
   }

src/applications/harbormaster/controller/HarbormasterBuildableListController.php

@@ -54,28 +54,13 @@ final class HarbormasterBuildableListController
 
       $list->addItem($item);
 
-
+      switch ($buildable->getBuildableStatus()) {
-
+        case HarbormasterBuildable::STATUS_PASSED:
-      // TODO: This is proof-of-concept for getting meaningful status
-      // information into this list, and should get an improvement pass
-      // once we're a little farther along.
-
-      $all_pass = true;
-      $any_fail = false;
-      foreach ($buildable->getBuilds() as $build) {
-        if ($build->getBuildStatus() != HarbormasterBuild::STATUS_PASSED) {
-          $all_pass = false;
-        }
-        if ($build->getBuildStatus() == HarbormasterBuild::STATUS_FAILED ||
-            $build->getBuildStatus() == HarbormasterBuild::STATUS_ERROR) {
-          $any_fail = true;
-        }
-      }
-
-      if ($any_fail) {
-        $item->setBarColor('red');
-      } else if ($all_pass) {
           $item->setBarColor('green');
+          break;
+        case HarbormasterBuildable::STATUS_FAILED:
+          $item->setBarColor('red');
+          break;
       }
     }
 

src/applications/harbormaster/engine/HarbormasterBuildEngine.php

@@ -9,6 +9,16 @@ final class HarbormasterBuildEngine extends Phobject {
   private $build;
   private $viewer;
   private $newBuildTargets = array();
+  private $forceBuildableUpdate;
+
+  public function setForceBuildableUpdate($force_buildable_update) {
+    $this->forceBuildableUpdate = $force_buildable_update;
+    return $this;
+  }
+
+  public function shouldForceBuildableUpdate() {
+    return $this->forceBuildableUpdate;
+  }
 
   public function queueNewBuildTarget(HarbormasterBuildTarget $target) {
     $this->newBuildTargets[] = $target;
@@ -44,6 +54,7 @@ final class HarbormasterBuildEngine extends Phobject {
     $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
 
     $build->reload();
+    $old_status = $build->getBuildStatus();
 
     try {
       $this->updateBuild($build);
@@ -69,6 +80,13 @@ final class HarbormasterBuildEngine extends Phobject {
           'targetID' => $target->getID(),
         ));
     }
+
+    // If the build changed status, we might need to update the overall status
+    // on the buildable.
+    $new_status = $build->getBuildStatus();
+    if ($new_status != $old_status || $this->shouldForceBuildableUpdate()) {
+      $this->updateBuildable($build->getBuildable());
+    }
   }
 
   private function updateBuild(HarbormasterBuild $build) {
@@ -330,4 +348,53 @@ final class HarbormasterBuildEngine extends Phobject {
     }
   }
 
+
+  /**
+   * Update the overall status of the buildable this build is attached to.
+   *
+   * After a build changes state (for example, passes or fails) it may affect
+   * the overall state of the associated buildable. Compute the new aggregate
+   * state and save it on the buildable.
+   *
+   * @param   HarbormasterBuild The buildable to update.
+   * @return  void
+   */
+  private function updateBuildable(HarbormasterBuildable $buildable) {
+    $lock_key = 'harbormaster.buildable:'.$buildable->getID();
+    $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
+
+    $buildable = id(new HarbormasterBuildableQuery())
+      ->setViewer($this->getViewer())
+      ->withIDs(array($buildable->getID()))
+      ->needBuilds(true)
+      ->executeOne();
+
+    $all_pass = true;
+    $any_fail = false;
+    foreach ($buildable->getBuilds() as $build) {
+      if ($build->getBuildStatus() != HarbormasterBuild::STATUS_PASSED) {
+        $all_pass = false;
+      }
+      if ($build->getBuildStatus() == HarbormasterBuild::STATUS_FAILED ||
+          $build->getBuildStatus() == HarbormasterBuild::STATUS_ERROR) {
+        $any_fail = true;
+      }
+    }
+
+    if ($any_fail) {
+      $new_status = HarbormasterBuildable::STATUS_FAILED;
+    } else if ($all_pass) {
+      $new_status = HarbormasterBuildable::STATUS_PASSED;
+    } else {
+      $new_status = HarbormasterBuildable::STATUS_BUILDING;
+    }
+
+    if ($buildable->getBuildableStatus() != $new_status) {
+      $buildable->setBuildableStatus($new_status);
+      $buildable->save();
+    }
+
+    $lock->unlock();
+  }
+
 }

src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php

@@ -28,7 +28,7 @@ final class HarbormasterManagementBuildWorkflow
     $names = $args->getArg('buildable');
     if (count($names) != 1) {
       throw new PhutilArgumentUsageException(
-        pht('Specify exactly one buildable, by object name.'));
+        pht('Specify exactly one buildable object, by object name.'));
     }
 
     $name = head($names);

src/applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php

@@ -0,0 +1,114 @@
+<?php
+
+final class HarbormasterManagementUpdateWorkflow
+  extends HarbormasterManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('update')
+      ->setExamples('**update** [__options__] __buildable__')
+      ->setSynopsis(pht('Explicitly update the builds for __buildable__.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'build',
+            'param' => 'id',
+            'help' => pht('Update only this build.'),
+          ),
+          array(
+            'name' => 'force',
+            'help' => pht(
+              'Force the buildable to update even if no build status '.
+              'changes occur during normal update.'),
+          ),
+          array(
+            'name' => 'background',
+            'help' => pht(
+              'If updating generates tasks, queue them for the daemons '.
+              'instead of executing them in this process.'),
+          ),
+          array(
+            'name'        => 'buildable',
+            'wildcard'    => true,
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $force_update = $args->getArg('force');
+
+    $names = $args->getArg('buildable');
+    if (count($names) != 1) {
+      throw new PhutilArgumentUsageException(
+        pht('Specify exactly one buildable, by object name.'));
+    }
+
+    $buildable = id(new PhabricatorObjectQuery())
+      ->setViewer($viewer)
+      ->withNames($names)
+      ->executeOne();
+
+    if (!$buildable) {
+      throw new PhutilArgumentUsageException(
+        pht('No such buildable "%s"!', head($names)));
+    }
+
+    if (!($buildable instanceof HarbormasterBuildable)) {
+      throw new PhutilArgumentUsageException(
+        pht('Object "%s" is not a Harbormaster Buildable!', head($names)));
+    }
+
+    // Reload the buildable directly to get builds.
+    $buildable = id(new HarbormasterBuildableQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($buildable->getID()))
+      ->needBuilds(true)
+      ->executeOne();
+
+    $builds = $buildable->getBuilds();
+    $builds = mpull($builds, null, 'getID');
+
+    $build_id = $args->getArg('build');
+    if ($build_id) {
+      $builds = array_select_keys($builds, array($build_id));
+      if (!$builds) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'The specified buildable does not have a build with ID "%s".',
+            $build_id));
+      }
+    }
+
+    $console = PhutilConsole::getConsole();
+
+    if (!$args->getArg('background')) {
+      PhabricatorWorker::setRunAllTasksInProcess(true);
+    }
+
+    foreach ($builds as $build) {
+      $console->writeOut(
+        "%s\n",
+        pht(
+          'Updating build %d of buildable %s...',
+          $build->getID(),
+          $buildable->getMonogram()));
+
+      $engine = id(new HarbormasterBuildEngine())
+        ->setViewer($viewer)
+        ->setBuild($build);
+
+      if ($force_update) {
+        $engine->setForceBuildableUpdate(true);
+      }
+
+      $engine->continueBuild();
+    }
+
+    $console->writeOut("%s\n", pht('Done.'));
+
+    return 0;
+  }
+
+}

src/applications/harbormaster/query/HarbormasterBuildableSearchEngine.php

@@ -52,8 +52,7 @@ final class HarbormasterBuildableSearchEngine
   public function buildQueryFromSavedQuery(PhabricatorSavedQuery $saved) {
     $query = id(new HarbormasterBuildableQuery())
       ->needContainerHandles(true)
-      ->needBuildableHandles(true)
+      ->needBuildableHandles(true);
-      ->needBuilds(true);
 
     $container_phids = $saved->getParameter('containerPHIDs', array());
     if ($container_phids) {

src/applications/harbormaster/storage/HarbormasterBuildable.php

@@ -16,12 +16,27 @@ final class HarbormasterBuildable extends HarbormasterDAO
   private $containerHandle = self::ATTACHABLE;
   private $builds = self::ATTACHABLE;
 
-  const STATUS_WHATEVER = 'whatever';
+  const STATUS_BUILDING = 'building';
+  const STATUS_PASSED = 'passed';
+  const STATUS_FAILED = 'failed';
+
+  public static function getBuildableStatusName($status) {
+    switch ($status) {
+      case self::STATUS_BUILDING:
+        return pht('Building');
+      case self::STATUS_PASSED:
+        return pht('Passed');
+      case self::STATUS_FAILED:
+        return pht('Failed');
+      default:
+        return pht('Unknown');
+    }
+  }
 
   public static function initializeNewBuildable(PhabricatorUser $actor) {
     return id(new HarbormasterBuildable())
       ->setIsManualBuildable(0)
-      ->setBuildableStatus(self::STATUS_WHATEVER);
+      ->setBuildableStatus(self::STATUS_BUILDING);
   }
 
   public function getMonogram() {