Show additional status information during repository import

Summary:
Ref T2350. Fixes T2231.

  - Adds log flags around discovery.
  - Adds message flags for "needs update". This is basically an out-of-band hint to the daemons that a repository should be pulled sooner than normal. We set the flag when users push a revision, and expose a Conduit method that `arc land` will be able to use.

Test Plan: See screenshots.

Reviewers: btrahan, chad

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2350, T2231

Differential Revision: https://secure.phabricator.com/D7467

src/__phutil_library_map__.php

@@ -159,6 +159,7 @@ phutil_register_library_map(array(
     'ConduitAPI_diffusion_getrecentcommitsbypath_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_getrecentcommitsbypath_Method.php',
     'ConduitAPI_diffusion_historyquery_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_historyquery_Method.php',
     'ConduitAPI_diffusion_lastmodifiedquery_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_lastmodifiedquery_Method.php',
+    'ConduitAPI_diffusion_looksoon_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_looksoon_Method.php',
     'ConduitAPI_diffusion_mergedcommitsquery_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_mergedcommitsquery_Method.php',
     'ConduitAPI_diffusion_rawdiffquery_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_rawdiffquery_Method.php',
     'ConduitAPI_diffusion_readmequery_Method' => 'applications/diffusion/conduit/ConduitAPI_diffusion_readmequery_Method.php',
@@ -2366,6 +2367,7 @@ phutil_register_library_map(array(
     'ConduitAPI_diffusion_getrecentcommitsbypath_Method' => 'ConduitAPI_diffusion_Method',
     'ConduitAPI_diffusion_historyquery_Method' => 'ConduitAPI_diffusion_abstractquery_Method',
     'ConduitAPI_diffusion_lastmodifiedquery_Method' => 'ConduitAPI_diffusion_abstractquery_Method',
+    'ConduitAPI_diffusion_looksoon_Method' => 'ConduitAPI_diffusion_Method',
     'ConduitAPI_diffusion_mergedcommitsquery_Method' => 'ConduitAPI_diffusion_abstractquery_Method',
     'ConduitAPI_diffusion_rawdiffquery_Method' => 'ConduitAPI_diffusion_abstractquery_Method',
     'ConduitAPI_diffusion_readmequery_Method' => 'ConduitAPI_diffusion_abstractquery_Method',

src/applications/diffusion/conduit/ConduitAPI_diffusion_looksoon_Method.php

@@ -0,0 +1,55 @@
+<?php
+
+final class ConduitAPI_diffusion_looksoon_Method
+  extends ConduitAPI_diffusion_Method {
+
+  public function getMethodStatus() {
+    return self::METHOD_STATUS_UNSTABLE;
+  }
+
+  public function getMethodDescription() {
+    return pht(
+      'Advises Phabricator to look for new commits in a repository as soon '.
+      'as possible. This advice is most useful if you have just pushed new '.
+      'commits to that repository.');
+  }
+
+  public function defineReturnType() {
+    return 'void';
+  }
+
+  public function defineParamTypes() {
+    return array(
+      'callsigns' => 'required list<string>',
+      'urgency' => 'optional string',
+    );
+  }
+
+  public function defineErrorTypes() {
+    return array();
+  }
+
+  protected function execute(ConduitAPIRequest $request) {
+    // NOTE: The "urgency" parameter does nothing, it is just a hilarious joke
+    // which exemplifies the boundless clever wit of this project.
+
+    $callsigns = $request->getValue('callsigns');
+    if (!$callsigns) {
+      return null;
+    }
+
+    $repositories = id(new PhabricatorRepositoryQuery())
+      ->setViewer($request->getUser())
+      ->withCallsigns($callsigns)
+      ->execute();
+
+    foreach ($repositories as $repository) {
+      $repository->writeStatusMessage(
+        PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
+        PhabricatorRepositoryStatusMessage::CODE_OKAY);
+    }
+
+    return null;
+  }
+
+}

src/applications/diffusion/controller/DiffusionController.php

@@ -140,14 +140,23 @@ abstract class DiffusionController extends PhabricatorController {
 
     switch ($repository->getVersionControlSystem()) {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-        return $this->serveGitRequest($repository);
+        $result = $this->serveGitRequest($repository);
       default:
+        $result = new PhabricatorVCSResponse(
+          999,
+          pht('TODO: Implement meaningful responses.'));
         break;
     }
 
-    return new PhabricatorVCSResponse(
+    $code = $result->getHTTPResponseCode();
-      999,
+
-      pht('TODO: Implement meaningful responses.'));
+    if ($is_push && ($code == 200)) {
+      $repository->writeStatusMessage(
+        PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
+        PhabricatorRepositoryStatusMessage::CODE_OKAY);
+    }
+
+    return $result;
   }
 
   private function isReadOnlyRequest(

src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php

@@ -563,6 +563,8 @@ final class DiffusionRepositoryEditMainController
   private function buildRepositoryStatus(
     PhabricatorRepository $repository) {
 
+    $viewer = $this->getRequest()->getUser();
+
     $view = new PHUIStatusListView();
 
     $messages = id(new PhabricatorRepositoryStatusMessage())
@@ -696,7 +698,7 @@ final class DiffusionRepositoryEditMainController
               id(new PHUIStatusItemView())
                 ->setIcon('time-green')
                 ->setTarget(pht('Initializing Working Copy'))
-                ->setNote(pht('Daemons are initilizing the working copy.')));
+                ->setNote(pht('Daemons are initializing the working copy.')));
             return $view;
           default:
             $view->addItem(
@@ -717,6 +719,92 @@ final class DiffusionRepositoryEditMainController
       }
     }
 
+    $message = idx($messages, PhabricatorRepositoryStatusMessage::TYPE_FETCH);
+    if ($message) {
+      switch ($message->getStatusCode()) {
+        case PhabricatorRepositoryStatusMessage::CODE_ERROR:
+          $view->addItem(
+            id(new PHUIStatusItemView())
+              ->setIcon('warning-red')
+              ->setTarget(pht('Update Error'))
+              ->setNote($message->getParameter('message')));
+          return $view;
+        case PhabricatorRepositoryStatusMessage::CODE_OKAY:
+          $view->addItem(
+            id(new PHUIStatusItemView())
+              ->setIcon('accept-green')
+              ->setTarget(pht('Updates OK'))
+              ->setNote(
+                pht(
+                  'Last updated %s.',
+                  phabricator_datetime($message->getEpoch(), $viewer))));
+          break;
+      }
+    } else {
+      $view->addItem(
+        id(new PHUIStatusItemView())
+          ->setIcon('time-orange')
+          ->setTarget(pht('Waiting For Update'))
+          ->setNote(
+            pht('Waiting for daemons to read updates.')));
+    }
+
+    if ($repository->isImporting()) {
+      $progress = queryfx_all(
+        $repository->establishConnection('r'),
+        'SELECT importStatus, count(*) N FROM %T WHERE repositoryID = %d
+          GROUP BY importStatus',
+        id(new PhabricatorRepositoryCommit())->getTableName(),
+        $repository->getID());
+
+      $done = 0;
+      $total = 0;
+      foreach ($progress as $row) {
+        $total += $row['N'] * 4;
+        $status = $row['importStatus'];
+        if ($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE) {
+          $done += $row['N'];
+        }
+        if ($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE) {
+          $done += $row['N'];
+        }
+        if ($status & PhabricatorRepositoryCommit::IMPORTED_OWNERS) {
+          $done += $row['N'];
+        }
+        if ($status & PhabricatorRepositoryCommit::IMPORTED_HERALD) {
+          $done += $row['N'];
+        }
+      }
+
+      if ($total) {
+        $percentage = 100 * ($done / $total);
+      } else {
+        $percentage = 0;
+      }
+
+      $percentage = sprintf('%.1f%%', $percentage);
+
+      $view->addItem(
+        id(new PHUIStatusItemView())
+          ->setIcon('time-green')
+          ->setTarget(pht('Importing'))
+          ->setNote(
+            pht('%s Complete', $percentage)));
+    } else {
+      $view->addItem(
+        id(new PHUIStatusItemView())
+          ->setIcon('accept-green')
+          ->setTarget(pht('Fully Imported')));
+    }
+
+    if (idx($messages, PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE)) {
+      $view->addItem(
+        id(new PHUIStatusItemView())
+          ->setIcon('up')
+          ->setTarget(pht('Prioritized'))
+          ->setNote(pht('This repository will be updated soon.')));
+    }
+
     return $view;
   }
 

src/applications/diffusion/ssh/DiffusionSSHGitReceivePackWorkflow.php

@@ -28,7 +28,15 @@ final class DiffusionSSHGitReceivePackWorkflow
     $future = new ExecFuture(
       'git-receive-pack %s',
       $repository->getLocalPath());
-    return $this->passthruIO($future);
+    $err = $this->passthruIO($future);
+
+    if (!$err) {
+      $repository->writeStatusMessage(
+        PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
+        PhabricatorRepositoryStatusMessage::CODE_OKAY);
+    }
+
+    return $err;
   }
 
 }

src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php

@@ -92,6 +92,15 @@ final class PhabricatorRepositoryPullLocalDaemon
       shuffle($repositories);
       $repositories = mpull($repositories, null, 'getID');
 
+      // If any repositories have the NEEDS_UPDATE flag set, pull them
+      // as soon as possible.
+      $type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE;
+      $need_update_messages = id(new PhabricatorRepositoryStatusMessage())
+        ->loadAllWhere('statusType = %s', $type_need_update);
+      foreach ($need_update_messages as $message) {
+        $retry_after[$message->getRepositoryID()] = time();
+      }
+
       // If any repositories were deleted, remove them from the retry timer map
       // so we don't end up with a retry timer that never gets updated and
       // causes us to sleep for the minimum amount of time.
@@ -135,9 +144,23 @@ final class PhabricatorRepositoryPullLocalDaemon
             $lock = PhabricatorGlobalLock::newLock($lock_name);
             $lock->lock();
 
+            $repository->writeStatusMessage(
+              PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
+              null);
+
             try {
               $this->discoverRepository($repository);
+              $repository->writeStatusMessage(
+                PhabricatorRepositoryStatusMessage::TYPE_FETCH,
+                PhabricatorRepositoryStatusMessage::CODE_OKAY);
             } catch (Exception $ex) {
+              $repository->writeStatusMessage(
+                PhabricatorRepositoryStatusMessage::TYPE_FETCH,
+                PhabricatorRepositoryStatusMessage::CODE_ERROR,
+                array(
+                  'message' => pht(
+                    'Error updating working copy: %s', $ex->getMessage()),
+                ));
               $lock->unlock();
               throw $ex;
             }
@@ -483,7 +506,8 @@ final class PhabricatorRepositoryPullLocalDaemon
     // Look for any commit which hasn't imported.
     $unparsed_commit = queryfx_one(
       $repository->establishConnection('r'),
-      'SELECT * FROM %T WHERE repositoryID = %d AND importStatus != %d',
+      'SELECT * FROM %T WHERE repositoryID = %d AND importStatus != %d
+        LIMIT 1',
       id(new PhabricatorRepositoryCommit())->getTableName(),
       $repository->getID(),
       PhabricatorRepositoryCommit::IMPORTED_ALL);