Improve UX for repository updates

Summary:
Fixes T5926. Fixes T5830. Ref T4767. Users currently sometimes have a hard time understanding repository update frequencies. This is compounded by aggressive backoff and incorrect backoff while importing repositories.

  - Don't back off while importing repositories. This prevents us from hanging at 99.99% for inactive repositories while waiting for the next update.
  - Back off less aggressively in general, and even more gradually during the first 3 days. This should make behavior around weekends better.
  - Show update frequency in the UI.
  - Provide an explicit "update now" button to call `diffusion.looksoon` in a more user-friendly way.
  - Document how backoff policies work and how to adjust behavior.

Test Plan:
  - Ran `bin/phd debug pulllocal` and verified backoff worked correctly from debugging output.
  - Clicked "Update Now" to get a hint, reloaded page to see it update.
  - Read documentation.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T4767, T5830, T5926

Differential Revision: https://secure.phabricator.com/D10323

src/__phutil_library_map__.php

@@ -475,6 +475,7 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditLocalController' => 'applications/diffusion/controller/DiffusionRepositoryEditLocalController.php',
     'DiffusionRepositoryEditMainController' => 'applications/diffusion/controller/DiffusionRepositoryEditMainController.php',
     'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php',
+    'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php',
     'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php',
     'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php',
     'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php',
@@ -3223,6 +3224,7 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditLocalController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryEditMainController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController',
+    'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController',
     'DiffusionRepositoryListController' => 'DiffusionController',
     'DiffusionRepositoryNewController' => 'DiffusionController',
     'DiffusionRepositoryRef' => 'Phobject',

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

@@ -92,6 +92,7 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
             'delete/' => 'DiffusionRepositoryEditDeleteController',
             'hosting/' => 'DiffusionRepositoryEditHostingController',
             '(?P<serve>serve)/' => 'DiffusionRepositoryEditHostingController',
+            'update/' => 'DiffusionRepositoryEditUpdateController',
           ),
           'pathtree/(?P<dblob>.*)' => 'DiffusionPathTreeController',
           'mirror/' => array(

src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php

@@ -207,6 +207,14 @@ final class DiffusionRepositoryEditMainController
       ->setHref($this->getRepositoryControllerURI($repository, 'edit/basic/'));
     $view->addAction($edit);
 
+    $edit = id(new PhabricatorActionView())
+      ->setIcon('fa-refresh')
+      ->setName(pht('Update Now'))
+      ->setWorkflow(true)
+      ->setHref(
+        $this->getRepositoryControllerURI($repository, 'edit/update/'));
+    $view->addAction($edit);
+
     $activate = id(new PhabricatorActionView())
       ->setHref(
         $this->getRepositoryControllerURI($repository, 'edit/activate/'))
@@ -280,6 +288,10 @@ final class DiffusionRepositoryEditMainController
       pht('Status'),
       $this->buildRepositoryStatus($repository));
 
+    $view->addProperty(
+      pht('Update Frequency'),
+      $this->buildRepositoryUpdateInterval($repository));
+
     $description = $repository->getDetail('description');
     $view->addSectionHeader(pht('Description'));
     if (!strlen($description)) {
@@ -942,14 +954,16 @@ final class DiffusionRepositoryEditMainController
               ->setNote($message->getParameter('message')));
           return $view;
         case PhabricatorRepositoryStatusMessage::CODE_OKAY:
+          $ago = (PhabricatorTime::getNow() - $message->getEpoch());
           $view->addItem(
             id(new PHUIStatusItemView())
               ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green')
               ->setTarget(pht('Updates OK'))
               ->setNote(
                 pht(
-                  'Last updated %s.',
+                  'Last updated %s (%s ago).',
-                  phabricator_datetime($message->getEpoch(), $viewer))));
+                  phabricator_datetime($message->getEpoch(), $viewer),
+                  phutil_format_relative_time_detailed($ago))));
           break;
       }
     } else {
@@ -1020,12 +1034,34 @@ final class DiffusionRepositoryEditMainController
         id(new PHUIStatusItemView())
           ->setIcon(PHUIStatusItemView::ICON_UP, 'indigo')
           ->setTarget(pht('Prioritized'))
-          ->setNote(pht('This repository will be updated soon.')));
+          ->setNote(pht('This repository will be updated soon!')));
     }
 
     return $view;
   }
 
+  private function buildRepositoryUpdateInterval(
+    PhabricatorRepository $repository) {
+
+    $smart_wait = $repository->loadUpdateInterval();
+
+    $doc_href = PhabricatorEnv::getDoclink(
+      'Diffusion User Guide: Repository Updates');
+
+    return array(
+      phutil_format_relative_time_detailed($smart_wait),
+      " \xC2\xB7 ",
+      phutil_tag(
+        'a',
+        array(
+          'href' => $doc_href,
+          'target' => '_blank',
+        ),
+        pht('Learn More')),
+    );
+  }
+
+
   private function buildMirrorActions(
     PhabricatorRepository $repository) {
 

src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php

@@ -0,0 +1,75 @@
+<?php
+
+final class DiffusionRepositoryEditUpdateController
+  extends DiffusionRepositoryEditController {
+
+  public function processRequest() {
+    $request = $this->getRequest();
+    $viewer = $request->getUser();
+    $drequest = $this->diffusionRequest;
+    $repository = $drequest->getRepository();
+
+    $repository = id(new PhabricatorRepositoryQuery())
+      ->setViewer($viewer)
+      ->requireCapabilities(
+        array(
+          PhabricatorPolicyCapability::CAN_VIEW,
+          PhabricatorPolicyCapability::CAN_EDIT,
+        ))
+      ->withIDs(array($repository->getID()))
+      ->executeOne();
+    if (!$repository) {
+      return new Aphront404Response();
+    }
+
+    $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/');
+
+    if ($request->isFormPost()) {
+      $params = array(
+        'callsigns' => array(
+          $repository->getCallsign(),
+        ),
+      );
+
+      id(new ConduitCall('diffusion.looksoon', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      return id(new AphrontRedirectResponse())->setURI($edit_uri);
+    }
+
+    $doc_name = 'Diffusion User Guide: Repository Updates';
+    $doc_href = PhabricatorEnv::getDoclink($doc_name);
+    $doc_link = phutil_tag(
+      'a',
+      array(
+        'href' => $doc_href,
+        'target' => '_blank',
+      ),
+      $doc_name);
+
+    return $this->newDialog()
+      ->setTitle(pht('Update Repository Now'))
+      ->appendParagraph(
+        pht(
+          'Normally, Phabricator automatically updates repositories '.
+          'based on how much time has elapsed since the last commit. '.
+          'This helps reduce load if you have a large number of mostly '.
+          'inactive repositories, which is common.'))
+      ->appendParagraph(
+        pht(
+          'You can manually schedule an update for this repository. The '.
+          'daemons will perform the update as soon as possible. This may '.
+          'be helpful if you have just made a commit to a rarely used '.
+          'repository.'))
+      ->appendParagraph(
+        pht(
+          'To learn more about how Phabricator updates repositories, '.
+          'read %s in the documentation.',
+          $doc_link))
+      ->addCancelButton($edit_uri)
+      ->addSubmitButton(pht('Schedule Update'));
+  }
+
+
+}

src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php

@@ -345,55 +345,23 @@ final class PhabricatorRepositoryPullLocalDaemon
       phlog($stderr_msg);
     }
 
-    $sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep);
+    // For now, continue respecting this deprecated setting for raising the
+    // minimum pull frequency.
+    // TODO: Remove this some day once this code has been completely stable
+    // for a while.
+    $sleep_for = (int)$repository->getDetail('pull-frequency');
+    $min_sleep = max($sleep_for, $min_sleep);
 
-    // Smart wait: pull rarely used repositories less frequently. Find the
+    $smart_wait = $repository->loadUpdateInterval($min_sleep);
-    // most recent commit which is older than the current time (this keeps us
-    // from spinning on repositories with a silly commit post-dated to some time
-    // in 2037), and adjust how frequently we pull based on how frequently this
-    // repository updates.
 
-    $table = id(new PhabricatorRepositoryCommit());
+    $this->log(
-    $last_commit = queryfx_one(
+      pht(
-      $table->establishConnection('w'),
+        'Based on activity in repository "%s", considering a wait of %s '.
-      'SELECT epoch FROM %T
+        'seconds before update.',
-        WHERE repositoryID = %d AND epoch <= %d
+        $repository->getMonogram(),
-        ORDER BY epoch DESC LIMIT 1',
+        new PhutilNumber($smart_wait)));
-      $table->getTableName(),
-      $repository->getID(),
-      time() + $min_sleep);
-    if ($last_commit) {
-      $time_since_commit = (time() + $min_sleep) - $last_commit['epoch'];
 
-      // Wait 0.5% of the time since the last commit before we pull. This gives
+    return time() + $smart_wait;
-      // us these wait times:
-      //
-      // 50 minutes or less: 15 seconds
-      // about 3 hours: 1 minute
-      // about 16 hours: 5 minutes
-      // about 2 days: 15 minutes
-      // 50 days or more: 6 hours
-
-      $smart_wait = ($time_since_commit / 200);
-      $smart_wait = min($smart_wait, phutil_units('6 hours in seconds'));
-
-      $this->log(
-        pht(
-          'Last commit to repository "%s" was %s seconds ago; considering '.
-          'a wait of %s seconds before update.',
-          $repository->getMonogram(),
-          new PhutilNumber($time_since_commit),
-          new PhutilNumber($smart_wait)));
-
-      $smart_wait = max(15, $smart_wait);
-      $sleep_for = max($smart_wait, $sleep_for);
-    }
-
-    if ($sleep_for < $min_sleep) {
-      $sleep_for = $min_sleep;
-    }
-
-    return time() + $sleep_for;
   }
 
 

src/applications/repository/storage/PhabricatorRepository.php

@@ -1319,6 +1319,69 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
   }
 
 
+  /**
+   * Load the pull frequency for this repository, based on the time since the
+   * last activity.
+   *
+   * We pull rarely used repositories less frequently. This finds the most
+   * recent commit which is older than the current time (which prevents us from
+   * spinning on repositories with a silly commit post-dated to some time in
+   * 2037). We adjust the pull frequency based on when the most recent commit
+   * occurred.
+   *
+   * @param   int   The minimum update interval to use, in seconds.
+   * @return  int   Repository update interval, in seconds.
+   */
+  public function loadUpdateInterval($minimum = 15) {
+    // If a repository is still importing, always pull it as frequently as
+    // possible. This prevents us from hanging for a long time at 99.9% when
+    // importing an inactive repository.
+    if ($this->isImporting()) {
+      return $minimum;
+    }
+
+    $window_start = (PhabricatorTime::getNow() + $minimum);
+
+    $table = id(new PhabricatorRepositoryCommit());
+    $last_commit = queryfx_one(
+      $table->establishConnection('r'),
+      'SELECT epoch FROM %T
+        WHERE repositoryID = %d AND epoch <= %d
+        ORDER BY epoch DESC LIMIT 1',
+      $table->getTableName(),
+      $this->getID(),
+      $window_start);
+    if ($last_commit) {
+      $time_since_commit = ($window_start - $last_commit['epoch']);
+
+      $last_few_days = phutil_units('3 days in seconds');
+
+      if ($time_since_commit <= $last_few_days) {
+        // For repositories with activity in the recent past, we wait one
+        // extra second for every 10 minutes since the last commit. This
+        // shorter backoff is intended to handle weekends and other short
+        // breaks from development.
+        $smart_wait = ($time_since_commit / 600);
+      } else {
+        // For repositories without recent activity, we wait one extra second
+        // for every 4 minutes since the last commit. This longer backoff
+        // handles rarely used repositories, up to the maximum.
+        $smart_wait = ($time_since_commit / 240);
+      }
+
+      // We'll never wait more than 6 hours to pull a repository.
+      $longest_wait = phutil_units('6 hours in seconds');
+      $smart_wait = min($smart_wait, $longest_wait);
+
+      $smart_wait = max($minimum, $smart_wait);
+    } else {
+      $smart_wait = $minimum;
+    }
+
+    return $smart_wait;
+  }
+
+
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
 

src/docs/user/userguide/diffusion_updates.diviner

@@ -0,0 +1,103 @@
+@title Diffusion User Guide: Repository Updates
+@group userguide
+
+Explains how Diffusion updates repositories to discover new changes.
+
+Overview
+========
+
+When Phabricator is configured to import repositories which are hosted
+elsewhere, it needs to poll those repositories for changes. If it polls too
+frequently, it can create too much load locally and on remote services. If it
+polls too rarely, it may take a long time for commits to show up in the web
+interface.
+
+This document describes the rules around polling and how to understand and
+adjust the behavior. In general:
+
+  - Phabricator chooses a default poll interval based on repository
+    activity. These intervals range from every 15 seconds (for active
+    repositories) to every 6 hours (for repositories with no commits in two
+    months).
+  - If you use `arc` to push commits, or you host repositories on Phabricator,
+    repositories automatically update after changes are pushed.
+  - If you don't use `arc` and your repository is hosted elsewhere, this
+    document describes ways you can make polling more responsive.
+
+
+Default Behavior
+================
+
+By default, Phabricator determines how frequently to poll repositories by
+examining how long it has been since the last commit. In most cases this is
+fairly accurate and produces good behavior. In particular, it automatically
+reduces the polling frequency for rarely-used repositories. This dramatically
+reduces load for installs with a large number of inactive repositories, which
+is common.
+
+For repositories with activity in the last 3 days, we wait 1 second for every
+10 minutes without activity. The table below has some examples.
+
+| Time Since Commit | Poll Interval    |
+|-------------------|------------------|
+| //Minimum//       | 15 seconds       |
+| 6h                | about 30 seconds |
+| 12h               | about 1 minute   |
+| 1 day             | about 2 minutes  |
+| 2 days            | about 5 minutes  |
+| 3 days            | about 7 minutes  |
+
+This means that you may need to wait about 2 minutes for the first commit to
+be imported in the morning, and about 5 minutes after a long weekend, but other
+commits to active repositories should usually be recognized in 30 seconds or
+less.
+
+For repositories with no activity in the last 3 days, we wait longer between
+updates (1 second for every 4 minutes without activity). The table below has
+some examples.
+
+| Time Since Commit | Poll Interval    |
+|-------------------|------------------|
+| 4 days            | about 30 minutes |
+| 7 days            | about 45 minutes |
+| 10 days           | about 1 hour     |
+| 20 days           | about 2 hours    |
+| 30 days           | about 3 hours    |
+| //Maximum//       | 6 hours          |
+
+You can find the exact default poll frequency of a repository in
+Diffusion > (Choose a Repository) > Edit Repository, under "Update Frequency".
+You can also see the time when the repository was last updated in this
+interface.
+
+Repositories that are currently importing are always updated at the minimum
+update frequency so the import finishes as quickly as possible.
+
+
+Triggering Repository Updates
+=============================
+
+If you want Phabricator to update a repository more quickly than the default
+update frequency (for example, because you just pushed a commit to it), you can
+tell Phabricator that it should schedule an update as soon as possible.
+
+There are several ways to do this:
+
+  - If you push changes with `arc land` or `arc commit`, this will be done
+    for you automatically. These commits should normally be recognized within
+    a few seconds.
+  - If your repository is hosted on Phabricator, this will also be done for you
+    automatically.
+  - You can schedule an update from the web interface, in Diffusion >
+    (Choose a Repository) > Edit Repository > Update Now.
+  - You can make a call to the Conduit API method `diffusion.looksoon`. This
+    hints to Phabricator that it should poll a repository as soon as it can.
+    All of the other mechanisms do this under the hood.
+
+In particular, you may be able to add a commit hook to your external repository
+which calls `diffusion.looksoon`. This should make an external repository about
+as responsive as a hosted repository.
+
+If a repository has an update scheduled, the Diffusion > (Choose a
+Repository) > Edit Repository interface will show that the the repository is
+prioritized and will be updated soon.