Give Asana feed publishing tasks a less aggressive retry/backoff schedule

Summary:
Ref T2852. Asana sync tasks currently have a standard retry/backoff schedule, but the defaults are quite aggressive (retry every 60s forever). Instead, retry at increasing intervals and stop retrying after a few tries.

  - Retry at intervals and stop retrying after a few iterations.
  - Modernize some interfaces.
  - Add better information about retry behaviors to the web UI.

Test Plan: {F49194}

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2852

Differential Revision: https://secure.phabricator.com/D6381

src/applications/daemon/controller/PhabricatorWorkerTaskDetailController.php

@@ -41,21 +41,34 @@ final class PhabricatorWorkerTaskDetailController
       $actions    = $this->buildActionListView($task);
       $properties = $this->buildPropertyListView($task);
 
+      $retry_head = id(new PhabricatorHeaderView())
+        ->setHeader(pht('Retries'));
+
+      $retry_info = $this->buildRetryListView($task);
+
       $content = array(
         $header,
         $actions,
         $properties,
+        $retry_head,
+        $retry_info,
       );
     }
 
-    $nav = $this->buildSideNavView();
-    $nav->selectFilter('');
-    $nav->appendChild($content);
+    $crumbs = $this->buildApplicationCrumbs();
+    $crumbs->addCrumb(
+      id(new PhabricatorCrumbView())
+        ->setName($title));
 
     return $this->buildApplicationPage(
-      $nav,
+      array(
+        $crumbs,
+        $content,
+      ),
       array(
         'title' => $title,
+        'dust' => true,
+        'device' => true,
       ));
   }
 
@@ -161,10 +174,6 @@ final class PhabricatorWorkerTaskDetailController
       pht('Lease Expires'),
       $expires);
 
-    $view->addProperty(
-      pht('Failure Count'),
-      $task->getFailureCount());
-
     if ($task->isArchived()) {
       $duration = number_format($task->getDuration()).' us';
     } else {
@@ -187,4 +196,63 @@ final class PhabricatorWorkerTaskDetailController
     return $view;
   }
 
+  private function buildRetryListView(PhabricatorWorkerTask $task) {
+    $view = new PhabricatorPropertyListView();
+
+    $data = id(new PhabricatorWorkerTaskData())->load($task->getDataID());
+    $task->setData($data->getData());
+    $worker = $task->getWorkerInstance();
+
+    $view->addProperty(
+      pht('Failure Count'),
+      $task->getFailureCount());
+
+    $retry_count = $worker->getMaximumRetryCount();
+    if ($retry_count === null) {
+      $max_retries = phutil_tag('em', array(), pht('Retries Forever'));
+      $retry_count = INF;
+    } else {
+      $max_retries = $retry_count;
+    }
+
+    $view->addProperty(
+      pht('Maximum Retries'),
+      $max_retries);
+
+    $projection = clone $task;
+    $projection->makeEphemeral();
+
+    $next = array();
+    for ($ii = $task->getFailureCount(); $ii < $retry_count; $ii++) {
+      $projection->setFailureCount($ii);
+      $next[] = $worker->getWaitBeforeRetry($projection);
+      if (count($next) > 10) {
+        break;
+      }
+    }
+
+    if ($next) {
+      $cumulative = 0;
+      foreach ($next as $key => $duration) {
+        if ($duration === null) {
+          $duration = 60;
+        }
+        $cumulative += $duration;
+        $next[$key] = phabricator_format_relative_time($cumulative);
+      }
+      if ($ii != $retry_count) {
+        $next[] = '...';
+      }
+      $retries_in = implode(', ', $next);
+    } else {
+      $retries_in = pht('No More Retries');
+    }
+
+    $view->addProperty(
+      pht('Retries After'),
+      $retries_in);
+
+    return $view;
+  }
+
 }

src/applications/doorkeeper/worker/DoorkeeperFeedWorkerAsana.php

@@ -602,5 +602,13 @@ final class DoorkeeperFeedWorkerAsana extends FeedPushWorker {
     return $ref;
   }
 
+  public function getMaximumRetryCount() {
+    return 4;
+  }
+
+  public function getWaitBeforeRetry(PhabricatorWorkerTask $task) {
+    $count = $task->getFailureCount();
+    return (5 * 60) * pow(8, $count);
+  }
 
 }

src/infrastructure/daemon/workers/PhabricatorWorker.php

@@ -59,7 +59,6 @@ abstract class PhabricatorWorker {
    *                                  retries, or to examine the execution
    *                                  exception if you want to react to
    *                                  different failures in different ways.
-   * @param   Exception               The exception which caused the failure.
    * @return  int|null                Number of seconds to wait between retries,
    *                                  or null for a default retry period
    *                                  (currently 60 seconds).