Queue large ICS files for background import

Summary: Ref T11801. When a file is larger than 512KB, queue it for background import instead of trying to do it in the foreground, sinc we risk hitting `max_execution_time`.

Test Plan: {F1906943}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11801

Differential Revision: https://secure.phabricator.com/D16805

src/__phutil_library_map__.php

@@ -2145,6 +2145,7 @@ phutil_register_library_map(array(
     'PhabricatorCalendarImportOrphanLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportOrphanLogType.php',
     'PhabricatorCalendarImportPHIDType' => 'applications/calendar/phid/PhabricatorCalendarImportPHIDType.php',
     'PhabricatorCalendarImportQuery' => 'applications/calendar/query/PhabricatorCalendarImportQuery.php',
+    'PhabricatorCalendarImportQueueLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php',
     'PhabricatorCalendarImportReloadController' => 'applications/calendar/controller/PhabricatorCalendarImportReloadController.php',
     'PhabricatorCalendarImportReloadTransaction' => 'applications/calendar/xaction/PhabricatorCalendarImportReloadTransaction.php',
     'PhabricatorCalendarImportReloadWorker' => 'applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php',
@@ -7012,6 +7013,7 @@ phutil_register_library_map(array(
     'PhabricatorCalendarImportOrphanLogType' => 'PhabricatorCalendarImportLogType',
     'PhabricatorCalendarImportPHIDType' => 'PhabricatorPHIDType',
     'PhabricatorCalendarImportQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorCalendarImportQueueLogType' => 'PhabricatorCalendarImportLogType',
     'PhabricatorCalendarImportReloadController' => 'PhabricatorCalendarController',
     'PhabricatorCalendarImportReloadTransaction' => 'PhabricatorCalendarImportTransactionType',
     'PhabricatorCalendarImportReloadWorker' => 'PhabricatorWorker',

src/applications/calendar/editor/PhabricatorCalendarImportEditor.php

@@ -54,7 +54,7 @@ final class PhabricatorCalendarImportEditor
 
     if ($should_reload) {
       $import_engine = $object->getEngine();
-      $import_engine->importEventsFromSource($actor, $object);
+      $import_engine->importEventsFromSource($actor, $object, true);
     }
 
     if ($should_trigger) {
@@ -107,6 +107,7 @@ final class PhabricatorCalendarImportEditor
             'class' => 'PhabricatorCalendarImportReloadWorker',
             'data' => array(
               'importPHID' => $object->getPHID(),
+              'via' => PhabricatorCalendarImportReloadWorker::VIA_TRIGGER,
             ),
             'options' => array(
               'objectPHID' => $object->getPHID(),

src/applications/calendar/import/PhabricatorCalendarICSFileImportEngine.php

@@ -65,7 +65,8 @@ final class PhabricatorCalendarICSFileImportEngine
 
   public function importEventsFromSource(
     PhabricatorUser $viewer,
-    PhabricatorCalendarImport $import) {
+    PhabricatorCalendarImport $import,
+    $should_queue) {
 
     $phid_key = PhabricatorCalendarImportICSFileTransaction::PARAMKEY_FILE;
     $file_phid = $import->getParameter($phid_key);
@@ -83,10 +84,13 @@ final class PhabricatorCalendarICSFileImportEngine
 
     $data = $file->loadFileData();
 
+    if ($should_queue && $this->shouldQueueDataImport($data)) {
+      return $this->queueDataImport($import, $data);
+    }
+
     return $this->importICSData($viewer, $import, $data);
   }
 
-
   public function canDisable(
     PhabricatorUser $viewer,
     PhabricatorCalendarImport $import) {

src/applications/calendar/import/PhabricatorCalendarICSURIImportEngine.php

@@ -77,7 +77,8 @@ final class PhabricatorCalendarICSURIImportEngine
 
   public function importEventsFromSource(
     PhabricatorUser $viewer,
-    PhabricatorCalendarImport $import) {
+    PhabricatorCalendarImport $import,
+    $should_queue) {
 
     $uri_key = PhabricatorCalendarImportICSURITransaction::PARAMKEY_URI;
     $uri = $import->getParameter($uri_key);
@@ -103,6 +104,10 @@ final class PhabricatorCalendarICSURIImportEngine
 
     $data = $file->loadFileData();
 
+    if ($should_queue && $this->shouldQueueDataImport($data)) {
+      return $this->queueDataImport($import, $data);
+    }
+
     return $this->importICSData($viewer, $import, $data);
   }
 

src/applications/calendar/import/PhabricatorCalendarImportEngine.php

@@ -3,11 +3,12 @@
 abstract class PhabricatorCalendarImportEngine
   extends Phobject {
 
+  const QUEUE_BYTE_LIMIT = 524288;
+
   final public function getImportEngineType() {
     return $this->getPhobjectClassConstant('ENGINETYPE', 64);
   }
 
-
   abstract public function getImportEngineName();
   abstract public function getImportEngineTypeName();
   abstract public function getImportEngineHint();
@@ -27,7 +28,8 @@ abstract class PhabricatorCalendarImportEngine
 
   abstract public function importEventsFromSource(
     PhabricatorUser $viewer,
-    PhabricatorCalendarImport $import);
+    PhabricatorCalendarImport $import,
+    $should_queue);
 
   abstract public function canDisable(
     PhabricatorUser $viewer,
@@ -568,4 +570,35 @@ abstract class PhabricatorCalendarImportEngine
     return (bool)$any_event;
   }
 
+  final protected function shouldQueueDataImport($data) {
+    return (strlen($data) > self::QUEUE_BYTE_LIMIT);
+  }
+
+  final protected function queueDataImport(
+    PhabricatorCalendarImport $import,
+    $data) {
+
+    $import->newLogMessage(
+      PhabricatorCalendarImportQueueLogType::LOGTYPE,
+      array(
+        'data.size' => strlen($data),
+        'data.limit' => self::QUEUE_BYTE_LIMIT,
+      ));
+
+    // When we queue on this pathway, we're queueing in response to an explicit
+    // user action (like uploading a big `.ics` file), so we queue at normal
+    // priority instead of bulk/import priority.
+
+    PhabricatorWorker::scheduleTask(
+      'PhabricatorCalendarImportReloadWorker',
+      array(
+        'importPHID' => $import->getPHID(),
+        'via' => PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND,
+      ),
+      array(
+        'objectPHID' => $import->getPHID(),
+      ));
+  }
+
+
 }

src/applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php

@@ -0,0 +1,40 @@
+<?php
+
+final class PhabricatorCalendarImportQueueLogType
+  extends PhabricatorCalendarImportLogType {
+
+  const LOGTYPE = 'queue';
+
+  public function getDisplayType(
+    PhabricatorUser $viewer,
+    PhabricatorCalendarImportLog $log) {
+    return pht('Queued');
+  }
+
+  public function getDisplayDescription(
+    PhabricatorUser $viewer,
+    PhabricatorCalendarImportLog $log) {
+
+    $size = $log->getParameter('data.size');
+    $limit = $log->getParameter('data.limit');
+
+    return pht(
+      'Queued for background import: data size (%s) exceeds limit for '.
+      'immediate processing (%s).',
+      phutil_format_bytes($size),
+      phutil_format_bytes($limit));
+  }
+
+  public function getDisplayIcon(
+    PhabricatorUser $viewer,
+    PhabricatorCalendarImportLog $log) {
+    return 'fa-sort-amount-desc';
+  }
+
+  public function getDisplayColor(
+    PhabricatorUser $viewer,
+    PhabricatorCalendarImportLog $log) {
+    return 'blue';
+  }
+
+}

src/applications/calendar/importlog/PhabricatorCalendarImportTriggerLogType.php

@@ -14,7 +14,15 @@ final class PhabricatorCalendarImportTriggerLogType
   public function getDisplayDescription(
     PhabricatorUser $viewer,
     PhabricatorCalendarImportLog $log) {
-    return pht('Triggered a periodic update.');
+
+    $via = $log->getParameter('via');
+    switch ($via) {
+      case PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND:
+        return pht('Started background processing.');
+      case PhabricatorCalendarImportReloadWorker::VIA_TRIGGER:
+      default:
+        return pht('Triggered a periodic update.');
+    }
   }
 
   public function getDisplayIcon(

src/applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php

@@ -2,6 +2,9 @@
 
 final class PhabricatorCalendarImportReloadWorker extends PhabricatorWorker {
 
+  const VIA_TRIGGER = 'trigger';
+  const VIA_BACKGROUND = 'background';
+
   protected function doWork() {
     $import = $this->loadImport();
     $viewer = PhabricatorUser::getOmnipotentUser();
@@ -18,11 +21,14 @@ final class PhabricatorCalendarImportReloadWorker extends PhabricatorWorker {
 
     $import_engine = $import->getEngine();
 
+    $data = $this->getTaskData();
     $import->newLogMessage(
       PhabricatorCalendarImportTriggerLogType::LOGTYPE,
-      array());
+      array(
+        'via' => idx($data, 'via', self::VIA_TRIGGER),
+      ));
 
-    $import_engine->importEventsFromSource($author, $import);
+    $import_engine->importEventsFromSource($author, $import, false);
   }
 
   private function loadImport() {