When exporting more than 1,000 records, export in the background

Summary:
Depends on D18961. Ref T13049. Currently, longer exports don't give the user any feedback, and exports that take longer than 30 seconds are likely to timeout.

For small exports (up to 1,000 rows) continue doing the export in the web process.

For large exports, queue a bulk job and do them in the workers instead. This sends the user through the bulk operation UI and is similar to bulk edits. It's a little clunky for now, but you get your data at the end, which is far better than hanging for 30 seconds and then fataling.

Test Plan: Exported small result sets, got the same workflow as before. Exported very large result sets, went through the bulk flow, got reasonable results out.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13049

Differential Revision: https://secure.phabricator.com/D18962

src/__phutil_library_map__.php

@@ -2845,6 +2845,8 @@ phutil_register_library_map(array(
     'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
     'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php',
     'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
+    'PhabricatorExportEngine' => 'infrastructure/export/engine/PhabricatorExportEngine.php',
+    'PhabricatorExportEngineBulkJobType' => 'infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php',
     'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php',
     'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php',
     'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php',
@@ -4419,6 +4421,7 @@ phutil_register_library_map(array(
     'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php',
     'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php',
     'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php',
+    'PhabricatorWorkerSingleBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php',
     'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
     'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
     'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php',
@@ -8286,6 +8289,8 @@ phutil_register_library_map(array(
     'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
     'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat',
     'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
+    'PhabricatorExportEngine' => 'Phobject',
+    'PhabricatorExportEngineBulkJobType' => 'PhabricatorWorkerSingleBulkJobType',
     'PhabricatorExportEngineExtension' => 'Phobject',
     'PhabricatorExportField' => 'Phobject',
     'PhabricatorExportFormat' => 'Phobject',
@@ -10154,6 +10159,7 @@ phutil_register_library_map(array(
     'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorWorkerPermanentFailureException' => 'Exception',
     'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec',
+    'PhabricatorWorkerSingleBulkJobType' => 'PhabricatorWorkerBulkJobType',
     'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
     'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
     'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController',

src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php

@@ -71,18 +71,10 @@ final class PhabricatorDaemonBulkJobViewController
     $viewer = $this->getViewer();
     $curtain = $this->newCurtainView($job);
 
-    if ($job->isConfirming()) {
-      $continue_uri = $job->getMonitorURI();
-    } else {
-      $continue_uri = $job->getDoneURI();
+    foreach ($job->getCurtainActions($viewer) as $action) {
+      $curtain->addAction($action);
     }
 
-    $curtain->addAction(
-      id(new PhabricatorActionView())
-        ->setHref($continue_uri)
-        ->setIcon('fa-arrow-circle-o-right')
-        ->setName(pht('Continue')));
-
     return $curtain;
   }
 

src/applications/search/controller/PhabricatorApplicationSearchController.php

@@ -444,6 +444,17 @@ final class PhabricatorApplicationSearchController
       $format_key = head_key($format_options);
     }
 
+    // Check if this is a large result set or not. If we're exporting a
+    // large amount of data, we'll build the actual export file in the daemons.
+
+    $threshold = 1000;
+    $query = $engine->buildQueryFromSavedQuery($saved_query);
+    $pager = $engine->newPagerForSavedQuery($saved_query);
+    $pager->setPageSize($threshold + 1);
+    $objects = $engine->executeQuery($query, $pager);
+    $object_count = count($objects);
+    $is_large_export = ($object_count > $threshold);
+
     $errors = array();
 
     $e_format = null;
@@ -475,50 +486,21 @@ final class PhabricatorApplicationSearchController
       if (!$errors) {
         $this->writeExportFormatPreference($format_key);
 
-        $query = $engine->buildQueryFromSavedQuery($saved_query);
-
-        // NOTE: We aren't reading the pager from the request. Exports always
-        // affect the entire result set.
-        $pager = $engine->newPagerForSavedQuery($saved_query);
-        $pager->setPageSize(0x7FFFFFFF);
-
-        $objects = $engine->executeQuery($query, $pager);
-
-        $extension = $format->getFileExtension();
-        $mime_type = $format->getMIMEContentType();
-        $filename = $filename.'.'.$extension;
-
-        $format = id(clone $format)
+        $export_engine = id(new PhabricatorExportEngine())
           ->setViewer($viewer)
-          ->setTitle($sheet_title);
+          ->setSearchEngine($engine)
+          ->setSavedQuery($saved_query)
+          ->setTitle($sheet_title)
+          ->setFilename($filename)
+          ->setExportFormat($format);
 
-        $export_data = $engine->newExport($objects);
-        $objects = array_values($objects);
+        if ($is_large_export) {
+          $job = $export_engine->newBulkJob($request);
 
-        $field_list = $engine->newExportFieldList();
-        $field_list = mpull($field_list, null, 'getKey');
-
-        $format->addHeaders($field_list);
-        for ($ii = 0; $ii < count($objects); $ii++) {
-          $format->addObject($objects[$ii], $field_list, $export_data[$ii]);
-        }
-
-        $export_result = $format->newFileData();
-
-        // We have all the data in one big string and aren't actually
-        // streaming it, but pretending that we are allows us to actviate
-        // the chunk engine and store large files.
-        $iterator = new ArrayIterator(array($export_result));
-
-        $source = id(new PhabricatorIteratorFileUploadSource())
-          ->setName($filename)
-          ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
-          ->setMIMEType($mime_type)
-          ->setRelativeTTL(phutil_units('60 minutes in seconds'))
-          ->setAuthorPHID($viewer->getPHID())
-          ->setIterator($iterator);
-
-        $file = $source->uploadFile();
+          return id(new AphrontRedirectResponse())
+            ->setURI($job->getMonitorURI());
+        } else {
+          $file = $export_engine->exportFile();
 
           return $this->newDialog()
             ->setTitle(pht('Download Results'))
@@ -530,6 +512,7 @@ final class PhabricatorApplicationSearchController
             ->addSubmitButton(pht('Download Data'));
         }
       }
+    }
 
     $export_form = id(new AphrontFormView())
       ->setViewer($viewer)

src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php

@@ -25,4 +25,24 @@ abstract class PhabricatorWorkerBulkJobType extends Phobject {
       ->execute();
   }
 
+  public function getCurtainActions(
+    PhabricatorUser $viewer,
+    PhabricatorWorkerBulkJob $job) {
+
+    if ($job->isConfirming()) {
+      $continue_uri = $job->getMonitorURI();
+    } else {
+      $continue_uri = $job->getDoneURI();
+    }
+
+    $continue = id(new PhabricatorActionView())
+      ->setHref($continue_uri)
+      ->setIcon('fa-arrow-circle-o-right')
+      ->setName(pht('Continue'));
+
+    return array(
+      $continue,
+    );
+  }
+
 }

src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php

@@ -77,6 +77,10 @@ abstract class PhabricatorWorkerBulkJobWorker
         pht('Job actor does not have permission to edit job.'));
     }
 
+    // Allow the worker to fill user caches inline; bulk jobs occasionally
+    // need to access user preferences.
+    $actor->setAllowInlineCacheGeneration(true);
+
     return $actor;
   }
 

src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php

@@ -0,0 +1,27 @@
+<?php
+
+/**
+ * An bulk job which can not be parallelized and executes only one task.
+ */
+abstract class PhabricatorWorkerSingleBulkJobType
+  extends PhabricatorWorkerBulkJobType {
+
+  public function getDescriptionForConfirm(PhabricatorWorkerBulkJob $job) {
+    return null;
+  }
+
+  public function getJobSize(PhabricatorWorkerBulkJob $job) {
+    return 1;
+  }
+
+  public function createTasks(PhabricatorWorkerBulkJob $job) {
+    $tasks = array();
+
+    $tasks[] = PhabricatorWorkerBulkTask::initializeNewTask(
+      $job,
+      $job->getPHID());
+
+    return $tasks;
+  }
+
+}

src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php

@@ -180,6 +180,10 @@ final class PhabricatorWorkerBulkJob
     return $this->getJobImplementation()->getJobName($this);
   }
 
+  public function getCurtainActions(PhabricatorUser $viewer) {
+    return $this->getJobImplementation()->getCurtainActions($viewer, $this);
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 

src/infrastructure/export/engine/PhabricatorExportEngine.php

@@ -0,0 +1,168 @@
+<?php
+
+final class PhabricatorExportEngine
+  extends Phobject {
+
+  private $viewer;
+  private $searchEngine;
+  private $savedQuery;
+  private $exportFormat;
+  private $filename;
+  private $title;
+
+  public function setViewer(PhabricatorUser $viewer) {
+    $this->viewer = $viewer;
+    return $this;
+  }
+
+  public function getViewer() {
+    return $this->viewer;
+  }
+
+  public function setSearchEngine(
+    PhabricatorApplicationSearchEngine $search_engine) {
+    $this->searchEngine = $search_engine;
+    return $this;
+  }
+
+  public function getSearchEngine() {
+    return $this->searchEngine;
+  }
+
+  public function setSavedQuery(PhabricatorSavedQuery $saved_query) {
+    $this->savedQuery = $saved_query;
+    return $this;
+  }
+
+  public function getSavedQuery() {
+    return $this->savedQuery;
+  }
+
+  public function setExportFormat(
+    PhabricatorExportFormat $export_format) {
+    $this->exportFormat = $export_format;
+    return $this;
+  }
+
+  public function getExportFormat() {
+    return $this->exportFormat;
+  }
+
+  public function setFilename($filename) {
+    $this->filename = $filename;
+    return $this;
+  }
+
+  public function getFilename() {
+    return $this->filename;
+  }
+
+  public function setTitle($title) {
+    $this->title = $title;
+    return $this;
+  }
+
+  public function getTitle() {
+    return $this->title;
+  }
+
+  public function newBulkJob(AphrontRequest $request) {
+    $viewer = $this->getViewer();
+    $engine = $this->getSearchEngine();
+    $saved_query = $this->getSavedQuery();
+    $format = $this->getExportFormat();
+
+    $params = array(
+      'engineClass' => get_class($engine),
+      'queryKey' => $saved_query->getQueryKey(),
+      'formatKey' => $format->getExportFormatKey(),
+      'title' => $this->getTitle(),
+      'filename' => $this->getFilename(),
+    );
+
+    $job = PhabricatorWorkerBulkJob::initializeNewJob(
+      $viewer,
+      new PhabricatorExportEngineBulkJobType(),
+      $params);
+
+    // We queue these jobs directly into STATUS_WAITING without requiring
+    // a confirmation from the user.
+
+    $xactions = array();
+
+    $xactions[] = id(new PhabricatorWorkerBulkJobTransaction())
+      ->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS)
+      ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING);
+
+    $editor = id(new PhabricatorWorkerBulkJobEditor())
+      ->setActor($viewer)
+      ->setContentSourceFromRequest($request)
+      ->setContinueOnMissingFields(true)
+      ->applyTransactions($job, $xactions);
+
+    return $job;
+  }
+
+  public function exportFile() {
+    $viewer = $this->getViewer();
+    $engine = $this->getSearchEngine();
+    $saved_query = $this->getSavedQuery();
+    $format = $this->getExportFormat();
+    $title = $this->getTitle();
+    $filename = $this->getFilename();
+
+    $query = $engine->buildQueryFromSavedQuery($saved_query);
+
+    $extension = $format->getFileExtension();
+    $mime_type = $format->getMIMEContentType();
+    $filename = $filename.'.'.$extension;
+
+    $format = id(clone $format)
+      ->setViewer($viewer)
+      ->setTitle($title);
+
+    $field_list = $engine->newExportFieldList();
+    $field_list = mpull($field_list, null, 'getKey');
+    $format->addHeaders($field_list);
+
+    // Iterate over the query results in large page so we don't have to hold
+    // too much stuff in memory.
+    $page_size = 1000;
+    $page_cursor = null;
+    do {
+      $pager = $engine->newPagerForSavedQuery($saved_query);
+      $pager->setPageSize($page_size);
+
+      if ($page_cursor !== null) {
+        $pager->setAfterID($page_cursor);
+      }
+
+      $objects = $engine->executeQuery($query, $pager);
+      $objects = array_values($objects);
+      $page_cursor = $pager->getNextPageID();
+
+      $export_data = $engine->newExport($objects);
+      for ($ii = 0; $ii < count($objects); $ii++) {
+        $format->addObject($objects[$ii], $field_list, $export_data[$ii]);
+      }
+    } while ($pager->getHasMoreResults());
+
+    $export_result = $format->newFileData();
+
+    // We have all the data in one big string and aren't actually
+    // streaming it, but pretending that we are allows us to actviate
+    // the chunk engine and store large files.
+    $iterator = new ArrayIterator(array($export_result));
+
+    $source = id(new PhabricatorIteratorFileUploadSource())
+      ->setName($filename)
+      ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
+      ->setMIMEType($mime_type)
+      ->setRelativeTTL(phutil_units('60 minutes in seconds'))
+      ->setAuthorPHID($viewer->getPHID())
+      ->setIterator($iterator);
+
+    return $source->uploadFile();
+  }
+
+}

src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php

@@ -0,0 +1,118 @@
+<?php
+
+final class PhabricatorExportEngineBulkJobType
+   extends PhabricatorWorkerSingleBulkJobType {
+
+  public function getBulkJobTypeKey() {
+    return 'export';
+  }
+
+  public function getJobName(PhabricatorWorkerBulkJob $job) {
+    return pht('Data Export');
+  }
+
+  public function getCurtainActions(
+    PhabricatorUser $viewer,
+    PhabricatorWorkerBulkJob $job) {
+    $actions = array();
+
+    $file_phid = $job->getParameter('filePHID');
+    if (!$file_phid) {
+      $actions[] = id(new PhabricatorActionView())
+        ->setHref('#')
+        ->setIcon('fa-download')
+        ->setDisabled(true)
+        ->setName(pht('Exporting Data...'));
+    } else {
+      $file = id(new PhabricatorFileQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($file_phid))
+        ->executeOne();
+      if (!$file) {
+        $actions[] = id(new PhabricatorActionView())
+          ->setHref('#')
+          ->setIcon('fa-download')
+          ->setDisabled(true)
+          ->setName(pht('Temporary File Expired'));
+      } else {
+        $actions[] = id(new PhabricatorActionView())
+          ->setRenderAsForm(true)
+          ->setHref($file->getDownloadURI())
+          ->setIcon('fa-download')
+          ->setName(pht('Download Data Export'));
+      }
+    }
+
+    return $actions;
+  }
+
+
+  public function runTask(
+    PhabricatorUser $actor,
+    PhabricatorWorkerBulkJob $job,
+    PhabricatorWorkerBulkTask $task) {
+
+    $engine_class = $job->getParameter('engineClass');
+    if (!is_subclass_of($engine_class, 'PhabricatorApplicationSearchEngine')) {
+      throw new Exception(
+        pht(
+          'Unknown search engine class "%s".',
+          $engine_class));
+    }
+
+    $engine = newv($engine_class, array())
+      ->setViewer($actor);
+
+    $query_key = $job->getParameter('queryKey');
+    if ($engine->isBuiltinQuery($query_key)) {
+      $saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
+    } else if ($query_key) {
+      $saved_query = id(new PhabricatorSavedQueryQuery())
+        ->setViewer($actor)
+        ->withQueryKeys(array($query_key))
+        ->executeOne();
+    } else {
+      $saved_query = null;
+    }
+
+    if (!$saved_query) {
+      throw new Exception(
+        pht(
+          'Failed to load saved query ("%s").',
+          $query_key));
+    }
+
+    $format_key = $job->getParameter('formatKey');
+
+    $all_formats = PhabricatorExportFormat::getAllExportFormats();
+    $format = idx($all_formats, $format_key);
+    if (!$format) {
+      throw new Exception(
+        pht(
+          'Unknown export format ("%s").',
+          $format_key));
+    }
+
+    if (!$format->isExportFormatEnabled()) {
+      throw new Exception(
+        pht(
+          'Export format ("%s") is not enabled.',
+          $format_key));
+    }
+
+    $export_engine = id(new PhabricatorExportEngine())
+      ->setViewer($actor)
+      ->setTitle($job->getParameter('title'))
+      ->setFilename($job->getParameter('filename'))
+      ->setSearchEngine($engine)
+      ->setSavedQuery($saved_query)
+      ->setExportFormat($format);
+
+    $file = $export_engine->exportFile();
+
+    $job
+      ->setParameter('filePHID', $file->getPHID())
+      ->save();
+  }
+
+}