From 84df1220858f61f13ee33506edfc8ec61af920c7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 10:29:49 -0800 Subject: [PATCH] When exporting more than 1,000 records, export in the background Summary: Depends on D18961. Ref T13049. Currently, longer exports don't give the user any feedback, and exports that take longer than 30 seconds are likely to timeout. For small exports (up to 1,000 rows) continue doing the export in the web process. For large exports, queue a bulk job and do them in the workers instead. This sends the user through the bulk operation UI and is similar to bulk edits. It's a little clunky for now, but you get your data at the end, which is far better than hanging for 30 seconds and then fataling. Test Plan: Exported small result sets, got the same workflow as before. Exported very large result sets, went through the bulk flow, got reasonable results out. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18962 --- src/__phutil_library_map__.php | 6 + ...PhabricatorDaemonBulkJobViewController.php | 12 +- ...PhabricatorApplicationSearchController.php | 79 ++++---- .../bulk/PhabricatorWorkerBulkJobType.php | 20 +++ .../bulk/PhabricatorWorkerBulkJobWorker.php | 4 + .../PhabricatorWorkerSingleBulkJobType.php | 27 +++ .../storage/PhabricatorWorkerBulkJob.php | 4 + .../export/engine/PhabricatorExportEngine.php | 168 ++++++++++++++++++ .../PhabricatorExportEngineBulkJobType.php | 118 ++++++++++++ 9 files changed, 380 insertions(+), 58 deletions(-) create mode 100644 src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php create mode 100644 src/infrastructure/export/engine/PhabricatorExportEngine.php create mode 100644 src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 1f71d97460..06f619d74e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2845,6 +2845,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', + 'PhabricatorExportEngine' => 'infrastructure/export/engine/PhabricatorExportEngine.php', + 'PhabricatorExportEngineBulkJobType' => 'infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php', 'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php', 'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php', @@ -4419,6 +4421,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php', 'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php', 'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php', + 'PhabricatorWorkerSingleBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php', 'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php', 'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php', 'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php', @@ -8286,6 +8289,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', + 'PhabricatorExportEngine' => 'Phobject', + 'PhabricatorExportEngineBulkJobType' => 'PhabricatorWorkerSingleBulkJobType', 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', 'PhabricatorExportFormat' => 'Phobject', @@ -10154,6 +10159,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorWorkerPermanentFailureException' => 'Exception', 'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec', + 'PhabricatorWorkerSingleBulkJobType' => 'PhabricatorWorkerBulkJobType', 'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController', diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php index f7aa396e94..00fb297fe0 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php @@ -71,18 +71,10 @@ final class PhabricatorDaemonBulkJobViewController $viewer = $this->getViewer(); $curtain = $this->newCurtainView($job); - if ($job->isConfirming()) { - $continue_uri = $job->getMonitorURI(); - } else { - $continue_uri = $job->getDoneURI(); + foreach ($job->getCurtainActions($viewer) as $action) { + $curtain->addAction($action); } - $curtain->addAction( - id(new PhabricatorActionView()) - ->setHref($continue_uri) - ->setIcon('fa-arrow-circle-o-right') - ->setName(pht('Continue'))); - return $curtain; } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index ea46886a15..6c7d80eafe 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -444,6 +444,17 @@ final class PhabricatorApplicationSearchController $format_key = head_key($format_options); } + // Check if this is a large result set or not. If we're exporting a + // large amount of data, we'll build the actual export file in the daemons. + + $threshold = 1000; + $query = $engine->buildQueryFromSavedQuery($saved_query); + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($threshold + 1); + $objects = $engine->executeQuery($query, $pager); + $object_count = count($objects); + $is_large_export = ($object_count > $threshold); + $errors = array(); $e_format = null; @@ -475,59 +486,31 @@ final class PhabricatorApplicationSearchController if (!$errors) { $this->writeExportFormatPreference($format_key); - $query = $engine->buildQueryFromSavedQuery($saved_query); - - // NOTE: We aren't reading the pager from the request. Exports always - // affect the entire result set. - $pager = $engine->newPagerForSavedQuery($saved_query); - $pager->setPageSize(0x7FFFFFFF); - - $objects = $engine->executeQuery($query, $pager); - - $extension = $format->getFileExtension(); - $mime_type = $format->getMIMEContentType(); - $filename = $filename.'.'.$extension; - - $format = id(clone $format) + $export_engine = id(new PhabricatorExportEngine()) ->setViewer($viewer) - ->setTitle($sheet_title); + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setTitle($sheet_title) + ->setFilename($filename) + ->setExportFormat($format); - $export_data = $engine->newExport($objects); - $objects = array_values($objects); + if ($is_large_export) { + $job = $export_engine->newBulkJob($request); - $field_list = $engine->newExportFieldList(); - $field_list = mpull($field_list, null, 'getKey'); + return id(new AphrontRedirectResponse()) + ->setURI($job->getMonitorURI()); + } else { + $file = $export_engine->exportFile(); - $format->addHeaders($field_list); - for ($ii = 0; $ii < count($objects); $ii++) { - $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + return $this->newDialog() + ->setTitle(pht('Download Results')) + ->appendParagraph( + pht('Click the download button to download the exported data.')) + ->addCancelButton($cancel_uri, pht('Done')) + ->setSubmitURI($file->getDownloadURI()) + ->setDisableWorkflowOnSubmit(true) + ->addSubmitButton(pht('Download Data')); } - - $export_result = $format->newFileData(); - - // We have all the data in one big string and aren't actually - // streaming it, but pretending that we are allows us to actviate - // the chunk engine and store large files. - $iterator = new ArrayIterator(array($export_result)); - - $source = id(new PhabricatorIteratorFileUploadSource()) - ->setName($filename) - ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) - ->setMIMEType($mime_type) - ->setRelativeTTL(phutil_units('60 minutes in seconds')) - ->setAuthorPHID($viewer->getPHID()) - ->setIterator($iterator); - - $file = $source->uploadFile(); - - return $this->newDialog() - ->setTitle(pht('Download Results')) - ->appendParagraph( - pht('Click the download button to download the exported data.')) - ->addCancelButton($cancel_uri, pht('Done')) - ->setSubmitURI($file->getDownloadURI()) - ->setDisableWorkflowOnSubmit(true) - ->addSubmitButton(pht('Download Data')); } } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php index a5e29bc101..7854730bea 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php @@ -25,4 +25,24 @@ abstract class PhabricatorWorkerBulkJobType extends Phobject { ->execute(); } + public function getCurtainActions( + PhabricatorUser $viewer, + PhabricatorWorkerBulkJob $job) { + + if ($job->isConfirming()) { + $continue_uri = $job->getMonitorURI(); + } else { + $continue_uri = $job->getDoneURI(); + } + + $continue = id(new PhabricatorActionView()) + ->setHref($continue_uri) + ->setIcon('fa-arrow-circle-o-right') + ->setName(pht('Continue')); + + return array( + $continue, + ); + } + } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php index 9117060da6..d11f7e81af 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php @@ -77,6 +77,10 @@ abstract class PhabricatorWorkerBulkJobWorker pht('Job actor does not have permission to edit job.')); } + // Allow the worker to fill user caches inline; bulk jobs occasionally + // need to access user preferences. + $actor->setAllowInlineCacheGeneration(true); + return $actor; } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php new file mode 100644 index 0000000000..f80411348c --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php @@ -0,0 +1,27 @@ +getPHID()); + + return $tasks; + } + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php index 40e43c2ec7..312281617d 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php @@ -180,6 +180,10 @@ final class PhabricatorWorkerBulkJob return $this->getJobImplementation()->getJobName($this); } + public function getCurtainActions(PhabricatorUser $viewer) { + return $this->getJobImplementation()->getCurtainActions($viewer, $this); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/infrastructure/export/engine/PhabricatorExportEngine.php b/src/infrastructure/export/engine/PhabricatorExportEngine.php new file mode 100644 index 0000000000..f2c6a1270b --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngine.php @@ -0,0 +1,168 @@ +viewer = $viewer; + return $this; + } + + public function getViewer() { + return $this->viewer; + } + + public function setSearchEngine( + PhabricatorApplicationSearchEngine $search_engine) { + $this->searchEngine = $search_engine; + return $this; + } + + public function getSearchEngine() { + return $this->searchEngine; + } + + public function setSavedQuery(PhabricatorSavedQuery $saved_query) { + $this->savedQuery = $saved_query; + return $this; + } + + public function getSavedQuery() { + return $this->savedQuery; + } + + public function setExportFormat( + PhabricatorExportFormat $export_format) { + $this->exportFormat = $export_format; + return $this; + } + + public function getExportFormat() { + return $this->exportFormat; + } + + public function setFilename($filename) { + $this->filename = $filename; + return $this; + } + + public function getFilename() { + return $this->filename; + } + + public function setTitle($title) { + $this->title = $title; + return $this; + } + + public function getTitle() { + return $this->title; + } + + public function newBulkJob(AphrontRequest $request) { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + + $params = array( + 'engineClass' => get_class($engine), + 'queryKey' => $saved_query->getQueryKey(), + 'formatKey' => $format->getExportFormatKey(), + 'title' => $this->getTitle(), + 'filename' => $this->getFilename(), + ); + + $job = PhabricatorWorkerBulkJob::initializeNewJob( + $viewer, + new PhabricatorExportEngineBulkJobType(), + $params); + + // We queue these jobs directly into STATUS_WAITING without requiring + // a confirmation from the user. + + $xactions = array(); + + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS) + ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + + return $job; + } + + public function exportFile() { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + $title = $this->getTitle(); + $filename = $this->getFilename(); + + $query = $engine->buildQueryFromSavedQuery($saved_query); + + $extension = $format->getFileExtension(); + $mime_type = $format->getMIMEContentType(); + $filename = $filename.'.'.$extension; + + $format = id(clone $format) + ->setViewer($viewer) + ->setTitle($title); + + $field_list = $engine->newExportFieldList(); + $field_list = mpull($field_list, null, 'getKey'); + $format->addHeaders($field_list); + + // Iterate over the query results in large page so we don't have to hold + // too much stuff in memory. + $page_size = 1000; + $page_cursor = null; + do { + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($page_size); + + if ($page_cursor !== null) { + $pager->setAfterID($page_cursor); + } + + $objects = $engine->executeQuery($query, $pager); + $objects = array_values($objects); + $page_cursor = $pager->getNextPageID(); + + $export_data = $engine->newExport($objects); + for ($ii = 0; $ii < count($objects); $ii++) { + $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + } + } while ($pager->getHasMoreResults()); + + $export_result = $format->newFileData(); + + // We have all the data in one big string and aren't actually + // streaming it, but pretending that we are allows us to actviate + // the chunk engine and store large files. + $iterator = new ArrayIterator(array($export_result)); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($filename) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($mime_type) + ->setRelativeTTL(phutil_units('60 minutes in seconds')) + ->setAuthorPHID($viewer->getPHID()) + ->setIterator($iterator); + + return $source->uploadFile(); + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php new file mode 100644 index 0000000000..712127f479 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php @@ -0,0 +1,118 @@ +getParameter('filePHID'); + if (!$file_phid) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Exporting Data...')); + } else { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if (!$file) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Temporary File Expired')); + } else { + $actions[] = id(new PhabricatorActionView()) + ->setRenderAsForm(true) + ->setHref($file->getDownloadURI()) + ->setIcon('fa-download') + ->setName(pht('Download Data Export')); + } + } + + return $actions; + } + + + public function runTask( + PhabricatorUser $actor, + PhabricatorWorkerBulkJob $job, + PhabricatorWorkerBulkTask $task) { + + $engine_class = $job->getParameter('engineClass'); + if (!is_subclass_of($engine_class, 'PhabricatorApplicationSearchEngine')) { + throw new Exception( + pht( + 'Unknown search engine class "%s".', + $engine_class)); + } + + $engine = newv($engine_class, array()) + ->setViewer($actor); + + $query_key = $job->getParameter('queryKey'); + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($actor) + ->withQueryKeys(array($query_key)) + ->executeOne(); + } else { + $saved_query = null; + } + + if (!$saved_query) { + throw new Exception( + pht( + 'Failed to load saved query ("%s").', + $query_key)); + } + + $format_key = $job->getParameter('formatKey'); + + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + $format = idx($all_formats, $format_key); + if (!$format) { + throw new Exception( + pht( + 'Unknown export format ("%s").', + $format_key)); + } + + if (!$format->isExportFormatEnabled()) { + throw new Exception( + pht( + 'Export format ("%s") is not enabled.', + $format_key)); + } + + $export_engine = id(new PhabricatorExportEngine()) + ->setViewer($actor) + ->setTitle($job->getParameter('title')) + ->setFilename($job->getParameter('filename')) + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setExportFormat($format); + + $file = $export_engine->exportFile(); + + $job + ->setParameter('filePHID', $file->getPHID()) + ->save(); + } + +}