diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 1f71d97460..06f619d74e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2845,6 +2845,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', + 'PhabricatorExportEngine' => 'infrastructure/export/engine/PhabricatorExportEngine.php', + 'PhabricatorExportEngineBulkJobType' => 'infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php', 'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php', 'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php', @@ -4419,6 +4421,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php', 'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php', 'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php', + 'PhabricatorWorkerSingleBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php', 'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php', 'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php', 'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php', @@ -8286,6 +8289,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', + 'PhabricatorExportEngine' => 'Phobject', + 'PhabricatorExportEngineBulkJobType' => 'PhabricatorWorkerSingleBulkJobType', 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', 'PhabricatorExportFormat' => 'Phobject', @@ -10154,6 +10159,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorWorkerPermanentFailureException' => 'Exception', 'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec', + 'PhabricatorWorkerSingleBulkJobType' => 'PhabricatorWorkerBulkJobType', 'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController', diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php index f7aa396e94..00fb297fe0 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php @@ -71,18 +71,10 @@ final class PhabricatorDaemonBulkJobViewController $viewer = $this->getViewer(); $curtain = $this->newCurtainView($job); - if ($job->isConfirming()) { - $continue_uri = $job->getMonitorURI(); - } else { - $continue_uri = $job->getDoneURI(); + foreach ($job->getCurtainActions($viewer) as $action) { + $curtain->addAction($action); } - $curtain->addAction( - id(new PhabricatorActionView()) - ->setHref($continue_uri) - ->setIcon('fa-arrow-circle-o-right') - ->setName(pht('Continue'))); - return $curtain; } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index ea46886a15..6c7d80eafe 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -444,6 +444,17 @@ final class PhabricatorApplicationSearchController $format_key = head_key($format_options); } + // Check if this is a large result set or not. If we're exporting a + // large amount of data, we'll build the actual export file in the daemons. + + $threshold = 1000; + $query = $engine->buildQueryFromSavedQuery($saved_query); + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($threshold + 1); + $objects = $engine->executeQuery($query, $pager); + $object_count = count($objects); + $is_large_export = ($object_count > $threshold); + $errors = array(); $e_format = null; @@ -475,59 +486,31 @@ final class PhabricatorApplicationSearchController if (!$errors) { $this->writeExportFormatPreference($format_key); - $query = $engine->buildQueryFromSavedQuery($saved_query); - - // NOTE: We aren't reading the pager from the request. Exports always - // affect the entire result set. - $pager = $engine->newPagerForSavedQuery($saved_query); - $pager->setPageSize(0x7FFFFFFF); - - $objects = $engine->executeQuery($query, $pager); - - $extension = $format->getFileExtension(); - $mime_type = $format->getMIMEContentType(); - $filename = $filename.'.'.$extension; - - $format = id(clone $format) + $export_engine = id(new PhabricatorExportEngine()) ->setViewer($viewer) - ->setTitle($sheet_title); + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setTitle($sheet_title) + ->setFilename($filename) + ->setExportFormat($format); - $export_data = $engine->newExport($objects); - $objects = array_values($objects); + if ($is_large_export) { + $job = $export_engine->newBulkJob($request); - $field_list = $engine->newExportFieldList(); - $field_list = mpull($field_list, null, 'getKey'); + return id(new AphrontRedirectResponse()) + ->setURI($job->getMonitorURI()); + } else { + $file = $export_engine->exportFile(); - $format->addHeaders($field_list); - for ($ii = 0; $ii < count($objects); $ii++) { - $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + return $this->newDialog() + ->setTitle(pht('Download Results')) + ->appendParagraph( + pht('Click the download button to download the exported data.')) + ->addCancelButton($cancel_uri, pht('Done')) + ->setSubmitURI($file->getDownloadURI()) + ->setDisableWorkflowOnSubmit(true) + ->addSubmitButton(pht('Download Data')); } - - $export_result = $format->newFileData(); - - // We have all the data in one big string and aren't actually - // streaming it, but pretending that we are allows us to actviate - // the chunk engine and store large files. - $iterator = new ArrayIterator(array($export_result)); - - $source = id(new PhabricatorIteratorFileUploadSource()) - ->setName($filename) - ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) - ->setMIMEType($mime_type) - ->setRelativeTTL(phutil_units('60 minutes in seconds')) - ->setAuthorPHID($viewer->getPHID()) - ->setIterator($iterator); - - $file = $source->uploadFile(); - - return $this->newDialog() - ->setTitle(pht('Download Results')) - ->appendParagraph( - pht('Click the download button to download the exported data.')) - ->addCancelButton($cancel_uri, pht('Done')) - ->setSubmitURI($file->getDownloadURI()) - ->setDisableWorkflowOnSubmit(true) - ->addSubmitButton(pht('Download Data')); } } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php index a5e29bc101..7854730bea 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php @@ -25,4 +25,24 @@ abstract class PhabricatorWorkerBulkJobType extends Phobject { ->execute(); } + public function getCurtainActions( + PhabricatorUser $viewer, + PhabricatorWorkerBulkJob $job) { + + if ($job->isConfirming()) { + $continue_uri = $job->getMonitorURI(); + } else { + $continue_uri = $job->getDoneURI(); + } + + $continue = id(new PhabricatorActionView()) + ->setHref($continue_uri) + ->setIcon('fa-arrow-circle-o-right') + ->setName(pht('Continue')); + + return array( + $continue, + ); + } + } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php index 9117060da6..d11f7e81af 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php @@ -77,6 +77,10 @@ abstract class PhabricatorWorkerBulkJobWorker pht('Job actor does not have permission to edit job.')); } + // Allow the worker to fill user caches inline; bulk jobs occasionally + // need to access user preferences. + $actor->setAllowInlineCacheGeneration(true); + return $actor; } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php new file mode 100644 index 0000000000..f80411348c --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php @@ -0,0 +1,27 @@ +getPHID()); + + return $tasks; + } + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php index 40e43c2ec7..312281617d 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php @@ -180,6 +180,10 @@ final class PhabricatorWorkerBulkJob return $this->getJobImplementation()->getJobName($this); } + public function getCurtainActions(PhabricatorUser $viewer) { + return $this->getJobImplementation()->getCurtainActions($viewer, $this); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/infrastructure/export/engine/PhabricatorExportEngine.php b/src/infrastructure/export/engine/PhabricatorExportEngine.php new file mode 100644 index 0000000000..f2c6a1270b --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngine.php @@ -0,0 +1,168 @@ +viewer = $viewer; + return $this; + } + + public function getViewer() { + return $this->viewer; + } + + public function setSearchEngine( + PhabricatorApplicationSearchEngine $search_engine) { + $this->searchEngine = $search_engine; + return $this; + } + + public function getSearchEngine() { + return $this->searchEngine; + } + + public function setSavedQuery(PhabricatorSavedQuery $saved_query) { + $this->savedQuery = $saved_query; + return $this; + } + + public function getSavedQuery() { + return $this->savedQuery; + } + + public function setExportFormat( + PhabricatorExportFormat $export_format) { + $this->exportFormat = $export_format; + return $this; + } + + public function getExportFormat() { + return $this->exportFormat; + } + + public function setFilename($filename) { + $this->filename = $filename; + return $this; + } + + public function getFilename() { + return $this->filename; + } + + public function setTitle($title) { + $this->title = $title; + return $this; + } + + public function getTitle() { + return $this->title; + } + + public function newBulkJob(AphrontRequest $request) { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + + $params = array( + 'engineClass' => get_class($engine), + 'queryKey' => $saved_query->getQueryKey(), + 'formatKey' => $format->getExportFormatKey(), + 'title' => $this->getTitle(), + 'filename' => $this->getFilename(), + ); + + $job = PhabricatorWorkerBulkJob::initializeNewJob( + $viewer, + new PhabricatorExportEngineBulkJobType(), + $params); + + // We queue these jobs directly into STATUS_WAITING without requiring + // a confirmation from the user. + + $xactions = array(); + + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS) + ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + + return $job; + } + + public function exportFile() { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + $title = $this->getTitle(); + $filename = $this->getFilename(); + + $query = $engine->buildQueryFromSavedQuery($saved_query); + + $extension = $format->getFileExtension(); + $mime_type = $format->getMIMEContentType(); + $filename = $filename.'.'.$extension; + + $format = id(clone $format) + ->setViewer($viewer) + ->setTitle($title); + + $field_list = $engine->newExportFieldList(); + $field_list = mpull($field_list, null, 'getKey'); + $format->addHeaders($field_list); + + // Iterate over the query results in large page so we don't have to hold + // too much stuff in memory. + $page_size = 1000; + $page_cursor = null; + do { + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($page_size); + + if ($page_cursor !== null) { + $pager->setAfterID($page_cursor); + } + + $objects = $engine->executeQuery($query, $pager); + $objects = array_values($objects); + $page_cursor = $pager->getNextPageID(); + + $export_data = $engine->newExport($objects); + for ($ii = 0; $ii < count($objects); $ii++) { + $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + } + } while ($pager->getHasMoreResults()); + + $export_result = $format->newFileData(); + + // We have all the data in one big string and aren't actually + // streaming it, but pretending that we are allows us to actviate + // the chunk engine and store large files. + $iterator = new ArrayIterator(array($export_result)); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($filename) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($mime_type) + ->setRelativeTTL(phutil_units('60 minutes in seconds')) + ->setAuthorPHID($viewer->getPHID()) + ->setIterator($iterator); + + return $source->uploadFile(); + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php new file mode 100644 index 0000000000..712127f479 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php @@ -0,0 +1,118 @@ +getParameter('filePHID'); + if (!$file_phid) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Exporting Data...')); + } else { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if (!$file) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Temporary File Expired')); + } else { + $actions[] = id(new PhabricatorActionView()) + ->setRenderAsForm(true) + ->setHref($file->getDownloadURI()) + ->setIcon('fa-download') + ->setName(pht('Download Data Export')); + } + } + + return $actions; + } + + + public function runTask( + PhabricatorUser $actor, + PhabricatorWorkerBulkJob $job, + PhabricatorWorkerBulkTask $task) { + + $engine_class = $job->getParameter('engineClass'); + if (!is_subclass_of($engine_class, 'PhabricatorApplicationSearchEngine')) { + throw new Exception( + pht( + 'Unknown search engine class "%s".', + $engine_class)); + } + + $engine = newv($engine_class, array()) + ->setViewer($actor); + + $query_key = $job->getParameter('queryKey'); + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($actor) + ->withQueryKeys(array($query_key)) + ->executeOne(); + } else { + $saved_query = null; + } + + if (!$saved_query) { + throw new Exception( + pht( + 'Failed to load saved query ("%s").', + $query_key)); + } + + $format_key = $job->getParameter('formatKey'); + + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + $format = idx($all_formats, $format_key); + if (!$format) { + throw new Exception( + pht( + 'Unknown export format ("%s").', + $format_key)); + } + + if (!$format->isExportFormatEnabled()) { + throw new Exception( + pht( + 'Export format ("%s") is not enabled.', + $format_key)); + } + + $export_engine = id(new PhabricatorExportEngine()) + ->setViewer($actor) + ->setTitle($job->getParameter('title')) + ->setFilename($job->getParameter('filename')) + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setExportFormat($format); + + $file = $export_engine->exportFile(); + + $job + ->setParameter('filePHID', $file->getPHID()) + ->save(); + } + +}