1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00

Add a basic, general-purpose export workflow for all objects with SearchEngine support

Summary:
Depends on D18918. Ref T13046. Ref T5954. Pull logs can currently be browsed in the web UI, but this isn't very powerful, especially if you have thousands of them.

Allow SearchEngine implementations to define exportable fields so that users can "Use Results > Export Data" on any query. In particular, they can use this workflow to download a file with pull logs.

In the future, this can replace the existing "Export to Excel" feature in Maniphest.

For now, we hard-code JSON as the only supported datatype and don't actually make any effort to format the data properly, but this leaves room to add more exporters (CSV, Excel) and data type awareness (integer casting, date formatting, etc) in the future.

For sufficiently large result sets, this will probably time out. At some point, I'll make this use the job queue (like bulk editing) when the export is "large" (affects more than 1K rows?).

Test Plan: Downloaded pull logs in JSON format.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13046, T5954

Differential Revision: https://secure.phabricator.com/D18919
This commit is contained in:
epriestley 2018-01-23 09:57:41 -08:00
parent 5058cfb972
commit c0b8e4784b
10 changed files with 274 additions and 2 deletions

View file

@ -2836,12 +2836,14 @@ phutil_register_library_map(array(
'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php',
'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php',
'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php',
'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php',
'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php',
'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php',
'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php',
'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php',
'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php',
'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
@ -3061,6 +3063,7 @@ phutil_register_library_map(array(
'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php',
'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php',
'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php',
'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php',
'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php',
'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php',
'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php',
@ -3412,6 +3415,7 @@ phutil_register_library_map(array(
'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php',
'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php',
'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php',
'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php',
'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php',
'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php',
'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php',
@ -4177,6 +4181,7 @@ phutil_register_library_map(array(
'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php',
'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php',
'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php',
'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php',
'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php',
'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php',
'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php',
@ -8255,12 +8260,14 @@ phutil_register_library_map(array(
'PhabricatorEnv' => 'Phobject',
'PhabricatorEnvTestCase' => 'PhabricatorTestCase',
'PhabricatorEpochEditField' => 'PhabricatorEditField',
'PhabricatorEpochExportField' => 'PhabricatorExportField',
'PhabricatorEvent' => 'PhutilEvent',
'PhabricatorEventEngine' => 'Phobject',
'PhabricatorEventListener' => 'PhutilEventListener',
'PhabricatorEventType' => 'PhutilEventType',
'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
'PhabricatorExportField' => 'Phobject',
'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorExternalAccount' => array(
@ -8521,6 +8528,7 @@ phutil_register_library_map(array(
'PhabricatorHomeProfileMenuItem' => 'PhabricatorProfileMenuItem',
'PhabricatorHovercardEngineExtension' => 'Phobject',
'PhabricatorHovercardEngineExtensionModule' => 'PhabricatorConfigModule',
'PhabricatorIDExportField' => 'PhabricatorExportField',
'PhabricatorIDsSearchEngineExtension' => 'PhabricatorSearchEngineExtension',
'PhabricatorIDsSearchField' => 'PhabricatorSearchField',
'PhabricatorIconDatasource' => 'PhabricatorTypeaheadDatasource',
@ -8911,6 +8919,7 @@ phutil_register_library_map(array(
'PhabricatorPHDConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorPHID' => 'Phobject',
'PhabricatorPHIDConstants' => 'Phobject',
'PhabricatorPHIDExportField' => 'PhabricatorExportField',
'PhabricatorPHIDListEditField' => 'PhabricatorEditField',
'PhabricatorPHIDListEditType' => 'PhabricatorEditType',
'PhabricatorPHIDResolver' => 'Phobject',
@ -9850,6 +9859,7 @@ phutil_register_library_map(array(
'PhabricatorStorageSchemaSpec' => 'PhabricatorConfigSchemaSpec',
'PhabricatorStorageSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorStringConfigType' => 'PhabricatorTextConfigType',
'PhabricatorStringExportField' => 'PhabricatorExportField',
'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType',
'PhabricatorStringListEditField' => 'PhabricatorEditField',
'PhabricatorStringSetting' => 'PhabricatorSetting',

View file

@ -623,7 +623,7 @@ abstract class PhabricatorApplication
}
protected function getQueryRoutePattern($base = null) {
return $base.'(?:query/(?P<queryKey>[^/]+)/)?';
return $base.'(?:query/(?P<queryKey>[^/]+)/(?:(?P<queryAction>[^/]+)/))?';
}
protected function getProfileMenuRouting($controller) {

View file

@ -47,6 +47,87 @@ final class DiffusionPullLogSearchEngine
);
}
protected function newExportFields() {
return array(
id(new PhabricatorIDExportField())
->setKey('id')
->setLabel(pht('ID')),
id(new PhabricatorPHIDExportField())
->setKey('phid')
->setLabel(pht('PHID')),
id(new PhabricatorPHIDExportField())
->setKey('repositoryPHID')
->setLabel(pht('Repository PHID')),
id(new PhabricatorStringExportField())
->setKey('repository')
->setLabel(pht('Repository')),
id(new PhabricatorPHIDExportField())
->setKey('pullerPHID')
->setLabel(pht('Puller PHID')),
id(new PhabricatorStringExportField())
->setKey('puller')
->setLabel(pht('Puller')),
id(new PhabricatorStringExportField())
->setKey('protocol')
->setLabel(pht('Protocol')),
id(new PhabricatorStringExportField())
->setKey('result')
->setLabel(pht('Result')),
id(new PhabricatorStringExportField())
->setKey('code')
->setLabel(pht('Code')),
id(new PhabricatorEpochExportField())
->setKey('date')
->setLabel(pht('Date')),
);
}
public function newExport(array $events) {
$viewer = $this->requireViewer();
$phids = array();
foreach ($events as $event) {
if ($event->getPullerPHID()) {
$phids[] = $event->getPullerPHID();
}
}
$handles = $viewer->loadHandles($phids);
$export = array();
foreach ($events as $event) {
$repository = $event->getRepository();
if ($repository) {
$repository_phid = $repository->getPHID();
$repository_name = $repository->getDisplayName();
} else {
$repository_phid = null;
$repository_name = null;
}
$puller_phid = $event->getPullerPHID();
if ($puller_phid) {
$puller_name = $handles[$puller_phid]->getName();
} else {
$puller_name = null;
}
$export[] = array(
'id' => $event->getID(),
'phid' => $event->getPHID(),
'repositoryPHID' => $repository_phid,
'repository' => $repository_name,
'pullerPHID' => $puller_phid,
'puller' => $puller_name,
'protocol' => $event->getRemoteProtocol(),
'result' => $event->getResultType(),
'code' => $event->getResultCode(),
'date' => $event->getEpoch(),
);
}
return $export;
}
protected function getURI($path) {
return '/diffusion/pulllog/'.$path;
}

View file

@ -66,6 +66,11 @@ final class PhabricatorApplicationSearchController
public function processRequest() {
$this->validateDelegatingController();
$query_action = $this->getRequest()->getURIData('queryAction');
if ($query_action == 'export') {
return $this->processExportRequest();
}
$key = $this->getQueryKey();
if ($key == 'edit') {
return $this->processEditRequest();
@ -374,6 +379,96 @@ final class PhabricatorApplicationSearchController
->appendChild($body);
}
private function processExportRequest() {
$viewer = $this->getViewer();
$engine = $this->getSearchEngine();
$request = $this->getRequest();
if (!$this->canExport()) {
return new Aphront404Response();
}
$query_key = $this->getQueryKey();
if ($engine->isBuiltinQuery($query_key)) {
$saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
} else if ($query_key) {
$saved_query = id(new PhabricatorSavedQueryQuery())
->setViewer($viewer)
->withQueryKeys(array($query_key))
->executeOne();
if (!$saved_query) {
return new Aphront404Response();
}
}
$cancel_uri = $engine->getQueryResultsPageURI($query_key);
$named_query = idx($engine->loadEnabledNamedQueries(), $query_key);
if ($named_query) {
$filename = $named_query->getQueryName();
} else {
$filename = $engine->getResultTypeDescription();
}
$filename = phutil_utf8_strtolower($filename);
$filename = PhabricatorFile::normalizeFileName($filename);
if ($request->isFormPost()) {
$query = $engine->buildQueryFromSavedQuery($saved_query);
// NOTE: We aren't reading the pager from the request. Exports always
// affect the entire result set.
$pager = $engine->newPagerForSavedQuery($saved_query);
$pager->setPageSize(0x7FFFFFFF);
$objects = $engine->executeQuery($query, $pager);
$extension = 'json';
$mime_type = 'application/json';
$filename = $filename.'.'.$extension;
$result = $engine->newExport($objects);
$result = id(new PhutilJSON())
->encodeAsList($result);
$file = PhabricatorFile::newFromFileData(
$result,
array(
'name' => $filename,
'authorPHID' => $viewer->getPHID(),
'ttl.relative' => phutil_units('15 minutes in seconds'),
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
'mime-type' => $mime_type,
));
return $this->newDialog()
->setTitle(pht('Download Results'))
->appendParagraph(
pht('Click the download button to download the exported data.'))
->addCancelButton($cancel_uri, pht('Done'))
->setSubmitURI($file->getDownloadURI())
->setDisableWorkflowOnSubmit(true)
->addSubmitButton(pht('Download Results'));
}
$export_form = id(new AphrontFormView())
->setViewer($viewer)
->appendControl(
id(new AphrontFormSelectControl())
->setName('format')
->setLabel(pht('Format'))
->setOptions(
array(
'json' => 'JSON',
)));
return $this->newDialog()
->setTitle(pht('Export Results'))
->appendForm($export_form)
->addCancelButton($cancel_uri)
->addSubmitButton(pht('Continue'));
}
private function processEditRequest() {
$parent = $this->getDelegatingController();
$request = $this->getRequest();
@ -720,7 +815,6 @@ final class PhabricatorApplicationSearchController
$viewer);
if ($can_use && $is_installed) {
$dashboard_uri = '/dashboard/install/';
$actions[] = id(new PhabricatorActionView())
->setIcon('fa-dashboard')
->setName(pht('Add to Dashboard'))
@ -728,6 +822,15 @@ final class PhabricatorApplicationSearchController
->setHref("/dashboard/panel/install/{$engine_class}/{$query_key}/");
}
if ($this->canExport()) {
$export_uri = $engine->getExportURI($query_key);
$actions[] = id(new PhabricatorActionView())
->setIcon('fa-download')
->setName(pht('Export Results'))
->setWorkflow(true)
->setHref($export_uri);
}
if ($is_dev) {
$engine = $this->getSearchEngine();
$nux_uri = $engine->getQueryBaseURI();
@ -753,4 +856,22 @@ final class PhabricatorApplicationSearchController
return $actions;
}
private function canExport() {
$engine = $this->getSearchEngine();
if (!$engine->canExport()) {
return false;
}
// Don't allow logged-out users to perform exports. There's no technical
// or policy reason they can't, but we don't normally give them access
// to write files or jobs. For now, just err on the side of caution.
$viewer = $this->getViewer();
if (!$viewer->getPHID()) {
return false;
}
return true;
}
}

View file

@ -413,6 +413,10 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
return $this->getURI('');
}
public function getExportURI($query_key) {
return $this->getURI('query/'.$query_key.'/export/');
}
/**
* Return the URI to a path within the application. Used to construct default
@ -1441,4 +1445,17 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
return array();
}
/* -( Export )------------------------------------------------------------- */
public function canExport() {
$fields = $this->newExportFields();
return (bool)$fields;
}
protected function newExportFields() {
return array();
}
}

View file

@ -0,0 +1,4 @@
<?php
final class PhabricatorEpochExportField
extends PhabricatorExportField {}

View file

@ -0,0 +1,27 @@
<?php
abstract class PhabricatorExportField
extends Phobject {
private $key;
private $label;
public function setKey($key) {
$this->key = $key;
return $this;
}
public function getKey() {
return $this->key;
}
public function setLabel($label) {
$this->label = $label;
return $this;
}
public function getLabel() {
return $this->label;
}
}

View file

@ -0,0 +1,4 @@
<?php
final class PhabricatorIDExportField
extends PhabricatorExportField {}

View file

@ -0,0 +1,4 @@
<?php
final class PhabricatorPHIDExportField
extends PhabricatorExportField {}

View file

@ -0,0 +1,4 @@
<?php
final class PhabricatorStringExportField
extends PhabricatorExportField {}