mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-10 00:42:41 +01:00
Add a basic, general-purpose export workflow for all objects with SearchEngine support
Summary: Depends on D18918. Ref T13046. Ref T5954. Pull logs can currently be browsed in the web UI, but this isn't very powerful, especially if you have thousands of them. Allow SearchEngine implementations to define exportable fields so that users can "Use Results > Export Data" on any query. In particular, they can use this workflow to download a file with pull logs. In the future, this can replace the existing "Export to Excel" feature in Maniphest. For now, we hard-code JSON as the only supported datatype and don't actually make any effort to format the data properly, but this leaves room to add more exporters (CSV, Excel) and data type awareness (integer casting, date formatting, etc) in the future. For sufficiently large result sets, this will probably time out. At some point, I'll make this use the job queue (like bulk editing) when the export is "large" (affects more than 1K rows?). Test Plan: Downloaded pull logs in JSON format. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13046, T5954 Differential Revision: https://secure.phabricator.com/D18919
This commit is contained in:
parent
5058cfb972
commit
c0b8e4784b
10 changed files with 274 additions and 2 deletions
|
@ -2836,12 +2836,14 @@ phutil_register_library_map(array(
|
|||
'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php',
|
||||
'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php',
|
||||
'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php',
|
||||
'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php',
|
||||
'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php',
|
||||
'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php',
|
||||
'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php',
|
||||
'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php',
|
||||
'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
|
||||
'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
|
||||
'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php',
|
||||
'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
|
||||
'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
|
||||
'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
|
||||
|
@ -3061,6 +3063,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php',
|
||||
'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php',
|
||||
'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php',
|
||||
'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php',
|
||||
'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php',
|
||||
'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php',
|
||||
'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php',
|
||||
|
@ -3412,6 +3415,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php',
|
||||
'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php',
|
||||
'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php',
|
||||
'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php',
|
||||
'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php',
|
||||
'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php',
|
||||
'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php',
|
||||
|
@ -4177,6 +4181,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php',
|
||||
'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php',
|
||||
'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php',
|
||||
'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php',
|
||||
'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php',
|
||||
'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php',
|
||||
'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php',
|
||||
|
@ -8255,12 +8260,14 @@ phutil_register_library_map(array(
|
|||
'PhabricatorEnv' => 'Phobject',
|
||||
'PhabricatorEnvTestCase' => 'PhabricatorTestCase',
|
||||
'PhabricatorEpochEditField' => 'PhabricatorEditField',
|
||||
'PhabricatorEpochExportField' => 'PhabricatorExportField',
|
||||
'PhabricatorEvent' => 'PhutilEvent',
|
||||
'PhabricatorEventEngine' => 'Phobject',
|
||||
'PhabricatorEventListener' => 'PhutilEventListener',
|
||||
'PhabricatorEventType' => 'PhutilEventType',
|
||||
'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
|
||||
'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
|
||||
'PhabricatorExportField' => 'Phobject',
|
||||
'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
|
||||
'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
|
||||
'PhabricatorExternalAccount' => array(
|
||||
|
@ -8521,6 +8528,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorHomeProfileMenuItem' => 'PhabricatorProfileMenuItem',
|
||||
'PhabricatorHovercardEngineExtension' => 'Phobject',
|
||||
'PhabricatorHovercardEngineExtensionModule' => 'PhabricatorConfigModule',
|
||||
'PhabricatorIDExportField' => 'PhabricatorExportField',
|
||||
'PhabricatorIDsSearchEngineExtension' => 'PhabricatorSearchEngineExtension',
|
||||
'PhabricatorIDsSearchField' => 'PhabricatorSearchField',
|
||||
'PhabricatorIconDatasource' => 'PhabricatorTypeaheadDatasource',
|
||||
|
@ -8911,6 +8919,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorPHDConfigOptions' => 'PhabricatorApplicationConfigOptions',
|
||||
'PhabricatorPHID' => 'Phobject',
|
||||
'PhabricatorPHIDConstants' => 'Phobject',
|
||||
'PhabricatorPHIDExportField' => 'PhabricatorExportField',
|
||||
'PhabricatorPHIDListEditField' => 'PhabricatorEditField',
|
||||
'PhabricatorPHIDListEditType' => 'PhabricatorEditType',
|
||||
'PhabricatorPHIDResolver' => 'Phobject',
|
||||
|
@ -9850,6 +9859,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorStorageSchemaSpec' => 'PhabricatorConfigSchemaSpec',
|
||||
'PhabricatorStorageSetupCheck' => 'PhabricatorSetupCheck',
|
||||
'PhabricatorStringConfigType' => 'PhabricatorTextConfigType',
|
||||
'PhabricatorStringExportField' => 'PhabricatorExportField',
|
||||
'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType',
|
||||
'PhabricatorStringListEditField' => 'PhabricatorEditField',
|
||||
'PhabricatorStringSetting' => 'PhabricatorSetting',
|
||||
|
|
|
@ -623,7 +623,7 @@ abstract class PhabricatorApplication
|
|||
}
|
||||
|
||||
protected function getQueryRoutePattern($base = null) {
|
||||
return $base.'(?:query/(?P<queryKey>[^/]+)/)?';
|
||||
return $base.'(?:query/(?P<queryKey>[^/]+)/(?:(?P<queryAction>[^/]+)/))?';
|
||||
}
|
||||
|
||||
protected function getProfileMenuRouting($controller) {
|
||||
|
|
|
@ -47,6 +47,87 @@ final class DiffusionPullLogSearchEngine
|
|||
);
|
||||
}
|
||||
|
||||
protected function newExportFields() {
|
||||
return array(
|
||||
id(new PhabricatorIDExportField())
|
||||
->setKey('id')
|
||||
->setLabel(pht('ID')),
|
||||
id(new PhabricatorPHIDExportField())
|
||||
->setKey('phid')
|
||||
->setLabel(pht('PHID')),
|
||||
id(new PhabricatorPHIDExportField())
|
||||
->setKey('repositoryPHID')
|
||||
->setLabel(pht('Repository PHID')),
|
||||
id(new PhabricatorStringExportField())
|
||||
->setKey('repository')
|
||||
->setLabel(pht('Repository')),
|
||||
id(new PhabricatorPHIDExportField())
|
||||
->setKey('pullerPHID')
|
||||
->setLabel(pht('Puller PHID')),
|
||||
id(new PhabricatorStringExportField())
|
||||
->setKey('puller')
|
||||
->setLabel(pht('Puller')),
|
||||
id(new PhabricatorStringExportField())
|
||||
->setKey('protocol')
|
||||
->setLabel(pht('Protocol')),
|
||||
id(new PhabricatorStringExportField())
|
||||
->setKey('result')
|
||||
->setLabel(pht('Result')),
|
||||
id(new PhabricatorStringExportField())
|
||||
->setKey('code')
|
||||
->setLabel(pht('Code')),
|
||||
id(new PhabricatorEpochExportField())
|
||||
->setKey('date')
|
||||
->setLabel(pht('Date')),
|
||||
);
|
||||
}
|
||||
|
||||
public function newExport(array $events) {
|
||||
$viewer = $this->requireViewer();
|
||||
|
||||
$phids = array();
|
||||
foreach ($events as $event) {
|
||||
if ($event->getPullerPHID()) {
|
||||
$phids[] = $event->getPullerPHID();
|
||||
}
|
||||
}
|
||||
$handles = $viewer->loadHandles($phids);
|
||||
|
||||
$export = array();
|
||||
foreach ($events as $event) {
|
||||
$repository = $event->getRepository();
|
||||
if ($repository) {
|
||||
$repository_phid = $repository->getPHID();
|
||||
$repository_name = $repository->getDisplayName();
|
||||
} else {
|
||||
$repository_phid = null;
|
||||
$repository_name = null;
|
||||
}
|
||||
|
||||
$puller_phid = $event->getPullerPHID();
|
||||
if ($puller_phid) {
|
||||
$puller_name = $handles[$puller_phid]->getName();
|
||||
} else {
|
||||
$puller_name = null;
|
||||
}
|
||||
|
||||
$export[] = array(
|
||||
'id' => $event->getID(),
|
||||
'phid' => $event->getPHID(),
|
||||
'repositoryPHID' => $repository_phid,
|
||||
'repository' => $repository_name,
|
||||
'pullerPHID' => $puller_phid,
|
||||
'puller' => $puller_name,
|
||||
'protocol' => $event->getRemoteProtocol(),
|
||||
'result' => $event->getResultType(),
|
||||
'code' => $event->getResultCode(),
|
||||
'date' => $event->getEpoch(),
|
||||
);
|
||||
}
|
||||
|
||||
return $export;
|
||||
}
|
||||
|
||||
protected function getURI($path) {
|
||||
return '/diffusion/pulllog/'.$path;
|
||||
}
|
||||
|
|
|
@ -66,6 +66,11 @@ final class PhabricatorApplicationSearchController
|
|||
public function processRequest() {
|
||||
$this->validateDelegatingController();
|
||||
|
||||
$query_action = $this->getRequest()->getURIData('queryAction');
|
||||
if ($query_action == 'export') {
|
||||
return $this->processExportRequest();
|
||||
}
|
||||
|
||||
$key = $this->getQueryKey();
|
||||
if ($key == 'edit') {
|
||||
return $this->processEditRequest();
|
||||
|
@ -374,6 +379,96 @@ final class PhabricatorApplicationSearchController
|
|||
->appendChild($body);
|
||||
}
|
||||
|
||||
private function processExportRequest() {
|
||||
$viewer = $this->getViewer();
|
||||
$engine = $this->getSearchEngine();
|
||||
$request = $this->getRequest();
|
||||
|
||||
if (!$this->canExport()) {
|
||||
return new Aphront404Response();
|
||||
}
|
||||
|
||||
$query_key = $this->getQueryKey();
|
||||
if ($engine->isBuiltinQuery($query_key)) {
|
||||
$saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
|
||||
} else if ($query_key) {
|
||||
$saved_query = id(new PhabricatorSavedQueryQuery())
|
||||
->setViewer($viewer)
|
||||
->withQueryKeys(array($query_key))
|
||||
->executeOne();
|
||||
if (!$saved_query) {
|
||||
return new Aphront404Response();
|
||||
}
|
||||
}
|
||||
|
||||
$cancel_uri = $engine->getQueryResultsPageURI($query_key);
|
||||
|
||||
$named_query = idx($engine->loadEnabledNamedQueries(), $query_key);
|
||||
|
||||
if ($named_query) {
|
||||
$filename = $named_query->getQueryName();
|
||||
} else {
|
||||
$filename = $engine->getResultTypeDescription();
|
||||
}
|
||||
$filename = phutil_utf8_strtolower($filename);
|
||||
$filename = PhabricatorFile::normalizeFileName($filename);
|
||||
|
||||
if ($request->isFormPost()) {
|
||||
$query = $engine->buildQueryFromSavedQuery($saved_query);
|
||||
|
||||
// NOTE: We aren't reading the pager from the request. Exports always
|
||||
// affect the entire result set.
|
||||
$pager = $engine->newPagerForSavedQuery($saved_query);
|
||||
$pager->setPageSize(0x7FFFFFFF);
|
||||
|
||||
$objects = $engine->executeQuery($query, $pager);
|
||||
|
||||
$extension = 'json';
|
||||
$mime_type = 'application/json';
|
||||
$filename = $filename.'.'.$extension;
|
||||
|
||||
$result = $engine->newExport($objects);
|
||||
$result = id(new PhutilJSON())
|
||||
->encodeAsList($result);
|
||||
|
||||
$file = PhabricatorFile::newFromFileData(
|
||||
$result,
|
||||
array(
|
||||
'name' => $filename,
|
||||
'authorPHID' => $viewer->getPHID(),
|
||||
'ttl.relative' => phutil_units('15 minutes in seconds'),
|
||||
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
|
||||
'mime-type' => $mime_type,
|
||||
));
|
||||
|
||||
return $this->newDialog()
|
||||
->setTitle(pht('Download Results'))
|
||||
->appendParagraph(
|
||||
pht('Click the download button to download the exported data.'))
|
||||
->addCancelButton($cancel_uri, pht('Done'))
|
||||
->setSubmitURI($file->getDownloadURI())
|
||||
->setDisableWorkflowOnSubmit(true)
|
||||
->addSubmitButton(pht('Download Results'));
|
||||
}
|
||||
|
||||
$export_form = id(new AphrontFormView())
|
||||
->setViewer($viewer)
|
||||
->appendControl(
|
||||
id(new AphrontFormSelectControl())
|
||||
->setName('format')
|
||||
->setLabel(pht('Format'))
|
||||
->setOptions(
|
||||
array(
|
||||
'json' => 'JSON',
|
||||
)));
|
||||
|
||||
return $this->newDialog()
|
||||
->setTitle(pht('Export Results'))
|
||||
->appendForm($export_form)
|
||||
->addCancelButton($cancel_uri)
|
||||
->addSubmitButton(pht('Continue'));
|
||||
}
|
||||
|
||||
private function processEditRequest() {
|
||||
$parent = $this->getDelegatingController();
|
||||
$request = $this->getRequest();
|
||||
|
@ -720,7 +815,6 @@ final class PhabricatorApplicationSearchController
|
|||
$viewer);
|
||||
|
||||
if ($can_use && $is_installed) {
|
||||
$dashboard_uri = '/dashboard/install/';
|
||||
$actions[] = id(new PhabricatorActionView())
|
||||
->setIcon('fa-dashboard')
|
||||
->setName(pht('Add to Dashboard'))
|
||||
|
@ -728,6 +822,15 @@ final class PhabricatorApplicationSearchController
|
|||
->setHref("/dashboard/panel/install/{$engine_class}/{$query_key}/");
|
||||
}
|
||||
|
||||
if ($this->canExport()) {
|
||||
$export_uri = $engine->getExportURI($query_key);
|
||||
$actions[] = id(new PhabricatorActionView())
|
||||
->setIcon('fa-download')
|
||||
->setName(pht('Export Results'))
|
||||
->setWorkflow(true)
|
||||
->setHref($export_uri);
|
||||
}
|
||||
|
||||
if ($is_dev) {
|
||||
$engine = $this->getSearchEngine();
|
||||
$nux_uri = $engine->getQueryBaseURI();
|
||||
|
@ -753,4 +856,22 @@ final class PhabricatorApplicationSearchController
|
|||
return $actions;
|
||||
}
|
||||
|
||||
private function canExport() {
|
||||
$engine = $this->getSearchEngine();
|
||||
if (!$engine->canExport()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Don't allow logged-out users to perform exports. There's no technical
|
||||
// or policy reason they can't, but we don't normally give them access
|
||||
// to write files or jobs. For now, just err on the side of caution.
|
||||
|
||||
$viewer = $this->getViewer();
|
||||
if (!$viewer->getPHID()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -413,6 +413,10 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
|
|||
return $this->getURI('');
|
||||
}
|
||||
|
||||
public function getExportURI($query_key) {
|
||||
return $this->getURI('query/'.$query_key.'/export/');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Return the URI to a path within the application. Used to construct default
|
||||
|
@ -1441,4 +1445,17 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
|
|||
return array();
|
||||
}
|
||||
|
||||
|
||||
/* -( Export )------------------------------------------------------------- */
|
||||
|
||||
|
||||
public function canExport() {
|
||||
$fields = $this->newExportFields();
|
||||
return (bool)$fields;
|
||||
}
|
||||
|
||||
protected function newExportFields() {
|
||||
return array();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,4 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorEpochExportField
|
||||
extends PhabricatorExportField {}
|
27
src/infrastructure/export/PhabricatorExportField.php
Normal file
27
src/infrastructure/export/PhabricatorExportField.php
Normal file
|
@ -0,0 +1,27 @@
|
|||
<?php
|
||||
|
||||
abstract class PhabricatorExportField
|
||||
extends Phobject {
|
||||
|
||||
private $key;
|
||||
private $label;
|
||||
|
||||
public function setKey($key) {
|
||||
$this->key = $key;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getKey() {
|
||||
return $this->key;
|
||||
}
|
||||
|
||||
public function setLabel($label) {
|
||||
$this->label = $label;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getLabel() {
|
||||
return $this->label;
|
||||
}
|
||||
|
||||
}
|
4
src/infrastructure/export/PhabricatorIDExportField.php
Normal file
4
src/infrastructure/export/PhabricatorIDExportField.php
Normal file
|
@ -0,0 +1,4 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorIDExportField
|
||||
extends PhabricatorExportField {}
|
4
src/infrastructure/export/PhabricatorPHIDExportField.php
Normal file
4
src/infrastructure/export/PhabricatorPHIDExportField.php
Normal file
|
@ -0,0 +1,4 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorPHIDExportField
|
||||
extends PhabricatorExportField {}
|
|
@ -0,0 +1,4 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorStringExportField
|
||||
extends PhabricatorExportField {}
|
Loading…
Reference in a new issue