Add a basic, general-purpose export workflow for all objects with SearchEngine support

Summary:
Depends on D18918. Ref T13046. Ref T5954. Pull logs can currently be browsed in the web UI, but this isn't very powerful, especially if you have thousands of them.

Allow SearchEngine implementations to define exportable fields so that users can "Use Results > Export Data" on any query. In particular, they can use this workflow to download a file with pull logs.

In the future, this can replace the existing "Export to Excel" feature in Maniphest.

For now, we hard-code JSON as the only supported datatype and don't actually make any effort to format the data properly, but this leaves room to add more exporters (CSV, Excel) and data type awareness (integer casting, date formatting, etc) in the future.

For sufficiently large result sets, this will probably time out. At some point, I'll make this use the job queue (like bulk editing) when the export is "large" (affects more than 1K rows?).

Test Plan: Downloaded pull logs in JSON format.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13046, T5954

Differential Revision: https://secure.phabricator.com/D18919

src/__phutil_library_map__.php

@@ -2836,12 +2836,14 @@ phutil_register_library_map(array(
     'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php',
     'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php',
     'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php',
+    'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php',
     'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php',
     'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php',
     'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php',
     'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php',
     'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
     'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
+    'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php',
     'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
     'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
     'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
@@ -3061,6 +3063,7 @@ phutil_register_library_map(array(
     'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php',
     'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php',
     'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php',
+    'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php',
     'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php',
     'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php',
     'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php',
@@ -3412,6 +3415,7 @@ phutil_register_library_map(array(
     'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php',
     'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php',
     'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php',
+    'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php',
     'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php',
     'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php',
     'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php',
@@ -4177,6 +4181,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php',
     'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php',
     'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php',
+    'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php',
     'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php',
     'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php',
     'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php',
@@ -8255,12 +8260,14 @@ phutil_register_library_map(array(
     'PhabricatorEnv' => 'Phobject',
     'PhabricatorEnvTestCase' => 'PhabricatorTestCase',
     'PhabricatorEpochEditField' => 'PhabricatorEditField',
+    'PhabricatorEpochExportField' => 'PhabricatorExportField',
     'PhabricatorEvent' => 'PhutilEvent',
     'PhabricatorEventEngine' => 'Phobject',
     'PhabricatorEventListener' => 'PhutilEventListener',
     'PhabricatorEventType' => 'PhutilEventType',
     'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
     'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
+    'PhabricatorExportField' => 'Phobject',
     'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorExternalAccount' => array(
@@ -8521,6 +8528,7 @@ phutil_register_library_map(array(
     'PhabricatorHomeProfileMenuItem' => 'PhabricatorProfileMenuItem',
     'PhabricatorHovercardEngineExtension' => 'Phobject',
     'PhabricatorHovercardEngineExtensionModule' => 'PhabricatorConfigModule',
+    'PhabricatorIDExportField' => 'PhabricatorExportField',
     'PhabricatorIDsSearchEngineExtension' => 'PhabricatorSearchEngineExtension',
     'PhabricatorIDsSearchField' => 'PhabricatorSearchField',
     'PhabricatorIconDatasource' => 'PhabricatorTypeaheadDatasource',
@@ -8911,6 +8919,7 @@ phutil_register_library_map(array(
     'PhabricatorPHDConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorPHID' => 'Phobject',
     'PhabricatorPHIDConstants' => 'Phobject',
+    'PhabricatorPHIDExportField' => 'PhabricatorExportField',
     'PhabricatorPHIDListEditField' => 'PhabricatorEditField',
     'PhabricatorPHIDListEditType' => 'PhabricatorEditType',
     'PhabricatorPHIDResolver' => 'Phobject',
@@ -9850,6 +9859,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorStorageSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorStringConfigType' => 'PhabricatorTextConfigType',
+    'PhabricatorStringExportField' => 'PhabricatorExportField',
     'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType',
     'PhabricatorStringListEditField' => 'PhabricatorEditField',
     'PhabricatorStringSetting' => 'PhabricatorSetting',

src/applications/base/PhabricatorApplication.php

@@ -623,7 +623,7 @@ abstract class PhabricatorApplication
   }
 
   protected function getQueryRoutePattern($base = null) {
-    return $base.'(?:query/(?P<queryKey>[^/]+)/)?';
+    return $base.'(?:query/(?P<queryKey>[^/]+)/(?:(?P<queryAction>[^/]+)/))?';
   }
 
   protected function getProfileMenuRouting($controller) {

src/applications/diffusion/query/DiffusionPullLogSearchEngine.php

@@ -47,6 +47,87 @@ final class DiffusionPullLogSearchEngine
     );
   }
 
+  protected function newExportFields() {
+    return array(
+      id(new PhabricatorIDExportField())
+        ->setKey('id')
+        ->setLabel(pht('ID')),
+      id(new PhabricatorPHIDExportField())
+        ->setKey('phid')
+        ->setLabel(pht('PHID')),
+      id(new PhabricatorPHIDExportField())
+        ->setKey('repositoryPHID')
+        ->setLabel(pht('Repository PHID')),
+      id(new PhabricatorStringExportField())
+        ->setKey('repository')
+        ->setLabel(pht('Repository')),
+      id(new PhabricatorPHIDExportField())
+        ->setKey('pullerPHID')
+        ->setLabel(pht('Puller PHID')),
+      id(new PhabricatorStringExportField())
+        ->setKey('puller')
+        ->setLabel(pht('Puller')),
+      id(new PhabricatorStringExportField())
+        ->setKey('protocol')
+        ->setLabel(pht('Protocol')),
+      id(new PhabricatorStringExportField())
+        ->setKey('result')
+        ->setLabel(pht('Result')),
+      id(new PhabricatorStringExportField())
+        ->setKey('code')
+        ->setLabel(pht('Code')),
+      id(new PhabricatorEpochExportField())
+        ->setKey('date')
+        ->setLabel(pht('Date')),
+    );
+  }
+
+  public function newExport(array $events) {
+    $viewer = $this->requireViewer();
+
+    $phids = array();
+    foreach ($events as $event) {
+      if ($event->getPullerPHID()) {
+        $phids[] = $event->getPullerPHID();
+      }
+    }
+    $handles = $viewer->loadHandles($phids);
+
+    $export = array();
+    foreach ($events as $event) {
+      $repository = $event->getRepository();
+      if ($repository) {
+        $repository_phid = $repository->getPHID();
+        $repository_name = $repository->getDisplayName();
+      } else {
+        $repository_phid = null;
+        $repository_name = null;
+      }
+
+      $puller_phid = $event->getPullerPHID();
+      if ($puller_phid) {
+        $puller_name = $handles[$puller_phid]->getName();
+      } else {
+        $puller_name = null;
+      }
+
+      $export[] = array(
+        'id' => $event->getID(),
+        'phid' => $event->getPHID(),
+        'repositoryPHID' => $repository_phid,
+        'repository' => $repository_name,
+        'pullerPHID' => $puller_phid,
+        'puller' => $puller_name,
+        'protocol' => $event->getRemoteProtocol(),
+        'result' => $event->getResultType(),
+        'code' => $event->getResultCode(),
+        'date' => $event->getEpoch(),
+      );
+    }
+
+    return $export;
+  }
+
   protected function getURI($path) {
     return '/diffusion/pulllog/'.$path;
   }

src/applications/search/controller/PhabricatorApplicationSearchController.php

@@ -66,6 +66,11 @@ final class PhabricatorApplicationSearchController
   public function processRequest() {
     $this->validateDelegatingController();
 
+    $query_action = $this->getRequest()->getURIData('queryAction');
+    if ($query_action == 'export') {
+      return $this->processExportRequest();
+    }
+
     $key = $this->getQueryKey();
     if ($key == 'edit') {
       return $this->processEditRequest();
@@ -374,6 +379,96 @@ final class PhabricatorApplicationSearchController
       ->appendChild($body);
   }
 
+  private function processExportRequest() {
+    $viewer = $this->getViewer();
+    $engine = $this->getSearchEngine();
+    $request = $this->getRequest();
+
+    if (!$this->canExport()) {
+      return new Aphront404Response();
+    }
+
+    $query_key = $this->getQueryKey();
+    if ($engine->isBuiltinQuery($query_key)) {
+      $saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
+    } else if ($query_key) {
+      $saved_query = id(new PhabricatorSavedQueryQuery())
+        ->setViewer($viewer)
+        ->withQueryKeys(array($query_key))
+        ->executeOne();
+      if (!$saved_query) {
+        return new Aphront404Response();
+      }
+    }
+
+    $cancel_uri = $engine->getQueryResultsPageURI($query_key);
+
+    $named_query = idx($engine->loadEnabledNamedQueries(), $query_key);
+
+    if ($named_query) {
+      $filename = $named_query->getQueryName();
+    } else {
+      $filename = $engine->getResultTypeDescription();
+    }
+    $filename = phutil_utf8_strtolower($filename);
+    $filename = PhabricatorFile::normalizeFileName($filename);
+
+    if ($request->isFormPost()) {
+      $query = $engine->buildQueryFromSavedQuery($saved_query);
+
+      // NOTE: We aren't reading the pager from the request. Exports always
+      // affect the entire result set.
+      $pager = $engine->newPagerForSavedQuery($saved_query);
+      $pager->setPageSize(0x7FFFFFFF);
+
+      $objects = $engine->executeQuery($query, $pager);
+
+      $extension = 'json';
+      $mime_type = 'application/json';
+      $filename = $filename.'.'.$extension;
+
+      $result = $engine->newExport($objects);
+      $result = id(new PhutilJSON())
+        ->encodeAsList($result);
+
+      $file = PhabricatorFile::newFromFileData(
+        $result,
+        array(
+          'name' => $filename,
+          'authorPHID' => $viewer->getPHID(),
+          'ttl.relative' => phutil_units('15 minutes in seconds'),
+          'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
+          'mime-type' => $mime_type,
+        ));
+
+      return $this->newDialog()
+        ->setTitle(pht('Download Results'))
+        ->appendParagraph(
+          pht('Click the download button to download the exported data.'))
+        ->addCancelButton($cancel_uri, pht('Done'))
+        ->setSubmitURI($file->getDownloadURI())
+        ->setDisableWorkflowOnSubmit(true)
+        ->addSubmitButton(pht('Download Results'));
+    }
+
+    $export_form = id(new AphrontFormView())
+      ->setViewer($viewer)
+      ->appendControl(
+        id(new AphrontFormSelectControl())
+          ->setName('format')
+          ->setLabel(pht('Format'))
+          ->setOptions(
+            array(
+              'json' => 'JSON',
+            )));
+
+    return $this->newDialog()
+      ->setTitle(pht('Export Results'))
+      ->appendForm($export_form)
+      ->addCancelButton($cancel_uri)
+      ->addSubmitButton(pht('Continue'));
+  }
+
   private function processEditRequest() {
     $parent = $this->getDelegatingController();
     $request = $this->getRequest();
@@ -720,7 +815,6 @@ final class PhabricatorApplicationSearchController
       $viewer);
 
     if ($can_use && $is_installed) {
-      $dashboard_uri = '/dashboard/install/';
       $actions[] = id(new PhabricatorActionView())
         ->setIcon('fa-dashboard')
         ->setName(pht('Add to Dashboard'))
@@ -728,6 +822,15 @@ final class PhabricatorApplicationSearchController
         ->setHref("/dashboard/panel/install/{$engine_class}/{$query_key}/");
     }
 
+    if ($this->canExport()) {
+      $export_uri = $engine->getExportURI($query_key);
+      $actions[] = id(new PhabricatorActionView())
+        ->setIcon('fa-download')
+        ->setName(pht('Export Results'))
+        ->setWorkflow(true)
+        ->setHref($export_uri);
+    }
+
     if ($is_dev) {
       $engine = $this->getSearchEngine();
       $nux_uri = $engine->getQueryBaseURI();
@@ -753,4 +856,22 @@ final class PhabricatorApplicationSearchController
     return $actions;
   }
 
+  private function canExport() {
+    $engine = $this->getSearchEngine();
+    if (!$engine->canExport()) {
+      return false;
+    }
+
+    // Don't allow logged-out users to perform exports. There's no technical
+    // or policy reason they can't, but we don't normally give them access
+    // to write files or jobs. For now, just err on the side of caution.
+
+    $viewer = $this->getViewer();
+    if (!$viewer->getPHID()) {
+      return false;
+    }
+
+    return true;
+  }
+
 }

src/applications/search/engine/PhabricatorApplicationSearchEngine.php

@@ -413,6 +413,10 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
     return $this->getURI('');
   }
 
+  public function getExportURI($query_key) {
+    return $this->getURI('query/'.$query_key.'/export/');
+  }
+
 
   /**
    * Return the URI to a path within the application. Used to construct default
@@ -1441,4 +1445,17 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
     return array();
   }
 
+
+/* -(  Export  )------------------------------------------------------------- */
+
+
+  public function canExport() {
+    $fields = $this->newExportFields();
+    return (bool)$fields;
+  }
+
+  protected function newExportFields() {
+    return array();
+  }
+
 }

src/infrastructure/export/PhabricatorEpochExportField.php

@@ -0,0 +1,4 @@
+<?php
+
+final class PhabricatorEpochExportField
+  extends PhabricatorExportField {}

src/infrastructure/export/PhabricatorExportField.php

@@ -0,0 +1,27 @@
+<?php
+
+abstract class PhabricatorExportField
+  extends Phobject {
+
+  private $key;
+  private $label;
+
+  public function setKey($key) {
+    $this->key = $key;
+    return $this;
+  }
+
+  public function getKey() {
+    return $this->key;
+  }
+
+  public function setLabel($label) {
+    $this->label = $label;
+    return $this;
+  }
+
+  public function getLabel() {
+    return $this->label;
+  }
+
+}

src/infrastructure/export/PhabricatorIDExportField.php

@@ -0,0 +1,4 @@
+<?php
+
+final class PhabricatorIDExportField
+  extends PhabricatorExportField {}

src/infrastructure/export/PhabricatorPHIDExportField.php

@@ -0,0 +1,4 @@
+<?php
+
+final class PhabricatorPHIDExportField
+  extends PhabricatorExportField {}

src/infrastructure/export/PhabricatorStringExportField.php

@@ -0,0 +1,4 @@
+<?php
+
+final class PhabricatorStringExportField
+  extends PhabricatorExportField {}