Support CSV, JSON, and tab-separated text as export formats

Summary: Depends on D18919. Ref T13046. Adds some simple modular exporters.

Test Plan: Exported pull logs in each format.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13046

Differential Revision: https://secure.phabricator.com/D18934

src/__phutil_library_map__.php

@@ -2231,6 +2231,7 @@ phutil_register_library_map(array(
     'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php',
     'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php',
     'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php',
+    'PhabricatorCSVExportFormat' => 'infrastructure/export/PhabricatorCSVExportFormat.php',
     'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php',
     'PhabricatorCacheEngine' => 'applications/system/engine/PhabricatorCacheEngine.php',
     'PhabricatorCacheEngineExtension' => 'applications/system/engine/PhabricatorCacheEngineExtension.php',
@@ -2844,6 +2845,7 @@ phutil_register_library_map(array(
     'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
     'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
     'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php',
+    'PhabricatorExportFormat' => 'infrastructure/export/PhabricatorExportFormat.php',
     'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
     'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
     'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
@@ -3087,6 +3089,7 @@ phutil_register_library_map(array(
     'PhabricatorInlineSummaryView' => 'infrastructure/diff/view/PhabricatorInlineSummaryView.php',
     'PhabricatorInstructionsEditField' => 'applications/transactions/editfield/PhabricatorInstructionsEditField.php',
     'PhabricatorIntConfigType' => 'applications/config/type/PhabricatorIntConfigType.php',
+    'PhabricatorIntExportField' => 'infrastructure/export/PhabricatorIntExportField.php',
     'PhabricatorInternalSetting' => 'applications/settings/setting/PhabricatorInternalSetting.php',
     'PhabricatorInternationalizationManagementExtractWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php',
     'PhabricatorInternationalizationManagementWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementWorkflow.php',
@@ -3096,6 +3099,7 @@ phutil_register_library_map(array(
     'PhabricatorIteratorFileUploadSource' => 'applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php',
     'PhabricatorJIRAAuthProvider' => 'applications/auth/provider/PhabricatorJIRAAuthProvider.php',
     'PhabricatorJSONConfigType' => 'applications/config/type/PhabricatorJSONConfigType.php',
+    'PhabricatorJSONExportFormat' => 'infrastructure/export/PhabricatorJSONExportFormat.php',
     'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
     'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
     'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php',
@@ -4245,6 +4249,7 @@ phutil_register_library_map(array(
     'PhabricatorTextAreaEditField' => 'applications/transactions/editfield/PhabricatorTextAreaEditField.php',
     'PhabricatorTextConfigType' => 'applications/config/type/PhabricatorTextConfigType.php',
     'PhabricatorTextEditField' => 'applications/transactions/editfield/PhabricatorTextEditField.php',
+    'PhabricatorTextExportFormat' => 'infrastructure/export/PhabricatorTextExportFormat.php',
     'PhabricatorTextListConfigType' => 'applications/config/type/PhabricatorTextListConfigType.php',
     'PhabricatorTime' => 'infrastructure/time/PhabricatorTime.php',
     'PhabricatorTimeFormatSetting' => 'applications/settings/setting/PhabricatorTimeFormatSetting.php',
@@ -7564,6 +7569,7 @@ phutil_register_library_map(array(
     'PhabricatorBulkEngine' => 'Phobject',
     'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow',
     'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow',
+    'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat',
     'PhabricatorCacheDAO' => 'PhabricatorLiskDAO',
     'PhabricatorCacheEngine' => 'Phobject',
     'PhabricatorCacheEngineExtension' => 'Phobject',
@@ -8268,6 +8274,7 @@ phutil_register_library_map(array(
     'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
     'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
     'PhabricatorExportField' => 'Phobject',
+    'PhabricatorExportFormat' => 'Phobject',
     'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorExternalAccount' => array(
@@ -8551,6 +8558,7 @@ phutil_register_library_map(array(
     'PhabricatorInlineSummaryView' => 'AphrontView',
     'PhabricatorInstructionsEditField' => 'PhabricatorEditField',
     'PhabricatorIntConfigType' => 'PhabricatorTextConfigType',
+    'PhabricatorIntExportField' => 'PhabricatorExportField',
     'PhabricatorInternalSetting' => 'PhabricatorSetting',
     'PhabricatorInternationalizationManagementExtractWorkflow' => 'PhabricatorInternationalizationManagementWorkflow',
     'PhabricatorInternationalizationManagementWorkflow' => 'PhabricatorManagementWorkflow',
@@ -8560,6 +8568,7 @@ phutil_register_library_map(array(
     'PhabricatorIteratorFileUploadSource' => 'PhabricatorFileUploadSource',
     'PhabricatorJIRAAuthProvider' => 'PhabricatorOAuth1AuthProvider',
     'PhabricatorJSONConfigType' => 'PhabricatorTextConfigType',
+    'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat',
     'PhabricatorJavelinLinter' => 'ArcanistLinter',
     'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
     'PhabricatorJumpNavHandler' => 'Phobject',
@@ -9922,6 +9931,7 @@ phutil_register_library_map(array(
     'PhabricatorTextAreaEditField' => 'PhabricatorEditField',
     'PhabricatorTextConfigType' => 'PhabricatorConfigType',
     'PhabricatorTextEditField' => 'PhabricatorEditField',
+    'PhabricatorTextExportFormat' => 'PhabricatorExportFormat',
     'PhabricatorTextListConfigType' => 'PhabricatorTextConfigType',
     'PhabricatorTime' => 'Phobject',
     'PhabricatorTimeFormatSetting' => 'PhabricatorSelectSetting',

src/applications/diffusion/query/DiffusionPullLogSearchEngine.php

@@ -73,7 +73,7 @@ final class DiffusionPullLogSearchEngine
       id(new PhabricatorStringExportField())
         ->setKey('result')
         ->setLabel(pht('Result')),
-      id(new PhabricatorStringExportField())
+      id(new PhabricatorIntExportField())
         ->setKey('code')
         ->setLabel(pht('Code')),
       id(new PhabricatorEpochExportField())

src/applications/search/controller/PhabricatorApplicationSearchController.php

@@ -413,42 +413,80 @@ final class PhabricatorApplicationSearchController
     $filename = phutil_utf8_strtolower($filename);
     $filename = PhabricatorFile::normalizeFileName($filename);
 
+    $formats = PhabricatorExportFormat::getAllEnabledExportFormats();
+    $format_options = mpull($formats, 'getExportFormatName');
+
+    $errors = array();
+
+    $e_format = null;
     if ($request->isFormPost()) {
-      $query = $engine->buildQueryFromSavedQuery($saved_query);
+      $format_key = $request->getStr('format');
+      $format = idx($formats, $format_key);
 
-      // NOTE: We aren't reading the pager from the request. Exports always
+      if (!$format) {
-      // affect the entire result set.
+        $e_format = pht('Invalid');
-      $pager = $engine->newPagerForSavedQuery($saved_query);
+        $errors[] = pht('Choose a valid export format.');
-      $pager->setPageSize(0x7FFFFFFF);
+      }
 
-      $objects = $engine->executeQuery($query, $pager);
+      if (!$errors) {
+        $query = $engine->buildQueryFromSavedQuery($saved_query);
 
-      $extension = 'json';
+        // NOTE: We aren't reading the pager from the request. Exports always
-      $mime_type = 'application/json';
+        // affect the entire result set.
-      $filename = $filename.'.'.$extension;
+        $pager = $engine->newPagerForSavedQuery($saved_query);
+        $pager->setPageSize(0x7FFFFFFF);
 
-      $result = $engine->newExport($objects);
+        $objects = $engine->executeQuery($query, $pager);
-      $result = id(new PhutilJSON())
-        ->encodeAsList($result);
 
-      $file = PhabricatorFile::newFromFileData(
+        $extension = $format->getFileExtension();
-        $result,
+        $mime_type = $format->getMIMEContentType();
-        array(
+        $filename = $filename.'.'.$extension;
-          'name' => $filename,
-          'authorPHID' => $viewer->getPHID(),
-          'ttl.relative' => phutil_units('15 minutes in seconds'),
-          'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
-          'mime-type' => $mime_type,
-        ));
 
-      return $this->newDialog()
+        $format = clone $format;
-        ->setTitle(pht('Download Results'))
+        $format->setViewer($viewer);
-        ->appendParagraph(
+
-          pht('Click the download button to download the exported data.'))
+        $export_data = $engine->newExport($objects);
-        ->addCancelButton($cancel_uri, pht('Done'))
+
-        ->setSubmitURI($file->getDownloadURI())
+        if (count($export_data) !== count($objects)) {
-        ->setDisableWorkflowOnSubmit(true)
+          throw new Exception(
-        ->addSubmitButton(pht('Download Results'));
+            pht(
+              'Search engine exported the wrong number of objects, expected '.
+              '%s but got %s.',
+              phutil_count($objects),
+              phutil_count($export_data)));
+        }
+
+        $objects = array_values($objects);
+        $export_data = array_values($export_data);
+
+        $field_list = $engine->newExportFieldList();
+        $field_list = mpull($field_list, null, 'getKey');
+
+        for ($ii = 0; $ii < count($objects); $ii++) {
+          $format->addObject($objects[$ii], $field_list, $export_data[$ii]);
+        }
+
+        $export_result = $format->newFileData();
+
+        $file = PhabricatorFile::newFromFileData(
+          $export_result,
+          array(
+            'name' => $filename,
+            'authorPHID' => $viewer->getPHID(),
+            'ttl.relative' => phutil_units('15 minutes in seconds'),
+            'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
+            'mime-type' => $mime_type,
+          ));
+
+        return $this->newDialog()
+          ->setTitle(pht('Download Results'))
+          ->appendParagraph(
+            pht('Click the download button to download the exported data.'))
+          ->addCancelButton($cancel_uri, pht('Done'))
+          ->setSubmitURI($file->getDownloadURI())
+          ->setDisableWorkflowOnSubmit(true)
+          ->addSubmitButton(pht('Download Data'));
+      }
     }
 
     $export_form = id(new AphrontFormView())
@@ -457,13 +495,12 @@ final class PhabricatorApplicationSearchController
         id(new AphrontFormSelectControl())
           ->setName('format')
           ->setLabel(pht('Format'))
-          ->setOptions(
+          ->setError($e_format)
-            array(
+          ->setOptions($format_options));
-              'json' => 'JSON',
-            )));
 
     return $this->newDialog()
       ->setTitle(pht('Export Results'))
+      ->setErrors($errors)
       ->appendForm($export_form)
       ->addCancelButton($cancel_uri)
       ->addSubmitButton(pht('Continue'));
@@ -826,7 +863,7 @@ final class PhabricatorApplicationSearchController
       $export_uri = $engine->getExportURI($query_key);
       $actions[] = id(new PhabricatorActionView())
         ->setIcon('fa-download')
-        ->setName(pht('Export Results'))
+        ->setName(pht('Export Data'))
         ->setWorkflow(true)
         ->setHref($export_uri);
     }

src/applications/search/engine/PhabricatorApplicationSearchEngine.php

@@ -1454,6 +1454,10 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject {
     return (bool)$fields;
   }
 
+  final public function newExportFieldList() {
+    return $this->newExportFields();
+  }
+
   protected function newExportFields() {
     return array();
   }

src/infrastructure/export/PhabricatorCSVExportFormat.php

@@ -0,0 +1,47 @@
+<?php
+
+final class PhabricatorCSVExportFormat
+  extends PhabricatorExportFormat {
+
+  const EXPORTKEY = 'csv';
+
+  private $rows = array();
+
+  public function getExportFormatName() {
+    return pht('Comma-Separated Values (.csv)');
+  }
+
+  public function isExportFormatEnabled() {
+    return true;
+  }
+
+  public function getFileExtension() {
+    return 'csv';
+  }
+
+  public function getMIMEContentType() {
+    return 'text/csv';
+  }
+
+  public function addObject($object, array $fields, array $map) {
+    $values = array();
+    foreach ($fields as $key => $field) {
+      $value = $map[$key];
+      $value = $field->getTextValue($value);
+
+      if (preg_match('/\s|,|\"/', $value)) {
+        $value = str_replace('"', '""', $value);
+        $value = '"'.$value.'"';
+      }
+
+      $values[] = $value;
+    }
+
+    $this->rows[] = implode(',', $values);
+  }
+
+  public function newFileData() {
+    return implode("\n", $this->rows);
+  }
+
+}

src/infrastructure/export/PhabricatorEpochExportField.php

@@ -1,4 +1,27 @@
 <?php
 
 final class PhabricatorEpochExportField
-  extends PhabricatorExportField {}
+  extends PhabricatorExportField {
+
+  private $zone;
+
+  public function getTextValue($value) {
+    if (!isset($this->zone)) {
+      $this->zone = new DateTimeZone('UTC');
+    }
+
+    try {
+      $date = new DateTime('@'.$value);
+    } catch (Exception $ex) {
+      return null;
+    }
+
+    $date->setTimezone($this->zone);
+    return $date->format('c');
+  }
+
+  public function getNaturalValue($value) {
+    return (int)$value;
+  }
+
+}

src/infrastructure/export/PhabricatorExportField.php

@@ -24,4 +24,12 @@ abstract class PhabricatorExportField
     return $this->label;
   }
 
+  public function getTextValue($value) {
+    return (string)$this->getNaturalValue($value);
+  }
+
+  public function getNaturalValue($value) {
+    return $value;
+  }
+
 }

src/infrastructure/export/PhabricatorExportFormat.php

@@ -0,0 +1,51 @@
+<?php
+
+abstract class PhabricatorExportFormat
+  extends Phobject {
+
+  private $viewer;
+
+  final public function getExportFormatKey() {
+    return $this->getPhobjectClassConstant('EXPORTKEY');
+  }
+
+  final public function setViewer(PhabricatorUser $viewer) {
+    $this->viewer = $viewer;
+    return $this;
+  }
+
+  final public function getViewer() {
+    return $this->viewer;
+  }
+
+  abstract public function getExportFormatName();
+  abstract public function getMIMEContentType();
+  abstract public function getFileExtension();
+
+  abstract public function addObject($object, array $fields, array $map);
+  abstract public function newFileData();
+
+  public function isExportFormatEnabled() {
+    return true;
+  }
+
+  final public static function getAllExportFormats() {
+    return id(new PhutilClassMapQuery())
+      ->setAncestorClass(__CLASS__)
+      ->setUniqueMethod('getExportFormatKey')
+      ->execute();
+  }
+
+  final public static function getAllEnabledExportFormats() {
+    $formats = self::getAllExportFormats();
+
+    foreach ($formats as $key => $format) {
+      if (!$format->isExportFormatEnabled()) {
+        unset($formats[$key]);
+      }
+    }
+
+    return $formats;
+  }
+
+}

src/infrastructure/export/PhabricatorIDExportField.php

@@ -1,4 +1,10 @@
 <?php
 
 final class PhabricatorIDExportField
-  extends PhabricatorExportField {}
+  extends PhabricatorExportField {
+
+  public function getNaturalValue($value) {
+    return (int)$value;
+  }
+
+}

src/infrastructure/export/PhabricatorIntExportField.php

@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorIntExportField
+  extends PhabricatorExportField {
+
+  public function getNaturalValue($value) {
+    return (int)$value;
+  }
+
+}

src/infrastructure/export/PhabricatorJSONExportFormat.php

@@ -0,0 +1,43 @@
+<?php
+
+final class PhabricatorJSONExportFormat
+  extends PhabricatorExportFormat {
+
+  const EXPORTKEY = 'json';
+
+  private $objects = array();
+
+  public function getExportFormatName() {
+    return 'JSON (.json)';
+  }
+
+  public function isExportFormatEnabled() {
+    return true;
+  }
+
+  public function getFileExtension() {
+    return 'json';
+  }
+
+  public function getMIMEContentType() {
+    return 'application/json';
+  }
+
+  public function addObject($object, array $fields, array $map) {
+    $values = array();
+    foreach ($fields as $key => $field) {
+      $value = $map[$key];
+      $value = $field->getNaturalValue($value);
+
+      $values[$key] = $value;
+    }
+
+    $this->objects[] = $values;
+  }
+
+  public function newFileData() {
+    return id(new PhutilJSON())
+      ->encodeAsList($this->objects);
+  }
+
+}

src/infrastructure/export/PhabricatorTextExportFormat.php

@@ -0,0 +1,43 @@
+<?php
+
+final class PhabricatorTextExportFormat
+  extends PhabricatorExportFormat {
+
+  const EXPORTKEY = 'text';
+
+  private $rows = array();
+
+  public function getExportFormatName() {
+    return 'Tab-Separated Text (.txt)';
+  }
+
+  public function isExportFormatEnabled() {
+    return true;
+  }
+
+  public function getFileExtension() {
+    return 'txt';
+  }
+
+  public function getMIMEContentType() {
+    return 'text/plain';
+  }
+
+  public function addObject($object, array $fields, array $map) {
+    $values = array();
+    foreach ($fields as $key => $field) {
+      $value = $map[$key];
+      $value = $field->getTextValue($value);
+      $value = addcslashes($value, "\0..\37\\\177..\377");
+
+      $values[] = $value;
+    }
+
+    $this->rows[] = implode("\t", $values);
+  }
+
+  public function newFileData() {
+    return implode("\n", $this->rows)."\n";
+  }
+
+}