Support text encoding and syntax highlighting options in document rendering

Summary: Depends on D19273. Ref T13105. Adds "Change Text Encoding..." and "Highlight As..." options when rendering documents, and makes an effort to automatically detect and handle text encoding.

Test Plan:
  - Uploaded a Shift-JIS file, saw it auto-detect as Shift-JIS.
  - Converted files between encodings.
  - Highlighted various things as "Rainbow", etc.

Maniphest Tasks: T13105

Differential Revision: https://secure.phabricator.com/D19274

resources/celerity/map.php

@@ -10,7 +10,7 @@ return array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
     'core.pkg.css' => '1dd5fa4b',
-    'core.pkg.js' => 'b9b4a943',
+    'core.pkg.js' => '1ea38af8',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
     'diffusion.pkg.css' => 'a2d17c7d',
@@ -392,7 +392,7 @@ return array(
     'rsrc/js/application/diffusion/behavior-pull-lastmodified.js' => 'f01586dc',
     'rsrc/js/application/doorkeeper/behavior-doorkeeper-tag.js' => '1db13e70',
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
-    'rsrc/js/application/files/behavior-document-engine.js' => '194cbe53',
+    'rsrc/js/application/files/behavior-document-engine.js' => '9108ee1a',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
     'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '191b4909',
@@ -508,7 +508,7 @@ return array(
     'rsrc/js/phui/behavior-phui-submenu.js' => 'a6f7a73b',
     'rsrc/js/phui/behavior-phui-tab-group.js' => '0a0b10e9',
     'rsrc/js/phuix/PHUIXActionListView.js' => 'b5c256b8',
-    'rsrc/js/phuix/PHUIXActionView.js' => 'ed18356a',
+    'rsrc/js/phuix/PHUIXActionView.js' => '8d4a8c72',
     'rsrc/js/phuix/PHUIXAutocomplete.js' => 'df1bbd34',
     'rsrc/js/phuix/PHUIXButtonView.js' => '8a91e1ac',
     'rsrc/js/phuix/PHUIXDropdownMenu.js' => '04b2ae03',
@@ -607,7 +607,7 @@ return array(
     'javelin-behavior-diffusion-jump-to' => '73d09eef',
     'javelin-behavior-diffusion-locate-file' => '6d3e1947',
     'javelin-behavior-diffusion-pull-lastmodified' => 'f01586dc',
-    'javelin-behavior-document-engine' => '194cbe53',
+    'javelin-behavior-document-engine' => '9108ee1a',
     'javelin-behavior-doorkeeper-tag' => '1db13e70',
     'javelin-behavior-drydock-live-operation-status' => '901935ef',
     'javelin-behavior-durable-column' => '2ae077e1',
@@ -864,7 +864,7 @@ return array(
     'phui-workcard-view-css' => 'cca5fa92',
     'phui-workpanel-view-css' => 'a3a63478',
     'phuix-action-list-view' => 'b5c256b8',
-    'phuix-action-view' => 'ed18356a',
+    'phuix-action-view' => '8d4a8c72',
     'phuix-autocomplete' => 'df1bbd34',
     'phuix-button-view' => '8a91e1ac',
     'phuix-dropdown-menu' => '04b2ae03',
@@ -983,11 +983,6 @@ return array(
     '191b4909' => array(
       'javelin-behavior',
     ),
-    '194cbe53' => array(
-      'javelin-behavior',
-      'javelin-dom',
-      'javelin-stratcom',
-    ),
     '1ad0a787' => array(
       'javelin-install',
       'javelin-reactor',
@@ -1619,6 +1614,11 @@ return array(
       'javelin-stratcom',
       'javelin-install',
     ),
+    '8d4a8c72' => array(
+      'javelin-install',
+      'javelin-dom',
+      'javelin-util',
+    ),
     '8e1baf68' => array(
       'phui-button-css',
     ),
@@ -1644,6 +1644,11 @@ return array(
       'javelin-stratcom',
       'javelin-vector',
     ),
+    '9108ee1a' => array(
+      'javelin-behavior',
+      'javelin-dom',
+      'javelin-stratcom',
+    ),
     '92b9ec77' => array(
       'javelin-behavior',
       'javelin-stratcom',
@@ -2125,11 +2130,6 @@ return array(
       'javelin-stratcom',
       'javelin-vector',
     ),
-    'ed18356a' => array(
-      'javelin-install',
-      'javelin-dom',
-      'javelin-util',
-    ),
     'edf8a145' => array(
       'javelin-behavior',
       'javelin-uri',

src/applications/files/controller/PhabricatorFileDocumentController.php

@@ -43,6 +43,16 @@ final class PhabricatorFileDocumentController
     $engine = $engines[$engine_key];
     $this->engine = $engine;
 
+    $encode_setting = $request->getStr('encode');
+    if (strlen($encode_setting)) {
+      $engine->setEncodingConfiguration($encode_setting);
+    }
+
+    $highlight_setting = $request->getStr('highlight');
+    if (strlen($highlight_setting)) {
+      $engine->setHighlightingConfiguration($highlight_setting);
+    }
+
     try {
       $content = $engine->newDocument($ref);
     } catch (Exception $ex) {

src/applications/files/controller/PhabricatorFileViewController.php

@@ -422,6 +422,16 @@ final class PhabricatorFileViewController extends PhabricatorFileController {
       $engine->setHighlightedLines(range($lines[0], $lines[1]));
     }
 
+    $encode_setting = $request->getStr('encode');
+    if (strlen($encode_setting)) {
+      $engine->setEncodingConfiguration($encode_setting);
+    }
+
+    $highlight_setting = $request->getStr('highlight');
+    if (strlen($highlight_setting)) {
+      $engine->setHighlightingConfiguration($highlight_setting);
+    }
+
     $views = array();
     foreach ($engines as $candidate_key => $candidate_engine) {
       $label = $candidate_engine->getViewAsLabel($ref);
@@ -443,6 +453,8 @@ final class PhabricatorFileViewController extends PhabricatorFileController {
         'engineURI' => $candidate_engine->getRenderURI($ref),
         'viewURI' => $view_uri,
         'loadingMarkup' => hsprintf('%s', $loading),
+        'canEncode' => $candidate_engine->canConfigureEncoding($ref),
+        'canHighlight' => $candidate_engine->CanConfigureHighlighting($ref),
       );
     }
 
@@ -474,6 +486,18 @@ final class PhabricatorFileViewController extends PhabricatorFileController {
       'viewKey' => $engine->getDocumentEngineKey(),
       'views' => $views,
       'standaloneURI' => $engine->getRenderURI($ref),
+      'encode' => array(
+        'icon' => 'fa-font',
+        'name' => pht('Change Text Encoding...'),
+        'uri' => '/services/encoding/',
+        'value' => $encode_setting,
+      ),
+      'highlight' => array(
+        'icon' => 'fa-lightbulb-o',
+        'name' => pht('Highlight As...'),
+        'uri' => '/services/highlight/',
+        'value' => $highlight_setting,
+      ),
     );
 
     $view_button = id(new PHUIButtonView())

src/applications/files/document/PhabricatorDocumentEngine.php

@@ -5,6 +5,8 @@ abstract class PhabricatorDocumentEngine
 
   private $viewer;
   private $highlightedLines = array();
+  private $encodingConfiguration;
+  private $highlightingConfiguration;
 
   final public function setViewer(PhabricatorUser $viewer) {
     $this->viewer = $viewer;
@@ -28,6 +30,32 @@ abstract class PhabricatorDocumentEngine
     return $this->canRenderDocumentType($ref);
   }
 
+  public function canConfigureEncoding(PhabricatorDocumentRef $ref) {
+    return false;
+  }
+
+  public function canConfigureHighlighting(PhabricatorDocumentRef $ref) {
+    return false;
+  }
+
+  final public function setEncodingConfiguration($config) {
+    $this->encodingConfiguration = $config;
+    return $this;
+  }
+
+  final public function getEncodingConfiguration() {
+    return $this->encodingConfiguration;
+  }
+
+  final public function setHighlightingConfiguration($config) {
+    $this->highlightingConfiguration = $config;
+    return $this;
+  }
+
+  final public function getHighlightingConfiguration() {
+    return $this->highlightingConfiguration;
+  }
+
   public function shouldRenderAsync(PhabricatorDocumentRef $ref) {
     return false;
   }

src/applications/files/document/PhabricatorSourceDocumentEngine.php

@@ -9,6 +9,10 @@ final class PhabricatorSourceDocumentEngine
     return pht('View as Source');
   }
 
+  public function canConfigureHighlighting(PhabricatorDocumentRef $ref) {
+    return true;
+  }
+
   protected function getDocumentIconIcon(PhabricatorDocumentRef $ref) {
     return 'fa-code';
   }
@@ -20,9 +24,16 @@ final class PhabricatorSourceDocumentEngine
   protected function newDocumentContent(PhabricatorDocumentRef $ref) {
     $content = $this->loadTextData($ref);
 
-    $content = PhabricatorSyntaxHighlighter::highlightWithFilename(
+    $highlighting = $this->getHighlightingConfiguration();
-      $ref->getName(),
+    if ($highlighting !== null) {
-      $content);
+      $content = PhabricatorSyntaxHighlighter::highlightWithLanguage(
+        $highlighting,
+        $content);
+    } else {
+      $content = PhabricatorSyntaxHighlighter::highlightWithFilename(
+        $ref->getName(),
+        $content);
+    }
 
     return $this->newTextDocumentContent($content);
   }

src/applications/files/document/PhabricatorTextDocumentEngine.php

@@ -3,10 +3,16 @@
 abstract class PhabricatorTextDocumentEngine
   extends PhabricatorDocumentEngine {
 
+  private $encodingMessage = null;
+
   protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
     return $ref->isProbablyText();
   }
 
+  public function canConfigureEncoding(PhabricatorDocumentRef $ref) {
+    return true;
+  }
+
   protected function newTextDocumentContent($content) {
     $lines = phutil_split_lines($content);
 
@@ -14,19 +20,69 @@ abstract class PhabricatorTextDocumentEngine
       ->setHighlights($this->getHighlightedLines())
       ->setLines($lines);
 
+    $message = null;
+    if ($this->encodingMessage !== null) {
+      $message = $this->newMessage($this->encodingMessage);
+    }
+
     $container = phutil_tag(
       'div',
       array(
         'class' => 'document-engine-text',
       ),
-      $view);
+      array(
+        $message,
+        $view,
+      ));
 
     return $container;
   }
 
   protected function loadTextData(PhabricatorDocumentRef $ref) {
     $content = $ref->loadData();
-    $content = phutil_utf8ize($content);
+
+    $encoding = $this->getEncodingConfiguration();
+    if ($encoding !== null) {
+      if (function_exists('mb_convert_encoding')) {
+        $content = mb_convert_encoding($content, 'UTF-8', $encoding);
+        $this->encodingMessage = pht(
+          'This document was converted from %s to UTF8 for display.',
+          $encoding);
+      } else {
+        $this->encodingMessage = pht(
+          'Unable to perform text encoding conversion: mbstring extension '.
+          'is not available.');
+      }
+    } else {
+      if (!phutil_is_utf8($content)) {
+        if (function_exists('mb_detect_encoding')) {
+          $try_encodings = array(
+            'JIS' => pht('JIS'),
+            'EUC-JP' => pht('EUC-JP'),
+            'SJIS' => pht('Shift JIS'),
+            'ISO-8859-1' => pht('ISO-8859-1 (Latin 1)'),
+          );
+
+          $guess = mb_detect_encoding($content, array_keys($try_encodings));
+          if ($guess) {
+            $content = mb_convert_encoding($content, 'UTF-8', $guess);
+            $this->encodingMessage = pht(
+              'This document is not UTF8. It was detected as %s and '.
+              'converted to UTF8 for display.',
+              idx($try_encodings, $guess, $guess));
+          }
+        }
+      }
+    }
+
+    if (!phutil_is_utf8($content)) {
+      $content = phutil_utf8ize($content);
+      $this->encodingMessage = pht(
+        'This document is not UTF8 and its text encoding could not be '.
+        'detected automatically. Use "Change Text Encoding..." to choose '.
+        'an encoding.');
+    }
+
     return $content;
   }
 

webroot/rsrc/js/application/files/behavior-document-engine.js

@@ -52,6 +52,61 @@ JX.behavior('document-engine', function(config, statics) {
       });
     }
 
+    list.addItem(
+      new JX.PHUIXActionView()
+        .setDivider(true));
+
+    var encode_item = new JX.PHUIXActionView()
+      .setName(data.encode.name)
+      .setIcon(data.encode.icon);
+
+    var onencode = JX.bind(null, function(data, e) {
+      e.prevent();
+
+      if (encode_item.getDisabled()) {
+        return;
+      }
+
+      new JX.Workflow(data.encode.uri, {encoding: data.encode.value})
+        .setHandler(function(r) {
+          data.encode.value = r.encoding;
+          onview(data);
+        })
+        .start();
+
+      menu.close();
+
+    }, data);
+
+    encode_item.setHandler(onencode);
+
+    list.addItem(encode_item);
+
+    var highlight_item = new JX.PHUIXActionView()
+      .setName(data.highlight.name)
+      .setIcon(data.highlight.icon);
+
+    var onhighlight = JX.bind(null, function(data, e) {
+      e.prevent();
+
+      if (highlight_item.getDisabled()) {
+        return;
+      }
+
+      new JX.Workflow(data.highlight.uri, {highlight: data.highlight.value})
+        .setHandler(function(r) {
+          data.highlight.value = r.highlight;
+          onview(data);
+        })
+        .start();
+
+      menu.close();
+    }, data);
+
+    highlight_item.setHandler(onhighlight);
+
+    list.addItem(highlight_item);
+
     menu.setContent(list.getNode());
 
     menu.listen('open', function() {
@@ -61,6 +116,11 @@ JX.behavior('document-engine', function(config, statics) {
         // Highlight the current rendering engine.
         var is_selected = (engine.spec.viewKey == data.viewKey);
         engine.view.setSelected(is_selected);
+
+        if (is_selected) {
+          encode_item.setDisabled(!engine.spec.canEncode);
+          highlight_item.setDisabled(!engine.spec.canHighlight);
+        }
       }
     });
 
@@ -68,13 +128,38 @@ JX.behavior('document-engine', function(config, statics) {
     menu.open();
   }
 
+  function add_params(uri, data) {
+    uri = JX.$U(uri);
+
+    if (data.highlight.value) {
+      uri.setQueryParam('highlight', data.highlight.value);
+    }
+
+    if (data.encode.value) {
+      uri.setQueryParam('encode', data.encode.value);
+    }
+
+    return uri.toString();
+  }
+
   function onview(data, spec, immediate) {
+    if (!spec) {
+      for (var ii = 0; ii < data.views.length; ii++) {
+        if (data.views[ii].viewKey == data.viewKey) {
+          spec = data.views[ii];
+          break;
+        }
+      }
+    }
+
     data.sequence = (data.sequence || 0) + 1;
     var handler = JX.bind(null, onrender, data, data.sequence);
 
     data.viewKey = spec.viewKey;
 
-    new JX.Request(spec.engineURI, handler)
+    var uri = add_params(spec.engineURI, data);
+
+    new JX.Request(uri, handler)
       .send();
 
     if (data.loadingView) {
@@ -93,7 +178,9 @@ JX.behavior('document-engine', function(config, statics) {
 
       // Replace the URI with the URI for the specific rendering the user
       // has selected.
-      JX.History.replace(spec.viewURI);
+
+      var view_uri = add_params(spec.viewURI, data);
+      JX.History.replace(view_uri);
     }
   }
 
@@ -134,13 +221,7 @@ JX.behavior('document-engine', function(config, statics) {
   if (config && config.renderControlID) {
     var control = JX.$(config.renderControlID);
     var data = JX.Stratcom.getData(control);
-
+    onview(data, null, true);
-    for (var ii = 0; ii < data.views.length; ii++) {
-      if (data.views[ii].viewKey == data.viewKey) {
-        onview(data, data.views[ii], true);
-        break;
-      }
-    }
   }
 
 });

webroot/rsrc/js/phuix/PHUIXActionView.js

@@ -34,6 +34,10 @@ JX.install('PHUIXActionView', {
       return this;
     },
 
+    getDisabled: function() {
+      return this._disabled;
+    },
+
     setLabel: function(label) {
       this._label = label;
       JX.DOM.alterClass(