Substantially support character encodings and "Highlight As" in changesets

Summary: Ref T5179. Ref T4045. Ref T832. We can now write non-utf8 hunks into the database, so try to do more reasonable things with them in the UI.

Test Plan: (See screenshots...)

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T832, T4045, T5179

Differential Revision: https://secure.phabricator.com/D9294

resources/celerity/map.php

@@ -11,7 +11,7 @@ return array(
     'core.pkg.js' => '07b01d4f',
     'darkconsole.pkg.js' => 'ca8671ce',
     'differential.pkg.css' => '4a93db37',
-    'differential.pkg.js' => 'eca39a2c',
+    'differential.pkg.js' => '2b128f3a',
     'diffusion.pkg.css' => '471bc9eb',
     'diffusion.pkg.js' => '077e3ad0',
     'maniphest.pkg.css' => 'f88a8402',
@@ -358,13 +358,13 @@ return array(
     'rsrc/js/application/dashboard/behavior-dashboard-move-panels.js' => 'fa187a68',
     'rsrc/js/application/dashboard/behavior-dashboard-query-panel-select.js' => '3be3eef5',
     'rsrc/js/application/dashboard/behavior-dashboard-tab-panel.js' => 'aa077691',
-    'rsrc/js/application/differential/ChangesetViewManager.js' => 'db09a523',
+    'rsrc/js/application/differential/ChangesetViewManager.js' => '1f304ef8',
     'rsrc/js/application/differential/DifferentialInlineCommentEditor.js' => 'f2441746',
     'rsrc/js/application/differential/behavior-add-reviewers-and-ccs.js' => '533a187b',
     'rsrc/js/application/differential/behavior-comment-jump.js' => '71755c79',
     'rsrc/js/application/differential/behavior-comment-preview.js' => '127f2018',
     'rsrc/js/application/differential/behavior-diff-radios.js' => 'e1ff79b1',
-    'rsrc/js/application/differential/behavior-dropdown-menus.js' => '64a79839',
+    'rsrc/js/application/differential/behavior-dropdown-menus.js' => '710f209e',
     'rsrc/js/application/differential/behavior-edit-inline-comments.js' => '00861799',
     'rsrc/js/application/differential/behavior-keyboard-nav.js' => '173ce7e7',
     'rsrc/js/application/differential/behavior-populate.js' => 'bdb3e4d0',
@@ -509,7 +509,7 @@ return array(
     'aphront-two-column-view-css' => '16ab3ad2',
     'aphront-typeahead-control-css' => 'a989b5b3',
     'auth-css' => '1e655982',
-    'changeset-view-manager' => 'db09a523',
+    'changeset-view-manager' => '1f304ef8',
     'config-options-css' => '7fedf08b',
     'conpherence-menu-css' => 'e1e0fdf1',
     'conpherence-message-pane-css' => '11a393ca',
@@ -564,7 +564,7 @@ return array(
     'javelin-behavior-differential-add-reviewers-and-ccs' => '533a187b',
     'javelin-behavior-differential-comment-jump' => '71755c79',
     'javelin-behavior-differential-diff-radios' => 'e1ff79b1',
-    'javelin-behavior-differential-dropdown-menus' => '64a79839',
+    'javelin-behavior-differential-dropdown-menus' => '710f209e',
     'javelin-behavior-differential-edit-inline-comments' => '00861799',
     'javelin-behavior-differential-feedback-preview' => '127f2018',
     'javelin-behavior-differential-keyboard-navigation' => '173ce7e7',
@@ -1001,6 +1001,17 @@ return array(
       5 => 'phabricator-drag-and-drop-file-upload',
       6 => 'phabricator-draggable-list',
     ),
+    '1f304ef8' =>
+    array(
+      0 => 'javelin-dom',
+      1 => 'javelin-util',
+      2 => 'javelin-stratcom',
+      3 => 'javelin-install',
+      4 => 'javelin-workflow',
+      5 => 'javelin-router',
+      6 => 'javelin-behavior-device',
+      7 => 'javelin-vector',
+    ),
     '2290aeef' =>
     array(
       0 => 'javelin-install',
@@ -1273,11 +1284,6 @@ return array(
       2 => 'javelin-util',
       3 => 'phabricator-shaped-request',
     ),
-    '7319e029' =>
-    array(
-      0 => 'javelin-behavior',
-      1 => 'javelin-dom',
-    ),
     '62e18640' =>
     array(
       0 => 'javelin-install',
@@ -1291,18 +1297,6 @@ return array(
       1 => 'javelin-dom',
       2 => 'javelin-fx',
     ),
-    '64a79839' =>
-    array(
-      0 => 'javelin-behavior',
-      1 => 'javelin-dom',
-      2 => 'javelin-util',
-      3 => 'javelin-stratcom',
-      4 => 'phuix-dropdown-menu',
-      5 => 'phuix-action-list-view',
-      6 => 'phuix-action-view',
-      7 => 'phabricator-phtize',
-      8 => 'changeset-view-manager',
-    ),
     '64ef2fd2' =>
     array(
       0 => 'javelin-behavior',
@@ -1344,12 +1338,30 @@ return array(
       1 => 'javelin-dom',
       2 => 'javelin-util',
     ),
+    '710f209e' =>
+    array(
+      0 => 'javelin-behavior',
+      1 => 'javelin-dom',
+      2 => 'javelin-util',
+      3 => 'javelin-stratcom',
+      4 => 'javelin-workflow',
+      5 => 'phuix-dropdown-menu',
+      6 => 'phuix-action-list-view',
+      7 => 'phuix-action-view',
+      8 => 'phabricator-phtize',
+      9 => 'changeset-view-manager',
+    ),
     '71755c79' =>
     array(
       0 => 'javelin-behavior',
       1 => 'javelin-stratcom',
       2 => 'javelin-dom',
     ),
+    '7319e029' =>
+    array(
+      0 => 'javelin-behavior',
+      1 => 'javelin-dom',
+    ),
     '76f4ebed' =>
     array(
       0 => 'javelin-install',
@@ -1916,17 +1928,6 @@ return array(
       1 => 'javelin-util',
       2 => 'javelin-stratcom',
     ),
-    'db09a523' =>
-    array(
-      0 => 'javelin-dom',
-      1 => 'javelin-util',
-      2 => 'javelin-stratcom',
-      3 => 'javelin-install',
-      4 => 'javelin-workflow',
-      5 => 'javelin-router',
-      6 => 'javelin-behavior-device',
-      7 => 'javelin-vector',
-    ),
     'dd7e8ef5' =>
     array(
       0 => 'javelin-behavior',

src/__phutil_library_map__.php

@@ -2258,6 +2258,8 @@ phutil_register_library_map(array(
     'PhabricatorSystemRemoveDestroyWorkflow' => 'applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php',
     'PhabricatorSystemRemoveLogWorkflow' => 'applications/system/management/PhabricatorSystemRemoveLogWorkflow.php',
     'PhabricatorSystemRemoveWorkflow' => 'applications/system/management/PhabricatorSystemRemoveWorkflow.php',
+    'PhabricatorSystemSelectEncodingController' => 'applications/system/controller/PhabricatorSystemSelectEncodingController.php',
+    'PhabricatorSystemSelectHighlightController' => 'applications/system/controller/PhabricatorSystemSelectHighlightController.php',
     'PhabricatorTaskmasterDaemon' => 'infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php',
     'PhabricatorTestCase' => 'infrastructure/testing/PhabricatorTestCase.php',
     'PhabricatorTestController' => 'applications/base/controller/__tests__/PhabricatorTestController.php',
@@ -5134,6 +5136,8 @@ phutil_register_library_map(array(
     'PhabricatorSystemRemoveDestroyWorkflow' => 'PhabricatorSystemRemoveWorkflow',
     'PhabricatorSystemRemoveLogWorkflow' => 'PhabricatorSystemRemoveWorkflow',
     'PhabricatorSystemRemoveWorkflow' => 'PhabricatorManagementWorkflow',
+    'PhabricatorSystemSelectEncodingController' => 'PhabricatorController',
+    'PhabricatorSystemSelectHighlightController' => 'PhabricatorController',
     'PhabricatorTaskmasterDaemon' => 'PhabricatorDaemon',
     'PhabricatorTestCase' => 'ArcanistPhutilTestCase',
     'PhabricatorTestController' => 'PhabricatorController',

src/applications/differential/controller/DifferentialChangesetViewController.php

@@ -155,6 +155,8 @@ final class DifferentialChangesetViewController extends DifferentialController {
     $parser->setRightSideCommentMapping($right_source, $right_new);
     $parser->setLeftSideCommentMapping($left_source, $left_new);
     $parser->setWhitespaceMode($request->getStr('whitespace'));
+    $parser->setCharacterEncoding($request->getStr('encoding'));
+    $parser->setHighlightAs($request->getStr('highlight'));
 
     if ($request->getStr('renderer') == '1up') {
       $parser->setRenderer(new DifferentialChangesetOneUpRenderer());

src/applications/differential/parser/DifferentialChangesetParser.php

@@ -41,6 +41,26 @@ final class DifferentialChangesetParser {
   private $highlightErrors;
   private $disableCache;
   private $renderer;
+  private $characterEncoding;
+  private $highlightAs;
+
+  public function setHighlightAs($highlight_as) {
+    $this->highlightAs = $highlight_as;
+    return $this;
+  }
+
+  public function getHighlightAs() {
+    return $this->highlightAs;
+  }
+
+  public function setCharacterEncoding($character_encoding) {
+    $this->characterEncoding = $character_encoding;
+    return $this;
+  }
+
+  public function getCharacterEncoding() {
+    return $this->characterEncoding;
+  }
 
   public function setRenderer($renderer) {
     $this->renderer = $renderer;
@@ -422,8 +442,15 @@ final class DifferentialChangesetParser {
   }
 
   private function getHighlightFuture($corpus) {
+    $language = $this->highlightAs;
+
+    if (!$language) {
+      $language = $this->highlightEngine->getLanguageFromFilename(
+        $this->filename);
+    }
+
     return $this->highlightEngine->getHighlightFuture(
-      $this->highlightEngine->getLanguageFromFilename($this->filename),
+      $language,
       $corpus);
   }
 
@@ -455,6 +482,14 @@ final class DifferentialChangesetParser {
       $skip_cache = true;
     }
 
+    if ($this->characterEncoding) {
+      $skip_cache = true;
+    }
+
+    if ($this->highlightAs) {
+      $skip_cache = true;
+    }
+
     $this->whitespaceMode = $whitespace_mode;
 
     $changeset = $this->changeset;
@@ -677,6 +712,25 @@ final class DifferentialChangesetParser {
     // requests.
     $this->isTopLevel = (($range_start === null) && ($range_len === null));
     $this->highlightEngine = PhabricatorSyntaxHighlighter::newEngine();
+
+    $encoding = null;
+    if ($this->characterEncoding) {
+      // We are forcing this changeset to be interpreted with a specific
+      // character encoding, so force all the hunks into that encoding and
+      // propagate it to the renderer.
+      $encoding = $this->characterEncoding;
+      foreach ($this->changeset->getHunks() as $hunk) {
+        $hunk->forceEncoding($this->characterEncoding);
+      }
+    } else {
+      // We're just using the default, so tell the renderer what that is
+      // (by reading the encoding from the first hunk).
+      foreach ($this->changeset->getHunks() as $hunk) {
+        $encoding = $hunk->getDataEncoding();
+        break;
+      }
+    }
+
     $this->tryCacheStuff();
     $render_pch = $this->shouldRenderPropertyChangeHeader($this->changeset);
 
@@ -700,7 +754,8 @@ final class DifferentialChangesetParser {
       ->setMarkupEngine($this->markupEngine)
       ->setHandles($this->handles)
       ->setOldLines($this->old)
-      ->setNewLines($this->new);
+      ->setNewLines($this->new)
+      ->setOriginalCharacterEncoding($encoding);
 
     if ($this->user) {
       $renderer->setUser($this->user);
@@ -1013,18 +1068,18 @@ final class DifferentialChangesetParser {
   private function isCommentVisibleOnRenderedDiff(
     PhabricatorInlineCommentInterface $comment) {
 
-      $changeset_id = $comment->getChangesetID();
+    $changeset_id = $comment->getChangesetID();
-      $is_new = $comment->getIsNewFile();
+    $is_new = $comment->getIsNewFile();
 
-      if ($changeset_id == $this->rightSideChangesetID &&
+    if ($changeset_id == $this->rightSideChangesetID &&
         $is_new == $this->rightSideAttachesToNewFile) {
-          return true;
+        return true;
-        }
+    }
 
-      if ($changeset_id == $this->leftSideChangesetID &&
+    if ($changeset_id == $this->leftSideChangesetID &&
         $is_new == $this->leftSideAttachesToNewFile) {
-          return true;
+        return true;
-        }
+    }
 
     return false;
   }

src/applications/differential/render/DifferentialChangesetHTMLRenderer.php

@@ -9,248 +9,275 @@ abstract class DifferentialChangesetHTMLRenderer
     $change = $changeset->getChangeType();
     $file = $changeset->getFileType();
 
-    $message = null;
+    $messages = array();
-    if ($change == DifferentialChangeType::TYPE_CHANGE &&
+    $none = hsprintf('');
-        $file   == DifferentialChangeType::FILE_TEXT) {
+    switch ($change) {
-      if ($force) {
+
-        // We have to force something to render because there were no changes
+      case DifferentialChangeType::TYPE_ADD:
-        // of other kinds.
+        switch ($file) {
-        $message = pht('This file was not modified.');
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht(
+              'This file was <strong>added</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht(
+              'This image was <strong>added</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht(
+              'This directory was <strong>added</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht(
+              'This binary file was <strong>added</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht(
+              'This symlink was <strong>added</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht(
+              'This submodule was <strong>added</strong>.',
+              $none);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_DELETE:
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht(
+              'This file was <strong>deleted</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht(
+              'This image was <strong>deleted</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht(
+              'This directory was <strong>deleted</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht(
+              'This binary file was <strong>deleted</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht(
+              'This symlink was <strong>deleted</strong>.',
+              $none);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht(
+              'This submodule was <strong>deleted</strong>.',
+              $none);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_MOVE_HERE:
+        $from = phutil_tag('strong', array(), $changeset->getOldFile());
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht('This file was moved from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht('This image was moved from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht('This directory was moved from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht('This binary file was moved from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht('This symlink was moved from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht('This submodule was moved from %s.', $from);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_COPY_HERE:
+        $from = phutil_tag('strong', array(), $changeset->getOldFile());
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht('This file was copied from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht('This image was copied from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht('This directory was copied from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht('This binary file was copied from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht('This symlink was copied from %s.', $from);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht('This submodule was copied from %s.', $from);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_MOVE_AWAY:
+        $paths = phutil_tag(
+          'strong',
+          array(),
+          implode(', ', $changeset->getAwayPaths()));
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht('This file was moved to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht('This image was moved to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht('This directory was moved to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht('This binary file was moved to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht('This symlink was moved to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht('This submodule was moved to %s.', $paths);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_COPY_AWAY:
+        $paths = phutil_tag(
+          'strong',
+          array(),
+          implode(', ', $changeset->getAwayPaths()));
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht('This file was copied to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht('This image was copied to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht('This directory was copied to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht('This binary file was copied to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht('This symlink was copied to %s.', $paths);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht('This submodule was copied to %s.', $paths);
+            break;
+        }
+        break;
+
+      case DifferentialChangeType::TYPE_MULTICOPY:
+        $paths = phutil_tag(
+          'strong',
+          array(),
+          implode(', ', $changeset->getAwayPaths()));
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            $messages[] = pht(
+              'This file was deleted after being copied to %s.',
+              $paths);
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht(
+              'This image was deleted after being copied to %s.',
+              $paths);
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht(
+              'This directory was deleted after being copied to %s.',
+              $paths);
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht(
+              'This binary file was deleted after being copied to %s.',
+              $paths);
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht(
+              'This symlink was deleted after being copied to %s.',
+              $paths);
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht(
+              'This submodule was deleted after being copied to %s.',
+              $paths);
+            break;
+        }
+        break;
+
+      default:
+        switch ($file) {
+          case DifferentialChangeType::FILE_TEXT:
+            // This is the default case, so we only render this header if
+            // forced to since it's not very useful.
+            if ($force) {
+              $messages[] = pht('This file was not modified.');
+            }
+            break;
+          case DifferentialChangeType::FILE_IMAGE:
+            $messages[] = pht('This is an image.');
+            break;
+          case DifferentialChangeType::FILE_DIRECTORY:
+            $messages[] = pht('This is a directory.');
+            break;
+          case DifferentialChangeType::FILE_BINARY:
+            $messages[] = pht('This is a binary file.');
+            break;
+          case DifferentialChangeType::FILE_SYMLINK:
+            $messages[] = pht('This is a symlink.');
+            break;
+          case DifferentialChangeType::FILE_SUBMODULE:
+            $messages[] = pht('This is a submodule.');
+            break;
+        }
+        break;
+    }
+
+    $encoding = $this->getOriginalCharacterEncoding();
+    if ($encoding != 'utf8' && ($file == DifferentialChangeType::FILE_TEXT)) {
+      if ($encoding) {
+        $messages[] = pht(
+          'This file was converted from %s for display.',
+          phutil_tag('strong', array(), $encoding));
       } else {
-        // Default case of changes to a text file, no metadata.
+        $messages[] = pht(
-        return null;
+          'This file uses an unknown character encoding.');
-      }
-    } else {
-      $none = hsprintf('');
-      switch ($change) {
-
-        case DifferentialChangeType::TYPE_ADD:
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was <strong>added</strong>.', $none);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was <strong>added</strong>.', $none);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht(
-                'This directory was <strong>added</strong>.',
-                $none);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht(
-                'This binary file was <strong>added</strong>.',
-                $none);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This symlink was <strong>added</strong>.', $none);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht(
-                'This submodule was <strong>added</strong>.',
-                $none);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_DELETE:
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was <strong>deleted</strong>.', $none);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was <strong>deleted</strong>.', $none);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht(
-                'This directory was <strong>deleted</strong>.',
-                $none);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht(
-                'This binary file was <strong>deleted</strong>.',
-                $none);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht(
-                'This symlink was <strong>deleted</strong>.',
-                $none);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht(
-                'This submodule was <strong>deleted</strong>.',
-                $none);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_MOVE_HERE:
-          $from = phutil_tag('strong', array(), $changeset->getOldFile());
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was moved from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was moved from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht('This directory was moved from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht('This binary file was moved from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This symlink was moved from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht('This submodule was moved from %s.', $from);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_COPY_HERE:
-          $from = phutil_tag('strong', array(), $changeset->getOldFile());
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was copied from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was copied from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht('This directory was copied from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht('This binary file was copied from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This symlink was copied from %s.', $from);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht('This submodule was copied from %s.', $from);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_MOVE_AWAY:
-          $paths = phutil_tag(
-            'strong',
-            array(),
-            implode(', ', $changeset->getAwayPaths()));
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was moved to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was moved to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht('This directory was moved to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht('This binary file was moved to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This symlink was moved to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht('This submodule was moved to %s.', $paths);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_COPY_AWAY:
-          $paths = phutil_tag(
-            'strong',
-            array(),
-            implode(', ', $changeset->getAwayPaths()));
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This file was copied to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This image was copied to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht('This directory was copied to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht('This binary file was copied to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This symlink was copied to %s.', $paths);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht('This submodule was copied to %s.', $paths);
-              break;
-          }
-          break;
-
-        case DifferentialChangeType::TYPE_MULTICOPY:
-          $paths = phutil_tag(
-            'strong',
-            array(),
-            implode(', ', $changeset->getAwayPaths()));
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht(
-                'This file was deleted after being copied to %s.',
-                $paths);
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht(
-                'This image was deleted after being copied to %s.',
-                $paths);
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht(
-                'This directory was deleted after being copied to %s.',
-                $paths);
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht(
-                'This binary file was deleted after being copied to %s.',
-                $paths);
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht(
-                'This symlink was deleted after being copied to %s.',
-                $paths);
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht(
-                'This submodule was deleted after being copied to %s.',
-                $paths);
-              break;
-          }
-          break;
-
-        default:
-          switch ($file) {
-            case DifferentialChangeType::FILE_TEXT:
-              $message = pht('This is a file.');
-              break;
-            case DifferentialChangeType::FILE_IMAGE:
-              $message = pht('This is an image.');
-              break;
-            case DifferentialChangeType::FILE_DIRECTORY:
-              $message = pht('This is a directory.');
-              break;
-            case DifferentialChangeType::FILE_BINARY:
-              $message = pht('This is a binary file.');
-              break;
-            case DifferentialChangeType::FILE_SYMLINK:
-              $message = pht('This is a symlink.');
-              break;
-            case DifferentialChangeType::FILE_SUBMODULE:
-              $message = pht('This is a submodule.');
-              break;
-          }
-          break;
       }
     }
 
-    return phutil_tag_div('differential-meta-notice', $message);
+    if (!$messages) {
+      return null;
+    }
+
+    foreach ($messages as $key => $message) {
+      $messages[$key] = phutil_tag('li', array(), $message);
+    }
+
+    return phutil_tag(
+      'ul',
+      array(
+        'class' => 'differential-meta-notice',
+      ),
+      $messages);
   }
 
   protected function renderPropertyChangeHeader() {

src/applications/differential/render/DifferentialChangesetRenderer.php

@@ -28,6 +28,16 @@ abstract class DifferentialChangesetRenderer {
   private $gaps;
   private $mask;
   private $depths;
+  private $originalCharacterEncoding;
+
+  public function setOriginalCharacterEncoding($original_character_encoding) {
+    $this->originalCharacterEncoding = $original_character_encoding;
+    return $this;
+  }
+
+  public function getOriginalCharacterEncoding() {
+    return $this->originalCharacterEncoding;
+  }
 
   public function setDepths($depths) {
     $this->depths = $depths;

src/applications/differential/storage/DifferentialHunkLegacy.php

@@ -8,4 +8,13 @@ final class DifferentialHunkLegacy extends DifferentialHunk {
     return 'differential_hunk';
   }
 
+  public function getDataEncoding() {
+    return 'utf8';
+  }
+
+  public function forceEncoding($encoding) {
+    // Not supported, these are always utf8.
+    return $this;
+  }
+
 }

src/applications/differential/storage/DifferentialHunkModern.php

@@ -14,6 +14,7 @@ final class DifferentialHunkModern extends DifferentialHunk {
   protected $data;
 
   private $rawData;
+  private $forcedEncoding;
 
   public function getTableName() {
     return 'differential_hunk_modern';
@@ -41,7 +42,12 @@ final class DifferentialHunkModern extends DifferentialHunk {
   public function getChanges() {
     return $this->getUTF8StringFromStorage(
       $this->getRawData(),
-      $this->getDataEncoding());
+      nonempty($this->forcedEncoding, $this->getDataEncoding()));
+  }
+
+  public function forceEncoding($encoding) {
+    $this->forcedEncoding = $encoding;
+    return $this;
   }
 
   public function save() {

src/applications/differential/view/DifferentialChangesetListView.php

@@ -127,6 +127,11 @@ final class DifferentialChangesetListView extends AphrontView {
           'Show Raw File (Left)' => pht('Show Raw File (Left)'),
           'Show Raw File (Right)' => pht('Show Raw File (Right)'),
           'Configure Editor' => pht('Configure Editor'),
+          'Load Changes' => pht('Load Changes'),
+          'View Side-by-Side' => pht('View Side-by-Side'),
+          'View Unified' => pht('View Unified (Barely Works!)'),
+          'Change Text Encoding...' => pht('Change Text Encoding...'),
+          'Highlight As...' => pht('Highlight As...'),
         ),
       ));
 

src/applications/diffusion/controller/DiffusionDiffController.php

@@ -68,6 +68,9 @@ final class DiffusionDiffController extends DiffusionController {
       array(
         'action' => 'rendering-ref')));
 
+    $parser->setCharacterEncoding($request->getStr('encoding'));
+    $parser->setHighlightAs($request->getStr('highlight'));
+
     $coverage = $drequest->loadCoverage();
     if ($coverage) {
       $parser->setCoverage($coverage);

src/applications/system/application/PhabricatorApplicationSystem.php

@@ -15,6 +15,10 @@ final class PhabricatorApplicationSystem extends PhabricatorApplication {
       '/status/' => 'PhabricatorStatusController',
       '/debug/' => 'PhabricatorDebugController',
       '/robots.txt' => 'PhabricatorRobotsController',
+      '/services/' => array(
+        'encoding/' => 'PhabricatorSystemSelectEncodingController',
+        'highlight/' => 'PhabricatorSystemSelectHighlightController',
+      ),
     );
   }
 

src/applications/system/controller/PhabricatorSystemSelectEncodingController.php

@@ -0,0 +1,55 @@
+<?php
+
+final class PhabricatorSystemSelectEncodingController
+  extends PhabricatorController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  public function processRequest() {
+    $request = $this->getRequest();
+
+    if (!function_exists('mb_list_encodings')) {
+      return $this->newDialog()
+        ->setTitle(pht('No Encoding Support'))
+        ->appendParagraph(
+          pht(
+            'This system does not have the "mbstring" extension installed, '.
+            'so character encodings are not supported. Install "mbstring" to '.
+            'enable support.'))
+        ->addCancelButton('/');
+    }
+
+    if ($request->isFormPost()) {
+      $result = array('encoding' => $request->getStr('encoding'));
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    $encodings = mb_list_encodings();
+    $encodings = array_fuse($encodings);
+    asort($encodings);
+    unset($encodings['pass']);
+    unset($encodings['auto']);
+
+    $encodings = array(
+      '' => pht('(Use Default)'),
+    ) + $encodings;
+
+    $form = id(new AphrontFormView())
+      ->setUser($this->getRequest()->getUser())
+      ->appendRemarkupInstructions(pht('Choose a text encoding to use.'))
+      ->appendChild(
+        id(new AphrontFormSelectControl())
+          ->setLabel(pht('Encoding'))
+          ->setName('encoding')
+          ->setValue($request->getStr('encoding'))
+          ->setOptions($encodings));
+
+    return $this->newDialog()
+      ->setTitle(pht('Select Character Encoding'))
+      ->appendChild($form->buildLayoutView())
+      ->addSubmitButton(pht('Choose Encoding'))
+      ->addCancelButton('/');
+  }
+}

src/applications/system/controller/PhabricatorSystemSelectHighlightController.php

@@ -0,0 +1,38 @@
+<?php
+
+final class PhabricatorSystemSelectHighlightController
+  extends PhabricatorController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  public function processRequest() {
+    $request = $this->getRequest();
+
+    if ($request->isFormPost()) {
+      $result = array('highlight' => $request->getStr('highlight'));
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    $languages = array(
+      '' => pht('(Use Default)'),
+    ) + PhabricatorEnv::getEnvConfig('pygments.dropdown-choices');
+
+    $form = id(new AphrontFormView())
+      ->setUser($this->getRequest()->getUser())
+      ->appendRemarkupInstructions(pht('Choose a syntax highlighting to use.'))
+      ->appendChild(
+        id(new AphrontFormSelectControl())
+          ->setLabel(pht('Highlighting'))
+          ->setName('highlight')
+          ->setValue($request->getStr('highlight'))
+          ->setOptions($languages));
+
+    return $this->newDialog()
+      ->setTitle(pht('Select Syntax Highlighting'))
+      ->appendChild($form->buildLayoutView())
+      ->addSubmitButton(pht('Choose Highlighting'))
+      ->addCancelButton('/');
+  }
+}

src/infrastructure/storage/lisk/PhabricatorLiskDAO.php

@@ -183,6 +183,29 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
     if ($encoding == 'utf8') {
       return $string;
     }
+
+    if (function_exists('mb_detect_encoding')) {
+      if (strlen($encoding)) {
+        $try_encodings = array(
+          $encoding,
+        );
+      } else {
+        // TODO: This is pretty much a guess, and probably needs to be
+        // configurable in the long run.
+        $try_encodings = array(
+          'JIS',
+          'EUC-JP',
+          'SJIS',
+          'ISO-8859-1',
+        );
+      }
+
+      $guess = mb_detect_encoding($string, $try_encodings);
+      if ($guess) {
+        return mb_convert_encoding($string, 'UTF-8', $guess);
+      }
+    }
+
     return phutil_utf8ize($string);
   }
 

webroot/rsrc/js/application/differential/ChangesetViewManager.js

@@ -22,6 +22,7 @@ JX.install('ChangesetViewManager', {
     this._whitespace = data.whitespace;
     this._renderer = data.renderer;
     this._highlight = data.highlight;
+    this._encoding = data.encoding;
   },
 
   members: {
@@ -35,6 +36,7 @@ JX.install('ChangesetViewManager', {
     _whitespace: null,
     _renderer: null,
     _highlight: null,
+    _encoding: null,
 
 
     /**
@@ -120,8 +122,10 @@ JX.install('ChangesetViewManager', {
 
       var params = {
         ref: this._ref,
-        whitespace: this._whitespace,
+        whitespace: this._whitespace || '',
-        renderer: this._getRenderer()
+        renderer: this.getRenderer() || '',
+        highlight: this._highlight || '',
+        encoding: this._encoding || ''
       };
 
       var workflow = new JX.Workflow(this._renderURI, params)
@@ -160,8 +164,16 @@ JX.install('ChangesetViewManager', {
       return JX.Router.getInstance().getRoutableByKey(this._getRoutableKey());
     },
 
+    setRenderer: function(renderer) {
+      this._renderer = renderer;
+      return this;
+    },
+
+    getRenderer: function() {
+      if (this._renderer !== null) {
+        return this._renderer;
+      }
 
-    _getRenderer: function() {
       // TODO: This is a big pile of TODOs.
 
       // NOTE: If you load the page at one device resolution and then resize to
@@ -176,6 +188,23 @@ JX.install('ChangesetViewManager', {
       return renderer;
     },
 
+    setEncoding: function(encoding) {
+      this._encoding = encoding;
+      return this;
+    },
+
+    getEncoding: function() {
+      return this._encoding;
+    },
+
+    setHighlight: function(highlight) {
+      this._highlight = highlight;
+      return this;
+    },
+
+    getHighlight: function() {
+      return this._highlight;
+    },
 
     _getNodeData: function() {
       return JX.Stratcom.getData(this._node);

webroot/rsrc/js/application/differential/behavior-dropdown-menus.js

@@ -4,6 +4,7 @@
  *           javelin-dom
  *           javelin-util
  *           javelin-stratcom
+ *           javelin-workflow
  *           phuix-dropdown-menu
  *           phuix-action-list-view
  *           phuix-action-view
@@ -48,13 +49,18 @@ JX.behavior('differential-dropdown-menus', function(config) {
   var buildmenu = function(e) {
     var button = e.getNode('differential-view-options');
     var data = JX.Stratcom.getData(button);
-
     if (data.menu) {
       return;
     }
 
     e.prevent();
 
+    var changeset = JX.DOM.findAbove(
+      button,
+      'div',
+      'differential-changeset');
+
+    var view = JX.ChangesetViewManager.getForNode(changeset);
     var menu = new JX.PHUIXDropdownMenu(button);
     var list = new JX.PHUIXActionListView();
 
@@ -103,12 +109,15 @@ JX.behavior('differential-dropdown-menus', function(config) {
 
     var up_item = new JX.PHUIXActionView()
       .setHandler(function(e) {
-        var changeset = JX.DOM.findAbove(
+        if (view.isLoaded()) {
-          button,
+          var renderer = view.getRenderer();
-          'div',
+          if (renderer == '1up') {
-          'differential-changeset');
+            renderer = '2up';
-
+          } else {
-        var view = JX.ChangesetViewManager.getForNode(changeset);
+            renderer = '1up';
+          }
+          view.setRenderer(renderer);
+        }
         view.reload();
 
         e.prevent();
@@ -116,22 +125,54 @@ JX.behavior('differential-dropdown-menus', function(config) {
       });
     list.addItem(up_item);
 
+    var encoding_item = new JX.PHUIXActionView()
+      .setIcon('fa-font')
+      .setName(pht('Change Text Encoding...'))
+      .setHandler(function(e) {
+        var params = {
+          encoding: view.getEncoding()
+        };
+
+        new JX.Workflow('/services/encoding/', params)
+          .setHandler(function(r) {
+            view.setEncoding(r.encoding);
+            view.reload();
+          })
+          .start();
+
+        e.prevent();
+        menu.close();
+      });
+    list.addItem(encoding_item);
+
+    var highlight_item = new JX.PHUIXActionView()
+      .setIcon('fa-sun-o')
+      .setName(pht('Highlight As...'))
+      .setHandler(function(e) {
+        var params = {
+          highlight: view.getHighlight()
+        };
+
+        new JX.Workflow('/services/highlight/', params)
+          .setHandler(function(r) {
+            view.setHighlight(r.highlight);
+            view.reload();
+          })
+          .start();
+
+        e.prevent();
+        menu.close();
+      });
+    list.addItem(highlight_item);
+
     add_link('fa-arrow-left', pht('Show Raw File (Left)'), data.leftURI);
     add_link('fa-arrow-right', pht('Show Raw File (Right)'), data.rightURI);
     add_link('fa-pencil', pht('Open in Editor'), data.editor, true);
     add_link('fa-wrench', pht('Configure Editor'), data.editorConfigure);
 
-
     menu.setContent(list.getNode());
 
     menu.listen('open', function() {
-      var changeset = JX.DOM.findAbove(
-        button,
-        'div',
-        'differential-changeset');
-
-      var view = JX.ChangesetViewManager.getForNode(changeset);
-
       // When the user opens the menu, check if there are any "Show More"
       // links in the changeset body. If there aren't, disable the "Show
       // Entire File" menu item since it won't change anything.
@@ -155,16 +196,23 @@ JX.behavior('differential-dropdown-menus', function(config) {
           .setHandler(function(e) { e.prevent(); });
       }
 
-      // TODO: This is temporary and just makes testing easier. It will do
+      encoding_item.setDisabled(!view.isLoaded());
-      // some mojo soon.
+      highlight_item.setDisabled(!view.isLoaded());
+
       if (view.isLoaded()) {
-        up_item
+        if (view.getRenderer() == '2up') {
-          .setIcon('fa-refresh')
+          up_item
-          .setName('Reload');
+            .setIcon('fa-list-alt')
+            .setName(pht('View Unified'));
+        } else {
+          up_item
+            .setIcon('fa-files-o')
+            .setName(pht('View Side-by-Side'));
+        }
       } else {
         up_item
           .setIcon('fa-refresh')
-          .setName('Load');
+          .setName(pht('Load Changes'));
       }
 
       visible_item
@@ -198,6 +246,7 @@ JX.behavior('differential-dropdown-menus', function(config) {
       }
 
     });
+
     data.menu = menu;
     menu.open();
   };