Add filesize limits for document rendering engines and support partial/complete rendering

Summary:
Depends on D19238. Ref T13105. Give document engines some reasonable automatic support for degrading gracefully when someone tries to hexdump a 100MB file or similar.

Also, make "Video" sort above "Audio" for files which could be rendered either way.

Test Plan: Viewed audio, video, image, and other files. Adjusted limits and saw full, partial, and fallback/error rendering.

Maniphest Tasks: T13105

Differential Revision: https://secure.phabricator.com/D19239

resources/celerity/map.php

@@ -9,8 +9,8 @@ return array(
   'names' => array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
-    'core.pkg.css' => '97dc0e74',
+    'core.pkg.css' => '6da3c0e5',
-    'core.pkg.js' => '8581cd02',
+    'core.pkg.js' => '932d60d4',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
     'diffusion.pkg.css' => 'a2d17c7d',
@@ -168,7 +168,7 @@ return array(
     'rsrc/css/phui/phui-object-box.css' => '9cff003c',
     'rsrc/css/phui/phui-pager.css' => 'edcbc226',
     'rsrc/css/phui/phui-pinboard-view.css' => '2495140e',
-    'rsrc/css/phui/phui-property-list-view.css' => 'ef864066',
+    'rsrc/css/phui/phui-property-list-view.css' => '6ef560df',
     'rsrc/css/phui/phui-remarkup-preview.css' => '54a34863',
     'rsrc/css/phui/phui-segment-bar-view.css' => 'b1d1b892',
     'rsrc/css/phui/phui-spacing.css' => '042804d6',
@@ -392,7 +392,7 @@ return array(
     'rsrc/js/application/diffusion/behavior-pull-lastmodified.js' => 'f01586dc',
     'rsrc/js/application/doorkeeper/behavior-doorkeeper-tag.js' => '1db13e70',
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
-    'rsrc/js/application/files/behavior-document-engine.js' => 'f6d6f389',
+    'rsrc/js/application/files/behavior-document-engine.js' => '396ef112',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
     'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '191b4909',
@@ -508,7 +508,7 @@ return array(
     'rsrc/js/phui/behavior-phui-submenu.js' => 'a6f7a73b',
     'rsrc/js/phui/behavior-phui-tab-group.js' => '0a0b10e9',
     'rsrc/js/phuix/PHUIXActionListView.js' => 'b5c256b8',
-    'rsrc/js/phuix/PHUIXActionView.js' => '442efd08',
+    'rsrc/js/phuix/PHUIXActionView.js' => 'ed18356a',
     'rsrc/js/phuix/PHUIXAutocomplete.js' => '7fa5c915',
     'rsrc/js/phuix/PHUIXButtonView.js' => '8a91e1ac',
     'rsrc/js/phuix/PHUIXDropdownMenu.js' => '04b2ae03',
@@ -607,7 +607,7 @@ return array(
     'javelin-behavior-diffusion-jump-to' => '73d09eef',
     'javelin-behavior-diffusion-locate-file' => '6d3e1947',
     'javelin-behavior-diffusion-pull-lastmodified' => 'f01586dc',
-    'javelin-behavior-document-engine' => 'f6d6f389',
+    'javelin-behavior-document-engine' => '396ef112',
     'javelin-behavior-doorkeeper-tag' => '1db13e70',
     'javelin-behavior-drydock-live-operation-status' => '901935ef',
     'javelin-behavior-durable-column' => '2ae077e1',
@@ -850,7 +850,7 @@ return array(
     'phui-oi-simple-ui-css' => 'a8beebea',
     'phui-pager-css' => 'edcbc226',
     'phui-pinboard-view-css' => '2495140e',
-    'phui-property-list-view-css' => 'ef864066',
+    'phui-property-list-view-css' => '6ef560df',
     'phui-remarkup-preview-css' => '54a34863',
     'phui-segment-bar-view-css' => 'b1d1b892',
     'phui-spacing-css' => '042804d6',
@@ -864,7 +864,7 @@ return array(
     'phui-workcard-view-css' => 'cca5fa92',
     'phui-workpanel-view-css' => 'a3a63478',
     'phuix-action-list-view' => 'b5c256b8',
-    'phuix-action-view' => '442efd08',
+    'phuix-action-view' => 'ed18356a',
     'phuix-autocomplete' => '7fa5c915',
     'phuix-button-view' => '8a91e1ac',
     'phuix-dropdown-menu' => '04b2ae03',
@@ -1114,6 +1114,11 @@ return array(
       'javelin-dom',
       'javelin-vector',
     ),
+    '396ef112' => array(
+      'javelin-behavior',
+      'javelin-dom',
+      'javelin-stratcom',
+    ),
     '3ab51e2c' => array(
       'javelin-behavior',
       'javelin-behavior-device',
@@ -1184,11 +1189,6 @@ return array(
       'javelin-workflow',
       'javelin-workboard-controller',
     ),
-    '442efd08' => array(
-      'javelin-install',
-      'javelin-dom',
-      'javelin-util',
-    ),
     '44959b73' => array(
       'javelin-util',
       'javelin-uri',
@@ -2125,6 +2125,11 @@ return array(
       'javelin-stratcom',
       'javelin-vector',
     ),
+    'ed18356a' => array(
+      'javelin-install',
+      'javelin-dom',
+      'javelin-util',
+    ),
     'edf8a145' => array(
       'javelin-behavior',
       'javelin-uri',
@@ -2153,11 +2158,6 @@ return array(
       'javelin-util',
       'javelin-reactor',
     ),
-    'f6d6f389' => array(
-      'javelin-behavior',
-      'javelin-dom',
-      'javelin-stratcom',
-    ),
     'f829edb3' => array(
       'javelin-view',
       'javelin-install',

src/applications/files/controller/PhabricatorFileInfoController.php

@@ -423,10 +423,12 @@ final class PhabricatorFileInfoController extends PhabricatorFileController {
       }
 
       $view_icon = $candidate_engine->getViewAsIconIcon($ref);
+      $view_color = $candidate_engine->getViewAsIconColor($ref);
 
       $views[] = array(
         'viewKey' => $candidate_engine->getDocumentEngineKey(),
         'icon' => $view_icon,
+        'color' => $view_color,
         'name' => $label,
         'engineURI' => $candidate_engine->getRenderURI($ref),
       );

src/applications/files/document/PhabricatorAudioDocumentEngine.php

@@ -13,6 +13,10 @@ final class PhabricatorAudioDocumentEngine
     return 'fa-file-sound-o';
   }
 
+  protected function getByteLengthLimit() {
+    return null;
+  }
+
   protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
     $file = $ref->getFile();
     if ($file) {

src/applications/files/document/PhabricatorDocumentEngine.php

@@ -22,6 +22,18 @@ abstract class PhabricatorDocumentEngine
     PhabricatorDocumentRef $ref);
 
   final public function newDocument(PhabricatorDocumentRef $ref) {
+    $can_complete = $this->canRenderCompleteDocument($ref);
+    $can_partial = $this->canRenderPartialDocument($ref);
+
+    if (!$can_complete && !$can_partial) {
+      return $this->newMessage(
+        pht(
+          'This document is too large to be rendered inline. (The document '.
+          'is %s bytes, the limit for this engine is %s bytes.)',
+          new PhutilNumber($ref->getByteLength()),
+          new PhutilNumber($this->getByteLengthLimit())));
+    }
+
     return $this->newDocumentContent($ref);
   }
 
@@ -51,20 +63,49 @@ abstract class PhabricatorDocumentEngine
   final public function newSortVector(PhabricatorDocumentRef $ref) {
     $content_score = $this->getContentScore($ref);
 
+    // Prefer engines which can render the entire file over engines which
+    // can only render a header, and engines which can render a header over
+    // engines which can't render anything.
+    if ($this->canRenderCompleteDocument($ref)) {
+      $limit_score = 0;
+    } else if ($this->canRenderPartialDocument($ref)) {
+      $limit_score = 1;
+    } else {
+      $limit_score = 2;
+    }
+
     return id(new PhutilSortVector())
+      ->addInt($limit_score)
       ->addInt(-$content_score);
   }
 
-  protected function getContentScore() {
+  protected function getContentScore(PhabricatorDocumentRef $ref) {
     return 2000;
   }
 
   abstract public function getViewAsLabel(PhabricatorDocumentRef $ref);
 
   public function getViewAsIconIcon(PhabricatorDocumentRef $ref) {
+    $can_complete = $this->canRenderCompleteDocument($ref);
+    $can_partial = $this->canRenderPartialDocument($ref);
+
+    if (!$can_complete && !$can_partial) {
+      return 'fa-times';
+    }
+
     return $this->getDocumentIconIcon($ref);
   }
 
+  public function getViewAsIconColor(PhabricatorDocumentRef $ref) {
+    $can_complete = $this->canRenderCompleteDocument($ref);
+
+    if (!$can_complete) {
+      return 'grey';
+    }
+
+    return null;
+  }
+
   public function getRenderURI(PhabricatorDocumentRef $ref) {
     $file = $ref->getFile();
     if (!$file) {
@@ -107,4 +148,33 @@ abstract class PhabricatorDocumentEngine
     return array_select_keys($engines, array_keys($vectors));
   }
 
+  protected function getByteLengthLimit() {
+    return (1024 * 1024 * 8);
+  }
+
+  protected function canRenderCompleteDocument(PhabricatorDocumentRef $ref) {
+    $limit = $this->getByteLengthLimit();
+    if ($limit) {
+      $length = $ref->getByteLength();
+      if ($length > $limit) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  protected function canRenderPartialDocument(PhabricatorDocumentRef $ref) {
+    return false;
+  }
+
+  protected function newMessage($message) {
+    return phutil_tag(
+      'div',
+      array(
+        'class' => 'document-engine-error',
+      ),
+      $message);
+  }
+
 }

src/applications/files/document/PhabricatorDocumentRef.php

@@ -56,7 +56,7 @@ final class PhabricatorDocumentRef
     return $this;
   }
 
-  public function getLength() {
+  public function getByteLength() {
     if ($this->byteLength !== null) {
       return $this->byteLength;
     }
@@ -68,9 +68,15 @@ final class PhabricatorDocumentRef
     return null;
   }
 
-  public function loadData() {
+  public function loadData($begin = null, $end = null) {
     if ($this->file) {
-      return $this->file->loadFileData();
+      $iterator = $this->file->getFileDataIterator($begin, $end);
+
+      $result = '';
+      foreach ($iterator as $chunk) {
+        $result .= $chunk;
+      }
+      return $result;
     }
 
     throw new PhutilMethodNotImplementedException();

src/applications/files/document/PhabricatorHexdumpDocumentEngine.php

@@ -13,7 +13,11 @@ final class PhabricatorHexdumpDocumentEngine
     return 'fa-microchip';
   }
 
-  protected function getContentScore() {
+  protected function getByteLengthLimit() {
+    return (1024 * 1024 * 1);
+  }
+
+  protected function getContentScore(PhabricatorDocumentRef $ref) {
     return 500;
   }
 
@@ -21,8 +25,23 @@ final class PhabricatorHexdumpDocumentEngine
     return true;
   }
 
+  protected function canRenderPartialDocument(PhabricatorDocumentRef $ref) {
+    return true;
+  }
+
   protected function newDocumentContent(PhabricatorDocumentRef $ref) {
-    $content = $ref->loadData();
+    $limit = $this->getByteLengthLimit();
+    $length = $ref->getByteLength();
+
+    $is_partial = false;
+    if ($limit) {
+      if ($length > $limit) {
+        $is_partial = true;
+        $length = $limit;
+      }
+    }
+
+    $content = $ref->loadData(null, $length);
 
     $output = array();
     $offset = 0;
@@ -48,7 +67,19 @@ final class PhabricatorHexdumpDocumentEngine
       ),
       $output);
 
-    return $container;
+    $message = null;
+    if ($is_partial) {
+      $message = $this->newMessage(
+        pht(
+          'This document is too large to be completely rendered inline. The '.
+          'first %s bytes are shown.',
+          new PhutilNumber($limit)));
+    }
+
+    return array(
+      $message,
+      $container,
+    );
   }
 
   private function renderHex($bytes) {

src/applications/files/document/PhabricatorImageDocumentEngine.php

@@ -13,6 +13,10 @@ final class PhabricatorImageDocumentEngine
     return 'fa-file-image-o';
   }
 
+  protected function getByteLengthLimit() {
+    return (1024 * 1024 * 64);
+  }
+
   protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
     $file = $ref->getFile();
     if ($file) {

src/applications/files/document/PhabricatorVideoDocumentEngine.php

@@ -9,6 +9,16 @@ final class PhabricatorVideoDocumentEngine
     return pht('View as Video');
   }
 
+  protected function getContentScore(PhabricatorDocumentRef $ref) {
+    // Some video documents can be rendered as either video or audio, but we
+    // want to prefer video.
+    return 2500;
+  }
+
+  protected function getByteLengthLimit() {
+    return null;
+  }
+
   protected function getDocumentIconIcon(PhabricatorDocumentRef $ref) {
     return 'fa-film';
   }

src/applications/files/document/PhabricatorVoidDocumentEngine.php

@@ -13,10 +13,14 @@ final class PhabricatorVoidDocumentEngine
     return 'fa-file';
   }
 
-  protected function getContentScore() {
+  protected function getContentScore(PhabricatorDocumentRef $ref) {
     return 1000;
   }
 
+  protected function getByteLengthLimit() {
+    return null;
+  }
+
   protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
     return true;
   }

webroot/rsrc/css/phui/phui-property-list-view.css

@@ -233,9 +233,11 @@ div.phui-property-list-stacked .phui-property-list-properties
 }
 
 .document-engine-error {
-  margin: 20px auto;
+  margin: 20px;
+  padding: 12px;
   text-align: center;
   color: {$redtext};
+  background: {$sh-redbackground};
 }
 
 .document-engine-hexdump {

webroot/rsrc/js/application/files/behavior-document-engine.js

@@ -27,6 +27,7 @@ JX.behavior('document-engine', function() {
       view = new JX.PHUIXActionView()
         .setName(spec.name)
         .setIcon(spec.icon)
+        .setIconColor(spec.color)
         .setHref(spec.engineURI);
 
       view.setHandler(JX.bind(null, function(spec, e) {

webroot/rsrc/js/phuix/PHUIXActionView.js

@@ -12,6 +12,7 @@ JX.install('PHUIXActionView', {
     _node: null,
     _name: null,
     _icon: 'none',
+    _iconColor: null,
     _disabled: false,
     _label: false,
     _handler: null,
@@ -79,6 +80,12 @@ JX.install('PHUIXActionView', {
       return this;
     },
 
+    setIconColor: function(color) {
+      this._iconColor = color;
+      this._buildIconNode(true);
+      return this;
+    },
+
     setHref: function(href) {
       this._href = href;
       this._buildNameNode(true);
@@ -129,6 +136,10 @@ JX.install('PHUIXActionView', {
           icon_class = icon_class + ' grey';
         }
 
+        if (this._iconColor) {
+          icon_class = icon_class + ' ' + this._iconColor;
+        }
+
         JX.DOM.alterClass(node, icon_class, true);
 
         if (this._iconNode && this._iconNode.parentNode) {