Enable the chunk storage engine

Summary: Ref T7149. This works now, so enable it.

Test Plan:
  - Uploaded large and small files in Firefox, Safari and Chrome.
  - Uploaded large files with `arc upload`.
  - Stopped/resumed large files with all clients.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12079

resources/celerity/map.php

@@ -11,7 +11,7 @@ return array(
     'core.pkg.js' => '69f7e542',
     'darkconsole.pkg.js' => '8ab24e01',
     'differential.pkg.css' => '1940be3f',
-    'differential.pkg.js' => 'be1e5f9b',
+    'differential.pkg.js' => '85fd84c6',
     'diffusion.pkg.css' => '591664fa',
     'diffusion.pkg.js' => 'bfc0737b',
     'maniphest.pkg.css' => '68d4dd3d',
@@ -437,7 +437,7 @@ return array(
     'rsrc/js/application/uiexample/gesture-example.js' => '558829c2',
     'rsrc/js/application/uiexample/notification-example.js' => '8ce821c5',
     'rsrc/js/core/Busy.js' => '6453c869',
-    'rsrc/js/core/DragAndDropFileUpload.js' => 'fd6ace61',
+    'rsrc/js/core/DragAndDropFileUpload.js' => '7fa4b248',
     'rsrc/js/core/DraggableList.js' => 'a16ec1c6',
     'rsrc/js/core/FileUpload.js' => '477359c8',
     'rsrc/js/core/Hovercard.js' => '7e8468ae',
@@ -717,7 +717,7 @@ return array(
     'phabricator-core-css' => '86bfbe8c',
     'phabricator-countdown-css' => '86b7b0a0',
     'phabricator-dashboard-css' => '17937d22',
-    'phabricator-drag-and-drop-file-upload' => 'fd6ace61',
+    'phabricator-drag-and-drop-file-upload' => '7fa4b248',
     'phabricator-draggable-list' => 'a16ec1c6',
     'phabricator-fatal-config-template-css' => '8e6c6fcd',
     'phabricator-feed-css' => 'b513b5f4',
@@ -1423,6 +1423,14 @@ return array(
       'javelin-behavior',
       'javelin-history',
     ),
+    '7fa4b248' => array(
+      'javelin-install',
+      'javelin-util',
+      'javelin-request',
+      'javelin-dom',
+      'javelin-uri',
+      'phabricator-file-upload',
+    ),
     82439934 => array(
       'javelin-behavior',
       'javelin-dom',
@@ -1992,14 +2000,6 @@ return array(
       'javelin-dom',
       'phortune-credit-card-form',
     ),
-    'fd6ace61' => array(
-      'javelin-install',
-      'javelin-util',
-      'javelin-request',
-      'javelin-dom',
-      'javelin-uri',
-      'phabricator-file-upload',
-    ),
     'fe287620' => array(
       'javelin-install',
       'javelin-dom',

src/applications/files/controller/PhabricatorFileDropUploadController.php

@@ -61,7 +61,6 @@ final class PhabricatorFileDropUploadController
     // vanilla upload, so we need it.
     $data = PhabricatorStartup::getRawInput();
 
-
     $is_chunk_upload = $request->getBool('uploadchunk');
     if ($is_chunk_upload) {
       $params = array(

src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php

@@ -14,8 +14,6 @@ final class PhabricatorChunkedFileStorageEngine
   /**
    * We can write chunks if we have at least one valid storage engine
    * underneath us.
-   *
-   * This engine must not also be a chunk engine.
    */
   public function canWriteFiles() {
     return (bool)$this->getWritableEngine();
@@ -29,11 +27,6 @@ final class PhabricatorChunkedFileStorageEngine
     return true;
   }
 
-  public function isTestEngine() {
-    // TODO: For now, prevent this from actually being selected.
-    return true;
-  }
-
   public function writeFile($data, array $params) {
     // The chunk engine does not support direct writes.
     throw new PhutilMethodNotImplementedException();
@@ -142,6 +135,12 @@ final class PhabricatorChunkedFileStorageEngine
     return $file;
   }
 
+  /**
+   * Find a storage engine which is suitable for storing chunks.
+   *
+   * This engine must be a writable engine, have a filesize limit larger than
+   * the chunk limit, and must not be a chunk engine itself.
+   */
   private function getWritableEngine() {
     // NOTE: We can't just load writable engines or we'll loop forever.
     $engines = PhabricatorFileStorageEngine::loadAllEngines();
@@ -172,9 +171,7 @@ final class PhabricatorChunkedFileStorageEngine
   }
 
   public function getChunkSize() {
-    // TODO: This is an artificially small size to make it easier to
-    // test chunking.
-    return 32;
+    return (4 * 1024 * 1024);
   }
 
   public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {

src/applications/files/engine/PhabricatorFileStorageEngine.php

@@ -82,7 +82,9 @@ abstract class PhabricatorFileStorageEngine {
    * @return bool `true` if the engine has a filesize limit.
    * @task meta
    */
-  abstract public function hasFilesizeLimit();
+  public function hasFilesizeLimit() {
+    return true;
+  }
 
 
   /**
@@ -92,11 +94,19 @@ abstract class PhabricatorFileStorageEngine {
    * an engine has a limit. Engines without a limit can store files of any
    * size.
    *
+   * By default, engines define a limit which supports chunked storage of
+   * large files. In most cases, you should not change this limit, even if an
+   * engine has vast storage capacity: chunked storage makes large files more
+   * manageable and enables features like resumable uploads.
+   *
    * @return int Maximum storable file size, in bytes.
    * @task meta
    */
   public function getFilesizeLimit() {
-    throw new PhutilMethodNotImplementedException();
+    // NOTE: This 8MB limit is selected to be larger than the 4MB chunk size,
+    // but not much larger. Files between 0MB and 8MB will be stored normally;
+    // files larger than 8MB will be chunked.
+    return (1024 * 1024 * 8);
   }
 
 

src/applications/files/engine/PhabricatorLocalDiskFileStorageEngine.php

@@ -30,11 +30,6 @@ final class PhabricatorLocalDiskFileStorageEngine
   }
 
 
-  public function hasFilesizeLimit() {
-    return false;
-  }
-
-
 /* -(  Managing File Data  )------------------------------------------------- */
 
 

src/applications/files/engine/PhabricatorS3FileStorageEngine.php

@@ -33,11 +33,6 @@ final class PhabricatorS3FileStorageEngine
   }
 
 
-  public function hasFilesizeLimit() {
-    return false;
-  }
-
-
 /* -(  Managing File Data  )------------------------------------------------- */
 
 

src/applications/files/query/PhabricatorFileSearchEngine.php

@@ -176,6 +176,10 @@ final class PhabricatorFileSearchEngine
         $item->addIcon('blame', pht('Temporary'));
       }
 
+      if ($file->getIsPartial()) {
+        $item->addIcon('fa-exclamation-triangle orange', pht('Partial'));
+      }
+
       if (isset($highlighted_ids[$id])) {
         $item->setEffect('highlighted');
       }

webroot/rsrc/js/core/DragAndDropFileUpload.js

@@ -264,31 +264,13 @@ JX.install('PhabricatorDragAndDropFileUpload', {
       for (var ii = 0; ii < chunks.length; ii++) {
         chunk = chunks[ii];
         if (!chunk.complete) {
-          this._readChunk(
-            file,
-            chunk,
-            JX.bind(this, this._didReadChunk, file, chunk));
+          this._uploadChunk(file, chunk);
           break;
         }
       }
     },
 
-    _readChunk: function(file, chunk, callback) {
-      var reader = new FileReader();
-      var blob = file.getRawFileObject().slice(chunk.byteStart, chunk.byteEnd);
-
-      reader.onload = function() {
-        callback(reader.result);
-      };
-
-      reader.onerror = function() {
-        this._failUpload(file, {error: reader.error.message});
-      };
-
-      reader.readAsBinaryString(blob);
-    },
-
-    _didReadChunk: function(file, chunk, data) {
+    _uploadChunk: function(file, chunk, callback) {
       file
         .setStatus('upload')
         .update();
@@ -316,8 +298,10 @@ JX.install('PhabricatorDragAndDropFileUpload', {
       req.listen('error', JX.bind(this, this._onUploadError, req, file));
       req.listen('uploadprogress', onprogress);
 
+      var blob = file.getRawFileObject().slice(chunk.byteStart, chunk.byteEnd);
+
       req
-        .setRawData(data)
+        .setRawData(blob)
         .send();
     },
 
@@ -390,7 +374,7 @@ JX.install('PhabricatorDragAndDropFileUpload', {
       this.invoke('didError', file);
     },
 
-    _onUploadError: function(file, req, error) {
+    _onUploadError: function(req, file, error) {
       file.setStatus('error');
 
       if (error) {