Rewrite file documentation to be chunk-aware

Summary:
Ref T7149. We can simplify configuration somewhat by removing the upload limit setting, now that we support arbitrarily large files.

  - Merge configuration documentation.
  - Tell users to set things to at least 32MB. This is 8MB maximum one-shot file + 4x headroom. Chunk sizes are 4MB.

Test Plan:
  - Faked all the setup warnings.
  - Read documentation.
  - Uploaded some files.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12083

src/applications/config/check/PhabricatorExtraConfigSetupCheck.php

@@ -214,6 +214,9 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
       'storage.engine-selector' => pht(
         'Phabricator now automatically discovers available storage engines '.
         'at runtime.'),
+      'storage.upload-size-limit' => pht(
+        'Phabricator now supports arbitrarily large files. Consult the '.
+        'documentation for configuration details.'),
     );
 
     return $ancient_config;

src/applications/config/check/PhabricatorMySQLSetupCheck.php

@@ -21,10 +21,13 @@ final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
 
   protected function executeChecks() {
     $max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
-    $recommended_minimum = 1024 * 1024;
+
+    // This primarily supports setting the filesize limit for MySQL to 8MB,
+    // which may produce a >16MB packet after escaping.
+    $recommended_minimum = (32 * 1024 * 1024);
     if ($max_allowed_packet < $recommended_minimum) {
       $message = pht(
-        "MySQL is configured with a very small 'max_allowed_packet' (%d), ".
+        "MySQL is configured with a small 'max_allowed_packet' (%d), ".
         "which may cause some large writes to fail. Strongly consider raising ".
         "this to at least %d in your MySQL configuration.",
         $max_allowed_packet,

src/applications/config/check/PhabricatorStorageSetupCheck.php

@@ -10,73 +10,116 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
    * @phutil-external-symbol class PhabricatorStartup
    */
   protected function executeChecks() {
-    $upload_limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
+    $chunk_engine_active = false;
-    if (!$upload_limit) {
+
+    $engines = PhabricatorFileStorageEngine::loadWritableEngines();
+    foreach ($engines as $engine) {
+      if ($engine->isChunkEngine()) {
+        $chunk_engine_active = true;
+        break;
+      }
+    }
+
+    if (!$chunk_engine_active) {
+      $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage');
+
       $message = pht(
-        'The Phabricator file upload limit is not configured. You may only '.
+        'Large file storage has not been configured, which will limit '.
-        'be able to upload very small files until you configure it, because '.
+        'the maximum size of file uploads. See %s for '.
-        'some PHP default limits are very low (as low as 2MB).');
+        'instructions on configuring uploads and storage.',
+        phutil_tag(
+          'a',
+          array(
+            'href' => $doc_href,
+            'target' => '_blank',
+          ),
+          pht('Configuring File Storage')));
 
       $this
-        ->newIssue('config.storage.upload-size-limit')
+        ->newIssue('large-files')
-        ->setShortName(pht('Upload Limit'))
+        ->setShortName(pht('Large Files'))
-        ->setName(pht('Upload Limit Not Yet Configured'))
+        ->setName(pht('Large File Storage Not Configured'))
+        ->setMessage($message);
+    }
+
+    $post_max_size = ini_get('post_max_size');
+    if ($post_max_size && ((int)$post_max_size > 0)) {
+      $post_max_bytes = phutil_parse_bytes($post_max_size);
+      $post_max_need = (32 * 1024 * 1024) * 100;
+      if ($post_max_need > $post_max_bytes) {
+        $summary = pht(
+          'Set %s in your PHP configuration to at least 32MB '.
+          'to support large file uploads.',
+          phutil_tag('tt', array(), 'post_max_size'));
+
+        $message = pht(
+          'Adjust %s in your PHP configuration to at least 32MB. When '.
+          'set to smaller value, large file uploads may not work properly.',
+          phutil_tag('tt', array(), 'post_max_size'));
+
+        $this
+          ->newIssue('php.post_max_size')
+          ->setName(pht('PHP post_max_size Not Configured'))
+          ->setSummary($summary)
           ->setMessage($message)
-        ->addPhabricatorConfig('storage.upload-size-limit');
+          ->setGroup(self::GROUP_PHP)
-    } else {
+          ->addPHPConfig('post_max_size');
+      }
+    }
+
+    // This is somewhat arbitrary, but make sure we have enough headroom to
+    // upload a default file at the chunk threshold (8MB), which may be
+    // base64 encoded, then JSON encoded in the request, and may need to be
+    // held in memory in the raw and as a query string.
+    $need_bytes = (64 * 1024 * 1024);
+
     $memory_limit = PhabricatorStartup::getOldMemoryLimit();
     if ($memory_limit && ((int)$memory_limit > 0)) {
       $memory_limit_bytes = phutil_parse_bytes($memory_limit);
       $memory_usage_bytes = memory_get_usage();
-        $upload_limit_bytes = phutil_parse_bytes($upload_limit);
 
       $available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
 
-        if ($upload_limit_bytes > $available_bytes) {
+      if ($need_bytes > $available_bytes) {
         $summary = pht(
           'Your PHP memory limit is configured in a way that may prevent '.
-            'you from uploading large files.');
+          'you from uploading large files or handling large requests.');
 
         $message = pht(
-            'When you upload a file via drag-and-drop or the API, the entire '.
+          'When you upload a file via drag-and-drop or the API, chunks must '.
-            'file is buffered into memory before being written to permanent '.
+          'be buffered into memory before being written to permanent '.
           'storage. Phabricator needs memory available to store these '.
-            'files while they are uploaded, but PHP is currently configured '.
+          'chunks while they are uploaded, but PHP is currently configured '.
-            'to limit the available memory.'.
+          'to severly limit the available memory.'.
           "\n\n".
-            'Your Phabricator %s is currently set to a larger value (%s) than '.
+          'PHP processes currently have very little free memory available '.
-            'the amount of available memory (%s) that a PHP process has '.
+          '(%s). To work well, processes should have at least %s.'.
-            'available to use, so uploads via drag-and-drop and the API will '.
-            'hit the memory limit before they hit other limits.'.
           "\n\n".
           '(Note that the application itself must also fit in available '.
           'memory, so not all of the memory under the memory limit is '.
-            'available for buffering file uploads.)'.
+          'available for running workloads.)'.
           "\n\n".
           "The easiest way to resolve this issue is to set %s to %s in your ".
           "PHP configuration, to disable the memory limit. There is ".
           "usually little or no value to using this option to limit ".
           "Phabricator process memory.".
           "\n\n".
-            "You can also increase the limit, or decrease %s, or ignore this ".
+          "You can also increase the limit or ignore this issue and accept ".
-            "issue and accept that these upload mechanisms will be limited ".
+          "that you may encounter problems uploading large files and ".
-            "in the size of files they can handle.",
+          "processing large requests.",
-            phutil_tag('tt', array(), 'storage.upload-size-limit'),
-            phutil_format_bytes($upload_limit_bytes),
           phutil_format_bytes($available_bytes),
+          phutil_format_bytes($need_bytes),
           phutil_tag('tt', array(), 'memory_limit'),
-            phutil_tag('tt', array(), '-1'),
+          phutil_tag('tt', array(), '-1'));
-            phutil_tag('tt', array(), 'storage.upload-size-limit'));
 
         $this
           ->newIssue('php.memory_limit.upload')
           ->setName(pht('Memory Limit Restricts File Uploads'))
           ->setSummary($summary)
           ->setMessage($message)
+          ->setGroup(self::GROUP_PHP)
           ->addPHPConfig('memory_limit')
-            ->addPHPConfigOriginalValue('memory_limit', $memory_limit)
+          ->addPHPConfigOriginalValue('memory_limit', $memory_limit);
-            ->addPhabricatorConfig('storage.upload-size-limit');
-        }
       }
     }
 

src/applications/files/config/PhabricatorFilesConfigOptions.php

@@ -147,26 +147,6 @@ final class PhabricatorFilesConfigOptions
             "Set this to a valid Amazon S3 bucket to store files there. You ".
             "must also configure S3 access keys in the 'Amazon Web Services' ".
             "group.")),
-     $this->newOption('storage.upload-size-limit', 'string', null)
-        ->setSummary(
-          pht('Limit to users in interfaces which allow uploading.'))
-        ->setDescription(
-          pht(
-            "Set the size of the largest file a user may upload. This is ".
-            "used to render text like 'Maximum file size: 10MB' on ".
-            "interfaces where users can upload files, and files larger than ".
-            "this size will be rejected. \n\n".
-            "NOTE: **Setting this to a large size is NOT sufficient to ".
-            "allow users to upload large files. You must also configure a ".
-            "number of other settings.** To configure file upload limits, ".
-            "consult the article 'Configuring File Upload Limits' in the ".
-            "documentation. Once you've configured some limit across all ".
-            "levels of the server, you can set this limit to an appropriate ".
-            "value and the UI will then reflect the actual configured ".
-            "limit.\n\n".
-            "Specify this limit in bytes, or using a 'K', 'M', or 'G' ".
-            "suffix."))
-        ->addExample('10M', pht('Allow Uploads 10MB or Smaller')),
      $this->newOption(
         'metamta.files.public-create-email',
         'string',

src/applications/files/controller/PhabricatorFileUploadController.php

@@ -57,8 +57,7 @@ final class PhabricatorFileUploadController extends PhabricatorFileController {
         id(new AphrontFormFileControl())
           ->setLabel(pht('File'))
           ->setName('file')
-          ->setError($e_file)
+          ->setError($e_file))
-          ->setCaption($this->renderUploadLimit()))
       ->appendChild(
         id(new AphrontFormTextControl())
           ->setLabel(pht('Name'))
@@ -102,25 +101,4 @@ final class PhabricatorFileUploadController extends PhabricatorFileController {
       ));
   }
 
-  private function renderUploadLimit() {
-    $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
-    $limit = phutil_parse_bytes($limit);
-    if ($limit) {
-      $formatted = phutil_format_bytes($limit);
-      return 'Maximum file size: '.$formatted;
-    }
-
-    $doc_href = PhabricatorEnv::getDocLink(
-      'Configuring File Upload Limits');
-    $doc_link = phutil_tag(
-      'a',
-      array(
-        'href'    => $doc_href,
-        'target'  => '_blank',
-      ),
-      'Configuring File Upload Limits');
-
-    return hsprintf('Upload limit is not configured, see %s.', $doc_link);
-  }
-
 }

src/applications/files/exception/PhabricatorFileUploadException.php

@@ -20,11 +20,6 @@ final class PhabricatorFileUploadException extends Exception {
         'Unable to write file: failed to write to temporary directory.',
       UPLOAD_ERR_EXTENSION =>
         'Unable to upload: a PHP extension stopped the upload.',
-
-      -1000 =>
-        pht("Uploaded file is too large: current limit is %s. To adjust this ".
-          "limit change 'storage.upload-size-limit' in the Phabricator config.",
-          PhabricatorEnv::getEnvConfig('storage.upload-size-limit')),
     );
 
     $message = idx($map, $code, 'Upload failed: unknown error.');

src/applications/files/storage/PhabricatorFile.php

@@ -162,8 +162,6 @@ final class PhabricatorFile extends PhabricatorFileDAO
       throw new Exception('File size disagrees with uploaded size.');
     }
 
-    self::validateFileSize(strlen($file_data));
-
     return $file_data;
   }
 
@@ -181,22 +179,9 @@ final class PhabricatorFile extends PhabricatorFileDAO
   }
 
   public static function newFromXHRUpload($data, array $params = array()) {
-    self::validateFileSize(strlen($data));
     return self::newFromFileData($data, $params);
   }
 
-  private static function validateFileSize($size) {
-    $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
-    if (!$limit) {
-      return;
-    }
-
-    $limit = phutil_parse_bytes($limit);
-    if ($size > $limit) {
-      throw new PhabricatorFileUploadException(-1000);
-    }
-  }
-
 
   /**
    * Given a block of data, try to load an existing file with the same content

src/docs/user/configuration/configuring_file_storage.diviner

@@ -1,45 +1,134 @@
 @title Configuring File Storage
 @group config
 
-Setup how Phabricator will store files.
+Setup file storage and support for large files.
 
 Overview
 ========
 
-Phabricator allows users to upload files, and several applications use file
+This document describes how to configure Phabricator to support large file
-storage (for instance, Maniphest allows you to attach files to tasks). You can
+uploads, and how to choose where Phabricator stores files.
-configure several different storage systems.
 
-| System | Setup | Cost | Notes |
+There are two major things to configure:
+
+  - set up PHP and your HTTP server to accept large requests;
+  - choose and configure a storage engine.
+
+The following sections will guide you through this configuration.
+
+
+How Phabricator Stores Files
+============================
+
+Phabricator stores files in "storage engines", which are modular backends
+that implement access to some storage system (like MySQL, the filesystem, or
+a cloud storage service like Amazon S3).
+
+Phabricator stores large files by breaking them up into many chunks (a few
+megabytes in size) and storing the chunks in an underlying storage engine.
+This makes it easier to implement new storage engines and gives Phabricator
+more flexibility in managing file data.
+
+The first section of this document discusses configuring your install so that
+PHP and your HTTP server will accept requests which are larger than the size of
+one file chunk. Without this configuration, file chunk data will be rejected.
+
+The second section discusses choosing and configuring storage engines, so data
+is stored where you want it to be.
+
+
+Configuring Upload Limits
+=========================
+
+File uploads are limited by several pieces of configuration at different layers
+of the stack. Generally, the minimum value of all the limits is the effective
+one.
+
+To upload large files, you need to increase all the limits to at least
+**32MB**. This will allow you to upload file chunks, which will let Phabricator
+store arbitrarily large files.
+
+The settings which limit file uploads are:
+
+**HTTP Server**: The HTTP server may set a limit on the maximum request size.
+If you exceed this limit, you'll see a default server page with an HTTP error.
+These directives limit the total size of the request body, so they must be
+somewhat larger than the desired maximum filesize.
+
+  - **Apache**: Apache limits requests with the Apache `LimitRequestBody`
+    directive.
+  - **nginx**: nginx limits requests with the nginx `client_max_body_size`
+    directive. This often defaults to `1M`.
+  - **lighttpd**: lighttpd limits requests with the lighttpd
+    `server.max-request-size` directive.
+
+Set the applicable limit to at least **32MB**. Phabricator can not read these
+settings, so it can not raise setup warnings if they are misconfigured.
+
+**PHP**: PHP has several directives which limit uploads. These directives are
+found in `php.ini`.
+
+  - **post_max_size**: Maximum POST request size PHP will accept. If you
+    exceed this, Phabricator will give you a useful error. This often defaults
+    to `8M`. Set this to at least `32MB`. Phabricator will give you a setup
+    warning about this if it is set too low.
+  - **memory_limit**: For some uploads, file data will be read into memory
+    before Phabricator can adjust the memory limit. If you exceed this, PHP
+    may give you a useful error, depending on your configuration. It is
+    recommended that you set this to `-1` to disable it. Phabricator will
+    give you a setup warning about this if it is set too low.
+
+You may also want to configure these PHP options:
+
+  - **max_input_vars**: When files are uploaded via HTML5 drag and drop file
+    upload APIs, PHP parses the file body as though it contained normal POST
+    parameters, and may trigger `max_input_vars` if a file has a lot of
+    brackets in it. You may need to set it to some astronomically high value.
+  - **upload_max_filesize**: Maximum file size PHP will accept in a raw file
+    upload. This is not normally used when uploading files via drag-and-drop,
+    but affects some other kinds of file uploads. If you exceed this,
+    Phabricator will give you a useful error. This often defaults to `2M`. Set
+    this to at least `32MB`.
+
+Once you've adjusted all this configuration, your server will be able to
+receive chunk uploads. As long as you have somewhere to store them, this will
+enable you to store arbitrarily large files.
+
+
+Storage Engines
+===============
+
+Phabricator supports several different file storage engines:
+
+| Engine | Setup | Cost | Notes |
 |========|=======|======|=======|
 | MySQL  | Automatic | Free | May not scale well. |
 | Local Disk | Easy | Free | Does not scale well. |
 | Amazon S3 | Easy | Cheap | Scales well. |
 | Custom | Hard | Varies | Implement a custom storage engine. |
 
+You can review available storage engines and their configuration by navigating
+to {nav Applications > Files > Help/Options > Storage Engines} in the web UI.
+
 By default, Phabricator is configured to store files up to 1MB in MySQL, and
 reject files larger than 1MB. To store larger files, you can either:
 
-  - configure local disk storage; or
+  - increase the MySQL limit to at least 8MB; or
-  - configure Amazon S3 storage; or
+  - configure another storage engine.
-  - raise the limits on MySQL.
 
-See the rest of this document for some additional discussion of engines.
+Doing either of these will enable the chunk storage engine and support for
+arbitrarily large files.
 
-You don't have to fully configure this immediately, the defaults are okay until
+The remaining sections of this document discuss the available storage engines
-you need to upload larger files and it's relatively easy to port files between
+and how to configure them.
-storage engines later.
 
-Storage Engines
-===============
 
-Builtin storage engines and information on how to configure them.
+Engine: MySQL
+=============
 
-== MySQL ==
+  - **Pros**: Low latency, no setup required.
-
+  - **Cons**: Storing files in a database is a classic bad idea. May become
-  - **Pros**: Fast, no setup required.
+    difficult to administrate if you have a large amount of data.
-  - **Cons**: Storing files in a database is a classic bad idea. Does not scale
-    well. Maximum file size is limited.
 
 MySQL storage is configured by default, for files up to (just under) 1MB. You
 can configure it with these keys:
@@ -49,37 +138,43 @@ can configure it with these keys:
 
 For most installs, it is reasonable to leave this engine as-is and let small
 files (like thumbnails and profile images) be stored in MySQL, which is usually
-the lowest-latency filestore.
+the lowest-latency filestore, even if you configure another storage engine.
 
-To support larger files, configure another engine or increase this limit.
+To support large files, increase this limit to at least **8MB**. This will
+activate chunk storage in MySQL.
 
-== Local Disk ==
+Engine: Local Disk
+==================
 
-  - **Pros**: Very simple. Almost no setup required.
+  - **Pros**: Simple to setup.
   - **Cons**: Doesn't scale to multiple web frontends without NFS.
 
-To upload larger files:
+To configure file storage on the local disk, set:
 
   - `storage.local-disk.path`: Set to some writable directory on local disk.
     Make that directory.
 
-== Amazon S3 ==
+Engine: Amazon S3
+=================
 
   - **Pros**: Scales well.
-  - **Cons**: More complicated and expensive than other approaches.
+  - **Cons**: Slightly more complicated than other engines, not free.
 
-To enable file storage in S3, set these key:
+To enable file storage in S3, set these keys:
 
-  - ##amazon-s3.access-key## Your AWS access key.
+  - `amazon-s3.access-key`: Your AWS access key.
-  - ##amazon-s3.secret-key## Your AWS secret key.
+  - `amazon-s3.secret-key`: Your AWS secret key.
-  - ##storage.s3.bucket## S3 bucket name where files should be stored.
+  - `storage.s3.bucket`: S3 bucket name where files should be stored.
 
-= Testing Storage Engines =
+Testing Storage Engines
+=======================
 
-You can test that things are correctly configured by going to the Files
+You can test that things are correctly configured by dragging and dropping
-application (##/file/##) and uploading files.
+a file onto the Phabricator home page. If engines have been configured
+properly, the file should upload.
 
-= Migrating Files Between Engines =
+Migrating Files Between Engines
+===============================
 
 If you want to move files between storage engines, you can use the `bin/files`
 script to perform migrations. For example, suppose you previously used MySQL but
@@ -95,10 +190,9 @@ If that works properly, you can then migrate everything:
 You can use `--dry-run` to show which migrations would be performed without
 taking any action. Run `bin/files help` for more options and information.
 
-= Next Steps =
+Next Steps
+==========
 
 Continue by:
 
-  - configuring file size upload limits with
-    @{article:Configuring File Upload Limits}; or
   - returning to the @{article:Configuration Guide}.

src/docs/user/configuration/configuring_file_upload_limits.diviner

@@ -1,77 +0,0 @@
-@title Configuring File Upload Limits
-@group config
-
-Explains limits on file upload sizes.
-
-= Overview =
-
-File uploads are limited by a large number of pieces of configuration, at
-multiple layers of the application. Generally, the minimum value of all the
-limits is the effective one. To upload large files, you need to increase all
-the limits above the maximum file size you want to support. The settings which
-limit uploads are:
-
-  - **HTTP Server**: The HTTP server may set a limit on the maximum request
-    size. If you exceed this limit, you'll see a default server page with an
-    HTTP error. These directives limit the total size of the request body,
-    so they must be somewhat larger than the desired maximum filesize.
-    - **Apache**: Apache limits requests with the Apache `LimitRequestBody`
-      directive.
-    - **nginx**: nginx limits requests with the nginx `client_max_body_size`
-      directive. This often defaults to `1M`.
-    - **lighttpd**: lighttpd limits requests with the lighttpd
-      `server.max-request-size` directive.
-  - **PHP**: PHP has several directives which limit uploads. These directives
-    are found in `php.ini`.
-    - **upload_max_filesize**: Maximum file size PHP will accept in a file
-      upload. If you exceed this, Phabricator will give you a useful error. This
-      often defaults to `2M`.
-    - **post_max_size**: Maximum POST request size PHP will accept. If you
-      exceed this, Phabricator will give you a useful error. This often defaults
-      to `8M`.
-    - **memory_limit**: For some uploads, file data will be read into memory
-      before Phabricator can adjust the memory limit. If you exceed this, PHP
-      may give you a useful error, depending on your configuration.
-    - **max_input_vars**: When files are uploaded via HTML5 drag and drop file
-      upload APIs, PHP parses the file body as though it contained normal POST
-      parameters, and may trigger `max_input_vars` if a file has a lot of
-      brackets in it. You may need to set it to some astronomically high value.
-  - **Storage Engines**: Some storage engines can be configured not to accept
-    files over a certain size. To upload a file, you must have at least one
-    configured storage engine which can accept it. Phabricator should give you
-    useful errors if any of these fail.
-    - **MySQL Engine**: Upload size is limited by the Phabricator setting
-      `storage.mysql-engine.max-size`.
-    - **Amazon S3**: Upload size is limited by Phabricator's implementation to
-      `5G`.
-    - **Local Disk**: Upload size is limited only by free disk space.
-  - **Resource Constraints**: File uploads are limited by resource constraints
-    on the application server. In particular, some uploaded files are written
-    to disk in their entirety before being moved to storage engines, and all
-    uploaded files are read into memory before being moved. These hard limits
-    should be large for most servers, but will fundamentally prevent Phabricator
-    from processing truly enormous files (GB/TB scale). Phabricator is probably
-    not the best application for this in any case.
-  - **Phabricator Master Limit**: The master limit, `storage.upload-size-limit`,
-    is used to show upload limits in the UI.
-
-Phabricator can't read some of these settings, so it can't figure out what the
-current limit is or be much help at all in configuring it. Thus, you need to
-manually configure all of these limits and then tell Phabricator what you set
-them to. Follow these steps:
-
-  - Pick some limit you want to set, like `100M`.
-  - Configure all of the settings mentioned above to be a bit bigger than the
-    limit you want to enforce (**note that there are some security implications
-    to raising these limits**; principally, your server may become easier to
-    attack with a denial-of-service).
-  - Set `storage.upload-size-limit` to the limit you want.
-  - The UI should now show your limit.
-  - Upload a big file to make sure it works.
-
-= Next Steps =
-
-Continue by:
-
-  - configuring file storage with @{article:Configuring File Storage}; or
-  - returning to the @{article:Configuration Guide}.