mirror of
https://we.phorge.it/source/phorge.git
synced 2024-12-24 06:20:56 +01:00
Rewrite file documentation to be chunk-aware
Summary: Ref T7149. We can simplify configuration somewhat by removing the upload limit setting, now that we support arbitrarily large files. - Merge configuration documentation. - Tell users to set things to at least 32MB. This is 8MB maximum one-shot file + 4x headroom. Chunk sizes are 4MB. Test Plan: - Faked all the setup warnings. - Read documentation. - Uploaded some files. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T7149 Differential Revision: https://secure.phabricator.com/D12083
This commit is contained in:
parent
21aa086b69
commit
7482d260b0
9 changed files with 244 additions and 240 deletions
|
@ -214,6 +214,9 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
|
||||||
'storage.engine-selector' => pht(
|
'storage.engine-selector' => pht(
|
||||||
'Phabricator now automatically discovers available storage engines '.
|
'Phabricator now automatically discovers available storage engines '.
|
||||||
'at runtime.'),
|
'at runtime.'),
|
||||||
|
'storage.upload-size-limit' => pht(
|
||||||
|
'Phabricator now supports arbitrarily large files. Consult the '.
|
||||||
|
'documentation for configuration details.'),
|
||||||
);
|
);
|
||||||
|
|
||||||
return $ancient_config;
|
return $ancient_config;
|
||||||
|
|
|
@ -21,10 +21,13 @@ final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
|
||||||
|
|
||||||
protected function executeChecks() {
|
protected function executeChecks() {
|
||||||
$max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
|
$max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
|
||||||
$recommended_minimum = 1024 * 1024;
|
|
||||||
|
// This primarily supports setting the filesize limit for MySQL to 8MB,
|
||||||
|
// which may produce a >16MB packet after escaping.
|
||||||
|
$recommended_minimum = (32 * 1024 * 1024);
|
||||||
if ($max_allowed_packet < $recommended_minimum) {
|
if ($max_allowed_packet < $recommended_minimum) {
|
||||||
$message = pht(
|
$message = pht(
|
||||||
"MySQL is configured with a very small 'max_allowed_packet' (%d), ".
|
"MySQL is configured with a small 'max_allowed_packet' (%d), ".
|
||||||
"which may cause some large writes to fail. Strongly consider raising ".
|
"which may cause some large writes to fail. Strongly consider raising ".
|
||||||
"this to at least %d in your MySQL configuration.",
|
"this to at least %d in your MySQL configuration.",
|
||||||
$max_allowed_packet,
|
$max_allowed_packet,
|
||||||
|
|
|
@ -10,73 +10,116 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
|
||||||
* @phutil-external-symbol class PhabricatorStartup
|
* @phutil-external-symbol class PhabricatorStartup
|
||||||
*/
|
*/
|
||||||
protected function executeChecks() {
|
protected function executeChecks() {
|
||||||
$upload_limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
|
$chunk_engine_active = false;
|
||||||
if (!$upload_limit) {
|
|
||||||
|
$engines = PhabricatorFileStorageEngine::loadWritableEngines();
|
||||||
|
foreach ($engines as $engine) {
|
||||||
|
if ($engine->isChunkEngine()) {
|
||||||
|
$chunk_engine_active = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$chunk_engine_active) {
|
||||||
|
$doc_href = PhabricatorEnv::getDocLink('Configuring File Storage');
|
||||||
|
|
||||||
$message = pht(
|
$message = pht(
|
||||||
'The Phabricator file upload limit is not configured. You may only '.
|
'Large file storage has not been configured, which will limit '.
|
||||||
'be able to upload very small files until you configure it, because '.
|
'the maximum size of file uploads. See %s for '.
|
||||||
'some PHP default limits are very low (as low as 2MB).');
|
'instructions on configuring uploads and storage.',
|
||||||
|
phutil_tag(
|
||||||
|
'a',
|
||||||
|
array(
|
||||||
|
'href' => $doc_href,
|
||||||
|
'target' => '_blank',
|
||||||
|
),
|
||||||
|
pht('Configuring File Storage')));
|
||||||
|
|
||||||
$this
|
$this
|
||||||
->newIssue('config.storage.upload-size-limit')
|
->newIssue('large-files')
|
||||||
->setShortName(pht('Upload Limit'))
|
->setShortName(pht('Large Files'))
|
||||||
->setName(pht('Upload Limit Not Yet Configured'))
|
->setName(pht('Large File Storage Not Configured'))
|
||||||
->setMessage($message)
|
->setMessage($message);
|
||||||
->addPhabricatorConfig('storage.upload-size-limit');
|
}
|
||||||
} else {
|
|
||||||
$memory_limit = PhabricatorStartup::getOldMemoryLimit();
|
|
||||||
if ($memory_limit && ((int)$memory_limit > 0)) {
|
|
||||||
$memory_limit_bytes = phutil_parse_bytes($memory_limit);
|
|
||||||
$memory_usage_bytes = memory_get_usage();
|
|
||||||
$upload_limit_bytes = phutil_parse_bytes($upload_limit);
|
|
||||||
|
|
||||||
$available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
|
$post_max_size = ini_get('post_max_size');
|
||||||
|
if ($post_max_size && ((int)$post_max_size > 0)) {
|
||||||
|
$post_max_bytes = phutil_parse_bytes($post_max_size);
|
||||||
|
$post_max_need = (32 * 1024 * 1024) * 100;
|
||||||
|
if ($post_max_need > $post_max_bytes) {
|
||||||
|
$summary = pht(
|
||||||
|
'Set %s in your PHP configuration to at least 32MB '.
|
||||||
|
'to support large file uploads.',
|
||||||
|
phutil_tag('tt', array(), 'post_max_size'));
|
||||||
|
|
||||||
if ($upload_limit_bytes > $available_bytes) {
|
$message = pht(
|
||||||
$summary = pht(
|
'Adjust %s in your PHP configuration to at least 32MB. When '.
|
||||||
'Your PHP memory limit is configured in a way that may prevent '.
|
'set to smaller value, large file uploads may not work properly.',
|
||||||
'you from uploading large files.');
|
phutil_tag('tt', array(), 'post_max_size'));
|
||||||
|
|
||||||
$message = pht(
|
$this
|
||||||
'When you upload a file via drag-and-drop or the API, the entire '.
|
->newIssue('php.post_max_size')
|
||||||
'file is buffered into memory before being written to permanent '.
|
->setName(pht('PHP post_max_size Not Configured'))
|
||||||
'storage. Phabricator needs memory available to store these '.
|
->setSummary($summary)
|
||||||
'files while they are uploaded, but PHP is currently configured '.
|
->setMessage($message)
|
||||||
'to limit the available memory.'.
|
->setGroup(self::GROUP_PHP)
|
||||||
"\n\n".
|
->addPHPConfig('post_max_size');
|
||||||
'Your Phabricator %s is currently set to a larger value (%s) than '.
|
}
|
||||||
'the amount of available memory (%s) that a PHP process has '.
|
}
|
||||||
'available to use, so uploads via drag-and-drop and the API will '.
|
|
||||||
'hit the memory limit before they hit other limits.'.
|
|
||||||
"\n\n".
|
|
||||||
'(Note that the application itself must also fit in available '.
|
|
||||||
'memory, so not all of the memory under the memory limit is '.
|
|
||||||
'available for buffering file uploads.)'.
|
|
||||||
"\n\n".
|
|
||||||
"The easiest way to resolve this issue is to set %s to %s in your ".
|
|
||||||
"PHP configuration, to disable the memory limit. There is ".
|
|
||||||
"usually little or no value to using this option to limit ".
|
|
||||||
"Phabricator process memory.".
|
|
||||||
"\n\n".
|
|
||||||
"You can also increase the limit, or decrease %s, or ignore this ".
|
|
||||||
"issue and accept that these upload mechanisms will be limited ".
|
|
||||||
"in the size of files they can handle.",
|
|
||||||
phutil_tag('tt', array(), 'storage.upload-size-limit'),
|
|
||||||
phutil_format_bytes($upload_limit_bytes),
|
|
||||||
phutil_format_bytes($available_bytes),
|
|
||||||
phutil_tag('tt', array(), 'memory_limit'),
|
|
||||||
phutil_tag('tt', array(), '-1'),
|
|
||||||
phutil_tag('tt', array(), 'storage.upload-size-limit'));
|
|
||||||
|
|
||||||
$this
|
// This is somewhat arbitrary, but make sure we have enough headroom to
|
||||||
->newIssue('php.memory_limit.upload')
|
// upload a default file at the chunk threshold (8MB), which may be
|
||||||
->setName(pht('Memory Limit Restricts File Uploads'))
|
// base64 encoded, then JSON encoded in the request, and may need to be
|
||||||
->setSummary($summary)
|
// held in memory in the raw and as a query string.
|
||||||
->setMessage($message)
|
$need_bytes = (64 * 1024 * 1024);
|
||||||
->addPHPConfig('memory_limit')
|
|
||||||
->addPHPConfigOriginalValue('memory_limit', $memory_limit)
|
$memory_limit = PhabricatorStartup::getOldMemoryLimit();
|
||||||
->addPhabricatorConfig('storage.upload-size-limit');
|
if ($memory_limit && ((int)$memory_limit > 0)) {
|
||||||
}
|
$memory_limit_bytes = phutil_parse_bytes($memory_limit);
|
||||||
|
$memory_usage_bytes = memory_get_usage();
|
||||||
|
|
||||||
|
$available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
|
||||||
|
|
||||||
|
if ($need_bytes > $available_bytes) {
|
||||||
|
$summary = pht(
|
||||||
|
'Your PHP memory limit is configured in a way that may prevent '.
|
||||||
|
'you from uploading large files or handling large requests.');
|
||||||
|
|
||||||
|
$message = pht(
|
||||||
|
'When you upload a file via drag-and-drop or the API, chunks must '.
|
||||||
|
'be buffered into memory before being written to permanent '.
|
||||||
|
'storage. Phabricator needs memory available to store these '.
|
||||||
|
'chunks while they are uploaded, but PHP is currently configured '.
|
||||||
|
'to severly limit the available memory.'.
|
||||||
|
"\n\n".
|
||||||
|
'PHP processes currently have very little free memory available '.
|
||||||
|
'(%s). To work well, processes should have at least %s.'.
|
||||||
|
"\n\n".
|
||||||
|
'(Note that the application itself must also fit in available '.
|
||||||
|
'memory, so not all of the memory under the memory limit is '.
|
||||||
|
'available for running workloads.)'.
|
||||||
|
"\n\n".
|
||||||
|
"The easiest way to resolve this issue is to set %s to %s in your ".
|
||||||
|
"PHP configuration, to disable the memory limit. There is ".
|
||||||
|
"usually little or no value to using this option to limit ".
|
||||||
|
"Phabricator process memory.".
|
||||||
|
"\n\n".
|
||||||
|
"You can also increase the limit or ignore this issue and accept ".
|
||||||
|
"that you may encounter problems uploading large files and ".
|
||||||
|
"processing large requests.",
|
||||||
|
phutil_format_bytes($available_bytes),
|
||||||
|
phutil_format_bytes($need_bytes),
|
||||||
|
phutil_tag('tt', array(), 'memory_limit'),
|
||||||
|
phutil_tag('tt', array(), '-1'));
|
||||||
|
|
||||||
|
$this
|
||||||
|
->newIssue('php.memory_limit.upload')
|
||||||
|
->setName(pht('Memory Limit Restricts File Uploads'))
|
||||||
|
->setSummary($summary)
|
||||||
|
->setMessage($message)
|
||||||
|
->setGroup(self::GROUP_PHP)
|
||||||
|
->addPHPConfig('memory_limit')
|
||||||
|
->addPHPConfigOriginalValue('memory_limit', $memory_limit);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -147,26 +147,6 @@ final class PhabricatorFilesConfigOptions
|
||||||
"Set this to a valid Amazon S3 bucket to store files there. You ".
|
"Set this to a valid Amazon S3 bucket to store files there. You ".
|
||||||
"must also configure S3 access keys in the 'Amazon Web Services' ".
|
"must also configure S3 access keys in the 'Amazon Web Services' ".
|
||||||
"group.")),
|
"group.")),
|
||||||
$this->newOption('storage.upload-size-limit', 'string', null)
|
|
||||||
->setSummary(
|
|
||||||
pht('Limit to users in interfaces which allow uploading.'))
|
|
||||||
->setDescription(
|
|
||||||
pht(
|
|
||||||
"Set the size of the largest file a user may upload. This is ".
|
|
||||||
"used to render text like 'Maximum file size: 10MB' on ".
|
|
||||||
"interfaces where users can upload files, and files larger than ".
|
|
||||||
"this size will be rejected. \n\n".
|
|
||||||
"NOTE: **Setting this to a large size is NOT sufficient to ".
|
|
||||||
"allow users to upload large files. You must also configure a ".
|
|
||||||
"number of other settings.** To configure file upload limits, ".
|
|
||||||
"consult the article 'Configuring File Upload Limits' in the ".
|
|
||||||
"documentation. Once you've configured some limit across all ".
|
|
||||||
"levels of the server, you can set this limit to an appropriate ".
|
|
||||||
"value and the UI will then reflect the actual configured ".
|
|
||||||
"limit.\n\n".
|
|
||||||
"Specify this limit in bytes, or using a 'K', 'M', or 'G' ".
|
|
||||||
"suffix."))
|
|
||||||
->addExample('10M', pht('Allow Uploads 10MB or Smaller')),
|
|
||||||
$this->newOption(
|
$this->newOption(
|
||||||
'metamta.files.public-create-email',
|
'metamta.files.public-create-email',
|
||||||
'string',
|
'string',
|
||||||
|
|
|
@ -57,8 +57,7 @@ final class PhabricatorFileUploadController extends PhabricatorFileController {
|
||||||
id(new AphrontFormFileControl())
|
id(new AphrontFormFileControl())
|
||||||
->setLabel(pht('File'))
|
->setLabel(pht('File'))
|
||||||
->setName('file')
|
->setName('file')
|
||||||
->setError($e_file)
|
->setError($e_file))
|
||||||
->setCaption($this->renderUploadLimit()))
|
|
||||||
->appendChild(
|
->appendChild(
|
||||||
id(new AphrontFormTextControl())
|
id(new AphrontFormTextControl())
|
||||||
->setLabel(pht('Name'))
|
->setLabel(pht('Name'))
|
||||||
|
@ -102,25 +101,4 @@ final class PhabricatorFileUploadController extends PhabricatorFileController {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
private function renderUploadLimit() {
|
|
||||||
$limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
|
|
||||||
$limit = phutil_parse_bytes($limit);
|
|
||||||
if ($limit) {
|
|
||||||
$formatted = phutil_format_bytes($limit);
|
|
||||||
return 'Maximum file size: '.$formatted;
|
|
||||||
}
|
|
||||||
|
|
||||||
$doc_href = PhabricatorEnv::getDocLink(
|
|
||||||
'Configuring File Upload Limits');
|
|
||||||
$doc_link = phutil_tag(
|
|
||||||
'a',
|
|
||||||
array(
|
|
||||||
'href' => $doc_href,
|
|
||||||
'target' => '_blank',
|
|
||||||
),
|
|
||||||
'Configuring File Upload Limits');
|
|
||||||
|
|
||||||
return hsprintf('Upload limit is not configured, see %s.', $doc_link);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,11 +20,6 @@ final class PhabricatorFileUploadException extends Exception {
|
||||||
'Unable to write file: failed to write to temporary directory.',
|
'Unable to write file: failed to write to temporary directory.',
|
||||||
UPLOAD_ERR_EXTENSION =>
|
UPLOAD_ERR_EXTENSION =>
|
||||||
'Unable to upload: a PHP extension stopped the upload.',
|
'Unable to upload: a PHP extension stopped the upload.',
|
||||||
|
|
||||||
-1000 =>
|
|
||||||
pht("Uploaded file is too large: current limit is %s. To adjust this ".
|
|
||||||
"limit change 'storage.upload-size-limit' in the Phabricator config.",
|
|
||||||
PhabricatorEnv::getEnvConfig('storage.upload-size-limit')),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
$message = idx($map, $code, 'Upload failed: unknown error.');
|
$message = idx($map, $code, 'Upload failed: unknown error.');
|
||||||
|
|
|
@ -162,8 +162,6 @@ final class PhabricatorFile extends PhabricatorFileDAO
|
||||||
throw new Exception('File size disagrees with uploaded size.');
|
throw new Exception('File size disagrees with uploaded size.');
|
||||||
}
|
}
|
||||||
|
|
||||||
self::validateFileSize(strlen($file_data));
|
|
||||||
|
|
||||||
return $file_data;
|
return $file_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -181,22 +179,9 @@ final class PhabricatorFile extends PhabricatorFileDAO
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function newFromXHRUpload($data, array $params = array()) {
|
public static function newFromXHRUpload($data, array $params = array()) {
|
||||||
self::validateFileSize(strlen($data));
|
|
||||||
return self::newFromFileData($data, $params);
|
return self::newFromFileData($data, $params);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static function validateFileSize($size) {
|
|
||||||
$limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
|
|
||||||
if (!$limit) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$limit = phutil_parse_bytes($limit);
|
|
||||||
if ($size > $limit) {
|
|
||||||
throw new PhabricatorFileUploadException(-1000);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Given a block of data, try to load an existing file with the same content
|
* Given a block of data, try to load an existing file with the same content
|
||||||
|
|
|
@ -1,45 +1,134 @@
|
||||||
@title Configuring File Storage
|
@title Configuring File Storage
|
||||||
@group config
|
@group config
|
||||||
|
|
||||||
Setup how Phabricator will store files.
|
Setup file storage and support for large files.
|
||||||
|
|
||||||
Overview
|
Overview
|
||||||
========
|
========
|
||||||
|
|
||||||
Phabricator allows users to upload files, and several applications use file
|
This document describes how to configure Phabricator to support large file
|
||||||
storage (for instance, Maniphest allows you to attach files to tasks). You can
|
uploads, and how to choose where Phabricator stores files.
|
||||||
configure several different storage systems.
|
|
||||||
|
|
||||||
| System | Setup | Cost | Notes |
|
There are two major things to configure:
|
||||||
|
|
||||||
|
- set up PHP and your HTTP server to accept large requests;
|
||||||
|
- choose and configure a storage engine.
|
||||||
|
|
||||||
|
The following sections will guide you through this configuration.
|
||||||
|
|
||||||
|
|
||||||
|
How Phabricator Stores Files
|
||||||
|
============================
|
||||||
|
|
||||||
|
Phabricator stores files in "storage engines", which are modular backends
|
||||||
|
that implement access to some storage system (like MySQL, the filesystem, or
|
||||||
|
a cloud storage service like Amazon S3).
|
||||||
|
|
||||||
|
Phabricator stores large files by breaking them up into many chunks (a few
|
||||||
|
megabytes in size) and storing the chunks in an underlying storage engine.
|
||||||
|
This makes it easier to implement new storage engines and gives Phabricator
|
||||||
|
more flexibility in managing file data.
|
||||||
|
|
||||||
|
The first section of this document discusses configuring your install so that
|
||||||
|
PHP and your HTTP server will accept requests which are larger than the size of
|
||||||
|
one file chunk. Without this configuration, file chunk data will be rejected.
|
||||||
|
|
||||||
|
The second section discusses choosing and configuring storage engines, so data
|
||||||
|
is stored where you want it to be.
|
||||||
|
|
||||||
|
|
||||||
|
Configuring Upload Limits
|
||||||
|
=========================
|
||||||
|
|
||||||
|
File uploads are limited by several pieces of configuration at different layers
|
||||||
|
of the stack. Generally, the minimum value of all the limits is the effective
|
||||||
|
one.
|
||||||
|
|
||||||
|
To upload large files, you need to increase all the limits to at least
|
||||||
|
**32MB**. This will allow you to upload file chunks, which will let Phabricator
|
||||||
|
store arbitrarily large files.
|
||||||
|
|
||||||
|
The settings which limit file uploads are:
|
||||||
|
|
||||||
|
**HTTP Server**: The HTTP server may set a limit on the maximum request size.
|
||||||
|
If you exceed this limit, you'll see a default server page with an HTTP error.
|
||||||
|
These directives limit the total size of the request body, so they must be
|
||||||
|
somewhat larger than the desired maximum filesize.
|
||||||
|
|
||||||
|
- **Apache**: Apache limits requests with the Apache `LimitRequestBody`
|
||||||
|
directive.
|
||||||
|
- **nginx**: nginx limits requests with the nginx `client_max_body_size`
|
||||||
|
directive. This often defaults to `1M`.
|
||||||
|
- **lighttpd**: lighttpd limits requests with the lighttpd
|
||||||
|
`server.max-request-size` directive.
|
||||||
|
|
||||||
|
Set the applicable limit to at least **32MB**. Phabricator can not read these
|
||||||
|
settings, so it can not raise setup warnings if they are misconfigured.
|
||||||
|
|
||||||
|
**PHP**: PHP has several directives which limit uploads. These directives are
|
||||||
|
found in `php.ini`.
|
||||||
|
|
||||||
|
- **post_max_size**: Maximum POST request size PHP will accept. If you
|
||||||
|
exceed this, Phabricator will give you a useful error. This often defaults
|
||||||
|
to `8M`. Set this to at least `32MB`. Phabricator will give you a setup
|
||||||
|
warning about this if it is set too low.
|
||||||
|
- **memory_limit**: For some uploads, file data will be read into memory
|
||||||
|
before Phabricator can adjust the memory limit. If you exceed this, PHP
|
||||||
|
may give you a useful error, depending on your configuration. It is
|
||||||
|
recommended that you set this to `-1` to disable it. Phabricator will
|
||||||
|
give you a setup warning about this if it is set too low.
|
||||||
|
|
||||||
|
You may also want to configure these PHP options:
|
||||||
|
|
||||||
|
- **max_input_vars**: When files are uploaded via HTML5 drag and drop file
|
||||||
|
upload APIs, PHP parses the file body as though it contained normal POST
|
||||||
|
parameters, and may trigger `max_input_vars` if a file has a lot of
|
||||||
|
brackets in it. You may need to set it to some astronomically high value.
|
||||||
|
- **upload_max_filesize**: Maximum file size PHP will accept in a raw file
|
||||||
|
upload. This is not normally used when uploading files via drag-and-drop,
|
||||||
|
but affects some other kinds of file uploads. If you exceed this,
|
||||||
|
Phabricator will give you a useful error. This often defaults to `2M`. Set
|
||||||
|
this to at least `32MB`.
|
||||||
|
|
||||||
|
Once you've adjusted all this configuration, your server will be able to
|
||||||
|
receive chunk uploads. As long as you have somewhere to store them, this will
|
||||||
|
enable you to store arbitrarily large files.
|
||||||
|
|
||||||
|
|
||||||
|
Storage Engines
|
||||||
|
===============
|
||||||
|
|
||||||
|
Phabricator supports several different file storage engines:
|
||||||
|
|
||||||
|
| Engine | Setup | Cost | Notes |
|
||||||
|========|=======|======|=======|
|
|========|=======|======|=======|
|
||||||
| MySQL | Automatic | Free | May not scale well. |
|
| MySQL | Automatic | Free | May not scale well. |
|
||||||
| Local Disk | Easy | Free | Does not scale well. |
|
| Local Disk | Easy | Free | Does not scale well. |
|
||||||
| Amazon S3 | Easy | Cheap | Scales well. |
|
| Amazon S3 | Easy | Cheap | Scales well. |
|
||||||
| Custom | Hard | Varies | Implement a custom storage engine. |
|
| Custom | Hard | Varies | Implement a custom storage engine. |
|
||||||
|
|
||||||
|
You can review available storage engines and their configuration by navigating
|
||||||
|
to {nav Applications > Files > Help/Options > Storage Engines} in the web UI.
|
||||||
|
|
||||||
By default, Phabricator is configured to store files up to 1MB in MySQL, and
|
By default, Phabricator is configured to store files up to 1MB in MySQL, and
|
||||||
reject files larger than 1MB. To store larger files, you can either:
|
reject files larger than 1MB. To store larger files, you can either:
|
||||||
|
|
||||||
- configure local disk storage; or
|
- increase the MySQL limit to at least 8MB; or
|
||||||
- configure Amazon S3 storage; or
|
- configure another storage engine.
|
||||||
- raise the limits on MySQL.
|
|
||||||
|
|
||||||
See the rest of this document for some additional discussion of engines.
|
Doing either of these will enable the chunk storage engine and support for
|
||||||
|
arbitrarily large files.
|
||||||
|
|
||||||
You don't have to fully configure this immediately, the defaults are okay until
|
The remaining sections of this document discuss the available storage engines
|
||||||
you need to upload larger files and it's relatively easy to port files between
|
and how to configure them.
|
||||||
storage engines later.
|
|
||||||
|
|
||||||
Storage Engines
|
|
||||||
===============
|
|
||||||
|
|
||||||
Builtin storage engines and information on how to configure them.
|
Engine: MySQL
|
||||||
|
=============
|
||||||
|
|
||||||
== MySQL ==
|
- **Pros**: Low latency, no setup required.
|
||||||
|
- **Cons**: Storing files in a database is a classic bad idea. May become
|
||||||
- **Pros**: Fast, no setup required.
|
difficult to administrate if you have a large amount of data.
|
||||||
- **Cons**: Storing files in a database is a classic bad idea. Does not scale
|
|
||||||
well. Maximum file size is limited.
|
|
||||||
|
|
||||||
MySQL storage is configured by default, for files up to (just under) 1MB. You
|
MySQL storage is configured by default, for files up to (just under) 1MB. You
|
||||||
can configure it with these keys:
|
can configure it with these keys:
|
||||||
|
@ -49,37 +138,43 @@ can configure it with these keys:
|
||||||
|
|
||||||
For most installs, it is reasonable to leave this engine as-is and let small
|
For most installs, it is reasonable to leave this engine as-is and let small
|
||||||
files (like thumbnails and profile images) be stored in MySQL, which is usually
|
files (like thumbnails and profile images) be stored in MySQL, which is usually
|
||||||
the lowest-latency filestore.
|
the lowest-latency filestore, even if you configure another storage engine.
|
||||||
|
|
||||||
To support larger files, configure another engine or increase this limit.
|
To support large files, increase this limit to at least **8MB**. This will
|
||||||
|
activate chunk storage in MySQL.
|
||||||
|
|
||||||
== Local Disk ==
|
Engine: Local Disk
|
||||||
|
==================
|
||||||
|
|
||||||
- **Pros**: Very simple. Almost no setup required.
|
- **Pros**: Simple to setup.
|
||||||
- **Cons**: Doesn't scale to multiple web frontends without NFS.
|
- **Cons**: Doesn't scale to multiple web frontends without NFS.
|
||||||
|
|
||||||
To upload larger files:
|
To configure file storage on the local disk, set:
|
||||||
|
|
||||||
- `storage.local-disk.path`: Set to some writable directory on local disk.
|
- `storage.local-disk.path`: Set to some writable directory on local disk.
|
||||||
Make that directory.
|
Make that directory.
|
||||||
|
|
||||||
== Amazon S3 ==
|
Engine: Amazon S3
|
||||||
|
=================
|
||||||
|
|
||||||
- **Pros**: Scales well.
|
- **Pros**: Scales well.
|
||||||
- **Cons**: More complicated and expensive than other approaches.
|
- **Cons**: Slightly more complicated than other engines, not free.
|
||||||
|
|
||||||
To enable file storage in S3, set these key:
|
To enable file storage in S3, set these keys:
|
||||||
|
|
||||||
- ##amazon-s3.access-key## Your AWS access key.
|
- `amazon-s3.access-key`: Your AWS access key.
|
||||||
- ##amazon-s3.secret-key## Your AWS secret key.
|
- `amazon-s3.secret-key`: Your AWS secret key.
|
||||||
- ##storage.s3.bucket## S3 bucket name where files should be stored.
|
- `storage.s3.bucket`: S3 bucket name where files should be stored.
|
||||||
|
|
||||||
= Testing Storage Engines =
|
Testing Storage Engines
|
||||||
|
=======================
|
||||||
|
|
||||||
You can test that things are correctly configured by going to the Files
|
You can test that things are correctly configured by dragging and dropping
|
||||||
application (##/file/##) and uploading files.
|
a file onto the Phabricator home page. If engines have been configured
|
||||||
|
properly, the file should upload.
|
||||||
|
|
||||||
= Migrating Files Between Engines =
|
Migrating Files Between Engines
|
||||||
|
===============================
|
||||||
|
|
||||||
If you want to move files between storage engines, you can use the `bin/files`
|
If you want to move files between storage engines, you can use the `bin/files`
|
||||||
script to perform migrations. For example, suppose you previously used MySQL but
|
script to perform migrations. For example, suppose you previously used MySQL but
|
||||||
|
@ -95,10 +190,9 @@ If that works properly, you can then migrate everything:
|
||||||
You can use `--dry-run` to show which migrations would be performed without
|
You can use `--dry-run` to show which migrations would be performed without
|
||||||
taking any action. Run `bin/files help` for more options and information.
|
taking any action. Run `bin/files help` for more options and information.
|
||||||
|
|
||||||
= Next Steps =
|
Next Steps
|
||||||
|
==========
|
||||||
|
|
||||||
Continue by:
|
Continue by:
|
||||||
|
|
||||||
- configuring file size upload limits with
|
|
||||||
@{article:Configuring File Upload Limits}; or
|
|
||||||
- returning to the @{article:Configuration Guide}.
|
- returning to the @{article:Configuration Guide}.
|
||||||
|
|
|
@ -1,77 +0,0 @@
|
||||||
@title Configuring File Upload Limits
|
|
||||||
@group config
|
|
||||||
|
|
||||||
Explains limits on file upload sizes.
|
|
||||||
|
|
||||||
= Overview =
|
|
||||||
|
|
||||||
File uploads are limited by a large number of pieces of configuration, at
|
|
||||||
multiple layers of the application. Generally, the minimum value of all the
|
|
||||||
limits is the effective one. To upload large files, you need to increase all
|
|
||||||
the limits above the maximum file size you want to support. The settings which
|
|
||||||
limit uploads are:
|
|
||||||
|
|
||||||
- **HTTP Server**: The HTTP server may set a limit on the maximum request
|
|
||||||
size. If you exceed this limit, you'll see a default server page with an
|
|
||||||
HTTP error. These directives limit the total size of the request body,
|
|
||||||
so they must be somewhat larger than the desired maximum filesize.
|
|
||||||
- **Apache**: Apache limits requests with the Apache `LimitRequestBody`
|
|
||||||
directive.
|
|
||||||
- **nginx**: nginx limits requests with the nginx `client_max_body_size`
|
|
||||||
directive. This often defaults to `1M`.
|
|
||||||
- **lighttpd**: lighttpd limits requests with the lighttpd
|
|
||||||
`server.max-request-size` directive.
|
|
||||||
- **PHP**: PHP has several directives which limit uploads. These directives
|
|
||||||
are found in `php.ini`.
|
|
||||||
- **upload_max_filesize**: Maximum file size PHP will accept in a file
|
|
||||||
upload. If you exceed this, Phabricator will give you a useful error. This
|
|
||||||
often defaults to `2M`.
|
|
||||||
- **post_max_size**: Maximum POST request size PHP will accept. If you
|
|
||||||
exceed this, Phabricator will give you a useful error. This often defaults
|
|
||||||
to `8M`.
|
|
||||||
- **memory_limit**: For some uploads, file data will be read into memory
|
|
||||||
before Phabricator can adjust the memory limit. If you exceed this, PHP
|
|
||||||
may give you a useful error, depending on your configuration.
|
|
||||||
- **max_input_vars**: When files are uploaded via HTML5 drag and drop file
|
|
||||||
upload APIs, PHP parses the file body as though it contained normal POST
|
|
||||||
parameters, and may trigger `max_input_vars` if a file has a lot of
|
|
||||||
brackets in it. You may need to set it to some astronomically high value.
|
|
||||||
- **Storage Engines**: Some storage engines can be configured not to accept
|
|
||||||
files over a certain size. To upload a file, you must have at least one
|
|
||||||
configured storage engine which can accept it. Phabricator should give you
|
|
||||||
useful errors if any of these fail.
|
|
||||||
- **MySQL Engine**: Upload size is limited by the Phabricator setting
|
|
||||||
`storage.mysql-engine.max-size`.
|
|
||||||
- **Amazon S3**: Upload size is limited by Phabricator's implementation to
|
|
||||||
`5G`.
|
|
||||||
- **Local Disk**: Upload size is limited only by free disk space.
|
|
||||||
- **Resource Constraints**: File uploads are limited by resource constraints
|
|
||||||
on the application server. In particular, some uploaded files are written
|
|
||||||
to disk in their entirety before being moved to storage engines, and all
|
|
||||||
uploaded files are read into memory before being moved. These hard limits
|
|
||||||
should be large for most servers, but will fundamentally prevent Phabricator
|
|
||||||
from processing truly enormous files (GB/TB scale). Phabricator is probably
|
|
||||||
not the best application for this in any case.
|
|
||||||
- **Phabricator Master Limit**: The master limit, `storage.upload-size-limit`,
|
|
||||||
is used to show upload limits in the UI.
|
|
||||||
|
|
||||||
Phabricator can't read some of these settings, so it can't figure out what the
|
|
||||||
current limit is or be much help at all in configuring it. Thus, you need to
|
|
||||||
manually configure all of these limits and then tell Phabricator what you set
|
|
||||||
them to. Follow these steps:
|
|
||||||
|
|
||||||
- Pick some limit you want to set, like `100M`.
|
|
||||||
- Configure all of the settings mentioned above to be a bit bigger than the
|
|
||||||
limit you want to enforce (**note that there are some security implications
|
|
||||||
to raising these limits**; principally, your server may become easier to
|
|
||||||
attack with a denial-of-service).
|
|
||||||
- Set `storage.upload-size-limit` to the limit you want.
|
|
||||||
- The UI should now show your limit.
|
|
||||||
- Upload a big file to make sure it works.
|
|
||||||
|
|
||||||
= Next Steps =
|
|
||||||
|
|
||||||
Continue by:
|
|
||||||
|
|
||||||
- configuring file storage with @{article:Configuring File Storage}; or
|
|
||||||
- returning to the @{article:Configuration Guide}.
|
|
Loading…
Reference in a new issue