From 6a6279705671a4c760e5913e57c2e073aa1ed381 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 19 Jan 2018 14:11:07 -0800 Subject: [PATCH] Fix some issues with Diffusion file data limits Summary: See . Three issues here: - When we finish reading `git cat-file ...` or whatever, we can end up with more than one chunk worth of bytes left in the internal buffer if the read is fast. Use `while` instead of `if` to make sure we write the whole buffer. - Limiting output with `setStdoutSizeLimit()` isn't really a reliable way to limit the size if we're also reading from the buffer. It's also pretty indirect and confusing. Instead, just let the `FileUploadSource` explicitly implement a byte limit in a straightforward way. - We weren't setting the time limit correctly on the main path. Overall, this could cause >4MB files to "write" as 4MB files, with the rest of the file left in the UploadSource buffer. Since these files were technically under the limit, they could return as valid. This was intermittent. Test Plan: - Pushed a ~4.2MB file. - Reloaded Diffusion a bunch, sometimes saw the `while/if` buffer race and produce a 4MB file with a prompt to download it. (Other times, the buffer worked right and the page just says "this file is too big, sorry"). - Applied patches. - Reloaded Diffusion a bunch, no longer saw bad behavior or truncated files. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18885 --- src/__phutil_library_map__.php | 2 ++ .../query/DiffusionFileFutureQuery.php | 24 +++++------------ .../PhabricatorFileUploadSource.php | 26 ++++++++++++++++--- ...atorFileUploadSourceByteLimitException.php | 4 +++ 4 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 src/applications/files/uploadsource/PhabricatorFileUploadSourceByteLimitException.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 55fe8ada3d..2b46a40970 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2938,6 +2938,7 @@ phutil_register_library_map(array( 'PhabricatorFileUploadDialogController' => 'applications/files/controller/PhabricatorFileUploadDialogController.php', 'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php', 'PhabricatorFileUploadSource' => 'applications/files/uploadsource/PhabricatorFileUploadSource.php', + 'PhabricatorFileUploadSourceByteLimitException' => 'applications/files/uploadsource/PhabricatorFileUploadSourceByteLimitException.php', 'PhabricatorFileinfoSetupCheck' => 'applications/config/check/PhabricatorFileinfoSetupCheck.php', 'PhabricatorFilesApplication' => 'applications/files/application/PhabricatorFilesApplication.php', 'PhabricatorFilesApplicationStorageEnginePanel' => 'applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php', @@ -8362,6 +8363,7 @@ phutil_register_library_map(array( 'PhabricatorFileUploadDialogController' => 'PhabricatorFileController', 'PhabricatorFileUploadException' => 'Exception', 'PhabricatorFileUploadSource' => 'Phobject', + 'PhabricatorFileUploadSourceByteLimitException' => 'Exception', 'PhabricatorFileinfoSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorFilesApplication' => 'PhabricatorApplication', 'PhabricatorFilesApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel', diff --git a/src/applications/diffusion/query/DiffusionFileFutureQuery.php b/src/applications/diffusion/query/DiffusionFileFutureQuery.php index 250e4962da..18779e5fb8 100644 --- a/src/applications/diffusion/query/DiffusionFileFutureQuery.php +++ b/src/applications/diffusion/query/DiffusionFileFutureQuery.php @@ -88,7 +88,7 @@ abstract class DiffusionFileFutureQuery } final protected function executeQuery() { - $future = $this->newQueryFuture(); + $future = $this->newConfiguredQueryFuture(); $drequest = $this->getRequest(); @@ -105,6 +105,11 @@ abstract class DiffusionFileFutureQuery ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) ->setExecFuture($future); + $byte_limit = $this->getByteLimit(); + if ($byte_limit) { + $source->setByteLimit($byte_limit); + } + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $file = $source->uploadFile(); unset($unguarded); @@ -116,18 +121,8 @@ abstract class DiffusionFileFutureQuery $this->didHitTimeLimit = true; $file = null; - } - - $byte_limit = $this->getByteLimit(); - - if ($byte_limit && ($file->getByteSize() > $byte_limit)) { + } catch (PhabricatorFileUploadSourceByteLimitException $ex) { $this->didHitByteLimit = true; - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - id(new PhabricatorDestructionEngine()) - ->destroyObject($file); - unset($unguarded); - $file = null; } @@ -141,11 +136,6 @@ abstract class DiffusionFileFutureQuery $future->setTimeout($this->getTimeout()); } - $byte_limit = $this->getByteLimit(); - if ($byte_limit) { - $future->setStdoutSizeLimit($byte_limit + 1); - } - return $future; } diff --git a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php index bbd93a455a..bf213a417e 100644 --- a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php +++ b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php @@ -12,6 +12,8 @@ abstract class PhabricatorFileUploadSource private $shouldChunk; private $didRewind; private $totalBytesWritten = 0; + private $totalBytesRead = 0; + private $byteLimit = 0; public function setName($name) { $this->name = $name; @@ -40,6 +42,15 @@ abstract class PhabricatorFileUploadSource return $this->viewPolicy; } + public function setByteLimit($byte_limit) { + $this->byteLimit = $byte_limit; + return $this; + } + + public function getByteLimit() { + return $this->byteLimit; + } + public function uploadFile() { if (!$this->shouldChunkFile()) { return $this->writeSingleFile(); @@ -81,8 +92,15 @@ abstract class PhabricatorFileUploadSource return false; } + $read_bytes = $data->current(); + $this->totalBytesRead += strlen($read_bytes); + + if ($this->byteLimit && ($this->totalBytesRead > $this->byteLimit)) { + throw new PhabricatorFileUploadSourceByteLimitException(); + } + $rope = $this->getRope(); - $rope->append($data->current()); + $rope->append($read_bytes); return true; } @@ -160,8 +178,10 @@ abstract class PhabricatorFileUploadSource } } - // If we have extra bytes at the end, write them. - if ($rope->getByteLength()) { + // If we have extra bytes at the end, write them. Note that it's possible + // that we have more than one chunk of bytes left if the read was very + // fast. + while ($rope->getByteLength()) { $this->writeChunk($file, $engine); } diff --git a/src/applications/files/uploadsource/PhabricatorFileUploadSourceByteLimitException.php b/src/applications/files/uploadsource/PhabricatorFileUploadSourceByteLimitException.php new file mode 100644 index 0000000000..d78c2189b6 --- /dev/null +++ b/src/applications/files/uploadsource/PhabricatorFileUploadSourceByteLimitException.php @@ -0,0 +1,4 @@ +