Make storage.mysql-engine.max-size independent on max_allowed_packet

Summary:
I like systems that just work. It is possible to store files larger than max_allowed_packet in MySQL and we shouldn't demand it.

It also fixes a problem when file was smaller than `storage.mysql-engine.max-size` but its escaped version was larger than `max_allowed_packet`.

Test Plan: Reduced the size to 5e4, uploaded 90 kB file, checked the queries in DarkConsole, downloaded the file.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Differential Revision: https://secure.phabricator.com/D3392

conf/default.conf.php

@@ -873,10 +873,8 @@ return array(
 
   // The largest filesize Phabricator will store in the MySQL BLOB storage
   // engine, which just uses a database table to store files. While this isn't a
-  // best practice, it's really easy to set up. This is hard-limited by the
+  // best practice, it's really easy to set up. Set this to 0 to disable use of
-  // value of 'max_allowed_packet' in MySQL (since this often defaults to 1MB,
+  // the MySQL blob engine.
-  // the default here is slightly smaller than 1MB). Set this to 0 to disable
-  // use of the MySQL blob engine.
   'storage.mysql-engine.max-size' => 1000000,
 
   // Phabricator provides a local disk storage engine, which just writes files

src/applications/files/storage/PhabricatorFileStorageBlob.php

@@ -22,7 +22,42 @@
  * @group filestorage
  */
 final class PhabricatorFileStorageBlob extends PhabricatorFileDAO {
+  // max_allowed_packet defaults to 1 MiB, escaping can make the data twice
+  // longer, query fits in the rest.
+  const CHUNK_SIZE = 5e5;
 
   protected $data;
 
+  private $fullData;
+
+  protected function willWriteData(array &$data) {
+    parent::willWriteData($data);
+
+    $this->fullData = $data['data'];
+    if (strlen($data['data']) > self::CHUNK_SIZE) {
+      $data['data'] = substr($data['data'], 0, self::CHUNK_SIZE);
+      $this->openTransaction();
+    }
+  }
+
+  protected function didWriteData() {
+    $size = self::CHUNK_SIZE;
+    $length = strlen($this->fullData);
+    if ($length > $size) {
+      $conn = $this->establishConnection('w');
+      for ($offset = $size; $offset < $length; $offset += $size) {
+        queryfx(
+          $conn,
+          'UPDATE %T SET data = CONCAT(data, %s) WHERE %C = %d',
+          $this->getTableName(),
+          substr($this->fullData, $offset, $size),
+          $this->getIDKeyForUse(),
+          $this->getID());
+      }
+      $this->saveTransaction();
+    }
+
+    parent::didWriteData();
+  }
+
 }

src/docs/configuration/configuring_file_storage.diviner

@@ -41,8 +41,7 @@ Builtin storage engines and information on how to configure them.
 MySQL storage is configured by default, for files up to (just under) 1MB. You
 can configure it with these keys:
 
-  - ##storage.mysql-engine.max-size##: Change the filesize limit. Note that
+  - ##storage.mysql-engine.max-size##: Change the filesize limit. Set to 0
-    this must be smaller than 'max_allowed_packet' on the server. Set to 0
     to disable.
 
 For most installs, it is recommended you configure local disk storage below,

src/docs/configuration/configuring_file_upload_limits.diviner

@@ -41,8 +41,7 @@ limit uploads are:
     configured storage engine which can accept it. Phabricator should give you
     useful errors if any of these fail.
     - **MySQL Engine**: Upload size is limited by the Phabricator setting
-      `storage.mysql-engine.max-size`, which is in turn limited by the MySQL
+      `storage.mysql-engine.max-size`.
-      setting `max_allowed_packet`. This often defaults to `1M`.
     - **Amazon S3**: Upload size is limited by Phabricator's implementation to
       `5G`.
     - **Local Disk**: Upload size is limited only by free disk space.

src/infrastructure/PhabricatorSetup.php

@@ -564,25 +564,6 @@ final class PhabricatorSetup {
       self::write(" okay  max_allowed_packet = {$max_allowed_packet}.\n");
     }
 
-    $mysql_key = 'storage.mysql-engine.max-size';
-    $mysql_limit = PhabricatorEnv::getEnvConfig($mysql_key);
-
-    if ($mysql_limit && ($mysql_limit + 8192) > $max_allowed_packet) {
-      self::writeFailure();
-      self::write(
-        "Setup failure! Your Phabricator 'storage.mysql-engine.max-size' ".
-        "configuration ('{$mysql_limit}') must be at least 8KB smaller ".
-        "than your MySQL 'max_allowed_packet' configuration ".
-        "('{$max_allowed_packet}'). Raise the 'max_allowed_packet' in your ".
-        "MySQL configuration, or reduce the maximum file size allowed by ".
-        "the Phabricator configuration.\n");
-      return;
-    } else if (!$mysql_limit) {
-      self::write(" skip  MySQL file storage engine not configured.\n");
-    } else {
-      self::write(" okay  MySQL file storage engine configuration okay.\n");
-    }
-
     $local_key = 'storage.local-disk.path';
     $local_path = PhabricatorEnv::getEnvConfig($local_key);
     if ($local_path) {