Allow modern hunks to be stored deflated

Summary: Ref T4045. Ref T5179. When saving a modern hunk, deflate it if we have the function and deflating it will save a nontrivial number of bytes.

Test Plan:
  - Used `bin/hunks migrate` to move some hunks over, saw ~70-80% compression on most standard hunks.
  - Viewed changesets using compressed hunks.
  - Profiled `gzinflate()` and verified the cost is trivial (<< 1ms) at least for normal diffs.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T4045, T5179

Differential Revision: https://secure.phabricator.com/D9292

src/applications/cache/PhabricatorCaches.php

@@ -3,6 +3,7 @@
 /**
  * @task immutable  Immutable Cache
  * @task setup      Setup Cache
+ * @task compress   Compression
  */
 final class PhabricatorCaches {
 
@@ -283,4 +284,64 @@ final class PhabricatorCaches {
     return $caches;
   }
 
+
+  /**
+   * Deflate a value, if deflation is available and has an impact.
+   *
+   * If the value is larger than 1KB, we have `gzdeflate()`, we successfully
+   * can deflate it, and it benefits from deflation, we deflate it. Otherwise
+   * we leave it as-is.
+   *
+   * Data can later be inflated with @{method:inflateData}.
+   *
+   * @param string String to attempt to deflate.
+   * @return string|null Deflated string, or null if it was not deflated.
+   * @task compress
+   */
+  public static function maybeDeflateData($value) {
+    $len = strlen($value);
+    if ($len <= 1024) {
+      return null;
+    }
+
+    if (!function_exists('gzdeflate')) {
+      return null;
+    }
+
+    $deflated = gzdeflate($value);
+    if ($deflated === false) {
+      return null;
+    }
+
+    $deflated_len = strlen($deflated);
+    if ($deflated_len >= ($len / 2)) {
+      return null;
+    }
+
+    return $deflated;
+  }
+
+
+  /**
+   * Inflate data previously deflated by @{method:maybeDeflateData}.
+   *
+   * @param string Deflated data, from @{method:maybeDeflateData}.
+   * @return string Original, uncompressed data.
+   * @task compress
+   */
+  public static function inflateData($value) {
+    if (!function_exists('gzinflate')) {
+      throw new Exception(
+        pht('gzinflate() is not available; unable to read deflated data!'));
+    }
+
+    $value = gzinflate($value);
+    if ($value === false) {
+      throw new Exception(pht('Failed to inflate data!'));
+    }
+
+    return $value;
+  }
+
+
 }

src/applications/cache/PhabricatorKeyValueDatabaseCache.php

@@ -145,18 +145,10 @@ final class PhabricatorKeyValueDatabaseCache
                      PhabricatorEnv::getEnvConfig('cache.enable-deflate');
     }
 
-    // If the value is larger than 1KB, we have gzdeflate(), we successfully
-    // can deflate it, and it benefits from deflation, store it deflated.
     if ($can_deflate) {
-      $len = strlen($value);
-      if ($len > 1024) {
-        $deflated = gzdeflate($value);
-        if ($deflated !== false) {
-          $deflated_len = strlen($deflated);
-          if ($deflated_len < ($len / 2)) {
-            return array(self::CACHE_FORMAT_DEFLATE, $deflated);
-          }
-        }
+      $deflated = PhabricatorCaches::maybeDeflateData($value);
+      if ($deflated !== null) {
+        return array(self::CACHE_FORMAT_DEFLATE, $deflated);
       }
     }
 
@@ -168,14 +160,7 @@ final class PhabricatorKeyValueDatabaseCache
       case self::CACHE_FORMAT_RAW:
         return $value;
       case self::CACHE_FORMAT_DEFLATE:
-        if (!function_exists('gzinflate')) {
-          throw new Exception("No gzinflate() to read deflated cache.");
-        }
-        $value = gzinflate($value);
-        if ($value === false) {
-          throw new Exception("Failed to deflate cache.");
-        }
-        return $value;
+        return PhabricatorCaches::inflateData($value);
       default:
         throw new Exception("Unknown cache format.");
     }

src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php

@@ -36,6 +36,20 @@ final class PhabricatorHunksManagementMigrateWorkflow
         $new_hunk->save();
         $hunk->delete();
       $hunk->saveTransaction();
+
+      $old_len = strlen($hunk->getChanges());
+      $new_len = strlen($new_hunk->getData());
+      if ($old_len) {
+        $diff_len = ($old_len - $new_len);
+        $console->writeOut(
+          "%s\n",
+          pht(
+            'Saved %s bytes (%s).',
+            new PhutilNumber($diff_len),
+            sprintf('%.1f%%', 100 * ($diff_len / $old_len))));
+      }
+
+      break;
     }
 
     if ($saw_any_rows) {

src/applications/differential/storage/DifferentialHunkModern.php

@@ -6,13 +6,15 @@ final class DifferentialHunkModern extends DifferentialHunk {
   const DATATYPE_FILE       = 'file';
 
   const DATAFORMAT_RAW      = 'byte';
-  const DATAFORMAT_DEFLATE  = 'gzde';
+  const DATAFORMAT_DEFLATED = 'gzde';
 
   protected $dataType;
   protected $dataEncoding;
   protected $dataFormat;
   protected $data;
 
+  private $rawData;
+
   public function getTableName() {
     return 'differential_hunk_modern';
   }
@@ -26,6 +28,8 @@ final class DifferentialHunkModern extends DifferentialHunk {
   }
 
   public function setChanges($text) {
+    $this->rawData = $text;
+
     $this->dataEncoding = $this->detectEncodingForStorage($text);
     $this->dataType = self::DATATYPE_TEXT;
     $this->dataFormat = self::DATAFORMAT_RAW;
@@ -40,34 +44,60 @@ final class DifferentialHunkModern extends DifferentialHunk {
       $this->getDataEncoding());
   }
 
-  private function getRawData() {
+  public function save() {
+
     $type = $this->getDataType();
-    $data = $this->getData();
-
-    switch ($type) {
-      case self::DATATYPE_TEXT:
-        // In this storage type, the changes are stored on the object.
-        $data = $data;
-        break;
-      case self::DATATYPE_FILE:
-      default:
-        throw new Exception(
-          pht('Hunk has unsupported data type "%s"!', $type));
-    }
-
     $format = $this->getDataFormat();
-    switch ($format) {
-      case self::DATAFORMAT_RAW:
-        // In this format, the changes are stored as-is.
-        $data = $data;
-        break;
-      case self::DATAFORMAT_DEFLATE:
-      default:
-        throw new Exception(
-          pht('Hunk has unsupported data encoding "%s"!', $type));
+
+    // Before saving the data, attempt to compress it.
+    if ($type == self::DATATYPE_TEXT) {
+      if ($format == self::DATAFORMAT_RAW) {
+        $data = $this->getData();
+        $deflated = PhabricatorCaches::maybeDeflateData($data);
+        if ($deflated !== null) {
+          $this->data = $deflated;
+          $this->dataFormat = self::DATAFORMAT_DEFLATED;
+        }
+      }
     }
 
-    return $data;
+    return parent::save();
+  }
+
+  private function getRawData() {
+    if ($this->rawData === null) {
+      $type = $this->getDataType();
+      $data = $this->getData();
+
+      switch ($type) {
+        case self::DATATYPE_TEXT:
+          // In this storage type, the changes are stored on the object.
+          $data = $data;
+          break;
+        case self::DATATYPE_FILE:
+        default:
+          throw new Exception(
+            pht('Hunk has unsupported data type "%s"!', $type));
+      }
+
+      $format = $this->getDataFormat();
+      switch ($format) {
+        case self::DATAFORMAT_RAW:
+          // In this format, the changes are stored as-is.
+          $data = $data;
+          break;
+        case self::DATAFORMAT_DEFLATED:
+          $data = PhabricatorCaches::inflateData($data);
+          break;
+        default:
+          throw new Exception(
+            pht('Hunk has unsupported data encoding "%s"!', $type));
+      }
+
+      $this->rawData = $data;
+    }
+
+    return $this->rawData;
   }
 
 }