Add an alternate "modern" hunk datastore

Summary:
Ref T4045. Ref T5179. Hunk storage has two major issues:

  - It's utf8, but actual diffs are binary.
  - It's huge and can't be compressed or archived.

This introduces a second datastore which solves these problems: by recording hunk encoding, supporting compression, and supporting alternate storage. There's no actual compression or storage support yet, but there's space in the table for them.

Since nothing actually uses hunk IDs, it's fine to have these tables exist at the same time and use the same IDs. We can migrate data between the tables gradually without requiring downtime or disrupting installs.

Test Plan:
  - There are no writes to the new table yet.
  - The only effect this has is making us issue one extra query when looking for hunks.
  - Observed the query issue, but everything else continue working fine.
  - Created a new diff.
  - Ran unit tests.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T4045, T5179

Differential Revision: https://secure.phabricator.com/D9290

resources/sql/autopatches/20140525.hunkmodern.sql

@@ -0,0 +1,18 @@
+CREATE TABLE {$NAMESPACE}_differential.differential_hunk_modern (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  changesetID INT UNSIGNED NOT NULL,
+  oldOffset INT UNSIGNED NOT NULL,
+  oldLen INT UNSIGNED NOT NULL,
+  newOffset INT UNSIGNED NOT NULL,
+  newLen INT UNSIGNED NOT NULL,
+  dataType CHAR(4) NOT NULL COLLATE latin1_bin,
+  dataEncoding VARCHAR(16) COLLATE latin1_bin,
+  dataFormat CHAR(4) NOT NULL COLLATE latin1_bin,
+  data LONGBLOB NOT NULL,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+
+  KEY `key_changeset` (changesetID),
+  KEY `key_created` (dateCreated)
+
+) ENGINE=InnoDB, COLLATE utf8_general_ci;

src/__phutil_library_map__.php

@@ -385,6 +385,8 @@ phutil_register_library_map(array(
     'DifferentialHostField' => 'applications/differential/customfield/DifferentialHostField.php',
     'DifferentialHovercardEventListener' => 'applications/differential/event/DifferentialHovercardEventListener.php',
     'DifferentialHunk' => 'applications/differential/storage/DifferentialHunk.php',
+    'DifferentialHunkLegacy' => 'applications/differential/storage/DifferentialHunkLegacy.php',
+    'DifferentialHunkModern' => 'applications/differential/storage/DifferentialHunkModern.php',
     'DifferentialHunkParser' => 'applications/differential/parser/DifferentialHunkParser.php',
     'DifferentialHunkParserTestCase' => 'applications/differential/parser/__tests__/DifferentialHunkParserTestCase.php',
     'DifferentialHunkQuery' => 'applications/differential/query/DifferentialHunkQuery.php',
@@ -3069,6 +3071,8 @@ phutil_register_library_map(array(
       0 => 'DifferentialDAO',
       1 => 'PhabricatorPolicyInterface',
     ),
+    'DifferentialHunkLegacy' => 'DifferentialHunk',
+    'DifferentialHunkModern' => 'DifferentialHunk',
     'DifferentialHunkParserTestCase' => 'PhabricatorTestCase',
     'DifferentialHunkQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'DifferentialHunkTestCase' => 'ArcanistPhutilTestCase',

src/applications/differential/parser/__tests__/DifferentialChangesetParserTestCase.php

@@ -3,7 +3,7 @@
 final class DifferentialChangesetParserTestCase extends PhabricatorTestCase {
 
   public function testDiffChangesets() {
-    $hunk = new DifferentialHunk();
+    $hunk = new DifferentialHunkLegacy();
     $hunk->setChanges("+a\n b\n-c");
     $hunk->setNewOffset(1);
     $hunk->setNewLen(2);
@@ -20,7 +20,7 @@ final class DifferentialChangesetParserTestCase extends PhabricatorTestCase {
     );
 
     foreach ($tests as $changes => $expected) {
-      $hunk = new DifferentialHunk();
+      $hunk = new DifferentialHunkLegacy();
       $hunk->setChanges($changes);
       $hunk->setNewOffset(11);
       $hunk->setNewLen(3);

src/applications/differential/parser/__tests__/DifferentialHunkParserTestCase.php

@@ -14,7 +14,7 @@ final class DifferentialHunkParserTestCase extends PhabricatorTestCase {
     $new_len,
     $changes) {
 
-    $hunk = id(new DifferentialHunk())
+    $hunk = id(new DifferentialHunkLegacy())
       ->setOldOffset($old_offset)
       ->setOldLen($old_len)
       ->setNewOffset($new_offset)

src/applications/differential/query/DifferentialHunkQuery.php

@@ -31,18 +31,38 @@ final class DifferentialHunkQuery
   }
 
   public function loadPage() {
-    $table = new DifferentialHunk();
+    $all_results = array();
+
+    // Load modern hunks.
+    $table = new DifferentialHunkModern();
     $conn_r = $table->establishConnection('r');
 
-    $data = queryfx_all(
+    $modern_data = queryfx_all(
       $conn_r,
       'SELECT * FROM %T %Q %Q %Q',
       $table->getTableName(),
       $this->buildWhereClause($conn_r),
       $this->buildOrderClause($conn_r),
       $this->buildLimitClause($conn_r));
+    $modern_results = $table->loadAllFromArray($modern_data);
 
-    return $table->loadAllFromArray($data);
+
+    // Now, load legacy hunks.
+    $table = new DifferentialHunkLegacy();
+    $conn_r = $table->establishConnection('r');
+
+    $legacy_data = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T %Q %Q %Q',
+      $table->getTableName(),
+      $this->buildWhereClause($conn_r),
+      $this->buildOrderClause($conn_r),
+      $this->buildLimitClause($conn_r));
+    $legacy_results = $table->loadAllFromArray($legacy_data);
+
+    // Strip all the IDs off since they're not unique and nothing should be
+    // using them.
+    return array_values(array_merge($legacy_results, $modern_results));
   }
 
   public function willFilterPage(array $hunks) {

src/applications/differential/storage/DifferentialChangeset.php

@@ -76,7 +76,7 @@ final class DifferentialChangeset extends DifferentialDAO
   public function delete() {
     $this->openTransaction();
 
-      $hunks = id(new DifferentialHunk())->loadAllWhere(
+      $hunks = id(new DifferentialHunkLegacy())->loadAllWhere(
         'changesetID = %d',
         $this->getID());
       foreach ($hunks as $hunk) {

src/applications/differential/storage/DifferentialDiff.php

@@ -132,7 +132,7 @@ final class DifferentialDiff
       $hunks = $change->getHunks();
       if ($hunks) {
         foreach ($hunks as $hunk) {
-          $dhunk = new DifferentialHunk();
+          $dhunk = new DifferentialHunkLegacy();
           $dhunk->setOldOffset($hunk->getOldOffset());
           $dhunk->setOldLen($hunk->getOldLength());
           $dhunk->setNewOffset($hunk->getNewOffset());

src/applications/differential/storage/DifferentialHunk.php

@@ -1,10 +1,9 @@
 <?php
 
-final class DifferentialHunk extends DifferentialDAO
+abstract class DifferentialHunk extends DifferentialDAO
   implements PhabricatorPolicyInterface {
 
   protected $changesetID;
-  protected $changes;
   protected $oldOffset;
   protected $oldLen;
   protected $newOffset;

src/applications/differential/storage/DifferentialHunkLegacy.php

@@ -0,0 +1,11 @@
+<?php
+
+final class DifferentialHunkLegacy extends DifferentialHunk {
+
+  protected $changes;
+
+  public function getTableName() {
+    return 'differential_hunk';
+  }
+
+}

src/applications/differential/storage/DifferentialHunkModern.php

@@ -0,0 +1,73 @@
+<?php
+
+final class DifferentialHunkModern extends DifferentialHunk {
+
+  const DATATYPE_TEXT       = 'text';
+  const DATATYPE_FILE       = 'file';
+
+  const DATAFORMAT_RAW      = 'byte';
+  const DATAFORMAT_DEFLATE  = 'gzde';
+
+  protected $dataType;
+  protected $dataEncoding;
+  protected $dataFormat;
+  protected $data;
+
+  public function getTableName() {
+    return 'differential_hunk_modern';
+  }
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_BINARY => array(
+        'data' => true,
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function setChanges($text) {
+    $this->dataEncoding = $this->detectEncodingForStorage($text);
+    $this->dataType = self::DATATYPE_TEXT;
+    $this->dataFormat = self::DATAFORMAT_RAW;
+    $this->data = $text;
+
+    return $this;
+  }
+
+  public function getChanges() {
+    return $this->getUTF8StringFromStorage(
+      $this->getRawData(),
+      $this->getDataEncoding());
+  }
+
+  private function getRawData() {
+    $type = $this->getDataType();
+    $data = $this->getData();
+
+    switch ($type) {
+      case self::DATATYPE_TEXT:
+        // In this storage type, the changes are stored on the object.
+        $data = $data;
+        break;
+      case self::DATATYPE_FILE:
+      default:
+        throw new Exception(
+          pht('Hunk has unsupported data type "%s"!', $type));
+    }
+
+    $format = $this->getDataFormat();
+    switch ($format) {
+      case self::DATAFORMAT_RAW:
+        // In this format, the changes are stored as-is.
+        $data = $data;
+        break;
+      case self::DATAFORMAT_DEFLATE:
+      default:
+        throw new Exception(
+          pht('Hunk has unsupported data encoding "%s"!', $type));
+    }
+
+    return $data;
+  }
+
+}

src/applications/differential/storage/__tests__/DifferentialHunkTestCase.php

@@ -5,7 +5,7 @@ final class DifferentialHunkTestCase extends ArcanistPhutilTestCase {
   public function testMakeChanges() {
     $root = dirname(__FILE__).'/hunk/';
 
-    $hunk = new DifferentialHunk();
+    $hunk = new DifferentialHunkLegacy();
     $hunk->setChanges(Filesystem::readFile($root.'basic.diff'));
     $hunk->setOldOffset(1);
     $hunk->setNewOffset(11);
@@ -23,7 +23,7 @@ final class DifferentialHunkTestCase extends ArcanistPhutilTestCase {
     );
     $this->assertEqual($added, $hunk->getAddedLines());
 
-    $hunk = new DifferentialHunk();
+    $hunk = new DifferentialHunkLegacy();
     $hunk->setChanges(Filesystem::readFile($root.'newline.diff'));
     $hunk->setOldOffset(1);
     $hunk->setNewOffset(11);

src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php

@@ -382,8 +382,8 @@ abstract class PhabricatorRepositoryCommitMessageParserWorker
         //   -echo "test";
         //   -(empty line)
 
-        $hunk = id(new DifferentialHunk())->setChanges($context);
-        $vs_hunk = id(new DifferentialHunk())->setChanges($vs_context);
+        $hunk = id(new DifferentialHunkLegacy())->setChanges($context);
+        $vs_hunk = id(new DifferentialHunkLegacy())->setChanges($vs_context);
         if ($hunk->makeOldFile() != $vs_hunk->makeOldFile() ||
             $hunk->makeNewFile() != $vs_hunk->makeNewFile()) {
           return $vs_diff;