Support storage of Differential hunk data in Files

Summary:
Ref T12932. For long-lived installs, one of the largest tables tends to be the hunk data table. Although it doesn't grow tremendously fast, it's also well suited to storage in Files instead of the database (infrequent access, relatively large blobs of data, mostly one-at-a-time access), and earlier work anticipated eventually adding support for Files storage.

Make Files storage work, and provide `bin/differential migrate-hunk` to manually test/migrate hunks. This is currently the only way hunks get moved to file storage, but I expect to add a GC step which moves them to File storage after 30 days shortly.

The immediate motivation for this is to relieve storage pressure on db001/db002 so we have more headroom for deploying the Ferret engine and its larger indexes (see also T12819).

Test Plan:
  - Used `bin/differential migrate-hunk` to move a hunk to and from file storage, verified it survived intact.
  - Downloaded the actual stored file, sanity-checked it. Verified permissions.
  - Destroyed a diff with `bin/remove destroy`, saw the hunk and file storage destroyed.
  - Verified that going from file -> text destroys the old file properly with `migrate-hunk --trace ...`.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T12932

Differential Revision: https://secure.phabricator.com/D18584

src/__phutil_library_map__.php

@@ -2667,6 +2667,7 @@ phutil_register_library_map(array(
     'PhabricatorDifferentialConfigOptions' => 'applications/differential/config/PhabricatorDifferentialConfigOptions.php',
     'PhabricatorDifferentialExtractWorkflow' => 'applications/differential/management/PhabricatorDifferentialExtractWorkflow.php',
     'PhabricatorDifferentialManagementWorkflow' => 'applications/differential/management/PhabricatorDifferentialManagementWorkflow.php',
+    'PhabricatorDifferentialMigrateHunkWorkflow' => 'applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php',
     'PhabricatorDifferentialRevisionTestDataGenerator' => 'applications/differential/lipsum/PhabricatorDifferentialRevisionTestDataGenerator.php',
     'PhabricatorDiffusionApplication' => 'applications/diffusion/application/PhabricatorDiffusionApplication.php',
     'PhabricatorDiffusionBlameSetting' => 'applications/settings/setting/PhabricatorDiffusionBlameSetting.php',
@@ -5367,6 +5368,7 @@ phutil_register_library_map(array(
     'DifferentialChangeset' => array(
       'DifferentialDAO',
       'PhabricatorPolicyInterface',
+      'PhabricatorDestructibleInterface',
     ),
     'DifferentialChangesetDetailView' => 'AphrontView',
     'DifferentialChangesetFileTreeSideNavBuilder' => 'Phobject',
@@ -5461,6 +5463,7 @@ phutil_register_library_map(array(
     'DifferentialHunk' => array(
       'DifferentialDAO',
       'PhabricatorPolicyInterface',
+      'PhabricatorDestructibleInterface',
     ),
     'DifferentialHunkParser' => 'Phobject',
     'DifferentialHunkParserTestCase' => 'PhabricatorTestCase',
@@ -7999,6 +8002,7 @@ phutil_register_library_map(array(
     'PhabricatorDifferentialConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorDifferentialExtractWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
     'PhabricatorDifferentialManagementWorkflow' => 'PhabricatorManagementWorkflow',
+    'PhabricatorDifferentialMigrateHunkWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
     'PhabricatorDifferentialRevisionTestDataGenerator' => 'PhabricatorTestDataGenerator',
     'PhabricatorDiffusionApplication' => 'PhabricatorApplication',
     'PhabricatorDiffusionBlameSetting' => 'PhabricatorInternalSetting',

src/applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php

@@ -0,0 +1,86 @@
+<?php
+
+final class PhabricatorDifferentialMigrateHunkWorkflow
+  extends PhabricatorDifferentialManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('migrate-hunk')
+      ->setExamples('**migrate-hunk** --id __hunk__ --to __storage__')
+      ->setSynopsis(pht('Migrate storage engines for a hunk.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'id',
+            'param' => 'id',
+            'help' => pht('Hunk ID to migrate.'),
+          ),
+          array(
+            'name' => 'to',
+            'param' => 'storage',
+            'help' => pht('Storage engine to migrate to.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $id = $args->getArg('id');
+    if (!$id) {
+      throw new PhutilArgumentUsageException(
+        pht('Specify a hunk to migrate with --id.'));
+    }
+
+    $storage = $args->getArg('to');
+    switch ($storage) {
+      case DifferentialModernHunk::DATATYPE_TEXT:
+      case DifferentialModernHunk::DATATYPE_FILE:
+        break;
+      default:
+        throw new PhutilArgumentUsageException(
+          pht('Specify a hunk storage engine with --to.'));
+    }
+
+    $hunk = $this->loadHunk($id);
+    $old_data = $hunk->getChanges();
+
+    switch ($storage) {
+      case DifferentialModernHunk::DATATYPE_TEXT:
+        $hunk->saveAsText();
+        $this->logOkay(
+          pht('TEXT'),
+          pht('Convereted hunk to text storage.'));
+        break;
+      case DifferentialModernHunk::DATATYPE_FILE:
+        $hunk->saveAsFile();
+        $this->logOkay(
+          pht('FILE'),
+          pht('Convereted hunk to file storage.'));
+        break;
+    }
+
+    $hunk = $this->loadHunk($id);
+    $new_data = $hunk->getChanges();
+
+    if ($old_data !== $new_data) {
+      throw new Exception(
+        pht(
+          'Integrity check failed: new file data differs fom old data!'));
+    }
+
+    return 0;
+  }
+
+  private function loadHunk($id) {
+    $hunk = id(new DifferentialModernHunk())->load($id);
+    if (!$hunk) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'No hunk exists with ID "%s".',
+          $id));
+    }
+
+    return $hunk;
+  }
+
+
+}

src/applications/differential/storage/DifferentialChangeset.php

@@ -1,7 +1,10 @@
 <?php
 
-final class DifferentialChangeset extends DifferentialDAO
-  implements PhabricatorPolicyInterface {
+final class DifferentialChangeset
+  extends DifferentialDAO
+  implements
+    PhabricatorPolicyInterface,
+    PhabricatorDestructibleInterface {
 
   protected $diffID;
   protected $oldFile;
@@ -236,4 +239,25 @@ final class DifferentialChangeset extends DifferentialDAO
     return $this->getDiff()->hasAutomaticCapability($capability, $viewer);
   }
 
+
+/* -(  PhabricatorDestructibleInterface  )----------------------------------- */
+
+
+  public function destroyObjectPermanently(
+    PhabricatorDestructionEngine $engine) {
+    $this->openTransaction();
+
+      $hunks = id(new DifferentialModernHunk())->loadAllWhere(
+        'changesetID = %d',
+        $this->getID());
+      foreach ($hunks as $hunk) {
+        $engine->destroyObject($hunk);
+      }
+
+      $this->delete();
+
+    $this->saveTransaction();
+  }
+
+
 }

src/applications/differential/storage/DifferentialDiff.php

@@ -727,7 +727,7 @@ final class DifferentialDiff
       $this->delete();
 
       foreach ($this->loadChangesets() as $changeset) {
-        $changeset->delete();
+        $engine->destroyObject($changeset);
       }
 
       $properties = id(new DifferentialDiffProperty())->loadAllWhere(

src/applications/differential/storage/DifferentialHunk.php

@@ -1,7 +1,10 @@
 <?php
 
-abstract class DifferentialHunk extends DifferentialDAO
-  implements PhabricatorPolicyInterface {
+abstract class DifferentialHunk
+  extends DifferentialDAO
+  implements
+    PhabricatorPolicyInterface,
+    PhabricatorDestructibleInterface {
 
   protected $changesetID;
   protected $oldOffset;
@@ -228,4 +231,14 @@ abstract class DifferentialHunk extends DifferentialDAO
     return $this->getChangeset()->hasAutomaticCapability($capability, $viewer);
   }
 
+
+/* -(  PhabricatorDestructibleInterface  )----------------------------------- */
+
+
+  public function destroyObjectPermanently(
+    PhabricatorDestructionEngine $engine) {
+    $this->delete();
+  }
+
+
 }

src/applications/differential/storage/DifferentialModernHunk.php

@@ -15,6 +15,7 @@ final class DifferentialModernHunk extends DifferentialHunk {
 
   private $rawData;
   private $forcedEncoding;
+  private $fileData;
 
   public function getTableName() {
     return 'differential_hunk_modern';
@@ -87,6 +88,57 @@ final class DifferentialModernHunk extends DifferentialHunk {
     return parent::save();
   }
 
+  public function saveAsText() {
+    $old_type = $this->getDataType();
+    $old_data = $this->getData();
+
+    if ($old_type == self::DATATYPE_TEXT) {
+      return $this;
+    }
+
+    $raw_data = $this->getRawData();
+
+    $this->setDataType(self::DATATYPE_TEXT);
+    $this->setData($raw_data);
+
+    $result = $this->save();
+
+    $this->destroyData($old_type, $old_data);
+
+    return $result;
+  }
+
+  public function saveAsFile() {
+    $old_type = $this->getDataType();
+    $old_data = $this->getData();
+
+    if ($old_type == self::DATATYPE_FILE) {
+      return $this;
+    }
+
+    $raw_data = $this->getRawData();
+
+    $file = PhabricatorFile::newFromFileData(
+      $raw_data,
+      array(
+        'name' => 'differential-hunk',
+        'mime-type' => 'application/octet-stream',
+        'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
+      ));
+
+    $this->setDataType(self::DATATYPE_FILE);
+    $this->setData($file->getPHID());
+
+    // NOTE: Because hunks don't have a PHID and we just load hunk data with
+    // the ominipotent viewer, we do not need to attach the file to anything.
+
+    $result = $this->save();
+
+    $this->destroyData($old_type, $old_data);
+
+    return $result;
+  }
+
   private function getRawData() {
     if ($this->rawData === null) {
       $type = $this->getDataType();
@@ -98,6 +150,8 @@ final class DifferentialModernHunk extends DifferentialHunk {
           $data = $data;
           break;
         case self::DATATYPE_FILE:
+          $data = $this->loadFileData();
+          break;
         default:
           throw new Exception(
             pht('Hunk has unsupported data type "%s"!', $type));
@@ -123,4 +177,75 @@ final class DifferentialModernHunk extends DifferentialHunk {
     return $this->rawData;
   }
 
+  private function loadFileData() {
+    if ($this->fileData === null) {
+      $type = $this->getDataType();
+      if ($type !== self::DATATYPE_FILE) {
+        throw new Exception(
+          pht(
+            'Unable to load file data for hunk with wrong data type ("%s").',
+            $type));
+      }
+
+      $file_phid = $this->getData();
+
+      $file = $this->loadRawFile($file_phid);
+      $data = $file->loadFileData();
+
+      $this->fileData = $data;
+    }
+
+    return $this->fileData;
+  }
+
+  private function loadRawFile($file_phid) {
+    $viewer = PhabricatorUser::getOmnipotentUser();
+
+
+    $files = id(new PhabricatorFileQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($file_phid))
+      ->execute();
+    if (!$files) {
+      throw new Exception(
+        pht(
+          'Failed to load file ("%s") with hunk data.',
+          $file_phid));
+    }
+
+    $file = head($files);
+
+    return $file;
+  }
+
+
+  public function destroyObjectPermanently(
+    PhabricatorDestructionEngine $engine) {
+
+    $type = $this->getDataType();
+    $data = $this->getData();
+
+    $this->destroyData($type, $data, $engine);
+
+    return parent::destroyObjectPermanently($engine);
+  }
+
+
+  private function destroyData(
+    $type,
+    $data,
+    PhabricatorDestructionEngine $engine = null) {
+
+    if (!$engine) {
+      $engine = new PhabricatorDestructionEngine();
+    }
+
+    switch ($type) {
+      case self::DATATYPE_FILE:
+        $file = $this->loadRawFile($data);
+        $engine->destroyObject($file);
+        break;
+    }
+  }
+
 }