Add support to Files for file storage formats, to support encryption-at-rest

Summary:
Ref T11140. When reading and writing files, we optionally apply a "storage format" to them.

The default format is "raw", which means we just store the raw data.

This change modularizes formats and adds a "rot13" format, which proves formatting works and is testable. In the future, I'll add real encryption formats.

Test Plan:
  - Added unit tests.
  - Viewed files in web UI.
  - Changed a file's format to rot13, saw the data get rotated on display.
  - Set default format to rot13:
    - Uploaded a small file, verified data was stored as rot13.
    - Uploaded a large file, verified metadata was stored as "raw" (just a type, no actual data) and blob data was stored as rot13.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11140

Differential Revision: https://secure.phabricator.com/D16122

src/__phutil_library_map__.php

@@ -2490,12 +2490,16 @@ phutil_register_library_map(array(
     'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php',
     'PhabricatorFileListController' => 'applications/files/controller/PhabricatorFileListController.php',
     'PhabricatorFileQuery' => 'applications/files/query/PhabricatorFileQuery.php',
+    'PhabricatorFileROT13StorageFormat' => 'applications/files/format/PhabricatorFileROT13StorageFormat.php',
+    'PhabricatorFileRawStorageFormat' => 'applications/files/format/PhabricatorFileRawStorageFormat.php',
     'PhabricatorFileSchemaSpec' => 'applications/files/storage/PhabricatorFileSchemaSpec.php',
     'PhabricatorFileSearchEngine' => 'applications/files/query/PhabricatorFileSearchEngine.php',
     'PhabricatorFileStorageBlob' => 'applications/files/storage/PhabricatorFileStorageBlob.php',
     'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php',
     'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php',
     'PhabricatorFileStorageEngineTestCase' => 'applications/files/engine/__tests__/PhabricatorFileStorageEngineTestCase.php',
+    'PhabricatorFileStorageFormat' => 'applications/files/format/PhabricatorFileStorageFormat.php',
+    'PhabricatorFileStorageFormatTestCase' => 'applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php',
     'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php',
     'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php',
     'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php',
@@ -7134,12 +7138,16 @@ phutil_register_library_map(array(
     'PhabricatorFileLinkView' => 'AphrontView',
     'PhabricatorFileListController' => 'PhabricatorFileController',
     'PhabricatorFileQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorFileROT13StorageFormat' => 'PhabricatorFileStorageFormat',
+    'PhabricatorFileRawStorageFormat' => 'PhabricatorFileStorageFormat',
     'PhabricatorFileSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorFileSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO',
     'PhabricatorFileStorageConfigurationException' => 'Exception',
     'PhabricatorFileStorageEngine' => 'Phobject',
     'PhabricatorFileStorageEngineTestCase' => 'PhabricatorTestCase',
+    'PhabricatorFileStorageFormat' => 'Phobject',
+    'PhabricatorFileStorageFormatTestCase' => 'PhabricatorTestCase',
     'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector',
     'PhabricatorFileTestCase' => 'PhabricatorTestCase',
     'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator',

src/applications/files/controller/PhabricatorFileInfoController.php

@@ -256,8 +256,10 @@ final class PhabricatorFileInfoController extends PhabricatorFileController {
       $types[] = pht('Profile');
     }
 
-    $types = implode(', ', $types);
-    $finfo->addProperty(pht('Attributes'), $types);
+    if ($types) {
+      $types = implode(', ', $types);
+      $finfo->addProperty(pht('Attributes'), $types);
+    }
 
     $storage_properties = new PHUIPropertyListView();
     $box->addPropertyList($storage_properties, pht('Storage'));
@@ -266,9 +268,14 @@ final class PhabricatorFileInfoController extends PhabricatorFileController {
       pht('Engine'),
       $file->getStorageEngine());
 
-    $storage_properties->addProperty(
-      pht('Format'),
-      $file->getStorageFormat());
+    $format_key = $file->getStorageFormat();
+    $format = PhabricatorFileStorageFormat::getFormat($format_key);
+    if ($format) {
+      $format_name = $format->getStorageFormatName();
+    } else {
+      $format_name = pht('Unknown ("%s")', $format_key);
+    }
+    $storage_properties->addProperty(pht('Format'), $format_name);
 
     $storage_properties->addProperty(
       pht('Handle'),

src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php

@@ -174,7 +174,7 @@ final class PhabricatorChunkedFileStorageEngine
     return (4 * 1024 * 1024);
   }
 
-  public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
+  public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
     $chunks = id(new PhabricatorFileChunkQuery())
       ->setViewer(PhabricatorUser::getOmnipotentUser())
       ->withChunkHandles(array($file->getStorageHandle()))

src/applications/files/engine/PhabricatorFileStorageEngine.php

@@ -325,10 +325,10 @@ abstract class PhabricatorFileStorageEngine extends Phobject {
     return $engine->getChunkSize();
   }
 
-  public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
+  public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
     // The default implementation is trivial and just loads the entire file
     // upfront.
-    $data = $file->loadFileData();
+    $data = $this->readFile($file->getStorageHandle());
 
     if ($begin !== null && $end !== null) {
       $data = substr($data, $begin, ($end - $begin));

src/applications/files/format/PhabricatorFileROT13StorageFormat.php

@@ -0,0 +1,44 @@
+<?php
+
+/**
+ * Trivial example of a file storage format for at-rest encryption.
+ *
+ * This format applies ROT13 encoding to file data as it is stored and
+ * reverses it on the way out. This encoding is trivially reversible. This
+ * format is for testing, developing, and understanding encoding formats and
+ * is not intended for production use.
+ */
+final class PhabricatorFileROT13StorageFormat
+  extends PhabricatorFileStorageFormat {
+
+  const FORMATKEY = 'rot13';
+
+  public function getStorageFormatName() {
+    return pht('Encoded (ROT13)');
+  }
+
+  public function newReadIterator($raw_iterator) {
+    $file = $this->getFile();
+    $iterations = $file->getStorageProperty('iterations', 1);
+
+    $value = $file->loadDataFromIterator($raw_iterator);
+    for ($ii = 0; $ii < $iterations; $ii++) {
+      $value = str_rot13($value);
+    }
+
+    return array($value);
+  }
+
+  public function newWriteIterator($raw_iterator) {
+    return $this->newReadIterator($raw_iterator);
+  }
+
+  public function newStorageProperties() {
+    // For extreme security, repeatedly encode the data using a random (odd)
+    // number of iterations.
+    return array(
+      'iterations' => (mt_rand(1, 3) * 2) - 1,
+    );
+  }
+
+}

src/applications/files/format/PhabricatorFileRawStorageFormat.php

@@ -0,0 +1,20 @@
+<?php
+
+final class PhabricatorFileRawStorageFormat
+  extends PhabricatorFileStorageFormat {
+
+  const FORMATKEY = 'raw';
+
+  public function getStorageFormatName() {
+    return pht('Raw Data');
+  }
+
+  public function newReadIterator($raw_iterator) {
+    return $raw_iterator;
+  }
+
+  public function newWriteIterator($raw_iterator) {
+    return $raw_iterator;
+  }
+
+}

src/applications/files/format/PhabricatorFileStorageFormat.php

@@ -0,0 +1,58 @@
+<?php
+
+abstract class PhabricatorFileStorageFormat
+  extends Phobject {
+
+  private $file;
+
+  final public function setFile(PhabricatorFile $file) {
+    $this->file = $file;
+    return $this;
+  }
+
+  final public function getFile() {
+    if (!$this->file) {
+      throw new PhutilInvalidStateException('setFile');
+    }
+    return $this->file;
+  }
+
+  abstract public function getStorageFormatName();
+
+  abstract public function newReadIterator($raw_iterator);
+  abstract public function newWriteIterator($raw_iterator);
+
+  public function newStorageProperties() {
+    return array();
+  }
+
+  final public function getStorageFormatKey() {
+    return $this->getPhobjectClassConstant('FORMATKEY');
+  }
+
+  final public static function getAllFormats() {
+    return id(new PhutilClassMapQuery())
+      ->setAncestorClass(__CLASS__)
+      ->setUniqueMethod('getStorageFormatKey')
+      ->execute();
+  }
+
+  final public static function getFormat($key) {
+    $formats = self::getAllFormats();
+    return idx($formats, $key);
+  }
+
+  final public static function requireFormat($key) {
+    $format = self::getFormat($key);
+
+    if (!$format) {
+      throw new Exception(
+        pht(
+          'No file storage format with key "%s" exists.',
+          $key));
+    }
+
+    return $format;
+  }
+
+}

src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php

@@ -0,0 +1,38 @@
+<?php
+
+final class PhabricatorFileStorageFormatTestCase extends PhabricatorTestCase {
+
+  protected function getPhabricatorTestCaseConfiguration() {
+    return array(
+      self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
+    );
+  }
+
+  public function testRot13Storage() {
+    $engine = new PhabricatorTestStorageEngine();
+    $rot13_format = PhabricatorFileROT13StorageFormat::FORMATKEY;
+
+    $data = 'The cow jumped over the full moon.';
+    $expect = 'Gur pbj whzcrq bire gur shyy zbba.';
+
+    $params = array(
+      'name' => 'test.dat',
+      'storageEngines' => array(
+        $engine,
+      ),
+      'format' => $rot13_format,
+    );
+
+    $file = PhabricatorFile::newFromFileData($data, $params);
+
+    // We should have a file stored as rot13, which reads back the input
+    // data correctly.
+    $this->assertEqual($rot13_format, $file->getStorageFormat());
+    $this->assertEqual($data, $file->loadFileData());
+
+    // The actual raw data in the storage engine should be encoded.
+    $raw_data = $engine->readFile($file->getStorageHandle());
+    $this->assertEqual($expect, $raw_data);
+  }
+
+}

src/applications/files/storage/PhabricatorFile.php

@@ -26,14 +26,13 @@ final class PhabricatorFile extends PhabricatorFileDAO
     PhabricatorPolicyInterface,
     PhabricatorDestructibleInterface {
 
-  const STORAGE_FORMAT_RAW  = 'raw';
-
   const METADATA_IMAGE_WIDTH  = 'width';
   const METADATA_IMAGE_HEIGHT = 'height';
   const METADATA_CAN_CDN = 'canCDN';
   const METADATA_BUILTIN = 'builtin';
   const METADATA_PARTIAL = 'partial';
   const METADATA_PROFILE = 'profile';
+  const METADATA_STORAGE = 'storage';
 
   protected $name;
   protected $mimeType;
@@ -233,10 +232,10 @@ final class PhabricatorFile extends PhabricatorFileDAO
       $hash);
 
     if ($file) {
-      // copy storageEngine, storageHandle, storageFormat
       $copy_of_storage_engine = $file->getStorageEngine();
       $copy_of_storage_handle = $file->getStorageHandle();
       $copy_of_storage_format = $file->getStorageFormat();
+      $copy_of_storage_properties = $file->getStorageProperties();
       $copy_of_byte_size = $file->getByteSize();
       $copy_of_mime_type = $file->getMimeType();
 
@@ -248,6 +247,7 @@ final class PhabricatorFile extends PhabricatorFileDAO
       $new_file->setStorageEngine($copy_of_storage_engine);
       $new_file->setStorageHandle($copy_of_storage_handle);
       $new_file->setStorageFormat($copy_of_storage_format);
+      $new_file->setStorageProperties($copy_of_storage_properties);
       $new_file->setMimeType($copy_of_mime_type);
       $new_file->copyDimensions($file);
 
@@ -290,7 +290,11 @@ final class PhabricatorFile extends PhabricatorFileDAO
 
     $file->setStorageEngine($engine->getEngineIdentifier());
     $file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
-    $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
+
+    // Chunked files are always stored raw because they do not actually store
+    // data. The chunks do, and can be individually formatted.
+    $file->setStorageFormat(PhabricatorFileRawStorageFormat::FORMATKEY);
+
     $file->setIsPartial(1);
 
     $file->readPropertiesFromParameters($params);
@@ -322,6 +326,16 @@ final class PhabricatorFile extends PhabricatorFileDAO
 
     $file = self::initializeNewFile();
 
+    $default_key = PhabricatorFileRawStorageFormat::FORMATKEY;
+    $format_key = idx($params, 'format', $default_key);
+
+    $format = id(clone PhabricatorFileStorageFormat::requireFormat($format_key))
+      ->setFile($file);
+
+    $properties = $format->newStorageProperties();
+    $file->setStorageFormat($format->getStorageFormatKey());
+    $file->setStorageProperties($properties);
+
     $data_handle = null;
     $engine_identifier = null;
     $exceptions = array();
@@ -361,10 +375,6 @@ final class PhabricatorFile extends PhabricatorFileDAO
     $file->setStorageEngine($engine_identifier);
     $file->setStorageHandle($data_handle);
 
-    // TODO: This is probably YAGNI, but allows for us to do encryption or
-    // compression later if we want.
-    $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
-
     $file->readPropertiesFromParameters($params);
 
     if (!$file->getMimeType()) {
@@ -434,7 +444,15 @@ final class PhabricatorFile extends PhabricatorFileDAO
 
     $engine_class = get_class($engine);
 
-    $data_handle = $engine->writeFile($data, $params);
+    $key = $this->getStorageFormat();
+    $format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
+      ->setFile($this);
+
+    $data_iterator = array($data);
+    $formatted_iterator = $format->newWriteIterator($data_iterator);
+    $formatted_data = $this->loadDataFromIterator($formatted_iterator);
+
+    $data_handle = $engine->writeFile($formatted_data, $params);
 
     if (!$data_handle || strlen($data_handle) > 255) {
       // This indicates an improperly implemented storage engine.
@@ -663,19 +681,8 @@ final class PhabricatorFile extends PhabricatorFileDAO
   }
 
   public function loadFileData() {
-
-    $engine = $this->instantiateStorageEngine();
-    $data = $engine->readFile($this->getStorageHandle());
-
-    switch ($this->getStorageFormat()) {
-      case self::STORAGE_FORMAT_RAW:
-        $data = $data;
-        break;
-      default:
-        throw new Exception(pht('Unknown storage format.'));
-    }
-
-    return $data;
+    $iterator = $this->getFileDataIterator();
+    return $this->loadDataFromIterator($iterator);
   }
 
 
@@ -688,7 +695,14 @@ final class PhabricatorFile extends PhabricatorFileDAO
    */
   public function getFileDataIterator($begin = null, $end = null) {
     $engine = $this->instantiateStorageEngine();
-    return $engine->getFileDataIterator($this, $begin, $end);
+    $raw_iterator = $engine->getRawFileDataIterator($this, $begin, $end);
+
+    $key = $this->getStorageFormat();
+
+    $format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
+      ->setFile($this);
+
+    return $format->newReadIterator($raw_iterator);
   }
 
 
@@ -917,6 +931,30 @@ final class PhabricatorFile extends PhabricatorFileDAO
     return Filesystem::readRandomCharacters(20);
   }
 
+  public function setStorageProperties(array $properties) {
+    $this->metadata[self::METADATA_STORAGE] = $properties;
+    return $this;
+  }
+
+  public function getStorageProperties() {
+    return idx($this->metadata, self::METADATA_STORAGE, array());
+  }
+
+  public function getStorageProperty($key, $default = null) {
+    $properties = $this->getStorageProperties();
+    return idx($properties, $key, $default);
+  }
+
+  public function loadDataFromIterator($iterator) {
+    $result = '';
+
+    foreach ($iterator as $chunk) {
+      $result .= $chunk;
+    }
+
+    return $result;
+  }
+
   public function updateDimensions($save = true) {
     if (!$this->isViewableImage()) {
       throw new Exception(pht('This file is not a viewable image.'));