Add an AES256 storage format for at-rest encryption

Summary:
Ref T11140. This doesn't do anything yet since there's no way to enable it and no way to store master keys.

Those are slightly tougher problems and I'm not totally satisfied that I have an approach I really like for either problem, so I may wait for a bit before tackling them. Once they're solved, this does the mechanical encrypt/decrypt stuff, though.

This design is substantially similar to the AWS S3 server-side encryption design, and intended as an analog for it. The decisions AWS has made in design generally seem reasonable to me.

Each block of file data is encrypted with a unique key and a unique IV, and then that key and IV are encrypted with the master key (and a distinct, unique IV). This is better than just encrypting with the master key directly because:

  - You can rotate the master key later and only need to re-encrypt a small amount of key data (about 48 bytes per file chunk), instead of re-encrypting all of the actual file data (up to 4MB per file chunk).
  - Instead of putting the master key on every server, you can put it on some dedicated keyserver which accepts encrypted keys, decrypts them, and returns plaintext keys, and can send it 32-byte keys for decryption instead of 4MB blocks of file data.
  - You have to compromise the master key, the database, AND the file store to get the file data. This is probably not much of a barrier realistically, but it does make attacks very slightly harder.

The "KeyRing" thing may change once I figure out how I want users to store master keys, but it was the simplest approach to get the unit tests working.

Test Plan:
  - Ran unit tests.
  - Dumped raw data, saw encrypted blob.
  - No way to actually use this in the real application yet so it can't be tested too extensively.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11140

Differential Revision: https://secure.phabricator.com/D16124

src/__phutil_library_map__.php

@@ -2468,6 +2468,7 @@ phutil_register_library_map(array(
     'PhabricatorFeedStoryPublisher' => 'applications/feed/PhabricatorFeedStoryPublisher.php',
     'PhabricatorFeedStoryReference' => 'applications/feed/storage/PhabricatorFeedStoryReference.php',
     'PhabricatorFile' => 'applications/files/storage/PhabricatorFile.php',
+    'PhabricatorFileAES256StorageFormat' => 'applications/files/format/PhabricatorFileAES256StorageFormat.php',
     'PhabricatorFileBundleLoader' => 'applications/files/query/PhabricatorFileBundleLoader.php',
     'PhabricatorFileChunk' => 'applications/files/storage/PhabricatorFileChunk.php',
     'PhabricatorFileChunkIterator' => 'applications/files/engine/PhabricatorFileChunkIterator.php',
@@ -7102,6 +7103,7 @@ phutil_register_library_map(array(
       'PhabricatorPolicyInterface',
       'PhabricatorDestructibleInterface',
     ),
+    'PhabricatorFileAES256StorageFormat' => 'PhabricatorFileStorageFormat',
     'PhabricatorFileBundleLoader' => 'Phobject',
     'PhabricatorFileChunk' => array(
       'PhabricatorFileDAO',

src/applications/files/format/PhabricatorFileAES256StorageFormat.php

@@ -0,0 +1,176 @@
+<?php
+
+/**
+ * At-rest encryption format using AES256 CBC.
+ */
+final class PhabricatorFileAES256StorageFormat
+  extends PhabricatorFileStorageFormat {
+
+  const FORMATKEY = 'aes-256-cbc';
+
+  private $keyName;
+  private static $keyRing = array();
+
+  public function getStorageFormatName() {
+    return pht('Encrypted (AES-256-CBC)');
+  }
+
+  public function newReadIterator($raw_iterator) {
+    $file = $this->getFile();
+    $data = $file->loadDataFromIterator($raw_iterator);
+
+    list($key, $iv) = $this->extractKeyAndIV($file);
+
+    $data = $this->decryptData($data, $key, $iv);
+
+    return array($data);
+  }
+
+  public function newWriteIterator($raw_iterator) {
+    $file = $this->getFile();
+    $data = $file->loadDataFromIterator($raw_iterator);
+
+    list($key, $iv) = $this->extractKeyAndIV($file);
+
+    $data = $this->encryptData($data, $key, $iv);
+
+    return array($data);
+  }
+
+  public function newStorageProperties() {
+    // Generate a unique key and IV for this block of data.
+    $key_envelope = self::newAES256Key();
+    $iv_envelope = self::newAES256IV();
+
+    // Encode the raw binary data with base64 so we can wrap it in JSON.
+    $data = array(
+      'iv.base64' => base64_encode($iv_envelope->openEnvelope()),
+      'key.base64' => base64_encode($key_envelope->openEnvelope()),
+    );
+
+    // Encode the base64 data with JSON.
+    $data_clear = phutil_json_encode($data);
+
+    // Encrypt the block key with the master key, using a unique IV.
+    $data_iv = self::newAES256IV();
+    $key_name = $this->getMasterKeyName();
+    $master_key = self::getMasterKeyFromKeyRing($key_name);
+    $data_cipher = $this->encryptData($data_clear, $master_key, $data_iv);
+
+    return array(
+      'key.name' => $key_name,
+      'iv.base64' => base64_encode($data_iv->openEnvelope()),
+      'payload.base64' => base64_encode($data_cipher),
+    );
+  }
+
+  private function extractKeyAndIV(PhabricatorFile $file) {
+    $outer_iv = $file->getStorageProperty('iv.base64');
+    $outer_iv = base64_decode($outer_iv);
+    $outer_iv = new PhutilOpaqueEnvelope($outer_iv);
+
+    $outer_payload = $file->getStorageProperty('payload.base64');
+    $outer_payload = base64_decode($outer_payload);
+
+    $outer_key_name = $file->getStorageProperty('key.name');
+    $outer_key = self::getMasterKeyFromKeyRing($outer_key_name);
+
+    $payload = $this->decryptData($outer_payload, $outer_key, $outer_iv);
+    $payload = phutil_json_decode($payload);
+
+    $inner_iv = $payload['iv.base64'];
+    $inner_iv = base64_decode($inner_iv);
+    $inner_iv = new PhutilOpaqueEnvelope($inner_iv);
+
+    $inner_key = $payload['key.base64'];
+    $inner_key = base64_decode($inner_key);
+    $inner_key = new PhutilOpaqueEnvelope($inner_key);
+
+    return array($inner_key, $inner_iv);
+  }
+
+  private function encryptData(
+    $data,
+    PhutilOpaqueEnvelope $key,
+    PhutilOpaqueEnvelope $iv) {
+
+    $method = 'aes-256-cbc';
+    $key = $key->openEnvelope();
+    $iv = $iv->openEnvelope();
+
+    $result = openssl_encrypt($data, $method, $key, OPENSSL_RAW_DATA, $iv);
+    if ($result === false) {
+      throw new Exception(
+        pht(
+          'Failed to openssl_encrypt() data: %s',
+          openssl_error_string()));
+    }
+
+    return $result;
+  }
+
+  private function decryptData(
+    $data,
+    PhutilOpaqueEnvelope $key,
+    PhutilOpaqueEnvelope $iv) {
+
+    $method = 'aes-256-cbc';
+    $key = $key->openEnvelope();
+    $iv = $iv->openEnvelope();
+
+    $result = openssl_decrypt($data, $method, $key, OPENSSL_RAW_DATA, $iv);
+    if ($result === false) {
+      throw new Exception(
+        pht(
+          'Failed to openssl_decrypt() data: %s',
+          openssl_error_string()));
+    }
+
+    return $result;
+  }
+
+  public static function newAES256Key() {
+    // Unsurprisingly, AES256 uses a 256 bit key.
+    $key = Filesystem::readRandomBytes(phutil_units('256 bits in bytes'));
+    return new PhutilOpaqueEnvelope($key);
+  }
+
+  public static function newAES256IV() {
+    // AES256 uses a 256 bit key, but the initialization vector length is
+    // only 128 bits.
+    $iv = Filesystem::readRandomBytes(phutil_units('128 bits in bytes'));
+    return new PhutilOpaqueEnvelope($iv);
+  }
+
+  public function selectKey($key_name) {
+    // Require that the key exist on the key ring.
+    self::getMasterKeyFromKeyRing($key_name);
+
+    $this->keyName = $key_name;
+    return $this;
+  }
+
+  public static function addKeyToKeyRing($name, PhutilOpaqueEnvelope $key) {
+    self::$keyRing[$name] = $key;
+  }
+
+  private function getMasterKeyName() {
+    if ($this->keyName === null) {
+      throw new Exception(pht('No master key selected for AES256 storage.'));
+    }
+
+    return $this->keyName;
+  }
+
+  private static function getMasterKeyFromKeyRing($key_name) {
+    if (!isset(self::$keyRing[$key_name])) {
+      throw new Exception(
+        pht(
+          'No master key "%s" exists in key ring for AES256 storage.',
+          $key_name));
+    }
+
+    return self::$keyRing[$key_name];
+  }
+
+}

src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php

@@ -35,4 +35,38 @@ final class PhabricatorFileStorageFormatTestCase extends PhabricatorTestCase {
     $this->assertEqual($expect, $raw_data);
   }
 
+  public function testAES256Storage() {
+    $engine = new PhabricatorTestStorageEngine();
+
+    $key_name = 'test.abcd';
+    $key_text = new PhutilOpaqueEnvelope('abcdefghijklmnopABCDEFGHIJKLMNOP');
+
+    PhabricatorFileAES256StorageFormat::addKeyToKeyRing($key_name, $key_text);
+
+    $format = id(new PhabricatorFileAES256StorageFormat())
+      ->selectKey($key_name);
+
+    $data = 'The cow jumped over the full moon.';
+
+    $params = array(
+      'name' => 'test.dat',
+      'storageEngines' => array(
+        $engine,
+      ),
+      'format' => $format,
+    );
+
+    $file = PhabricatorFile::newFromFileData($data, $params);
+
+    // We should have a file stored as AES256.
+    $format_key = $format->getStorageFormatKey();
+    $this->assertEqual($format_key, $file->getStorageFormat());
+    $this->assertEqual($data, $file->loadFileData());
+
+    // The actual raw data in the storage engine should be encrypted. We
+    // can't really test this, but we can make sure it's not the same as the
+    // input data.
+    $raw_data = $engine->readFile($file->getStorageHandle());
+    $this->assertTrue($data !== $raw_data);
+  }
 }

src/applications/files/storage/PhabricatorFile.php

@@ -327,10 +327,17 @@ final class PhabricatorFile extends PhabricatorFileDAO
     $file = self::initializeNewFile();
 
     $default_key = PhabricatorFileRawStorageFormat::FORMATKEY;
-    $format_key = idx($params, 'format', $default_key);
+    $key = idx($params, 'format', $default_key);
 
-    $format = id(clone PhabricatorFileStorageFormat::requireFormat($format_key))
-      ->setFile($file);
+    // Callers can pass in an object explicitly instead of a key. This is
+    // primarily useful for unit tests.
+    if ($key instanceof PhabricatorFileStorageFormat) {
+      $format = clone $key;
+    } else {
+      $format = clone PhabricatorFileStorageFormat::requireFormat($key);
+    }
+
+    $format->setFile($file);
 
     $properties = $format->newStorageProperties();
     $file->setStorageFormat($format->getStorageFormatKey());