From 67084a69533c861f34510f5f695b598077ec5632 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 15 Jun 2016 17:40:32 -0700 Subject: [PATCH] Support AES256 at-rest encryption in Files Summary: Ref T11140. This makes encryption actually work: - Provide a new configuation option, `keyring`, for specifying encryption keys. - One key may be marked as `default`. This activates AES256 encryption for Files. - Add `bin/files generate-key`. This is helps when generating valid encryption keys. - Add `bin/files encode`. This changes the storage encoding of a file, and helps test encodings and migrate existing data. - Add `bin/files cycle`. This re-encodes the block key with a new master key, if your master key leaks or you're just paraonid. - Document all these options and behaviors. Test Plan: - Configured a bad `keyring`, hit a bunch of different errors. - Used `bin/files generate-key` to try to generate bad keys, got appropriate errors ("raw doesn't support keys", etc). - Used `bin/files generate-key` to generate an AES256 key. - Put the new AES256 key into the `keyring`, without `default`. - Uploaded a new file, verified it still uploaded as raw data (no `default` key yet). - Used `bin/files encode` to change a file to ROT13 and back to raw. Verified old data got deleted and new data got stored properly. - Used `bin/files encode --key ...` to explicitly convert a file to AES256 with my non-default key. - Forced a re-encode of an AES256 file, verified the old data was deleted and a new key and IV were generated. - Used `bin/files cycle` to try to cycle raw/rot13 files, got errors. - Used `bin/files cycle` to cycle AES256 files. Verified metadata changed but file data did not. Verified file data was still decryptable with metadata. - Ran `bin/files cycle --all`. - Ran `encode` and `cycle` on chunked files, saw commands fail properly. These commands operate on the underlying data blocks, not the chunk metadata. - Set key to `default`, uploaded a file, saw it stored as AES256. - Read documentation. Reviewers: chad Reviewed By: chad Maniphest Tasks: T11140 Differential Revision: https://secure.phabricator.com/D16127 --- src/__phutil_library_map__.php | 10 + .../PhabricatorSecurityConfigOptions.php | 12 ++ .../PhabricatorFileAES256StorageFormat.php | 65 ++++-- .../format/PhabricatorFileStorageFormat.php | 23 ++ .../PhabricatorFileStorageFormatTestCase.php | 11 +- .../files/keyring/PhabricatorKeyring.php | 52 +++++ .../PhabricatorKeyringConfigOptionType.php | 111 ++++++++++ ...habricatorFilesManagementCycleWorkflow.php | 132 ++++++++++++ ...abricatorFilesManagementEncodeWorkflow.php | 151 ++++++++++++++ ...atorFilesManagementGenerateKeyWorkflow.php | 63 ++++++ .../files/storage/PhabricatorFile.php | 55 ++++- .../configuring_encryption.diviner | 196 ++++++++++++++++++ .../configuring_file_storage.diviner | 2 + 13 files changed, 858 insertions(+), 25 deletions(-) create mode 100644 src/applications/files/keyring/PhabricatorKeyring.php create mode 100644 src/applications/files/keyring/PhabricatorKeyringConfigOptionType.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php create mode 100644 src/applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php create mode 100644 src/docs/user/configuration/configuring_encryption.diviner diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index b1961900b1..79e7ebf8e5 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2524,7 +2524,10 @@ phutil_register_library_map(array( 'PhabricatorFilesConfigOptions' => 'applications/files/config/PhabricatorFilesConfigOptions.php', 'PhabricatorFilesManagementCatWorkflow' => 'applications/files/management/PhabricatorFilesManagementCatWorkflow.php', 'PhabricatorFilesManagementCompactWorkflow' => 'applications/files/management/PhabricatorFilesManagementCompactWorkflow.php', + 'PhabricatorFilesManagementCycleWorkflow' => 'applications/files/management/PhabricatorFilesManagementCycleWorkflow.php', + 'PhabricatorFilesManagementEncodeWorkflow' => 'applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php', 'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php', + 'PhabricatorFilesManagementGenerateKeyWorkflow' => 'applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php', 'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php', 'PhabricatorFilesManagementPurgeWorkflow' => 'applications/files/management/PhabricatorFilesManagementPurgeWorkflow.php', 'PhabricatorFilesManagementRebuildWorkflow' => 'applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php', @@ -2627,6 +2630,8 @@ phutil_register_library_map(array( 'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php', 'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php', 'PhabricatorKeyValueDatabaseCache' => 'applications/cache/PhabricatorKeyValueDatabaseCache.php', + 'PhabricatorKeyring' => 'applications/files/keyring/PhabricatorKeyring.php', + 'PhabricatorKeyringConfigOptionType' => 'applications/files/keyring/PhabricatorKeyringConfigOptionType.php', 'PhabricatorLDAPAuthProvider' => 'applications/auth/provider/PhabricatorLDAPAuthProvider.php', 'PhabricatorLegalpadApplication' => 'applications/legalpad/application/PhabricatorLegalpadApplication.php', 'PhabricatorLegalpadConfigOptions' => 'applications/legalpad/config/PhabricatorLegalpadConfigOptions.php', @@ -7173,7 +7178,10 @@ phutil_register_library_map(array( 'PhabricatorFilesConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorFilesManagementCatWorkflow' => 'PhabricatorFilesManagementWorkflow', 'PhabricatorFilesManagementCompactWorkflow' => 'PhabricatorFilesManagementWorkflow', + 'PhabricatorFilesManagementCycleWorkflow' => 'PhabricatorFilesManagementWorkflow', + 'PhabricatorFilesManagementEncodeWorkflow' => 'PhabricatorFilesManagementWorkflow', 'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow', + 'PhabricatorFilesManagementGenerateKeyWorkflow' => 'PhabricatorFilesManagementWorkflow', 'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow', 'PhabricatorFilesManagementPurgeWorkflow' => 'PhabricatorFilesManagementWorkflow', 'PhabricatorFilesManagementRebuildWorkflow' => 'PhabricatorFilesManagementWorkflow', @@ -7283,6 +7291,8 @@ phutil_register_library_map(array( 'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorJumpNavHandler' => 'Phobject', 'PhabricatorKeyValueDatabaseCache' => 'PhutilKeyValueCache', + 'PhabricatorKeyring' => 'Phobject', + 'PhabricatorKeyringConfigOptionType' => 'PhabricatorConfigJSONOptionType', 'PhabricatorLDAPAuthProvider' => 'PhabricatorAuthProvider', 'PhabricatorLegalpadApplication' => 'PhabricatorApplication', 'PhabricatorLegalpadConfigOptions' => 'PhabricatorApplicationConfigOptions', diff --git a/src/applications/config/option/PhabricatorSecurityConfigOptions.php b/src/applications/config/option/PhabricatorSecurityConfigOptions.php index 311c2d30b6..dde1d0d7be 100644 --- a/src/applications/config/option/PhabricatorSecurityConfigOptions.php +++ b/src/applications/config/option/PhabricatorSecurityConfigOptions.php @@ -43,6 +43,14 @@ final class PhabricatorSecurityConfigOptions '255.255.255.255/32', ); + $keyring_type = 'custom:PhabricatorKeyringConfigOptionType'; + $keyring_description = $this->deformat(pht(<<newOption('security.alternate-file-domain', 'string', null) ->setLocked(true) @@ -276,6 +284,10 @@ final class PhabricatorSecurityConfigOptions 'unsecured content over plain HTTP. It is very difficult to '. 'undo this change once users\' browsers have accepted the '. 'setting.')), + $this->newOption('keyring', $keyring_type, array()) + ->setHidden(true) + ->setSummary(pht('Configure master encryption keys.')) + ->setDescription($keyring_description), ); } diff --git a/src/applications/files/format/PhabricatorFileAES256StorageFormat.php b/src/applications/files/format/PhabricatorFileAES256StorageFormat.php index e0b02bdad1..6c8a0273bd 100644 --- a/src/applications/files/format/PhabricatorFileAES256StorageFormat.php +++ b/src/applications/files/format/PhabricatorFileAES256StorageFormat.php @@ -9,12 +9,31 @@ final class PhabricatorFileAES256StorageFormat const FORMATKEY = 'aes-256-cbc'; private $keyName; - private static $keyRing = array(); public function getStorageFormatName() { return pht('Encrypted (AES-256-CBC)'); } + public function canGenerateNewKeyMaterial() { + return true; + } + + public function generateNewKeyMaterial() { + $envelope = self::newAES256Key(); + $material = $envelope->openEnvelope(); + return base64_encode($material); + } + + public function canCycleMasterKey() { + return true; + } + + public function cycleStorageProperties() { + $file = $this->getFile(); + list($key, $iv) = $this->extractKeyAndIV($file); + return $this->formatStorageProperties($key, $iv); + } + public function newReadIterator($raw_iterator) { $file = $this->getFile(); $data = $file->loadDataFromIterator($raw_iterator); @@ -42,6 +61,13 @@ final class PhabricatorFileAES256StorageFormat $key_envelope = self::newAES256Key(); $iv_envelope = self::newAES256IV(); + return $this->formatStorageProperties($key_envelope, $iv_envelope); + } + + private function formatStorageProperties( + PhutilOpaqueEnvelope $key_envelope, + PhutilOpaqueEnvelope $iv_envelope) { + // Encode the raw binary data with base64 so we can wrap it in JSON. $data = array( 'iv.base64' => base64_encode($iv_envelope->openEnvelope()), @@ -54,7 +80,7 @@ final class PhabricatorFileAES256StorageFormat // Encrypt the block key with the master key, using a unique IV. $data_iv = self::newAES256IV(); $key_name = $this->getMasterKeyName(); - $master_key = self::getMasterKeyFromKeyRing($key_name); + $master_key = $this->getMasterKeyMaterial($key_name); $data_cipher = $this->encryptData($data_clear, $master_key, $data_iv); return array( @@ -73,7 +99,7 @@ final class PhabricatorFileAES256StorageFormat $outer_payload = base64_decode($outer_payload); $outer_key_name = $file->getStorageProperty('key.name'); - $outer_key = self::getMasterKeyFromKeyRing($outer_key_name); + $outer_key = $this->getMasterKeyMaterial($outer_key_name); $payload = $this->decryptData($outer_payload, $outer_key, $outer_iv); $payload = phutil_json_decode($payload); @@ -142,35 +168,32 @@ final class PhabricatorFileAES256StorageFormat return new PhutilOpaqueEnvelope($iv); } - public function selectKey($key_name) { + public function selectMasterKey($key_name) { // Require that the key exist on the key ring. - self::getMasterKeyFromKeyRing($key_name); + $this->getMasterKeyMaterial($key_name); $this->keyName = $key_name; return $this; } - public static function addKeyToKeyRing($name, PhutilOpaqueEnvelope $key) { - self::$keyRing[$name] = $key; - } - private function getMasterKeyName() { - if ($this->keyName === null) { - throw new Exception(pht('No master key selected for AES256 storage.')); + if ($this->keyName !== null) { + return $this->keyName; } - return $this->keyName; + $default = PhabricatorKeyring::getDefaultKeyName(self::FORMATKEY); + if ($default !== null) { + return $default; + } + + throw new Exception( + pht( + 'No AES256 key is specified in the keyring as a default encryption '. + 'key, and no encryption key has been explicitly selected.')); } - private static function getMasterKeyFromKeyRing($key_name) { - if (!isset(self::$keyRing[$key_name])) { - throw new Exception( - pht( - 'No master key "%s" exists in key ring for AES256 storage.', - $key_name)); - } - - return self::$keyRing[$key_name]; + private function getMasterKeyMaterial($key_name) { + return PhabricatorKeyring::getKey($key_name, self::FORMATKEY); } } diff --git a/src/applications/files/format/PhabricatorFileStorageFormat.php b/src/applications/files/format/PhabricatorFileStorageFormat.php index fd3480a81f..0d286ecd64 100644 --- a/src/applications/files/format/PhabricatorFileStorageFormat.php +++ b/src/applications/files/format/PhabricatorFileStorageFormat.php @@ -26,6 +26,29 @@ abstract class PhabricatorFileStorageFormat return array(); } + public function canGenerateNewKeyMaterial() { + return false; + } + + public function generateNewKeyMaterial() { + throw new PhutilMethodNotImplementedException(); + } + + public function canCycleMasterKey() { + return false; + } + + public function cycleStorageProperties() { + throw new PhutilMethodNotImplementedException(); + } + + public function selectMasterKey($key_name) { + throw new Exception( + pht( + 'This storage format ("%s") does not support key selection.', + $this->getStorageFormatName())); + } + final public function getStorageFormatKey() { return $this->getPhobjectClassConstant('FORMATKEY'); } diff --git a/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php index 25f68b83e1..18364fd2a4 100644 --- a/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php +++ b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php @@ -39,12 +39,17 @@ final class PhabricatorFileStorageFormatTestCase extends PhabricatorTestCase { $engine = new PhabricatorTestStorageEngine(); $key_name = 'test.abcd'; - $key_text = new PhutilOpaqueEnvelope('abcdefghijklmnopABCDEFGHIJKLMNOP'); + $key_text = 'abcdefghijklmnopABCDEFGHIJKLMNOP'; - PhabricatorFileAES256StorageFormat::addKeyToKeyRing($key_name, $key_text); + PhabricatorKeyring::addKey( + array( + 'name' => $key_name, + 'type' => 'aes-256-cbc', + 'material.base64' => base64_encode($key_text), + )); $format = id(new PhabricatorFileAES256StorageFormat()) - ->selectKey($key_name); + ->selectMasterKey($key_name); $data = 'The cow jumped over the full moon.'; diff --git a/src/applications/files/keyring/PhabricatorKeyring.php b/src/applications/files/keyring/PhabricatorKeyring.php new file mode 100644 index 0000000000..1772c145c2 --- /dev/null +++ b/src/applications/files/keyring/PhabricatorKeyring.php @@ -0,0 +1,52 @@ + $key) { + if (!empty($key['default'])) { + return $name; + } + } + + return null; + } + + private static function readConfiguration() { + if (self::$hasReadConfiguration) { + return true; + } + + self::$hasReadConfiguration = true; + + foreach (PhabricatorEnv::getEnvConfig('keyring') as $spec) { + self::addKey($spec); + } + } + +} diff --git a/src/applications/files/keyring/PhabricatorKeyringConfigOptionType.php b/src/applications/files/keyring/PhabricatorKeyringConfigOptionType.php new file mode 100644 index 0000000000..2550d17b44 --- /dev/null +++ b/src/applications/files/keyring/PhabricatorKeyringConfigOptionType.php @@ -0,0 +1,111 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Keyring configuration is not valid: each entry in the list must '. + 'be a dictionary describing an encryption key, but the value '. + 'with index "%s" is not a dictionary.', + $index)); + } + } + + + $map = array(); + $defaults = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'name' => 'string', + 'type' => 'string', + 'material.base64' => 'string', + 'default' => 'optional bool', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Keyring configuration has an invalid key specification (at '. + 'index "%s"): %s.', + $index, + $ex->getMessage())); + } + + $name = $spec['name']; + if (isset($map[$name])) { + throw new Exception( + pht( + 'Keyring configuration is invalid: it describes multiple keys '. + 'with the same name ("%s"). Each key must have a unique name.', + $name)); + } + $map[$name] = true; + + if (idx($spec, 'default')) { + $defaults[] = $name; + } + + $type = $spec['type']; + switch ($type) { + case 'aes-256-cbc': + if (!function_exists('openssl_encrypt')) { + throw new Exception( + pht( + 'Keyring is configured with a "%s" key, but the PHP OpenSSL '. + 'extension is not installed. Install the OpenSSL extension '. + 'to enable encryption.', + $type)); + } + + $material = $spec['material.base64']; + $material = base64_decode($material, true); + if ($material === false) { + throw new Exception( + pht( + 'Keyring specifies an invalid key ("%s"): key material '. + 'should be base64 encoded.', + $name)); + } + + if (strlen($material) != 32) { + throw new Exception( + pht( + 'Keyring specifies an invalid key ("%s"): key material '. + 'should be 32 bytes (256 bits) but has length %s.', + $name, + new PhutilNumber(strlen($material)))); + } + break; + default: + throw new Exception( + pht( + 'Keyring configuration is invalid: it describes a key with '. + 'type "%s", but this type is unknown.', + $type)); + } + } + + if (count($defaults) > 1) { + throw new Exception( + pht( + 'Keyring configuration is invalid: it describes multiple default '. + 'encryption keys. No more than one key may be the default key. '. + 'Keys currently configured as defaults: %s.', + implode(', ', $defaults))); + } + } + +} diff --git a/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php new file mode 100644 index 0000000000..6d574d633a --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php @@ -0,0 +1,132 @@ +setName('cycle') + ->setSynopsis( + pht('Cycle master key for encrypted files.')) + ->setArguments( + array( + array( + 'name' => 'key', + 'param' => 'keyname', + 'help' => pht('Select a specific storage key to cycle to.'), + ), + array( + 'name' => 'all', + 'help' => pht('Change encoding for all files.'), + ), + array( + 'name' => 'names', + 'wildcard' => true, + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $iterator = $this->buildIterator($args); + if (!$iterator) { + throw new PhutilArgumentUsageException( + pht( + 'Either specify a list of files to cycle, or use --all to cycle '. + 'all files.')); + } + + $format_map = PhabricatorFileStorageFormat::getAllFormats(); + $engines = PhabricatorFileStorageEngine::loadAllEngines(); + + $key_name = $args->getArg('key'); + + $failed = array(); + foreach ($iterator as $file) { + $monogram = $file->getMonogram(); + + $engine_key = $file->getStorageEngine(); + $engine = idx($engines, $engine_key); + + if (!$engine) { + echo tsprintf( + "%s\n", + pht( + '%s: Uses unknown storage engine "%s".', + $monogram, + $engine_key)); + $failed[] = $file; + continue; + } + + if ($engine->isChunkEngine()) { + echo tsprintf( + "%s\n", + pht( + '%s: Stored as chunks, declining to cycle directly.', + $monogram)); + continue; + } + + $format_key = $file->getStorageFormat(); + if (empty($format_map[$format_key])) { + echo tsprintf( + "%s\n", + pht( + '%s: Uses unknown storage format "%s".', + $monogram, + $format_key)); + $failed[] = $file; + continue; + } + + $format = clone $format_map[$format_key]; + $format->setFile($file); + + if (!$format->canCycleMasterKey()) { + echo tsprintf( + "%s\n", + pht( + '%s: Storage format ("%s") does not support key cycling.', + $monogram, + $format->getStorageFormatName())); + continue; + } + + echo tsprintf( + "%s\n", + pht( + '%s: Cycling master key.', + $monogram)); + + try { + if ($key_name) { + $format->selectMasterKey($key_name); + } + + $file->cycleMasterStorageKey($format); + + echo tsprintf( + "%s\n", + pht('Done.')); + } catch (Exception $ex) { + echo tsprintf( + "%B\n", + pht('Failed! %s', (string)$ex)); + $failed[] = $file; + } + } + + if ($failed) { + $monograms = mpull($failed, 'getMonogram'); + + echo tsprintf( + "%s\n", + pht('Failures: %s.', implode(', ', $monograms))); + + return 1; + } + + return 0; + } + +} diff --git a/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php new file mode 100644 index 0000000000..1d972326da --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php @@ -0,0 +1,151 @@ +setName('encode') + ->setSynopsis( + pht('Change the storage encoding of files.')) + ->setArguments( + array( + array( + 'name' => 'as', + 'param' => 'format', + 'help' => pht('Select the storage format to use.'), + ), + array( + 'name' => 'key', + 'param' => 'keyname', + 'help' => pht('Select a specific storage key.'), + ), + array( + 'name' => 'all', + 'help' => pht('Change encoding for all files.'), + ), + array( + 'name' => 'force', + 'help' => pht( + 'Re-encode files which are already stored in the target '. + 'encoding.'), + ), + array( + 'name' => 'names', + 'wildcard' => true, + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $iterator = $this->buildIterator($args); + if (!$iterator) { + throw new PhutilArgumentUsageException( + pht( + 'Either specify a list of files to encode, or use --all to '. + 'encode all files.')); + } + + $force = (bool)$args->getArg('force'); + + $format_list = PhabricatorFileStorageFormat::getAllFormats(); + $format_list = array_keys($format_list); + $format_list = implode(', ', $format_list); + + $format_key = $args->getArg('as'); + if (!strlen($format_key)) { + throw new PhutilArgumentUsageException( + pht( + 'Use --as to select a target encoding format. Available '. + 'formats are: %s.', + $format_list)); + } + + $format = PhabricatorFileStorageFormat::getFormat($format_key); + if (!$format) { + throw new PhutilArgumentUsageException( + pht( + 'Storage format "%s" is not valid. Available formats are: %s.', + $format_key, + $format_list)); + } + + $key_name = $args->getArg('key'); + if (strlen($key_name)) { + $format->selectMasterKey($key_name); + } + + $engines = PhabricatorFileStorageEngine::loadAllEngines(); + + $failed = array(); + foreach ($iterator as $file) { + $monogram = $file->getMonogram(); + + $engine_key = $file->getStorageEngine(); + $engine = idx($engines, $engine_key); + + if (!$engine) { + echo tsprintf( + "%s\n", + pht( + '%s: Uses unknown storage engine "%s".', + $monogram, + $engine_key)); + $failed[] = $file; + continue; + } + + if ($engine->isChunkEngine()) { + echo tsprintf( + "%s\n", + pht( + '%s: Stored as chunks, no data to encode directly.', + $monogram)); + continue; + } + + if (($file->getStorageFormat() == $format_key) && !$force) { + echo tsprintf( + "%s\n", + pht( + '%s: Already encoded in target format.', + $monogram)); + continue; + } + + echo tsprintf( + "%s\n", + pht( + '%s: Changing encoding from "%s" to "%s".', + $monogram, + $file->getStorageFormat(), + $format_key)); + + try { + $file->migrateToStorageFormat($format); + + echo tsprintf( + "%s\n", + pht('Done.')); + } catch (Exception $ex) { + echo tsprintf( + "%B\n", + pht('Failed! %s', (string)$ex)); + $failed[] = $file; + } + } + + if ($failed) { + $monograms = mpull($failed, 'getMonogram'); + + echo tsprintf( + "%s\n", + pht('Failures: %s.', implode(', ', $monograms))); + + return 1; + } + + return 0; + } + +} diff --git a/src/applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php new file mode 100644 index 0000000000..710c3586a6 --- /dev/null +++ b/src/applications/files/management/PhabricatorFilesManagementGenerateKeyWorkflow.php @@ -0,0 +1,63 @@ +setName('generate-key') + ->setSynopsis( + pht('Generate an encryption key.')) + ->setArguments( + array( + array( + 'name' => 'type', + 'param' => 'keytype', + 'help' => pht('Select the type of key to generate.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $type = $args->getArg('type'); + if (!strlen($type)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify the type of key to generate with --type.')); + } + + $format = PhabricatorFileStorageFormat::getFormat($type); + if (!$format) { + throw new PhutilArgumentUsageException( + pht( + 'No key type "%s" exists.', + $type)); + } + + if (!$format->canGenerateNewKeyMaterial()) { + throw new PhutilArgumentUsageException( + pht( + 'Storage format "%s" can not generate keys.', + $format->getStorageFormatName())); + } + + $material = $format->generateNewKeyMaterial(); + + $structure = array( + 'name' => 'generated-key-'.Filesystem::readRandomCharacters(12), + 'type' => $type, + 'material.base64' => $material, + ); + + $json = id(new PhutilJSON())->encodeFormatted($structure); + + echo tsprintf( + "%s: %s\n\n%B\n", + pht('Key Material'), + $format->getStorageFormatName(), + $json); + + return 0; + } + +} diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php index ea4d3845e4..4e43e3dd18 100644 --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -326,7 +326,13 @@ final class PhabricatorFile extends PhabricatorFileDAO $file = self::initializeNewFile(); - $default_key = PhabricatorFileRawStorageFormat::FORMATKEY; + $aes_type = PhabricatorFileAES256StorageFormat::FORMATKEY; + $has_aes = PhabricatorKeyring::getDefaultKeyName($aes_type); + if ($has_aes !== null) { + $default_key = PhabricatorFileAES256StorageFormat::FORMATKEY; + } else { + $default_key = PhabricatorFileRawStorageFormat::FORMATKEY; + } $key = idx($params, 'format', $default_key); // Callers can pass in an object explicitly instead of a key. This is @@ -444,6 +450,53 @@ final class PhabricatorFile extends PhabricatorFileDAO return $this; } + public function migrateToStorageFormat(PhabricatorFileStorageFormat $format) { + if (!$this->getID() || !$this->getStorageHandle()) { + throw new Exception( + pht("You can not migrate a file which hasn't yet been saved.")); + } + + $data = $this->loadFileData(); + $params = array( + 'name' => $this->getName(), + ); + + $engine = $this->instantiateStorageEngine(); + $old_handle = $this->getStorageHandle(); + + $properties = $format->newStorageProperties(); + $this->setStorageFormat($format->getStorageFormatKey()); + $this->setStorageProperties($properties); + + list($identifier, $new_handle) = $this->writeToEngine( + $engine, + $data, + $params); + + $this->setStorageHandle($new_handle); + $this->save(); + + $this->deleteFileDataIfUnused( + $engine, + $identifier, + $old_handle); + + return $this; + } + + public function cycleMasterStorageKey(PhabricatorFileStorageFormat $format) { + if (!$this->getID() || !$this->getStorageHandle()) { + throw new Exception( + pht("You can not cycle keys for a file which hasn't yet been saved.")); + } + + $properties = $format->cycleStorageProperties(); + $this->setStorageProperties($properties); + $this->save(); + + return $this; + } + private function writeToEngine( PhabricatorFileStorageEngine $engine, $data, diff --git a/src/docs/user/configuration/configuring_encryption.diviner b/src/docs/user/configuration/configuring_encryption.diviner new file mode 100644 index 0000000000..1e62071509 --- /dev/null +++ b/src/docs/user/configuration/configuring_encryption.diviner @@ -0,0 +1,196 @@ +@title Configuring Encryption +@group config + +Setup guide for configuring encryption. + +Overview +======== + +Phabricator supports at-rest encryption of uploaded file data stored in the +"Files" application. + +Configuring at-rest file data encryption does not encrypt any other data or +resources. In particular, it does not encrypt the database and does not encrypt +Passphrase credentials. + +Attackers who compromise a Phabricator host can read the master key and decrypt +the data. In most configurations, this does not represent a significant +barrier above and beyond accessing the file data. Thus, configuring at-rest +encryption is primarily useful for two types of installs: + + - If you maintain your own webserver and database hardware but want to use + Amazon S3 or a similar cloud provider as a blind storage server, file data + encryption can let you do so without needing to trust the cloud provider. + - If you face a regulatory or compliance need to encrypt data at rest but do + not need to actually secure this data, encrypting the data and placing the + master key in plaintext next to it may satisfy compliance requirements. + +The remainder of this document discusses how to configure at-rest encryption. + + +Quick Start +=========== + +To configure encryption, you will generally follow these steps: + + - Generate a master key with `bin/files generate-key`. + - Add the master key it to the `keyring`, but don't mark it as `default` yet. + - Use `bin/files encode ...` to test encrypting a few files. + - Mark the key as `default` to automatically encrypt new files. + - Use `bin/files encode --all ...` to encrypt any existing files. + +See the following sections for detailed guidance on these steps. + + +Configuring a Keyring +===================== + +To configure a keyring, set `keyring` with `bin/config` or by using another +configuration source. This option should be a list of keys in this format: + +```lang=json +... +"keyring": [ + { + "name": "master.key", + "type": "aes-256-cbc", + "material.base64": "UcHUJqq8MhZRwhvDV8sJwHj7bNJoM4tWfOIi..." + "default": true + }, + ... +] +... +``` + +Each key should have these properties: + + - `name`: //Required string.// A unique key name. + - `type`: //Required string.// Type of the key. Only `aes-256-cbc` is + supported. + - `material.base64`: //Required string.// The key material. See below for + details. + - `default`: //Optional bool.// Optionally, mark exactly one key as the + default key to enable encryption of newly uploaded file data. + +The key material is sensitive an an attacker who learns it can decrypt data +from the storage engine. + + +Format: Raw Data +================ + +The `raw` storage format is automatically selected for all newly uploaded +file data if no key is makred as the `default` key in the keyring. This is +the behavior of Phabricator if you haven't configured anything. + +This format stores raw data without modification. + + +Format: AES256 +============== + +The `aes-256-cbc` storage format is automatically selected for all newly +uploaded file data if an AES256 key is marked as the `default` key in the +keyring. + +This format uses AES256 in CBC mode. Each block of file data is encrypted with +a unique, randomly generated private key. That key is then encrypted with the +master key. Among other motivations, this strategy allows the master key to be +cycled relatively cheaply later (see "Cycling Master Keys" below). + +AES256 keys should be randomly generated and 256 bits (32 characters) in +length, then base64 encoded when represented in `keyring`. + +You can generate a valid, properly encoded AES256 master key with this command: + +``` +phabricator/ $ ./bin/files generate-key --type aes-256-cbc +``` + +This mode is generally similar to the default server-side encryption mode +supported by Amazon S3. + + +Format: ROT13 +============= + +The `rot13` format is a test format that is never selected by default. You can +select this format explicitly with `bin/files encode` to test storage and +encryption behavior. + +This format applies ROT13 encoding to file data. + + +Changing File Storage Formats +============================= + +To test configuration, you can explicitly change the storage format of a file. + +This will read the file data, decrypt it if necessary, write a new copy of the +data with the desired encryption, then update the file to point at the new +data. You can use this to make sure encryption works before turning it on by +default. + +To change the format of an individual file, run this command: + +``` +phabricator/ $ ./bin/files encode --as F123 [--key ] +``` + +This will change the storage format of the sepcified file. + + +Verifying Storage Formats +========================= + +You can review the storage format of a file from the web UI, in the +{nav Storage} tab under "Format". You can also use the "Engine" and "Handle" +properties to identify where the underlying data is stored and verify that +it is encrypted or encoded in the way you expect. + +See @{article:Configuring File Storage} for more information on storage +engines. + + +Cycling Master Keys +=================== + +If you need to cycle your master key, some storage formats support key cycling. + +Cycling a file's encryption key decodes the local key for the data using the +old master key, then re-encodes it using the new master key. This is primarily +useful if you believe your master key may have been compromised. + +First, add a new key to the keyring and mark it as the default key. You need +to leave the old key in place for now so existing data can be decrypted. + +To cycle an individual file, run this command: + +``` +phabricator/ $ ./bin/files cycle F123 +``` + +Verify that cycling worked properly by examining the command output and +accessing the file to check that the data is present and decryptable. You +can cycle additional files to gain additional confidence. + +You can cycle all files with this command: + +``` +phabricator/ $ ./bin/files cycle --all +``` + +Once all files have been cycled, remove the old master key from the keyring. + +Not all storage formats support key cycling: cycling a file only has an effect +if the storage format is an encrypted format. For example, cycling a file that +uses the `raw` storage format has no effect. + + +Next Steps +========== + +Continue by: + + - understanding storage engines with @{article:Configuring File Storage}; or + - returning to the @{article:Configuration Guide}. diff --git a/src/docs/user/configuration/configuring_file_storage.diviner b/src/docs/user/configuration/configuring_file_storage.diviner index 8ea60080ac..64942b493e 100644 --- a/src/docs/user/configuration/configuring_file_storage.diviner +++ b/src/docs/user/configuration/configuring_file_storage.diviner @@ -197,4 +197,6 @@ Next Steps Continue by: + - reviewing at-rest encryption options with + @{article:Configuring Encryption}; or - returning to the @{article:Configuration Guide}.