Support HMAC+SHA256 with automatic key generation and management

Summary:
Ref T12509. This adds support for HMAC+SHA256 (instead of HMAC+SHA1). Although HMAC+SHA1 is not currently broken in any sense, SHA1 has a well-known collision and it's good to look at moving away from HMAC+SHA1.

The new mechanism also automatically generates and stores HMAC keys.

Currently, HMAC keys largely use a per-install constant defined in `security.hmac-key`. In theory this can be changed, but in practice essentially no install changes it.

We generally (in fact, always, I think?) don't use HMAC digests in a way where it matters that this key is well-known, but it's slightly better if this key is unique per class of use cases. Principally, if use cases have unique HMAC keys they are generally less vulnerable to precomputation attacks where an attacker might generate a large number of HMAC hashes of well-known values and use them in a nefarious way. The actual threat here is probably close to nonexistent, but we can harden against it without much extra effort.

Beyond that, this isn't something users should really have to think about or bother configuring.

Test Plan:
  - Added unit tests.
  - Used `bin/files integrity` to verify, strip, and recompute hashes.
  - Tampered with a generated HMAC key, verified it invalidated hashes.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T12509

Differential Revision: https://secure.phabricator.com/D17630

resources/sql/autopatches/20170406.hmac.01.keystore.sql

@@ -0,0 +1,8 @@
+CREATE TABLE {$NAMESPACE}_auth.auth_hmackey (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  keyName VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT},
+  keyValue VARCHAR(128) NOT NULL COLLATE {$COLLATE_TEXT},
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY `key_name` (keyName)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -1920,6 +1920,7 @@ phutil_register_library_map(array(
     'PhabricatorAuthFactorConfig' => 'applications/auth/storage/PhabricatorAuthFactorConfig.php',
     'PhabricatorAuthFactorTestCase' => 'applications/auth/factor/__tests__/PhabricatorAuthFactorTestCase.php',
     'PhabricatorAuthFinishController' => 'applications/auth/controller/PhabricatorAuthFinishController.php',
+    'PhabricatorAuthHMACKey' => 'applications/auth/storage/PhabricatorAuthHMACKey.php',
     'PhabricatorAuthHighSecurityRequiredException' => 'applications/auth/exception/PhabricatorAuthHighSecurityRequiredException.php',
     'PhabricatorAuthHighSecurityToken' => 'applications/auth/data/PhabricatorAuthHighSecurityToken.php',
     'PhabricatorAuthInvite' => 'applications/auth/storage/PhabricatorAuthInvite.php',
@@ -2864,6 +2865,7 @@ phutil_register_library_map(array(
     'PhabricatorGuideModule' => 'applications/guides/module/PhabricatorGuideModule.php',
     'PhabricatorGuideModuleController' => 'applications/guides/controller/PhabricatorGuideModuleController.php',
     'PhabricatorGuideQuickStartModule' => 'applications/guides/module/PhabricatorGuideQuickStartModule.php',
+    'PhabricatorHMACTestCase' => 'infrastructure/util/__tests__/PhabricatorHMACTestCase.php',
     'PhabricatorHTTPParameterTypeTableView' => 'applications/config/view/PhabricatorHTTPParameterTypeTableView.php',
     'PhabricatorHandleList' => 'applications/phid/handle/pool/PhabricatorHandleList.php',
     'PhabricatorHandleObjectSelectorDataView' => 'applications/phid/handle/view/PhabricatorHandleObjectSelectorDataView.php',
@@ -6900,6 +6902,7 @@ phutil_register_library_map(array(
     'PhabricatorAuthFactorConfig' => 'PhabricatorAuthDAO',
     'PhabricatorAuthFactorTestCase' => 'PhabricatorTestCase',
     'PhabricatorAuthFinishController' => 'PhabricatorAuthController',
+    'PhabricatorAuthHMACKey' => 'PhabricatorAuthDAO',
     'PhabricatorAuthHighSecurityRequiredException' => 'Exception',
     'PhabricatorAuthHighSecurityToken' => 'Phobject',
     'PhabricatorAuthInvite' => array(
@@ -7998,6 +8001,7 @@ phutil_register_library_map(array(
     'PhabricatorGuideModule' => 'Phobject',
     'PhabricatorGuideModuleController' => 'PhabricatorGuideController',
     'PhabricatorGuideQuickStartModule' => 'PhabricatorGuideModule',
+    'PhabricatorHMACTestCase' => 'PhabricatorTestCase',
     'PhabricatorHTTPParameterTypeTableView' => 'AphrontView',
     'PhabricatorHandleList' => array(
       'Phobject',

src/applications/auth/storage/PhabricatorAuthHMACKey.php

@@ -0,0 +1,24 @@
+<?php
+
+final class PhabricatorAuthHMACKey
+  extends PhabricatorAuthDAO {
+
+  protected $keyName;
+  protected $keyValue;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'keyName' => 'text64',
+        'keyValue' => 'text128',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_name' => array(
+          'columns' => array('keyName'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+}

src/applications/config/option/PhabricatorSecurityConfigOptions.php

@@ -109,7 +109,8 @@ EOTEXT
             'Default key for HMAC digests where the key is not important '.
             '(i.e., the hash itself is secret). You can change this if you '.
             'want (to any other string), but doing so will break existing '.
-            'sessions and CSRF tokens.')),
+            'sessions and CSRF tokens. This option is deprecated. Newer '.
+            'code automatically manages HMAC keys.')),
       $this->newOption('security.require-https', 'bool', false)
         ->setLocked(true)
         ->setSummary(

src/applications/files/engine/PhabricatorFileStorageEngine.php

@@ -16,6 +16,8 @@
  */
 abstract class PhabricatorFileStorageEngine extends Phobject {
 
+  const HMAC_INTEGRITY = 'file.integrity';
+
   /**
    * Construct a new storage engine.
    *
@@ -367,12 +369,14 @@ abstract class PhabricatorFileStorageEngine extends Phobject {
     $data,
     PhabricatorFileStorageFormat $format) {
 
-    $data_hash = PhabricatorHash::digest($data);
+    $hmac_name = self::HMAC_INTEGRITY;
+
+    $data_hash = PhabricatorHash::digestWithNamedKey($data, $hmac_name);
     $format_hash = $format->newFormatIntegrityHash();
 
     $full_hash = "{$data_hash}/{$format_hash}";
 
-    return PhabricatorHash::digest($full_hash);
+    return PhabricatorHash::digestWithNamedKey($full_hash, $hmac_name);
   }
 
 }

src/applications/files/format/PhabricatorFileAES256StorageFormat.php

@@ -67,7 +67,9 @@ final class PhabricatorFileAES256StorageFormat
 
     $input = self::FORMATKEY.'/iv:'.$iv_envelope->openEnvelope();
 
-    return PhabricatorHash::digest($input);
+    return PhabricatorHash::digestWithNamedKey(
+      $input,
+      PhabricatorFileStorageEngine::HMAC_INTEGRITY);
   }
 
   public function newStorageProperties() {

src/infrastructure/util/PhabricatorHash.php

@@ -139,5 +139,89 @@ final class PhabricatorHash extends Phobject {
     return $prefix.'-'.$hash;
   }
 
+  public static function digestWithNamedKey($message, $key_name) {
+    $key_bytes = self::getNamedHMACKey($key_name);
+    return self::digestHMACSHA256($message, $key_bytes);
+  }
+
+  public static function digestHMACSHA256($message, $key) {
+    if (!strlen($key)) {
+      throw new Exception(
+        pht('HMAC-SHA256 requires a nonempty key.'));
+    }
+
+    $result = hash_hmac('sha256', $message, $key, $raw_output = false);
+
+    if ($result === false) {
+      throw new Exception(
+        pht('Unable to compute HMAC-SHA256 digest of message.'));
+    }
+
+    return $result;
+  }
+
+
+/* -(  HMAC Key Management  )------------------------------------------------ */
+
+
+  private static function getNamedHMACKey($hmac_name) {
+    $cache = PhabricatorCaches::getImmutableCache();
+
+    $cache_key = "hmac.key({$hmac_name})";
+
+    $hmac_key = $cache->getKey($cache_key);
+    if (!strlen($hmac_key)) {
+      $hmac_key = self::readHMACKey($hmac_name);
+
+      if ($hmac_key === null) {
+        $hmac_key = self::newHMACKey($hmac_name);
+        self::writeHMACKey($hmac_name, $hmac_key);
+      }
+
+      $cache->setKey($cache_key, $hmac_key);
+    }
+
+    // The "hex2bin()" function doesn't exist until PHP 5.4.0 so just
+    // implement it inline.
+    $result = '';
+    for ($ii = 0; $ii < strlen($hmac_key); $ii += 2) {
+      $result .= pack('H*', substr($hmac_key, $ii, 2));
+    }
+
+    return $result;
+  }
+
+  private static function newHMACKey($hmac_name) {
+    $hmac_key = Filesystem::readRandomBytes(64);
+    return bin2hex($hmac_key);
+  }
+
+  private static function writeHMACKey($hmac_name, $hmac_key) {
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+
+      id(new PhabricatorAuthHMACKey())
+        ->setKeyName($hmac_name)
+        ->setKeyValue($hmac_key)
+        ->save();
+
+    unset($unguarded);
+  }
+
+  private static function readHMACKey($hmac_name) {
+    $table = new PhabricatorAuthHMACKey();
+    $conn = $table->establishConnection('r');
+
+    $row = queryfx_one(
+      $conn,
+      'SELECT keyValue FROM %T WHERE keyName = %s',
+      $table->getTableName(),
+      $hmac_name);
+    if (!$row) {
+      return null;
+    }
+
+    return $row['keyValue'];
+  }
+
 
 }

src/infrastructure/util/__tests__/PhabricatorHMACTestCase.php

@@ -0,0 +1,40 @@
+<?php
+
+final class PhabricatorHMACTestCase extends PhabricatorTestCase {
+
+  protected function getPhabricatorTestCaseConfiguration() {
+    return array(
+      self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
+    );
+  }
+
+  public function testHMACKeyGeneration() {
+    $input = 'quack';
+
+    $hash_1 = PhabricatorHash::digestWithNamedKey($input, 'test');
+    $hash_2 = PhabricatorHash::digestWithNamedKey($input, 'test');
+
+    $this->assertEqual($hash_1, $hash_2);
+  }
+
+  public function testSHA256Hashing() {
+    $input = 'quack';
+    $key = 'duck';
+    $expect =
+      '5274473dc34fc61bd7a6a5ff258e6505'.
+      '4b26644fb7a272d74f276ab677361b9a';
+
+    $hash = PhabricatorHash::digestHMACSHA256($input, $key);
+    $this->assertEqual($expect, $hash);
+
+    $input = 'The quick brown fox jumps over the lazy dog';
+    $key = 'key';
+    $expect =
+      'f7bc83f430538424b13298e6aa6fb143'.
+      'ef4d59a14946175997479dbc2d1a3cd8';
+
+    $hash = PhabricatorHash::digestHMACSHA256($input, $key);
+    $this->assertEqual($expect, $hash);
+  }
+
+}