Implement storage of a host ID and a public key for authorizing Conduit between servers

Summary:
Ref T4209.  This creates storage for public keys against authorized hosts, such that servers can be authorized to make Conduit calls as the omnipotent user.

Servers are registered into this system by running the following command once:

```
bin/almanac register
```

NOTE: This doesn't implement authorization between servers, just the storage of public keys.

Placing this against Almanac seemed like the most sensible place, since I'm imagining in future that the `register` command will accept more information (like the hostname of the server so it can be found in the service directory).

Test Plan: Ran `bin/almanac register` and saw the host (and public key information) appear in the database.

Reviewers: #blessed_reviewers, epriestley

Reviewed By: #blessed_reviewers, epriestley

Subscribers: epriestley, Korvin

Maniphest Tasks: T4209

Differential Revision: https://secure.phabricator.com/D10400

.gitignore

@@ -13,6 +13,8 @@
 /conf/local/local.json
 /conf/local/ENVIRONMENT
 /conf/local/VERSION
+/conf/local/HOSTKEY
+/conf/local/HOSTID
 
 # Impact Font
 /resources/font/impact.ttf

bin/almanac

@@ -0,0 +1 @@
+../scripts/almanac/manage_almanac.php

resources/sql/autopatches/20140902.almanacdevice.1.sql

@@ -0,0 +1,18 @@
+CREATE TABLE {$NAMESPACE}_almanac.almanac_device (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARBINARY(64) NOT NULL,
+  name VARCHAR(255) NOT NULL COLLATE {$COLLATE_TEXT},
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY `key_phid` (phid)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+
+CREATE TABLE {$NAMESPACE}_almanac.almanac_deviceproperty (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  devicePHID VARBINARY(64) NOT NULL,
+  `key` VARCHAR(128) NOT NULL COLLATE {$COLLATE_TEXT},
+  value LONGTEXT NOT NULL,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  KEY `key_device` (devicePHID, `key`)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

scripts/almanac/manage_almanac.php

@@ -0,0 +1,21 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('manage host directory');
+$args->setSynopsis(<<<EOSYNOPSIS
+**almanac** __commmand__ [__options__]
+    Manage Almanac stuff. NEW AND EXPERIMENTAL.
+
+EOSYNOPSIS
+);
+$args->parseStandardArguments();
+
+$workflows = id(new PhutilSymbolLoader())
+  ->setAncestorClass('AlmanacManagementWorkflow')
+  ->loadObjects();
+$workflows[] = new PhutilHelpArgumentWorkflow();
+$args->parseWorkflows($workflows);

src/__phutil_library_map__.php

@@ -9,6 +9,14 @@
 phutil_register_library_map(array(
   '__library_version__' => 2,
   'class' => array(
+    'AlmanacConduitUtil' => 'applications/almanac/util/AlmanacConduitUtil.php',
+    'AlmanacDAO' => 'applications/almanac/storage/AlmanacDAO.php',
+    'AlmanacDevice' => 'applications/almanac/storage/AlmanacDevice.php',
+    'AlmanacDevicePHIDType' => 'applications/almanac/phid/AlmanacDevicePHIDType.php',
+    'AlmanacDeviceProperty' => 'applications/almanac/storage/AlmanacDeviceProperty.php',
+    'AlmanacDeviceQuery' => 'applications/almanac/query/AlmanacDeviceQuery.php',
+    'AlmanacManagementRegisterWorkflow' => 'applications/almanac/management/AlmanacManagementRegisterWorkflow.php',
+    'AlmanacManagementWorkflow' => 'applications/almanac/management/AlmanacManagementWorkflow.php',
     'Aphront304Response' => 'aphront/response/Aphront304Response.php',
     'Aphront400Response' => 'aphront/response/Aphront400Response.php',
     'Aphront403Response' => 'aphront/response/Aphront403Response.php',
@@ -1126,6 +1134,7 @@ phutil_register_library_map(array(
     'PhabricatorActionListView' => 'view/layout/PhabricatorActionListView.php',
     'PhabricatorActionView' => 'view/layout/PhabricatorActionView.php',
     'PhabricatorAllCapsTranslation' => 'infrastructure/internationalization/translation/PhabricatorAllCapsTranslation.php',
+    'PhabricatorAlmanacApplication' => 'applications/almanac/application/PhabricatorAlmanacApplication.php',
     'PhabricatorAmazonAuthProvider' => 'applications/auth/provider/PhabricatorAmazonAuthProvider.php',
     'PhabricatorAnchorView' => 'view/layout/PhabricatorAnchorView.php',
     'PhabricatorAphlictManagementBuildWorkflow' => 'applications/aphlict/management/PhabricatorAphlictManagementBuildWorkflow.php',
@@ -2847,6 +2856,17 @@ phutil_register_library_map(array(
     'require_celerity_resource' => 'infrastructure/celerity/api.php',
   ),
   'xmap' => array(
+    'AlmanacConduitUtil' => 'Phobject',
+    'AlmanacDAO' => 'PhabricatorLiskDAO',
+    'AlmanacDevice' => array(
+      'AlmanacDAO',
+      'PhabricatorPolicyInterface',
+    ),
+    'AlmanacDevicePHIDType' => 'PhabricatorPHIDType',
+    'AlmanacDeviceProperty' => 'AlmanacDAO',
+    'AlmanacDeviceQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'AlmanacManagementRegisterWorkflow' => 'AlmanacManagementWorkflow',
+    'AlmanacManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'Aphront304Response' => 'AphrontResponse',
     'Aphront400Response' => 'AphrontResponse',
     'Aphront403Response' => 'AphrontHTMLResponse',
@@ -4031,6 +4051,7 @@ phutil_register_library_map(array(
     'PhabricatorActionListView' => 'AphrontView',
     'PhabricatorActionView' => 'AphrontView',
     'PhabricatorAllCapsTranslation' => 'PhabricatorTranslation',
+    'PhabricatorAlmanacApplication' => 'PhabricatorApplication',
     'PhabricatorAmazonAuthProvider' => 'PhabricatorOAuth2AuthProvider',
     'PhabricatorAnchorView' => 'AphrontView',
     'PhabricatorAphlictManagementBuildWorkflow' => 'PhabricatorAphlictManagementWorkflow',

src/applications/almanac/application/PhabricatorAlmanacApplication.php

@@ -0,0 +1,41 @@
+<?php
+
+final class PhabricatorAlmanacApplication extends PhabricatorApplication {
+
+  public function getBaseURI() {
+    return '/almanac/';
+  }
+
+  public function getName() {
+    return pht('Almanac');
+  }
+
+  public function getShortDescription() {
+    return pht('Service Directory');
+  }
+
+  public function getIconName() {
+    return 'almanac';
+  }
+
+  public function getTitleGlyph() {
+    return "\xE2\x98\x82";
+  }
+
+  public function getApplicationGroup() {
+    return self::GROUP_UTILITIES;
+  }
+
+  public function isPrototype() {
+    return true;
+  }
+
+  public function isLaunchable() {
+    return false;
+  }
+
+  public function getRoutes() {
+    return array();
+  }
+
+}

src/applications/almanac/management/AlmanacManagementRegisterWorkflow.php

@@ -0,0 +1,64 @@
+<?php
+
+final class AlmanacManagementRegisterWorkflow
+  extends AlmanacManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('register')
+      ->setSynopsis(pht('Register this host for authorized Conduit access.'))
+      ->setArguments(array());
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $console = PhutilConsole::getConsole();
+
+    if (Filesystem::pathExists(AlmanacConduitUtil::getHostPrivateKeyPath())) {
+      throw new Exception(
+        'This host already has a private key for Conduit access.');
+    }
+
+    $pair = PhabricatorSSHKeyGenerator::generateKeypair();
+    list($public_key, $private_key) = $pair;
+
+    $host = id(new AlmanacDevice())
+      ->setName(php_uname('n'))
+      ->save();
+
+    id(new AlmanacDeviceProperty())
+      ->setDevicePHID($host->getPHID())
+      ->setKey('conduitPublicOpenSSHKey')
+      ->setValue($public_key)
+      ->save();
+
+    id(new AlmanacDeviceProperty())
+      ->setDevicePHID($host->getPHID())
+      ->setKey('conduitPublicOpenSSLKey')
+      ->setValue($this->convertToOpenSSLPublicKey($public_key))
+      ->save();
+
+    Filesystem::writeFile(
+      AlmanacConduitUtil::getHostPrivateKeyPath(),
+      $private_key);
+
+    Filesystem::writeFile(
+      AlmanacConduitUtil::getHostIDPath(),
+      $host->getID());
+
+    $console->writeOut("Registered as device %d.\n", $host->getID());
+  }
+
+  private function convertToOpenSSLPublicKey($openssh_public_key) {
+    $ssh_public_key_file = new TempFile();
+    Filesystem::writeFile($ssh_public_key_file, $openssh_public_key);
+
+    list($public_key, $stderr) = id(new ExecFuture(
+      'ssh-keygen -e -f %s -m pkcs8',
+      $ssh_public_key_file))->resolvex();
+
+    unset($ssh_public_key_file);
+
+    return $public_key;
+  }
+
+}

src/applications/almanac/management/AlmanacManagementWorkflow.php

@@ -0,0 +1,4 @@
+<?php
+
+abstract class AlmanacManagementWorkflow
+  extends PhabricatorManagementWorkflow {}

src/applications/almanac/phid/AlmanacDevicePHIDType.php

@@ -0,0 +1,39 @@
+<?php
+
+final class AlmanacDevicePHIDType extends PhabricatorPHIDType {
+
+  const TYPECONST = 'ADEV';
+
+  public function getTypeName() {
+    return pht('Almanac Device');
+  }
+
+  public function newObject() {
+    return new AlmanacDevice();
+  }
+
+  protected function buildQueryForObjects(
+    PhabricatorObjectQuery $query,
+    array $phids) {
+
+    return id(new AlmanacDeviceQuery())
+      ->withPHIDs($phids);
+  }
+
+  public function loadHandles(
+    PhabricatorHandleQuery $query,
+    array $handles,
+    array $objects) {
+
+    foreach ($handles as $phid => $handle) {
+      $device = $objects[$phid];
+
+      $id = $device->getID();
+      $name = $device->getName();
+
+      $handle->setObjectName(pht('Device %d', $id));
+      $handle->setName($name);
+    }
+  }
+
+}

src/applications/almanac/query/AlmanacDeviceQuery.php

@@ -0,0 +1,60 @@
+<?php
+
+final class AlmanacDeviceQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $ids;
+  private $phids;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withPHIDs(array $phids) {
+    $this->phids = $phids;
+    return $this;
+  }
+
+  protected function loadPage() {
+    $table = new AlmanacDevice();
+    $conn_r = $table->establishConnection('r');
+
+    $data = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T %Q %Q %Q',
+      $table->getTableName(),
+      $this->buildWhereClause($conn_r),
+      $this->buildOrderClause($conn_r),
+      $this->buildLimitClause($conn_r));
+
+    return $table->loadAllFromArray($data);
+  }
+
+  protected function buildWhereClause($conn_r) {
+    $where = array();
+
+    if ($this->ids !== null) {
+      $where[] = qsprintf(
+        $conn_r,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->phids !== null) {
+      $where[] = qsprintf(
+        $conn_r,
+        'phid IN (%Ls)',
+        $this->phids);
+    }
+
+    $where[] = $this->buildPagingClause($conn_r);
+
+    return $this->formatWhereClause($where);
+  }
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorAlmanacApplication';
+  }
+
+}

src/applications/almanac/storage/AlmanacDAO.php

@@ -0,0 +1,9 @@
+<?php
+
+abstract class AlmanacDAO extends PhabricatorLiskDAO {
+
+  public function getApplicationName() {
+    return 'almanac';
+  }
+
+}

src/applications/almanac/storage/AlmanacDevice.php

@@ -0,0 +1,50 @@
+<?php
+
+final class AlmanacDevice
+  extends AlmanacDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $name;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_AUX_PHID => true,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'name' => 'text255',
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function generatePHID() {
+    return PhabricatorPHID::generateNewPHID(AlmanacDevicePHIDType::TYPECONST);
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    switch ($capability) {
+      case PhabricatorPolicyCapability::CAN_VIEW:
+        // Until we get a clearer idea on what's going to be stored in this
+        // table, don't allow anyone (other than the omnipotent user) to find
+        // these objects.
+        return PhabricatorPolicies::POLICY_NOONE;
+    }
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return false;
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return null;
+  }
+
+}

src/applications/almanac/storage/AlmanacDeviceProperty.php

@@ -0,0 +1,25 @@
+<?php
+
+final class AlmanacDeviceProperty extends AlmanacDAO {
+
+  protected $devicePHID;
+  protected $key;
+  protected $value;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_SERIALIZATION => array(
+        'value'    => self::SERIALIZATION_JSON,
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'key' => 'text128',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_device' => array(
+          'columns' => array('devicePHID', 'key'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+}

src/applications/almanac/util/AlmanacConduitUtil.php

@@ -0,0 +1,17 @@
+<?php
+
+final class AlmanacConduitUtil extends Phobject {
+
+  public static function getHostPrivateKeyPath() {
+    $root = dirname(phutil_get_library_root('phabricator'));
+    $path = $root.'/conf/local/HOSTKEY';
+    return $path;
+  }
+
+  public static function getHostIDPath() {
+    $root = dirname(phutil_get_library_root('phabricator'));
+    $path = $root.'/conf/local/HOSTID';
+    return $path;
+  }
+
+}

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -120,6 +120,7 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
       'db.dashboard' => array(),
       'db.system' => array(),
       'db.fund' => array(),
+      'db.almanac' => array(),
       '0000.legacy.sql' => array(
         'legacy' => 0,
       ),