Make Facts more modern, DRY, and dimensional

Summary:
Ref T13083. Facts has a fair amount of weird hardcoding and duplication of responsibilities. Reduce this somewhat: no more hard-coded fact aggregates, no more database-driven list of available facts, etc. Generally, derive all objective truth from FactEngines. This is more similar to how most other modern applications work.

For clarity, hopefully: rename "FactSpec" to "Fact". Rename "RawFact" to "Datapoint".

Split the fairly optimistic "RawFact" table into an "IntDatapoint" table with less stuff in it, then dimension tables for the object PHIDs and key names. This is primarily aimed at reducing the row size of each datapoint. At the time I originally wrote this code we hadn't experimented much with storing similar data in multiple tables, but this is now more common and has worked well elsewhere (CustomFields, Edges, Ferret) so I don't anticipate this causing issues. If we need more complex or multidimension/multivalue tables later we can accommodate them. The queries a single table supports (like "all facts of all kinds in some time window") don't make any sense as far as I can tell and could likely be UNION ALL'd anyway.

Remove all the aggregation stuff for now, it's not really clear to me what this should look like.

Test Plan: Ran `bin/fact analyze` and viewed web UI. Nothing exploded too violently.

Subscribers: yelirekim

Maniphest Tasks: T13083

Differential Revision: https://secure.phabricator.com/D19119

resources/sql/autopatches/20180218.fact.01.dim.key.sql

@@ -0,0 +1,5 @@
+CREATE TABLE {$NAMESPACE}_fact.fact_keydimension (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  factKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT},
+  UNIQUE KEY `key_factkey` (factKey)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20180218.fact.02.dim.obj.sql

@@ -0,0 +1,5 @@
+CREATE TABLE {$NAMESPACE}_fact.fact_objectdimension (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  objectPHID VARBINARY(64) NOT NULL,
+  UNIQUE KEY `key_object` (objectPHID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20180218.fact.03.data.int.sql

@@ -0,0 +1,8 @@
+CREATE TABLE {$NAMESPACE}_fact.fact_intdatapoint (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  keyID INT UNSIGNED NOT NULL,
+  objectID INT UNSIGNED NOT NULL,
+  dimensionID INT UNSIGNED,
+  value BIGINT SIGNED NOT NULL,
+  epoch INT UNSIGNED NOT NULL
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -2907,27 +2907,28 @@ phutil_register_library_map(array(
     'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php',
     'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php',
     'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php',
+    'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php',
     'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php',
     'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php',
     'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php',
     'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php',
-    'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php',
     'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
     'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
     'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
+    'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
     'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
     'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
     'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php',
-    'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php',
+    'PhabricatorFactIntDatapoint' => 'applications/fact/storage/PhabricatorFactIntDatapoint.php',
+    'PhabricatorFactKeyDimension' => 'applications/fact/storage/PhabricatorFactKeyDimension.php',
     'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php',
     'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php',
     'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php',
     'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
-    'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php',
     'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
+    'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
+    'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
     'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
-    'PhabricatorFactSimpleSpec' => 'applications/fact/spec/PhabricatorFactSimpleSpec.php',
-    'PhabricatorFactSpec' => 'applications/fact/spec/PhabricatorFactSpec.php',
     'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
     'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php',
     'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php',
@@ -3716,6 +3717,7 @@ phutil_register_library_map(array(
     'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php',
     'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php',
     'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php',
+    'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php',
     'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php',
     'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php',
     'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php',
@@ -8433,27 +8435,28 @@ phutil_register_library_map(array(
     'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel',
     'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider',
+    'PhabricatorFact' => 'Phobject',
     'PhabricatorFactAggregate' => 'PhabricatorFactDAO',
     'PhabricatorFactApplication' => 'PhabricatorApplication',
     'PhabricatorFactChartController' => 'PhabricatorFactController',
     'PhabricatorFactController' => 'PhabricatorController',
-    'PhabricatorFactCountEngine' => 'PhabricatorFactEngine',
     'PhabricatorFactCursor' => 'PhabricatorFactDAO',
     'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
     'PhabricatorFactDaemon' => 'PhabricatorDaemon',
+    'PhabricatorFactDimension' => 'PhabricatorFactDAO',
     'PhabricatorFactEngine' => 'Phobject',
     'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
     'PhabricatorFactHomeController' => 'PhabricatorFactController',
-    'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine',
+    'PhabricatorFactIntDatapoint' => 'PhabricatorFactDAO',
+    'PhabricatorFactKeyDimension' => 'PhabricatorFactDimension',
     'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
-    'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
+    'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
+    'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
     'PhabricatorFactRaw' => 'PhabricatorFactDAO',
-    'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec',
-    'PhabricatorFactSpec' => 'Phobject',
     'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',
     'PhabricatorFavoritesApplication' => 'PhabricatorApplication',
     'PhabricatorFavoritesController' => 'PhabricatorController',
@@ -9373,6 +9376,7 @@ phutil_register_library_map(array(
     'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation',
     'PhabricatorPlatformSite' => 'PhabricatorSite',
     'PhabricatorPointsEditField' => 'PhabricatorEditField',
+    'PhabricatorPointsFact' => 'PhabricatorFact',
     'PhabricatorPolicies' => 'PhabricatorPolicyConstants',
     'PhabricatorPolicy' => array(
       'PhabricatorPolicyDAO',

src/applications/differential/application/PhabricatorDifferentialApplication.php

@@ -35,12 +35,6 @@ final class PhabricatorDifferentialApplication extends PhabricatorApplication {
     );
   }
 
-  public function getFactObjectsForAnalysis() {
-    return array(
-      new DifferentialRevision(),
-    );
-  }
-
   public function getTitleGlyph() {
     return "\xE2\x9A\x99";
   }

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

@@ -39,12 +39,6 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
     );
   }
 
-  public function getFactObjectsForAnalysis() {
-    return array(
-      new PhabricatorRepositoryCommit(),
-    );
-  }
-
   public function getRemarkupRules() {
     return array(
       new DiffusionCommitRemarkupRule(),

src/applications/fact/controller/PhabricatorFactChartController.php

@@ -5,27 +5,32 @@ final class PhabricatorFactChartController extends PhabricatorFactController {
   public function handleRequest(AphrontRequest $request) {
     $viewer = $request->getViewer();
 
-    $table = new PhabricatorFactRaw();
+    $series = $request->getStr('y1');
+
+    $facts = PhabricatorFact::getAllFacts();
+    $fact = idx($facts, $series);
+
+    if (!$fact) {
+      return new Aphront404Response();
+    }
+
+    $key_id = id(new PhabricatorFactKeyDimension())
+      ->newDimensionID($fact->getKey());
+
+    $table = $fact->newDatapoint();
     $conn_r = $table->establishConnection('r');
     $table_name = $table->getTableName();
 
-    $series = $request->getStr('y1');
-
-    $specs = PhabricatorFactSpec::newSpecsForFactTypes(
-      PhabricatorFactEngine::loadAllEngines(),
-      array($series));
-    $spec = idx($specs, $series);
-
     $data = queryfx_all(
       $conn_r,
-      'SELECT valueX, epoch FROM %T WHERE factType = %s ORDER BY epoch ASC',
+      'SELECT value, epoch FROM %T WHERE keyID = %d ORDER BY epoch ASC',
       $table_name,
-      $series);
+      $key_id);
 
     $points = array();
     $sum = 0;
     foreach ($data as $key => $row) {
-      $sum += (int)$row['valueX'];
+      $sum += (int)$row['value'];
       $points[(int)$row['epoch']] = $sum;
     }
 
@@ -71,7 +76,7 @@ final class PhabricatorFactChartController extends PhabricatorFactController {
     ));
 
     $box = id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('Count of %s', $spec->getName()))
+      ->setHeaderText(pht('Count of %s', $fact->getName()))
       ->appendChild($chart);
 
     $crumbs = $this->buildApplicationCrumbs();

src/applications/fact/controller/PhabricatorFactHomeController.php

@@ -15,45 +15,6 @@ final class PhabricatorFactHomeController extends PhabricatorFactController {
       return id(new AphrontRedirectResponse())->setURI($uri);
     }
 
-    $types = array(
-      '+N:*',
-      '+N:DREV',
-      'updated',
-    );
-
-    $engines = PhabricatorFactEngine::loadAllEngines();
-    $specs = PhabricatorFactSpec::newSpecsForFactTypes($engines, $types);
-
-    $facts = id(new PhabricatorFactAggregate())->loadAllWhere(
-      'factType IN (%Ls)',
-      $types);
-
-    $rows = array();
-    foreach ($facts as $fact) {
-      $spec = $specs[$fact->getFactType()];
-
-      $name = $spec->getName();
-      $value = $spec->formatValueForDisplay($viewer, $fact->getValueX());
-
-      $rows[] = array($name, $value);
-    }
-
-    $table = new AphrontTableView($rows);
-    $table->setHeaders(
-      array(
-        pht('Fact'),
-        pht('Value'),
-      ));
-    $table->setColumnClasses(
-      array(
-        'wide',
-        'n',
-      ));
-
-    $panel = new PHUIObjectBoxView();
-    $panel->setHeaderText(pht('Facts'));
-    $panel->setTable($table);
-
     $chart_form = $this->buildChartForm();
 
     $crumbs = $this->buildApplicationCrumbs();
@@ -64,46 +25,18 @@ final class PhabricatorFactHomeController extends PhabricatorFactController {
     return $this->newPage()
       ->setTitle($title)
       ->setCrumbs($crumbs)
-      ->appendChild(array(
-        $chart_form,
-        $panel,
-      ));
-
+      ->appendChild(
+        array(
+          $chart_form,
+        ));
   }
 
   private function buildChartForm() {
     $request = $this->getRequest();
     $viewer = $request->getUser();
 
-    $table = new PhabricatorFactRaw();
-    $conn_r = $table->establishConnection('r');
-    $table_name = $table->getTableName();
-
-    $facts = queryfx_all(
-      $conn_r,
-      'SELECT DISTINCT factType from %T',
-      $table_name);
-
-    $specs = PhabricatorFactSpec::newSpecsForFactTypes(
-      PhabricatorFactEngine::loadAllEngines(),
-      ipull($facts, 'factType'));
-
-    $options = array();
-    foreach ($specs as $spec) {
-      if ($spec->getUnit() == PhabricatorFactSpec::UNIT_COUNT) {
-        $options[$spec->getType()] = $spec->getName();
-      }
-    }
-
-    if (!$options) {
-      return id(new PHUIInfoView())
-        ->setSeverity(PHUIInfoView::SEVERITY_NODATA)
-        ->setTitle(pht('No Chartable Facts'))
-        ->appendChild(phutil_tag(
-          'p',
-          array(),
-          pht('There are no facts that can be plotted yet.')));
-    }
+    $specs = PhabricatorFact::getAllFacts();
+    $options = mpull($specs, 'getName', 'getKey');
 
     $form = id(new AphrontFormView())
       ->setUser($viewer)

src/applications/fact/daemon/PhabricatorFactDaemon.php

@@ -4,8 +4,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
 
   private $engines;
 
-  const RAW_FACT_BUFFER_LIMIT = 128;
-
   protected function run() {
     $this->setEngines(PhabricatorFactEngine::loadAllEngines());
     while (!$this->shouldExit()) {
@@ -15,7 +13,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
       foreach ($iterators as $iterator_name => $iterator) {
         $this->processIteratorWithCursor($iterator_name, $iterator);
       }
-      $this->processAggregates();
 
       $this->log(pht('Zzz...'));
       $this->sleep(60 * 5);
@@ -72,59 +69,41 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
   public function processIterator($iterator) {
     $result = null;
 
-    $raw_facts = array();
+    $datapoints = array();
     foreach ($iterator as $key => $object) {
       $phid = $object->getPHID();
       $this->log(pht('Processing %s...', $phid));
-      $raw_facts[$phid] = $this->computeRawFacts($object);
-      if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) {
-        $this->updateRawFacts($raw_facts);
-        $raw_facts = array();
+      $datapoints[$phid] = $this->newDatapoints($object);
+      if (count($datapoints) > 1024) {
+        $this->updateDatapoints($datapoints);
+        $datapoints = array();
       }
       $result = $key;
     }
 
-    if ($raw_facts) {
-      $this->updateRawFacts($raw_facts);
-      $raw_facts = array();
+    if ($datapoints) {
+      $this->updateDatapoints($datapoints);
+      $datapoints = array();
     }
 
     return $result;
   }
 
-  public function processAggregates() {
-    $this->log(pht('Processing aggregates.'));
-
-    $facts = $this->computeAggregateFacts();
-    $this->updateAggregateFacts($facts);
-  }
-
-  private function computeAggregateFacts() {
+  private function newDatapoints(PhabricatorLiskDAO $object) {
     $facts = array();
     foreach ($this->engines as $engine) {
-      if (!$engine->shouldComputeAggregateFacts()) {
+      if (!$engine->supportsDatapointsForObject($object)) {
         continue;
       }
-      $facts[] = $engine->computeAggregateFacts();
-    }
-    return array_mergev($facts);
-  }
-
-  private function computeRawFacts(PhabricatorLiskDAO $object) {
-    $facts = array();
-    foreach ($this->engines as $engine) {
-      if (!$engine->shouldComputeRawFactsForObject($object)) {
-        continue;
-      }
-      $facts[] = $engine->computeRawFactsForObject($object);
+      $facts[] = $engine->newDatapointsForObject($object);
     }
 
     return array_mergev($facts);
   }
 
-  private function updateRawFacts(array $map) {
+  private function updateDatapoints(array $map) {
     foreach ($map as $phid => $facts) {
-      assert_instances_of($facts, 'PhabricatorFactRaw');
+      assert_instances_of($facts, 'PhabricatorFactIntDatapoint');
     }
 
     $phids = array_keys($map);
@@ -132,76 +111,79 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
       return;
     }
 
-    $table = new PhabricatorFactRaw();
+
+    $fact_keys = array();
+    $objects = array();
+    foreach ($map as $phid => $facts) {
+      foreach ($facts as $fact) {
+        $fact_keys[$fact->getKey()] = true;
+
+        $object_phid = $fact->getObjectPHID();
+        $objects[$object_phid] = $object_phid;
+
+        $dimension_phid = $fact->getDimensionPHID();
+        if ($dimension_phid !== null) {
+          $objects[$dimension_phid] = $dimension_phid;
+        }
+      }
+    }
+
+    $key_map = id(new PhabricatorFactKeyDimension())
+      ->newDimensionMap(array_keys($fact_keys));
+    $object_map = id(new PhabricatorFactObjectDimension())
+      ->newDimensionMap(array_keys($objects));
+
+    $table = new PhabricatorFactIntDatapoint();
     $conn = $table->establishConnection('w');
     $table_name = $table->getTableName();
 
     $sql = array();
     foreach ($map as $phid => $facts) {
       foreach ($facts as $fact) {
+        $key_id = $key_map[$fact->getKey()];
+        $object_id = $object_map[$fact->getObjectPHID()];
+
+        $dimension_phid = $fact->getDimensionPHID();
+        if ($dimension_phid !== null) {
+          $dimension_id = $object_map[$dimension_phid];
+        } else {
+          $dimension_id = null;
+        }
+
         $sql[] = qsprintf(
           $conn,
-          '(%s, %s, %s, %d, %d, %d)',
-          $fact->getFactType(),
-          $fact->getObjectPHID(),
-          $fact->getObjectA(),
-          $fact->getValueX(),
-          $fact->getValueY(),
+          '(%d, %d, %nd, %d, %d)',
+          $key_id,
+          $object_id,
+          $dimension_id,
+          $fact->getValue(),
           $fact->getEpoch());
       }
     }
 
+    $rebuilt_ids = array_select_keys($object_map, $phids);
+
     $table->openTransaction();
 
       queryfx(
         $conn,
-        'DELETE FROM %T WHERE objectPHID IN (%Ls)',
+        'DELETE FROM %T WHERE objectID IN (%Ld)',
         $table_name,
-        $phids);
+        $rebuilt_ids);
 
       if ($sql) {
-        foreach (array_chunk($sql, 256) as $chunk) {
+        foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
           queryfx(
             $conn,
             'INSERT INTO %T
-              (factType, objectPHID, objectA, valueX, valueY, epoch)
+              (keyID, objectID, dimensionID, value, epoch)
               VALUES %Q',
             $table_name,
-            implode(', ', $chunk));
+            $chunk);
         }
       }
 
     $table->saveTransaction();
   }
 
-  private function updateAggregateFacts(array $facts) {
-    if (!$facts) {
-      return;
-    }
-
-    $table = new PhabricatorFactAggregate();
-    $conn = $table->establishConnection('w');
-    $table_name = $table->getTableName();
-
-    $sql = array();
-    foreach ($facts as $fact) {
-      $sql[] = qsprintf(
-        $conn,
-        '(%s, %s, %d)',
-        $fact->getFactType(),
-        $fact->getObjectPHID(),
-        $fact->getValueX());
-    }
-
-    foreach (array_chunk($sql, 256) as $chunk) {
-      queryfx(
-        $conn,
-        'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q
-          ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)',
-        $table_name,
-        implode(', ', $chunk));
-    }
-
-  }
-
 }

src/applications/fact/engine/PhabricatorFactCountEngine.php

@@ -1,86 +0,0 @@
-<?php
-
-/**
- * Simple fact engine which counts objects.
- */
-final class PhabricatorFactCountEngine extends PhabricatorFactEngine {
-
-  public function getFactSpecs(array $fact_types) {
-    $results = array();
-    foreach ($fact_types as $type) {
-      if (!strncmp($type, '+N:', 3)) {
-        if ($type == '+N:*') {
-          $name = pht('Total Objects');
-        } else {
-          $name = pht('Total Objects of type %s', substr($type, 3));
-        }
-
-        $results[] = id(new PhabricatorFactSimpleSpec($type))
-          ->setName($name)
-          ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
-      }
-
-      if (!strncmp($type, 'N:', 2)) {
-        if ($type == 'N:*') {
-          $name = pht('Objects');
-        } else {
-          $name = pht('Objects of type %s', substr($type, 2));
-        }
-        $results[] = id(new PhabricatorFactSimpleSpec($type))
-          ->setName($name)
-          ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
-      }
-
-    }
-    return $results;
-  }
-
-  public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) {
-    return true;
-  }
-
-  public function computeRawFactsForObject(PhabricatorLiskDAO $object) {
-    $facts = array();
-
-    $phid = $object->getPHID();
-    $type = phid_get_type($phid);
-
-    foreach (array('N:*', 'N:'.$type) as $fact_type) {
-      $facts[] = id(new PhabricatorFactRaw())
-        ->setFactType($fact_type)
-        ->setObjectPHID($phid)
-        ->setValueX(1)
-        ->setEpoch($object->getDateCreated());
-    }
-
-    return $facts;
-  }
-
-  public function shouldComputeAggregateFacts() {
-    return true;
-  }
-
-  public function computeAggregateFacts() {
-    $table = new PhabricatorFactRaw();
-    $table_name = $table->getTableName();
-    $conn = $table->establishConnection('r');
-
-    $counts = queryfx_all(
-      $conn,
-      'SELECT factType, SUM(valueX) N FROM %T WHERE factType LIKE %>
-        GROUP BY factType',
-      $table_name,
-      'N:');
-
-    $facts = array();
-    foreach ($counts as $count) {
-      $facts[] = id(new PhabricatorFactAggregate())
-        ->setFactType('+'.$count['factType'])
-        ->setValueX($count['N']);
-    }
-
-    return $facts;
-  }
-
-
-}

src/applications/fact/engine/PhabricatorFactEngine.php

@@ -2,30 +2,37 @@
 
 abstract class PhabricatorFactEngine extends Phobject {
 
+  private $factMap;
+
   final public static function loadAllEngines() {
     return id(new PhutilClassMapQuery())
       ->setAncestorClass(__CLASS__)
       ->execute();
   }
 
-  public function getFactSpecs(array $fact_types) {
-    return array();
-  }
+  abstract public function newFacts();
 
-  public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) {
-    return false;
-  }
+  abstract public function supportsDatapointsForObject(
+    PhabricatorLiskDAO $object);
 
-  public function computeRawFactsForObject(PhabricatorLiskDAO $object) {
-    return array();
-  }
+  abstract public function newDatapointsForObject(PhabricatorLiskDAO $object);
 
-  public function shouldComputeAggregateFacts() {
-    return false;
-  }
+  final protected function getFact($key) {
+    if ($this->factMap === null) {
+      $facts = $this->newFacts();
+      $facts = mpull($facts, null, 'getKey');
+      $this->factMap = $facts;
+    }
 
-  public function computeAggregateFacts() {
-    return array();
+    if (!isset($this->factMap[$key])) {
+      throw new Exception(
+        pht(
+          'Unknown fact ("%s") for engine "%s".',
+          $key,
+          get_class($this)));
+    }
+
+    return $this->factMap[$key];
   }
 
 }

src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php

@@ -1,34 +0,0 @@
-<?php
-
-/**
- * Engine that records the time facts were last updated.
- */
-final class PhabricatorFactLastUpdatedEngine extends PhabricatorFactEngine {
-
-  public function getFactSpecs(array $fact_types) {
-    $results = array();
-    foreach ($fact_types as $type) {
-      if ($type == 'updated') {
-        $results[] = id(new PhabricatorFactSimpleSpec($type))
-          ->setName(pht('Facts Last Updated'))
-          ->setUnit(PhabricatorFactSimpleSpec::UNIT_EPOCH);
-      }
-    }
-    return $results;
-  }
-
-  public function shouldComputeAggregateFacts() {
-    return true;
-  }
-
-  public function computeAggregateFacts() {
-    $facts = array();
-
-    $facts[] = id(new PhabricatorFactAggregate())
-      ->setFactType('updated')
-      ->setValueX(time());
-
-    return $facts;
-  }
-
-}

src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php

@@ -0,0 +1,34 @@
+<?php
+
+final class PhabricatorFactManiphestTaskEngine
+  extends PhabricatorFactEngine {
+
+  public function newFacts() {
+    return array(
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.count.open'),
+    );
+  }
+
+  public function supportsDatapointsForObject(PhabricatorLiskDAO $object) {
+    return ($object instanceof ManiphestTask);
+  }
+
+  public function newDatapointsForObject(PhabricatorLiskDAO $object) {
+    $datapoints = array();
+
+    $phid = $object->getPHID();
+    $type = phid_get_type($phid);
+
+    $datapoint = $this->getFact('tasks.count.open')
+      ->newDatapoint();
+
+    $datapoints[] = $datapoint
+      ->setObjectPHID($phid)
+      ->setValue(1)
+      ->setEpoch($object->getDateCreated());
+
+    return $datapoints;
+  }
+
+}

src/applications/fact/fact/PhabricatorFact.php

@@ -0,0 +1,40 @@
+<?php
+
+abstract class PhabricatorFact extends Phobject {
+
+  private $key;
+
+  public static function getAllFacts() {
+    $engines = PhabricatorFactEngine::loadAllEngines();
+
+    $map = array();
+    foreach ($engines as $engine) {
+      $facts = $engine->newFacts();
+      $facts = mpull($facts, null, 'getKey');
+      $map += $facts;
+    }
+
+    return $map;
+  }
+
+  final public function setKey($key) {
+    $this->key = $key;
+    return $this;
+  }
+
+  final public function getKey() {
+    return $this->key;
+  }
+
+  final public function getName() {
+    return pht('Fact "%s"', $this->getKey());
+  }
+
+  final public function newDatapoint() {
+    return $this->newTemplateDatapoint()
+      ->setKey($this->getKey());
+  }
+
+  abstract protected function newTemplateDatapoint();
+
+}

src/applications/fact/fact/PhabricatorPointsFact.php

@@ -0,0 +1,9 @@
+<?php
+
+final class PhabricatorPointsFact extends PhabricatorFact {
+
+  protected function newTemplateDatapoint() {
+    return new PhabricatorFactIntDatapoint();
+  }
+
+}

src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php

@@ -58,10 +58,6 @@ final class PhabricatorFactManagementAnalyzeWorkflow
       }
     }
 
-    if (!$args->getArg('skip-aggregates')) {
-      $daemon->processAggregates();
-    }
-
     return 0;
   }
 

src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php

@@ -23,8 +23,13 @@ final class PhabricatorFactManagementDestroyWorkflow
     }
 
     $tables = array();
-    $tables[] = new PhabricatorFactRaw();
-    $tables[] = new PhabricatorFactAggregate();
+    $tables[] = new PhabricatorFactCursor();
+
+    $tables[] = new PhabricatorFactIntDatapoint();
+
+    $tables[] = new PhabricatorFactObjectDimension();
+    $tables[] = new PhabricatorFactKeyDimension();
+
     foreach ($tables as $table) {
       $conn = $table->establishConnection('w');
       $name = $table->getTableName();

src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php

@@ -1,47 +0,0 @@
-<?php
-
-final class PhabricatorFactManagementStatusWorkflow
-  extends PhabricatorFactManagementWorkflow {
-
-  protected function didConstruct() {
-    $this
-      ->setName('status')
-      ->setSynopsis(pht('Show status of fact data.'))
-      ->setArguments(array());
-  }
-
-  public function execute(PhutilArgumentParser $args) {
-    $console = PhutilConsole::getConsole();
-
-    $map = array(
-      'raw' => new PhabricatorFactRaw(),
-      'agg' => new PhabricatorFactAggregate(),
-    );
-
-    foreach ($map as $type => $table) {
-      $conn = $table->establishConnection('r');
-      $name = $table->getTableName();
-
-      $row = queryfx_one(
-        $conn,
-        'SELECT COUNT(*) N FROM %T',
-        $name);
-
-      $n = $row['N'];
-
-      switch ($type) {
-        case 'raw':
-          $desc = pht('There are %d raw fact(s) in storage.', $n);
-          break;
-        case 'agg':
-          $desc = pht('There are %d aggregate fact(s) in storage.', $n);
-          break;
-      }
-
-      $console->writeOut("%s\n", $desc);
-    }
-
-    return 0;
-  }
-
-}

src/applications/fact/spec/PhabricatorFactSimpleSpec.php

@@ -1,38 +0,0 @@
-<?php
-
-final class PhabricatorFactSimpleSpec extends PhabricatorFactSpec {
-
-  private $type;
-  private $name;
-  private $unit;
-
-  public function __construct($type) {
-    $this->type = $type;
-  }
-
-  public function getType() {
-    return $this->type;
-  }
-
-  public function setUnit($unit) {
-    $this->unit = $unit;
-    return $this;
-  }
-
-  public function getUnit() {
-    return $this->unit;
-  }
-
-  public function setName($name) {
-    $this->name = $name;
-    return $this;
-  }
-
-  public function getName() {
-    if ($this->name !== null) {
-      return $this->name;
-    }
-    return parent::getName();
-  }
-
-}

src/applications/fact/spec/PhabricatorFactSpec.php

@@ -1,53 +0,0 @@
-<?php
-
-abstract class PhabricatorFactSpec extends Phobject {
-
-  const UNIT_COUNT = 'unit-count';
-  const UNIT_EPOCH = 'unit-epoch';
-
-  public static function newSpecsForFactTypes(
-    array $engines,
-    array $fact_types) {
-    assert_instances_of($engines, 'PhabricatorFactEngine');
-
-    $map = array();
-    foreach ($engines as $engine) {
-      $specs = $engine->getFactSpecs($fact_types);
-      $specs = mpull($specs, null, 'getType');
-      $map += $specs;
-    }
-
-    foreach ($fact_types as $type) {
-      if (empty($map[$type])) {
-        $map[$type] = new PhabricatorFactSimpleSpec($type);
-      }
-    }
-
-    return $map;
-  }
-
-  abstract public function getType();
-
-  public function getUnit() {
-    return null;
-  }
-
-  public function getName() {
-    return pht(
-      'Fact (%s)',
-      $this->getType());
-  }
-
-  public function formatValueForDisplay(PhabricatorUser $user, $value) {
-    $unit = $this->getUnit();
-    switch ($unit) {
-      case self::UNIT_COUNT:
-        return number_format($value);
-      case self::UNIT_EPOCH:
-        return phabricator_datetime($value, $user);
-      default:
-        return $value;
-    }
-  }
-
-}

src/applications/fact/storage/PhabricatorFactDimension.php

@@ -0,0 +1,85 @@
+<?php
+
+abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
+
+  abstract protected function getDimensionColumnName();
+
+  final public function newDimensionID($key) {
+    $map = $this->newDimensionMap(array($key));
+    return $map[$key];
+  }
+
+  final public function newDimensionMap(array $keys) {
+    if (!$keys) {
+      return array();
+    }
+
+    $conn = $this->establishConnection('r');
+    $column = $this->getDimensionColumnName();
+
+    $rows = queryfx_all(
+      $conn,
+      'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
+      $column,
+      $this->getTableName(),
+      $column,
+      $keys);
+    $rows = ipull($rows, 'id', $column);
+
+    $map = array();
+    $need = array();
+    foreach ($keys as $key) {
+      if (isset($rows[$key])) {
+        $map[$key] = (int)$rows[$key];
+      } else {
+        $need[] = $key;
+      }
+    }
+
+    if (!$need) {
+      return $map;
+    }
+
+    $sql = array();
+    foreach ($need as $key) {
+      $sql[] = qsprintf(
+        $conn,
+        '(%s)',
+        $key);
+    }
+
+    foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+      queryfx(
+        $conn,
+        'INSERT IGNORE INTO %T (%C) VALUES %Q',
+        $this->getTableName(),
+        $column,
+        $chunk);
+    }
+
+    $rows = queryfx_all(
+      $conn,
+      'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
+      $column,
+      $this->getTableName(),
+      $column,
+      $need);
+    $rows = ipull($rows, 'id', $column);
+
+    foreach ($keys as $key) {
+      if (isset($rows[$key])) {
+        $map[$key] = (int)$rows[$key];
+      } else {
+        throw new Exception(
+          pht(
+            'Failed to load or generate dimension ID ("%s") for dimension '.
+            'key "%s".',
+            get_class($this),
+            $key));
+      }
+    }
+
+    return $map;
+  }
+
+}

src/applications/fact/storage/PhabricatorFactIntDatapoint.php

@@ -0,0 +1,61 @@
+<?php
+
+final class PhabricatorFactIntDatapoint extends PhabricatorFactDAO {
+
+  protected $keyID;
+  protected $objectID;
+  protected $dimensionID;
+  protected $value;
+  protected $epoch;
+
+  private $key;
+  private $objectPHID;
+  private $dimensionPHID;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'id' => 'auto64',
+        'dimensionID' => 'id?',
+        'value' => 'sint64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_dimension' => array(
+          'columns' => array('keyID', 'dimensionID'),
+        ),
+        'key_object' => array(
+          'columns' => array('objectID'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function setKey($key) {
+    $this->key = $key;
+    return $this;
+  }
+
+  public function getKey() {
+    return $this->key;
+  }
+
+  public function setObjectPHID($object_phid) {
+    $this->objectPHID = $object_phid;
+    return $this;
+  }
+
+  public function getObjectPHID() {
+    return $this->objectPHID;
+  }
+
+  public function setDimensionPHID($dimension_phid) {
+    $this->dimensionPHID = $dimension_phid;
+    return $this;
+  }
+
+  public function getDimensionPHID() {
+    return $this->dimensionPHID;
+  }
+
+}

src/applications/fact/storage/PhabricatorFactKeyDimension.php

@@ -0,0 +1,27 @@
+<?php
+
+final class PhabricatorFactKeyDimension
+  extends PhabricatorFactDimension {
+
+  protected $factKey;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'factKey' => 'text64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_factkey' => array(
+          'columns' => array('factKey'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  protected function getDimensionColumnName() {
+    return 'factKey';
+  }
+
+}

src/applications/fact/storage/PhabricatorFactObjectDimension.php

@@ -0,0 +1,25 @@
+<?php
+
+final class PhabricatorFactObjectDimension
+  extends PhabricatorFactDimension {
+
+  protected $objectPHID;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_object' => array(
+          'columns' => array('objectPHID'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  protected function getDimensionColumnName() {
+    return 'objectPHID';
+  }
+
+}

src/applications/ponder/application/PhabricatorPonderApplication.php

@@ -18,12 +18,6 @@ final class PhabricatorPonderApplication extends PhabricatorApplication {
     return 'fa-university';
   }
 
-  public function getFactObjectsForAnalysis() {
-    return array(
-      new PonderQuestion(),
-    );
-  }
-
   public function getTitleGlyph() {
     return "\xE2\x97\xB3";
   }