Provide a page for examining the facts an object generates

Summary:
Depends on D19120. Ref T13083. When you write a fact engine, it's currently somewhat difficult to figure out exactly what it's doing. It would also be difficult to diagnose bugs or report them to the upstream.

To ease this, add a page which shows all the facts an object generates. This allows you to iterate on an engine quickly without needing to reanalyze facts, take a screenshot, easily compare the timeline to the fact view, etc.

Test Plan: Viewed the object fact page for several objects.

Subscribers: yelirekim

Maniphest Tasks: T13083

Differential Revision: https://secure.phabricator.com/D19121

src/__phutil_library_map__.php

@@ -2916,6 +2916,7 @@ phutil_register_library_map(array(
     'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
     'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
     'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
+    'PhabricatorFactDatapointQuery' => 'applications/fact/query/PhabricatorFactDatapointQuery.php',
     'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
     'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
     'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
@@ -2928,6 +2929,7 @@ phutil_register_library_map(array(
     'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
     'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
     'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
+    'PhabricatorFactObjectController' => 'applications/fact/controller/PhabricatorFactObjectController.php',
     'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
     'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
     'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
@@ -8445,6 +8447,7 @@ phutil_register_library_map(array(
     'PhabricatorFactCursor' => 'PhabricatorFactDAO',
     'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
     'PhabricatorFactDaemon' => 'PhabricatorDaemon',
+    'PhabricatorFactDatapointQuery' => 'Phobject',
     'PhabricatorFactDimension' => 'PhabricatorFactDAO',
     'PhabricatorFactEngine' => 'Phobject',
     'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
@@ -8457,6 +8460,7 @@ phutil_register_library_map(array(
     'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
+    'PhabricatorFactObjectController' => 'PhabricatorFactController',
     'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
     'PhabricatorFactRaw' => 'PhabricatorFactDAO',
     'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',

src/applications/fact/application/PhabricatorFactApplication.php

@@ -31,6 +31,7 @@ final class PhabricatorFactApplication extends PhabricatorApplication {
       '/fact/' => array(
         '' => 'PhabricatorFactHomeController',
         'chart/' => 'PhabricatorFactChartController',
+        'object/(?<phid>[^/]+)/' => 'PhabricatorFactObjectController',
       ),
     );
   }

src/applications/fact/controller/PhabricatorFactObjectController.php

@@ -0,0 +1,267 @@
+<?php
+
+final class PhabricatorFactObjectController
+  extends PhabricatorFactController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $viewer = $request->getViewer();
+
+    $phid = $request->getURIData('phid');
+    $object = id(new PhabricatorObjectQuery())
+      ->setViewer($viewer)
+      ->withNames(array($phid))
+      ->executeOne();
+    if (!$object) {
+      return new Aphront404Response();
+    }
+
+    $engines = PhabricatorFactEngine::loadAllEngines();
+    foreach ($engines as $key => $engine) {
+      if (!$engine->supportsDatapointsForObject($object)) {
+        unset($engines[$key]);
+      }
+    }
+
+    if (!$engines) {
+      return $this->newDialog()
+        ->setTitle(pht('No Engines'))
+        ->appendParagraph(
+          pht(
+            'No fact engines support generating facts for this object.'))
+        ->addCancelButton($this->getApplicationURI());
+    }
+
+    $key_dimension = new PhabricatorFactKeyDimension();
+    $object_phid = $object->getPHID();
+
+    $facts = array();
+    $generated_datapoints = array();
+    $timings = array();
+    foreach ($engines as $key => $engine) {
+      $engine_facts = $engine->newFacts();
+      $engine_facts = mpull($engine_facts, null, 'getKey');
+      $facts[$key] = $engine_facts;
+
+      $t_start = microtime(true);
+      $generated_datapoints[$key] = $engine->newDatapointsForObject($object);
+      $t_end = microtime(true);
+
+      $timings[$key] = ($t_end - $t_start);
+    }
+
+    $object_id = id(new PhabricatorFactObjectDimension())
+      ->newDimensionID($object_phid, true);
+
+    $stored_datapoints = id(new PhabricatorFactDatapointQuery())
+      ->withFacts(array_mergev($facts))
+      ->withObjectPHIDs(array($object_phid))
+      ->needVectors(true)
+      ->execute();
+
+    $stored_groups = array();
+    foreach ($stored_datapoints as $stored_datapoint) {
+      $stored_groups[$stored_datapoint['key']][] = $stored_datapoint;
+    }
+
+    $stored_map = array();
+    foreach ($engines as $key => $engine) {
+      $stored_map[$key] = array();
+      foreach ($facts[$key] as $fact) {
+        $fact_datapoints = idx($stored_groups, $fact->getKey(), array());
+        $fact_datapoints = igroup($fact_datapoints, 'vector');
+        $stored_map[$key] += $fact_datapoints;
+      }
+    }
+
+    $handle_phids = array();
+    $handle_phids[] = $object->getPHID();
+    foreach ($generated_datapoints as $key => $datapoint_set) {
+      foreach ($datapoint_set as $datapoint) {
+        $dimension_phid = $datapoint->getDimensionPHID();
+        if ($dimension_phid !== null) {
+          $handle_phids[$dimension_phid] = $dimension_phid;
+        }
+      }
+    }
+
+    foreach ($stored_map as $key => $stored_datapoints) {
+      foreach ($stored_datapoints as $vector_key => $datapoints) {
+        foreach ($datapoints as $datapoint) {
+          $dimension_phid = $datapoint['dimensionPHID'];
+          if ($dimension_phid !== null) {
+            $handle_phids[$dimension_phid] = $dimension_phid;
+          }
+        }
+      }
+    }
+
+    $handles = $viewer->loadHandles($handle_phids);
+
+    $dimension_map = id(new PhabricatorFactObjectDimension())
+      ->newDimensionMap($handle_phids, true);
+
+    $content = array();
+
+    $object_list = id(new PHUIPropertyListView())
+      ->setViewer($viewer)
+      ->addProperty(
+        pht('Object'),
+        $handles[$object->getPHID()]->renderLink());
+
+    $total_cost = array_sum($timings);
+    $total_cost = pht('%sms', new PhutilNumber((int)(1000 * $total_cost)));
+    $object_list->addProperty(pht('Total Cost'), $total_cost);
+
+    $object_box = id(new PHUIObjectBoxView())
+      ->setHeaderText(pht('Fact Extraction Report'))
+      ->addPropertyList($object_list);
+
+    $content[] = $object_box;
+
+    $icon_fact = id(new PHUIIconView())
+      ->setIcon('fa-line-chart green')
+      ->setTooltip(pht('Consistent Fact'));
+
+    $icon_nodata = id(new PHUIIconView())
+      ->setIcon('fa-question-circle-o violet')
+      ->setTooltip(pht('No Stored Datapoints'));
+
+    $icon_new = id(new PHUIIconView())
+      ->setIcon('fa-plus red')
+      ->setTooltip(pht('Not Stored'));
+
+    $icon_surplus = id(new PHUIIconView())
+      ->setIcon('fa-minus red')
+      ->setTooltip(pht('Not Generated'));
+
+    foreach ($engines as $key => $engine) {
+      $rows = array();
+      foreach ($generated_datapoints[$key] as $datapoint) {
+        $dimension_phid = $datapoint->getDimensionPHID();
+        if ($dimension_phid !== null) {
+          $dimension = $handles[$datapoint->getDimensionPHID()]->renderLink();
+        } else {
+          $dimension = null;
+        }
+
+        $fact_key = $datapoint->getKey();
+
+        $fact = idx($facts[$key], $fact_key, null);
+        if ($fact) {
+          $fact_label = $fact->getName();
+        } else {
+          $fact_label = $fact_key;
+        }
+
+        $vector_key = $datapoint->newDatapointVector();
+        if (isset($stored_map[$key][$vector_key])) {
+          unset($stored_map[$key][$vector_key]);
+          $icon = $icon_fact;
+        } else {
+          $icon = $icon_new;
+        }
+
+        $rows[] = array(
+          $icon,
+          $fact_label,
+          $dimension,
+          $datapoint->getValue(),
+          phabricator_datetime($datapoint->getEpoch(), $viewer),
+        );
+      }
+
+      foreach ($stored_map[$key] as $vector_key => $datapoints) {
+        foreach ($datapoints as $datapoint) {
+          $dimension_phid = $datapoint['dimensionPHID'];
+          if ($dimension_phid !== null) {
+            $dimension = $handles[$dimension_phid]->renderLink();
+          } else {
+            $dimension = null;
+          }
+
+          $fact_key = $datapoint['key'];
+          $fact = idx($facts[$key], $fact_key, null);
+          if ($fact) {
+            $fact_label = $fact->getName();
+          } else {
+            $fact_label = $fact_key;
+          }
+
+          $rows[] = array(
+            $icon_surplus,
+            $fact_label,
+            $dimension,
+            $datapoint['value'],
+            phabricator_datetime($datapoint['epoch'], $viewer),
+          );
+        }
+      }
+
+      foreach ($facts[$key] as $fact) {
+        $has_any = id(new PhabricatorFactDatapointQuery())
+          ->withFacts(array($fact))
+          ->setLimit(1)
+          ->execute();
+        if ($has_any) {
+          continue;
+        }
+
+        if (!$has_any) {
+          $rows[] = array(
+            $icon_nodata,
+            $fact->getName(),
+            null,
+            null,
+            null,
+          );
+        }
+      }
+
+      $table = id(new AphrontTableView($rows))
+        ->setHeaders(
+          array(
+            null,
+            pht('Fact'),
+            pht('Dimension'),
+            pht('Value'),
+            pht('Date'),
+          ))
+        ->setColumnClasses(
+          array(
+            '',
+            '',
+            '',
+            'n wide right',
+            'right',
+          ));
+
+      $extraction_cost = $timings[$key];
+      $extraction_cost = pht(
+        '%sms',
+        new PhutilNumber((int)(1000 * $extraction_cost)));
+
+      $header = pht(
+        '%s (%s)',
+        get_class($engine),
+        $extraction_cost);
+
+      $box = id(new PHUIObjectBoxView())
+        ->setHeaderText($header)
+        ->setTable($table);
+
+      $content[] = $box;
+    }
+
+    $crumbs = $this->buildApplicationCrumbs()
+      ->addTextCrumb(pht('Chart'));
+
+    $title = pht('Chart');
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs($crumbs)
+      ->appendChild($content);
+
+  }
+
+}

src/applications/fact/query/PhabricatorFactDatapointQuery.php

@@ -0,0 +1,181 @@
+<?php
+
+final class PhabricatorFactDatapointQuery extends Phobject {
+
+  private $facts;
+  private $objectPHIDs;
+  private $limit;
+
+  private $needVectors;
+
+  private $keyMap = array();
+  private $dimensionMap = array();
+
+  public function withFacts(array $facts) {
+    $this->facts = $facts;
+    return $this;
+  }
+
+  public function withObjectPHIDs(array $object_phids) {
+    $this->objectPHIDs = $object_phids;
+    return $this;
+  }
+
+  public function setLimit($limit) {
+    $this->limit = $limit;
+    return $this;
+  }
+
+  public function needVectors($need) {
+    $this->needVectors = $need;
+    return $this;
+  }
+
+  public function execute() {
+    $facts = mpull($this->facts, null, 'getKey');
+    if (!$facts) {
+      throw new Exception(pht('Executing a fact query requires facts.'));
+    }
+
+    $table_map = array();
+    foreach ($facts as $fact) {
+      $datapoint = $fact->newDatapoint();
+      $table = $datapoint->getTableName();
+
+      if (!isset($table_map[$table])) {
+        $table_map[$table] = array(
+          'table' => $datapoint,
+          'facts' => array(),
+        );
+      }
+
+      $table_map[$table]['facts'][] = $fact;
+    }
+
+    $rows = array();
+    foreach ($table_map as $spec) {
+      $rows[] = $this->executeWithTable($spec);
+    }
+    $rows = array_mergev($rows);
+
+    $key_unmap = array_flip($this->keyMap);
+    $dimension_unmap = array_flip($this->dimensionMap);
+
+    $groups = array();
+    $need_phids = array();
+    foreach ($rows as $row) {
+      $groups[$row['keyID']][] = $row;
+
+      $object_id = $row['objectID'];
+      if (!isset($dimension_unmap[$object_id])) {
+        $need_phids[$object_id] = $object_id;
+      }
+
+      $dimension_id = $row['dimensionID'];
+      if ($dimension_id && !isset($dimension_unmap[$dimension_id])) {
+        $need_phids[$dimension_id] = $dimension_id;
+      }
+    }
+
+    $dimension_unmap += id(new PhabricatorFactObjectDimension())
+      ->newDimensionUnmap($need_phids);
+
+    $results = array();
+    foreach ($groups as $key_id => $rows) {
+      $key = $key_unmap[$key_id];
+      $fact = $facts[$key];
+      $datapoint = $fact->newDatapoint();
+      foreach ($rows as $row) {
+        $dimension_id = $row['dimensionID'];
+        if ($dimension_id) {
+          if (!isset($dimension_unmap[$dimension_id])) {
+            continue;
+          } else {
+            $dimension_phid = $dimension_unmap[$dimension_id];
+          }
+        } else {
+          $dimension_phid = null;
+        }
+
+        $object_id = $row['objectID'];
+        if (!isset($dimension_unmap[$object_id])) {
+          continue;
+        } else {
+          $object_phid = $dimension_unmap[$object_id];
+        }
+
+        $result = array(
+          'key' => $key,
+          'objectPHID' => $object_phid,
+          'dimensionPHID' => $dimension_phid,
+          'value' => (int)$row['value'],
+          'epoch' => $row['epoch'],
+        );
+
+        if ($this->needVectors) {
+          $result['vector'] = $datapoint->newRawVector($result);
+        }
+
+        $results[] = $result;
+      }
+    }
+
+    return $results;
+  }
+
+  private function executeWithTable(array $spec) {
+    $table = $spec['table'];
+    $facts = $spec['facts'];
+    $conn = $table->establishConnection('r');
+
+    $fact_keys = mpull($facts, 'getKey');
+    $this->keyMap = id(new PhabricatorFactKeyDimension())
+      ->newDimensionMap($fact_keys);
+
+    if (!$this->keyMap) {
+      return array();
+    }
+
+    $where = array();
+
+    $where[] = qsprintf(
+      $conn,
+      'keyID IN (%Ld)',
+      $this->keyMap);
+
+    if ($this->objectPHIDs) {
+      $object_map = id(new PhabricatorFactObjectDimension())
+        ->newDimensionMap($this->objectPHIDs);
+      if (!$object_map) {
+        return array();
+      }
+
+      $this->dimensionMap = $object_map;
+
+      $where[] = qsprintf(
+        $conn,
+        'objectID IN (%Ld)',
+        $this->dimensionMap);
+    }
+
+    $where = '('.implode(') AND (', $where).')';
+
+    if ($this->limit) {
+      $limit = qsprintf(
+        $conn,
+        'LIMIT %d',
+        $this->limit);
+    } else {
+      $limit = '';
+    }
+
+    return queryfx_all(
+      $conn,
+      'SELECT keyID, objectID, dimensionID, value, epoch
+        FROM %T WHERE %Q %Q',
+      $table->getTableName(),
+      $where,
+      $limit);
+  }
+
+}

src/applications/fact/storage/PhabricatorFactDimension.php

@@ -4,11 +4,30 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
 
   abstract protected function getDimensionColumnName();
 
-  final public function newDimensionID($key) {
-    $map = $this->newDimensionMap(array($key));
+  final public function newDimensionID($key, $create = false) {
+    $map = $this->newDimensionMap(array($key), $create);
     return idx($map, $key);
   }
 
+  final public function newDimensionUnmap(array $ids) {
+    if (!$ids) {
+      return array();
+    }
+
+    $conn = $this->establishConnection('r');
+    $column = $this->getDimensionColumnName();
+
+    $rows = queryfx_all(
+      $conn,
+      'SELECT id, %C FROM %T WHERE id IN (%Ld)',
+      $column,
+      $this->getTableName(),
+      $ids);
+    $rows = ipull($rows, $column, 'id');
+
+    return $rows;
+  }
+
   final public function newDimensionMap(array $keys, $create = false) {
     if (!$keys) {
       return array();
@@ -52,14 +71,16 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
         $key);
     }
 
-    foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
-      queryfx(
-        $conn,
-        'INSERT IGNORE INTO %T (%C) VALUES %Q',
-        $this->getTableName(),
-        $column,
-        $chunk);
-    }
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+      foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+        queryfx(
+          $conn,
+          'INSERT IGNORE INTO %T (%C) VALUES %Q',
+          $this->getTableName(),
+          $column,
+          $chunk);
+      }
+    unset($unguarded);
 
     $rows = queryfx_all(
       $conn,

src/applications/fact/storage/PhabricatorFactIntDatapoint.php

@@ -58,4 +58,30 @@ final class PhabricatorFactIntDatapoint extends PhabricatorFactDAO {
     return $this->dimensionPHID;
   }
 
+  public function newDatapointVector() {
+    return $this->formatVector(
+      array(
+        $this->key,
+        $this->objectPHID,
+        $this->dimensionPHID,
+        $this->value,
+        $this->epoch,
+      ));
+  }
+
+  public function newRawVector(array $spec) {
+    return $this->formatVector(
+      array(
+        $spec['key'],
+        $spec['objectPHID'],
+        $spec['dimensionPHID'],
+        $spec['value'],
+        $spec['epoch'],
+      ));
+  }
+
+  private function formatVector(array $vector) {
+    return implode(':', $vector);
+  }
+
 }