1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-24 14:30:56 +01:00

Provide a page for examining the facts an object generates

Summary:
Depends on D19120. Ref T13083. When you write a fact engine, it's currently somewhat difficult to figure out exactly what it's doing. It would also be difficult to diagnose bugs or report them to the upstream.

To ease this, add a page which shows all the facts an object generates. This allows you to iterate on an engine quickly without needing to reanalyze facts, take a screenshot, easily compare the timeline to the fact view, etc.

Test Plan: Viewed the object fact page for several objects.

Subscribers: yelirekim

Maniphest Tasks: T13083

Differential Revision: https://secure.phabricator.com/D19121
This commit is contained in:
epriestley 2018-02-19 09:35:17 -08:00
parent e3a1a32444
commit 46ce4c7aef
6 changed files with 510 additions and 10 deletions

View file

@ -2916,6 +2916,7 @@ phutil_register_library_map(array(
'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
'PhabricatorFactDatapointQuery' => 'applications/fact/query/PhabricatorFactDatapointQuery.php',
'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php', 'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php', 'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
@ -2928,6 +2929,7 @@ phutil_register_library_map(array(
'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php', 'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
'PhabricatorFactObjectController' => 'applications/fact/controller/PhabricatorFactObjectController.php',
'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php', 'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
@ -8445,6 +8447,7 @@ phutil_register_library_map(array(
'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactCursor' => 'PhabricatorFactDAO',
'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
'PhabricatorFactDaemon' => 'PhabricatorDaemon', 'PhabricatorFactDaemon' => 'PhabricatorDaemon',
'PhabricatorFactDatapointQuery' => 'Phobject',
'PhabricatorFactDimension' => 'PhabricatorFactDAO', 'PhabricatorFactDimension' => 'PhabricatorFactDAO',
'PhabricatorFactEngine' => 'Phobject', 'PhabricatorFactEngine' => 'Phobject',
'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
@ -8457,6 +8460,7 @@ phutil_register_library_map(array(
'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine', 'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
'PhabricatorFactObjectController' => 'PhabricatorFactController',
'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension', 'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
'PhabricatorFactRaw' => 'PhabricatorFactDAO', 'PhabricatorFactRaw' => 'PhabricatorFactDAO',
'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',

View file

@ -31,6 +31,7 @@ final class PhabricatorFactApplication extends PhabricatorApplication {
'/fact/' => array( '/fact/' => array(
'' => 'PhabricatorFactHomeController', '' => 'PhabricatorFactHomeController',
'chart/' => 'PhabricatorFactChartController', 'chart/' => 'PhabricatorFactChartController',
'object/(?<phid>[^/]+)/' => 'PhabricatorFactObjectController',
), ),
); );
} }

View file

@ -0,0 +1,267 @@
<?php
final class PhabricatorFactObjectController
extends PhabricatorFactController {
public function handleRequest(AphrontRequest $request) {
$viewer = $request->getViewer();
$phid = $request->getURIData('phid');
$object = id(new PhabricatorObjectQuery())
->setViewer($viewer)
->withNames(array($phid))
->executeOne();
if (!$object) {
return new Aphront404Response();
}
$engines = PhabricatorFactEngine::loadAllEngines();
foreach ($engines as $key => $engine) {
if (!$engine->supportsDatapointsForObject($object)) {
unset($engines[$key]);
}
}
if (!$engines) {
return $this->newDialog()
->setTitle(pht('No Engines'))
->appendParagraph(
pht(
'No fact engines support generating facts for this object.'))
->addCancelButton($this->getApplicationURI());
}
$key_dimension = new PhabricatorFactKeyDimension();
$object_phid = $object->getPHID();
$facts = array();
$generated_datapoints = array();
$timings = array();
foreach ($engines as $key => $engine) {
$engine_facts = $engine->newFacts();
$engine_facts = mpull($engine_facts, null, 'getKey');
$facts[$key] = $engine_facts;
$t_start = microtime(true);
$generated_datapoints[$key] = $engine->newDatapointsForObject($object);
$t_end = microtime(true);
$timings[$key] = ($t_end - $t_start);
}
$object_id = id(new PhabricatorFactObjectDimension())
->newDimensionID($object_phid, true);
$stored_datapoints = id(new PhabricatorFactDatapointQuery())
->withFacts(array_mergev($facts))
->withObjectPHIDs(array($object_phid))
->needVectors(true)
->execute();
$stored_groups = array();
foreach ($stored_datapoints as $stored_datapoint) {
$stored_groups[$stored_datapoint['key']][] = $stored_datapoint;
}
$stored_map = array();
foreach ($engines as $key => $engine) {
$stored_map[$key] = array();
foreach ($facts[$key] as $fact) {
$fact_datapoints = idx($stored_groups, $fact->getKey(), array());
$fact_datapoints = igroup($fact_datapoints, 'vector');
$stored_map[$key] += $fact_datapoints;
}
}
$handle_phids = array();
$handle_phids[] = $object->getPHID();
foreach ($generated_datapoints as $key => $datapoint_set) {
foreach ($datapoint_set as $datapoint) {
$dimension_phid = $datapoint->getDimensionPHID();
if ($dimension_phid !== null) {
$handle_phids[$dimension_phid] = $dimension_phid;
}
}
}
foreach ($stored_map as $key => $stored_datapoints) {
foreach ($stored_datapoints as $vector_key => $datapoints) {
foreach ($datapoints as $datapoint) {
$dimension_phid = $datapoint['dimensionPHID'];
if ($dimension_phid !== null) {
$handle_phids[$dimension_phid] = $dimension_phid;
}
}
}
}
$handles = $viewer->loadHandles($handle_phids);
$dimension_map = id(new PhabricatorFactObjectDimension())
->newDimensionMap($handle_phids, true);
$content = array();
$object_list = id(new PHUIPropertyListView())
->setViewer($viewer)
->addProperty(
pht('Object'),
$handles[$object->getPHID()]->renderLink());
$total_cost = array_sum($timings);
$total_cost = pht('%sms', new PhutilNumber((int)(1000 * $total_cost)));
$object_list->addProperty(pht('Total Cost'), $total_cost);
$object_box = id(new PHUIObjectBoxView())
->setHeaderText(pht('Fact Extraction Report'))
->addPropertyList($object_list);
$content[] = $object_box;
$icon_fact = id(new PHUIIconView())
->setIcon('fa-line-chart green')
->setTooltip(pht('Consistent Fact'));
$icon_nodata = id(new PHUIIconView())
->setIcon('fa-question-circle-o violet')
->setTooltip(pht('No Stored Datapoints'));
$icon_new = id(new PHUIIconView())
->setIcon('fa-plus red')
->setTooltip(pht('Not Stored'));
$icon_surplus = id(new PHUIIconView())
->setIcon('fa-minus red')
->setTooltip(pht('Not Generated'));
foreach ($engines as $key => $engine) {
$rows = array();
foreach ($generated_datapoints[$key] as $datapoint) {
$dimension_phid = $datapoint->getDimensionPHID();
if ($dimension_phid !== null) {
$dimension = $handles[$datapoint->getDimensionPHID()]->renderLink();
} else {
$dimension = null;
}
$fact_key = $datapoint->getKey();
$fact = idx($facts[$key], $fact_key, null);
if ($fact) {
$fact_label = $fact->getName();
} else {
$fact_label = $fact_key;
}
$vector_key = $datapoint->newDatapointVector();
if (isset($stored_map[$key][$vector_key])) {
unset($stored_map[$key][$vector_key]);
$icon = $icon_fact;
} else {
$icon = $icon_new;
}
$rows[] = array(
$icon,
$fact_label,
$dimension,
$datapoint->getValue(),
phabricator_datetime($datapoint->getEpoch(), $viewer),
);
}
foreach ($stored_map[$key] as $vector_key => $datapoints) {
foreach ($datapoints as $datapoint) {
$dimension_phid = $datapoint['dimensionPHID'];
if ($dimension_phid !== null) {
$dimension = $handles[$dimension_phid]->renderLink();
} else {
$dimension = null;
}
$fact_key = $datapoint['key'];
$fact = idx($facts[$key], $fact_key, null);
if ($fact) {
$fact_label = $fact->getName();
} else {
$fact_label = $fact_key;
}
$rows[] = array(
$icon_surplus,
$fact_label,
$dimension,
$datapoint['value'],
phabricator_datetime($datapoint['epoch'], $viewer),
);
}
}
foreach ($facts[$key] as $fact) {
$has_any = id(new PhabricatorFactDatapointQuery())
->withFacts(array($fact))
->setLimit(1)
->execute();
if ($has_any) {
continue;
}
if (!$has_any) {
$rows[] = array(
$icon_nodata,
$fact->getName(),
null,
null,
null,
);
}
}
$table = id(new AphrontTableView($rows))
->setHeaders(
array(
null,
pht('Fact'),
pht('Dimension'),
pht('Value'),
pht('Date'),
))
->setColumnClasses(
array(
'',
'',
'',
'n wide right',
'right',
));
$extraction_cost = $timings[$key];
$extraction_cost = pht(
'%sms',
new PhutilNumber((int)(1000 * $extraction_cost)));
$header = pht(
'%s (%s)',
get_class($engine),
$extraction_cost);
$box = id(new PHUIObjectBoxView())
->setHeaderText($header)
->setTable($table);
$content[] = $box;
}
$crumbs = $this->buildApplicationCrumbs()
->addTextCrumb(pht('Chart'));
$title = pht('Chart');
return $this->newPage()
->setTitle($title)
->setCrumbs($crumbs)
->appendChild($content);
}
}

View file

@ -0,0 +1,181 @@
<?php
final class PhabricatorFactDatapointQuery extends Phobject {
private $facts;
private $objectPHIDs;
private $limit;
private $needVectors;
private $keyMap = array();
private $dimensionMap = array();
public function withFacts(array $facts) {
$this->facts = $facts;
return $this;
}
public function withObjectPHIDs(array $object_phids) {
$this->objectPHIDs = $object_phids;
return $this;
}
public function setLimit($limit) {
$this->limit = $limit;
return $this;
}
public function needVectors($need) {
$this->needVectors = $need;
return $this;
}
public function execute() {
$facts = mpull($this->facts, null, 'getKey');
if (!$facts) {
throw new Exception(pht('Executing a fact query requires facts.'));
}
$table_map = array();
foreach ($facts as $fact) {
$datapoint = $fact->newDatapoint();
$table = $datapoint->getTableName();
if (!isset($table_map[$table])) {
$table_map[$table] = array(
'table' => $datapoint,
'facts' => array(),
);
}
$table_map[$table]['facts'][] = $fact;
}
$rows = array();
foreach ($table_map as $spec) {
$rows[] = $this->executeWithTable($spec);
}
$rows = array_mergev($rows);
$key_unmap = array_flip($this->keyMap);
$dimension_unmap = array_flip($this->dimensionMap);
$groups = array();
$need_phids = array();
foreach ($rows as $row) {
$groups[$row['keyID']][] = $row;
$object_id = $row['objectID'];
if (!isset($dimension_unmap[$object_id])) {
$need_phids[$object_id] = $object_id;
}
$dimension_id = $row['dimensionID'];
if ($dimension_id && !isset($dimension_unmap[$dimension_id])) {
$need_phids[$dimension_id] = $dimension_id;
}
}
$dimension_unmap += id(new PhabricatorFactObjectDimension())
->newDimensionUnmap($need_phids);
$results = array();
foreach ($groups as $key_id => $rows) {
$key = $key_unmap[$key_id];
$fact = $facts[$key];
$datapoint = $fact->newDatapoint();
foreach ($rows as $row) {
$dimension_id = $row['dimensionID'];
if ($dimension_id) {
if (!isset($dimension_unmap[$dimension_id])) {
continue;
} else {
$dimension_phid = $dimension_unmap[$dimension_id];
}
} else {
$dimension_phid = null;
}
$object_id = $row['objectID'];
if (!isset($dimension_unmap[$object_id])) {
continue;
} else {
$object_phid = $dimension_unmap[$object_id];
}
$result = array(
'key' => $key,
'objectPHID' => $object_phid,
'dimensionPHID' => $dimension_phid,
'value' => (int)$row['value'],
'epoch' => $row['epoch'],
);
if ($this->needVectors) {
$result['vector'] = $datapoint->newRawVector($result);
}
$results[] = $result;
}
}
return $results;
}
private function executeWithTable(array $spec) {
$table = $spec['table'];
$facts = $spec['facts'];
$conn = $table->establishConnection('r');
$fact_keys = mpull($facts, 'getKey');
$this->keyMap = id(new PhabricatorFactKeyDimension())
->newDimensionMap($fact_keys);
if (!$this->keyMap) {
return array();
}
$where = array();
$where[] = qsprintf(
$conn,
'keyID IN (%Ld)',
$this->keyMap);
if ($this->objectPHIDs) {
$object_map = id(new PhabricatorFactObjectDimension())
->newDimensionMap($this->objectPHIDs);
if (!$object_map) {
return array();
}
$this->dimensionMap = $object_map;
$where[] = qsprintf(
$conn,
'objectID IN (%Ld)',
$this->dimensionMap);
}
$where = '('.implode(') AND (', $where).')';
if ($this->limit) {
$limit = qsprintf(
$conn,
'LIMIT %d',
$this->limit);
} else {
$limit = '';
}
return queryfx_all(
$conn,
'SELECT keyID, objectID, dimensionID, value, epoch
FROM %T WHERE %Q %Q',
$table->getTableName(),
$where,
$limit);
}
}

View file

@ -4,11 +4,30 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
abstract protected function getDimensionColumnName(); abstract protected function getDimensionColumnName();
final public function newDimensionID($key) { final public function newDimensionID($key, $create = false) {
$map = $this->newDimensionMap(array($key)); $map = $this->newDimensionMap(array($key), $create);
return idx($map, $key); return idx($map, $key);
} }
final public function newDimensionUnmap(array $ids) {
if (!$ids) {
return array();
}
$conn = $this->establishConnection('r');
$column = $this->getDimensionColumnName();
$rows = queryfx_all(
$conn,
'SELECT id, %C FROM %T WHERE id IN (%Ld)',
$column,
$this->getTableName(),
$ids);
$rows = ipull($rows, $column, 'id');
return $rows;
}
final public function newDimensionMap(array $keys, $create = false) { final public function newDimensionMap(array $keys, $create = false) {
if (!$keys) { if (!$keys) {
return array(); return array();
@ -52,6 +71,7 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
$key); $key);
} }
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx( queryfx(
$conn, $conn,
@ -60,6 +80,7 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
$column, $column,
$chunk); $chunk);
} }
unset($unguarded);
$rows = queryfx_all( $rows = queryfx_all(
$conn, $conn,

View file

@ -58,4 +58,30 @@ final class PhabricatorFactIntDatapoint extends PhabricatorFactDAO {
return $this->dimensionPHID; return $this->dimensionPHID;
} }
public function newDatapointVector() {
return $this->formatVector(
array(
$this->key,
$this->objectPHID,
$this->dimensionPHID,
$this->value,
$this->epoch,
));
}
public function newRawVector(array $spec) {
return $this->formatVector(
array(
$spec['key'],
$spec['objectPHID'],
$spec['dimensionPHID'],
$spec['value'],
$spec['epoch'],
));
}
private function formatVector(array $vector) {
return implode(':', $vector);
}
} }