1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-18 11:30:55 +01:00

Make Facts more modern, DRY, and dimensional

Summary:
Ref T13083. Facts has a fair amount of weird hardcoding and duplication of responsibilities. Reduce this somewhat: no more hard-coded fact aggregates, no more database-driven list of available facts, etc. Generally, derive all objective truth from FactEngines. This is more similar to how most other modern applications work.

For clarity, hopefully: rename "FactSpec" to "Fact". Rename "RawFact" to "Datapoint".

Split the fairly optimistic "RawFact" table into an "IntDatapoint" table with less stuff in it, then dimension tables for the object PHIDs and key names. This is primarily aimed at reducing the row size of each datapoint. At the time I originally wrote this code we hadn't experimented much with storing similar data in multiple tables, but this is now more common and has worked well elsewhere (CustomFields, Edges, Ferret) so I don't anticipate this causing issues. If we need more complex or multidimension/multivalue tables later we can accommodate them. The queries a single table supports (like "all facts of all kinds in some time window") don't make any sense as far as I can tell and could likely be UNION ALL'd anyway.

Remove all the aggregation stuff for now, it's not really clear to me what this should look like.

Test Plan: Ran `bin/fact analyze` and viewed web UI. Nothing exploded too violently.

Subscribers: yelirekim

Maniphest Tasks: T13083

Differential Revision: https://secure.phabricator.com/D19119
This commit is contained in:
epriestley 2018-02-18 06:31:28 -08:00
parent 05a4c55c52
commit 0dee34b3fa
25 changed files with 422 additions and 467 deletions

View file

@ -0,0 +1,5 @@
CREATE TABLE {$NAMESPACE}_fact.fact_keydimension (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
factKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT},
UNIQUE KEY `key_factkey` (factKey)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,5 @@
CREATE TABLE {$NAMESPACE}_fact.fact_objectdimension (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
objectPHID VARBINARY(64) NOT NULL,
UNIQUE KEY `key_object` (objectPHID)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,8 @@
CREATE TABLE {$NAMESPACE}_fact.fact_intdatapoint (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
keyID INT UNSIGNED NOT NULL,
objectID INT UNSIGNED NOT NULL,
dimensionID INT UNSIGNED,
value BIGINT SIGNED NOT NULL,
epoch INT UNSIGNED NOT NULL
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -2907,27 +2907,28 @@ phutil_register_library_map(array(
'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php', 'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php',
'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php', 'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php',
'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php', 'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php',
'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php',
'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php', 'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php',
'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php', 'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php',
'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php', 'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php',
'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php', 'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php',
'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php',
'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php', 'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php', 'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php',
'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php', 'PhabricatorFactIntDatapoint' => 'applications/fact/storage/PhabricatorFactIntDatapoint.php',
'PhabricatorFactKeyDimension' => 'applications/fact/storage/PhabricatorFactKeyDimension.php',
'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php',
'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php', 'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php',
'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php',
'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php',
'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
'PhabricatorFactSimpleSpec' => 'applications/fact/spec/PhabricatorFactSimpleSpec.php',
'PhabricatorFactSpec' => 'applications/fact/spec/PhabricatorFactSpec.php',
'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php', 'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php',
'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php', 'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php',
@ -3716,6 +3717,7 @@ phutil_register_library_map(array(
'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php', 'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php',
'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php', 'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php',
'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php', 'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php',
'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php',
'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php', 'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php',
'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php', 'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php',
'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php', 'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php',
@ -8433,27 +8435,28 @@ phutil_register_library_map(array(
'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel',
'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider', 'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider',
'PhabricatorFact' => 'Phobject',
'PhabricatorFactAggregate' => 'PhabricatorFactDAO', 'PhabricatorFactAggregate' => 'PhabricatorFactDAO',
'PhabricatorFactApplication' => 'PhabricatorApplication', 'PhabricatorFactApplication' => 'PhabricatorApplication',
'PhabricatorFactChartController' => 'PhabricatorFactController', 'PhabricatorFactChartController' => 'PhabricatorFactController',
'PhabricatorFactController' => 'PhabricatorController', 'PhabricatorFactController' => 'PhabricatorController',
'PhabricatorFactCountEngine' => 'PhabricatorFactEngine',
'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactCursor' => 'PhabricatorFactDAO',
'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
'PhabricatorFactDaemon' => 'PhabricatorDaemon', 'PhabricatorFactDaemon' => 'PhabricatorDaemon',
'PhabricatorFactDimension' => 'PhabricatorFactDAO',
'PhabricatorFactEngine' => 'Phobject', 'PhabricatorFactEngine' => 'Phobject',
'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
'PhabricatorFactHomeController' => 'PhabricatorFactController', 'PhabricatorFactHomeController' => 'PhabricatorFactController',
'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine', 'PhabricatorFactIntDatapoint' => 'PhabricatorFactDAO',
'PhabricatorFactKeyDimension' => 'PhabricatorFactDimension',
'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow',
'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
'PhabricatorFactRaw' => 'PhabricatorFactDAO', 'PhabricatorFactRaw' => 'PhabricatorFactDAO',
'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec',
'PhabricatorFactSpec' => 'Phobject',
'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',
'PhabricatorFavoritesApplication' => 'PhabricatorApplication', 'PhabricatorFavoritesApplication' => 'PhabricatorApplication',
'PhabricatorFavoritesController' => 'PhabricatorController', 'PhabricatorFavoritesController' => 'PhabricatorController',
@ -9373,6 +9376,7 @@ phutil_register_library_map(array(
'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation', 'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation',
'PhabricatorPlatformSite' => 'PhabricatorSite', 'PhabricatorPlatformSite' => 'PhabricatorSite',
'PhabricatorPointsEditField' => 'PhabricatorEditField', 'PhabricatorPointsEditField' => 'PhabricatorEditField',
'PhabricatorPointsFact' => 'PhabricatorFact',
'PhabricatorPolicies' => 'PhabricatorPolicyConstants', 'PhabricatorPolicies' => 'PhabricatorPolicyConstants',
'PhabricatorPolicy' => array( 'PhabricatorPolicy' => array(
'PhabricatorPolicyDAO', 'PhabricatorPolicyDAO',

View file

@ -35,12 +35,6 @@ final class PhabricatorDifferentialApplication extends PhabricatorApplication {
); );
} }
public function getFactObjectsForAnalysis() {
return array(
new DifferentialRevision(),
);
}
public function getTitleGlyph() { public function getTitleGlyph() {
return "\xE2\x9A\x99"; return "\xE2\x9A\x99";
} }

View file

@ -39,12 +39,6 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
); );
} }
public function getFactObjectsForAnalysis() {
return array(
new PhabricatorRepositoryCommit(),
);
}
public function getRemarkupRules() { public function getRemarkupRules() {
return array( return array(
new DiffusionCommitRemarkupRule(), new DiffusionCommitRemarkupRule(),

View file

@ -5,27 +5,32 @@ final class PhabricatorFactChartController extends PhabricatorFactController {
public function handleRequest(AphrontRequest $request) { public function handleRequest(AphrontRequest $request) {
$viewer = $request->getViewer(); $viewer = $request->getViewer();
$table = new PhabricatorFactRaw(); $series = $request->getStr('y1');
$facts = PhabricatorFact::getAllFacts();
$fact = idx($facts, $series);
if (!$fact) {
return new Aphront404Response();
}
$key_id = id(new PhabricatorFactKeyDimension())
->newDimensionID($fact->getKey());
$table = $fact->newDatapoint();
$conn_r = $table->establishConnection('r'); $conn_r = $table->establishConnection('r');
$table_name = $table->getTableName(); $table_name = $table->getTableName();
$series = $request->getStr('y1');
$specs = PhabricatorFactSpec::newSpecsForFactTypes(
PhabricatorFactEngine::loadAllEngines(),
array($series));
$spec = idx($specs, $series);
$data = queryfx_all( $data = queryfx_all(
$conn_r, $conn_r,
'SELECT valueX, epoch FROM %T WHERE factType = %s ORDER BY epoch ASC', 'SELECT value, epoch FROM %T WHERE keyID = %d ORDER BY epoch ASC',
$table_name, $table_name,
$series); $key_id);
$points = array(); $points = array();
$sum = 0; $sum = 0;
foreach ($data as $key => $row) { foreach ($data as $key => $row) {
$sum += (int)$row['valueX']; $sum += (int)$row['value'];
$points[(int)$row['epoch']] = $sum; $points[(int)$row['epoch']] = $sum;
} }
@ -71,7 +76,7 @@ final class PhabricatorFactChartController extends PhabricatorFactController {
)); ));
$box = id(new PHUIObjectBoxView()) $box = id(new PHUIObjectBoxView())
->setHeaderText(pht('Count of %s', $spec->getName())) ->setHeaderText(pht('Count of %s', $fact->getName()))
->appendChild($chart); ->appendChild($chart);
$crumbs = $this->buildApplicationCrumbs(); $crumbs = $this->buildApplicationCrumbs();

View file

@ -15,45 +15,6 @@ final class PhabricatorFactHomeController extends PhabricatorFactController {
return id(new AphrontRedirectResponse())->setURI($uri); return id(new AphrontRedirectResponse())->setURI($uri);
} }
$types = array(
'+N:*',
'+N:DREV',
'updated',
);
$engines = PhabricatorFactEngine::loadAllEngines();
$specs = PhabricatorFactSpec::newSpecsForFactTypes($engines, $types);
$facts = id(new PhabricatorFactAggregate())->loadAllWhere(
'factType IN (%Ls)',
$types);
$rows = array();
foreach ($facts as $fact) {
$spec = $specs[$fact->getFactType()];
$name = $spec->getName();
$value = $spec->formatValueForDisplay($viewer, $fact->getValueX());
$rows[] = array($name, $value);
}
$table = new AphrontTableView($rows);
$table->setHeaders(
array(
pht('Fact'),
pht('Value'),
));
$table->setColumnClasses(
array(
'wide',
'n',
));
$panel = new PHUIObjectBoxView();
$panel->setHeaderText(pht('Facts'));
$panel->setTable($table);
$chart_form = $this->buildChartForm(); $chart_form = $this->buildChartForm();
$crumbs = $this->buildApplicationCrumbs(); $crumbs = $this->buildApplicationCrumbs();
@ -64,46 +25,18 @@ final class PhabricatorFactHomeController extends PhabricatorFactController {
return $this->newPage() return $this->newPage()
->setTitle($title) ->setTitle($title)
->setCrumbs($crumbs) ->setCrumbs($crumbs)
->appendChild(array( ->appendChild(
array(
$chart_form, $chart_form,
$panel,
)); ));
} }
private function buildChartForm() { private function buildChartForm() {
$request = $this->getRequest(); $request = $this->getRequest();
$viewer = $request->getUser(); $viewer = $request->getUser();
$table = new PhabricatorFactRaw(); $specs = PhabricatorFact::getAllFacts();
$conn_r = $table->establishConnection('r'); $options = mpull($specs, 'getName', 'getKey');
$table_name = $table->getTableName();
$facts = queryfx_all(
$conn_r,
'SELECT DISTINCT factType from %T',
$table_name);
$specs = PhabricatorFactSpec::newSpecsForFactTypes(
PhabricatorFactEngine::loadAllEngines(),
ipull($facts, 'factType'));
$options = array();
foreach ($specs as $spec) {
if ($spec->getUnit() == PhabricatorFactSpec::UNIT_COUNT) {
$options[$spec->getType()] = $spec->getName();
}
}
if (!$options) {
return id(new PHUIInfoView())
->setSeverity(PHUIInfoView::SEVERITY_NODATA)
->setTitle(pht('No Chartable Facts'))
->appendChild(phutil_tag(
'p',
array(),
pht('There are no facts that can be plotted yet.')));
}
$form = id(new AphrontFormView()) $form = id(new AphrontFormView())
->setUser($viewer) ->setUser($viewer)

View file

@ -4,8 +4,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
private $engines; private $engines;
const RAW_FACT_BUFFER_LIMIT = 128;
protected function run() { protected function run() {
$this->setEngines(PhabricatorFactEngine::loadAllEngines()); $this->setEngines(PhabricatorFactEngine::loadAllEngines());
while (!$this->shouldExit()) { while (!$this->shouldExit()) {
@ -15,7 +13,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
foreach ($iterators as $iterator_name => $iterator) { foreach ($iterators as $iterator_name => $iterator) {
$this->processIteratorWithCursor($iterator_name, $iterator); $this->processIteratorWithCursor($iterator_name, $iterator);
} }
$this->processAggregates();
$this->log(pht('Zzz...')); $this->log(pht('Zzz...'));
$this->sleep(60 * 5); $this->sleep(60 * 5);
@ -72,59 +69,41 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
public function processIterator($iterator) { public function processIterator($iterator) {
$result = null; $result = null;
$raw_facts = array(); $datapoints = array();
foreach ($iterator as $key => $object) { foreach ($iterator as $key => $object) {
$phid = $object->getPHID(); $phid = $object->getPHID();
$this->log(pht('Processing %s...', $phid)); $this->log(pht('Processing %s...', $phid));
$raw_facts[$phid] = $this->computeRawFacts($object); $datapoints[$phid] = $this->newDatapoints($object);
if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) { if (count($datapoints) > 1024) {
$this->updateRawFacts($raw_facts); $this->updateDatapoints($datapoints);
$raw_facts = array(); $datapoints = array();
} }
$result = $key; $result = $key;
} }
if ($raw_facts) { if ($datapoints) {
$this->updateRawFacts($raw_facts); $this->updateDatapoints($datapoints);
$raw_facts = array(); $datapoints = array();
} }
return $result; return $result;
} }
public function processAggregates() { private function newDatapoints(PhabricatorLiskDAO $object) {
$this->log(pht('Processing aggregates.'));
$facts = $this->computeAggregateFacts();
$this->updateAggregateFacts($facts);
}
private function computeAggregateFacts() {
$facts = array(); $facts = array();
foreach ($this->engines as $engine) { foreach ($this->engines as $engine) {
if (!$engine->shouldComputeAggregateFacts()) { if (!$engine->supportsDatapointsForObject($object)) {
continue; continue;
} }
$facts[] = $engine->computeAggregateFacts(); $facts[] = $engine->newDatapointsForObject($object);
}
return array_mergev($facts);
}
private function computeRawFacts(PhabricatorLiskDAO $object) {
$facts = array();
foreach ($this->engines as $engine) {
if (!$engine->shouldComputeRawFactsForObject($object)) {
continue;
}
$facts[] = $engine->computeRawFactsForObject($object);
} }
return array_mergev($facts); return array_mergev($facts);
} }
private function updateRawFacts(array $map) { private function updateDatapoints(array $map) {
foreach ($map as $phid => $facts) { foreach ($map as $phid => $facts) {
assert_instances_of($facts, 'PhabricatorFactRaw'); assert_instances_of($facts, 'PhabricatorFactIntDatapoint');
} }
$phids = array_keys($map); $phids = array_keys($map);
@ -132,76 +111,79 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
return; return;
} }
$table = new PhabricatorFactRaw();
$fact_keys = array();
$objects = array();
foreach ($map as $phid => $facts) {
foreach ($facts as $fact) {
$fact_keys[$fact->getKey()] = true;
$object_phid = $fact->getObjectPHID();
$objects[$object_phid] = $object_phid;
$dimension_phid = $fact->getDimensionPHID();
if ($dimension_phid !== null) {
$objects[$dimension_phid] = $dimension_phid;
}
}
}
$key_map = id(new PhabricatorFactKeyDimension())
->newDimensionMap(array_keys($fact_keys));
$object_map = id(new PhabricatorFactObjectDimension())
->newDimensionMap(array_keys($objects));
$table = new PhabricatorFactIntDatapoint();
$conn = $table->establishConnection('w'); $conn = $table->establishConnection('w');
$table_name = $table->getTableName(); $table_name = $table->getTableName();
$sql = array(); $sql = array();
foreach ($map as $phid => $facts) { foreach ($map as $phid => $facts) {
foreach ($facts as $fact) { foreach ($facts as $fact) {
$key_id = $key_map[$fact->getKey()];
$object_id = $object_map[$fact->getObjectPHID()];
$dimension_phid = $fact->getDimensionPHID();
if ($dimension_phid !== null) {
$dimension_id = $object_map[$dimension_phid];
} else {
$dimension_id = null;
}
$sql[] = qsprintf( $sql[] = qsprintf(
$conn, $conn,
'(%s, %s, %s, %d, %d, %d)', '(%d, %d, %nd, %d, %d)',
$fact->getFactType(), $key_id,
$fact->getObjectPHID(), $object_id,
$fact->getObjectA(), $dimension_id,
$fact->getValueX(), $fact->getValue(),
$fact->getValueY(),
$fact->getEpoch()); $fact->getEpoch());
} }
} }
$rebuilt_ids = array_select_keys($object_map, $phids);
$table->openTransaction(); $table->openTransaction();
queryfx( queryfx(
$conn, $conn,
'DELETE FROM %T WHERE objectPHID IN (%Ls)', 'DELETE FROM %T WHERE objectID IN (%Ld)',
$table_name, $table_name,
$phids); $rebuilt_ids);
if ($sql) { if ($sql) {
foreach (array_chunk($sql, 256) as $chunk) { foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx( queryfx(
$conn, $conn,
'INSERT INTO %T 'INSERT INTO %T
(factType, objectPHID, objectA, valueX, valueY, epoch) (keyID, objectID, dimensionID, value, epoch)
VALUES %Q', VALUES %Q',
$table_name, $table_name,
implode(', ', $chunk)); $chunk);
} }
} }
$table->saveTransaction(); $table->saveTransaction();
} }
private function updateAggregateFacts(array $facts) {
if (!$facts) {
return;
}
$table = new PhabricatorFactAggregate();
$conn = $table->establishConnection('w');
$table_name = $table->getTableName();
$sql = array();
foreach ($facts as $fact) {
$sql[] = qsprintf(
$conn,
'(%s, %s, %d)',
$fact->getFactType(),
$fact->getObjectPHID(),
$fact->getValueX());
}
foreach (array_chunk($sql, 256) as $chunk) {
queryfx(
$conn,
'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q
ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)',
$table_name,
implode(', ', $chunk));
}
}
} }

View file

@ -1,86 +0,0 @@
<?php
/**
* Simple fact engine which counts objects.
*/
final class PhabricatorFactCountEngine extends PhabricatorFactEngine {
public function getFactSpecs(array $fact_types) {
$results = array();
foreach ($fact_types as $type) {
if (!strncmp($type, '+N:', 3)) {
if ($type == '+N:*') {
$name = pht('Total Objects');
} else {
$name = pht('Total Objects of type %s', substr($type, 3));
}
$results[] = id(new PhabricatorFactSimpleSpec($type))
->setName($name)
->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
}
if (!strncmp($type, 'N:', 2)) {
if ($type == 'N:*') {
$name = pht('Objects');
} else {
$name = pht('Objects of type %s', substr($type, 2));
}
$results[] = id(new PhabricatorFactSimpleSpec($type))
->setName($name)
->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
}
}
return $results;
}
public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) {
return true;
}
public function computeRawFactsForObject(PhabricatorLiskDAO $object) {
$facts = array();
$phid = $object->getPHID();
$type = phid_get_type($phid);
foreach (array('N:*', 'N:'.$type) as $fact_type) {
$facts[] = id(new PhabricatorFactRaw())
->setFactType($fact_type)
->setObjectPHID($phid)
->setValueX(1)
->setEpoch($object->getDateCreated());
}
return $facts;
}
public function shouldComputeAggregateFacts() {
return true;
}
public function computeAggregateFacts() {
$table = new PhabricatorFactRaw();
$table_name = $table->getTableName();
$conn = $table->establishConnection('r');
$counts = queryfx_all(
$conn,
'SELECT factType, SUM(valueX) N FROM %T WHERE factType LIKE %>
GROUP BY factType',
$table_name,
'N:');
$facts = array();
foreach ($counts as $count) {
$facts[] = id(new PhabricatorFactAggregate())
->setFactType('+'.$count['factType'])
->setValueX($count['N']);
}
return $facts;
}
}

View file

@ -2,30 +2,37 @@
abstract class PhabricatorFactEngine extends Phobject { abstract class PhabricatorFactEngine extends Phobject {
private $factMap;
final public static function loadAllEngines() { final public static function loadAllEngines() {
return id(new PhutilClassMapQuery()) return id(new PhutilClassMapQuery())
->setAncestorClass(__CLASS__) ->setAncestorClass(__CLASS__)
->execute(); ->execute();
} }
public function getFactSpecs(array $fact_types) { abstract public function newFacts();
return array();
abstract public function supportsDatapointsForObject(
PhabricatorLiskDAO $object);
abstract public function newDatapointsForObject(PhabricatorLiskDAO $object);
final protected function getFact($key) {
if ($this->factMap === null) {
$facts = $this->newFacts();
$facts = mpull($facts, null, 'getKey');
$this->factMap = $facts;
} }
public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { if (!isset($this->factMap[$key])) {
return false; throw new Exception(
pht(
'Unknown fact ("%s") for engine "%s".',
$key,
get_class($this)));
} }
public function computeRawFactsForObject(PhabricatorLiskDAO $object) { return $this->factMap[$key];
return array();
}
public function shouldComputeAggregateFacts() {
return false;
}
public function computeAggregateFacts() {
return array();
} }
} }

View file

@ -1,34 +0,0 @@
<?php
/**
* Engine that records the time facts were last updated.
*/
final class PhabricatorFactLastUpdatedEngine extends PhabricatorFactEngine {
public function getFactSpecs(array $fact_types) {
$results = array();
foreach ($fact_types as $type) {
if ($type == 'updated') {
$results[] = id(new PhabricatorFactSimpleSpec($type))
->setName(pht('Facts Last Updated'))
->setUnit(PhabricatorFactSimpleSpec::UNIT_EPOCH);
}
}
return $results;
}
public function shouldComputeAggregateFacts() {
return true;
}
public function computeAggregateFacts() {
$facts = array();
$facts[] = id(new PhabricatorFactAggregate())
->setFactType('updated')
->setValueX(time());
return $facts;
}
}

View file

@ -0,0 +1,34 @@
<?php
final class PhabricatorFactManiphestTaskEngine
extends PhabricatorFactEngine {
public function newFacts() {
return array(
id(new PhabricatorPointsFact())
->setKey('tasks.count.open'),
);
}
public function supportsDatapointsForObject(PhabricatorLiskDAO $object) {
return ($object instanceof ManiphestTask);
}
public function newDatapointsForObject(PhabricatorLiskDAO $object) {
$datapoints = array();
$phid = $object->getPHID();
$type = phid_get_type($phid);
$datapoint = $this->getFact('tasks.count.open')
->newDatapoint();
$datapoints[] = $datapoint
->setObjectPHID($phid)
->setValue(1)
->setEpoch($object->getDateCreated());
return $datapoints;
}
}

View file

@ -0,0 +1,40 @@
<?php
abstract class PhabricatorFact extends Phobject {
private $key;
public static function getAllFacts() {
$engines = PhabricatorFactEngine::loadAllEngines();
$map = array();
foreach ($engines as $engine) {
$facts = $engine->newFacts();
$facts = mpull($facts, null, 'getKey');
$map += $facts;
}
return $map;
}
final public function setKey($key) {
$this->key = $key;
return $this;
}
final public function getKey() {
return $this->key;
}
final public function getName() {
return pht('Fact "%s"', $this->getKey());
}
final public function newDatapoint() {
return $this->newTemplateDatapoint()
->setKey($this->getKey());
}
abstract protected function newTemplateDatapoint();
}

View file

@ -0,0 +1,9 @@
<?php
final class PhabricatorPointsFact extends PhabricatorFact {
protected function newTemplateDatapoint() {
return new PhabricatorFactIntDatapoint();
}
}

View file

@ -58,10 +58,6 @@ final class PhabricatorFactManagementAnalyzeWorkflow
} }
} }
if (!$args->getArg('skip-aggregates')) {
$daemon->processAggregates();
}
return 0; return 0;
} }

View file

@ -23,8 +23,13 @@ final class PhabricatorFactManagementDestroyWorkflow
} }
$tables = array(); $tables = array();
$tables[] = new PhabricatorFactRaw(); $tables[] = new PhabricatorFactCursor();
$tables[] = new PhabricatorFactAggregate();
$tables[] = new PhabricatorFactIntDatapoint();
$tables[] = new PhabricatorFactObjectDimension();
$tables[] = new PhabricatorFactKeyDimension();
foreach ($tables as $table) { foreach ($tables as $table) {
$conn = $table->establishConnection('w'); $conn = $table->establishConnection('w');
$name = $table->getTableName(); $name = $table->getTableName();

View file

@ -1,47 +0,0 @@
<?php
final class PhabricatorFactManagementStatusWorkflow
extends PhabricatorFactManagementWorkflow {
protected function didConstruct() {
$this
->setName('status')
->setSynopsis(pht('Show status of fact data.'))
->setArguments(array());
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$map = array(
'raw' => new PhabricatorFactRaw(),
'agg' => new PhabricatorFactAggregate(),
);
foreach ($map as $type => $table) {
$conn = $table->establishConnection('r');
$name = $table->getTableName();
$row = queryfx_one(
$conn,
'SELECT COUNT(*) N FROM %T',
$name);
$n = $row['N'];
switch ($type) {
case 'raw':
$desc = pht('There are %d raw fact(s) in storage.', $n);
break;
case 'agg':
$desc = pht('There are %d aggregate fact(s) in storage.', $n);
break;
}
$console->writeOut("%s\n", $desc);
}
return 0;
}
}

View file

@ -1,38 +0,0 @@
<?php
final class PhabricatorFactSimpleSpec extends PhabricatorFactSpec {
private $type;
private $name;
private $unit;
public function __construct($type) {
$this->type = $type;
}
public function getType() {
return $this->type;
}
public function setUnit($unit) {
$this->unit = $unit;
return $this;
}
public function getUnit() {
return $this->unit;
}
public function setName($name) {
$this->name = $name;
return $this;
}
public function getName() {
if ($this->name !== null) {
return $this->name;
}
return parent::getName();
}
}

View file

@ -1,53 +0,0 @@
<?php
abstract class PhabricatorFactSpec extends Phobject {
const UNIT_COUNT = 'unit-count';
const UNIT_EPOCH = 'unit-epoch';
public static function newSpecsForFactTypes(
array $engines,
array $fact_types) {
assert_instances_of($engines, 'PhabricatorFactEngine');
$map = array();
foreach ($engines as $engine) {
$specs = $engine->getFactSpecs($fact_types);
$specs = mpull($specs, null, 'getType');
$map += $specs;
}
foreach ($fact_types as $type) {
if (empty($map[$type])) {
$map[$type] = new PhabricatorFactSimpleSpec($type);
}
}
return $map;
}
abstract public function getType();
public function getUnit() {
return null;
}
public function getName() {
return pht(
'Fact (%s)',
$this->getType());
}
public function formatValueForDisplay(PhabricatorUser $user, $value) {
$unit = $this->getUnit();
switch ($unit) {
case self::UNIT_COUNT:
return number_format($value);
case self::UNIT_EPOCH:
return phabricator_datetime($value, $user);
default:
return $value;
}
}
}

View file

@ -0,0 +1,85 @@
<?php
abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
abstract protected function getDimensionColumnName();
final public function newDimensionID($key) {
$map = $this->newDimensionMap(array($key));
return $map[$key];
}
final public function newDimensionMap(array $keys) {
if (!$keys) {
return array();
}
$conn = $this->establishConnection('r');
$column = $this->getDimensionColumnName();
$rows = queryfx_all(
$conn,
'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
$column,
$this->getTableName(),
$column,
$keys);
$rows = ipull($rows, 'id', $column);
$map = array();
$need = array();
foreach ($keys as $key) {
if (isset($rows[$key])) {
$map[$key] = (int)$rows[$key];
} else {
$need[] = $key;
}
}
if (!$need) {
return $map;
}
$sql = array();
foreach ($need as $key) {
$sql[] = qsprintf(
$conn,
'(%s)',
$key);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT IGNORE INTO %T (%C) VALUES %Q',
$this->getTableName(),
$column,
$chunk);
}
$rows = queryfx_all(
$conn,
'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
$column,
$this->getTableName(),
$column,
$need);
$rows = ipull($rows, 'id', $column);
foreach ($keys as $key) {
if (isset($rows[$key])) {
$map[$key] = (int)$rows[$key];
} else {
throw new Exception(
pht(
'Failed to load or generate dimension ID ("%s") for dimension '.
'key "%s".',
get_class($this),
$key));
}
}
return $map;
}
}

View file

@ -0,0 +1,61 @@
<?php
final class PhabricatorFactIntDatapoint extends PhabricatorFactDAO {
protected $keyID;
protected $objectID;
protected $dimensionID;
protected $value;
protected $epoch;
private $key;
private $objectPHID;
private $dimensionPHID;
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(
'id' => 'auto64',
'dimensionID' => 'id?',
'value' => 'sint64',
),
self::CONFIG_KEY_SCHEMA => array(
'key_dimension' => array(
'columns' => array('keyID', 'dimensionID'),
),
'key_object' => array(
'columns' => array('objectID'),
),
),
) + parent::getConfiguration();
}
public function setKey($key) {
$this->key = $key;
return $this;
}
public function getKey() {
return $this->key;
}
public function setObjectPHID($object_phid) {
$this->objectPHID = $object_phid;
return $this;
}
public function getObjectPHID() {
return $this->objectPHID;
}
public function setDimensionPHID($dimension_phid) {
$this->dimensionPHID = $dimension_phid;
return $this;
}
public function getDimensionPHID() {
return $this->dimensionPHID;
}
}

View file

@ -0,0 +1,27 @@
<?php
final class PhabricatorFactKeyDimension
extends PhabricatorFactDimension {
protected $factKey;
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(
'factKey' => 'text64',
),
self::CONFIG_KEY_SCHEMA => array(
'key_factkey' => array(
'columns' => array('factKey'),
'unique' => true,
),
),
) + parent::getConfiguration();
}
protected function getDimensionColumnName() {
return 'factKey';
}
}

View file

@ -0,0 +1,25 @@
<?php
final class PhabricatorFactObjectDimension
extends PhabricatorFactDimension {
protected $objectPHID;
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(),
self::CONFIG_KEY_SCHEMA => array(
'key_object' => array(
'columns' => array('objectPHID'),
'unique' => true,
),
),
) + parent::getConfiguration();
}
protected function getDimensionColumnName() {
return 'objectPHID';
}
}

View file

@ -18,12 +18,6 @@ final class PhabricatorPonderApplication extends PhabricatorApplication {
return 'fa-university'; return 'fa-university';
} }
public function getFactObjectsForAnalysis() {
return array(
new PonderQuestion(),
);
}
public function getTitleGlyph() { public function getTitleGlyph() {
return "\xE2\x97\xB3"; return "\xE2\x97\xB3";
} }