From 0dee34b3fafd2d7fd5011ea23fae08f9bb8eabf5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 18 Feb 2018 06:31:28 -0800 Subject: [PATCH] Make Facts more modern, DRY, and dimensional Summary: Ref T13083. Facts has a fair amount of weird hardcoding and duplication of responsibilities. Reduce this somewhat: no more hard-coded fact aggregates, no more database-driven list of available facts, etc. Generally, derive all objective truth from FactEngines. This is more similar to how most other modern applications work. For clarity, hopefully: rename "FactSpec" to "Fact". Rename "RawFact" to "Datapoint". Split the fairly optimistic "RawFact" table into an "IntDatapoint" table with less stuff in it, then dimension tables for the object PHIDs and key names. This is primarily aimed at reducing the row size of each datapoint. At the time I originally wrote this code we hadn't experimented much with storing similar data in multiple tables, but this is now more common and has worked well elsewhere (CustomFields, Edges, Ferret) so I don't anticipate this causing issues. If we need more complex or multidimension/multivalue tables later we can accommodate them. The queries a single table supports (like "all facts of all kinds in some time window") don't make any sense as far as I can tell and could likely be UNION ALL'd anyway. Remove all the aggregation stuff for now, it's not really clear to me what this should look like. Test Plan: Ran `bin/fact analyze` and viewed web UI. Nothing exploded too violently. Subscribers: yelirekim Maniphest Tasks: T13083 Differential Revision: https://secure.phabricator.com/D19119 --- .../autopatches/20180218.fact.01.dim.key.sql | 5 + .../autopatches/20180218.fact.02.dim.obj.sql | 5 + .../autopatches/20180218.fact.03.data.int.sql | 8 ++ src/__phutil_library_map__.php | 24 ++-- .../PhabricatorDifferentialApplication.php | 6 - .../PhabricatorDiffusionApplication.php | 6 - .../PhabricatorFactChartController.php | 29 ++-- .../PhabricatorFactHomeController.php | 79 +---------- .../fact/daemon/PhabricatorFactDaemon.php | 134 ++++++++---------- .../engine/PhabricatorFactCountEngine.php | 86 ----------- .../fact/engine/PhabricatorFactEngine.php | 35 +++-- .../PhabricatorFactLastUpdatedEngine.php | 34 ----- .../PhabricatorFactManiphestTaskEngine.php | 34 +++++ .../fact/fact/PhabricatorFact.php | 40 ++++++ .../fact/fact/PhabricatorPointsFact.php | 9 ++ ...abricatorFactManagementAnalyzeWorkflow.php | 4 - ...abricatorFactManagementDestroyWorkflow.php | 9 +- ...habricatorFactManagementStatusWorkflow.php | 47 ------ .../fact/spec/PhabricatorFactSimpleSpec.php | 38 ----- .../fact/spec/PhabricatorFactSpec.php | 53 ------- .../fact/storage/PhabricatorFactDimension.php | 85 +++++++++++ .../storage/PhabricatorFactIntDatapoint.php | 61 ++++++++ .../storage/PhabricatorFactKeyDimension.php | 27 ++++ .../PhabricatorFactObjectDimension.php | 25 ++++ .../PhabricatorPonderApplication.php | 6 - 25 files changed, 422 insertions(+), 467 deletions(-) create mode 100644 resources/sql/autopatches/20180218.fact.01.dim.key.sql create mode 100644 resources/sql/autopatches/20180218.fact.02.dim.obj.sql create mode 100644 resources/sql/autopatches/20180218.fact.03.data.int.sql delete mode 100644 src/applications/fact/engine/PhabricatorFactCountEngine.php delete mode 100644 src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php create mode 100644 src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php create mode 100644 src/applications/fact/fact/PhabricatorFact.php create mode 100644 src/applications/fact/fact/PhabricatorPointsFact.php delete mode 100644 src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php delete mode 100644 src/applications/fact/spec/PhabricatorFactSimpleSpec.php delete mode 100644 src/applications/fact/spec/PhabricatorFactSpec.php create mode 100644 src/applications/fact/storage/PhabricatorFactDimension.php create mode 100644 src/applications/fact/storage/PhabricatorFactIntDatapoint.php create mode 100644 src/applications/fact/storage/PhabricatorFactKeyDimension.php create mode 100644 src/applications/fact/storage/PhabricatorFactObjectDimension.php diff --git a/resources/sql/autopatches/20180218.fact.01.dim.key.sql b/resources/sql/autopatches/20180218.fact.01.dim.key.sql new file mode 100644 index 0000000000..3a81915026 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.01.dim.key.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_keydimension ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + factKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT}, + UNIQUE KEY `key_factkey` (factKey) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180218.fact.02.dim.obj.sql b/resources/sql/autopatches/20180218.fact.02.dim.obj.sql new file mode 100644 index 0000000000..6b38062b29 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.02.dim.obj.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_objectdimension ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + UNIQUE KEY `key_object` (objectPHID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180218.fact.03.data.int.sql b/resources/sql/autopatches/20180218.fact.03.data.int.sql new file mode 100644 index 0000000000..d93d546733 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.03.data.int.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_intdatapoint ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + keyID INT UNSIGNED NOT NULL, + objectID INT UNSIGNED NOT NULL, + dimensionID INT UNSIGNED, + value BIGINT SIGNED NOT NULL, + epoch INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 29a957ae6e..3fe0cba001 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2907,27 +2907,28 @@ phutil_register_library_map(array( 'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php', 'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php', 'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php', + 'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php', 'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php', 'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php', 'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php', 'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php', - 'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php', 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', + 'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php', 'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php', - 'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php', + 'PhabricatorFactIntDatapoint' => 'applications/fact/storage/PhabricatorFactIntDatapoint.php', + 'PhabricatorFactKeyDimension' => 'applications/fact/storage/PhabricatorFactKeyDimension.php', 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', 'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php', 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', - 'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php', 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', + 'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php', + 'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php', 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', - 'PhabricatorFactSimpleSpec' => 'applications/fact/spec/PhabricatorFactSimpleSpec.php', - 'PhabricatorFactSpec' => 'applications/fact/spec/PhabricatorFactSpec.php', 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', 'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php', 'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php', @@ -3716,6 +3717,7 @@ phutil_register_library_map(array( 'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php', 'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php', 'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php', + 'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php', 'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php', 'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php', 'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php', @@ -8433,27 +8435,28 @@ phutil_register_library_map(array( 'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider', + 'PhabricatorFact' => 'Phobject', 'PhabricatorFactAggregate' => 'PhabricatorFactDAO', 'PhabricatorFactApplication' => 'PhabricatorApplication', 'PhabricatorFactChartController' => 'PhabricatorFactController', 'PhabricatorFactController' => 'PhabricatorController', - 'PhabricatorFactCountEngine' => 'PhabricatorFactEngine', 'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDaemon' => 'PhabricatorDaemon', + 'PhabricatorFactDimension' => 'PhabricatorFactDAO', 'PhabricatorFactEngine' => 'Phobject', 'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorFactHomeController' => 'PhabricatorFactController', - 'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactIntDatapoint' => 'PhabricatorFactDAO', + 'PhabricatorFactKeyDimension' => 'PhabricatorFactDimension', 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', - 'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension', 'PhabricatorFactRaw' => 'PhabricatorFactDAO', - 'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec', - 'PhabricatorFactSpec' => 'Phobject', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFavoritesApplication' => 'PhabricatorApplication', 'PhabricatorFavoritesController' => 'PhabricatorController', @@ -9373,6 +9376,7 @@ phutil_register_library_map(array( 'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation', 'PhabricatorPlatformSite' => 'PhabricatorSite', 'PhabricatorPointsEditField' => 'PhabricatorEditField', + 'PhabricatorPointsFact' => 'PhabricatorFact', 'PhabricatorPolicies' => 'PhabricatorPolicyConstants', 'PhabricatorPolicy' => array( 'PhabricatorPolicyDAO', diff --git a/src/applications/differential/application/PhabricatorDifferentialApplication.php b/src/applications/differential/application/PhabricatorDifferentialApplication.php index 1c8926f585..4141cf8539 100644 --- a/src/applications/differential/application/PhabricatorDifferentialApplication.php +++ b/src/applications/differential/application/PhabricatorDifferentialApplication.php @@ -35,12 +35,6 @@ final class PhabricatorDifferentialApplication extends PhabricatorApplication { ); } - public function getFactObjectsForAnalysis() { - return array( - new DifferentialRevision(), - ); - } - public function getTitleGlyph() { return "\xE2\x9A\x99"; } diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php index e619ecb1ad..d42e58b747 100644 --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -39,12 +39,6 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication { ); } - public function getFactObjectsForAnalysis() { - return array( - new PhabricatorRepositoryCommit(), - ); - } - public function getRemarkupRules() { return array( new DiffusionCommitRemarkupRule(), diff --git a/src/applications/fact/controller/PhabricatorFactChartController.php b/src/applications/fact/controller/PhabricatorFactChartController.php index 16df8fca69..b6ae64b4b5 100644 --- a/src/applications/fact/controller/PhabricatorFactChartController.php +++ b/src/applications/fact/controller/PhabricatorFactChartController.php @@ -5,27 +5,32 @@ final class PhabricatorFactChartController extends PhabricatorFactController { public function handleRequest(AphrontRequest $request) { $viewer = $request->getViewer(); - $table = new PhabricatorFactRaw(); + $series = $request->getStr('y1'); + + $facts = PhabricatorFact::getAllFacts(); + $fact = idx($facts, $series); + + if (!$fact) { + return new Aphront404Response(); + } + + $key_id = id(new PhabricatorFactKeyDimension()) + ->newDimensionID($fact->getKey()); + + $table = $fact->newDatapoint(); $conn_r = $table->establishConnection('r'); $table_name = $table->getTableName(); - $series = $request->getStr('y1'); - - $specs = PhabricatorFactSpec::newSpecsForFactTypes( - PhabricatorFactEngine::loadAllEngines(), - array($series)); - $spec = idx($specs, $series); - $data = queryfx_all( $conn_r, - 'SELECT valueX, epoch FROM %T WHERE factType = %s ORDER BY epoch ASC', + 'SELECT value, epoch FROM %T WHERE keyID = %d ORDER BY epoch ASC', $table_name, - $series); + $key_id); $points = array(); $sum = 0; foreach ($data as $key => $row) { - $sum += (int)$row['valueX']; + $sum += (int)$row['value']; $points[(int)$row['epoch']] = $sum; } @@ -71,7 +76,7 @@ final class PhabricatorFactChartController extends PhabricatorFactController { )); $box = id(new PHUIObjectBoxView()) - ->setHeaderText(pht('Count of %s', $spec->getName())) + ->setHeaderText(pht('Count of %s', $fact->getName())) ->appendChild($chart); $crumbs = $this->buildApplicationCrumbs(); diff --git a/src/applications/fact/controller/PhabricatorFactHomeController.php b/src/applications/fact/controller/PhabricatorFactHomeController.php index f4eeca0387..82f6a0905b 100644 --- a/src/applications/fact/controller/PhabricatorFactHomeController.php +++ b/src/applications/fact/controller/PhabricatorFactHomeController.php @@ -15,45 +15,6 @@ final class PhabricatorFactHomeController extends PhabricatorFactController { return id(new AphrontRedirectResponse())->setURI($uri); } - $types = array( - '+N:*', - '+N:DREV', - 'updated', - ); - - $engines = PhabricatorFactEngine::loadAllEngines(); - $specs = PhabricatorFactSpec::newSpecsForFactTypes($engines, $types); - - $facts = id(new PhabricatorFactAggregate())->loadAllWhere( - 'factType IN (%Ls)', - $types); - - $rows = array(); - foreach ($facts as $fact) { - $spec = $specs[$fact->getFactType()]; - - $name = $spec->getName(); - $value = $spec->formatValueForDisplay($viewer, $fact->getValueX()); - - $rows[] = array($name, $value); - } - - $table = new AphrontTableView($rows); - $table->setHeaders( - array( - pht('Fact'), - pht('Value'), - )); - $table->setColumnClasses( - array( - 'wide', - 'n', - )); - - $panel = new PHUIObjectBoxView(); - $panel->setHeaderText(pht('Facts')); - $panel->setTable($table); - $chart_form = $this->buildChartForm(); $crumbs = $this->buildApplicationCrumbs(); @@ -64,46 +25,18 @@ final class PhabricatorFactHomeController extends PhabricatorFactController { return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) - ->appendChild(array( - $chart_form, - $panel, - )); - + ->appendChild( + array( + $chart_form, + )); } private function buildChartForm() { $request = $this->getRequest(); $viewer = $request->getUser(); - $table = new PhabricatorFactRaw(); - $conn_r = $table->establishConnection('r'); - $table_name = $table->getTableName(); - - $facts = queryfx_all( - $conn_r, - 'SELECT DISTINCT factType from %T', - $table_name); - - $specs = PhabricatorFactSpec::newSpecsForFactTypes( - PhabricatorFactEngine::loadAllEngines(), - ipull($facts, 'factType')); - - $options = array(); - foreach ($specs as $spec) { - if ($spec->getUnit() == PhabricatorFactSpec::UNIT_COUNT) { - $options[$spec->getType()] = $spec->getName(); - } - } - - if (!$options) { - return id(new PHUIInfoView()) - ->setSeverity(PHUIInfoView::SEVERITY_NODATA) - ->setTitle(pht('No Chartable Facts')) - ->appendChild(phutil_tag( - 'p', - array(), - pht('There are no facts that can be plotted yet.'))); - } + $specs = PhabricatorFact::getAllFacts(); + $options = mpull($specs, 'getName', 'getKey'); $form = id(new AphrontFormView()) ->setUser($viewer) diff --git a/src/applications/fact/daemon/PhabricatorFactDaemon.php b/src/applications/fact/daemon/PhabricatorFactDaemon.php index 384bb56df4..8a9214c40b 100644 --- a/src/applications/fact/daemon/PhabricatorFactDaemon.php +++ b/src/applications/fact/daemon/PhabricatorFactDaemon.php @@ -4,8 +4,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { private $engines; - const RAW_FACT_BUFFER_LIMIT = 128; - protected function run() { $this->setEngines(PhabricatorFactEngine::loadAllEngines()); while (!$this->shouldExit()) { @@ -15,7 +13,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { foreach ($iterators as $iterator_name => $iterator) { $this->processIteratorWithCursor($iterator_name, $iterator); } - $this->processAggregates(); $this->log(pht('Zzz...')); $this->sleep(60 * 5); @@ -72,59 +69,41 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { public function processIterator($iterator) { $result = null; - $raw_facts = array(); + $datapoints = array(); foreach ($iterator as $key => $object) { $phid = $object->getPHID(); $this->log(pht('Processing %s...', $phid)); - $raw_facts[$phid] = $this->computeRawFacts($object); - if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) { - $this->updateRawFacts($raw_facts); - $raw_facts = array(); + $datapoints[$phid] = $this->newDatapoints($object); + if (count($datapoints) > 1024) { + $this->updateDatapoints($datapoints); + $datapoints = array(); } $result = $key; } - if ($raw_facts) { - $this->updateRawFacts($raw_facts); - $raw_facts = array(); + if ($datapoints) { + $this->updateDatapoints($datapoints); + $datapoints = array(); } return $result; } - public function processAggregates() { - $this->log(pht('Processing aggregates.')); - - $facts = $this->computeAggregateFacts(); - $this->updateAggregateFacts($facts); - } - - private function computeAggregateFacts() { + private function newDatapoints(PhabricatorLiskDAO $object) { $facts = array(); foreach ($this->engines as $engine) { - if (!$engine->shouldComputeAggregateFacts()) { + if (!$engine->supportsDatapointsForObject($object)) { continue; } - $facts[] = $engine->computeAggregateFacts(); - } - return array_mergev($facts); - } - - private function computeRawFacts(PhabricatorLiskDAO $object) { - $facts = array(); - foreach ($this->engines as $engine) { - if (!$engine->shouldComputeRawFactsForObject($object)) { - continue; - } - $facts[] = $engine->computeRawFactsForObject($object); + $facts[] = $engine->newDatapointsForObject($object); } return array_mergev($facts); } - private function updateRawFacts(array $map) { + private function updateDatapoints(array $map) { foreach ($map as $phid => $facts) { - assert_instances_of($facts, 'PhabricatorFactRaw'); + assert_instances_of($facts, 'PhabricatorFactIntDatapoint'); } $phids = array_keys($map); @@ -132,76 +111,79 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { return; } - $table = new PhabricatorFactRaw(); + + $fact_keys = array(); + $objects = array(); + foreach ($map as $phid => $facts) { + foreach ($facts as $fact) { + $fact_keys[$fact->getKey()] = true; + + $object_phid = $fact->getObjectPHID(); + $objects[$object_phid] = $object_phid; + + $dimension_phid = $fact->getDimensionPHID(); + if ($dimension_phid !== null) { + $objects[$dimension_phid] = $dimension_phid; + } + } + } + + $key_map = id(new PhabricatorFactKeyDimension()) + ->newDimensionMap(array_keys($fact_keys)); + $object_map = id(new PhabricatorFactObjectDimension()) + ->newDimensionMap(array_keys($objects)); + + $table = new PhabricatorFactIntDatapoint(); $conn = $table->establishConnection('w'); $table_name = $table->getTableName(); $sql = array(); foreach ($map as $phid => $facts) { foreach ($facts as $fact) { + $key_id = $key_map[$fact->getKey()]; + $object_id = $object_map[$fact->getObjectPHID()]; + + $dimension_phid = $fact->getDimensionPHID(); + if ($dimension_phid !== null) { + $dimension_id = $object_map[$dimension_phid]; + } else { + $dimension_id = null; + } + $sql[] = qsprintf( $conn, - '(%s, %s, %s, %d, %d, %d)', - $fact->getFactType(), - $fact->getObjectPHID(), - $fact->getObjectA(), - $fact->getValueX(), - $fact->getValueY(), + '(%d, %d, %nd, %d, %d)', + $key_id, + $object_id, + $dimension_id, + $fact->getValue(), $fact->getEpoch()); } } + $rebuilt_ids = array_select_keys($object_map, $phids); + $table->openTransaction(); queryfx( $conn, - 'DELETE FROM %T WHERE objectPHID IN (%Ls)', + 'DELETE FROM %T WHERE objectID IN (%Ld)', $table_name, - $phids); + $rebuilt_ids); if ($sql) { - foreach (array_chunk($sql, 256) as $chunk) { + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { queryfx( $conn, 'INSERT INTO %T - (factType, objectPHID, objectA, valueX, valueY, epoch) + (keyID, objectID, dimensionID, value, epoch) VALUES %Q', $table_name, - implode(', ', $chunk)); + $chunk); } } $table->saveTransaction(); } - private function updateAggregateFacts(array $facts) { - if (!$facts) { - return; - } - - $table = new PhabricatorFactAggregate(); - $conn = $table->establishConnection('w'); - $table_name = $table->getTableName(); - - $sql = array(); - foreach ($facts as $fact) { - $sql[] = qsprintf( - $conn, - '(%s, %s, %d)', - $fact->getFactType(), - $fact->getObjectPHID(), - $fact->getValueX()); - } - - foreach (array_chunk($sql, 256) as $chunk) { - queryfx( - $conn, - 'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q - ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)', - $table_name, - implode(', ', $chunk)); - } - - } - } diff --git a/src/applications/fact/engine/PhabricatorFactCountEngine.php b/src/applications/fact/engine/PhabricatorFactCountEngine.php deleted file mode 100644 index f24068646d..0000000000 --- a/src/applications/fact/engine/PhabricatorFactCountEngine.php +++ /dev/null @@ -1,86 +0,0 @@ -setName($name) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT); - } - - if (!strncmp($type, 'N:', 2)) { - if ($type == 'N:*') { - $name = pht('Objects'); - } else { - $name = pht('Objects of type %s', substr($type, 2)); - } - $results[] = id(new PhabricatorFactSimpleSpec($type)) - ->setName($name) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT); - } - - } - return $results; - } - - public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { - return true; - } - - public function computeRawFactsForObject(PhabricatorLiskDAO $object) { - $facts = array(); - - $phid = $object->getPHID(); - $type = phid_get_type($phid); - - foreach (array('N:*', 'N:'.$type) as $fact_type) { - $facts[] = id(new PhabricatorFactRaw()) - ->setFactType($fact_type) - ->setObjectPHID($phid) - ->setValueX(1) - ->setEpoch($object->getDateCreated()); - } - - return $facts; - } - - public function shouldComputeAggregateFacts() { - return true; - } - - public function computeAggregateFacts() { - $table = new PhabricatorFactRaw(); - $table_name = $table->getTableName(); - $conn = $table->establishConnection('r'); - - $counts = queryfx_all( - $conn, - 'SELECT factType, SUM(valueX) N FROM %T WHERE factType LIKE %> - GROUP BY factType', - $table_name, - 'N:'); - - $facts = array(); - foreach ($counts as $count) { - $facts[] = id(new PhabricatorFactAggregate()) - ->setFactType('+'.$count['factType']) - ->setValueX($count['N']); - } - - return $facts; - } - - -} diff --git a/src/applications/fact/engine/PhabricatorFactEngine.php b/src/applications/fact/engine/PhabricatorFactEngine.php index a87cabf56d..b8389f7b23 100644 --- a/src/applications/fact/engine/PhabricatorFactEngine.php +++ b/src/applications/fact/engine/PhabricatorFactEngine.php @@ -2,30 +2,37 @@ abstract class PhabricatorFactEngine extends Phobject { + private $factMap; + final public static function loadAllEngines() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) ->execute(); } - public function getFactSpecs(array $fact_types) { - return array(); - } + abstract public function newFacts(); - public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { - return false; - } + abstract public function supportsDatapointsForObject( + PhabricatorLiskDAO $object); - public function computeRawFactsForObject(PhabricatorLiskDAO $object) { - return array(); - } + abstract public function newDatapointsForObject(PhabricatorLiskDAO $object); - public function shouldComputeAggregateFacts() { - return false; - } + final protected function getFact($key) { + if ($this->factMap === null) { + $facts = $this->newFacts(); + $facts = mpull($facts, null, 'getKey'); + $this->factMap = $facts; + } - public function computeAggregateFacts() { - return array(); + if (!isset($this->factMap[$key])) { + throw new Exception( + pht( + 'Unknown fact ("%s") for engine "%s".', + $key, + get_class($this))); + } + + return $this->factMap[$key]; } } diff --git a/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php b/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php deleted file mode 100644 index 5ea99e6232..0000000000 --- a/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php +++ /dev/null @@ -1,34 +0,0 @@ -setName(pht('Facts Last Updated')) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_EPOCH); - } - } - return $results; - } - - public function shouldComputeAggregateFacts() { - return true; - } - - public function computeAggregateFacts() { - $facts = array(); - - $facts[] = id(new PhabricatorFactAggregate()) - ->setFactType('updated') - ->setValueX(time()); - - return $facts; - } - -} diff --git a/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php b/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php new file mode 100644 index 0000000000..c2ed81d72b --- /dev/null +++ b/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php @@ -0,0 +1,34 @@ +setKey('tasks.count.open'), + ); + } + + public function supportsDatapointsForObject(PhabricatorLiskDAO $object) { + return ($object instanceof ManiphestTask); + } + + public function newDatapointsForObject(PhabricatorLiskDAO $object) { + $datapoints = array(); + + $phid = $object->getPHID(); + $type = phid_get_type($phid); + + $datapoint = $this->getFact('tasks.count.open') + ->newDatapoint(); + + $datapoints[] = $datapoint + ->setObjectPHID($phid) + ->setValue(1) + ->setEpoch($object->getDateCreated()); + + return $datapoints; + } + +} diff --git a/src/applications/fact/fact/PhabricatorFact.php b/src/applications/fact/fact/PhabricatorFact.php new file mode 100644 index 0000000000..2e33a029f3 --- /dev/null +++ b/src/applications/fact/fact/PhabricatorFact.php @@ -0,0 +1,40 @@ +newFacts(); + $facts = mpull($facts, null, 'getKey'); + $map += $facts; + } + + return $map; + } + + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final public function getName() { + return pht('Fact "%s"', $this->getKey()); + } + + final public function newDatapoint() { + return $this->newTemplateDatapoint() + ->setKey($this->getKey()); + } + + abstract protected function newTemplateDatapoint(); + +} diff --git a/src/applications/fact/fact/PhabricatorPointsFact.php b/src/applications/fact/fact/PhabricatorPointsFact.php new file mode 100644 index 0000000000..a80f45d132 --- /dev/null +++ b/src/applications/fact/fact/PhabricatorPointsFact.php @@ -0,0 +1,9 @@ +getArg('skip-aggregates')) { - $daemon->processAggregates(); - } - return 0; } diff --git a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php index 8a2e5f1ed8..7a4997ab35 100644 --- a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php @@ -23,8 +23,13 @@ final class PhabricatorFactManagementDestroyWorkflow } $tables = array(); - $tables[] = new PhabricatorFactRaw(); - $tables[] = new PhabricatorFactAggregate(); + $tables[] = new PhabricatorFactCursor(); + + $tables[] = new PhabricatorFactIntDatapoint(); + + $tables[] = new PhabricatorFactObjectDimension(); + $tables[] = new PhabricatorFactKeyDimension(); + foreach ($tables as $table) { $conn = $table->establishConnection('w'); $name = $table->getTableName(); diff --git a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php deleted file mode 100644 index 604a40b6ee..0000000000 --- a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php +++ /dev/null @@ -1,47 +0,0 @@ -setName('status') - ->setSynopsis(pht('Show status of fact data.')) - ->setArguments(array()); - } - - public function execute(PhutilArgumentParser $args) { - $console = PhutilConsole::getConsole(); - - $map = array( - 'raw' => new PhabricatorFactRaw(), - 'agg' => new PhabricatorFactAggregate(), - ); - - foreach ($map as $type => $table) { - $conn = $table->establishConnection('r'); - $name = $table->getTableName(); - - $row = queryfx_one( - $conn, - 'SELECT COUNT(*) N FROM %T', - $name); - - $n = $row['N']; - - switch ($type) { - case 'raw': - $desc = pht('There are %d raw fact(s) in storage.', $n); - break; - case 'agg': - $desc = pht('There are %d aggregate fact(s) in storage.', $n); - break; - } - - $console->writeOut("%s\n", $desc); - } - - return 0; - } - -} diff --git a/src/applications/fact/spec/PhabricatorFactSimpleSpec.php b/src/applications/fact/spec/PhabricatorFactSimpleSpec.php deleted file mode 100644 index 350b6367f1..0000000000 --- a/src/applications/fact/spec/PhabricatorFactSimpleSpec.php +++ /dev/null @@ -1,38 +0,0 @@ -type = $type; - } - - public function getType() { - return $this->type; - } - - public function setUnit($unit) { - $this->unit = $unit; - return $this; - } - - public function getUnit() { - return $this->unit; - } - - public function setName($name) { - $this->name = $name; - return $this; - } - - public function getName() { - if ($this->name !== null) { - return $this->name; - } - return parent::getName(); - } - -} diff --git a/src/applications/fact/spec/PhabricatorFactSpec.php b/src/applications/fact/spec/PhabricatorFactSpec.php deleted file mode 100644 index 47fcc01d8b..0000000000 --- a/src/applications/fact/spec/PhabricatorFactSpec.php +++ /dev/null @@ -1,53 +0,0 @@ -getFactSpecs($fact_types); - $specs = mpull($specs, null, 'getType'); - $map += $specs; - } - - foreach ($fact_types as $type) { - if (empty($map[$type])) { - $map[$type] = new PhabricatorFactSimpleSpec($type); - } - } - - return $map; - } - - abstract public function getType(); - - public function getUnit() { - return null; - } - - public function getName() { - return pht( - 'Fact (%s)', - $this->getType()); - } - - public function formatValueForDisplay(PhabricatorUser $user, $value) { - $unit = $this->getUnit(); - switch ($unit) { - case self::UNIT_COUNT: - return number_format($value); - case self::UNIT_EPOCH: - return phabricator_datetime($value, $user); - default: - return $value; - } - } - -} diff --git a/src/applications/fact/storage/PhabricatorFactDimension.php b/src/applications/fact/storage/PhabricatorFactDimension.php new file mode 100644 index 0000000000..d2bf2420dc --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactDimension.php @@ -0,0 +1,85 @@ +newDimensionMap(array($key)); + return $map[$key]; + } + + final public function newDimensionMap(array $keys) { + if (!$keys) { + return array(); + } + + $conn = $this->establishConnection('r'); + $column = $this->getDimensionColumnName(); + + $rows = queryfx_all( + $conn, + 'SELECT id, %C FROM %T WHERE %C IN (%Ls)', + $column, + $this->getTableName(), + $column, + $keys); + $rows = ipull($rows, 'id', $column); + + $map = array(); + $need = array(); + foreach ($keys as $key) { + if (isset($rows[$key])) { + $map[$key] = (int)$rows[$key]; + } else { + $need[] = $key; + } + } + + if (!$need) { + return $map; + } + + $sql = array(); + foreach ($need as $key) { + $sql[] = qsprintf( + $conn, + '(%s)', + $key); + } + + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn, + 'INSERT IGNORE INTO %T (%C) VALUES %Q', + $this->getTableName(), + $column, + $chunk); + } + + $rows = queryfx_all( + $conn, + 'SELECT id, %C FROM %T WHERE %C IN (%Ls)', + $column, + $this->getTableName(), + $column, + $need); + $rows = ipull($rows, 'id', $column); + + foreach ($keys as $key) { + if (isset($rows[$key])) { + $map[$key] = (int)$rows[$key]; + } else { + throw new Exception( + pht( + 'Failed to load or generate dimension ID ("%s") for dimension '. + 'key "%s".', + get_class($this), + $key)); + } + } + + return $map; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactIntDatapoint.php b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php new file mode 100644 index 0000000000..7895052d5d --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php @@ -0,0 +1,61 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'id' => 'auto64', + 'dimensionID' => 'id?', + 'value' => 'sint64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_dimension' => array( + 'columns' => array('keyID', 'dimensionID'), + ), + 'key_object' => array( + 'columns' => array('objectID'), + ), + ), + ) + parent::getConfiguration(); + } + + public function setKey($key) { + $this->key = $key; + return $this; + } + + public function getKey() { + return $this->key; + } + + public function setObjectPHID($object_phid) { + $this->objectPHID = $object_phid; + return $this; + } + + public function getObjectPHID() { + return $this->objectPHID; + } + + public function setDimensionPHID($dimension_phid) { + $this->dimensionPHID = $dimension_phid; + return $this; + } + + public function getDimensionPHID() { + return $this->dimensionPHID; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactKeyDimension.php b/src/applications/fact/storage/PhabricatorFactKeyDimension.php new file mode 100644 index 0000000000..b58ba94400 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactKeyDimension.php @@ -0,0 +1,27 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'factKey' => 'text64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_factkey' => array( + 'columns' => array('factKey'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + protected function getDimensionColumnName() { + return 'factKey'; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactObjectDimension.php b/src/applications/fact/storage/PhabricatorFactObjectDimension.php new file mode 100644 index 0000000000..e7319724a4 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactObjectDimension.php @@ -0,0 +1,25 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array(), + self::CONFIG_KEY_SCHEMA => array( + 'key_object' => array( + 'columns' => array('objectPHID'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + protected function getDimensionColumnName() { + return 'objectPHID'; + } + +} diff --git a/src/applications/ponder/application/PhabricatorPonderApplication.php b/src/applications/ponder/application/PhabricatorPonderApplication.php index ed37c7ef6e..56973447f9 100644 --- a/src/applications/ponder/application/PhabricatorPonderApplication.php +++ b/src/applications/ponder/application/PhabricatorPonderApplication.php @@ -18,12 +18,6 @@ final class PhabricatorPonderApplication extends PhabricatorApplication { return 'fa-university'; } - public function getFactObjectsForAnalysis() { - return array( - new PonderQuestion(), - ); - } - public function getTitleGlyph() { return "\xE2\x97\xB3"; }