From 486f7c1e8e2385f7b64bfda29456032ed2d03c9a Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 27 Jul 2012 13:46:01 -0700 Subject: [PATCH] Add aggregated facts to the Facts application Summary: Some facts are aggregations of other facts. For example, we may compute how many times each macro is used in each object as a "raw fact": Dnnn uses macro "psyduck" 6 times. But we want to present this data in aggregate form, e.g. "order macros by popularity". We can do this at runtime and it probably won't be too awful a query, but we can also aggregate it cheaply: Macro "psyduck" is used 3920 times across all objects. ...and then do a query like "select macros ordered by usage". "Aggregate" facts support facts like this. The aggregate facts I've implemented are: - Count of all objects. - Count of objects of type X. - Last time facts were updated. These clearly fit the "aggregate" facts template well. I'm not 100% sure macros do. We can use this table to answer a question like "What are the most popular macros, ordered by use?" We can also use it to answer a question like "What are the most popular macros in the last 6 months?", if we build a specific fact for that. But we can't use it to answer a question like "What are the most popular macros between times X and Y?". Maybe that's important; maybe not. This seems like a good fit for at least some types of facts. I'll de-magic the keys a bit in the next diff. Test Plan: Ran the engines and got some aggregated facts about other facts. Reviewers: vrana, btrahan Reviewed By: vrana CC: aran Maniphest Tasks: T1562 Differential Revision: https://secure.phabricator.com/D3089 --- src/__phutil_library_map__.php | 8 +++ ...AphrontDefaultApplicationConfiguration.php | 4 ++ .../controller/PhabricatorFactController.php | 34 +++++++++++ .../PhabricatorFactHomeController.php | 59 +++++++++++++++++++ .../fact/daemon/PhabricatorFactDaemon.php | 46 +++++++++++++++ .../engine/PhabricatorFactCountEngine.php | 27 +++++++++ .../fact/engine/PhabricatorFactEngine.php | 8 +++ .../PhabricatorFactLastUpdatedEngine.php | 38 ++++++++++++ ...abricatorFactManagementAnalyzeWorkflow.php | 2 + ...abricatorFactManagementDestroyWorkflow.php | 1 + ...habricatorFactManagementStatusWorkflow.php | 4 ++ .../fact/storage/PhabricatorFactAggregate.php | 26 ++++++++ .../PhabricatorBaseEnglishTranslation.php | 5 ++ 13 files changed, 262 insertions(+) create mode 100644 src/applications/fact/controller/PhabricatorFactController.php create mode 100644 src/applications/fact/controller/PhabricatorFactHomeController.php create mode 100644 src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php create mode 100644 src/applications/fact/storage/PhabricatorFactAggregate.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 97cb7f8a83..6fb37525ab 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -624,10 +624,14 @@ phutil_register_library_map(array( 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', + 'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php', + 'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php', 'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', + 'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php', + 'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php', 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', @@ -1660,9 +1664,13 @@ phutil_register_library_map(array( 'PhabricatorEvent' => 'PhutilEvent', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhutilEventListener', + 'PhabricatorFactAggregate' => 'PhabricatorFactDAO', + 'PhabricatorFactController' => 'PhabricatorController', 'PhabricatorFactCountEngine' => 'PhabricatorFactEngine', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDaemon' => 'PhabricatorDaemon', + 'PhabricatorFactHomeController' => 'PhabricatorFactController', + 'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine', 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', diff --git a/src/aphront/configuration/AphrontDefaultApplicationConfiguration.php b/src/aphront/configuration/AphrontDefaultApplicationConfiguration.php index b22a7ac68c..9b9c4a151d 100644 --- a/src/aphront/configuration/AphrontDefaultApplicationConfiguration.php +++ b/src/aphront/configuration/AphrontDefaultApplicationConfiguration.php @@ -463,6 +463,10 @@ class AphrontDefaultApplicationConfiguration '/emailverify/(?P[^/]+)/' => 'PhabricatorEmailVerificationController', + + '/fact/' => array( + '' => 'PhabricatorFactHomeController', + ), ); } diff --git a/src/applications/fact/controller/PhabricatorFactController.php b/src/applications/fact/controller/PhabricatorFactController.php new file mode 100644 index 0000000000..3f1cfc7e1c --- /dev/null +++ b/src/applications/fact/controller/PhabricatorFactController.php @@ -0,0 +1,34 @@ +buildStandardPageView(); + + $page->setBaseURI('/fact/'); + $page->setTitle(idx($data, 'title')); + + $page->setGlyph("\xCE\xA3"); + $page->appendChild($view); + + $response = new AphrontWebpageResponse(); + return $response->setContent($page->render()); + } + +} diff --git a/src/applications/fact/controller/PhabricatorFactHomeController.php b/src/applications/fact/controller/PhabricatorFactHomeController.php new file mode 100644 index 0000000000..3783b20671 --- /dev/null +++ b/src/applications/fact/controller/PhabricatorFactHomeController.php @@ -0,0 +1,59 @@ +loadAllWhere( + 'factType LIKE %> OR factType = %s', + '+N:', + 'updated'); + + $rows = array(); + foreach ($facts as $fact) { + $rows[] = array( + phutil_escape_html($fact->getFactType()), + (int)$fact->getValueX(), + ); + } + + $table = new AphrontTableView($rows); + $table->setHeaders( + array( + 'Fact', + 'Value', + )); + $table->setColumnClasses( + array( + 'wide', + 'n', + )); + + $panel = new AphrontPanelView(); + $panel->setHeader('Facts!'); + $panel->appendChild($table); + + return $this->buildStandardPageResponse( + $panel, + array( + 'title' => 'Facts!', + )); + } + +} diff --git a/src/applications/fact/daemon/PhabricatorFactDaemon.php b/src/applications/fact/daemon/PhabricatorFactDaemon.php index 4ef792e758..7d15c514f2 100644 --- a/src/applications/fact/daemon/PhabricatorFactDaemon.php +++ b/src/applications/fact/daemon/PhabricatorFactDaemon.php @@ -54,6 +54,22 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { return $result; } + public function processAggregates() { + $facts = $this->computeAggregateFacts(); + $this->updateAggregateFacts($facts); + } + + private function computeAggregateFacts() { + $facts = array(); + foreach ($this->engines as $engine) { + if (!$engine->shouldComputeAggregateFacts()) { + continue; + } + $facts[] = $engine->computeAggregateFacts(); + } + return array_mergev($facts); + } + private function computeRawFacts(PhabricatorLiskDAO $object) { $facts = array(); foreach ($this->engines as $engine) { @@ -118,4 +134,34 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { $table->saveTransaction(); } + private function updateAggregateFacts(array $facts) { + if (!$facts) { + return; + } + + $table = new PhabricatorFactAggregate(); + $conn = $table->establishConnection('w'); + $table_name = $table->getTableName(); + + $sql = array(); + foreach ($facts as $fact) { + $sql[] = qsprintf( + $conn, + '(%s, %s, %d)', + $fact->getFactType(), + $fact->getObjectPHID(), + $fact->getValueX()); + } + + foreach (array_chunk($sql, 256) as $chunk) { + queryfx( + $conn, + 'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q + ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)', + $table_name, + implode(', ', $chunk)); + } + + } + } diff --git a/src/applications/fact/engine/PhabricatorFactCountEngine.php b/src/applications/fact/engine/PhabricatorFactCountEngine.php index d20df85ad8..9792a79c32 100644 --- a/src/applications/fact/engine/PhabricatorFactCountEngine.php +++ b/src/applications/fact/engine/PhabricatorFactCountEngine.php @@ -41,4 +41,31 @@ final class PhabricatorFactCountEngine extends PhabricatorFactEngine { return $facts; } + public function shouldComputeAggregateFacts() { + return true; + } + + public function computeAggregateFacts() { + $table = new PhabricatorFactRaw(); + $table_name = $table->getTableName(); + $conn = $table->establishConnection('r'); + + $counts = queryfx_all( + $conn, + 'SELECT factType, count(*) N FROM %T WHERE factType LIKE %> + GROUP BY factType', + $table_name, + 'N:'); + + $facts = array(); + foreach ($counts as $count) { + $facts[] = id(new PhabricatorFactAggregate()) + ->setFactType('+'.$count['factType']) + ->setValueX($count['N']); + } + + return $facts; + } + + } diff --git a/src/applications/fact/engine/PhabricatorFactEngine.php b/src/applications/fact/engine/PhabricatorFactEngine.php index cd951e913f..a868a36671 100644 --- a/src/applications/fact/engine/PhabricatorFactEngine.php +++ b/src/applications/fact/engine/PhabricatorFactEngine.php @@ -40,4 +40,12 @@ abstract class PhabricatorFactEngine { return array(); } + public function shouldComputeAggregateFacts() { + return false; + } + + public function computeAggregateFacts() { + return array(); + } + } diff --git a/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php b/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php new file mode 100644 index 0000000000..cd9f1024ce --- /dev/null +++ b/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php @@ -0,0 +1,38 @@ +setFactType('updated') + ->setValueX(time()); + + return $facts; + } + +} diff --git a/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php index c864277741..e0c3f70fb8 100644 --- a/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php @@ -41,6 +41,8 @@ final class PhabricatorFactManagementAnalyzeWorkflow $daemon->processIterator($iterator); } + $daemon->processAggregates(); + return 0; } diff --git a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php index 686c12fbbd..711dd26e85 100644 --- a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php @@ -40,6 +40,7 @@ final class PhabricatorFactManagementDestroyWorkflow $tables = array(); $tables[] = new PhabricatorFactRaw(); + $tables[] = new PhabricatorFactAggregate(); foreach ($tables as $table) { $conn = $table->establishConnection('w'); $name = $table->getTableName(); diff --git a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php index 114484f244..07a69a1bde 100644 --- a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php @@ -31,6 +31,7 @@ final class PhabricatorFactManagementStatusWorkflow $map = array( 'raw' => new PhabricatorFactRaw(), + 'agg' => new PhabricatorFactAggregate(), ); foreach ($map as $type => $table) { @@ -48,6 +49,9 @@ final class PhabricatorFactManagementStatusWorkflow case 'raw': $desc = pht('There are %d raw fact(s) in storage.', $n); break; + case 'agg': + $desc = pht('There are %d aggregate fact(s) in storage.', $n); + break; } $console->writeOut("%s\n", $desc); diff --git a/src/applications/fact/storage/PhabricatorFactAggregate.php b/src/applications/fact/storage/PhabricatorFactAggregate.php new file mode 100644 index 0000000000..45af1dc66e --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactAggregate.php @@ -0,0 +1,26 @@ + array( + 'There is %d aggregate fact in storage.', + 'There are %d aggregate facts in storage.', + ), + ); }