From f0af2731656313ad5ac456427307190d3581c801 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 30 Jul 2012 10:43:49 -0700 Subject: [PATCH] Add FactCursors and application fact datasources Summary: - Add PhabricatorApplication. This is a general class that I have grand designs for, but used here to allow applications to provide objects for analysis by the facts appliction. - Add FactCursors, to keep track of where iterators are. - Make the daemon do something sort of useful. - Add `bin/fact cursors` for showing and managing objects and cursors. - Add some options to `bin/fact analyze`. Test Plan: - `bin/fact cursors`, `bin/fact cursors --reset DifferentialRevision`, `bin/fact cursors --reset X` - `bin/fact analyze`, `bin/fact analyze --all`, `bin/fact analyze --iterator DifferentialRevision --skip-aggregates` - `bin/phd debug fact` Reviewers: vrana, btrahan Reviewed By: btrahan CC: aran Maniphest Tasks: T1562 Differential Revision: https://secure.phabricator.com/D3098 --- scripts/fact/manage_facts.php | 1 + src/__phutil_library_map__.php | 10 +++ .../base/PhabricatorApplication.php | 70 ++++++++++++++++ .../PhabricatorApplicationDifferential.php | 28 +++++++ .../fact/daemon/PhabricatorFactDaemon.php | 58 ++++++++++++- .../extract/PhabricatorFactUpdateIterator.php | 12 +-- ...abricatorFactManagementAnalyzeWorkflow.php | 49 +++++++++-- ...abricatorFactManagementCursorsWorkflow.php | 82 +++++++++++++++++++ .../fact/storage/PhabricatorFactCursor.php | 25 ++++++ .../PhabricatorApplicationManiphest.php | 32 ++++++++ 10 files changed, 353 insertions(+), 14 deletions(-) create mode 100644 src/applications/base/PhabricatorApplication.php create mode 100644 src/applications/differential/application/PhabricatorApplicationDifferential.php create mode 100644 src/applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php create mode 100644 src/applications/fact/storage/PhabricatorFactCursor.php create mode 100644 src/applications/maniphest/application/PhabricatorApplicationManiphest.php diff --git a/scripts/fact/manage_facts.php b/scripts/fact/manage_facts.php index fd075896d3..d37be22ed5 100755 --- a/scripts/fact/manage_facts.php +++ b/scripts/fact/manage_facts.php @@ -36,6 +36,7 @@ $workflows = array( new PhabricatorFactManagementAnalyzeWorkflow(), new PhabricatorFactManagementStatusWorkflow(), new PhabricatorFactManagementListWorkflow(), + new PhabricatorFactManagementCursorsWorkflow(), new PhutilHelpArgumentWorkflow(), ); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 8729bf61c9..4fe9c3383a 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -530,6 +530,9 @@ phutil_register_library_map(array( 'PackageModifyMail' => 'applications/owners/mail/PackageModifyMail.php', 'Phabricator404Controller' => 'applications/base/controller/Phabricator404Controller.php', 'PhabricatorAccessLog' => 'infrastructure/PhabricatorAccessLog.php', + 'PhabricatorApplication' => 'applications/base/PhabricatorApplication.php', + 'PhabricatorApplicationDifferential' => 'applications/differential/application/PhabricatorApplicationDifferential.php', + 'PhabricatorApplicationManiphest' => 'applications/maniphest/application/PhabricatorApplicationManiphest.php', 'PhabricatorAuditActionConstants' => 'applications/audit/constants/PhabricatorAuditActionConstants.php', 'PhabricatorAuditAddCommentController' => 'applications/audit/controller/PhabricatorAuditAddCommentController.php', 'PhabricatorAuditComment' => 'applications/audit/storage/PhabricatorAuditComment.php', @@ -627,12 +630,14 @@ phutil_register_library_map(array( 'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php', 'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php', 'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php', + 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php', 'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php', 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', + 'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php', 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', 'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php', @@ -1579,6 +1584,8 @@ phutil_register_library_map(array( 'PackageDeleteMail' => 'PackageMail', 'PackageModifyMail' => 'PackageMail', 'Phabricator404Controller' => 'PhabricatorController', + 'PhabricatorApplicationDifferential' => 'PhabricatorApplication', + 'PhabricatorApplicationManiphest' => 'PhabricatorApplication', 'PhabricatorAuditAddCommentController' => 'PhabricatorAuditController', 'PhabricatorAuditComment' => 'PhabricatorAuditDAO', 'PhabricatorAuditCommitListView' => 'AphrontView', @@ -1669,16 +1676,19 @@ phutil_register_library_map(array( 'PhabricatorFactAggregate' => 'PhabricatorFactDAO', 'PhabricatorFactController' => 'PhabricatorController', 'PhabricatorFactCountEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDaemon' => 'PhabricatorDaemon', 'PhabricatorFactHomeController' => 'PhabricatorFactController', 'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine', 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', + 'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhutilArgumentWorkflow', 'PhabricatorFactRaw' => 'PhabricatorFactDAO', + 'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFeedController' => 'PhabricatorController', 'PhabricatorFeedDAO' => 'PhabricatorLiskDAO', diff --git a/src/applications/base/PhabricatorApplication.php b/src/applications/base/PhabricatorApplication.php new file mode 100644 index 0000000000..88ad71c947 --- /dev/null +++ b/src/applications/base/PhabricatorApplication.php @@ -0,0 +1,70 @@ +setAncestorClass(__CLASS__) + ->setConcreteOnly(true) + ->selectAndLoadSymbols(); + + $apps = array(); + foreach ($classes as $class) { + $app = newv($class['name'], array()); + if (!$app->isEnabled()) { + continue; + } + $apps[] = $app; + } + + return $apps; + } + + +} diff --git a/src/applications/differential/application/PhabricatorApplicationDifferential.php b/src/applications/differential/application/PhabricatorApplicationDifferential.php new file mode 100644 index 0000000000..f618e10b84 --- /dev/null +++ b/src/applications/differential/application/PhabricatorApplicationDifferential.php @@ -0,0 +1,28 @@ +setEngines(PhabricatorFactEngine::loadAllEngines()); + while (true) { + $iterators = $this->getAllApplicationIterators(); + foreach ($iterators as $iterator_name => $iterator) { + $this->processIteratorWithCursor($iterator_name, $iterator); + } + $this->processAggregates(); + + $this->log("Zzz..."); + $this->sleep(60 * 5); + } + } + + public static function getAllApplicationIterators() { + $apps = PhabricatorApplication::getAllInstalledApplications(); + + $iterators = array(); + foreach ($apps as $app) { + foreach ($app->getFactObjectsForAnalysis() as $object) { + $iterator = new PhabricatorFactUpdateIterator($object); + $iterators[get_class($object)] = $iterator; + } + } + + return $iterators; + } + + public function processIteratorWithCursor($iterator_name, $iterator) { + $this->log("Processing cursor '{$iterator_name}'."); + + $cursor = id(new PhabricatorFactCursor())->loadOneWhere( + 'name = %s', + $iterator_name); + if (!$cursor) { + $cursor = new PhabricatorFactCursor(); + $cursor->setName($iterator_name); + $position = null; + } else { + $position = $cursor->getPosition(); + } + + if ($position) { + $iterator->setPosition($position); + } + + $new_cursor_position = $this->processIterator($iterator); + + if ($new_cursor_position) { + $cursor->setPosition($new_cursor_position); + $cursor->save(); + } } public function setEngines(array $engines) { @@ -38,7 +88,9 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { $raw_facts = array(); foreach ($iterator as $key => $object) { - $raw_facts[$object->getPHID()] = $this->computeRawFacts($object); + $phid = $object->getPHID(); + $this->log("Processing {$phid}..."); + $raw_facts[$phid] = $this->computeRawFacts($object); if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) { $this->updateRawFacts($raw_facts); $raw_facts = array(); @@ -55,6 +107,8 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon { } public function processAggregates() { + $this->log("Processing aggregates."); + $facts = $this->computeAggregateFacts(); $this->updateAggregateFacts($facts); } diff --git a/src/applications/fact/extract/PhabricatorFactUpdateIterator.php b/src/applications/fact/extract/PhabricatorFactUpdateIterator.php index b70a74a702..af2db70d36 100644 --- a/src/applications/fact/extract/PhabricatorFactUpdateIterator.php +++ b/src/applications/fact/extract/PhabricatorFactUpdateIterator.php @@ -28,12 +28,14 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator { private $start; private $ignoreUpdatesDuration = 15; - public function __construct(LiskDAO $object, $start = null) { - if ($start === null) { - $start = '0:0'; - } + public function __construct(LiskDAO $object) { $this->object = $object; - $this->start = $start; + $this->start = '0:0'; + } + + public function setPosition($position) { + $this->start = $position; + return $this; } protected function didRewind() { diff --git a/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php index e0c3f70fb8..bd4d415ebf 100644 --- a/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php @@ -23,7 +23,23 @@ final class PhabricatorFactManagementAnalyzeWorkflow $this ->setName('analyze') ->setSynopsis(pht('Manually invoke fact analyzers.')) - ->setArguments(array()); + ->setArguments( + array( + array( + 'name' => 'iterator', + 'param' => 'name', + 'repeat' => true, + 'help' => 'Process only __iterator__.', + ), + array( + 'name' => 'all', + 'help' => 'Analyze from the beginning, ignoring cursors.', + ), + array( + 'name' => 'skip-aggregates', + 'help' => 'Skip analysis of aggreate facts.', + ), + )); } public function execute(PhutilArgumentParser $args) { @@ -33,15 +49,34 @@ final class PhabricatorFactManagementAnalyzeWorkflow $daemon->setVerbose(true); $daemon->setEngines(PhabricatorFactEngine::loadAllEngines()); - $iterators = array( - new PhabricatorFactUpdateIterator(new DifferentialRevision()), - ); + $iterators = PhabricatorFactDaemon::getAllApplicationIterators(); - foreach ($iterators as $iterator) { - $daemon->processIterator($iterator); + $selected = $args->getArg('iterator'); + if ($selected) { + $use = array(); + foreach ($selected as $iterator_name) { + if (isset($iterators[$iterator_name])) { + $use[$iterator_name] = $iterators[$iterator_name]; + } else { + $console->writeErr( + "%s\n", + pht("Iterator '%s' does not exist.", $iterator_name)); + } + } + $iterators = $use; } - $daemon->processAggregates(); + foreach ($iterators as $iterator_name => $iterator) { + if ($args->getArg('all')) { + $daemon->processIterator($iterator); + } else { + $daemon->processIteratorWithCursor($iterator_name, $iterator); + } + } + + if (!$args->getArg('skip-aggregates')) { + $daemon->processAggregates(); + } return 0; } diff --git a/src/applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php new file mode 100644 index 0000000000..b6c69e368d --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php @@ -0,0 +1,82 @@ +setName('cursors') + ->setSynopsis(pht('Show a list of fact iterators and cursors.')) + ->setExamples( + "**cursors**\n". + "**cursors** --reset __cursor__") + ->setArguments( + array( + array( + 'name' => 'reset', + 'param' => 'cursor', + 'repeat' => true, + 'help' => 'Reset cursor __cursor__.', + ) + )); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $reset = $args->getArg('reset'); + if ($reset) { + foreach ($reset as $name) { + $cursor = id(new PhabricatorFactCursor())->loadOneWhere( + 'name = %s', + $name); + if ($cursor) { + $console->writeOut("%s\n", pht("Resetting cursor %s...", $name)); + $cursor->delete(); + } else { + $console->writeErr( + "%s\n", + pht("Cursor %s does not exist or is already reset.", $name)); + } + } + return 0; + } + + $iterator_map = PhabricatorFactDaemon::getAllApplicationIterators(); + if (!$iterator_map) { + $console->writeErr("%s\n", pht("No cursors.")); + return 0; + } + + $cursors = id(new PhabricatorFactCursor())->loadAllWhere( + 'name IN (%Ls)', + array_keys($iterator_map)); + $cursors = mpull($cursors, 'getPosition', 'getName'); + + foreach ($iterator_map as $iterator_name => $iterator) { + $console->writeOut( + "%s (%s)\n", + $iterator_name, + idx($cursors, $iterator_name, 'start')); + } + + return 0; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactCursor.php b/src/applications/fact/storage/PhabricatorFactCursor.php new file mode 100644 index 0000000000..1a68c7b645 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactCursor.php @@ -0,0 +1,25 @@ +