diff --git a/bin/fact b/bin/fact new file mode 120000 index 0000000000..9a84ebad20 --- /dev/null +++ b/bin/fact @@ -0,0 +1 @@ +../scripts/fact/manage_facts.php \ No newline at end of file diff --git a/scripts/fact/manage_facts.php b/scripts/fact/manage_facts.php new file mode 100755 index 0000000000..fd075896d3 --- /dev/null +++ b/scripts/fact/manage_facts.php @@ -0,0 +1,42 @@ +#!/usr/bin/env php +setTagline('manage fact configuration'); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = array( + new PhabricatorFactManagementDestroyWorkflow(), + new PhabricatorFactManagementAnalyzeWorkflow(), + new PhabricatorFactManagementStatusWorkflow(), + new PhabricatorFactManagementListWorkflow(), + new PhutilHelpArgumentWorkflow(), +); + +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index b72e2107eb..97cb7f8a83 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -624,7 +624,17 @@ phutil_register_library_map(array( 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', - 'PhabricatorFactsUpdateIterator' => 'applications/facts/extract/PhabricatorFactsUpdateIterator.php', + 'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php', + 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', + 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', + 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', + 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', + 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', + 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', + 'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php', + 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', + 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', + 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', 'PhabricatorFeedBuilder' => 'applications/feed/builder/PhabricatorFeedBuilder.php', 'PhabricatorFeedConstants' => 'applications/feed/constants/PhabricatorFeedConstants.php', 'PhabricatorFeedController' => 'applications/feed/controller/PhabricatorFeedController.php', @@ -1650,7 +1660,16 @@ phutil_register_library_map(array( 'PhabricatorEvent' => 'PhutilEvent', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhutilEventListener', - 'PhabricatorFactsUpdateIterator' => 'PhutilBufferedIterator', + 'PhabricatorFactCountEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', + 'PhabricatorFactDaemon' => 'PhabricatorDaemon', + 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', + 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', + 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', + 'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow', + 'PhabricatorFactManagementWorkflow' => 'PhutilArgumentWorkflow', + 'PhabricatorFactRaw' => 'PhabricatorFactDAO', + 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFeedController' => 'PhabricatorController', 'PhabricatorFeedDAO' => 'PhabricatorLiskDAO', 'PhabricatorFeedPublicStreamController' => 'PhabricatorFeedController', diff --git a/src/applications/fact/daemon/PhabricatorFactDaemon.php b/src/applications/fact/daemon/PhabricatorFactDaemon.php new file mode 100644 index 0000000000..4ef792e758 --- /dev/null +++ b/src/applications/fact/daemon/PhabricatorFactDaemon.php @@ -0,0 +1,121 @@ +engines = $engines; + return $this; + } + + public function processIterator($iterator) { + $result = null; + + $raw_facts = array(); + foreach ($iterator as $key => $object) { + $raw_facts[$object->getPHID()] = $this->computeRawFacts($object); + if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) { + $this->updateRawFacts($raw_facts); + $raw_facts = array(); + } + $result = $key; + } + + if ($raw_facts) { + $this->updateRawFacts($raw_facts); + $raw_facts = array(); + } + + return $result; + } + + private function computeRawFacts(PhabricatorLiskDAO $object) { + $facts = array(); + foreach ($this->engines as $engine) { + if (!$engine->shouldComputeRawFactsForObject($object)) { + continue; + } + $facts[] = $engine->computeRawFactsForObject($object); + } + + return array_mergev($facts); + } + + private function updateRawFacts(array $map) { + foreach ($map as $phid => $facts) { + assert_instances_of($facts, 'PhabricatorFactRaw'); + } + + $phids = array_keys($map); + if (!$phids) { + return; + } + + $table = new PhabricatorFactRaw(); + $conn = $table->establishConnection('w'); + $table_name = $table->getTableName(); + + $sql = array(); + foreach ($map as $phid => $facts) { + foreach ($facts as $fact) { + $sql[] = qsprintf( + $conn, + '(%s, %s, %s, %d, %d, %d)', + $fact->getFactType(), + $fact->getObjectPHID(), + $fact->getObjectA(), + $fact->getValueX(), + $fact->getValueY(), + $fact->getEpoch()); + } + } + + $table->openTransaction(); + + queryfx( + $conn, + 'DELETE FROM %T WHERE objectPHID IN (%Ls)', + $table_name, + $phids); + + if ($sql) { + foreach (array_chunk($sql, 256) as $chunk) { + queryfx( + $conn, + 'INSERT INTO %T + (factType, objectPHID, objectA, valueX, valueY, epoch) + VALUES %Q', + $table_name, + implode(', ', $chunk)); + } + } + + $table->saveTransaction(); + } + +} diff --git a/src/applications/fact/engine/PhabricatorFactCountEngine.php b/src/applications/fact/engine/PhabricatorFactCountEngine.php new file mode 100644 index 0000000000..d20df85ad8 --- /dev/null +++ b/src/applications/fact/engine/PhabricatorFactCountEngine.php @@ -0,0 +1,44 @@ +getPHID(); + $type = phid_get_type($phid); + + foreach (array('N:*', 'N:'.$type) as $fact_type) { + $facts[] = id(new PhabricatorFactRaw()) + ->setFactType($fact_type) + ->setObjectPHID($phid) + ->setEpoch($object->getDateCreated()); + } + + return $facts; + } + +} diff --git a/src/applications/fact/engine/PhabricatorFactEngine.php b/src/applications/fact/engine/PhabricatorFactEngine.php new file mode 100644 index 0000000000..cd951e913f --- /dev/null +++ b/src/applications/fact/engine/PhabricatorFactEngine.php @@ -0,0 +1,43 @@ +setAncestorClass(__CLASS__) + ->setConcreteOnly(true) + ->selectAndLoadSymbols(); + + $objects = array(); + foreach ($classes as $class) { + $objects[] = newv($class['name'], array()); + } + + return $objects; + } + + public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { + return false; + } + + public function computeRawFactsForObject(PhabricatorLiskDAO $object) { + return array(); + } + +} diff --git a/src/applications/facts/extract/PhabricatorFactsUpdateIterator.php b/src/applications/fact/extract/PhabricatorFactUpdateIterator.php similarity index 97% rename from src/applications/facts/extract/PhabricatorFactsUpdateIterator.php rename to src/applications/fact/extract/PhabricatorFactUpdateIterator.php index 19b1757396..b70a74a702 100644 --- a/src/applications/facts/extract/PhabricatorFactsUpdateIterator.php +++ b/src/applications/fact/extract/PhabricatorFactUpdateIterator.php @@ -21,7 +21,7 @@ * for "normal" Lisk objects: objects with an autoincrement ID and a * dateModified column. */ -final class PhabricatorFactsUpdateIterator extends PhutilBufferedIterator { +final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator { private $cursor; private $object; diff --git a/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php new file mode 100644 index 0000000000..c864277741 --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php @@ -0,0 +1,47 @@ +setName('analyze') + ->setSynopsis(pht('Manually invoke fact analyzers.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $daemon = new PhabricatorFactDaemon(array()); + $daemon->setVerbose(true); + $daemon->setEngines(PhabricatorFactEngine::loadAllEngines()); + + $iterators = array( + new PhabricatorFactUpdateIterator(new DifferentialRevision()), + ); + + foreach ($iterators as $iterator) { + $daemon->processIterator($iterator); + } + + return 0; + } + +} diff --git a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php new file mode 100644 index 0000000000..686c12fbbd --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php @@ -0,0 +1,58 @@ +setName('destroy') + ->setSynopsis(pht('Destroy all facts.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $question = pht( + 'Really destroy all facts? They will need to be rebuilt through '. + 'analysis, which may take some time.'); + + $ok = $console->confirm($question, $default = false); + if (!$ok) { + return 1; + } + + $tables = array(); + $tables[] = new PhabricatorFactRaw(); + foreach ($tables as $table) { + $conn = $table->establishConnection('w'); + $name = $table->getTableName(); + + $console->writeOut("%s\n", pht("Destroying table '%s'...", $name)); + + queryfx( + $conn, + 'TRUNCATE TABLE %T', + $name); + } + + $console->writeOut("%s\n", pht('Done.')); + } + +} diff --git a/src/applications/fact/management/PhabricatorFactManagementListWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementListWorkflow.php new file mode 100644 index 0000000000..b98a19cbc9 --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementListWorkflow.php @@ -0,0 +1,40 @@ +setName('list') + ->setSynopsis(pht('Show a list of fact engines.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $engines = PhabricatorFactEngine::loadAllEngines(); + foreach ($engines as $engine) { + $console->writeOut("%s\n", get_class($engine)); + } + + return 0; + } + +} diff --git a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php new file mode 100644 index 0000000000..114484f244 --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php @@ -0,0 +1,59 @@ +setName('status') + ->setSynopsis(pht('Show status of fact data.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $map = array( + 'raw' => new PhabricatorFactRaw(), + ); + + foreach ($map as $type => $table) { + $conn = $table->establishConnection('r'); + $name = $table->getTableName(); + + $row = queryfx_one( + $conn, + 'SELECT COUNT(*) N FROM %T', + $name); + + $n = $row['N']; + + switch ($type) { + case 'raw': + $desc = pht('There are %d raw fact(s) in storage.', $n); + break; + } + + $console->writeOut("%s\n", $desc); + } + + return 0; + } + +} diff --git a/src/applications/fact/management/PhabricatorFactManagementWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementWorkflow.php new file mode 100644 index 0000000000..5400a4ff0a --- /dev/null +++ b/src/applications/fact/management/PhabricatorFactManagementWorkflow.php @@ -0,0 +1,26 @@ + false, + ) + parent::getConfiguration(); + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactRaw.php b/src/applications/fact/storage/PhabricatorFactRaw.php new file mode 100644 index 0000000000..446d8f5afc --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactRaw.php @@ -0,0 +1,32 @@ + 'changed revisions, added %3$s; removed %5$s', + 'There are %d raw fact(s) in storage.' => array( + 'There is %d raw fact in storage.', + 'There are %d raw facts in storage.', + ), + ); }