Add FactCursors and application fact datasources

Summary:
  - Add PhabricatorApplication. This is a general class that I have grand designs for, but used here to allow applications to provide objects for analysis by the facts appliction.
  - Add FactCursors, to keep track of where iterators are.
  - Make the daemon do something sort of useful.
  - Add `bin/fact cursors` for showing and managing objects and cursors.
  - Add some options to `bin/fact analyze`.

Test Plan:
  - `bin/fact cursors`, `bin/fact cursors --reset DifferentialRevision`, `bin/fact cursors --reset X`
  - `bin/fact analyze`, `bin/fact analyze --all`, `bin/fact analyze --iterator DifferentialRevision --skip-aggregates`
  - `bin/phd debug fact`

Reviewers: vrana, btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T1562

Differential Revision: https://secure.phabricator.com/D3098

scripts/fact/manage_facts.php

@@ -36,6 +36,7 @@ $workflows = array(
   new PhabricatorFactManagementAnalyzeWorkflow(),
   new PhabricatorFactManagementStatusWorkflow(),
   new PhabricatorFactManagementListWorkflow(),
+  new PhabricatorFactManagementCursorsWorkflow(),
   new PhutilHelpArgumentWorkflow(),
 );
 

src/__phutil_library_map__.php

@@ -530,6 +530,9 @@ phutil_register_library_map(array(
     'PackageModifyMail' => 'applications/owners/mail/PackageModifyMail.php',
     'Phabricator404Controller' => 'applications/base/controller/Phabricator404Controller.php',
     'PhabricatorAccessLog' => 'infrastructure/PhabricatorAccessLog.php',
+    'PhabricatorApplication' => 'applications/base/PhabricatorApplication.php',
+    'PhabricatorApplicationDifferential' => 'applications/differential/application/PhabricatorApplicationDifferential.php',
+    'PhabricatorApplicationManiphest' => 'applications/maniphest/application/PhabricatorApplicationManiphest.php',
     'PhabricatorAuditActionConstants' => 'applications/audit/constants/PhabricatorAuditActionConstants.php',
     'PhabricatorAuditAddCommentController' => 'applications/audit/controller/PhabricatorAuditAddCommentController.php',
     'PhabricatorAuditComment' => 'applications/audit/storage/PhabricatorAuditComment.php',
@@ -627,12 +630,14 @@ phutil_register_library_map(array(
     'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php',
     'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php',
     'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php',
+    'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
     'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
     'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
     'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
     'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php',
     'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php',
     'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php',
+    'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php',
     'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php',
     'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
     'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php',
@@ -1579,6 +1584,8 @@ phutil_register_library_map(array(
     'PackageDeleteMail' => 'PackageMail',
     'PackageModifyMail' => 'PackageMail',
     'Phabricator404Controller' => 'PhabricatorController',
+    'PhabricatorApplicationDifferential' => 'PhabricatorApplication',
+    'PhabricatorApplicationManiphest' => 'PhabricatorApplication',
     'PhabricatorAuditAddCommentController' => 'PhabricatorAuditController',
     'PhabricatorAuditComment' => 'PhabricatorAuditDAO',
     'PhabricatorAuditCommitListView' => 'AphrontView',
@@ -1669,16 +1676,19 @@ phutil_register_library_map(array(
     'PhabricatorFactAggregate' => 'PhabricatorFactDAO',
     'PhabricatorFactController' => 'PhabricatorController',
     'PhabricatorFactCountEngine' => 'PhabricatorFactEngine',
+    'PhabricatorFactCursor' => 'PhabricatorFactDAO',
     'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
     'PhabricatorFactDaemon' => 'PhabricatorDaemon',
     'PhabricatorFactHomeController' => 'PhabricatorFactController',
     'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine',
     'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow',
+    'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow',
     'PhabricatorFactManagementWorkflow' => 'PhutilArgumentWorkflow',
     'PhabricatorFactRaw' => 'PhabricatorFactDAO',
+    'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec',
     'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',
     'PhabricatorFeedController' => 'PhabricatorController',
     'PhabricatorFeedDAO' => 'PhabricatorLiskDAO',

src/applications/base/PhabricatorApplication.php

@@ -0,0 +1,70 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @task  info  Application Information
+ * @task  fact  Fact Integration
+ * @task  meta  Application Management
+ * @group apps
+ */
+abstract class PhabricatorApplication {
+
+
+/* -(  Application Information  )-------------------------------------------- */
+
+
+  public function getName() {
+    return substr(__CLASS__, strlen('PhabricatorApplication'));
+  }
+
+  public function isEnabled() {
+    return true;
+  }
+
+
+/* -(  Fact Integration  )--------------------------------------------------- */
+
+
+  public function getFactObjectsForAnalysis() {
+    return array();
+  }
+
+
+/* -(  Application Management  )--------------------------------------------- */
+
+
+  public static function getAllInstalledApplications() {
+    $classes = id(new PhutilSymbolLoader())
+      ->setAncestorClass(__CLASS__)
+      ->setConcreteOnly(true)
+      ->selectAndLoadSymbols();
+
+    $apps = array();
+    foreach ($classes as $class) {
+      $app = newv($class['name'], array());
+      if (!$app->isEnabled()) {
+        continue;
+      }
+      $apps[] = $app;
+    }
+
+    return $apps;
+  }
+
+
+}

src/applications/differential/application/PhabricatorApplicationDifferential.php

@@ -0,0 +1,28 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorApplicationDifferential extends PhabricatorApplication {
+
+  public function getFactObjectsForAnalysis() {
+    return array(
+      new DifferentialRevision(),
+    );
+  }
+
+}
+

src/applications/fact/daemon/PhabricatorFactDaemon.php

@@ -23,7 +23,57 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
   const RAW_FACT_BUFFER_LIMIT = 128;
 
   public function run() {
-    throw new Exception("This daemon doesn't do anything yet!");
+    $this->setEngines(PhabricatorFactEngine::loadAllEngines());
+    while (true) {
+      $iterators = $this->getAllApplicationIterators();
+      foreach ($iterators as $iterator_name => $iterator) {
+        $this->processIteratorWithCursor($iterator_name, $iterator);
+      }
+      $this->processAggregates();
+
+      $this->log("Zzz...");
+      $this->sleep(60 * 5);
+    }
+  }
+
+  public static function getAllApplicationIterators() {
+    $apps = PhabricatorApplication::getAllInstalledApplications();
+
+    $iterators = array();
+    foreach ($apps as $app) {
+      foreach ($app->getFactObjectsForAnalysis() as $object) {
+        $iterator = new PhabricatorFactUpdateIterator($object);
+        $iterators[get_class($object)] = $iterator;
+      }
+    }
+
+    return $iterators;
+  }
+
+  public function processIteratorWithCursor($iterator_name, $iterator) {
+    $this->log("Processing cursor '{$iterator_name}'.");
+
+    $cursor = id(new PhabricatorFactCursor())->loadOneWhere(
+      'name = %s',
+      $iterator_name);
+    if (!$cursor) {
+      $cursor = new PhabricatorFactCursor();
+      $cursor->setName($iterator_name);
+      $position = null;
+    } else {
+      $position = $cursor->getPosition();
+    }
+
+    if ($position) {
+      $iterator->setPosition($position);
+    }
+
+    $new_cursor_position = $this->processIterator($iterator);
+
+    if ($new_cursor_position) {
+      $cursor->setPosition($new_cursor_position);
+      $cursor->save();
+    }
   }
 
   public function setEngines(array $engines) {
@@ -38,7 +88,9 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
 
     $raw_facts = array();
     foreach ($iterator as $key => $object) {
-      $raw_facts[$object->getPHID()] = $this->computeRawFacts($object);
+      $phid = $object->getPHID();
+      $this->log("Processing {$phid}...");
+      $raw_facts[$phid] = $this->computeRawFacts($object);
       if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) {
         $this->updateRawFacts($raw_facts);
         $raw_facts = array();
@@ -55,6 +107,8 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
   }
 
   public function processAggregates() {
+    $this->log("Processing aggregates.");
+
     $facts = $this->computeAggregateFacts();
     $this->updateAggregateFacts($facts);
   }

src/applications/fact/extract/PhabricatorFactUpdateIterator.php

@@ -28,12 +28,14 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
   private $start;
   private $ignoreUpdatesDuration = 15;
 
-  public function __construct(LiskDAO $object, $start = null) {
+  public function __construct(LiskDAO $object) {
-    if ($start === null) {
-      $start = '0:0';
-    }
     $this->object = $object;
-    $this->start  = $start;
+    $this->start = '0:0';
+  }
+
+  public function setPosition($position) {
+    $this->start = $position;
+    return $this;
   }
 
   protected function didRewind() {

src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php

@@ -23,7 +23,23 @@ final class PhabricatorFactManagementAnalyzeWorkflow
     $this
       ->setName('analyze')
       ->setSynopsis(pht('Manually invoke fact analyzers.'))
-      ->setArguments(array());
+      ->setArguments(
+        array(
+          array(
+            'name'    => 'iterator',
+            'param'   => 'name',
+            'repeat'  => true,
+            'help'    => 'Process only __iterator__.',
+          ),
+          array(
+            'name'    => 'all',
+            'help'    => 'Analyze from the beginning, ignoring cursors.',
+          ),
+          array(
+            'name'    => 'skip-aggregates',
+            'help'    => 'Skip analysis of aggreate facts.',
+          ),
+        ));
   }
 
   public function execute(PhutilArgumentParser $args) {
@@ -33,15 +49,34 @@ final class PhabricatorFactManagementAnalyzeWorkflow
     $daemon->setVerbose(true);
     $daemon->setEngines(PhabricatorFactEngine::loadAllEngines());
 
-    $iterators = array(
+    $iterators = PhabricatorFactDaemon::getAllApplicationIterators();
-      new PhabricatorFactUpdateIterator(new DifferentialRevision()),
-    );
 
-    foreach ($iterators as $iterator) {
+    $selected = $args->getArg('iterator');
-      $daemon->processIterator($iterator);
+    if ($selected) {
+      $use = array();
+      foreach ($selected as $iterator_name) {
+        if (isset($iterators[$iterator_name])) {
+          $use[$iterator_name] = $iterators[$iterator_name];
+        } else {
+          $console->writeErr(
+            "%s\n",
+            pht("Iterator '%s' does not exist.", $iterator_name));
+        }
+      }
+      $iterators = $use;
     }
 
+    foreach ($iterators as $iterator_name => $iterator) {
+      if ($args->getArg('all')) {
+        $daemon->processIterator($iterator);
+      } else {
+        $daemon->processIteratorWithCursor($iterator_name, $iterator);
+      }
+    }
+
+    if (!$args->getArg('skip-aggregates')) {
       $daemon->processAggregates();
+    }
 
     return 0;
   }

src/applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php

@@ -0,0 +1,82 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorFactManagementCursorsWorkflow
+  extends PhabricatorFactManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('cursors')
+      ->setSynopsis(pht('Show a list of fact iterators and cursors.'))
+      ->setExamples(
+        "**cursors**\n".
+        "**cursors** --reset __cursor__")
+      ->setArguments(
+        array(
+          array(
+            'name'    => 'reset',
+            'param'   => 'cursor',
+            'repeat'  => true,
+            'help'    => 'Reset cursor __cursor__.',
+          )
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $console = PhutilConsole::getConsole();
+
+    $reset = $args->getArg('reset');
+    if ($reset) {
+      foreach ($reset as $name) {
+        $cursor = id(new PhabricatorFactCursor())->loadOneWhere(
+          'name = %s',
+          $name);
+        if ($cursor) {
+          $console->writeOut("%s\n", pht("Resetting cursor %s...", $name));
+          $cursor->delete();
+        } else {
+          $console->writeErr(
+            "%s\n",
+            pht("Cursor %s does not exist or is already reset.", $name));
+        }
+      }
+      return 0;
+    }
+
+    $iterator_map = PhabricatorFactDaemon::getAllApplicationIterators();
+    if (!$iterator_map) {
+      $console->writeErr("%s\n", pht("No cursors."));
+      return 0;
+    }
+
+    $cursors = id(new PhabricatorFactCursor())->loadAllWhere(
+      'name IN (%Ls)',
+      array_keys($iterator_map));
+    $cursors = mpull($cursors, 'getPosition', 'getName');
+
+    foreach ($iterator_map as $iterator_name => $iterator) {
+      $console->writeOut(
+        "%s (%s)\n",
+        $iterator_name,
+        idx($cursors, $iterator_name, 'start'));
+    }
+
+    return 0;
+  }
+
+}

src/applications/fact/storage/PhabricatorFactCursor.php

@@ -0,0 +1,25 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorFactCursor extends PhabricatorFactDAO {
+
+  protected $id;
+  protected $name;
+  protected $position;
+
+}

src/applications/maniphest/application/PhabricatorApplicationManiphest.php

@@ -0,0 +1,32 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorApplicationManiphest extends PhabricatorApplication {
+
+  public function isEnabled() {
+    return PhabricatorEnv::getEnvConfig('maniphest.enabled');
+  }
+
+  public function getFactObjectsForAnalysis() {
+    return array(
+      new ManiphestTask(),
+    );
+  }
+
+}
+