Allow Fact analysis of commits

Summary: For immutable objects, just use the ID as a cursor.

Test Plan:
  - Analyzed commits from an empty cursor.
  - Checked that cursor was good.
  - Pulled some more commits.
  - Analyzed commits again, verified it only hit the new ones.
  - Verified the graph of "Count of CMIT" looked reasonable.

Reviewers: vrana

Reviewed By: vrana

CC: aran

Maniphest Tasks: T1866

Differential Revision: https://secure.phabricator.com/D3656

src/applications/fact/extract/PhabricatorFactUpdateIterator.php

@@ -33,7 +33,6 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
   public function __construct(LiskDAO $object) {
     $this->set = new LiskDAOSet();
     $this->object = $object->putInSet($this->set);
-    $this->position = '0:0';
   }
 
   public function setPosition($position) {
@@ -46,7 +45,11 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
   }
 
   protected function getCursorFromObject($object) {
-    return $object->getDateModified().':'.$object->getID();
+    if ($object->hasProperty('dateModified')) {
+      return $object->getDateModified().':'.$object->getID();
+    } else {
+      return $object->getID();
+    }
   }
 
   public function key() {
@@ -54,32 +57,50 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
   }
 
   protected function loadPage() {
-    list($after_epoch, $after_id) = explode(':', $this->cursor);
-
     $this->set->clearSet();
 
-    // NOTE: We ignore recent updates because once we process an update we'll
+    if ($this->object->hasProperty('dateModified')) {
-    // never process rows behind it again. We need to read only rows which
+      if ($this->cursor) {
-    // we're sure no new rows will be inserted behind. If we read a row that
+        list($after_epoch, $after_id) = explode(':', $this->cursor);
-    // was updated on the current second, another update later on in this second
+      } else {
-    // could affect an object with a lower ID, and we'd skip that update. To
+        $after_epoch = 0;
-    // avoid this, just ignore any rows which have been updated in the last few
+        $after_id = 0;
-    // seconds. This also reduces the amount of work we need to do if an object
+      }
-    // is repeatedly updated; we will just look at the end state without
-    // processing the intermediate states. Finally, this gives us reasonable
-    // protections against clock skew between the machine the daemon is running
-    // on and any machines performing writes.
 
-    $page = $this->object->loadAllWhere(
+      // NOTE: We ignore recent updates because once we process an update we'll
-      '((dateModified > %d) OR (dateModified = %d AND id > %d))
+      // never process rows behind it again. We need to read only rows which
-        AND (dateModified < %d - %d)
+      // we're sure no new rows will be inserted behind. If we read a row that
-        ORDER BY dateModified ASC, id ASC LIMIT %d',
+      // was updated on the current second, another update later on in this
-      $after_epoch,
+      // second could affect an object with a lower ID, and we'd skip that
-      $after_epoch,
+      // update. To avoid this, just ignore any rows which have been updated in
-      $after_id,
+      // the last few seconds. This also reduces the amount of work we need to
-      time(),
+      // do if an object is repeatedly updated; we will just look at the end
-      $this->ignoreUpdatesDuration,
+      // state without processing the intermediate states. Finally, this gives
-      $this->getPageSize());
+      // us reasonable protections against clock skew between the machine the
+      // daemon is running on and any machines performing writes.
+
+      $page = $this->object->loadAllWhere(
+        '((dateModified > %d) OR (dateModified = %d AND id > %d))
+          AND (dateModified < %d - %d)
+          ORDER BY dateModified ASC, id ASC LIMIT %d',
+        $after_epoch,
+        $after_epoch,
+        $after_id,
+        time(),
+        $this->ignoreUpdatesDuration,
+        $this->getPageSize());
+    } else {
+      if ($this->cursor) {
+        $after_id = $this->cursor;
+      } else {
+        $after_id = 0;
+      }
+
+      $page = $this->object->loadAllWhere(
+        'id > %d ORDER BY id ASC LIMIT %d',
+        $after_id,
+        $this->getPageSize());
+    }
 
     if ($page) {
       $this->cursor = $this->getCursorFromObject(end($page));

src/applications/repository/storage/PhabricatorRepositoryCommit.php

@@ -102,6 +102,11 @@ final class PhabricatorRepositoryCommit extends PhabricatorRepositoryDAO {
     return $result;
   }
 
+  public function getDateCreated() {
+    // This is primarily to make analysis of commits with the Fact engine work.
+    return $this->getEpoch();
+  }
+
   /**
    * Synchronize a commit's overall audit status with the individual audit
    * triggers.
@@ -144,6 +149,4 @@ final class PhabricatorRepositoryCommit extends PhabricatorRepositoryDAO {
     return $this->setAuditStatus($status);
   }
 
-
-
 }

src/infrastructure/storage/lisk/LiskDAO.php

@@ -647,7 +647,7 @@ abstract class LiskDAO {
           // already determined it's not valid. We don't need to check again.
           continue;
         }
-        $valid_properties[$k] = (bool)$this->checkProperty($k);
+        $valid_properties[$k] = $this->hasProperty($k);
         if (!$valid_properties[$k]) {
           continue;
         }
@@ -896,6 +896,18 @@ abstract class LiskDAO {
   }
 
 
+  /**
+   * Test if a property exists.
+   *
+   * @param   string    Property name.
+   * @return  bool      True if the property exists.
+   * @task info
+   */
+  public function hasProperty($property) {
+    return (bool)$this->checkProperty($property);
+  }
+
+
   /**
    * Retrieve a list of all object properties. This list only includes
    * properties that are declared as protected, and it is expected that