diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index ecbd7f7c6b..f32565e78d 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -624,6 +624,7 @@ phutil_register_library_map(array( 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', + 'PhabricatorFactsUpdateIterator' => 'applications/facts/extract/PhabricatorFactsUpdateIterator.php', 'PhabricatorFeedBuilder' => 'applications/feed/builder/PhabricatorFeedBuilder.php', 'PhabricatorFeedConstants' => 'applications/feed/constants/PhabricatorFeedConstants.php', 'PhabricatorFeedController' => 'applications/feed/controller/PhabricatorFeedController.php', @@ -1507,7 +1508,7 @@ phutil_register_library_map(array( 'LiskIsolationTestCase' => 'PhabricatorTestCase', 'LiskIsolationTestDAO' => 'LiskDAO', 'LiskIsolationTestDAOException' => 'Exception', - 'LiskMigrationIterator' => 'Iterator', + 'LiskMigrationIterator' => 'PhutilBufferedIterator', 'ManiphestAction' => 'ManiphestConstants', 'ManiphestAuxiliaryFieldDefaultSpecification' => 'ManiphestAuxiliaryFieldSpecification', 'ManiphestAuxiliaryFieldTypeException' => 'Exception', @@ -1648,6 +1649,7 @@ phutil_register_library_map(array( 'PhabricatorEvent' => 'PhutilEvent', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhutilEventListener', + 'PhabricatorFactsUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFeedController' => 'PhabricatorController', 'PhabricatorFeedDAO' => 'PhabricatorLiskDAO', 'PhabricatorFeedPublicStreamController' => 'PhabricatorFeedController', diff --git a/src/applications/facts/extract/PhabricatorFactsUpdateIterator.php b/src/applications/facts/extract/PhabricatorFactsUpdateIterator.php new file mode 100644 index 0000000000..19b1757396 --- /dev/null +++ b/src/applications/facts/extract/PhabricatorFactsUpdateIterator.php @@ -0,0 +1,84 @@ +object = $object; + $this->start = $start; + } + + protected function didRewind() { + $this->cursor = $this->start; + } + + protected function getCursorFromObject($object) { + return $object->getDateModified().':'.$object->getID(); + } + + public function key() { + return $this->getCursorFromObject($this->current()); + } + + protected function loadPage() { + list($after_epoch, $after_id) = explode(':', $this->cursor); + + // NOTE: We ignore recent updates because once we process an update we'll + // never process rows behind it again. We need to read only rows which + // we're sure no new rows will be inserted behind. If we read a row that + // was updated on the current second, another update later on in this second + // could affect an object with a lower ID, and we'd skip that update. To + // avoid this, just ignore any rows which have been updated in the last few + // seconds. This also reduces the amount of work we need to do if an object + // is repeatedly updated; we will just look at the end state without + // processing the intermediate states. Finally, this gives us reasonable + // protections against clock skew between the machine the daemon is running + // on and any machines performing writes. + + $page = $this->object->loadAllWhere( + '((dateModified > %d) OR (dateModified = %d AND id > %d)) + AND (dateModified < %d - %d) + ORDER BY dateModified ASC, id ASC LIMIT %d', + $after_epoch, + $after_epoch, + $after_id, + time(), + $this->ignoreUpdatesDuration, + $this->getPageSize()); + + if ($page) { + $this->cursor = $this->getCursorFromObject(end($page)); + } + + return $page; + } + +}