diff --git a/resources/sql/autopatches/20160616.repo.01.oldref.sql b/resources/sql/autopatches/20160616.repo.01.oldref.sql new file mode 100644 index 0000000000..63bced8aab --- /dev/null +++ b/resources/sql/autopatches/20160616.repo.01.oldref.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_oldref ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + repositoryPHID VARBINARY(64) NOT NULL, + commitIdentifier VARCHAR(40) NOT NULL COLLATE {$COLLATE_TEXT}, + KEY `key_repository` (repositoryPHID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c4a590fb2b..4e9dd481a6 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3277,6 +3277,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php', 'PhabricatorRepositoryMirror' => 'applications/repository/storage/PhabricatorRepositoryMirror.php', 'PhabricatorRepositoryMirrorEngine' => 'applications/repository/engine/PhabricatorRepositoryMirrorEngine.php', + 'PhabricatorRepositoryOldRef' => 'applications/repository/storage/PhabricatorRepositoryOldRef.php', 'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php', 'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php', 'PhabricatorRepositoryPullEvent' => 'applications/repository/storage/PhabricatorRepositoryPullEvent.php', @@ -8067,6 +8068,10 @@ phutil_register_library_map(array( 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', 'PhabricatorRepositoryMirror' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryMirrorEngine' => 'PhabricatorRepositoryEngine', + 'PhabricatorRepositoryOldRef' => array( + 'PhabricatorRepositoryDAO', + 'PhabricatorPolicyInterface', + ), 'PhabricatorRepositoryParsedChange' => 'Phobject', 'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine', 'PhabricatorRepositoryPullEvent' => array( diff --git a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php index e8c58ed52f..cdb7e11d03 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php @@ -105,6 +105,8 @@ final class PhabricatorRepositoryDiscoveryEngine $this->commitCache[$ref->getIdentifier()] = true; } + $this->markUnreachableCommits($repository); + $version = $this->getObservedVersion($repository); if ($version !== null) { id(new DiffusionRepositoryClusterEngine()) @@ -731,4 +733,136 @@ final class PhabricatorRepositoryDiscoveryEngine return (int)$version['version']; } + private function markUnreachableCommits(PhabricatorRepository $repository) { + // For now, this is only supported for Git. + if (!$repository->isGit()) { + return; + } + + // Find older versions of refs which we haven't processed yet. We're going + // to make sure their commits are still reachable. + $old_refs = id(new PhabricatorRepositoryOldRef())->loadAllWhere( + 'repositoryPHID = %s', + $repository->getPHID()); + + // We can share a single graph stream across all the checks we need to do. + $stream = new PhabricatorGitGraphStream($repository); + + foreach ($old_refs as $old_ref) { + $identifier = $old_ref->getCommitIdentifier(); + $this->markUnreachableFrom($repository, $stream, $identifier); + + // If nothing threw an exception, we're all done with this ref. + $old_ref->delete(); + } + } + + private function markUnreachableFrom( + PhabricatorRepository $repository, + PhabricatorGitGraphStream $stream, + $identifier) { + + $unreachable = array(); + + $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %s AND commitIdentifier = %s', + $repository->getID(), + $identifier); + if (!$commit) { + return; + } + + $look = array($commit); + $seen = array(); + while ($look) { + $target = array_pop($look); + + // If we've already checked this commit (for example, because history + // branches and then merges) we don't need to check it again. + $target_identifier = $target->getCommitIdentifier(); + if (isset($seen[$target_identifier])) { + continue; + } + + $seen[$target_identifier] = true; + + try { + $stream->getCommitDate($target_identifier); + $reachable = true; + } catch (Exception $ex) { + $reachable = false; + } + + if ($reachable) { + // This commit is reachable, so we don't need to go any further + // down this road. + continue; + } + + $unreachable[] = $target; + + // Find the commit's parents and check them for reachability, too. We + // have to look in the database since we no may longer have the commit + // in the repository. + $rows = queryfx_all( + $commit->establishConnection('w'), + 'SELECT commit.* FROM %T commit + JOIN %T parents ON commit.id = parents.parentCommitID + WHERE parents.childCommitID = %d', + $commit->getTableName(), + PhabricatorRepository::TABLE_PARENTS, + $target->getID()); + if (!$rows) { + continue; + } + + $parents = id(new PhabricatorRepositoryCommit()) + ->loadAllFromArray($rows); + foreach ($parents as $parent) { + $look[] = $parent; + } + } + + $unreachable = array_reverse($unreachable); + + $flag = PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE; + foreach ($unreachable as $unreachable_commit) { + $unreachable_commit->writeImportStatusFlag($flag); + } + + // If anything was unreachable, just rebuild the whole summary table. + // We can't really update it incrementally when a commit becomes + // unreachable. + if ($unreachable) { + $this->rebuildSummaryTable($repository); + } + } + + private function rebuildSummaryTable(PhabricatorRepository $repository) { + $conn_w = $repository->establishConnection('w'); + + $data = queryfx_one( + $conn_w, + 'SELECT COUNT(*) N, MAX(id) id, MAX(epoch) epoch + FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d', + id(new PhabricatorRepositoryCommit())->getTableName(), + $repository->getID(), + PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE, + PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE); + + queryfx( + $conn_w, + 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) + VALUES (%d, %d, %d, %d) + ON DUPLICATE KEY UPDATE + size = VALUES(size), + lastCommitID = VALUES(lastCommitID), + epoch = VALUES(epoch)', + PhabricatorRepository::TABLE_SUMMARY, + $repository->getID(), + $data['N'], + $data['id'], + $data['epoch']); + } + } diff --git a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php index f486eb442c..af07ea052a 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php @@ -85,6 +85,13 @@ final class PhabricatorRepositoryRefEngine $ref->save(); } foreach ($this->deadRefs as $ref) { + // Shove this ref into the old refs table so the discovery engine + // can check if any commits have been rendered unreachable. + id(new PhabricatorRepositoryOldRef()) + ->setRepositoryPHID($repository->getPHID()) + ->setCommitIdentifier($ref->getCommitIdentifier()) + ->save(); + $ref->delete(); } $repository->saveTransaction(); diff --git a/src/applications/repository/storage/PhabricatorRepositoryOldRef.php b/src/applications/repository/storage/PhabricatorRepositoryOldRef.php new file mode 100644 index 0000000000..65cba3c48b --- /dev/null +++ b/src/applications/repository/storage/PhabricatorRepositoryOldRef.php @@ -0,0 +1,52 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'commitIdentifier' => 'text40', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_repository' => array( + 'columns' => array('repositoryPHID'), + ), + ), + ) + parent::getConfiguration(); + } + + +/* -( PhabricatorPolicyInterface )----------------------------------------- */ + + + public function getCapabilities() { + return array( + PhabricatorPolicyCapability::CAN_VIEW, + ); + } + + public function getPolicy($capability) { + return PhabricatorPolicies::getMostOpenPolicy(); + } + + public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { + return false; + } + + public function describeAutomaticCapability($capability) { + return null; + } + +}