When a ref is moved or deleted, put it on a list; later, check for reachability

Summary:
Ref T9028. This allows us to detect when commits are unreachable:

  - When a ref (tag, branch, etc) is moved or deleted, store the old thing it pointed at in a list.
  - After discovery, go through the list and check if all the stuff on it is still reachable.
  - If something isn't, try to follow its ancestors back until we find something that is reachable.
  - Then, mark everything we found as unreachable.
  - Finally, rebuild the repository summary table to correct the commit count.

Test Plan:
  - Deleted a ref, ran `pull` + `refs`, saw oldref in database.
  - Ran `discover`, saw it process the oldref, mark the unreachable commit, and update the summary table.
  - Visited commit page, saw it properly marked.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9028

Differential Revision: https://secure.phabricator.com/D16133

resources/sql/autopatches/20160616.repo.01.oldref.sql

@@ -0,0 +1,6 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_oldref (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  repositoryPHID VARBINARY(64) NOT NULL,
+  commitIdentifier VARCHAR(40) NOT NULL COLLATE {$COLLATE_TEXT},
+  KEY `key_repository` (repositoryPHID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -3277,6 +3277,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php',
     'PhabricatorRepositoryMirror' => 'applications/repository/storage/PhabricatorRepositoryMirror.php',
     'PhabricatorRepositoryMirrorEngine' => 'applications/repository/engine/PhabricatorRepositoryMirrorEngine.php',
+    'PhabricatorRepositoryOldRef' => 'applications/repository/storage/PhabricatorRepositoryOldRef.php',
     'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
     'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
     'PhabricatorRepositoryPullEvent' => 'applications/repository/storage/PhabricatorRepositoryPullEvent.php',
@@ -8067,6 +8068,10 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorRepositoryMirror' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryMirrorEngine' => 'PhabricatorRepositoryEngine',
+    'PhabricatorRepositoryOldRef' => array(
+      'PhabricatorRepositoryDAO',
+      'PhabricatorPolicyInterface',
+    ),
     'PhabricatorRepositoryParsedChange' => 'Phobject',
     'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
     'PhabricatorRepositoryPullEvent' => array(

src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php

@@ -105,6 +105,8 @@ final class PhabricatorRepositoryDiscoveryEngine
       $this->commitCache[$ref->getIdentifier()] = true;
     }
 
+    $this->markUnreachableCommits($repository);
+
     $version = $this->getObservedVersion($repository);
     if ($version !== null) {
       id(new DiffusionRepositoryClusterEngine())
@@ -731,4 +733,136 @@ final class PhabricatorRepositoryDiscoveryEngine
     return (int)$version['version'];
   }
 
+  private function markUnreachableCommits(PhabricatorRepository $repository) {
+    // For now, this is only supported for Git.
+    if (!$repository->isGit()) {
+      return;
+    }
+
+    // Find older versions of refs which we haven't processed yet. We're going
+    // to make sure their commits are still reachable.
+    $old_refs = id(new PhabricatorRepositoryOldRef())->loadAllWhere(
+      'repositoryPHID = %s',
+      $repository->getPHID());
+
+    // We can share a single graph stream across all the checks we need to do.
+    $stream = new PhabricatorGitGraphStream($repository);
+
+    foreach ($old_refs as $old_ref) {
+      $identifier = $old_ref->getCommitIdentifier();
+      $this->markUnreachableFrom($repository, $stream, $identifier);
+
+      // If nothing threw an exception, we're all done with this ref.
+      $old_ref->delete();
+    }
+  }
+
+  private function markUnreachableFrom(
+    PhabricatorRepository $repository,
+    PhabricatorGitGraphStream $stream,
+    $identifier) {
+
+    $unreachable = array();
+
+    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+      'repositoryID = %s AND commitIdentifier = %s',
+      $repository->getID(),
+      $identifier);
+    if (!$commit) {
+      return;
+    }
+
+    $look = array($commit);
+    $seen = array();
+    while ($look) {
+      $target = array_pop($look);
+
+      // If we've already checked this commit (for example, because history
+      // branches and then merges) we don't need to check it again.
+      $target_identifier = $target->getCommitIdentifier();
+      if (isset($seen[$target_identifier])) {
+        continue;
+      }
+
+      $seen[$target_identifier] = true;
+
+      try {
+        $stream->getCommitDate($target_identifier);
+        $reachable = true;
+      } catch (Exception $ex) {
+        $reachable = false;
+      }
+
+      if ($reachable) {
+        // This commit is reachable, so we don't need to go any further
+        // down this road.
+        continue;
+      }
+
+      $unreachable[] = $target;
+
+      // Find the commit's parents and check them for reachability, too. We
+      // have to look in the database since we no may longer have the commit
+      // in the repository.
+      $rows = queryfx_all(
+        $commit->establishConnection('w'),
+        'SELECT commit.* FROM %T commit
+          JOIN %T parents ON commit.id = parents.parentCommitID
+          WHERE parents.childCommitID = %d',
+        $commit->getTableName(),
+        PhabricatorRepository::TABLE_PARENTS,
+        $target->getID());
+      if (!$rows) {
+        continue;
+      }
+
+      $parents = id(new PhabricatorRepositoryCommit())
+        ->loadAllFromArray($rows);
+      foreach ($parents as $parent) {
+        $look[] = $parent;
+      }
+    }
+
+    $unreachable = array_reverse($unreachable);
+
+    $flag = PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE;
+    foreach ($unreachable as $unreachable_commit) {
+      $unreachable_commit->writeImportStatusFlag($flag);
+    }
+
+    // If anything was unreachable, just rebuild the whole summary table.
+    // We can't really update it incrementally when a commit becomes
+    // unreachable.
+    if ($unreachable) {
+      $this->rebuildSummaryTable($repository);
+    }
+  }
+
+  private function rebuildSummaryTable(PhabricatorRepository $repository) {
+    $conn_w = $repository->establishConnection('w');
+
+    $data = queryfx_one(
+      $conn_w,
+      'SELECT COUNT(*) N, MAX(id) id, MAX(epoch) epoch
+        FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d',
+      id(new PhabricatorRepositoryCommit())->getTableName(),
+      $repository->getID(),
+      PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE,
+      PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE);
+
+    queryfx(
+      $conn_w,
+      'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
+        VALUES (%d, %d, %d, %d)
+        ON DUPLICATE KEY UPDATE
+          size = VALUES(size),
+          lastCommitID = VALUES(lastCommitID),
+          epoch = VALUES(epoch)',
+      PhabricatorRepository::TABLE_SUMMARY,
+      $repository->getID(),
+      $data['N'],
+      $data['id'],
+      $data['epoch']);
+  }
+
 }

src/applications/repository/engine/PhabricatorRepositoryRefEngine.php

@@ -85,6 +85,13 @@ final class PhabricatorRepositoryRefEngine
           $ref->save();
         }
         foreach ($this->deadRefs as $ref) {
+          // Shove this ref into the old refs table so the discovery engine
+          // can check if any commits have been rendered unreachable.
+          id(new PhabricatorRepositoryOldRef())
+            ->setRepositoryPHID($repository->getPHID())
+            ->setCommitIdentifier($ref->getCommitIdentifier())
+            ->save();
+
           $ref->delete();
         }
       $repository->saveTransaction();

src/applications/repository/storage/PhabricatorRepositoryOldRef.php

@@ -0,0 +1,52 @@
+<?php
+
+/**
+ * Stores outdated refs which need to be checked for reachability.
+ *
+ * When a branch is deleted, the old HEAD ends up here and the discovery
+ * engine marks all the commits that previously appeared on it as unreachable.
+ */
+final class PhabricatorRepositoryOldRef
+  extends PhabricatorRepositoryDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $repositoryPHID;
+  protected $commitIdentifier;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'commitIdentifier' => 'text40',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_repository' => array(
+          'columns' => array('repositoryPHID'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return PhabricatorPolicies::getMostOpenPolicy();
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return false;
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return null;
+  }
+
+}