Introduce ref cursors for repository parsing

Summary:
Ref T4327. I want to make change parsing testable; one thing which is blocking this is that the Git discovery process is still part of `PullLocal` daemon instead of being part of `DiscoveryEngine`. The unit test stuff which I want to use for change parsing relies on `DiscoveryEngine` to discover repositories during unit tests.

The major reason git discovery isn't part of `DiscoveryEngine` is that it relies on the messy "autoclose" logic, which we never implemented for Mercurial. Generally, I don't like how autoclose was implemented: it's complicated and gross and too hard to figure out and extend.

Instead, I want to do something more similar to what we do for pushes, which is cleaner overall. Basically this means remembering the old branch heads from the last time we parsed a repository, and figuring out what's new by comparing the old and new branch heads. This should give us several advantages:

  - It should be simpler to understand than the autoclose stuff, which is pretty mind-numbing, at least for me.
  - It will let us satisfy branch and tag queries cheaply (from the database) instead of having to go to the repository. We could also satisfy some ref-resolve queries from the database.
  - It should be easier to extend to Mercurial.

This implements the basics -- pretty much a table to store the cursors, which we update only for Git for now.

Test Plan:
  - Ran migration.
  - Ran `bin/repository discover X --trace --verbose` on various repositories with branches and tags, before and after modifying pushes.
  - Pushed commits to a git repo.
  - Looked at database tables.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T4327

Differential Revision: https://secure.phabricator.com/D7982

resources/sql/autopatches/20140116.reporefcursor.sql

@@ -0,0 +1,11 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_refcursor (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  repositoryPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  refType VARCHAR(32) NOT NULL COLLATE utf8_bin,
+  refNameHash VARCHAR(12) NOT NULL COLLATE latin1_bin,
+  refNameRaw LONGTEXT NOT NULL COLLATE latin1_bin,
+  refNameEncoding VARCHAR(16) COLLATE utf8_bin,
+  commitIdentifier VARCHAR(40) NOT NULL COLLATE utf8_bin,
+
+  KEY `key_cursor` (repositoryPHID, refType, refNameHash)
+) ENGINE=InnoDB, COLLATE=utf8_general_ci;

src/__phutil_library_map__.php

@@ -1856,6 +1856,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php',
     'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php',
+    'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php',
     'PhabricatorRepositoryManagementWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementWorkflow.php',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php',
@@ -1872,6 +1873,9 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPushLogQuery' => 'applications/repository/query/PhabricatorRepositoryPushLogQuery.php',
     'PhabricatorRepositoryPushLogSearchEngine' => 'applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php',
     'PhabricatorRepositoryQuery' => 'applications/repository/query/PhabricatorRepositoryQuery.php',
+    'PhabricatorRepositoryRefCursor' => 'applications/repository/storage/PhabricatorRepositoryRefCursor.php',
+    'PhabricatorRepositoryRefCursorQuery' => 'applications/repository/query/PhabricatorRepositoryRefCursorQuery.php',
+    'PhabricatorRepositoryRefEngine' => 'applications/repository/engine/PhabricatorRepositoryRefEngine.php',
     'PhabricatorRepositorySearchEngine' => 'applications/repository/query/PhabricatorRepositorySearchEngine.php',
     'PhabricatorRepositoryStatusMessage' => 'applications/repository/storage/PhabricatorRepositoryStatusMessage.php',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php',
@@ -4521,6 +4525,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
+    'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
@@ -4545,6 +4550,13 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPushLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositoryPushLogSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorRepositoryQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorRepositoryRefCursor' =>
+    array(
+      0 => 'PhabricatorRepositoryDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'PhabricatorRepositoryRefCursorQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorRepositoryRefEngine' => 'PhabricatorRepositoryEngine',
     'PhabricatorRepositorySearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorRepositoryStatusMessage' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',

src/applications/diffusion/data/DiffusionBranchInformation.php

@@ -42,4 +42,15 @@ final class DiffusionBranchInformation {
     );
   }
 
+  // TODO: These are hacks to make this compatible with DiffusionRepositoryRef
+  // for PhabricatorRepositoryRefEngine. The two classes should be merged.
+
+  public function getShortName() {
+    return $this->getName();
+  }
+
+  public function getCommitIdentifier() {
+    return $this->getHeadCommitIdentifier();
+  }
+
 }

src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php

@@ -149,6 +149,7 @@ final class PhabricatorRepositoryPullLocalDaemon
                 PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
                 null);
               $this->discoverRepository($repository);
+              $this->updateRepositoryRefs($repository);
               $repository->writeStatusMessage(
                 PhabricatorRepositoryStatusMessage::TYPE_FETCH,
                 PhabricatorRepositoryStatusMessage::CODE_OKAY);
@@ -267,6 +268,12 @@ final class PhabricatorRepositoryPullLocalDaemon
     }
   }
 
+  private function updateRepositoryRefs(PhabricatorRepository $repository) {
+    id(new PhabricatorRepositoryRefEngine())
+      ->setRepository($repository)
+      ->updateRefs();
+  }
+
   private function getDiscoveryEngine(PhabricatorRepository $repository) {
     $id = $repository->getID();
     if (empty($this->discoveryEngines[$id])) {
@@ -615,6 +622,7 @@ final class PhabricatorRepositoryPullLocalDaemon
   }
 
 
+
   /**
    * @task git
    */

src/applications/repository/engine/PhabricatorRepositoryRefEngine.php

@@ -0,0 +1,223 @@
+<?php
+
+/**
+ * Update the ref cursors for a repository, which track the positions of
+ * branches, bookmarks, and tags.
+ */
+final class PhabricatorRepositoryRefEngine
+  extends PhabricatorRepositoryEngine {
+
+  private $newRefs = array();
+  private $deadRefs = array();
+
+  public function updateRefs() {
+    $this->newRefs = array();
+    $this->deadRefs = array();
+
+    $repository = $this->getRepository();
+
+    $vcs = $repository->getVersionControlSystem();
+    switch ($vcs) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+        // No meaningful refs of any type in Subversion.
+        $branches = array();
+        $bookmarks = array();
+        $tags = array();
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $branches = $this->loadMercurialBranchPositions($repository);
+        $bookmarks = $this->loadMercurialBookmarkPositions($repository);
+        $tags = array();
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        $branches = $this->loadGitBranchPositions($repository);
+        $bookmarks = array();
+        $tags = $this->loadGitTagPositions($repository);
+        break;
+      default:
+        throw new Exception(pht('Unknown VCS "%s"!', $vcs));
+    }
+
+    $maps = array(
+      PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
+      PhabricatorRepositoryRefCursor::TYPE_TAG => $tags,
+      PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
+    );
+
+    $all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withRepositoryPHIDs(array($repository->getPHID()))
+      ->execute();
+    $cursor_groups = mgroup($all_cursors, 'getRefType');
+
+    foreach ($maps as $type => $refs) {
+      $cursor_group = idx($cursor_groups, $type, array());
+      $this->updateCursors($cursor_group, $refs, $type);
+    }
+
+    if ($this->newRefs || $this->deadRefs) {
+      $repository->openTransaction();
+        foreach ($this->newRefs as $ref) {
+          $ref->save();
+        }
+        foreach ($this->deadRefs as $ref) {
+          $ref->delete();
+        }
+      $repository->saveTransaction();
+
+      $this->newRefs = array();
+      $this->deadRefs = array();
+    }
+  }
+
+  private function markRefNew(PhabricatorRepositoryRefCursor $cursor) {
+    $this->newRefs[] = $cursor;
+    return $this;
+  }
+
+  private function markRefDead(PhabricatorRepositoryRefCursor $cursor) {
+    $this->deadRefs[] = $cursor;
+    return $this;
+  }
+
+  private function updateCursors(
+    array $cursors,
+    array $new_refs,
+    $ref_type) {
+    $repository = $this->getRepository();
+
+    // NOTE: Mercurial branches may have multiple branch heads; this logic
+    // is complex primarily to account for that.
+
+    // Group all the cursors by their ref name, like "master". Since Mercurial
+    // branches may have multiple heads, there could be several cursors with
+    // the same name.
+    $cursor_groups = mgroup($cursors, 'getRefNameRaw');
+
+    // Group all the new ref values by their name. As above, these groups may
+    // have multiple members in Mercurial.
+    $ref_groups = mgroup($new_refs, 'getShortName');
+
+    foreach ($ref_groups as $name => $refs) {
+      $new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
+
+      $ref_cursors = idx($cursor_groups, $name, array());
+      $old_commits = mpull($ref_cursors, null, 'getCommitIdentifier');
+
+      // We're going to delete all the cursors pointing at commits which are
+      // no longer associated with the refs. This primarily makes the Mercurial
+      // multiple head case easier, and means that when we update a ref we
+      // delete the old one and write a new one.
+      foreach ($ref_cursors as $cursor) {
+        if (isset($new_commits[$cursor->getCommitIdentifier()])) {
+          // This ref previously pointed at this commit, and still does.
+          $this->log(
+            pht(
+              'Ref %s "%s" still points at %s.',
+              $ref_type,
+              $name,
+              $cursor->getCommitIdentifier()));
+        } else {
+          // This ref previously pointed at this commit, but no longer does.
+          $this->log(
+            pht(
+              'Ref %s "%s" no longer points at %s.',
+              $ref_type,
+              $name,
+              $cursor->getCommitIdentifier()));
+
+          // Nuke the obsolete cursor.
+          $this->markRefDead($cursor);
+        }
+      }
+
+      // Now, we're going to insert new cursors for all the commits which are
+      // associated with this ref that don't currently have cursors.
+      $added_commits = array_diff_key($new_commits, $old_commits);
+      foreach ($added_commits as $identifier) {
+        $this->log(
+          pht(
+            'Ref %s "%s" now points at %s.',
+            $ref_type,
+            $name,
+            $identifier));
+        $this->markRefNew(
+          id(new PhabricatorRepositoryRefCursor())
+            ->setRepositoryPHID($repository->getPHID())
+            ->setRefType($ref_type)
+            ->setRefName($name)
+            ->setCommitIdentifier($identifier));
+      }
+
+      foreach ($added_commits as $identifier) {
+        // TODO: Do autoclose stuff here.
+      }
+    }
+
+    // Find any cursors for refs which no longer exist. This happens when a
+    // branch, tag or bookmark is deleted.
+
+    foreach ($cursor_groups as $name => $cursor_group) {
+      if (idx($ref_groups, $name) === null) {
+        $this->log(
+          pht(
+            'Ref %s "%s" no longer exists.',
+            $cursor->getRefType(),
+            $cursor->getRefName()));
+        foreach ($cursor_group as $cursor) {
+          $this->markRefDead($cursor);
+        }
+      }
+    }
+  }
+
+
+/* -(  Updating Git Refs  )-------------------------------------------------- */
+
+
+  /**
+   * @task git
+   */
+  private function loadGitBranchPositions(PhabricatorRepository $repository) {
+    return id(new DiffusionLowLevelGitRefQuery())
+      ->setRepository($repository)
+      ->withIsOriginBranch(true)
+      ->execute();
+  }
+
+
+  /**
+   * @task git
+   */
+  private function loadGitTagPositions(PhabricatorRepository $repository) {
+    return id(new DiffusionLowLevelGitRefQuery())
+      ->setRepository($repository)
+      ->withIsTag(true)
+      ->execute();
+  }
+
+
+/* -(  Updating Mercurial Refs  )-------------------------------------------- */
+
+
+  /**
+   * @task hg
+   */
+  private function loadMercurialBranchPositions(
+    PhabricatorRepository $repository) {
+    return id(new DiffusionLowLevelMercurialBranchesQuery())
+      ->setRepository($repository)
+      ->execute();
+  }
+
+
+  /**
+   * @task hg
+   */
+  private function loadMercurialBookmarkPositions(
+    PhabricatorRepository $repository) {
+    // TODO: Implement support for Mercurial bookmarks.
+    return array();
+  }
+
+}

src/applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php

@@ -0,0 +1,49 @@
+<?php
+
+final class PhabricatorRepositoryManagementRefsWorkflow
+  extends PhabricatorRepositoryManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('refs')
+      ->setExamples('**refs** [__options__] __repository__ ...')
+      ->setSynopsis('Update refs in __repository__, named by callsign.')
+      ->setArguments(
+        array(
+          array(
+            'name'        => 'verbose',
+            'help'        => 'Show additional debugging information.',
+          ),
+          array(
+            'name'        => 'repos',
+            'wildcard'    => true,
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $repos = $this->loadRepositories($args, 'repos');
+
+    if (!$repos) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          "Specify one or more repositories to update refs for, ".
+          "by callsign."));
+    }
+
+    $console = PhutilConsole::getConsole();
+    foreach ($repos as $repo) {
+      $console->writeOut("Updating refs in '%s'...\n", $repo->getCallsign());
+
+      $engine = id(new PhabricatorRepositoryRefEngine())
+        ->setRepository($repo)
+        ->setVerbose($args->getArg('verbose'))
+        ->updateRefs();
+    }
+
+    $console->writeOut("Done.\n");
+
+    return 0;
+  }
+
+}

src/applications/repository/query/PhabricatorRepositoryRefCursorQuery.php

@@ -0,0 +1,83 @@
+<?php
+
+final class PhabricatorRepositoryRefCursorQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $repositoryPHIDs;
+  private $refTypes;
+
+  public function withRepositoryPHIDs(array $phids) {
+    $this->repositoryPHIDs = $phids;
+    return $this;
+  }
+
+  public function withRefTypes(array $types) {
+    $this->refTypes = $types;
+    return $this;
+  }
+
+  protected function loadPage() {
+    $table = new PhabricatorRepositoryRefCursor();
+    $conn_r = $table->establishConnection('r');
+
+    $data = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T r %Q %Q %Q',
+      $table->getTableName(),
+      $this->buildWhereClause($conn_r),
+      $this->buildOrderClause($conn_r),
+      $this->buildLimitClause($conn_r));
+
+    return $table->loadAllFromArray($data);
+  }
+
+  public function willFilterPage(array $refs) {
+    $repository_phids = mpull($refs, 'getRepositoryPHID');
+
+    $repositories = id(new PhabricatorRepositoryQuery())
+      ->setViewer($this->getViewer())
+      ->setParentQuery($this)
+      ->withPHIDs($repository_phids)
+      ->execute();
+    $repositories = mpull($repositories, null, 'getPHID');
+
+    foreach ($refs as $key => $ref) {
+      $repository = idx($repositories, $ref->getRepositoryPHID());
+      if (!$repository) {
+        unset($refs[$key]);
+        continue;
+      }
+      $ref->attachRepository($repository);
+    }
+
+    return $refs;
+  }
+
+  private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
+    $where = array();
+
+    if ($this->repositoryPHIDs) {
+      $where[] = qsprintf(
+        $conn_r,
+        'repositoryPHID IN (%Ls)',
+        $this->repositoryPHIDs);
+    }
+
+    if ($this->refTypes) {
+      $where[] = qsprintf(
+        $conn_r,
+        'refType IN (%Ls)',
+        $this->refTypes);
+    }
+
+    $where[] = $this->buildPagingClause($conn_r);
+
+    return $this->formatWhereClause($where);
+  }
+
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorApplicationDiffusion';
+  }
+
+}

src/applications/repository/storage/PhabricatorRepository.php

@@ -802,6 +802,12 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
         $mirror->delete();
       }
 
+      $ref_cursors = id(new PhabricatorRepositoryRefCursor())
+        ->loadAllWhere('repositoryPHID = %s', $this->getPHID());
+      foreach ($ref_cursors as $cursor) {
+        $cursor->delete();
+      }
+
       $conn_w = $this->establishConnection('w');
 
       queryfx(

src/applications/repository/storage/PhabricatorRepositoryCommit.php

@@ -24,6 +24,8 @@ final class PhabricatorRepositoryCommit
   const IMPORTED_HERALD = 8;
   const IMPORTED_ALL = 15;
 
+  const IMPORTED_CLOSEABLE = 1024;
+
   private $commitData = self::ATTACHABLE;
   private $audits;
   private $repository = self::ATTACHABLE;
@@ -42,7 +44,7 @@ final class PhabricatorRepositoryCommit
   }
 
   public function isImported() {
-    return ($this->getImportStatus() == self::IMPORTED_ALL);
+    return $this->isPartiallyImported(self::IMPORTED_ALL);
   }
 
   public function writeImportStatusFlag($flag) {

src/applications/repository/storage/PhabricatorRepositoryPushLog.php

@@ -77,18 +77,15 @@ final class PhabricatorRepositoryPushLog
   }
 
   public function getRefName() {
-    if ($this->getRefNameEncoding() == 'utf8') {
+    return $this->getUTF8StringFromStorage(
-      return $this->getRefNameRaw();
+      $this->getRefNameRaw(),
-    }
+      $this->getRefNameEncoding());
-    return phutil_utf8ize($this->getRefNameRaw());
   }
 
   public function setRefName($ref_raw) {
-    $encoding = phutil_is_utf8($ref_raw) ? 'utf8' : null;
-
     $this->setRefNameRaw($ref_raw);
     $this->setRefNameHash(PhabricatorHash::digestForIndex($ref_raw));
-    $this->setRefNameEncoding($encoding);
+    $this->setRefNameEncoding($this->detectEncodingForStorage($ref_raw));
 
     return $this;
   }

src/applications/repository/storage/PhabricatorRepositoryRefCursor.php

@@ -0,0 +1,75 @@
+<?php
+
+/**
+ * Stores the previous value of a ref (like a branch or tag) so we can figure
+ * out how a repository has changed when we discover new commits or branch
+ * heads.
+ */
+final class PhabricatorRepositoryRefCursor extends PhabricatorRepositoryDAO
+  implements PhabricatorPolicyInterface {
+
+  const TYPE_BRANCH = 'branch';
+  const TYPE_TAG = 'tag';
+  const TYPE_BOOKMARK = 'bookmark';
+
+  protected $repositoryPHID;
+  protected $refType;
+  protected $refNameHash;
+  protected $refNameRaw;
+  protected $refNameEncoding;
+  protected $commitIdentifier;
+
+  private $repository = self::ATTACHABLE;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+    ) + parent::getConfiguration();
+  }
+
+  public function getRefName() {
+    return $this->getUTF8StringFromStorage(
+      $this->getRefNameRaw(),
+      $this->getRefNameEncoding());
+  }
+
+  public function setRefName($ref_raw) {
+    $this->setRefNameRaw($ref_raw);
+    $this->setRefNameHash(PhabricatorHash::digestForIndex($ref_raw));
+    $this->setRefNameEncoding($this->detectEncodingForStorage($ref_raw));
+
+    return $this;
+  }
+
+  public function attachRepository(PhabricatorRepository $repository) {
+    $this->repository = $repository;
+    return $this;
+  }
+
+  public function getRepository() {
+    return $this->assertAttached($this->repository);
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return $this->getRepository()->getPolicy($capability);
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return $this->getRepository()->hasAutomaticCapability($capability, $viewer);
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return pht('Repository refs have the same policies as their repository.');
+  }
+
+}

src/infrastructure/storage/lisk/PhabricatorLiskDAO.php

@@ -175,4 +175,15 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
     return $value[$key];
   }
 
+  protected function detectEncodingForStorage($string) {
+    return phutil_is_utf8($string) ? 'utf8' : null;
+  }
+
+  protected function getUTF8StringFromStorage($string, $encoding) {
+    if ($encoding == 'utf8') {
+      return $string;
+    }
+    return phutil_utf8ize($string);
+  }
+
 }