Provide a fast path for resolving repository refs

Summary:
Ref T2783. With service-oriented calls, we take a larger performacne hit than necessary resolving refs.

Instead of resolving refs over the wire, try to resolve them from the database first. This can resolve almost all refs (commit hashes, branch and tag names).

This can't resolve weird refs like `master~50`, and obviously can't resolve invalid refs. In those cases we'll go back to the old logic, call `diffusion.resolverefs`, and end up with the right result.

Test Plan:
  - Browsed repositories in Diffusion.
  - Verified that service repositories no longer make unnecessary `diffusion.resolverefs` calls for common refs (branch names, commit hashes).
  - Resolved refs like `master~50`, saw call to underlying VCS and correct result.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T2783

Differential Revision: https://secure.phabricator.com/D11476

src/__phutil_library_map__.php

@@ -464,6 +464,7 @@ phutil_register_library_map(array(
     'DiffusionBrowseResultSet' => 'applications/diffusion/data/DiffusionBrowseResultSet.php',
     'DiffusionBrowseSearchController' => 'applications/diffusion/controller/DiffusionBrowseSearchController.php',
     'DiffusionBrowseTableView' => 'applications/diffusion/view/DiffusionBrowseTableView.php',
+    'DiffusionCachedResolveRefsQuery' => 'applications/diffusion/query/DiffusionCachedResolveRefsQuery.php',
     'DiffusionChangeController' => 'applications/diffusion/controller/DiffusionChangeController.php',
     'DiffusionCommitBranchesController' => 'applications/diffusion/controller/DiffusionCommitBranchesController.php',
     'DiffusionCommitChangeTableView' => 'applications/diffusion/view/DiffusionCommitChangeTableView.php',
@@ -3560,6 +3561,7 @@ phutil_register_library_map(array(
     'DiffusionBrowseQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
     'DiffusionBrowseSearchController' => 'DiffusionBrowseController',
     'DiffusionBrowseTableView' => 'DiffusionView',
+    'DiffusionCachedResolveRefsQuery' => 'DiffusionLowLevelQuery',
     'DiffusionChangeController' => 'DiffusionController',
     'DiffusionCommitBranchesController' => 'DiffusionController',
     'DiffusionCommitChangeTableView' => 'DiffusionView',

src/applications/diffusion/query/DiffusionCachedResolveRefsQuery.php

@@ -0,0 +1,174 @@
+<?php
+
+/**
+ * Resolves references into canonical, stable commit identifiers by examining
+ * database caches.
+ *
+ * This is a counterpart to @{class:DiffusionLowLevelResolveRefsQuery}. This
+ * query offers fast resolution, but can not resolve everything that the
+ * low-level query can.
+ *
+ * This class can resolve the most common refs (commits, branches, tags) and
+ * can do so cheapy (by examining the database, without needing to make calls
+ * to the VCS or the service host).
+ */
+final class DiffusionCachedResolveRefsQuery
+  extends DiffusionLowLevelQuery {
+
+  private $refs;
+
+  public function withRefs(array $refs) {
+    $this->refs = $refs;
+    return $this;
+  }
+
+  protected function executeQuery() {
+    if (!$this->refs) {
+      return array();
+    }
+
+    switch ($this->getRepository()->getVersionControlSystem()) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $result = $this->resolveGitAndMercurialRefs();
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+        $result = $this->resolveSubversionRefs();
+        break;
+      default:
+        throw new Exception('Unsupported repository type!');
+    }
+
+    return $result;
+  }
+
+  /**
+   * Resolve refs in Git and Mercurial repositories.
+   *
+   * We can resolve commit hashes from the commits table, and branch and tag
+   * names from the refcursor table.
+   */
+  private function resolveGitAndMercurialRefs() {
+    $repository = $this->getRepository();
+
+    $conn_r = $repository->establishConnection('r');
+
+    $results = array();
+
+    $prefixes = array();
+    foreach ($this->refs as $ref) {
+      // We require refs to look like hashes and be at least 4 characters
+      // long. This is similar to the behavior of git.
+      if (preg_match('/^[a-f0-9]{4,}$/', $ref)) {
+        $prefixes[] = qsprintf(
+          $conn_r,
+          '(commitIdentifier LIKE %>)',
+          $ref);
+      }
+    }
+
+    if ($prefixes) {
+      $commits = queryfx_all(
+        $conn_r,
+        'SELECT commitIdentifier FROM %T
+          WHERE repositoryID = %s AND %Q',
+        id(new PhabricatorRepositoryCommit())->getTableName(),
+        $repository->getID(),
+        implode(' OR ', $prefixes));
+
+      foreach ($commits as $commit) {
+        $hash = $commit['commitIdentifier'];
+        foreach ($this->refs as $ref) {
+          if (!strncmp($hash, $ref, strlen($ref))) {
+            $results[$ref][] = array(
+              'type' => 'commit',
+              'identifier' => $hash,
+            );
+          }
+        }
+      }
+    }
+
+    $name_hashes = array();
+    foreach ($this->refs as $ref) {
+      $name_hashes[PhabricatorHash::digestForIndex($ref)] = $ref;
+    }
+
+    $cursors = queryfx_all(
+      $conn_r,
+      'SELECT refNameHash, refType, commitIdentifier FROM %T
+        WHERE repositoryPHID = %s AND refNameHash IN (%Ls)',
+      id(new PhabricatorRepositoryRefCursor())->getTableName(),
+      $repository->getPHID(),
+      array_keys($name_hashes));
+
+    foreach ($cursors as $cursor) {
+      if (isset($name_hashes[$cursor['refNameHash']])) {
+        $results[$name_hashes[$cursor['refNameHash']]][] = array(
+          'type' => $cursor['refType'],
+          'identifier' => $cursor['commitIdentifier'],
+        );
+
+        // TODO: In Git, we don't store (and thus don't return) the hash
+        // of the tag itself. It would be vaguely nice to do this.
+      }
+    }
+
+    return $results;
+  }
+
+
+  /**
+   * Resolve refs in Subversion repositories.
+   *
+   * We can resolve all numeric identifiers and the keyword `HEAD`.
+   */
+  private function resolveSubversionRefs() {
+    $repository = $this->getRepository();
+
+    $max_commit = id(new PhabricatorRepositoryCommit())
+      ->loadOneWhere(
+        'repositoryID = %d ORDER BY epoch DESC, id DESC LIMIT 1',
+        $repository->getID());
+    if (!$max_commit) {
+      // This repository is empty or hasn't parsed yet, so none of the refs are
+      // going to resolve.
+      return array();
+    }
+
+    $max_commit_id = (int)$max_commit->getCommitIdentifier();
+
+    $results = array();
+    foreach ($this->refs as $ref) {
+      if ($ref == 'HEAD') {
+        // Resolve "HEAD" to mean "the most recent commit".
+        $results[$ref][] = array(
+          'type' => 'commit',
+          'identifier' => $max_commit_id,
+        );
+        continue;
+      }
+
+      if (!preg_match('/^\d+$/', $ref)) {
+        // This ref is non-numeric, so it doesn't resolve to anything.
+        continue;
+      }
+
+      // Resolve other commits if we can deduce their existence.
+
+      // TODO: When we import only part of a repository, we won't necessarily
+      // have all of the smaller commits. Should we fail to resolve them here
+      // for repositories with a subpath? It might let us simplify other things
+      // elsewhere.
+      if ((int)$ref <= $max_commit_id) {
+        $results[$ref][] = array(
+          'type' => 'commit',
+          'identifier' => (int)$ref,
+        );
+      }
+    }
+
+    return $results;
+  }
+
+}

src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php

@@ -4,6 +4,11 @@
  * Resolves references (like short commit names, branch names, tag names, etc.)
  * into canonical, stable commit identifiers. This query works for all
  * repository types.
+ *
+ * This query will always resolve refs which can be resolved, but may need to
+ * perform VCS operations. A faster (but less complete) counterpart query is
+ * available in @{class:DiffusionCachedResolveRefsQuery}; that query can
+ * resolve most refs without VCS operations.
  */
 final class DiffusionLowLevelResolveRefsQuery
   extends DiffusionLowLevelQuery {
@@ -169,51 +174,12 @@ final class DiffusionLowLevelResolveRefsQuery
   }
 
   private function resolveSubversionRefs() {
-    $repository = $this->getRepository();
-
-    $max_commit = id(new PhabricatorRepositoryCommit())
-      ->loadOneWhere(
-        'repositoryID = %d ORDER BY epoch DESC, id DESC LIMIT 1',
-        $repository->getID());
-    if (!$max_commit) {
-      // This repository is empty or hasn't parsed yet, so none of the refs are
-      // going to resolve.
-      return array();
-    }
-
-    $max_commit_id = (int)$max_commit->getCommitIdentifier();
-
-    $results = array();
-    foreach ($this->refs as $ref) {
-      if ($ref == 'HEAD') {
-        // Resolve "HEAD" to mean "the most recent commit".
-        $results[$ref][] = array(
-          'type' => 'commit',
-          'identifier' => $max_commit_id,
-        );
-        continue;
-      }
-
-      if (!preg_match('/^\d+$/', $ref)) {
-        // This ref is non-numeric, so it doesn't resolve to anything.
-        continue;
-      }
-
-      // Resolve other commits if we can deduce their existence.
-
-      // TODO: When we import only part of a repository, we won't necessarily
-      // have all of the smaller commits. Should we fail to resolve them here
-      // for repositories with a subpath? It might let us simplify other things
-      // elsewhere.
-      if ((int)$ref <= $max_commit_id) {
-        $results[$ref][] = array(
-          'type' => 'commit',
-          'identifier' => (int)$ref,
-        );
-      }
-    }
-
-    return $results;
+    // We don't have any VCS logic for Subversion, so just use the cached
+    // query.
+    return id(new DiffusionCachedResolveRefsQuery())
+      ->setRepository($this->getRepository())
+      ->withRefs($this->refs)
+      ->execute();
   }
 
 }

src/applications/diffusion/request/DiffusionRequest.php

@@ -750,20 +750,42 @@ abstract class DiffusionRequest {
   }
 
   private function resolveRefs(array $refs) {
-    if ($this->shouldInitFromConduit()) {
-      return DiffusionQuery::callConduitWithDiffusionRequest(
-        $this->getUser(),
-        $this,
-        'diffusion.resolverefs',
-        array(
-          'refs' => $refs,
-        ));
-    } else {
-      return id(new DiffusionLowLevelResolveRefsQuery())
-        ->setRepository($this->getRepository())
-        ->withRefs($refs)
-        ->execute();
+    // First, try to resolve refs from fast cache sources.
+    $cached_results = id(new DiffusionCachedResolveRefsQuery())
+      ->setRepository($this->getRepository())
+      ->withRefs($refs)
+      ->execute();
+
+    // Throw away all the refs we resolved. Hopefully, we'll throw away
+    // everything here.
+    foreach ($refs as $key => $ref) {
+      if (isset($cached_results[$ref])) {
+        unset($refs[$key]);
+      }
     }
+
+    // If we couldn't pull everything out of the cache, execute the underlying
+    // VCS operation.
+    if ($refs) {
+      if ($this->shouldInitFromConduit()) {
+        $vcs_results = DiffusionQuery::callConduitWithDiffusionRequest(
+          $this->getUser(),
+          $this,
+          'diffusion.resolverefs',
+          array(
+            'refs' => $refs,
+          ));
+      } else {
+        $vcs_results = id(new DiffusionLowLevelResolveRefsQuery())
+          ->setRepository($this->getRepository())
+          ->withRefs($refs)
+          ->execute();
+      }
+    } else {
+      $vcs_results = array();
+    }
+
+    return $vcs_results + $cached_results;
   }
 
   public function setIsClusterRequest($is_cluster_request) {