Record parent relationships when discovering commits

Summary:
Ref T4455. This adds a `repository_parents` table which stores `<childCommitID, parentCommitID>` relationships.

For new commits, it is populated when commits are discovered.

For older commits, there's a `bin/repository parents` script to rebuild the data.

Right now, there's no UI suggestion that you should run the script. I haven't come up with a super clean way to do this, and this table will only improve performance for now, so it's not important that we get everyone to run the script right away. I'm just leaving it for the moment, and we can figure out how to tell admins to run it later.

The ultimate goal is to solve T2683, but solving T4455 gets us some stuff anyway (for example, we can serve `diffusion.commitparentsquery` faster out of this cache).

Test Plan:
  - Used `bin/repository discover` to discover new commits in Git, SVN and Mercurial repositories.
  - Used `bin/repository parents` to rebuild Git and Mercurial repositories (SVN repos just exit with a message).
  - Verified that the table appears to be sensible.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: jhurwitz, epriestley

Maniphest Tasks: T4455

Differential Revision: https://secure.phabricator.com/D9044

resources/sql/autopatches/20140512.dparents.1.sql

@@ -0,0 +1,7 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_parents (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  childCommitID INT UNSIGNED NOT NULL,
+  parentCommitID INT UNSIGNED NOT NULL,
+  UNIQUE `key_child` (childCommitID, parentCommitID),
+  KEY `key_parent` (parentCommitID)
+) ENGINE=InnoDB, COLLATE utf8_general_ci;

src/__phutil_library_map__.php

@@ -1993,6 +1993,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php',
     'PhabricatorRepositoryManagementMirrorWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMirrorWorkflow.php',
+    'PhabricatorRepositoryManagementParentsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php',
     'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php',
     'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php',
     'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php',
@@ -4835,6 +4836,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementMirrorWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
+    'PhabricatorRepositoryManagementParentsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
     'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow',

src/applications/repository/engine/PhabricatorRepositoryCommitRef.php

@@ -6,6 +6,7 @@ final class PhabricatorRepositoryCommitRef {
   private $epoch;
   private $branch;
   private $canCloseImmediately;
+  private $parents = array();
 
   public function setIdentifier($identifier) {
     $this->identifier = $identifier;
@@ -43,4 +44,13 @@ final class PhabricatorRepositoryCommitRef {
     return $this->canCloseImmediately;
   }
 
+  public function setParents(array $parents) {
+    $this->parents = $parents;
+    return $this;
+  }
+
+  public function getParents() {
+    return $this->parents;
+  }
+
 }

src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php

@@ -61,7 +61,8 @@ final class PhabricatorRepositoryDiscoveryEngine
         $repository,
         $ref->getIdentifier(),
         $ref->getEpoch(),
-        $ref->getCanCloseImmediately());
+        $ref->getCanCloseImmediately(),
+        $ref->getParents());
 
       $this->commitCache[$ref->getIdentifier()] = true;
     }
@@ -436,7 +437,8 @@ final class PhabricatorRepositoryDiscoveryEngine
       $refs[] = id(new PhabricatorRepositoryCommitRef())
         ->setIdentifier($commit)
         ->setEpoch($stream->getCommitDate($commit))
-        ->setCanCloseImmediately($close_immediately);
+        ->setCanCloseImmediately($close_immediately)
+        ->setParents($stream->getParents($commit));
     }
 
     return $refs;
@@ -534,7 +536,8 @@ final class PhabricatorRepositoryDiscoveryEngine
     PhabricatorRepository $repository,
     $commit_identifier,
     $epoch,
-    $close_immediately) {
+    $close_immediately,
+    array $parents) {
 
     $commit = new PhabricatorRepositoryCommit();
     $commit->setRepositoryID($repository->getID());
@@ -546,17 +549,55 @@ final class PhabricatorRepositoryDiscoveryEngine
 
     $data = new PhabricatorRepositoryCommitData();
 
+    $conn_w = $repository->establishConnection('w');
+
     try {
+
+      // If this commit has parents, look up their IDs. The parent commits
+      // should always exist already.
+
+      $parent_ids = array();
+      if ($parents) {
+        $parent_rows = queryfx_all(
+          $conn_w,
+          'SELECT id, commitIdentifier FROM %T
+            WHERE commitIdentifier IN (%Ls) AND repositoryID = %d',
+          $commit->getTableName(),
+          $parents,
+          $repository->getID());
+
+        $parent_map = ipull($parent_rows, 'id', 'commitIdentifier');
+
+        foreach ($parents as $parent) {
+          if (empty($parent_map[$parent])) {
+            throw new Exception(
+              pht('Unable to identify parent "%s"!', $parent));
+          }
+          $parent_ids[] = $parent_map[$parent];
+        }
+      }
+
       $commit->openTransaction();
         $commit->save();
+
         $data->setCommitID($commit->getID());
         $data->save();
+
+        foreach ($parent_ids as $parent_id) {
+          queryfx(
+            $conn_w,
+            'INSERT IGNORE INTO %T (childCommitID, parentCommitID)
+              VALUES (%d, %d)',
+            PhabricatorRepository::TABLE_PARENTS,
+            $commit->getID(),
+            $parent_id);
+        }
       $commit->saveTransaction();
 
       $this->insertTask($repository, $commit);
 
       queryfx(
-        $repository->establishConnection('w'),
+        $conn_w,
         'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
           VALUES (%d, 1, %d, %d)
           ON DUPLICATE KEY UPDATE
@@ -583,6 +624,8 @@ final class PhabricatorRepositoryDiscoveryEngine
             'commit'      => $commit,
           )));
 
+
+
     } catch (AphrontQueryDuplicateKeyException $ex) {
       $commit->killTransaction();
       // Ignore. This can happen because we discover the same new commit

src/applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php

@@ -0,0 +1,149 @@
+<?php
+
+final class PhabricatorRepositoryManagementParentsWorkflow
+  extends PhabricatorRepositoryManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('parents')
+      ->setExamples('**parents** [options] [__repository__] ...')
+      ->setSynopsis(
+        pht(
+          'Build parent caches in repositories that are missing the data, '.
+          'or rebuild them in a specific __repository__.'))
+      ->setArguments(
+        array(
+          array(
+            'name'        => 'repos',
+            'wildcard'    => true,
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $repos = $this->loadRepositories($args, 'repos');
+    if (!$repos) {
+      $repos = id(new PhabricatorRepositoryQuery())
+        ->setViewer($this->getViewer())
+        ->execute();
+    }
+
+    $console = PhutilConsole::getConsole();
+    foreach ($repos as $repo) {
+      $monogram = $repo->getMonogram();
+      if ($repo->isSVN()) {
+        $console->writeOut(
+          "%s\n",
+          pht(
+            'Skipping "%s": Subversion repositories do not require this '.
+            'cache to be built.',
+            $monogram));
+        continue;
+      }
+      $this->rebuildRepository($repo);
+    }
+
+    return 0;
+  }
+
+  private function rebuildRepository(PhabricatorRepository $repo) {
+    $console = PhutilConsole::getConsole();
+    $console->writeOut("%s\n", pht('Rebuilding "%s"...', $repo->getMonogram()));
+
+    $refs = id(new PhabricatorRepositoryRefCursorQuery())
+      ->setViewer($this->getViewer())
+      ->withRefTypes(array(PhabricatorRepositoryRefCursor::TYPE_BRANCH))
+      ->withRepositoryPHIDs(array($repo->getPHID()))
+      ->execute();
+
+    $graph = array();
+    foreach ($refs as $ref) {
+      $console->writeOut(
+        "%s\n",
+        pht('Rebuilding branch "%s"...', $ref->getRefName()));
+
+      $commit = $ref->getCommitIdentifier();
+
+      if ($repo->isGit()) {
+        $stream = new PhabricatorGitGraphStream($repo, $commit);
+      } else {
+        $stream = new PhabricatorMercurialGraphStream($repo, $commit);
+      }
+
+      $discover = array($commit);
+      while ($discover) {
+        $target = array_pop($discover);
+        if (isset($graph[$target])) {
+          continue;
+        }
+        $graph[$target] = $stream->getParents($target);
+        foreach ($graph[$target] as $parent) {
+          $discover[] = $parent;
+        }
+      }
+    }
+
+    $console->writeOut(
+      "%s\n",
+      pht(
+        'Found %s total commit(s); updating...',
+        new PhutilNumber(count($graph))));
+
+    $commit_table = id(new PhabricatorRepositoryCommit());
+    $commit_table_name = $commit_table->getTableName();
+    $conn_w = $commit_table->establishConnection('w');
+
+    $bar = id(new PhutilConsoleProgressBar())
+      ->setTotal(count($graph));
+
+    foreach ($graph as $child => $parents) {
+      $names = $parents;
+      $names[] = $child;
+
+      $rows = queryfx_all(
+        $conn_w,
+        'SELECT id, commitIdentifier FROM %T
+          WHERE commitIdentifier IN (%Ls) AND repositoryID = %d',
+        $commit_table_name,
+        $names,
+        $repo->getID());
+
+      $map = ipull($rows, 'id', 'commitIdentifier');
+      foreach ($names as $name) {
+        if (empty($map[$name])) {
+          throw new Exception(pht('Unknown commit "%s"!', $name));
+        }
+      }
+
+      $sql = array();
+      foreach ($parents as $parent) {
+        $sql[] = qsprintf(
+          $conn_w,
+          '(%d, %d)',
+          $map[$child],
+          $map[$parent]);
+      }
+
+      $commit_table->openTransaction();
+        queryfx(
+          $conn_w,
+          'DELETE FROM %T WHERE childCommitID = %d',
+          PhabricatorRepository::TABLE_PARENTS,
+          $map[$child]);
+
+        if ($sql) {
+          queryfx(
+            $conn_w,
+            'INSERT INTO %T (childCommitID, parentCommitID) VALUES %Q',
+            PhabricatorRepository::TABLE_PARENTS,
+            implode(', ', $sql));
+        }
+      $commit_table->saveTransaction();
+
+      $bar->update(1);
+    }
+
+    $bar->done();
+  }
+
+}

src/applications/repository/storage/PhabricatorRepository.php

@@ -25,6 +25,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
   const TABLE_SUMMARY = 'repository_summary';
   const TABLE_BADCOMMIT = 'repository_badcommit';
   const TABLE_LINTMESSAGE = 'repository_lintmessage';
+  const TABLE_PARENTS = 'repository_parents';
 
   const SERVE_OFF = 'off';
   const SERVE_READONLY = 'readonly';