Begin making change parsers testable

Summary:
Ref T4327. There are a bunch of other probably-related tasks too, some linked there.

We have some rare/unusual bugs in the change parsers, mostly in Subversion, but it's terrifying to touch them because they're complicated and fragile and have no test coverage.

To fix this stuff, I want to make them more testable. In particular, they basically end with this big INSERT right now. Instead, I'm going to make them return objects representing the data to be inserted, then have the common infrastructure do the insert. This gives us two benefits:

  - Reduced code duplication on the insert;
  - we can stop before the insert and have unit tests examine the objects.

This swaps the Git parser over, but doesn't swap the hg/svn parsers yet. I'll do those separately, the SVN one looks a bit tricky.

Test Plan:
  - Used `scripts/repository/reparse.php` to reparse a Git commit, with `--trace`. Verified it looked the same as before and the SQL that was executed seemed reasonable.
  - Did the same for `hg` / `svn` commits, to make sure I didn't derp anything. These aren't expected to do anything differently.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T4327

Differential Revision: https://secure.phabricator.com/D7980

src/__phutil_library_map__.php

@@ -1866,6 +1866,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPHIDTypeMirror' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeMirror.php',
     'PhabricatorRepositoryPHIDTypePushLog' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypePushLog.php',
     'PhabricatorRepositoryPHIDTypeRepository' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeRepository.php',
+    'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
     'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
     'PhabricatorRepositoryPushLog' => 'applications/repository/storage/PhabricatorRepositoryPushLog.php',
@@ -4542,6 +4543,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPHIDTypeMirror' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypePushLog' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPHIDTypeRepository' => 'PhabricatorPHIDType',
+    'PhabricatorRepositoryParsedChange' => 'Phobject',
     'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryPushLog' =>

src/applications/repository/data/PhabricatorRepositoryParsedChange.php

@@ -0,0 +1,77 @@
+<?php
+
+final class PhabricatorRepositoryParsedChange extends Phobject {
+
+  private $pathID;
+  private $targetPathID;
+  private $targetCommitID;
+  private $changeType;
+  private $fileType;
+  private $isDirect;
+  private $commitSequence;
+
+  public function setPathID($path_id) {
+    $this->pathID = $path_id;
+    return $this;
+  }
+
+  public function getPathID() {
+    return $this->pathID;
+  }
+
+  public function setCommitSequence($commit_sequence) {
+    $this->commitSequence = $commit_sequence;
+    return $this;
+  }
+
+  public function getCommitSequence() {
+    return $this->commitSequence;
+  }
+
+  public function setIsDirect($is_direct) {
+    $this->isDirect = $is_direct;
+    return $this;
+  }
+
+  public function getIsDirect() {
+    return $this->isDirect;
+  }
+
+  public function setFileType($file_type) {
+    $this->fileType = $file_type;
+    return $this;
+  }
+
+  public function getFileType() {
+    return $this->fileType;
+  }
+
+  public function setChangeType($change_type) {
+    $this->changeType = $change_type;
+    return $this;
+  }
+
+  public function getChangeType() {
+    return $this->changeType;
+  }
+
+  public function setTargetCommitID($target_commit_id) {
+    $this->targetCommitID = $target_commit_id;
+    return $this;
+  }
+
+  public function getTargetCommitID() {
+    return $this->targetCommitID;
+  }
+
+
+  public function setTargetPathID($target_path_id) {
+    $this->targetPathID = $target_path_id;
+    return $this;
+  }
+
+  public function getTargetPathID() {
+    return $this->targetPathID;
+  }
+
+}

src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php

@@ -40,8 +40,7 @@ abstract class PhabricatorRepositoryCommitParserWorker
     }
 
     $this->repository = $repository;
-
+    $this->parseCommit($repository, $this->commit);
-    return $this->parseCommit($repository, $this->commit);
   }
 
   final protected function shouldQueueFollowupTasks() {

src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php

@@ -24,14 +24,15 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
     $this->log("Parsing %s...\n", $full_name);
     if ($this->isBadCommit($full_name)) {
       $this->log("This commit is marked bad!");
-      $result = null;
+      return;
-    } else {
+    }
-      $result = $this->parseCommitChanges($repository, $commit);
+
+    $results = $this->parseCommitChanges($repository, $commit);
+    if ($results) {
+      $this->writeCommitChanges($repository, $commit, $results);
     }
 
     $this->finishParse();
-
-    return $result;
   }
 
   public static function lookupOrCreatePaths(array $paths) {
@@ -99,4 +100,52 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
     }
   }
 
+  private function writeCommitChanges(
+    PhabricatorRepository $repository,
+    PhabricatorRepositoryCommit $commit,
+    array $changes) {
+
+    $repository_id = (int)$repository->getID();
+    $commit_id = (int)$commit->getID();
+
+    // NOTE: This SQL is being built manually instead of with qsprintf()
+    // because some SVN changes affect an enormous number of paths (millions)
+    // and this showed up as significantly slow on a profile at some point.
+
+    $changes_sql = array();
+    foreach ($changes as $change) {
+      $values = array(
+        $repository_id,
+        (int)$change->getPathID(),
+        $commit_id,
+        nonempty((int)$change->getTargetPathID(), 'null'),
+        nonempty((int)$change->getTargetCommitID(), 'null'),
+        (int)$change->getChangeType(),
+        (int)$change->getFileType(),
+        (int)$change->getIsDirect(),
+        (int)$change->getCommitSequence(),
+      );
+      $changes_sql[] = '('.implode(', ', $values).')';
+    }
+
+    $conn_w = $repository->establishConnection('w');
+
+    queryfx(
+      $conn_w,
+      'DELETE FROM %T WHERE commitID = %d',
+      PhabricatorRepository::TABLE_PATHCHANGE,
+      $commit_id);
+
+    foreach (PhabricatorLiskDAO::chunkSQL($changes_sql) as $chunk) {
+      queryfx(
+        $conn_w,
+        'INSERT INTO %T
+          (repositoryID, pathID, commitID, targetPathID, targetCommitID,
+            changeType, fileType, isDirect, commitSequence)
+          VALUES %Q',
+        PhabricatorRepository::TABLE_PATHCHANGE,
+        $chunk);
+    }
+  }
+
 }

src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php

@@ -228,43 +228,30 @@ final class PhabricatorRepositoryGitCommitChangeParserWorker
       }
     }
 
-    $conn_w = $repository->establishConnection('w');
 
-    $changes_sql = array();
+    $results = array();
     foreach ($changes as $change) {
-      $values = array(
+      $target_path_id = $change['targetPathID']
-        (int)$change['repositoryID'],
+        ? (int)$change['targetPathID']
-        (int)$change['pathID'],
+        : null;
-        (int)$change['commitID'],
+
-        $change['targetPathID']
+      $target_commit_id = $change['targetCommitID']
-          ? (int)$change['targetPathID']
+        ? (int)$change['targetCommitID']
-          : 'null',
+        : null;
-        $change['targetCommitID']
+
-          ? (int)$change['targetCommitID']
+      $result = id(new PhabricatorRepositoryParsedChange())
-          : 'null',
+        ->setPathID((int)$change['pathID'])
-        (int)$change['changeType'],
+        ->setTargetPathID($target_path_id)
-        (int)$change['fileType'],
+        ->setTargetCommitID($target_commit_id)
-        (int)$change['isDirect'],
+        ->setChangeType((int)$change['changeType'])
-        (int)$change['commitSequence'],
+        ->setFileType($change['fileType'])
-      );
+        ->setIsDirect((int)$change['isDirect'])
-      $changes_sql[] = '('.implode(', ', $values).')';
+        ->setCommitSequence((int)$change['commitSequence']);
+
+      $results[] = $result;
     }
 
-    queryfx(
+    return $results;
-      $conn_w,
-      'DELETE FROM %T WHERE commitID = %d',
-      PhabricatorRepository::TABLE_PATHCHANGE,
-      $commit->getID());
-    foreach (array_chunk($changes_sql, 256) as $sql_chunk) {
-      queryfx(
-        $conn_w,
-        'INSERT INTO %T
-          (repositoryID, pathID, commitID, targetPathID, targetCommitID,
-            changeType, fileType, isDirect, commitSequence)
-          VALUES %Q',
-        PhabricatorRepository::TABLE_PATHCHANGE,
-        implode(', ', $sql_chunk));
-    }
   }
 
 }

src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php

@@ -300,6 +300,8 @@ final class PhabricatorRepositoryMercurialCommitChangeParserWorker
         PhabricatorRepository::TABLE_PATHCHANGE,
         implode(', ', $sql_chunk));
     }
+
+    return array();
   }
 
   private function mercurialPathExists(

src/applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php

@@ -359,6 +359,8 @@ final class PhabricatorRepositorySvnCommitChangeParserWorker
 
     $this->writeChanges($repository, $commit, $effects, $path_map, $commit_map);
     $this->writeBrowse($repository, $commit, $effects, $path_map);
+
+    return array();
   }
 
   private function writeChanges(