From 6d74736a7ec8d5bda259234a4d2d70331a0331e5 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 2 Oct 2019 13:09:52 -0700
Subject: [PATCH] When a large number of commits need to be marked as
 published, issue the lookup query in chunks

Summary: See PHI1474. This query can become large enough to exceed reasonable packet limits. Chunk the query so it is split up if we have too many identifiers.

Test Plan: Ran `bin/repository refs ...` on a repository with no new commits and a repository with some new commits.

Differential Revision: https://secure.phabricator.com/D20853
---
 .../engine/PhabricatorRepositoryRefEngine.php | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
index f823ead6d9..8d8e7f45d8 100644
--- a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
+++ b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
@@ -498,7 +498,7 @@ final class PhabricatorRepositoryRefEngine
   private function setCloseFlagOnCommits(array $identifiers) {
     $repository = $this->getRepository();
     $commit_table = new PhabricatorRepositoryCommit();
-    $conn_w = $commit_table->establishConnection('w');
+    $conn = $commit_table->establishConnection('w');
 
     $vcs = $repository->getVersionControlSystem();
     switch ($vcs) {
@@ -515,13 +515,27 @@ final class PhabricatorRepositoryRefEngine
         throw new Exception(pht("Unknown repository type '%s'!", $vcs));
     }
 
-    $all_commits = queryfx_all(
-      $conn_w,
-      'SELECT id, phid, commitIdentifier, importStatus FROM %T
-        WHERE repositoryID = %d AND commitIdentifier IN (%Ls)',
-      $commit_table->getTableName(),
-      $repository->getID(),
-      $identifiers);
+    $identifier_tokens = array();
+    foreach ($identifiers as $identifier) {
+      $identifier_tokens[] = qsprintf(
+        $conn,
+        '%s',
+        $identifier);
+    }
+
+    $all_commits = array();
+    foreach (PhabricatorLiskDAO::chunkSQL($identifier_tokens) as $chunk) {
+      $rows = queryfx_all(
+        $conn,
+        'SELECT id, phid, commitIdentifier, importStatus FROM %T
+          WHERE repositoryID = %d AND commitIdentifier IN (%LQ)',
+        $commit_table->getTableName(),
+        $repository->getID(),
+        $chunk);
+      foreach ($rows as $row) {
+        $all_commits[] = $row;
+      }
+    }
 
     $closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE;
 
@@ -539,7 +553,7 @@ final class PhabricatorRepositoryRefEngine
 
       if (!($row['importStatus'] & $closeable_flag)) {
         queryfx(
-          $conn_w,
+          $conn,
           'UPDATE %T SET importStatus = (importStatus | %d) WHERE id = %d',
           $commit_table->getTableName(),
           $closeable_flag,