When waiting for long-running Harbormaster futures to resolve, close idle database connections

Summary:
Ref T13216. See PHI916. Harbormaster builds may be long-running, particularly if they effectively wrap `ssh ... ./run-huge-build.sh`. If we spend more than a few seconds waiting for futures to resolve, close idle database connections.

The general goal here is to reduce the held connection load for installs with a very large number of test runners.

Test Plan: Added debugging code to `phlog()` closures, saw connections closed while running builds.

Reviewers: amckinley

Reviewed By: amckinley

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13216

Differential Revision: https://secure.phabricator.com/D19824

src/applications/feed/query/PhabricatorFeedQuery.php

@@ -78,8 +78,8 @@ final class PhabricatorFeedQuery
 
       $where[] = qsprintf(
         $conn,
-        'ref.chronologicalKey IN (%Q)',
-        implode(', ', $keys));
+        'ref.chronologicalKey IN (%LQ)',
+        $keys);
     }
 
     // NOTE: We may not have 64-bit PHP, so do the shifts in MySQL instead.

src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php

@@ -253,12 +253,32 @@ abstract class HarbormasterBuildStepImplementation extends Phobject {
     HarbormasterBuildTarget $target,
     array $futures) {
 
+    $did_close = false;
+    $wait_start = PhabricatorTime::getNow();
+
     $futures = new FutureIterator($futures);
     foreach ($futures->setUpdateInterval(5) as $key => $future) {
-      if ($future === null) {
-        $build->reload();
-        if ($this->shouldAbort($build, $target)) {
-          throw new HarbormasterBuildAbortedException();
+      if ($future !== null) {
+        continue;
+      }
+
+      $build->reload();
+      if ($this->shouldAbort($build, $target)) {
+        throw new HarbormasterBuildAbortedException();
+      }
+
+      // See PHI916. If we're waiting on a remote system for a while, clean
+      // up database connections to reduce the cost of having a large number
+      // of processes babysitting an `ssh ... ./run-huge-build.sh` process on
+      // a build host.
+      if (!$did_close) {
+        $now = PhabricatorTime::getNow();
+        $elapsed = ($now - $wait_start);
+        $idle_limit = 5;
+
+        if ($elapsed >= $idle_limit) {
+          LiskDAO::closeIdleConnections();
+          $did_close = true;
         }
       }
     }

src/applications/search/index/PhabricatorIndexEngine.php

@@ -141,10 +141,10 @@ final class PhabricatorIndexEngine extends Phobject {
     queryfx(
       $conn_w,
       'INSERT INTO %T (objectPHID, extensionKey, version)
-        VALUES %Q
+        VALUES %LQ
         ON DUPLICATE KEY UPDATE version = VALUES(version)',
       $table->getTableName(),
-      implode(', ', $sql));
+      $sql);
   }
 
 }

src/applications/search/ngrams/PhabricatorSearchNgrams.php

@@ -102,9 +102,9 @@ abstract class PhabricatorSearchNgrams
     if ($sql) {
       queryfx(
         $conn_w,
-        'INSERT INTO %T (objectID, ngram) VALUES %Q',
+        'INSERT INTO %T (objectID, ngram) VALUES %LQ',
         $this->getTableName(),
-        implode(', ', $sql));
+        $sql);
     }
 
     return $this;

src/infrastructure/storage/lisk/LiskDAO.php

@@ -1652,6 +1652,11 @@ abstract class LiskDAO extends Phobject
 
     $now = PhabricatorTime::getNow();
     foreach ($connections as $key => $connection) {
+      // If the connection is not idle, never consider it inactive.
+      if (!$connection->isIdle()) {
+        continue;
+      }
+
       $last_active = $connection->getLastActiveEpoch();
 
       $idle_duration = ($now - $last_active);
@@ -1672,6 +1677,18 @@ abstract class LiskDAO extends Phobject
     }
   }
 
+  public static function closeIdleConnections() {
+    $connections = self::$connections;
+
+    foreach ($connections as $key => $connection) {
+      if (!$connection->isIdle()) {
+        continue;
+      }
+
+      self::closeConnection($key);
+    }
+  }
+
   private static function closeConnection($key) {
     if (empty(self::$connections[$key])) {
       throw new Exception(