Add retry loop when trying to establish db connection, log retries

Summary:
We retried if a db connection was lost when executing a query, but not when
establishing a connection. I've seen a lot of failures establishing connections
in our install (they go away when retrying), so this diff retries when
establishing connections, and logs when we retry.

Test Plan:
- Loaded phabricator in a sandbox
- Temporarily added a check in the try block to throw if there were still
  retries (to test logging, retry logic)

Reviewers: epriestley, blair

Reviewed By: epriestley

CC: aran, btrahan

Differential Revision: https://secure.phabricator.com/D1460

conf/default.conf.php

@@ -117,6 +117,9 @@ return array(
   // (e.g., db.example.com:1234).
   'mysql.host' => 'localhost',
 
+  // The number of times to try reconnecting to the MySQL database
+  'mysql.connection-retries' => 3,
+
 
 // -- Email ----------------------------------------------------------------- //
 

src/storage/connection/mysql/AphrontMySQLDatabaseConnection.php

@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -137,33 +137,43 @@ class AphrontMySQLDatabaseConnection extends AphrontDatabaseConnection {
         'database'  => $database,
       ));
 
-    try {
+    $retries = max(1, PhabricatorEnv::getEnvConfig('mysql.connection-retries'));
-      $conn = @mysql_connect(
+    while ($retries--) {
-        $host,
+      try {
-        $user,
+        $conn = @mysql_connect(
-        $this->getConfiguration('pass'),
+          $host,
-        $new_link = true,
+          $user,
-        $flags = 0);
+          $this->getConfiguration('pass'),
+          $new_link = true,
+          $flags = 0);
 
-      if (!$conn) {
+        if (!$conn) {
-        $errno = mysql_errno();
+          $errno = mysql_errno();
-        $error = mysql_error();
+          $error = mysql_error();
-        throw new AphrontQueryConnectionException(
+          throw new AphrontQueryConnectionException(
-          "Attempt to connect to {$user}@{$host} failed with error #{$errno}: ".
+            "Attempt to connect to {$user}@{$host} failed with error ".
-          "{$error}.");
+            "#{$errno}: {$error}.", $errno);
-      }
+        }
 
-      if ($database !== null) {
+        if ($database !== null) {
-        $ret = @mysql_select_db($database, $conn);
+          $ret = @mysql_select_db($database, $conn);
-        if (!$ret) {
+          if (!$ret) {
-          $this->throwQueryException($conn);
+            $this->throwQueryException($conn);
+          }
+        }
+
+        $profiler->endServiceCall($call_id, array());
+        break;
+      } catch (Exception $ex) {
+        if ($retries && $ex->getCode() == 2003) {
+          $class = get_class($ex);
+          $message = $ex->getMessage();
+          phlog("Retrying ({$retries}) after {$class}: {$message}");
+        } else {
+          $profiler->endServiceCall($call_id, array());
+          throw $ex;
         }
       }
-
-      $profiler->endServiceCall($call_id, array());
-    } catch (Exception $ex) {
-      $profiler->endServiceCall($call_id, array());
-      throw $ex;
     }
 
     self::$connectionCache[$key] = $conn;
@@ -203,7 +213,7 @@ class AphrontMySQLDatabaseConnection extends AphrontDatabaseConnection {
 
   public function executeRawQuery($raw_query) {
     $this->lastResult = null;
-    $retries = 3;
+    $retries = max(1, PhabricatorEnv::getEnvConfig('mysql.connection-retries'));
     while ($retries--) {
       try {
         $this->requireConnection();
@@ -242,6 +252,9 @@ class AphrontMySQLDatabaseConnection extends AphrontDatabaseConnection {
         if ($this->isInsideTransaction()) {
           throw $ex;
         }
+        $class = get_class($ex);
+        $message = $ex->getMessage();
+        phlog("Retrying ({$retries}) after {$class}: {$message}");
         $this->closeConnection();
       }
     }

src/storage/connection/mysql/__init__.php

@@ -7,6 +7,7 @@
 
 
 phutil_require_module('phabricator', 'aphront/writeguard');
+phutil_require_module('phabricator', 'infrastructure/env');
 phutil_require_module('phabricator', 'storage/connection/base');
 phutil_require_module('phabricator', 'storage/exception/accessdenied');
 phutil_require_module('phabricator', 'storage/exception/base');
@@ -15,6 +16,7 @@ phutil_require_module('phabricator', 'storage/exception/connectionlost');
 phutil_require_module('phabricator', 'storage/exception/duplicatekey');
 phutil_require_module('phabricator', 'storage/exception/recoverable');
 
+phutil_require_module('phutil', 'error');
 phutil_require_module('phutil', 'serviceprofiler');
 phutil_require_module('phutil', 'utils');