Implement a MySQL-backed global lock

Summary: Implementation is a little crazy but this seems to work as advertised.

Test Plan: Acquired locks with "lock.php". Verified they held as long as the process reamined open and released properly on kill -9, ^C, etc.

Reviewers: nh, jungejason, vrana, btrahan, Girish, edward

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T1400

Differential Revision: https://secure.phabricator.com/D2864

src/__phutil_library_map__.php

@@ -687,6 +687,7 @@ phutil_register_library_map(array(
     'PhabricatorFormExample' => 'applications/uiexample/examples/PhabricatorFormExample.php',
     'PhabricatorGarbageCollectorDaemon' => 'infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php',
     'PhabricatorGitGraphStream' => 'applications/repository/daemon/PhabricatorGitGraphStream.php',
+    'PhabricatorGlobalLock' => 'infrastructure/util/PhabricatorGlobalLock.php',
     'PhabricatorGoodForNothingWorker' => 'infrastructure/daemon/workers/worker/PhabricatorGoodForNothingWorker.php',
     'PhabricatorHandleObjectSelectorDataView' => 'applications/phid/handle/view/PhabricatorHandleObjectSelectorDataView.php',
     'PhabricatorHash' => 'infrastructure/util/PhabricatorHash.php',
@@ -1685,6 +1686,7 @@ phutil_register_library_map(array(
     'PhabricatorFlagListView' => 'AphrontView',
     'PhabricatorFormExample' => 'PhabricatorUIExample',
     'PhabricatorGarbageCollectorDaemon' => 'PhabricatorDaemon',
+    'PhabricatorGlobalLock' => 'PhutilLock',
     'PhabricatorGoodForNothingWorker' => 'PhabricatorWorker',
     'PhabricatorHelpController' => 'PhabricatorController',
     'PhabricatorHelpKeyboardShortcutController' => 'PhabricatorHelpController',

src/infrastructure/util/PhabricatorGlobalLock.php

@@ -0,0 +1,115 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Global, MySQL-backed lock. This is a high-reliability, low-performance
+ * global lock.
+ *
+ * The lock is maintained by using GET_LOCK() in MySQL, and automatically
+ * released when the connection terminates. Thus, this lock can safely be used
+ * to control access to shared resources without implementing any sort of
+ * timeout or override logic: the lock can't normally be stuck in a locked state
+ * with no process actually holding the lock.
+ *
+ * However, acquiring the lock is moderately expensive (several network
+ * roundtrips). This makes it unsuitable for tasks where lock performance is
+ * important.
+ *
+ *    $lock = PhabricatorGlobalLock::newLock('example');
+ *    $lock->lock();
+ *      do_contentious_things();
+ *    $lock->unlock();
+ *
+ * @task construct  Constructing Locks
+ * @task impl       Implementation
+ */
+final class PhabricatorGlobalLock extends PhutilLock {
+
+  private $lockname;
+  private $conn;
+
+
+/* -(  Constructing Locks  )------------------------------------------------- */
+
+
+  public static function newLock($name) {
+    $full_name = 'global:'.$name;
+
+    $lock = self::getLock($full_name);
+    if (!$lock) {
+      $lock = new PhabricatorGlobalLock($full_name);
+      $lock->lockname = $name;
+      self::registerLock($lock);
+    }
+
+    return $lock;
+  }
+
+
+/* -(  Implementation  )----------------------------------------------------- */
+
+  protected function doLock() {
+    $conn = $this->conn;
+    if (!$conn) {
+      // NOTE: Using the 'repository' database somewhat arbitrarily, mostly
+      // because the first client of locks is the repository daemons. We must
+      // always use the same database for all locks, but don't access any
+      // tables so we could use any valid database. We could build a
+      // database-free connection instead, but that's kind of messy and we
+      // might forget about it in the future if we vertically partition the
+      // application.
+      $dao = new PhabricatorRepository();
+
+      // NOTE: Using "force_new" to make sure each lock is on its own
+      // connection.
+      $conn = $dao->establishConnection('w', $force_new = true);
+
+      // NOTE: Since MySQL will disconnect us if we're idle for too long, we set
+      // the wait_timeout to an enormous value, to allow us to hold the
+      // connection open indefinitely (or, at least, for a year).
+      queryfx($conn, 'SET wait_timeout = %d', 365 * 24 * 60 * 60);
+    }
+
+    $result = queryfx_one(
+      $conn,
+      'SELECT GET_LOCK(%s, %d)',
+      'phabricator:'.$this->lockname,
+      0);
+
+    $ok = head($result);
+    if (!$ok) {
+      throw new PhutilLockException($this->getName());
+    }
+
+    $this->conn = $conn;
+  }
+
+  protected function doUnlock() {
+    queryfx(
+      $this->conn,
+      'SELECT RELEASE_LOCK(%s)',
+      'phabricator:'.$this->lockname);
+
+    // TODO: There's no explicit close() method on connections right now. Once
+    // we have one, we could close the connection here. Since we don't have
+    // such a method, we need to keep the connection around in case lock() is
+    // called again, so that long-running daemons don't gradually open
+    // an unbounded number of connections.
+  }
+
+}

src/storage/lisk/LiskDAO.php

@@ -957,11 +957,14 @@ abstract class LiskDAO {
   /**
    * Get or build the database connection for this object.
    *
+   * @param  string 'r' for read, 'w' for read/write.
+   * @param  bool True to force a new connection. The connection will not
+   *              be retrieved from or saved into the connection cache.
    * @return LiskDatabaseConnection   Lisk connection object.
    *
    * @task   info
    */
-  public function establishConnection($mode) {
+  public function establishConnection($mode, $force_new = false) {
     if ($mode != 'r' && $mode != 'w') {
       throw new Exception("Unknown mode '{$mode}', should be 'r' or 'w'.");
     }
@@ -988,15 +991,17 @@ abstract class LiskDAO {
     // TODO: There is currently no protection on 'r' queries against writing.
 
     $connection = null;
-    if ($mode == 'r') {
+    if (!$force_new) {
-      // If we're requesting a read connection but already have a write
+      if ($mode == 'r') {
-      // connection, reuse the write connection so that reads can take place
+        // If we're requesting a read connection but already have a write
-      // inside transactions.
+        // connection, reuse the write connection so that reads can take place
-      $connection = $this->getEstablishedConnection('w');
+        // inside transactions.
-    }
+        $connection = $this->getEstablishedConnection('w');
+      }
 
-    if (!$connection) {
+      if (!$connection) {
-      $connection = $this->getEstablishedConnection($mode);
+        $connection = $this->getEstablishedConnection($mode);
+      }
     }
 
     if (!$connection) {
@@ -1004,7 +1009,9 @@ abstract class LiskDAO {
       if (self::shouldIsolateAllLiskEffectsToTransactions()) {
         $connection->openTransaction();
       }
-      $this->setEstablishedConnection($mode, $connection);
+      if (!$force_new) {
+        $this->setEstablishedConnection($mode, $connection);
+      }
     }
 
     return $connection;