Add a "metronome" for spreading service call load

Summary:
Ref T13244. See D20080. Rather than randomly jittering service calls, we can give each host a "metronome" that ticks every 60 seconds to get load to spread out after one cycle.

For example, web001 ticks (and makes a service call) when the second hand points at 0:17, web002 at 0:43, web003 at 0:04, etc.

For now I'm just planning to seed the metronomes randomly based on hostname, but we could conceivably give each host an assigned offset some day if we want perfectly smooth service call rates.

Test Plan: Ran unit tests.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13244

Differential Revision: https://secure.phabricator.com/D20087

src/__phutil_library_map__.php

@@ -3552,6 +3552,8 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTASchemaSpec' => 'applications/metamta/storage/PhabricatorMetaMTASchemaSpec.php',
     'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php',
     'PhabricatorMetaMTAWorker' => 'applications/metamta/PhabricatorMetaMTAWorker.php',
+    'PhabricatorMetronome' => 'infrastructure/util/PhabricatorMetronome.php',
+    'PhabricatorMetronomeTestCase' => 'infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php',
     'PhabricatorMetronomicTriggerClock' => 'infrastructure/daemon/workers/clock/PhabricatorMetronomicTriggerClock.php',
     'PhabricatorModularTransaction' => 'applications/transactions/storage/PhabricatorModularTransaction.php',
     'PhabricatorModularTransactionType' => 'applications/transactions/storage/PhabricatorModularTransactionType.php',
@@ -9477,6 +9479,8 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTASchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAWorker' => 'PhabricatorWorker',
+    'PhabricatorMetronome' => 'Phobject',
+    'PhabricatorMetronomeTestCase' => 'PhabricatorTestCase',
     'PhabricatorMetronomicTriggerClock' => 'PhabricatorTriggerClock',
     'PhabricatorModularTransaction' => 'PhabricatorApplicationTransaction',
     'PhabricatorModularTransactionType' => 'Phobject',

src/infrastructure/util/PhabricatorMetronome.php

@@ -0,0 +1,92 @@
+<?php
+
+/**
+ * Tick at a given frequency with a specifiable offset.
+ *
+ * One use case for this is to flatten out load spikes caused by periodic
+ * service calls. Give each host a metronome that ticks at the same frequency,
+ * but with different offsets. Then, have hosts make service calls only after
+ * their metronome ticks. This spreads service calls out evenly more quickly
+ * and more predictably than adding random jitter.
+ */
+final class PhabricatorMetronome
+  extends Phobject {
+
+  private $offset = 0;
+  private $frequency;
+
+  public function setOffset($offset) {
+    if (!is_int($offset)) {
+      throw new Exception(pht('Metronome offset must be an integer.'));
+    }
+
+    if ($offset < 0) {
+      throw new Exception(pht('Metronome offset must be 0 or more.'));
+    }
+
+    // We're not requiring that the offset be smaller than the frequency. If
+    // the offset is larger, we'll just clamp it to the frequency before we
+    // use it. This allows the offset to be configured before the frequency
+    // is configured, which is useful for using a hostname as an offset seed.
+
+    $this->offset = $offset;
+
+    return $this;
+  }
+
+  public function setFrequency($frequency) {
+    if (!is_int($frequency)) {
+      throw new Exception(pht('Metronome frequency must be an integer.'));
+    }
+
+    if ($frequency < 1) {
+      throw new Exception(pht('Metronome frequency must be 1 or more.'));
+    }
+
+    $this->frequency = $frequency;
+
+    return $this;
+  }
+
+  public function setOffsetFromSeed($seed) {
+    $offset = PhabricatorHash::digestToRange($seed, 0, PHP_INT_MAX);
+    return $this->setOffset($offset);
+  }
+
+  public function getFrequency() {
+    if ($this->frequency === null) {
+      throw new PhutilInvalidStateException('setFrequency');
+    }
+    return $this->frequency;
+  }
+
+  public function getOffset() {
+    $frequency = $this->getFrequency();
+    return ($this->offset % $frequency);
+  }
+
+  public function getNextTickAfter($epoch) {
+    $frequency = $this->getFrequency();
+    $offset = $this->getOffset();
+
+    $remainder = ($epoch % $frequency);
+
+    if ($remainder < $offset) {
+      return ($epoch - $remainder) + $offset;
+    } else {
+      return ($epoch - $remainder) + $frequency + $offset;
+    }
+  }
+
+  public function didTickBetween($min, $max) {
+    if ($max < $min) {
+      throw new Exception(
+        pht(
+          'Maximum tick window must not be smaller than minimum tick window.'));
+    }
+
+    $next = $this->getNextTickAfter($min);
+    return ($next <= $max);
+  }
+
+}

src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php

@@ -0,0 +1,61 @@
+<?php
+
+final class PhabricatorMetronomeTestCase
+  extends PhabricatorTestCase {
+
+  public function testMetronomeOffsets() {
+    $cases = array(
+      'web001.example.net' => 44,
+      'web002.example.net' => 36,
+      'web003.example.net' => 25,
+      'web004.example.net' => 25,
+      'web005.example.net' => 16,
+      'web006.example.net' => 26,
+      'web007.example.net' => 35,
+      'web008.example.net' => 14,
+    );
+
+    $metronome = id(new PhabricatorMetronome())
+      ->setFrequency(60);
+
+    foreach ($cases as $input => $expect) {
+      $metronome->setOffsetFromSeed($input);
+
+      $this->assertEqual(
+        $expect,
+        $metronome->getOffset(),
+        pht('Offset for: %s', $input));
+    }
+  }
+
+  public function testMetronomeTicks() {
+    $metronome = id(new PhabricatorMetronome())
+      ->setFrequency(60)
+      ->setOffset(13);
+
+    $tick_epoch = strtotime('2000-01-01 11:11:13 AM UTC');
+
+    // Since the epoch is at "0:13" on the clock, the metronome should tick
+    // then.
+    $this->assertEqual(
+      $tick_epoch,
+      $metronome->getNextTickAfter($tick_epoch - 1),
+      pht('Tick at 11:11:13 AM.'));
+
+    // The next tick should be a minute later.
+    $this->assertEqual(
+      $tick_epoch + 60,
+      $metronome->getNextTickAfter($tick_epoch),
+      pht('Tick at 11:12:13 AM.'));
+
+
+    // There's no tick in the next 59 seconds.
+    $this->assertFalse(
+      $metronome->didTickBetween($tick_epoch, $tick_epoch + 59));
+
+    $this->assertTrue(
+      $metronome->didTickBetween($tick_epoch, $tick_epoch + 60));
+  }
+
+
+}