From 4675306615da8848e0f196820af7bcc6f9fa6118 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 4 Feb 2019 16:52:58 -0800 Subject: [PATCH] Add a "metronome" for spreading service call load Summary: Ref T13244. See D20080. Rather than randomly jittering service calls, we can give each host a "metronome" that ticks every 60 seconds to get load to spread out after one cycle. For example, web001 ticks (and makes a service call) when the second hand points at 0:17, web002 at 0:43, web003 at 0:04, etc. For now I'm just planning to seed the metronomes randomly based on hostname, but we could conceivably give each host an assigned offset some day if we want perfectly smooth service call rates. Test Plan: Ran unit tests. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13244 Differential Revision: https://secure.phabricator.com/D20087 --- src/__phutil_library_map__.php | 4 + .../util/PhabricatorMetronome.php | 92 +++++++++++++++++++ .../PhabricatorMetronomeTestCase.php | 61 ++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 src/infrastructure/util/PhabricatorMetronome.php create mode 100644 src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 324fc3c5de..fe77880417 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3552,6 +3552,8 @@ phutil_register_library_map(array( 'PhabricatorMetaMTASchemaSpec' => 'applications/metamta/storage/PhabricatorMetaMTASchemaSpec.php', 'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php', 'PhabricatorMetaMTAWorker' => 'applications/metamta/PhabricatorMetaMTAWorker.php', + 'PhabricatorMetronome' => 'infrastructure/util/PhabricatorMetronome.php', + 'PhabricatorMetronomeTestCase' => 'infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php', 'PhabricatorMetronomicTriggerClock' => 'infrastructure/daemon/workers/clock/PhabricatorMetronomicTriggerClock.php', 'PhabricatorModularTransaction' => 'applications/transactions/storage/PhabricatorModularTransaction.php', 'PhabricatorModularTransactionType' => 'applications/transactions/storage/PhabricatorModularTransactionType.php', @@ -9477,6 +9479,8 @@ phutil_register_library_map(array( 'PhabricatorMetaMTASchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController', 'PhabricatorMetaMTAWorker' => 'PhabricatorWorker', + 'PhabricatorMetronome' => 'Phobject', + 'PhabricatorMetronomeTestCase' => 'PhabricatorTestCase', 'PhabricatorMetronomicTriggerClock' => 'PhabricatorTriggerClock', 'PhabricatorModularTransaction' => 'PhabricatorApplicationTransaction', 'PhabricatorModularTransactionType' => 'Phobject', diff --git a/src/infrastructure/util/PhabricatorMetronome.php b/src/infrastructure/util/PhabricatorMetronome.php new file mode 100644 index 0000000000..24f58127f6 --- /dev/null +++ b/src/infrastructure/util/PhabricatorMetronome.php @@ -0,0 +1,92 @@ +offset = $offset; + + return $this; + } + + public function setFrequency($frequency) { + if (!is_int($frequency)) { + throw new Exception(pht('Metronome frequency must be an integer.')); + } + + if ($frequency < 1) { + throw new Exception(pht('Metronome frequency must be 1 or more.')); + } + + $this->frequency = $frequency; + + return $this; + } + + public function setOffsetFromSeed($seed) { + $offset = PhabricatorHash::digestToRange($seed, 0, PHP_INT_MAX); + return $this->setOffset($offset); + } + + public function getFrequency() { + if ($this->frequency === null) { + throw new PhutilInvalidStateException('setFrequency'); + } + return $this->frequency; + } + + public function getOffset() { + $frequency = $this->getFrequency(); + return ($this->offset % $frequency); + } + + public function getNextTickAfter($epoch) { + $frequency = $this->getFrequency(); + $offset = $this->getOffset(); + + $remainder = ($epoch % $frequency); + + if ($remainder < $offset) { + return ($epoch - $remainder) + $offset; + } else { + return ($epoch - $remainder) + $frequency + $offset; + } + } + + public function didTickBetween($min, $max) { + if ($max < $min) { + throw new Exception( + pht( + 'Maximum tick window must not be smaller than minimum tick window.')); + } + + $next = $this->getNextTickAfter($min); + return ($next <= $max); + } + +} diff --git a/src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php b/src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php new file mode 100644 index 0000000000..9ad74e2b90 --- /dev/null +++ b/src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php @@ -0,0 +1,61 @@ + 44, + 'web002.example.net' => 36, + 'web003.example.net' => 25, + 'web004.example.net' => 25, + 'web005.example.net' => 16, + 'web006.example.net' => 26, + 'web007.example.net' => 35, + 'web008.example.net' => 14, + ); + + $metronome = id(new PhabricatorMetronome()) + ->setFrequency(60); + + foreach ($cases as $input => $expect) { + $metronome->setOffsetFromSeed($input); + + $this->assertEqual( + $expect, + $metronome->getOffset(), + pht('Offset for: %s', $input)); + } + } + + public function testMetronomeTicks() { + $metronome = id(new PhabricatorMetronome()) + ->setFrequency(60) + ->setOffset(13); + + $tick_epoch = strtotime('2000-01-01 11:11:13 AM UTC'); + + // Since the epoch is at "0:13" on the clock, the metronome should tick + // then. + $this->assertEqual( + $tick_epoch, + $metronome->getNextTickAfter($tick_epoch - 1), + pht('Tick at 11:11:13 AM.')); + + // The next tick should be a minute later. + $this->assertEqual( + $tick_epoch + 60, + $metronome->getNextTickAfter($tick_epoch), + pht('Tick at 11:12:13 AM.')); + + + // There's no tick in the next 59 seconds. + $this->assertFalse( + $metronome->didTickBetween($tick_epoch, $tick_epoch + 59)); + + $this->assertTrue( + $metronome->didTickBetween($tick_epoch, $tick_epoch + 60)); + } + + +}