From c402d7d3076d81889c678fe439e09b624456c10e Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 16 Apr 2014 13:01:04 -0700 Subject: [PATCH] Implement smart waits for rarely updated repositories Summary: Ref T4605. When figuring out how long to wait to update a repository, factor in when it was last pushed. For rarely updated repositories, wait longer between updates. (A slightly funky thing about this is that empty repos update every 15 seconds, but that seems OK for the moment.) Test Plan: Ran `bin/phd debug pulllocal` and saw sensible calculations and output: ``` ... PhabricatorRepositoryPullLocalDaemon Last commit to repository "rPOEMS" was 1,239,608 seconds ago; considering a wait of 6,198 seconds before update. >>> [79] SELECT * FROM `repository` r ORDER BY r.id DESC <<< [79] 514 us >>> [80] SELECT * FROM `repository_statusmessage` WHERE statusType = 'needs-update' <<< [80] 406 us PhabricatorRepositoryPullLocalDaemon Repository "rINIH" is not due for an update for 8,754 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rDUCK" is not due for an update for 14 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rMTESTX" is not due for an update for 21,598 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rQWER" is not due for an update for 14 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rBT" is not due for an update for 13 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rSVNX" is not due for an update for 21,598 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rINIG" is not due for an update for 13 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rHGTEST" is not due for an update for 21,598 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rBTX" is not due for an update for 14 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rGX" is not due for an update for 13 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rMTX" is currently updating. PhabricatorRepositoryPullLocalDaemon Repository "rPOEMS" is not due for an update for 6,198 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rPHU" is currently updating. PhabricatorRepositoryPullLocalDaemon Repository "rSVN" is not due for an update for 21,598 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rPHY" is currently updating. PhabricatorRepositoryPullLocalDaemon Repository "rGTEST" is not due for an update for 21,598 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rINIS" is not due for an update for 6,894 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rARCLINT" is not due for an update for 21,599 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rLPHX" is not due for an update for 1,979 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rARC" is not due for an update for 1,824 second(s). PhabricatorRepositoryPullLocalDaemon Repository "rINIHG" is not due for an update for 21,599 second(s). ... ``` Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T4605 Differential Revision: https://secure.phabricator.com/D8782 --- .../PhabricatorRepositoryPullLocalDaemon.php | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index f3627c2e9c..673ae4b8eb 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -346,6 +346,49 @@ final class PhabricatorRepositoryPullLocalDaemon } $sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep); + + // Smart wait: pull rarely used repositories less frequently. Find the + // most recent commit which is older than the current time (this keeps us + // from spinning on repositories with a silly commit post-dated to some time + // in 2037), and adjust how frequently we pull based on how frequently this + // repository updates. + + $table = id(new PhabricatorRepositoryCommit()); + $last_commit = queryfx_one( + $table->establishConnection('w'), + 'SELECT epoch FROM %T + WHERE repositoryID = %d AND epoch <= %d + ORDER BY epoch DESC LIMIT 1', + $table->getTableName(), + $repository->getID(), + time() + $min_sleep); + if ($last_commit) { + $time_since_commit = (time() + $min_sleep) - $last_commit['epoch']; + + // Wait 0.5% of the time since the last commit before we pull. This gives + // us these wait times: + // + // 50 minutes or less: 15 seconds + // about 3 hours: 1 minute + // about 16 hours: 5 minutes + // about 2 days: 15 minutes + // 50 days or more: 6 hours + + $smart_wait = ($time_since_commit / 200); + $smart_wait = min($smart_wait, phutil_units('6 hours in seconds')); + + $this->log( + pht( + 'Last commit to repository "%s" was %s seconds ago; considering '. + 'a wait of %s seconds before update.', + $repository->getMonogram(), + new PhutilNumber($time_since_commit), + new PhutilNumber($smart_wait))); + + $smart_wait = max(15, $smart_wait); + $sleep_for = max($smart_wait, $sleep_for); + } + if ($sleep_for < $min_sleep) { $sleep_for = $min_sleep; }