From 0216fac30a38cb9abc349da26cd0566ad718e33c Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 11 Apr 2016 09:35:31 -0700 Subject: [PATCH] Make PullLocal smart about which repositories it should pull Summary: Ref T10756. When repositories are properly configured for the cluster (which is hard to set up today), be smart about which repositories are expected to exist on the current host, and only pull them. This generally allows daemons to pretty much do the right thing no matter how many copies are running, although there may still be some lock contention issues that need to be sorted out. Test Plan: {F1214483} Reviewers: chad Reviewed By: chad Maniphest Tasks: T10756 Differential Revision: https://secure.phabricator.com/D15682 --- .../AlmanacDeviceViewController.php | 6 +- src/applications/almanac/util/AlmanacKeys.php | 29 +++++ .../PhabricatorRepositoryPullLocalDaemon.php | 111 +++++++++++++++++- 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/src/applications/almanac/controller/AlmanacDeviceViewController.php b/src/applications/almanac/controller/AlmanacDeviceViewController.php index efc4334132..000c8f8971 100644 --- a/src/applications/almanac/controller/AlmanacDeviceViewController.php +++ b/src/applications/almanac/controller/AlmanacDeviceViewController.php @@ -117,7 +117,7 @@ final class AlmanacDeviceViewController ->setCanEdit($can_edit); $header = id(new PHUIHeaderView()) - ->setHeader(pht('DEVICE INTERFACES')) + ->setHeader(pht('Device Interfaces')) ->addActionLink( id(new PHUIButtonView()) ->setTag('a') @@ -167,7 +167,7 @@ final class AlmanacDeviceViewController $upload_uri = '/auth/sshkey/upload/?objectPHID='.$device_phid; $header = id(new PHUIHeaderView()) - ->setHeader(pht('SSH PUBLIC KEYS')) + ->setHeader(pht('SSH Public Keys')) ->addActionLink( id(new PHUIButtonView()) ->setTag('a') @@ -238,7 +238,7 @@ final class AlmanacDeviceViewController )); return id(new PHUIObjectBoxView()) - ->setHeaderText(pht('BOUND SERVICES')) + ->setHeaderText(pht('Bound Services')) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setTable($table); } diff --git a/src/applications/almanac/util/AlmanacKeys.php b/src/applications/almanac/util/AlmanacKeys.php index dec49a08a7..d15d3cb439 100644 --- a/src/applications/almanac/util/AlmanacKeys.php +++ b/src/applications/almanac/util/AlmanacKeys.php @@ -19,4 +19,33 @@ final class AlmanacKeys extends Phobject { return null; } + public static function getLiveDevice() { + $device_id = self::getDeviceID(); + if (!$device_id) { + return null; + } + + $cache = PhabricatorCaches::getRequestCache(); + $cache_key = 'almanac.device.self'; + + $device = $cache->getKey($cache_key); + if (!$device) { + $viewer = PhabricatorUser::getOmnipotentUser(); + $device = id(new AlmanacDeviceQuery()) + ->setViewer($viewer) + ->withNames(array($device_id)) + ->executeOne(); + if (!$device) { + throw new Exception( + pht( + 'This host has device ID "%s", but there is no corresponding '. + 'device record in Almanac.', + $device_id)); + } + $cache->setKey($cache_key, $device); + } + + return $device; + } + } diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index 21b0e13413..1eb3eda929 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -74,7 +74,9 @@ final class PhabricatorRepositoryPullLocalDaemon while (!$this->shouldExit()) { PhabricatorCaches::destroyRequestCache(); - $pullable = $this->loadPullableRepositories($include, $exclude); + $device = AlmanacKeys::getLiveDevice(); + + $pullable = $this->loadPullableRepositories($include, $exclude, $device); // If any repositories have the NEEDS_UPDATE flag set, pull them // as soon as possible. @@ -297,7 +299,11 @@ final class PhabricatorRepositoryPullLocalDaemon /** * @task pull */ - private function loadPullableRepositories(array $include, array $exclude) { + private function loadPullableRepositories( + array $include, + array $exclude, + AlmanacDevice $device = null) { + $query = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()); @@ -348,6 +354,107 @@ final class PhabricatorRepositoryPullLocalDaemon } } + $service_phids = array(); + foreach ($repositories as $key => $repository) { + $service_phid = $repository->getAlmanacServicePHID(); + + // If the repository is bound to a service but this host is not a + // recognized device, or vice versa, don't pull the repository. + $is_cluster_repo = (bool)$service_phid; + $is_cluster_device = (bool)$device; + if ($is_cluster_repo != $is_cluster_device) { + if ($is_cluster_device) { + $this->log( + pht( + 'Repository "%s" is not a cluster repository, but the current '. + 'host is a cluster device ("%s"), so the repository will not '. + 'be updated on this host.', + $repository->getDisplayName(), + $device->getName())); + } else { + $this->log( + pht( + 'Repository "%s" is a cluster repository, but the current '. + 'host is not a cluster device (it has no device ID), so the '. + 'repository will not be updated on this host.', + $repository->getDisplayName())); + } + unset($repositories[$key]); + continue; + } + + if ($service_phid) { + $service_phids[] = $service_phid; + } + } + + if ($device) { + $device_phid = $device->getPHID(); + + if ($service_phids) { + // We could include `withDevicePHIDs()` here to pull a smaller result + // set, but we can provide more helpful diagnostic messages below if + // we fetch a little more data. + $services = id(new AlmanacServiceQuery()) + ->setViewer($this->getViewer()) + ->withPHIDs($service_phids) + ->needBindings(true) + ->execute(); + $services = mpull($services, null, 'getPHID'); + } else { + $services = array(); + } + + foreach ($repositories as $key => $repository) { + $service_phid = $repository->getAlmanacServicePHID(); + + $service = idx($services, $service_phid); + if (!$service) { + $this->log( + pht( + 'Repository "%s" is on cluster service "%s", but that service '. + 'could not be loaded, so the repository will not be updated '. + 'on this host.', + $repository->getDisplayName(), + $service_phid)); + unset($repositories[$key]); + continue; + } + + $bindings = $service->getBindings(); + $bindings = mpull($bindings, null, 'getDevicePHID'); + $binding = idx($bindings, $device_phid); + if (!$binding) { + $this->log( + pht( + 'Repository "%s" is on cluster service "%s", but that service '. + 'is not bound to this device ("%s"), so the repository will '. + 'not be updated on this host.', + $repository->getDisplayName(), + $service->getName(), + $device->getName())); + unset($repositories[$key]); + continue; + } + + if ($binding->getIsDisabled()) { + $this->log( + pht( + 'Repository "%s" is on cluster service "%s", but the binding '. + 'between that service and this device ("%s") is disabled, so '. + 'the not be updated on this host.', + $repository->getDisplayName(), + $service->getName(), + $device->getName())); + unset($repositories[$key]); + continue; + } + + // We have a valid service that is actively bound to the current host + // device, so we're good to go. + } + } + // Shuffle the repositories, then re-key the array since shuffle() // discards keys. This is mostly for startup, we'll use soft priorities // later.