1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-24 14:30:56 +01:00

Make PullLocal smart about which repositories it should pull

Summary:
Ref T10756. When repositories are properly configured for the cluster (which is hard to set up today), be smart about which repositories are expected to exist on the current host, and only pull them.

This generally allows daemons to pretty much do the right thing no matter how many copies are running, although there may still be some lock contention issues that need to be sorted out.

Test Plan: {F1214483}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10756

Differential Revision: https://secure.phabricator.com/D15682
This commit is contained in:
epriestley 2016-04-11 09:35:31 -07:00
parent 3f9e8e675b
commit 0216fac30a
3 changed files with 141 additions and 5 deletions

View file

@ -117,7 +117,7 @@ final class AlmanacDeviceViewController
->setCanEdit($can_edit); ->setCanEdit($can_edit);
$header = id(new PHUIHeaderView()) $header = id(new PHUIHeaderView())
->setHeader(pht('DEVICE INTERFACES')) ->setHeader(pht('Device Interfaces'))
->addActionLink( ->addActionLink(
id(new PHUIButtonView()) id(new PHUIButtonView())
->setTag('a') ->setTag('a')
@ -167,7 +167,7 @@ final class AlmanacDeviceViewController
$upload_uri = '/auth/sshkey/upload/?objectPHID='.$device_phid; $upload_uri = '/auth/sshkey/upload/?objectPHID='.$device_phid;
$header = id(new PHUIHeaderView()) $header = id(new PHUIHeaderView())
->setHeader(pht('SSH PUBLIC KEYS')) ->setHeader(pht('SSH Public Keys'))
->addActionLink( ->addActionLink(
id(new PHUIButtonView()) id(new PHUIButtonView())
->setTag('a') ->setTag('a')
@ -238,7 +238,7 @@ final class AlmanacDeviceViewController
)); ));
return id(new PHUIObjectBoxView()) return id(new PHUIObjectBoxView())
->setHeaderText(pht('BOUND SERVICES')) ->setHeaderText(pht('Bound Services'))
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->setTable($table); ->setTable($table);
} }

View file

@ -19,4 +19,33 @@ final class AlmanacKeys extends Phobject {
return null; return null;
} }
public static function getLiveDevice() {
$device_id = self::getDeviceID();
if (!$device_id) {
return null;
}
$cache = PhabricatorCaches::getRequestCache();
$cache_key = 'almanac.device.self';
$device = $cache->getKey($cache_key);
if (!$device) {
$viewer = PhabricatorUser::getOmnipotentUser();
$device = id(new AlmanacDeviceQuery())
->setViewer($viewer)
->withNames(array($device_id))
->executeOne();
if (!$device) {
throw new Exception(
pht(
'This host has device ID "%s", but there is no corresponding '.
'device record in Almanac.',
$device_id));
}
$cache->setKey($cache_key, $device);
}
return $device;
}
} }

View file

@ -74,7 +74,9 @@ final class PhabricatorRepositoryPullLocalDaemon
while (!$this->shouldExit()) { while (!$this->shouldExit()) {
PhabricatorCaches::destroyRequestCache(); PhabricatorCaches::destroyRequestCache();
$pullable = $this->loadPullableRepositories($include, $exclude); $device = AlmanacKeys::getLiveDevice();
$pullable = $this->loadPullableRepositories($include, $exclude, $device);
// If any repositories have the NEEDS_UPDATE flag set, pull them // If any repositories have the NEEDS_UPDATE flag set, pull them
// as soon as possible. // as soon as possible.
@ -297,7 +299,11 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task pull * @task pull
*/ */
private function loadPullableRepositories(array $include, array $exclude) { private function loadPullableRepositories(
array $include,
array $exclude,
AlmanacDevice $device = null) {
$query = id(new PhabricatorRepositoryQuery()) $query = id(new PhabricatorRepositoryQuery())
->setViewer($this->getViewer()); ->setViewer($this->getViewer());
@ -348,6 +354,107 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
} }
$service_phids = array();
foreach ($repositories as $key => $repository) {
$service_phid = $repository->getAlmanacServicePHID();
// If the repository is bound to a service but this host is not a
// recognized device, or vice versa, don't pull the repository.
$is_cluster_repo = (bool)$service_phid;
$is_cluster_device = (bool)$device;
if ($is_cluster_repo != $is_cluster_device) {
if ($is_cluster_device) {
$this->log(
pht(
'Repository "%s" is not a cluster repository, but the current '.
'host is a cluster device ("%s"), so the repository will not '.
'be updated on this host.',
$repository->getDisplayName(),
$device->getName()));
} else {
$this->log(
pht(
'Repository "%s" is a cluster repository, but the current '.
'host is not a cluster device (it has no device ID), so the '.
'repository will not be updated on this host.',
$repository->getDisplayName()));
}
unset($repositories[$key]);
continue;
}
if ($service_phid) {
$service_phids[] = $service_phid;
}
}
if ($device) {
$device_phid = $device->getPHID();
if ($service_phids) {
// We could include `withDevicePHIDs()` here to pull a smaller result
// set, but we can provide more helpful diagnostic messages below if
// we fetch a little more data.
$services = id(new AlmanacServiceQuery())
->setViewer($this->getViewer())
->withPHIDs($service_phids)
->needBindings(true)
->execute();
$services = mpull($services, null, 'getPHID');
} else {
$services = array();
}
foreach ($repositories as $key => $repository) {
$service_phid = $repository->getAlmanacServicePHID();
$service = idx($services, $service_phid);
if (!$service) {
$this->log(
pht(
'Repository "%s" is on cluster service "%s", but that service '.
'could not be loaded, so the repository will not be updated '.
'on this host.',
$repository->getDisplayName(),
$service_phid));
unset($repositories[$key]);
continue;
}
$bindings = $service->getBindings();
$bindings = mpull($bindings, null, 'getDevicePHID');
$binding = idx($bindings, $device_phid);
if (!$binding) {
$this->log(
pht(
'Repository "%s" is on cluster service "%s", but that service '.
'is not bound to this device ("%s"), so the repository will '.
'not be updated on this host.',
$repository->getDisplayName(),
$service->getName(),
$device->getName()));
unset($repositories[$key]);
continue;
}
if ($binding->getIsDisabled()) {
$this->log(
pht(
'Repository "%s" is on cluster service "%s", but the binding '.
'between that service and this device ("%s") is disabled, so '.
'the not be updated on this host.',
$repository->getDisplayName(),
$service->getName(),
$device->getName()));
unset($repositories[$key]);
continue;
}
// We have a valid service that is actively bound to the current host
// device, so we're good to go.
}
}
// Shuffle the repositories, then re-key the array since shuffle() // Shuffle the repositories, then re-key the array since shuffle()
// discards keys. This is mostly for startup, we'll use soft priorities // discards keys. This is mostly for startup, we'll use soft priorities
// later. // later.