From 58eef68b7c60c7bc77f4cbbe408b086201fc2cc5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 11 Apr 2016 11:37:41 -0700 Subject: [PATCH] Rough cut of repository cluster status panel Summary: Ref T4292. This adds some very basic cluster/device data to the new management view. Nothing interesting yet. Also deal with disabled bindings a little more cleanly. Test Plan: {F1214619} Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15685 --- ...icatorConfigClusterDatabasesController.php | 2 +- ...fusionRepositoryClusterManagementPanel.php | 111 +++++++++++++++++- .../PhabricatorRepositoryPullLocalDaemon.php | 20 +++- src/docs/user/cluster/cluster.diviner | 17 +++ .../user/cluster/cluster_repositories.diviner | 86 ++++++++++++++ 5 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 src/docs/user/cluster/cluster_repositories.diviner diff --git a/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php index 1b3da10cae..c60b24b089 100644 --- a/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php +++ b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php @@ -203,7 +203,7 @@ final class PhabricatorConfigClusterDatabasesController ->setIcon('fa-book') ->setHref($doc_href) ->setTag('a') - ->setText(pht('Database Clustering Documentation'))); + ->setText(pht('Documentation'))); return id(new PHUIObjectBoxView()) ->setHeader($header) diff --git a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php index 5993f74355..2a6841ebdb 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php @@ -14,7 +14,116 @@ final class DiffusionRepositoryClusterManagementPanel } public function buildManagementPanelContent() { - return pht('TODO: Cluster configuration management.'); + $repository = $this->getRepository(); + $viewer = $this->getViewer(); + + $service_phid = $repository->getAlmanacServicePHID(); + if ($service_phid) { + $service = id(new AlmanacServiceQuery()) + ->setViewer($viewer) + ->withServiceTypes( + array( + AlmanacClusterRepositoryServiceType::SERVICETYPE, + )) + ->withPHIDs(array($service_phid)) + ->needBindings(true) + ->executeOne(); + if (!$service) { + // TODO: Viewer may not have permission to see the service, or it may + // be invalid? Raise some more useful error here? + throw new Exception(pht('Unable to load cluster service.')); + } + } else { + $service = null; + } + + Javelin::initBehavior('phabricator-tooltips'); + + $rows = array(); + if ($service) { + $bindings = $service->getBindings(); + $bindings = mgroup($bindings, 'getDevicePHID'); + + foreach ($bindings as $binding_group) { + $all_disabled = true; + foreach ($binding_group as $binding) { + if (!$binding->getIsDisabled()) { + $all_disabled = false; + break; + } + } + + $any_binding = head($binding_group); + + if ($all_disabled) { + $binding_icon = 'fa-times grey'; + $binding_tip = pht('Disabled'); + } else { + $binding_icon = 'fa-folder-open green'; + $binding_tip = pht('Active'); + } + + $binding_icon = id(new PHUIIconView()) + ->setIcon($binding_icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $binding_tip, + )); + + $device = $any_binding->getDevice(); + + $rows[] = array( + $binding_icon, + phutil_tag( + 'a', + array( + 'href' => $device->getURI(), + ), + $device->getName()), + ); + } + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString(pht('This is not a cluster repository.')) + ->setHeaders( + array( + null, + pht('Device'), + )) + ->setColumnClasses( + array( + null, + 'wide', + )); + + $doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories'); + + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Cluster Status')) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + if ($service) { + $header->setSubheader( + pht( + 'This repository is hosted on %s.', + phutil_tag( + 'a', + array( + 'href' => $service->getURI(), + ), + $service->getName()))); + } + + return id(new PHUIObjectBoxView()) + ->setHeader($header) + ->setTable($table); } } diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index 1eb3eda929..da8d9336d2 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -398,6 +398,10 @@ final class PhabricatorRepositoryPullLocalDaemon $services = id(new AlmanacServiceQuery()) ->setViewer($this->getViewer()) ->withPHIDs($service_phids) + ->withServiceTypes( + array( + AlmanacClusterRepositoryServiceType::SERVICETYPE, + )) ->needBindings(true) ->execute(); $services = mpull($services, null, 'getPHID'); @@ -422,9 +426,9 @@ final class PhabricatorRepositoryPullLocalDaemon } $bindings = $service->getBindings(); - $bindings = mpull($bindings, null, 'getDevicePHID'); - $binding = idx($bindings, $device_phid); - if (!$binding) { + $bindings = mgroup($bindings, 'getDevicePHID'); + $bindings = idx($bindings, $device_phid); + if (!$bindings) { $this->log( pht( 'Repository "%s" is on cluster service "%s", but that service '. @@ -437,7 +441,15 @@ final class PhabricatorRepositoryPullLocalDaemon continue; } - if ($binding->getIsDisabled()) { + $all_disabled = true; + foreach ($bindings as $binding) { + if (!$binding->getIsDisabled()) { + $all_disabled = false; + break; + } + } + + if ($all_disabled) { $this->log( pht( 'Repository "%s" is on cluster service "%s", but the binding '. diff --git a/src/docs/user/cluster/cluster.diviner b/src/docs/user/cluster/cluster.diviner index 2b6376d49e..a05381af92 100644 --- a/src/docs/user/cluster/cluster.diviner +++ b/src/docs/user/cluster/cluster.diviner @@ -26,6 +26,7 @@ operations personnel who need this high degree of flexibility. The remainder of this document summarizes how to add redundancy to each service and where your efforts are likely to have the greatest impact. + Cluster: Databases ================= @@ -38,3 +39,19 @@ Configuring replicas allows Phabricator to run in read-only mode if you lose the master, and to quickly promote the replica as a replacement. For details, see @{article:Cluster: Databases}. + + +Cluster: Repositories +===================== + +Configuring multiple repository hosts is complex. + +Repository replicas are important for availability if you host repositories +on Phabricator, but less important if you host repositories elsewhere +(instead, you should focus on making that service more available). + +The distributed nature of Git and Mercurial tend to mean that they are +naturally somewhat resistant to data loss: every clone of a repository includes +the entire history. + +For details, see @{article:Cluster: Repositories}. diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner new file mode 100644 index 0000000000..d9e859fd42 --- /dev/null +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -0,0 +1,86 @@ +@title Cluster: Repositories +@group intro + +Configuring Phabricator to use multiple repository hosts. + +Overview +======== + +WARNING: This feature is a very early prototype; the features this document +describes are mostly speculative fantasy. + +If you use Git or Mercurial, you can deploy Phabricator with multiple +repository hosts, configured so that each host is readable and writable. The +advantages of doing this are: + + - you can completely survive the loss of repository hosts; + - reads and writes can scale across multiple machines; and + - read and write performance across multiple geographic regions may improve. + +This configuration is complex, and many installs do not need to pursue it. + +This configuration is not currently supported with Subversion. + + +Repository Hosts +================ + +Repository hosts must run a complete, fully configured copy of Phabricator, +including a webserver. If you make repositories available over SSH, they must +also run a properly configured `sshd`. + +Generally, these hosts will run the same set of services and configuration that +web hosts run. If you prefer, you can overlay these services and put web and +repository services on the same hosts. + +When a user requests information about a repository that can only be satisfied +by examining a repository working copy, the webserver receiving the reqeust +will make an HTTP service call to a repository server which hosts the +repository to retrieve the data it needs. It will use the result of this query +to respond to the user. + + +How Reads and Writes Work +========================= + +Phabricator repository replicas are multi-master: every node is readable and +writable, and a cluster of nodes can (almost always) survive the loss of any +arbitrary subset of nodes so long as at least one node is still alive. + +Phabricator maintains an internal version for each repository, and increments +it when the repository is mutated. + +Before responding to a read, replicas make sure their version of the repository +is up to date (no node in the cluster has a newer version of the repository). +If it isn't, they block the read until they can complete a fetch. + +Before responding to a write, replicas obtain a global lock, perform the same +version check and fetch if necessary, then allow the write to continue. + + +Backups +====== + +Even if you configure clustering, you should still consider retaining separate +backup snapshots. Replicas protect you from data loss if you lose a host, but +they do not let you rewind time to recover from data mutation mistakes. + +If something issues a `--force` push that destroys branch heads, the mutation +will propagate to the replicas. + +You may be able to manually restore the branches by using tools like the +Phabricator push log or the Git reflog so it is less important to retain +repository snapshots than database snapshots, but it is still possible for +data to be lost permanently, especially if you don't notice the problem for +some time. + +Retaining separate backup snapshots will improve your ability to recover more +data more easily in a wider range of disaster situations. + + +Next Steps +========== + +Continue by: + + - returning to @{article:Clustering Introduction}.