From dd1023e5a8f8aca512c798b9645e5dfe33edb87d Mon Sep 17 00:00:00 2001 From: June Rhodes Date: Sat, 16 Apr 2016 02:07:03 +0000 Subject: [PATCH 01/50] Support relative links in Phriction Summary: Resolves T7691. This turned out more complex than I really wanted, mainly because I needed to feed the slug information through to both the document renderer and the preview window that appears in the edit controller. After this change, you can now create relative links in Phriction by doing `[[ ./../some/relative/path ]]`. Relative paths aren't handled anywhere else (they'll still render, but the dots are turned into a literal 'dot' as per existing behaviour). Test Plan: Created some Phriction documents with relative links, saw them all link correctly. Reviewers: #blessed_reviewers, epriestley Reviewed By: #blessed_reviewers, epriestley Subscribers: Korvin Maniphest Tasks: T7691 Differential Revision: https://secure.phabricator.com/D15732 --- src/__phutil_library_map__.php | 2 ++ .../PhabricatorPhrictionApplication.php | 2 +- .../controller/PhrictionEditController.php | 2 +- .../PhrictionMarkupPreviewController.php | 28 ++++++++++++++++ .../markup/PhrictionRemarkupRule.php | 33 +++++++++++++++++++ .../phriction/storage/PhrictionContent.php | 3 +- 6 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/applications/phriction/controller/PhrictionMarkupPreviewController.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index e76e768774..ac506ce0f5 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3905,6 +3905,7 @@ phutil_register_library_map(array( 'PhrictionHistoryController' => 'applications/phriction/controller/PhrictionHistoryController.php', 'PhrictionInfoConduitAPIMethod' => 'applications/phriction/conduit/PhrictionInfoConduitAPIMethod.php', 'PhrictionListController' => 'applications/phriction/controller/PhrictionListController.php', + 'PhrictionMarkupPreviewController' => 'applications/phriction/controller/PhrictionMarkupPreviewController.php', 'PhrictionMoveController' => 'applications/phriction/controller/PhrictionMoveController.php', 'PhrictionNewController' => 'applications/phriction/controller/PhrictionNewController.php', 'PhrictionRemarkupRule' => 'applications/phriction/markup/PhrictionRemarkupRule.php', @@ -8702,6 +8703,7 @@ phutil_register_library_map(array( 'PhrictionHistoryController' => 'PhrictionController', 'PhrictionInfoConduitAPIMethod' => 'PhrictionConduitAPIMethod', 'PhrictionListController' => 'PhrictionController', + 'PhrictionMarkupPreviewController' => 'PhabricatorController', 'PhrictionMoveController' => 'PhrictionController', 'PhrictionNewController' => 'PhrictionController', 'PhrictionRemarkupRule' => 'PhutilRemarkupRule', diff --git a/src/applications/phriction/application/PhabricatorPhrictionApplication.php b/src/applications/phriction/application/PhabricatorPhrictionApplication.php index 7721749c36..c996365c67 100644 --- a/src/applications/phriction/application/PhabricatorPhrictionApplication.php +++ b/src/applications/phriction/application/PhabricatorPhrictionApplication.php @@ -59,7 +59,7 @@ final class PhabricatorPhrictionApplication extends PhabricatorApplication { 'new/' => 'PhrictionNewController', 'move/(?P[1-9]\d*)/' => 'PhrictionMoveController', - 'preview/' => 'PhabricatorMarkupPreviewController', + 'preview/(?P.+/)' => 'PhrictionMarkupPreviewController', 'diff/(?P[1-9]\d*)/' => 'PhrictionDiffController', ), ); diff --git a/src/applications/phriction/controller/PhrictionEditController.php b/src/applications/phriction/controller/PhrictionEditController.php index fd88bf9408..e0e27ad796 100644 --- a/src/applications/phriction/controller/PhrictionEditController.php +++ b/src/applications/phriction/controller/PhrictionEditController.php @@ -272,7 +272,7 @@ final class PhrictionEditController $preview = id(new PHUIRemarkupPreviewPanel()) ->setHeader($content->getTitle()) - ->setPreviewURI('/phriction/preview/') + ->setPreviewURI('/phriction/preview/'.$document->getSlug()) ->setControlID('document-textarea') ->setPreviewType(PHUIRemarkupPreviewPanel::DOCUMENT); diff --git a/src/applications/phriction/controller/PhrictionMarkupPreviewController.php b/src/applications/phriction/controller/PhrictionMarkupPreviewController.php new file mode 100644 index 0000000000..b5c97e5c64 --- /dev/null +++ b/src/applications/phriction/controller/PhrictionMarkupPreviewController.php @@ -0,0 +1,28 @@ +getRequest(); + $viewer = $request->getUser(); + + $text = $request->getStr('text'); + $slug = $request->getURIData('slug'); + + $output = PhabricatorMarkupEngine::renderOneObject( + id(new PhabricatorMarkupOneOff()) + ->setPreserveLinebreaks(true) + ->setDisableCache(true) + ->setContent($text), + 'default', + $viewer, + array( + 'phriction.isPreview' => true, + 'phriction.slug' => $slug, + )); + + return id(new AphrontAjaxResponse()) + ->setContent($output); + } +} diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index 6b66dd95d1..c3d9aaf6a0 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -15,6 +15,39 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { public function markupDocumentLink(array $matches) { $link = trim($matches[1]); + + // Handle relative links. + if (substr($link, 0, 2) === './') { + $base = null; + $context = $this->getEngine()->getConfig('contextObject'); + if ($context !== null && $context instanceof PhrictionContent) { + // Handle content when it's being rendered in document view. + $base = $context->getSlug(); + } + if ($context !== null && is_array($context) && + idx($context, 'phriction.isPreview')) { + // Handle content when it's a preview for the Phriction editor. + $base = idx($context, 'phriction.slug'); + } + if ($base !== null) { + $base_parts = explode('/', rtrim($base, '/')); + $rel_parts = explode('/', substr(rtrim($link, '/'), 2)); + foreach ($rel_parts as $part) { + if ($part === '.') { + // Consume standalone dots in a relative path, and do + // nothing with them. + } else if ($part === '..') { + if (count($base_parts) > 0) { + array_pop($base_parts); + } + } else { + array_push($base_parts, $part); + } + } + $link = implode('/', $base_parts).'/'; + } + } + $name = trim(idx($matches, 2, $link)); if (empty($matches[2])) { $name = explode('/', trim($name, '/')); diff --git a/src/applications/phriction/storage/PhrictionContent.php b/src/applications/phriction/storage/PhrictionContent.php index 99aaf0c7ec..3a2e20aa29 100644 --- a/src/applications/phriction/storage/PhrictionContent.php +++ b/src/applications/phriction/storage/PhrictionContent.php @@ -27,7 +27,8 @@ final class PhrictionContent extends PhrictionDAO return PhabricatorMarkupEngine::renderOneObject( $this, self::MARKUP_FIELD_BODY, - $viewer); + $viewer, + $this); } protected function getConfiguration() { From 27227b8010eed6e6fc256f3f7e83103df62a0d80 Mon Sep 17 00:00:00 2001 From: June Rhodes Date: Sat, 16 Apr 2016 03:02:39 +0000 Subject: [PATCH 02/50] Show missing Phriction documents as red links, invisible documents with a lock Summary: Ref T7691 (errata). This shows links to Phriction documents in red if they're missing, and links to Phriction documents in grey with a lock icon if the user doesn't have the correct permissions to see the document. Test Plan: Tested a bunch of different configurations: ``` [[ ./../ ]] Back to Main Document [[ ./../subdocument_2]] Mmmm more documents [[ ./../invisible_document]] Mmmm more documents [[ ./../ | Explicit Title ]] Back to Main Document [[ ./../subdocument_2 | Explicit Title ]] Mmmm more documents [[ ./../invisible_document | Explicit Title ]] Mmmm more documents [[ ]] Absolute link [[ subdocument_2 ]] Absolute link [[ invisible_document ]] Absolute link [[ | Explicit Title ]] Absolute link [[ subdocument_2 | Explicit Title ]] Absolute link [[ invisible_document | Explicit Title ]] Absolute link ``` Got the expected result: {F1221106} Reviewers: epriestley, chad, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T7691 Differential Revision: https://secure.phabricator.com/D15733 --- resources/celerity/map.php | 4 +- .../markup/PhrictionRemarkupRule.php | 136 ++++++++++++++---- .../phriction/phriction-document-css.css | 10 ++ 3 files changed, 124 insertions(+), 26 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index fb5a93fcc9..49f38b3c51 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -88,7 +88,7 @@ return array( 'rsrc/css/application/phortune/phortune-credit-card-form.css' => '8391eb02', 'rsrc/css/application/phortune/phortune.css' => '9149f103', 'rsrc/css/application/phrequent/phrequent.css' => 'ffc185ad', - 'rsrc/css/application/phriction/phriction-document-css.css' => 'd1861e06', + 'rsrc/css/application/phriction/phriction-document-css.css' => '55446c91', 'rsrc/css/application/policy/policy-edit.css' => '815c66f7', 'rsrc/css/application/policy/policy-transaction-detail.css' => '82100a43', 'rsrc/css/application/policy/policy.css' => '957ea14c', @@ -807,7 +807,7 @@ return array( 'phortune-credit-card-form-css' => '8391eb02', 'phortune-css' => '9149f103', 'phrequent-css' => 'ffc185ad', - 'phriction-document-css' => 'd1861e06', + 'phriction-document-css' => '55446c91', 'phui-action-panel-css' => '91c7b835', 'phui-badge-view-css' => '3baef8db', 'phui-big-info-view-css' => 'bd903741', diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index c3d9aaf6a0..99a9d1bdd4 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -2,6 +2,8 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { + const KEY_RULE_PHRICTION_LINK = 'phriction.link'; + public function getPriority() { return 175.0; } @@ -48,37 +50,123 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { } } - $name = trim(idx($matches, 2, $link)); + $name = trim(idx($matches, 2, '')); if (empty($matches[2])) { - $name = explode('/', trim($name, '/')); - $name = end($name); + $name = null; } - $uri = new PhutilURI($link); - $slug = $uri->getPath(); - $fragment = $uri->getFragment(); - $slug = PhabricatorSlug::normalize($slug); - $slug = PhrictionDocument::getSlugURI($slug); - $href = (string)id(new PhutilURI($slug))->setFragment($fragment); + // Link is now used for slug detection, so append a slash if one + // is needed. + $link = rtrim($link, '/').'/'; - $text_mode = $this->getEngine()->isTextMode(); - $mail_mode = $this->getEngine()->isHTMLMailMode(); + $engine = $this->getEngine(); + $token = $engine->storeText('x'); + $metadata = $engine->getTextMetadata( + self::KEY_RULE_PHRICTION_LINK, + array()); + $metadata[] = array( + 'token' => $token, + 'link' => $link, + 'explicitName' => $name, + ); + $engine->setTextMetadata(self::KEY_RULE_PHRICTION_LINK, $metadata); - if ($this->getEngine()->getState('toc')) { - $text = $name; - } else if ($text_mode || $mail_mode) { - return PhabricatorEnv::getProductionURI($href); - } else { - $text = $this->newTag( - 'a', - array( - 'href' => $href, - 'class' => 'phriction-link', - ), - $name); + return $token; + } + + public function didMarkupText() { + $engine = $this->getEngine(); + $metadata = $engine->getTextMetadata( + self::KEY_RULE_PHRICTION_LINK, + array()); + + if (!$metadata) { + return; } - return $this->getEngine()->storeText($text); + $slugs = ipull($metadata, 'link'); + + // We have to make two queries here to distinguish between + // documents the user can't see, and documents that don't + // exist. + $visible_documents = id(new PhrictionDocumentQuery()) + ->setViewer($engine->getConfig('viewer')) + ->withSlugs($slugs) + ->needContent(true) + ->execute(); + $existant_documents = id(new PhrictionDocumentQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withSlugs($slugs) + ->execute(); + + $visible_documents = mpull($visible_documents, null, 'getSlug'); + $existant_documents = mpull($existant_documents, null, 'getSlug'); + + foreach ($metadata as $spec) { + $link = $spec['link']; + $name = $spec['explicitName']; + $class = 'phriction-link'; + + if (idx($existant_documents, $link) === null) { + // The target document doesn't exist. + if ($name === null) { + $name = explode('/', trim($link, '/')); + $name = end($name); + } + $class = 'phriction-link-missing'; + } else if (idx($visible_documents, $link) === null) { + // The document exists, but the user can't see it. + if ($name === null) { + $name = explode('/', trim($link, '/')); + $name = end($name); + } + $class = 'phriction-link-lock'; + } else { + if ($name === null) { + // Use the title of the document if no name is set. + $name = $visible_documents[$link] + ->getContent() + ->getTitle(); + } + } + + $uri = new PhutilURI($link); + $slug = $uri->getPath(); + $fragment = $uri->getFragment(); + $slug = PhabricatorSlug::normalize($slug); + $slug = PhrictionDocument::getSlugURI($slug); + $href = (string)id(new PhutilURI($slug))->setFragment($fragment); + + $text_mode = $this->getEngine()->isTextMode(); + $mail_mode = $this->getEngine()->isHTMLMailMode(); + + if ($this->getEngine()->getState('toc')) { + $text = $name; + } else if ($text_mode || $mail_mode) { + return PhabricatorEnv::getProductionURI($href); + } else { + if ($class === 'phriction-link-lock') { + $name = array( + $this->newTag( + 'i', + array( + 'class' => 'phui-icon-view phui-font-fa fa-lock', + ), + ''), + ' ', + $name, + ); + } + $text = $this->newTag( + 'a', + array( + 'href' => $href, + 'class' => $class, + ), + $name); + $this->getEngine()->overwriteStoredText($spec['token'], $text); + } + } } } diff --git a/webroot/rsrc/css/application/phriction/phriction-document-css.css b/webroot/rsrc/css/application/phriction/phriction-document-css.css index 95b9fb0d4e..7728c8dd77 100644 --- a/webroot/rsrc/css/application/phriction/phriction-document-css.css +++ b/webroot/rsrc/css/application/phriction/phriction-document-css.css @@ -36,3 +36,13 @@ .phui-document-content .phriction-link { font-weight: bold; } + +.phui-document-content .phriction-link-missing { + font-weight: bold; + color: {$red}; +} + +.phui-document-content .phriction-link-lock { + font-weight: bold; + color: {$greytext}; +} From b2d2f03dea7a6f0cb36d3e3370185a15bdf45397 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Sun, 17 Apr 2016 01:50:26 +0000 Subject: [PATCH 03/50] Tell users to avoid magical CloudFlare nonsense in the CDN documentation Summary: Fixes T9716. Doesn't go into too much detail, but will hopefully save some pain. Test Plan: Read all the wonderful text. Reviewers: #blessed_committers, epriestley, #blessed_reviewers Reviewed By: #blessed_committers, epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T9716 Differential Revision: https://secure.phabricator.com/D15738 --- .../user/configuration/configuring_file_domain.diviner | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/docs/user/configuration/configuring_file_domain.diviner b/src/docs/user/configuration/configuring_file_domain.diviner index d6e81bb6b1..6f7c410435 100644 --- a/src/docs/user/configuration/configuring_file_domain.diviner +++ b/src/docs/user/configuration/configuring_file_domain.diviner @@ -65,6 +65,13 @@ Continue to "Configuring Phabricator", below. Approach: CloudFlare ======== +WARNING: You should review all your CloudFlare settings, and be very +sure to turn off all JavaScript, HTML, CSS minification and +optimization features, including systems like "Rocket Loader". These +features will break Phabricator in strange and mysterious ways that +are unpredictable. Only allow CloudFlare to cache files, and never +optimize them. + [[ https://cloudflare.com | CloudFlare ]] is a general-purpose CDN service. To set up CloudFlare, you'll need to register a second domain and go through From 025b243e27419a3a471cae677fbcd8df019a7284 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 16 Apr 2016 08:48:05 -0700 Subject: [PATCH 04/50] Document wiki relative link syntax Summary: Also make `../` work to start relative a link so I don't have to document it as `./../path`. Test Plan: - Used `./`, `../`. `./../`, and normal links (proper title pickup). - Used bad links (red). - Regenerated documentation: {F1221692} Reviewers: hach-que Reviewed By: hach-que Differential Revision: https://secure.phabricator.com/D15734 --- .../markup/PhrictionRemarkupRule.php | 4 ++-- src/docs/user/userguide/remarkup.diviner | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index 99a9d1bdd4..2c9ab76284 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -19,7 +19,7 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { $link = trim($matches[1]); // Handle relative links. - if (substr($link, 0, 2) === './') { + if ((substr($link, 0, 2) === './') || (substr($link, 0, 3) === '../')) { $base = null; $context = $this->getEngine()->getConfig('contextObject'); if ($context !== null && $context instanceof PhrictionContent) { @@ -33,7 +33,7 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { } if ($base !== null) { $base_parts = explode('/', rtrim($base, '/')); - $rel_parts = explode('/', substr(rtrim($link, '/'), 2)); + $rel_parts = explode('/', rtrim($link, '/')); foreach ($rel_parts as $part) { if ($part === '.') { // Consume standalone dots in a relative path, and do diff --git a/src/docs/user/userguide/remarkup.diviner b/src/docs/user/userguide/remarkup.diviner index 379c09e5c3..606a9db3e7 100644 --- a/src/docs/user/userguide/remarkup.diviner +++ b/src/docs/user/userguide/remarkup.diviner @@ -512,11 +512,34 @@ You can link to Phriction documents with a name or path: Make sure you sign and date your [[legal/Letter of Marque and Reprisal]]! +By default, the link will render with the document title as the link name. With a pipe (`|`), you can retitle the link. Use this to mislead your opponents: Check out these [[legal/boring_documents/ | exciting legal documents]]! +Links to pages which do not exist are shown in red. Links to pages which exist +but which the viewer does not have permission to see are shown with a lock +icon, and the link will not disclose the page title. + +If you begin a link path with `./` or `../`, the remainder of the path will be +evaluated relative to the current wiki page. For example, if you are writing +content for the document `fruit/` a link to `[[./guava]]` is the same as a link +to `[[fruit/guava]]` from elsewhere. + +Relative links may use `../` to transverse up the document tree. From the +`produce/vegetables/` page, you can use `[[../fruit/guava]]` to link to the +`produce/fruit/guava` page. + +Relative links do not work when used outside of wiki pages. For example, +you can't use a relative link in a comment on a task, because there is no +reasonable place for the link to start resolving from. + +When documents are moved, relative links are not automatically updated: they +are preserved as currently written. After moving a document, you may need to +review and adjust any relative links it contains. + + = Literal Blocks = To place text in a literal block use `%%%`: From eef2172161166830bf4fcc7ba9f206c9e3b6a2ff Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 16 Apr 2016 18:49:37 -0700 Subject: [PATCH 05/50] When a user tries to regsiter while logged in, just send them home Summary: This error message is pointless and dead-ends logged-in users needlessly if they're sent to the register page by documentation or Advanced Enterprise Sales Funnels. Test Plan: Visited `/auth/register/` while logged in, was sent home. Reviewers: chad Reviewed By: chad Differential Revision: https://secure.phabricator.com/D15739 --- .../auth/controller/PhabricatorAuthRegisterController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/auth/controller/PhabricatorAuthRegisterController.php b/src/applications/auth/controller/PhabricatorAuthRegisterController.php index 27d54e4ad6..1d036b13a9 100644 --- a/src/applications/auth/controller/PhabricatorAuthRegisterController.php +++ b/src/applications/auth/controller/PhabricatorAuthRegisterController.php @@ -12,7 +12,7 @@ final class PhabricatorAuthRegisterController $account_key = $request->getURIData('akey'); if ($request->getUser()->isLoggedIn()) { - return $this->renderError(pht('You are already logged in.')); + return id(new AphrontRedirectResponse())->setURI('/'); } $is_setup = false; From adf42db5ea2dcc3b77f03f588142eac9e6820e48 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 16 Apr 2016 16:45:19 -0700 Subject: [PATCH 06/50] Trivially implement RepositoryEditEngine and API methods Summary: Ref T10748. Ref T10337. This technically implements this stuff, but it does not do anything useful yet. This skips all the hard stuff. Test Plan: - Technically used `diffusion.repository.search` to get repository information. - Technically used `diffusion.repository.edit` to change a repository name. - Used `editpro/` to edit a repository name. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10337, T10748 Differential Revision: https://secure.phabricator.com/D15736 --- src/__phutil_library_map__.php | 9 ++ .../PhabricatorDiffusionApplication.php | 4 +- ...iffusionRepositoryEditConduitAPIMethod.php | 20 +++++ ...fusionRepositorySearchConduitAPIMethod.php | 18 ++++ .../DiffusionRepositoryEditproController.php | 12 +++ .../editor/DiffusionRepositoryEditEngine.php | 84 +++++++++++++++++++ .../storage/PhabricatorRepository.php | 41 ++++++++- 7 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 src/applications/diffusion/conduit/DiffusionRepositoryEditConduitAPIMethod.php create mode 100644 src/applications/diffusion/conduit/DiffusionRepositorySearchConduitAPIMethod.php create mode 100644 src/applications/diffusion/controller/DiffusionRepositoryEditproController.php create mode 100644 src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index ac506ce0f5..0d12e403c4 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -750,16 +750,19 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php', 'DiffusionRepositoryEditBasicController' => 'applications/diffusion/controller/DiffusionRepositoryEditBasicController.php', 'DiffusionRepositoryEditBranchesController' => 'applications/diffusion/controller/DiffusionRepositoryEditBranchesController.php', + 'DiffusionRepositoryEditConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRepositoryEditConduitAPIMethod.php', 'DiffusionRepositoryEditController' => 'applications/diffusion/controller/DiffusionRepositoryEditController.php', 'DiffusionRepositoryEditDangerousController' => 'applications/diffusion/controller/DiffusionRepositoryEditDangerousController.php', 'DiffusionRepositoryEditDeleteController' => 'applications/diffusion/controller/DiffusionRepositoryEditDeleteController.php', 'DiffusionRepositoryEditEncodingController' => 'applications/diffusion/controller/DiffusionRepositoryEditEncodingController.php', + 'DiffusionRepositoryEditEngine' => 'applications/diffusion/editor/DiffusionRepositoryEditEngine.php', 'DiffusionRepositoryEditHostingController' => 'applications/diffusion/controller/DiffusionRepositoryEditHostingController.php', 'DiffusionRepositoryEditMainController' => 'applications/diffusion/controller/DiffusionRepositoryEditMainController.php', 'DiffusionRepositoryEditStagingController' => 'applications/diffusion/controller/DiffusionRepositoryEditStagingController.php', 'DiffusionRepositoryEditStorageController' => 'applications/diffusion/controller/DiffusionRepositoryEditStorageController.php', 'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php', 'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php', + 'DiffusionRepositoryEditproController' => 'applications/diffusion/controller/DiffusionRepositoryEditproController.php', 'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php', 'DiffusionRepositoryManageController' => 'applications/diffusion/controller/DiffusionRepositoryManageController.php', 'DiffusionRepositoryManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryManagementPanel.php', @@ -767,6 +770,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php', 'DiffusionRepositoryRef' => 'applications/diffusion/data/DiffusionRepositoryRef.php', 'DiffusionRepositoryRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryRemarkupRule.php', + 'DiffusionRepositorySearchConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRepositorySearchConduitAPIMethod.php', 'DiffusionRepositorySymbolsController' => 'applications/diffusion/controller/DiffusionRepositorySymbolsController.php', 'DiffusionRepositoryTag' => 'applications/diffusion/data/DiffusionRepositoryTag.php', 'DiffusionRepositoryTestAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php', @@ -4941,16 +4945,19 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditAutomationController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditBasicController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditBranchesController' => 'DiffusionRepositoryEditController', + 'DiffusionRepositoryEditConduitAPIMethod' => 'PhabricatorEditEngineAPIMethod', 'DiffusionRepositoryEditController' => 'DiffusionController', 'DiffusionRepositoryEditDangerousController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditDeleteController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditEncodingController' => 'DiffusionRepositoryEditController', + 'DiffusionRepositoryEditEngine' => 'PhabricatorEditEngine', 'DiffusionRepositoryEditHostingController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditMainController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditStagingController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditStorageController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController', + 'DiffusionRepositoryEditproController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryListController' => 'DiffusionController', 'DiffusionRepositoryManageController' => 'DiffusionController', 'DiffusionRepositoryManagementPanel' => 'Phobject', @@ -4958,6 +4965,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryPath' => 'Phobject', 'DiffusionRepositoryRef' => 'Phobject', 'DiffusionRepositoryRemarkupRule' => 'PhabricatorObjectRemarkupRule', + 'DiffusionRepositorySearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', 'DiffusionRepositorySymbolsController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryTag' => 'Phobject', 'DiffusionRepositoryTestAutomationController' => 'DiffusionRepositoryEditController', @@ -7747,6 +7755,7 @@ phutil_register_library_map(array( 'PhabricatorDestructibleInterface', 'PhabricatorProjectInterface', 'PhabricatorSpacesInterface', + 'PhabricatorConduitResultInterface', ), 'PhabricatorRepositoryAuditRequest' => array( 'PhabricatorRepositoryDAO', diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php index b65ec43937..775bdc203e 100644 --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -55,8 +55,10 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication { => 'DiffusionCommitController', '/diffusion/' => array( - '(?:query/(?P[^/]+)/)?' + $this->getQueryRoutePattern() => 'DiffusionRepositoryListController', + $this->getEditRoutePattern('editpro/') => + 'DiffusionRepositoryEditproController', 'new/' => 'DiffusionRepositoryNewController', '(?Pcreate)/' => 'DiffusionRepositoryCreateController', '(?Pimport)/' => 'DiffusionRepositoryCreateController', diff --git a/src/applications/diffusion/conduit/DiffusionRepositoryEditConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionRepositoryEditConduitAPIMethod.php new file mode 100644 index 0000000000..af4cc05bb8 --- /dev/null +++ b/src/applications/diffusion/conduit/DiffusionRepositoryEditConduitAPIMethod.php @@ -0,0 +1,20 @@ +setController($this) + ->buildResponse(); + } + +} diff --git a/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php b/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php new file mode 100644 index 0000000000..2bf99c1bc5 --- /dev/null +++ b/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php @@ -0,0 +1,84 @@ +getViewer(); + return PhabricatorRepository::initializeNewRepository($viewer); + } + + protected function newObjectQuery() { + return new PhabricatorRepositoryQuery(); + } + + protected function getObjectCreateTitleText($object) { + return pht('Create Repository'); + } + + protected function getObjectCreateButtonText($object) { + return pht('Create Repository'); + } + + protected function getObjectEditTitleText($object) { + return pht('Edit Repository: %s', $object->getName()); + } + + protected function getObjectEditShortText($object) { + return $object->getDisplayName(); + } + + protected function getObjectCreateShortText() { + return pht('Create Repository'); + } + + protected function getObjectName() { + return pht('Repository'); + } + + protected function getObjectViewURI($object) { + return $object->getPathURI('manage/'); + } + + protected function getCreateNewObjectPolicy() { + return $this->getApplication()->getPolicy( + DiffusionCreateRepositoriesCapability::CAPABILITY); + } + + protected function buildCustomEditFields($object) { + return array( + id(new PhabricatorTextEditField()) + ->setKey('name') + ->setLabel(pht('Name')) + ->setIsRequired(true) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_NAME) + ->setDescription(pht('The repository name.')) + ->setConduitDescription(pht('Rename the repository.')) + ->setConduitTypeDescription(pht('New repository name.')) + ->setValue($object->getName()), + ); + } + +} diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 22a28daaa3..9bcc740f83 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -13,7 +13,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO PhabricatorMarkupInterface, PhabricatorDestructibleInterface, PhabricatorProjectInterface, - PhabricatorSpacesInterface { + PhabricatorSpacesInterface, + PhabricatorConduitResultInterface { /** * Shortest hash we'll recognize in raw "a829f32" form. @@ -2626,4 +2627,42 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return $this->spacePHID; } +/* -( PhabricatorConduitResultInterface )---------------------------------- */ + + + public function getFieldSpecificationsForConduit() { + return array( + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('name') + ->setType('string') + ->setDescription(pht('The repository name.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('vcs') + ->setType('string') + ->setDescription( + pht('The VCS this repository uses ("git", "hg" or "svn").')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('callsign') + ->setType('string') + ->setDescription(pht('The repository callsign, if it has one.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('shortName') + ->setType('string') + ->setDescription(pht('Unique short name, if the repository has one.')), + ); + } + + public function getFieldValuesForConduit() { + return array( + 'name' => $this->getName(), + 'vcs' => $this->getVersionControlSystem(), + 'callsign' => $this->getCallsign(), + 'shortName' => $this->getRepositorySlug(), + ); + } + + public function getConduitSearchAttachments() { + return array(); + } + } From e582e9172bcd561034fe4af62406d5c50ff6aa2b Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 16 Apr 2016 17:33:27 -0700 Subject: [PATCH 07/50] Rough in basics + policies + history repository management panels Summary: Ref T10748. This is roughly where I'm headed, if it makes some kind of sense? The "Edit" links in sub-sections don't work yet since I haven't built the thing. Probably depends on D15736. Test Plan: Manually navigated to `/manage/`, clicked around. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10748 Differential Revision: https://secure.phabricator.com/D15737 --- src/__phutil_library_map__.php | 6 + .../DiffusionRepositoryManageController.php | 32 ++++- ...ffusionRepositoryBasicsManagementPanel.php | 122 ++++++++++++++++++ ...fusionRepositoryClusterManagementPanel.php | 2 +- ...fusionRepositoryHistoryManagementPanel.php | 21 +++ .../DiffusionRepositoryManagementPanel.php | 58 +++++++++ ...usionRepositoryPoliciesManagementPanel.php | 73 +++++++++++ 7 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php create mode 100644 src/applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php create mode 100644 src/applications/diffusion/management/DiffusionRepositoryPoliciesManagementPanel.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 0d12e403c4..3496529362 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -739,6 +739,7 @@ phutil_register_library_map(array( 'DiffusionRefTableController' => 'applications/diffusion/controller/DiffusionRefTableController.php', 'DiffusionRefsQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRefsQueryConduitAPIMethod.php', 'DiffusionRenameHistoryQuery' => 'applications/diffusion/query/DiffusionRenameHistoryQuery.php', + 'DiffusionRepositoryBasicsManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php', 'DiffusionRepositoryByIDRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryByIDRemarkupRule.php', 'DiffusionRepositoryClusterManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php', 'DiffusionRepositoryController' => 'applications/diffusion/controller/DiffusionRepositoryController.php', @@ -763,11 +764,13 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php', 'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php', 'DiffusionRepositoryEditproController' => 'applications/diffusion/controller/DiffusionRepositoryEditproController.php', + 'DiffusionRepositoryHistoryManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php', 'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php', 'DiffusionRepositoryManageController' => 'applications/diffusion/controller/DiffusionRepositoryManageController.php', 'DiffusionRepositoryManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryManagementPanel.php', 'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php', 'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php', + 'DiffusionRepositoryPoliciesManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryPoliciesManagementPanel.php', 'DiffusionRepositoryRef' => 'applications/diffusion/data/DiffusionRepositoryRef.php', 'DiffusionRepositoryRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryRemarkupRule.php', 'DiffusionRepositorySearchConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRepositorySearchConduitAPIMethod.php', @@ -4934,6 +4937,7 @@ phutil_register_library_map(array( 'DiffusionRefTableController' => 'DiffusionController', 'DiffusionRefsQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 'DiffusionRenameHistoryQuery' => 'Phobject', + 'DiffusionRepositoryBasicsManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRepositoryByIDRemarkupRule' => 'PhabricatorObjectRemarkupRule', 'DiffusionRepositoryClusterManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRepositoryController' => 'DiffusionController', @@ -4958,11 +4962,13 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditproController' => 'DiffusionRepositoryEditController', + 'DiffusionRepositoryHistoryManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRepositoryListController' => 'DiffusionController', 'DiffusionRepositoryManageController' => 'DiffusionController', 'DiffusionRepositoryManagementPanel' => 'Phobject', 'DiffusionRepositoryNewController' => 'DiffusionController', 'DiffusionRepositoryPath' => 'Phobject', + 'DiffusionRepositoryPoliciesManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRepositoryRef' => 'Phobject', 'DiffusionRepositoryRemarkupRule' => 'PhabricatorObjectRemarkupRule', 'DiffusionRepositorySearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', diff --git a/src/applications/diffusion/controller/DiffusionRepositoryManageController.php b/src/applications/diffusion/controller/DiffusionRepositoryManageController.php index d4ba97440d..aa34f8f909 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryManageController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryManageController.php @@ -30,7 +30,8 @@ final class DiffusionRepositoryManageController foreach ($panels as $panel) { $panel ->setViewer($viewer) - ->setRepository($repository); + ->setRepository($repository) + ->setController($this); } $selected = $request->getURIData('panel'); @@ -63,10 +64,30 @@ final class DiffusionRepositoryManageController $repository->getPathURI('manage/')); $crumbs->addTextCrumb($panel->getManagementPanelLabel()); + $header_text = pht( + '%s: %s', + $repository->getDisplayName(), + $panel->getManagementPanelLabel()); + + $header = id(new PHUIHeaderView()) + ->setHeader($header_text) + ->setHeaderIcon('fa-pencil'); + if ($repository->isTracked()) { + $header->setStatus('fa-check', 'bluegrey', pht('Active')); + } else { + $header->setStatus('fa-ban', 'dark', pht('Inactive')); + } + $view = id(new PHUITwoColumnView()) + ->setHeader($header) ->setNavigation($nav) ->setMainColumn($content); + $curtain = $panel->buildManagementPanelCurtain(); + if ($curtain) { + $view->setCurtain($curtain); + } + return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) @@ -95,5 +116,14 @@ final class DiffusionRepositoryManageController return $nav; } + public function newTimeline(PhabricatorRepository $repository) { + $timeline = $this->buildTransactionTimeline( + $repository, + new PhabricatorRepositoryTransactionQuery()); + $timeline->setShouldTerminate(true); + + return $timeline; + } + } diff --git a/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php new file mode 100644 index 0000000000..2810d0dc9a --- /dev/null +++ b/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php @@ -0,0 +1,122 @@ +getRepository(); + $viewer = $this->getViewer(); + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $repository, + PhabricatorPolicyCapability::CAN_EDIT); + + $edit_uri = $repository->getPathURI('manage/'); + $activate_uri = $repository->getPathURI('edit/activate/'); + $delete_uri = $repository->getPathURI('edit/delete/'); + + if ($repository->isTracked()) { + $activate_icon = 'fa-pause'; + $activate_label = pht('Deactivate Repository'); + } else { + $activate_icon = 'fa-play'; + $activate_label = pht('Activate Repository'); + } + + return array( + id(new PhabricatorActionView()) + ->setIcon('fa-pencil') + ->setName(pht('Edit Basic Information')) + ->setHref($edit_uri) + ->setDisabled(!$can_edit) + ->setWorkflow(!$can_edit), + id(new PhabricatorActionView()) + ->setHref($activate_uri) + ->setIcon($activate_icon) + ->setName($activate_label) + ->setDisabled(!$can_edit) + ->setWorkflow(true), + id(new PhabricatorActionView()) + ->setName(pht('Delete Repository')) + ->setIcon('fa-times') + ->setHref($delete_uri) + ->setDisabled(true) + ->setWorkflow(true), + ); + } + + public function buildManagementPanelContent() { + $result = array(); + + $result[] = $this->newBox(pht('Repository Basics'), $this->buildBasics()); + + $description = $this->buildDescription(); + if ($description) { + $result[] = $this->newBox(pht('Description'), $description); + } + + return $result; + } + + private function buildBasics() { + $repository = $this->getRepository(); + $viewer = $this->getViewer(); + + $view = id(new PHUIPropertyListView()) + ->setViewer($viewer) + ->setActionList($this->newActions()); + + $name = $repository->getName(); + $view->addProperty(pht('Name'), $name); + + $type = PhabricatorRepositoryType::getNameForRepositoryType( + $repository->getVersionControlSystem()); + $view->addProperty(pht('Type'), $type); + + $callsign = $repository->getCallsign(); + if (!strlen($callsign)) { + $callsign = phutil_tag('em', array(), pht('No Callsign')); + } + $view->addProperty(pht('Callsign'), $callsign); + + $short_name = $repository->getRepositorySlug(); + if ($short_name === null) { + $short_name = $repository->getCloneName(); + $short_name = phutil_tag('em', array(), $short_name); + } + $view->addProperty(pht('Short Name'), $short_name); + + return $view; + } + + + private function buildDescription() { + $repository = $this->getRepository(); + $viewer = $this->getViewer(); + + $description = $repository->getDetail('description'); + + $view = id(new PHUIPropertyListView()) + ->setViewer($viewer); + if (!strlen($description)) { + $description = phutil_tag('em', array(), pht('No description provided.')); + } else { + $description = new PHUIRemarkupView($viewer, $description); + } + $view->addTextContent($description); + + return $view; + } + +} diff --git a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php index 27991f62c7..5ab907924b 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php @@ -10,7 +10,7 @@ final class DiffusionRepositoryClusterManagementPanel } public function getManagementPanelOrder() { - return 12345; + return 600; } public function buildManagementPanelContent() { diff --git a/src/applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php new file mode 100644 index 0000000000..c98d3d76ca --- /dev/null +++ b/src/applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php @@ -0,0 +1,21 @@ +newTimeline(); + } + + +} diff --git a/src/applications/diffusion/management/DiffusionRepositoryManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryManagementPanel.php index b72411cd58..f278c3323a 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryManagementPanel.php @@ -5,6 +5,7 @@ abstract class DiffusionRepositoryManagementPanel private $viewer; private $repository; + private $controller; final public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; @@ -24,6 +25,11 @@ abstract class DiffusionRepositoryManagementPanel return $this->repository; } + final public function setController(PhabricatorController $controller) { + $this->controller = $controller; + return $this; + } + final public function getManagementPanelKey() { return $this->getPhobjectClassConstant('PANELKEY'); } @@ -32,6 +38,47 @@ abstract class DiffusionRepositoryManagementPanel abstract public function getManagementPanelOrder(); abstract public function buildManagementPanelContent(); + protected function buildManagementPanelActions() { + return array(); + } + + final protected function newActions() { + $actions = $this->buildManagementPanelActions(); + if (!$actions) { + return null; + } + + $viewer = $this->getViewer(); + + $action_list = id(new PhabricatorActionListView()) + ->setViewer($viewer); + + foreach ($actions as $action) { + $action_list->addAction($action); + } + + return $action_list; + } + + public function buildManagementPanelCurtain() { + // TODO: Delete or fix this, curtains always render in the left gutter + // at the moment. + return null; + + $actions = $this->newActions(); + if (!$actions) { + return null; + } + + $viewer = $this->getViewer(); + + $curtain = id(new PHUICurtainView()) + ->setViewer($viewer) + ->setActionList($actions); + + return $curtain; + } + public static function getAllPanels() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) @@ -40,4 +87,15 @@ abstract class DiffusionRepositoryManagementPanel ->execute(); } + final protected function newBox($header_text, $body) { + return id(new PHUIObjectBoxView()) + ->setHeaderText($header_text) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->appendChild($body); + } + + final protected function newTimeline() { + return $this->controller->newTimeline($this->getRepository()); + } + } diff --git a/src/applications/diffusion/management/DiffusionRepositoryPoliciesManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryPoliciesManagementPanel.php new file mode 100644 index 0000000000..9bfe572070 --- /dev/null +++ b/src/applications/diffusion/management/DiffusionRepositoryPoliciesManagementPanel.php @@ -0,0 +1,73 @@ +getRepository(); + $viewer = $this->getViewer(); + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $repository, + PhabricatorPolicyCapability::CAN_EDIT); + + $edit_uri = $repository->getPathURI('manage/'); + + return array( + id(new PhabricatorActionView()) + ->setIcon('fa-pencil') + ->setName(pht('Edit Policies')) + ->setHref($edit_uri) + ->setDisabled(!$can_edit) + ->setWorkflow(!$can_edit), + ); + } + + public function buildManagementPanelContent() { + $repository = $this->getRepository(); + $viewer = $this->getViewer(); + + $view = id(new PHUIPropertyListView()) + ->setViewer($viewer) + ->setActionList($this->newActions()); + + $descriptions = PhabricatorPolicyQuery::renderPolicyDescriptions( + $viewer, + $repository); + + $view_parts = array(); + if (PhabricatorSpacesNamespaceQuery::getViewerSpacesExist($viewer)) { + $space_phid = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $repository); + $view_parts[] = $viewer->renderHandle($space_phid); + } + $view_parts[] = $descriptions[PhabricatorPolicyCapability::CAN_VIEW]; + + $view->addProperty( + pht('Visible To'), + phutil_implode_html(" \xC2\xB7 ", $view_parts)); + + $view->addProperty( + pht('Editable By'), + $descriptions[PhabricatorPolicyCapability::CAN_EDIT]); + + $pushable = $repository->isHosted() + ? $descriptions[DiffusionPushCapability::CAPABILITY] + : phutil_tag('em', array(), pht('Not a Hosted Repository')); + $view->addProperty(pht('Pushable By'), $pushable); + + return $this->newBox(pht('Policies'), $view); + } + +} From 51838f990f8a3e0a9539d03eccbcd456d3b926e0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 17 Apr 2016 05:10:18 -0700 Subject: [PATCH 08/50] Copy repository status to a management panel Summary: Ref T10748. Pretty straightforward. I'd like to put a little "!" icon in the menu if there's a warning/error eventually, but can deal with that latre. Test Plan: {F1223096} Reviewers: chad Reviewed By: chad Maniphest Tasks: T10748 Differential Revision: https://secure.phabricator.com/D15741 --- src/__phutil_library_map__.php | 2 + ...ffusionRepositoryBasicsManagementPanel.php | 13 + ...fusionRepositoryClusterManagementPanel.php | 1 + ...ffusionRepositoryStatusManagementPanel.php | 457 ++++++++++++++++++ 4 files changed, 473 insertions(+) create mode 100644 src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 3496529362..f8ea620f85 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -774,6 +774,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryRef' => 'applications/diffusion/data/DiffusionRepositoryRef.php', 'DiffusionRepositoryRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryRemarkupRule.php', 'DiffusionRepositorySearchConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionRepositorySearchConduitAPIMethod.php', + 'DiffusionRepositoryStatusManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php', 'DiffusionRepositorySymbolsController' => 'applications/diffusion/controller/DiffusionRepositorySymbolsController.php', 'DiffusionRepositoryTag' => 'applications/diffusion/data/DiffusionRepositoryTag.php', 'DiffusionRepositoryTestAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php', @@ -4972,6 +4973,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryRef' => 'Phobject', 'DiffusionRepositoryRemarkupRule' => 'PhabricatorObjectRemarkupRule', 'DiffusionRepositorySearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', + 'DiffusionRepositoryStatusManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRepositorySymbolsController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryTag' => 'Phobject', 'DiffusionRepositoryTestAutomationController' => 'DiffusionRepositoryEditController', diff --git a/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php index 2810d0dc9a..d8614923ce 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php @@ -25,6 +25,7 @@ final class DiffusionRepositoryBasicsManagementPanel $edit_uri = $repository->getPathURI('manage/'); $activate_uri = $repository->getPathURI('edit/activate/'); $delete_uri = $repository->getPathURI('edit/delete/'); + $encoding_uri = $repository->getPathURI('edit/encoding/'); if ($repository->isTracked()) { $activate_icon = 'fa-pause'; @@ -41,6 +42,12 @@ final class DiffusionRepositoryBasicsManagementPanel ->setHref($edit_uri) ->setDisabled(!$can_edit) ->setWorkflow(!$can_edit), + id(new PhabricatorActionView()) + ->setIcon('fa-text-width') + ->setName(pht('Edit Text Encoding')) + ->setHref($encoding_uri) + ->setDisabled(!$can_edit) + ->setWorkflow(!$can_edit), id(new PhabricatorActionView()) ->setHref($activate_uri) ->setIcon($activate_icon) @@ -97,6 +104,12 @@ final class DiffusionRepositoryBasicsManagementPanel } $view->addProperty(pht('Short Name'), $short_name); + $encoding = $repository->getDetail('encoding'); + if (!$encoding) { + $encoding = phutil_tag('em', array(), pht('Use Default (UTF-8)')); + } + $view->addProperty(pht('Encoding'), $encoding); + return $view; } diff --git a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php index 5ab907924b..2017871453 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php @@ -160,6 +160,7 @@ final class DiffusionRepositoryClusterManagementPanel return id(new PHUIObjectBoxView()) ->setHeader($header) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setTable($table); } diff --git a/src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php new file mode 100644 index 0000000000..db31e0d71e --- /dev/null +++ b/src/applications/diffusion/management/DiffusionRepositoryStatusManagementPanel.php @@ -0,0 +1,457 @@ +getRepository(); + $viewer = $this->getViewer(); + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $repository, + PhabricatorPolicyCapability::CAN_EDIT); + + $update_uri = $repository->getPathURI('edit/update/'); + + return array( + id(new PhabricatorActionView()) + ->setIcon('fa-refresh') + ->setName(pht('Update Now')) + ->setWorkflow(true) + ->setDisabled(!$can_edit) + ->setHref($update_uri), + ); + } + + public function buildManagementPanelContent() { + $repository = $this->getRepository(); + $viewer = $this->getViewer(); + + $view = id(new PHUIPropertyListView()) + ->setViewer($viewer) + ->setActionList($this->newActions()); + + $view->addProperty( + pht('Update Frequency'), + $this->buildRepositoryUpdateInterval($repository)); + + + list($status, $raw_error) = $this->buildRepositoryStatus($repository); + + $view->addProperty(pht('Status'), $status); + if ($raw_error) { + $view->addSectionHeader(pht('Raw Error')); + $view->addTextContent($raw_error); + } + + return $this->newBox(pht('Status'), $view); + } + + private function buildRepositoryUpdateInterval( + PhabricatorRepository $repository) { + + $smart_wait = $repository->loadUpdateInterval(); + + $doc_href = PhabricatorEnv::getDoclink( + 'Diffusion User Guide: Repository Updates'); + + return array( + phutil_format_relative_time_detailed($smart_wait), + " \xC2\xB7 ", + phutil_tag( + 'a', + array( + 'href' => $doc_href, + 'target' => '_blank', + ), + pht('Learn More')), + ); + } + + private function buildRepositoryStatus( + PhabricatorRepository $repository) { + + $viewer = $this->getViewer(); + $is_cluster = $repository->getAlmanacServicePHID(); + + $view = new PHUIStatusListView(); + + $messages = id(new PhabricatorRepositoryStatusMessage()) + ->loadAllWhere('repositoryID = %d', $repository->getID()); + $messages = mpull($messages, null, 'getStatusType'); + + if ($repository->isTracked()) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Repository Active'))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'bluegrey') + ->setTarget(pht('Repository Inactive')) + ->setNote( + pht('Activate this repository to begin or resume import.'))); + return $view; + } + + $binaries = array(); + $svnlook_check = false; + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $binaries[] = 'git'; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $binaries[] = 'svn'; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + $binaries[] = 'hg'; + break; + } + + if ($repository->isHosted()) { + if ($repository->getServeOverHTTP() != PhabricatorRepository::SERVE_OFF) { + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $binaries[] = 'git-http-backend'; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $binaries[] = 'svnserve'; + $binaries[] = 'svnadmin'; + $binaries[] = 'svnlook'; + $svnlook_check = true; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + $binaries[] = 'hg'; + break; + } + } + if ($repository->getServeOverSSH() != PhabricatorRepository::SERVE_OFF) { + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $binaries[] = 'git-receive-pack'; + $binaries[] = 'git-upload-pack'; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $binaries[] = 'svnserve'; + $binaries[] = 'svnadmin'; + $binaries[] = 'svnlook'; + $svnlook_check = true; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + $binaries[] = 'hg'; + break; + } + } + } + + $binaries = array_unique($binaries); + if (!$is_cluster) { + // We're only checking for binaries if we aren't running with a cluster + // configuration. In theory, we could check for binaries on the + // repository host machine, but we'd need to make this more complicated + // to do that. + + foreach ($binaries as $binary) { + $where = Filesystem::resolveBinary($binary); + if (!$where) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget( + pht('Missing Binary %s', phutil_tag('tt', array(), $binary))) + ->setNote(pht( + "Unable to find this binary in the webserver's PATH. You may ". + "need to configure %s.", + $this->getEnvConfigLink()))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget( + pht('Found Binary %s', phutil_tag('tt', array(), $binary))) + ->setNote(phutil_tag('tt', array(), $where))); + } + } + + // This gets checked generically above. However, for svn commit hooks, we + // need this to be in environment.append-paths because subversion strips + // PATH. + if ($svnlook_check) { + $where = Filesystem::resolveBinary('svnlook'); + if ($where) { + $path = substr($where, 0, strlen($where) - strlen('svnlook')); + $dirs = PhabricatorEnv::getEnvConfig('environment.append-paths'); + $in_path = false; + foreach ($dirs as $dir) { + if (Filesystem::isDescendant($path, $dir)) { + $in_path = true; + break; + } + } + if (!$in_path) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget( + pht('Missing Binary %s', phutil_tag('tt', array(), $binary))) + ->setNote(pht( + 'Unable to find this binary in `%s`. '. + 'You need to configure %s and include %s.', + 'environment.append-paths', + $this->getEnvConfigLink(), + $path))); + } + } + } + } + + $doc_href = PhabricatorEnv::getDocLink('Managing Daemons with phd'); + + $daemon_instructions = pht( + 'Use %s to start daemons. See %s.', + phutil_tag('tt', array(), 'bin/phd start'), + phutil_tag( + 'a', + array( + 'href' => $doc_href, + ), + pht('Managing Daemons with phd'))); + + + $pull_daemon = id(new PhabricatorDaemonLogQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE) + ->withDaemonClasses(array('PhabricatorRepositoryPullLocalDaemon')) + ->setLimit(1) + ->execute(); + + if ($pull_daemon) { + + // TODO: In a cluster environment, we need a daemon on this repository's + // host, specifically, and we aren't checking for that right now. This + // is a reasonable proxy for things being more-or-less correctly set up, + // though. + + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Pull Daemon Running'))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Pull Daemon Not Running')) + ->setNote($daemon_instructions)); + } + + + $task_daemon = id(new PhabricatorDaemonLogQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE) + ->withDaemonClasses(array('PhabricatorTaskmasterDaemon')) + ->setLimit(1) + ->execute(); + if ($task_daemon) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Task Daemon Running'))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Task Daemon Not Running')) + ->setNote($daemon_instructions)); + } + + + if ($is_cluster) { + // Just omit this status check for now in cluster environments. We + // could make a service call and pull it from the repository host + // eventually. + } else if ($repository->usesLocalWorkingCopy()) { + $local_parent = dirname($repository->getLocalPath()); + if (Filesystem::pathExists($local_parent)) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Storage Directory OK')) + ->setNote(phutil_tag('tt', array(), $local_parent))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('No Storage Directory')) + ->setNote( + pht( + 'Storage directory %s does not exist, or is not readable by '. + 'the webserver. Create this directory or make it readable.', + phutil_tag('tt', array(), $local_parent)))); + return $view; + } + + $local_path = $repository->getLocalPath(); + $message = idx($messages, PhabricatorRepositoryStatusMessage::TYPE_INIT); + if ($message) { + switch ($message->getStatusCode()) { + case PhabricatorRepositoryStatusMessage::CODE_ERROR: + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Initialization Error')) + ->setNote($message->getParameter('message'))); + return $view; + case PhabricatorRepositoryStatusMessage::CODE_OKAY: + if (Filesystem::pathExists($local_path)) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Working Copy OK')) + ->setNote(phutil_tag('tt', array(), $local_path))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Working Copy Error')) + ->setNote( + pht( + 'Working copy %s has been deleted, or is not '. + 'readable by the webserver. Make this directory '. + 'readable. If it has been deleted, the daemons should '. + 'restore it automatically.', + phutil_tag('tt', array(), $local_path)))); + return $view; + } + break; + case PhabricatorRepositoryStatusMessage::CODE_WORKING: + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_CLOCK, 'green') + ->setTarget(pht('Initializing Working Copy')) + ->setNote(pht('Daemons are initializing the working copy.'))); + return $view; + default: + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Unknown Init Status')) + ->setNote($message->getStatusCode())); + return $view; + } + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_CLOCK, 'orange') + ->setTarget(pht('No Working Copy Yet')) + ->setNote( + pht('Waiting for daemons to build a working copy.'))); + return $view; + } + } + + $raw_error = null; + + $message = idx($messages, PhabricatorRepositoryStatusMessage::TYPE_FETCH); + if ($message) { + switch ($message->getStatusCode()) { + case PhabricatorRepositoryStatusMessage::CODE_ERROR: + $message = $message->getParameter('message'); + + $suggestion = null; + if (preg_match('/Permission denied \(publickey\)./', $message)) { + $suggestion = pht( + 'Public Key Error: This error usually indicates that the '. + 'keypair you have configured does not have permission to '. + 'access the repository.'); + } + + $raw_error = $message; + + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_WARNING, 'red') + ->setTarget(pht('Update Error')) + ->setNote($suggestion)); + return $view; + case PhabricatorRepositoryStatusMessage::CODE_OKAY: + $ago = (PhabricatorTime::getNow() - $message->getEpoch()); + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Updates OK')) + ->setNote( + pht( + 'Last updated %s (%s ago).', + phabricator_datetime($message->getEpoch(), $viewer), + phutil_format_relative_time_detailed($ago)))); + break; + } + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_CLOCK, 'orange') + ->setTarget(pht('Waiting For Update')) + ->setNote( + pht('Waiting for daemons to read updates.'))); + } + + if ($repository->isImporting()) { + $ratio = $repository->loadImportProgress(); + $percentage = sprintf('%.2f%%', 100 * $ratio); + + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_CLOCK, 'green') + ->setTarget(pht('Importing')) + ->setNote( + pht('%s Complete', $percentage))); + } else { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') + ->setTarget(pht('Fully Imported'))); + } + + if (idx($messages, PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE)) { + $view->addItem( + id(new PHUIStatusItemView()) + ->setIcon(PHUIStatusItemView::ICON_UP, 'indigo') + ->setTarget(pht('Prioritized')) + ->setNote(pht('This repository will be updated soon!'))); + } + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $repository, + PhabricatorPolicyCapability::CAN_EDIT); + + if ($raw_error !== null) { + if (!$can_edit) { + $raw_message = pht( + 'You must be able to edit a repository to see raw error messages '. + 'because they sometimes disclose sensitive information.'); + $raw_message = phutil_tag('em', array(), $raw_message); + } else { + $raw_message = phutil_escape_html_newlines($raw_error); + } + } else { + $raw_message = null; + } + + return array($view, $raw_message); + } + + +} From 92c50de8aac5cd21e9c9bd6041002b68b5e72bed Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 17 Apr 2016 08:17:49 -0700 Subject: [PATCH 09/50] Rough in the new custom URI panel Summary: Ref T10748. Ref T10366. No support for editing and no impact on the UI, but get some of the basics in place. Test Plan: {F1223279} Reviewers: chad Reviewed By: chad Maniphest Tasks: T10366, T10748 Differential Revision: https://secure.phabricator.com/D15742 --- src/__phutil_library_map__.php | 4 + ...DiffusionRepositoryURIsManagementPanel.php | 126 ++++++++ .../storage/PhabricatorRepository.php | 94 ++++++ .../storage/PhabricatorRepositoryURI.php | 300 ++++++++++++++++++ .../user/userguide/diffusion_uris.diviner | 47 +++ 5 files changed, 571 insertions(+) create mode 100644 src/applications/diffusion/management/DiffusionRepositoryURIsManagementPanel.php create mode 100644 src/applications/repository/storage/PhabricatorRepositoryURI.php create mode 100644 src/docs/user/userguide/diffusion_uris.diviner diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index f8ea620f85..4361f12911 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -779,6 +779,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryTag' => 'applications/diffusion/data/DiffusionRepositoryTag.php', 'DiffusionRepositoryTestAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php', 'DiffusionRepositoryURIsIndexEngineExtension' => 'applications/diffusion/engineextension/DiffusionRepositoryURIsIndexEngineExtension.php', + 'DiffusionRepositoryURIsManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryURIsManagementPanel.php', 'DiffusionRequest' => 'applications/diffusion/request/DiffusionRequest.php', 'DiffusionResolveRefsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionResolveRefsConduitAPIMethod.php', 'DiffusionResolveUserQuery' => 'applications/diffusion/query/DiffusionResolveUserQuery.php', @@ -3216,6 +3217,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryTransaction' => 'applications/repository/storage/PhabricatorRepositoryTransaction.php', 'PhabricatorRepositoryTransactionQuery' => 'applications/repository/query/PhabricatorRepositoryTransactionQuery.php', 'PhabricatorRepositoryType' => 'applications/repository/constants/PhabricatorRepositoryType.php', + 'PhabricatorRepositoryURI' => 'applications/repository/storage/PhabricatorRepositoryURI.php', 'PhabricatorRepositoryURIIndex' => 'applications/repository/storage/PhabricatorRepositoryURIIndex.php', 'PhabricatorRepositoryURINormalizer' => 'applications/repository/data/PhabricatorRepositoryURINormalizer.php', 'PhabricatorRepositoryURINormalizerTestCase' => 'applications/repository/data/__tests__/PhabricatorRepositoryURINormalizerTestCase.php', @@ -4978,6 +4980,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryTag' => 'Phobject', 'DiffusionRepositoryTestAutomationController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryURIsIndexEngineExtension' => 'PhabricatorIndexEngineExtension', + 'DiffusionRepositoryURIsManagementPanel' => 'DiffusionRepositoryManagementPanel', 'DiffusionRequest' => 'Phobject', 'DiffusionResolveRefsConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 'DiffusionResolveUserQuery' => 'Phobject', @@ -7875,6 +7878,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryTransaction' => 'PhabricatorApplicationTransaction', 'PhabricatorRepositoryTransactionQuery' => 'PhabricatorApplicationTransactionQuery', 'PhabricatorRepositoryType' => 'Phobject', + 'PhabricatorRepositoryURI' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryURIIndex' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryURINormalizer' => 'Phobject', 'PhabricatorRepositoryURINormalizerTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/diffusion/management/DiffusionRepositoryURIsManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryURIsManagementPanel.php new file mode 100644 index 0000000000..3535c64bbc --- /dev/null +++ b/src/applications/diffusion/management/DiffusionRepositoryURIsManagementPanel.php @@ -0,0 +1,126 @@ +getRepository(); + $viewer = $this->getViewer(); + + $repository->attachURIs(array()); + $uris = $repository->getURIs(); + + Javelin::initBehavior('phabricator-tooltips'); + $rows = array(); + foreach ($uris as $uri) { + + $uri_name = $uri->getDisplayURI(); + + if ($uri->getIsDisabled()) { + $status_icon = 'fa-times grey'; + } else { + $status_icon = 'fa-check green'; + } + + $uri_status = id(new PHUIIconView())->setIcon($status_icon); + + switch ($uri->getEffectiveIOType()) { + case PhabricatorRepositoryURI::IO_OBSERVE: + $io_icon = 'fa-download green'; + $io_label = pht('Observe'); + break; + case PhabricatorRepositoryURI::IO_MIRROR: + $io_icon = 'fa-upload green'; + $io_label = pht('Mirror'); + break; + case PhabricatorRepositoryURI::IO_NONE: + $io_icon = 'fa-times grey'; + $io_label = pht('No I/O'); + break; + case PhabricatorRepositoryURI::IO_READ: + $io_icon = 'fa-folder blue'; + $io_label = pht('Read Only'); + break; + case PhabricatorRepositoryURI::IO_READWRITE: + $io_icon = 'fa-folder-open blue'; + $io_label = pht('Read/Write'); + break; + } + + $uri_io = array( + id(new PHUIIconView())->setIcon($io_icon), + ' ', + $io_label, + ); + + switch ($uri->getEffectiveDisplayType()) { + case PhabricatorRepositoryURI::DISPLAY_NEVER: + $display_icon = 'fa-eye-slash grey'; + $display_label = pht('Hidden'); + break; + case PhabricatorRepositoryURI::DISPLAY_ALWAYS: + $display_icon = 'fa-eye green'; + $display_label = pht('Visible'); + break; + } + + $uri_display = array( + id(new PHUIIconView())->setIcon($display_icon), + ' ', + $display_label, + ); + + $rows[] = array( + $uri_status, + $uri_name, + $uri_io, + $uri_display, + ); + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString(pht('This repository has no URIs.')) + ->setHeaders( + array( + null, + pht('URI'), + pht('I/O'), + pht('Display'), + )) + ->setColumnClasses( + array( + null, + 'pri wide', + null, + null, + )); + + $doc_href = PhabricatorEnv::getDoclink( + 'Diffusion User Guide: Repository URIs'); + + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Repository URIs')) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + return id(new PHUIObjectBoxView()) + ->setHeader($header) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->setTable($table); + } + +} diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 9bcc740f83..3c298fb286 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -63,10 +63,12 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private $commitCount = self::ATTACHABLE; private $mostRecentCommit = self::ATTACHABLE; private $projectPHIDs = self::ATTACHABLE; + private $uris = self::ATTACHABLE; private $clusterWriteLock; private $clusterWriteVersion; + public static function initializeNewRepository(PhabricatorUser $actor) { $app = id(new PhabricatorApplicationQuery()) ->setViewer($actor) @@ -2266,6 +2268,98 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return $client; } +/* -( Repository URIs )---------------------------------------------------- */ + + + public function attachURIs(array $uris) { + $custom_map = array(); + foreach ($uris as $key => $uri) { + $builtin_key = $uri->getRepositoryURIBuiltinKey(); + if ($builtin_key !== null) { + $custom_map[$builtin_key] = $key; + } + } + + $builtin_uris = $this->newBuiltinURIs(); + $seen_builtins = array(); + foreach ($builtin_uris as $builtin_uri) { + $builtin_key = $builtin_uri->getRepositoryURIBuiltinKey(); + $seen_builtins[$builtin_key] = true; + + // If this builtin URI is disabled, don't attach it and remove the + // persisted version if it exists. + if ($builtin_uri->getIsDisabled()) { + if (isset($custom_map[$builtin_key])) { + unset($uris[$custom_map[$builtin_key]]); + } + continue; + } + + // If we don't have a persisted version of the URI, add the builtin + // version. + if (empty($custom_map[$builtin_key])) { + $uris[] = $builtin_uri; + } + } + + // Remove any builtins which no longer exist. + foreach ($custom_map as $builtin_key => $key) { + if (empty($seen_builtins[$builtin_key])) { + unset($uris[$key]); + } + } + + $this->uris = $uris; + + return $this; + } + + public function getURIs() { + return $this->assertAttached($this->uris); + } + + protected function newBuiltinURIs() { + $has_callsign = ($this->getCallsign() !== null); + $has_shortname = ($this->getRepositorySlug() !== null); + + $identifier_map = array( + PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_CALLSIGN => $has_callsign, + PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_SHORTNAME => $has_shortname, + PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_ID => true, + ); + + $allow_http = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth'); + + $base_uri = PhabricatorEnv::getURI('/'); + $base_uri = new PhutilURI($base_uri); + $has_https = ($base_uri->getProtocol() == 'https'); + $has_https = ($has_https && $allow_http); + + $has_http = !PhabricatorEnv::getEnvConfig('security.require-https'); + $has_http = ($has_http && $allow_http); + + // TODO: Maybe allow users to disable this by default somehow? + $has_ssh = true; + + $protocol_map = array( + PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH => $has_ssh, + PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTPS => $has_https, + PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTP => $has_http, + ); + + $uris = array(); + foreach ($protocol_map as $protocol => $proto_supported) { + foreach ($identifier_map as $identifier => $id_supported) { + $uris[] = PhabricatorRepositoryURI::initializeNewURI($this) + ->setBuiltinProtocol($protocol) + ->setBuiltinIdentifier($identifier) + ->setIsDisabled(!$proto_supported || !$id_supported); + } + } + + return $uris; + } + /* -( Cluster Synchronization )-------------------------------------------- */ diff --git a/src/applications/repository/storage/PhabricatorRepositoryURI.php b/src/applications/repository/storage/PhabricatorRepositoryURI.php new file mode 100644 index 0000000000..c91861918f --- /dev/null +++ b/src/applications/repository/storage/PhabricatorRepositoryURI.php @@ -0,0 +1,300 @@ + true, + self::CONFIG_COLUMN_SCHEMA => array( + 'uri' => 'text', + 'builtinProtocol' => 'text32?', + 'builtinIdentifier' => 'text32?', + 'ioType' => 'text32', + 'displayType' => 'text32', + 'isDisabled' => 'bool', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_builtin' => array( + 'columns' => array( + 'repositoryPHID', + 'builtinProtocol', + 'builtinIdentifier', + ), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + public static function initializeNewURI(PhabricatorRepository $repository) { + return id(new self()) + ->attachRepository($repository) + ->setRepositoryPHID($repository->getPHID()) + ->setIoType(self::IO_DEFAULT) + ->setDisplayType(self::DISPLAY_DEFAULT) + ->setIsDisabled(0); + } + + public function attachRepository(PhabricatorRepository $repository) { + $this->repository = $repository; + return $this; + } + + public function getRepository() { + return $this->assertAttached($this->repository); + } + + public function getRepositoryURIBuiltinKey() { + if (!$this->getBuiltinProtocol()) { + return null; + } + + $parts = array( + $this->getBuiltinProtocol(), + $this->getBuiltinIdentifier(), + ); + return implode('.', $parts); + } + + public function isBuiltin() { + return (bool)$this->getBuiltinProtocol(); + } + + public function getEffectiveDisplayType() { + $display = $this->getDisplayType(); + + if ($display != self::IO_DEFAULT) { + return $display; + } + + switch ($this->getEffectiveIOType()) { + case self::IO_MIRROR: + case self::IO_OBSERVE: + return self::DISPLAY_NEVER; + case self::IO_NONE: + if ($this->isBuiltin()) { + return self::DISPLAY_NEVER; + } else { + return self::DISPLAY_ALWAYS; + } + case self::IO_READ: + case self::IO_READWRITE: + // By default, only show the "best" version of the builtin URI, not the + // other redundant versions. + if ($this->isBuiltin()) { + $repository = $this->getRepository(); + $other_uris = $repository->getURIs(); + + $identifier_value = array( + self::BUILTIN_IDENTIFIER_CALLSIGN => 3, + self::BUILTIN_IDENTIFIER_SHORTNAME => 2, + self::BUILTIN_IDENTIFIER_ID => 1, + ); + + $have_identifiers = array(); + foreach ($other_uris as $other_uri) { + if ($other_uri->getIsDisabled()) { + continue; + } + + $identifier = $other_uri->getBuiltinIdentifier(); + if (!$identifier) { + continue; + } + + $have_identifiers[$identifier] = $identifier_value[$identifier]; + } + + $best_identifier = max($have_identifiers); + $this_identifier = $identifier_value[$this->getBuiltinIdentifier()]; + + if ($this_identifier < $best_identifier) { + return self::DISPLAY_NEVER; + } + } + + return self::DISPLAY_ALWAYS; + } + } + + + public function getEffectiveIOType() { + $io = $this->getIoType(); + + if ($io != self::IO_DEFAULT) { + return $io; + } + + if ($this->isBuiltin()) { + $repository = $this->getRepository(); + $other_uris = $repository->getURIs(); + + $any_observe = false; + foreach ($other_uris as $other_uri) { + if ($other_uri->getIoType() == self::IO_OBSERVE) { + $any_observe = true; + break; + } + } + + if ($any_observe) { + return self::IO_READ; + } else { + return self::IO_READWRITE; + } + } + + return self::IO_IGNORE; + } + + + public function getDisplayURI() { + $uri = new PhutilURI($this->getURI()); + + $protocol = $this->getForcedProtocol(); + if ($protocol) { + $uri->setProtocol($protocol); + } + + $user = $this->getForcedUser(); + if ($user) { + $uri->setUser($user); + } + + $host = $this->getForcedHost(); + if ($host) { + $uri->setDomain($host); + } + + $port = $this->getForcedPort(); + if ($port) { + $uri->setPort($port); + } + + $path = $this->getForcedPath(); + if ($path) { + $uri->setPath($path); + } + + return $uri; + } + + private function getForcedProtocol() { + switch ($this->getBuiltinProtocol()) { + case self::BUILTIN_PROTOCOL_SSH: + return 'ssh'; + case self::BUILTIN_PROTOCOL_HTTP: + return 'http'; + case self::BUILTIN_PROTOCOL_HTTPS: + return 'https'; + default: + return null; + } + } + + private function getForcedUser() { + switch ($this->getBuiltinProtocol()) { + case self::BUILTIN_PROTOCOL_SSH: + return PhabricatorEnv::getEnvConfig('diffusion.ssh-user'); + default: + return null; + } + } + + private function getForcedHost() { + $phabricator_uri = PhabricatorEnv::getURI('/'); + $phabricator_uri = new PhutilURI($phabricator_uri); + + $phabricator_host = $phabricator_uri->getDomain(); + + switch ($this->getBuiltinProtocol()) { + case self::BUILTIN_PROTOCOL_SSH: + $ssh_host = PhabricatorEnv::getEnvConfig('diffusion.ssh-host'); + if ($ssh_host !== null) { + return $ssh_host; + } + return $phabricator_host; + case self::BUILTIN_PROTOCOL_HTTP: + case self::BUILTIN_PROTOCOL_HTTPS: + return $phabricator_host; + default: + return null; + } + } + + private function getForcedPort() { + switch ($this->getBuiltinProtocol()) { + case self::BUILTIN_PROTOCOL_SSH: + return PhabricatorEnv::getEnvConfig('diffusion.ssh-port'); + case self::BUILTIN_PROTOCOL_HTTP: + case self::BUILTIN_PROTOCOL_HTTPS: + default: + return null; + } + } + + private function getForcedPath() { + if (!$this->isBuiltin()) { + return null; + } + + $repository = $this->getRepository(); + + $id = $repository->getID(); + $callsign = $repository->getCallsign(); + $short_name = $repository->getRepositorySlug(); + + $clone_name = $repository->getCloneName(); + + if ($repository->isGit()) { + $suffix = '.git'; + } else if ($repository->isHg()) { + $suffix = '/'; + } else { + $suffix = ''; + } + + switch ($this->getBuiltinIdentifier()) { + case self::BUILTIN_IDENTIFIER_ID: + return "/diffusion/{$id}/{$clone_name}{$suffix}"; + case self::BUILTIN_IDENTIFIER_SHORTNAME: + return "/source/{$short_name}{$suffix}"; + case self::BUILTIN_IDENTIFIER_CALLSIGN: + return "/diffusion/{$callsign}/{$clone_name}{$suffix}"; + default: + return null; + } + } + +} diff --git a/src/docs/user/userguide/diffusion_uris.diviner b/src/docs/user/userguide/diffusion_uris.diviner new file mode 100644 index 0000000000..3ff122e85e --- /dev/null +++ b/src/docs/user/userguide/diffusion_uris.diviner @@ -0,0 +1,47 @@ +@title Diffusion User Guide: URIs +@group userguide + +Guide to configuring repository URIs for fetching, cloning and mirroring. + +Overview +======== + +WARNING: This document describes a feature which is still under development, +and is not necessarily accurate or complete. + +Phabricator can host, observe, mirror, and proxy repositories. For example, +here are some supported use cases: + +**Host Repositories**: Phabricator can host repositories locally. Phabricator +maintains the writable master version of the repository, and you can push and +pull the repository. This is the most straightforward kind of repository +configuration, and similar to repositories on other services like GitHub or +Bitbucket. + +**Observe Repositories**: Phabricator can create a copy of an repository which +is hosted elsewhere (like GitHub or Bitbucket) and track updates to the remote +repository. This will create a read-only copy of the repository in Phabricator. + +**Mirror Repositories**: Phabricator can publish any repository to mirrors, +updating the mirrors as changes are made to the repository. This works with +both local hosted repositories and remote repositories that Phabricator is +observing. + +**Proxy Repositories**: If you are observing a repository, you can allow users +to read Phabricator's copy of the repository. Phabricator supports granular +read permissions, so this can let you open a private repository up a little +bit in a flexible way. + +**Import Repositories**: If you have a repository elsewhere that you want to +host on Phabricator, you can observe the remote repository first, then turn +the tracking off once the repository fully synchronizes. This allows you to +copy an existing repository and begin hosting it in Phabricator. + +You can also import repositories by creating an empty hosted repository and +then pushing everything to the repository directly. + +You configure the behavior of a Phabricator repository by adding and +configuring URIs and marking them to be fetched from, mirrored to, clonable, +and so on. By configuring all the URIs that a repository should interact with +and expose to users, you configure the read, write, and mirroring behavior +of the repository. From fbfe7304521464151986aee99d71fa07b20e0768 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 16 Apr 2016 18:17:13 -0700 Subject: [PATCH 10/50] Support more transactions types in RepositoryEditEngine Summary: Ref T10748. This supports more transaction types in the modern editor and improves validation so Conduit benefits. You can technically create repositories via `diffusion.repository.edit` now, although they aren't very useful. Test Plan: - Used `diffusion.repository.edit` to create and edit repositories. - Used `/editpro/` to edit repositories. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10748 Differential Revision: https://secure.phabricator.com/D15740 --- .../DiffusionRepositoryCreateController.php | 8 +- ...fusionRepositoryEditActivateController.php | 9 +- .../editor/DiffusionRepositoryEditEngine.php | 88 ++++++++++++ .../constants/PhabricatorRepositoryType.php | 8 +- .../editor/PhabricatorRepositoryEditor.php | 131 +++++++++++++++--- .../storage/PhabricatorRepository.php | 49 ++++++- .../PhabricatorRepositoryTransaction.php | 30 ++-- 7 files changed, 284 insertions(+), 39 deletions(-) diff --git a/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php b/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php index ae28bf3993..09fdd23565 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php @@ -162,9 +162,15 @@ final class DiffusionRepositoryCreateController $activate = $form->getPage('done') ->getControl('activate')->getValue(); + if ($activate == 'start') { + $initial_status = PhabricatorRepository::STATUS_ACTIVE; + } else { + $initial_status = PhabricatorRepository::STATUS_INACTIVE; + } + $xactions[] = id(clone $template) ->setTransactionType($type_activate) - ->setNewValue(($activate == 'start')); + ->setNewValue($initial_status); if ($service) { $xactions[] = id(clone $template) diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditActivateController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditActivateController.php index c8428833ee..55caaa59b9 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryEditActivateController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryEditActivateController.php @@ -16,12 +16,19 @@ final class DiffusionRepositoryEditActivateController $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/'); if ($request->isFormPost()) { + if (!$repository->isTracked()) { + $new_status = PhabricatorRepository::STATUS_ACTIVE; + } else { + $new_status = PhabricatorRepository::STATUS_INACTIVE; + } + $xaction = id(new PhabricatorRepositoryTransaction()) ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_ACTIVATE) - ->setNewValue(!$repository->isTracked()); + ->setNewValue($new_status); $editor = id(new PhabricatorRepositoryEditor()) ->setContinueOnNoEffect(true) + ->setContinueOnMissingFields(true) ->setContentSourceFromRequest($request) ->setActor($viewer) ->applyTransactions($repository, array($xaction)); diff --git a/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php b/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php index 2bf99c1bc5..602a3059d6 100644 --- a/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php +++ b/src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php @@ -68,7 +68,28 @@ final class DiffusionRepositoryEditEngine } protected function buildCustomEditFields($object) { + $viewer = $this->getViewer(); + + $policies = id(new PhabricatorPolicyQuery()) + ->setViewer($viewer) + ->setObject($object) + ->execute(); + return array( + id(new PhabricatorSelectEditField()) + ->setKey('vcs') + ->setLabel(pht('Version Control System')) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_VCS) + ->setIsConduitOnly(true) + ->setIsCopyable(true) + ->setOptions(PhabricatorRepositoryType::getAllRepositoryTypes()) + ->setDescription(pht('Underlying repository version control system.')) + ->setConduitDescription( + pht( + 'Choose which version control system to use when creating a '. + 'repository.')) + ->setConduitTypeDescription(pht('Version control system selection.')) + ->setValue($object->getVersionControlSystem()), id(new PhabricatorTextEditField()) ->setKey('name') ->setLabel(pht('Name')) @@ -78,6 +99,73 @@ final class DiffusionRepositoryEditEngine ->setConduitDescription(pht('Rename the repository.')) ->setConduitTypeDescription(pht('New repository name.')) ->setValue($object->getName()), + id(new PhabricatorTextEditField()) + ->setKey('callsign') + ->setLabel(pht('Callsign')) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_CALLSIGN) + ->setDescription(pht('The repository callsign.')) + ->setConduitDescription(pht('Change the repository callsign.')) + ->setConduitTypeDescription(pht('New repository callsign.')) + ->setValue($object->getCallsign()), + id(new PhabricatorTextEditField()) + ->setKey('shortName') + ->setLabel(pht('Short Name')) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_SLUG) + ->setDescription(pht('Short, unique repository name.')) + ->setConduitDescription(pht('Change the repository short name.')) + ->setConduitTypeDescription(pht('New short name for the repository.')) + ->setValue($object->getRepositorySlug()), + id(new PhabricatorRemarkupEditField()) + ->setKey('description') + ->setLabel(pht('Description')) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_DESCRIPTION) + ->setDescription(pht('Repository description.')) + ->setConduitDescription(pht('Change the repository description.')) + ->setConduitTypeDescription(pht('New repository description.')) + ->setValue($object->getDetail('description')), + id(new PhabricatorTextEditField()) + ->setKey('encoding') + ->setLabel(pht('Text Encoding')) + ->setIsCopyable(true) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_ENCODING) + ->setDescription(pht('Default text encoding.')) + ->setConduitDescription(pht('Change the default text encoding.')) + ->setConduitTypeDescription(pht('New text encoding.')) + ->setValue($object->getDetail('encoding')), + id(new PhabricatorSelectEditField()) + ->setKey('status') + ->setLabel(pht('Status')) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_ACTIVATE) + ->setIsConduitOnly(true) + ->setOptions(PhabricatorRepository::getStatusNameMap()) + ->setDescription(pht('Active or inactive status.')) + ->setConduitDescription(pht('Active or deactivate the repository.')) + ->setConduitTypeDescription(pht('New repository status.')) + ->setValue($object->getStatus()), + id(new PhabricatorTextEditField()) + ->setKey('defaultBranch') + ->setLabel(pht('Default Branch')) + ->setTransactionType( + PhabricatorRepositoryTransaction::TYPE_DEFAULT_BRANCH) + ->setIsCopyable(true) + ->setDescription(pht('Default branch name.')) + ->setConduitDescription(pht('Set the default branch name.')) + ->setConduitTypeDescription(pht('New default branch name.')) + ->setValue($object->getDetail('default-branch')), + id(new PhabricatorPolicyEditField()) + ->setKey('policy.push') + ->setLabel(pht('Push Policy')) + ->setAliases(array('push')) + ->setIsCopyable(true) + ->setCapability(DiffusionPushCapability::CAPABILITY) + ->setPolicies($policies) + ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY) + ->setDescription( + pht('Controls who can push changes to the repository.')) + ->setConduitDescription( + pht('Change the push policy of the repository.')) + ->setConduitTypeDescription(pht('New policy PHID or constant.')) + ->setValue($object->getPolicy(DiffusionPushCapability::CAPABILITY)), ); } diff --git a/src/applications/repository/constants/PhabricatorRepositoryType.php b/src/applications/repository/constants/PhabricatorRepositoryType.php index 557d77ac07..4861641411 100644 --- a/src/applications/repository/constants/PhabricatorRepositoryType.php +++ b/src/applications/repository/constants/PhabricatorRepositoryType.php @@ -7,10 +7,10 @@ final class PhabricatorRepositoryType extends Phobject { const REPOSITORY_TYPE_MERCURIAL = 'hg'; public static function getAllRepositoryTypes() { - static $map = array( - self::REPOSITORY_TYPE_GIT => 'Git', - self::REPOSITORY_TYPE_SVN => 'Subversion', - self::REPOSITORY_TYPE_MERCURIAL => 'Mercurial', + $map = array( + self::REPOSITORY_TYPE_GIT => pht('Git'), + self::REPOSITORY_TYPE_MERCURIAL => pht('Mercurial'), + self::REPOSITORY_TYPE_SVN => pht('Subversion'), ); return $map; } diff --git a/src/applications/repository/editor/PhabricatorRepositoryEditor.php b/src/applications/repository/editor/PhabricatorRepositoryEditor.php index fa585a826a..5de7f068e7 100644 --- a/src/applications/repository/editor/PhabricatorRepositoryEditor.php +++ b/src/applications/repository/editor/PhabricatorRepositoryEditor.php @@ -248,25 +248,7 @@ final class PhabricatorRepositoryEditor $object->setCallsign($xaction->getNewValue()); return; case PhabricatorRepositoryTransaction::TYPE_ENCODING: - // Make sure the encoding is valid by converting to UTF-8. This tests - // that the user has mbstring installed, and also that they didn't type - // a garbage encoding name. Note that we're converting from UTF-8 to - // the target encoding, because mbstring is fine with converting from - // a nonsense encoding. - $encoding = $xaction->getNewValue(); - if (strlen($encoding)) { - try { - phutil_utf8_convert('.', $encoding, 'UTF-8'); - } catch (Exception $ex) { - throw new PhutilProxyException( - pht( - "Error setting repository encoding '%s': %s'", - $encoding, - $ex->getMessage()), - $ex); - } - } - $object->setDetail('encoding', $encoding); + $object->setDetail('encoding', $xaction->getNewValue()); break; } } @@ -461,6 +443,117 @@ final class PhabricatorRepositoryEditor } break; + case PhabricatorRepositoryTransaction::TYPE_VCS: + $vcs_map = PhabricatorRepositoryType::getAllRepositoryTypes(); + $current_vcs = $object->getVersionControlSystem(); + + if (!$this->getIsNewObject()) { + foreach ($xactions as $xaction) { + if ($xaction->getNewValue() == $current_vcs) { + continue; + } + + $errors[] = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Immutable'), + pht( + 'You can not change the version control system an existing '. + 'repository uses. It can only be set when a repository is '. + 'first created.'), + $xaction); + } + } else { + $value = $object->getVersionControlSystem(); + foreach ($xactions as $xaction) { + $value = $xaction->getNewValue(); + + if (empty($vcs_map[$value])) { + $errors[] = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Invalid'), + pht( + 'Specified version control system must be a VCS '. + 'recognized by Phabricator: %s.', + implode(', ', array_keys($vcs_map))), + $xaction); + } + } + + if (!strlen($value)) { + $error = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Required'), + pht( + 'When creating a repository, you must specify a valid '. + 'underlying version control system: %s.', + implode(', ', array_keys($vcs_map))), + nonempty(last($xactions), null)); + $error->setIsMissingFieldError(true); + $errors[] = $error; + } + } + break; + + case PhabricatorRepositoryTransaction::TYPE_NAME: + $missing = $this->validateIsEmptyTextField( + $object->getName(), + $xactions); + + if ($missing) { + $error = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Required'), + pht('Repository name is required.'), + nonempty(last($xactions), null)); + + $error->setIsMissingFieldError(true); + $errors[] = $error; + } + break; + + case PhabricatorRepositoryTransaction::TYPE_ACTIVATE: + $status_map = PhabricatorRepository::getStatusMap(); + foreach ($xactions as $xaction) { + $status = $xaction->getNewValue(); + if (empty($status_map[$status])) { + $errors[] = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Invalid'), + pht( + 'Repository status "%s" is not valid.', + $status), + $xaction); + } + } + break; + + case PhabricatorRepositoryTransaction::TYPE_ENCODING: + foreach ($xactions as $xaction) { + // Make sure the encoding is valid by converting to UTF-8. This tests + // that the user has mbstring installed, and also that they didn't + // type a garbage encoding name. Note that we're converting from + // UTF-8 to the target encoding, because mbstring is fine with + // converting from a nonsense encoding. + $encoding = $xaction->getNewValue(); + if (!strlen($encoding)) { + continue; + } + + try { + phutil_utf8_convert('.', $encoding, 'UTF-8'); + } catch (Exception $ex) { + $errors[] = new PhabricatorApplicationTransactionValidationError( + $type, + pht('Invalid'), + pht( + 'Repository encoding "%s" is not valid: %s', + $encoding, + $ex->getMessage()), + $xaction); + } + } + break; + case PhabricatorRepositoryTransaction::TYPE_SLUG: foreach ($xactions as $xaction) { $old = $xaction->getOldValue(); diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 3c298fb286..18e45359ce 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -46,6 +46,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO const BECAUSE_BRANCH_NOT_AUTOCLOSE = 'auto/noclose'; const BECAUSE_AUTOCLOSE_FORCED = 'auto/forced'; + const STATUS_ACTIVE = 'active'; + const STATUS_INACTIVE = 'inactive'; + protected $name; protected $callsign; protected $repositorySlug; @@ -132,6 +135,31 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO PhabricatorRepositoryRepositoryPHIDType::TYPECONST); } + public static function getStatusMap() { + return array( + self::STATUS_ACTIVE => array( + 'name' => pht('Active'), + 'isTracked' => 1, + ), + self::STATUS_INACTIVE => array( + 'name' => pht('Inactive'), + 'isTracked' => 0, + ), + ); + } + + public static function getStatusNameMap() { + return ipull(self::getStatusMap(), 'name'); + } + + public function getStatus() { + if ($this->isTracked()) { + return self::STATUS_ACTIVE; + } else { + return self::STATUS_INACTIVE; + } + } + public function toDictionary() { return array( 'id' => $this->getID(), @@ -146,7 +174,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO 'isActive' => $this->isTracked(), 'isHosted' => $this->isHosted(), 'isImporting' => $this->isImporting(), - 'encoding' => $this->getDetail('encoding'), + 'encoding' => $this->getDefaultTextEncoding(), 'staging' => array( 'supported' => $this->supportsStaging(), 'prefix' => 'phabricator', @@ -155,6 +183,10 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO ); } + public function getDefaultTextEncoding() { + return $this->getDetail('encoding', 'UTF-8'); + } + public function getMonogram() { $callsign = $this->getCallsign(); if (strlen($callsign)) { @@ -994,7 +1026,20 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } public function isTracked() { - return $this->getDetail('tracking-enabled', false); + $status = $this->getDetail('tracking-enabled'); + $map = self::getStatusMap(); + $spec = idx($map, $status); + + if (!$spec) { + if ($status) { + $status = self::STATUS_ACTIVE; + } else { + $status = self::STATUS_INACTIVE; + } + $spec = idx($map, $status); + } + + return (bool)idx($spec, 'isTracked', false); } public function getDefaultBranch() { diff --git a/src/applications/repository/storage/PhabricatorRepositoryTransaction.php b/src/applications/repository/storage/PhabricatorRepositoryTransaction.php index 20c975cb33..76e75a70b0 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryTransaction.php +++ b/src/applications/repository/storage/PhabricatorRepositoryTransaction.php @@ -4,24 +4,17 @@ final class PhabricatorRepositoryTransaction extends PhabricatorApplicationTransaction { const TYPE_VCS = 'repo:vcs'; - const TYPE_ACTIVATE = 'repo:activate'; - const TYPE_NAME = 'repo:name'; - const TYPE_DESCRIPTION = 'repo:description'; - const TYPE_ENCODING = 'repo:encoding'; + const TYPE_ACTIVATE = 'repo:activate'; + const TYPE_NAME = 'repo:name'; + const TYPE_DESCRIPTION = 'repo:description'; + const TYPE_ENCODING = 'repo:encoding'; const TYPE_DEFAULT_BRANCH = 'repo:default-branch'; const TYPE_TRACK_ONLY = 'repo:track-only'; const TYPE_AUTOCLOSE_ONLY = 'repo:autoclose-only'; const TYPE_SVN_SUBPATH = 'repo:svn-subpath'; - const TYPE_UUID = 'repo:uuid'; const TYPE_NOTIFY = 'repo:notify'; const TYPE_AUTOCLOSE = 'repo:autoclose'; - const TYPE_REMOTE_URI = 'repo:remote-uri'; - const TYPE_LOCAL_PATH = 'repo:local-path'; - const TYPE_HOSTING = 'repo:hosting'; - const TYPE_PROTOCOL_HTTP = 'repo:serve-http'; - const TYPE_PROTOCOL_SSH = 'repo:serve-ssh'; const TYPE_PUSH_POLICY = 'repo:push-policy'; - const TYPE_CREDENTIAL = 'repo:credential'; const TYPE_DANGEROUS = 'repo:dangerous'; const TYPE_SLUG = 'repo:slug'; const TYPE_SERVICE = 'repo:service'; @@ -37,6 +30,13 @@ final class PhabricatorRepositoryTransaction const TYPE_SSH_KEYFILE = 'repo:ssh-keyfile'; const TYPE_HTTP_LOGIN = 'repo:http-login'; const TYPE_HTTP_PASS = 'repo:http-pass'; + const TYPE_CREDENTIAL = 'repo:credential'; + const TYPE_PROTOCOL_HTTP = 'repo:serve-http'; + const TYPE_PROTOCOL_SSH = 'repo:serve-ssh'; + const TYPE_HOSTING = 'repo:hosting'; + const TYPE_LOCAL_PATH = 'repo:local-path'; + const TYPE_REMOTE_URI = 'repo:remote-uri'; + const TYPE_UUID = 'repo:uuid'; public function getApplicationName() { return 'repository'; @@ -134,7 +134,13 @@ final class PhabricatorRepositoryTransaction '%s created this repository.', $this->renderHandleLink($author_phid)); case self::TYPE_ACTIVATE: - if ($new) { + // TODO: Old versions of this transaction use a boolean value, but + // should be migrated. + $is_deactivate = + (!$new) || + ($new == PhabricatorRepository::STATUS_INACTIVE); + + if (!$is_deactivate) { return pht( '%s activated this repository.', $this->renderHandleLink($author_phid)); From 9352ed8abbd2d2aeb782db87d74695667bc1bc9c Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 04:44:45 -0700 Subject: [PATCH 11/50] Add missing RepositoryURI table + run storage adjustments in tests Summary: Fixes T10830. Ref T10366. I wasn't writing to this table yet so I didn't build it, but the fact that `bin/storage adjust` would complain slipped my mind. - Add the table. - Make the tests run `adjust`. This is a little slow (a few extra seconds) but we could eventually move some steps like this to run server-side only. Test Plan: Ran `bin/storage upgrade -f`, got a clean `adjust`. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10366, T10830 Differential Revision: https://secure.phabricator.com/D15744 --- resources/sql/autopatches/20160418.repouri.1.sql | 14 ++++++++++++++ .../storage/PhabricatorRepositoryURI.php | 2 +- .../PhabricatorStorageFixtureScopeGuard.php | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 resources/sql/autopatches/20160418.repouri.1.sql diff --git a/resources/sql/autopatches/20160418.repouri.1.sql b/resources/sql/autopatches/20160418.repouri.1.sql new file mode 100644 index 0000000000..89f48b4291 --- /dev/null +++ b/resources/sql/autopatches/20160418.repouri.1.sql @@ -0,0 +1,14 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_uri ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + repositoryPHID VARBINARY(64) NOT NULL, + uri VARCHAR(255) NOT NULL COLLATE {$COLLATE_TEXT}, + builtinProtocol VARCHAR(32) COLLATE {$COLLATE_TEXT}, + builtinIdentifier VARCHAR(32) COLLATE {$COLLATE_TEXT}, + ioType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + displayType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + isDisabled BOOL NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_builtin` (repositoryPHID, builtinProtocol, builtinIdentifier) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/applications/repository/storage/PhabricatorRepositoryURI.php b/src/applications/repository/storage/PhabricatorRepositoryURI.php index c91861918f..9a93e261e4 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryURI.php +++ b/src/applications/repository/storage/PhabricatorRepositoryURI.php @@ -37,7 +37,7 @@ final class PhabricatorRepositoryURI return array( self::CONFIG_AUX_PHID => true, self::CONFIG_COLUMN_SCHEMA => array( - 'uri' => 'text', + 'uri' => 'text255', 'builtinProtocol' => 'text32?', 'builtinIdentifier' => 'text32?', 'ioType' => 'text32', diff --git a/src/infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php b/src/infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php index 2624427dc8..3a10f77bc7 100644 --- a/src/infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php +++ b/src/infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php @@ -11,7 +11,7 @@ final class PhabricatorStorageFixtureScopeGuard extends Phobject { $this->name = $name; execx( - 'php %s upgrade --force --no-adjust --namespace %s', + 'php %s upgrade --force --namespace %s', $this->getStorageBinPath(), $this->name); From d844e5112794421d55484da0d540120aa2e09421 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 05:38:09 -0700 Subject: [PATCH 12/50] Warn users about remote code execution in older Git Summary: Ref T10832. Raise a setup warning for out-of-date versions of `git`. Test Plan: {F1224632} Reviewers: chad Reviewed By: chad Maniphest Tasks: T10832 Differential Revision: https://secure.phabricator.com/D15745 --- .../check/PhabricatorBinariesSetupCheck.php | 126 +++++++++--------- 1 file changed, 60 insertions(+), 66 deletions(-) diff --git a/src/applications/config/check/PhabricatorBinariesSetupCheck.php b/src/applications/config/check/PhabricatorBinariesSetupCheck.php index c33dcae36f..c3c0740cfa 100644 --- a/src/applications/config/check/PhabricatorBinariesSetupCheck.php +++ b/src/applications/config/check/PhabricatorBinariesSetupCheck.php @@ -102,15 +102,24 @@ final class PhabricatorBinariesSetupCheck extends PhabricatorSetupCheck { $version = null; switch ($vcs['versionControlSystem']) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $minimum_version = null; - $bad_versions = array(); + $bad_versions = array( + '< 2.7.4' => pht( + 'Prior to 2.7.4, Git contains two remote code execution '. + 'vulnerabilities which allow an attacker to take control of a '. + 'system by crafting a commit which affects very long paths, '. + 'then pushing it or tricking a victim into fetching it. This '. + 'is a severe security vulnerability.'), + ); list($err, $stdout, $stderr) = exec_manual('git --version'); $version = trim(substr($stdout, strlen('git version '))); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - $minimum_version = '1.5'; $bad_versions = array( - '1.7.1' => pht( + // We need 1.5 for "--depth", see T7228. + '< 1.5' => pht( + 'The minimum supported version of Subversion is 1.5, which '. + 'was released in 2008.'), + '= 1.7.1' => pht( 'This version of Subversion has a bug where `%s` does not work '. 'for files added in rN (Subversion issue #2873), fixed in 1.7.2.', 'svn diff -c N'), @@ -119,12 +128,15 @@ final class PhabricatorBinariesSetupCheck extends PhabricatorSetupCheck { $version = trim($stdout); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - $minimum_version = '1.9'; $bad_versions = array( - '2.1' => pht( + // We need 1.9 for HTTP cloning, see T3046. + '< 1.9' => pht( + 'The minimum supported version of Mercurial is 1.9, which was '. + 'released in 2011.'), + '= 2.1' => pht( 'This version of Mercurial returns a bad exit code '. 'after a successful pull.'), - '2.2' => pht( + '= 2.2' => pht( 'This version of Mercurial has a significant memory leak, fixed '. 'in 2.2.1. Pushing fails with this version as well; see %s.', 'T3046#54922'), @@ -136,20 +148,25 @@ final class PhabricatorBinariesSetupCheck extends PhabricatorSetupCheck { if ($version === null) { $this->raiseUnknownVersionWarning($binary); } else { - if ($minimum_version && - version_compare($version, $minimum_version, '<')) { - $this->raiseMinimumVersionWarning( - $binary, - $minimum_version, - $version); + $version_details = array(); + + foreach ($bad_versions as $spec => $details) { + list($operator, $bad_version) = explode(' ', $spec, 2); + $is_bad = version_compare($version, $bad_version, $operator); + if ($is_bad) { + $version_details[] = pht( + '(%s%s) %s', + $operator, + $bad_version, + $details); + } } - foreach ($bad_versions as $bad_version => $details) { - if ($bad_version === $version) { - $this->raiseBadVersionWarning( - $binary, - $bad_version); - } + if ($version_details) { + $this->raiseBadVersionWarning( + $binary, + $version, + $version_details); } } } @@ -223,57 +240,34 @@ final class PhabricatorBinariesSetupCheck extends PhabricatorSetupCheck { pht('Report this Issue to the Upstream')); } - private function raiseMinimumVersionWarning( - $binary, - $minimum_version, - $version) { + private function raiseBadVersionWarning($binary, $version, array $problems) { + $summary = pht( + 'This server has a known bad version of "%s".', + $binary); - switch ($binary) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - $summary = pht( - "The '%s' binary is version %s and Phabricator requires version ". - "%s or higher.", - $binary, - $version, - $minimum_version); - $message = pht( - "Please upgrade the '%s' binary to a more modern version.", - $binary); - $this->newIssue('bin.'.$binary) - ->setShortName(pht("Unsupported '%s' Version", $binary)) - ->setName(pht("Unsupported '%s' Version", $binary)) - ->setSummary($summary) - ->setMessage($summary.' '.$message); - break; - } - } + $message = array(); - private function raiseBadVersionWarning($binary, $bad_version) { - switch ($binary) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - $summary = pht( - "The '%s' binary is version %s which has bugs that break ". - "Phabricator.", - $binary, - $bad_version); - $message = pht( - "Please upgrade the '%s' binary to a more modern version.", - $binary); - $this->newIssue('bin.'.$binary) - ->setShortName(pht("Unsupported '%s' Version", $binary)) - ->setName(pht("Unsupported '%s' Version", $binary)) - ->setSummary($summary) - ->setMessage($summary.' '.$message); - break; - } + $message[] = pht( + 'This server has a known bad version of "%s" installed ("%s"). This '. + 'version is not supported, or contains important bugs or security '. + 'vulnerabilities which are fixed in a newer version.', + $binary, + $version); + $message[] = pht('You should upgrade this software.'); + $message[] = pht('The known issues with this old version are:'); + + foreach ($problems as $problem) { + $message[] = $problem; + } + + $message = implode("\n\n", $message); + + $this->newIssue("bin.{$binary}.bad-version") + ->setName(pht('Unsupported/Insecure "%s" Version', $binary)) + ->setSummary($summary) + ->setMessage($message); } } From 595f203816913dd31d45ad608813cdcc9ede2f60 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 08:03:11 -0700 Subject: [PATCH 13/50] Correct RepositoryURI schema and propagate `adjust` exit code correctly Summary: Fixes T10830. - The return code from `storage adjust` did not propagate correct. - There was one column issue which I missed the first time around because I had a bunch of unrelated stuff locally. Test Plan: - Ran `bin/storage upgrade -f` with failures, used `echo $?` to make sure it exited nonzero. - Got fully clean `bin/storage adjust` by dropping all my extra local tables. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10830 Differential Revision: https://secure.phabricator.com/D15746 --- resources/sql/autopatches/20160418.repouri.2.sql | 2 ++ .../repository/storage/PhabricatorRepositoryURI.php | 1 + .../workflow/PhabricatorStorageManagementWorkflow.php | 6 ++++-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 resources/sql/autopatches/20160418.repouri.2.sql diff --git a/resources/sql/autopatches/20160418.repouri.2.sql b/resources/sql/autopatches/20160418.repouri.2.sql new file mode 100644 index 0000000000..03884a3dfc --- /dev/null +++ b/resources/sql/autopatches/20160418.repouri.2.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_repository.repository_uri + ADD credentialPHID VARBINARY(64); diff --git a/src/applications/repository/storage/PhabricatorRepositoryURI.php b/src/applications/repository/storage/PhabricatorRepositoryURI.php index 9a93e261e4..163eafa120 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryURI.php +++ b/src/applications/repository/storage/PhabricatorRepositoryURI.php @@ -40,6 +40,7 @@ final class PhabricatorRepositoryURI 'uri' => 'text255', 'builtinProtocol' => 'text32?', 'builtinIdentifier' => 'text32?', + 'credentialPHID' => 'phid?', 'ioType' => 'text32', 'displayType' => 'text32', 'isDisabled' => 'bool', diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php index b500599956..7f969793ec 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php @@ -61,7 +61,7 @@ abstract class PhabricatorStorageManagementWorkflow } } - $this->didExecute($args); + return $this->didExecute($args); } public function didExecute(PhutilArgumentParser $args) {} @@ -81,13 +81,15 @@ abstract class PhabricatorStorageManagementWorkflow $lock = $this->lock(); try { - $this->doAdjustSchemata($unsafe); + $err = $this->doAdjustSchemata($unsafe); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); + + return $err; } final private function doAdjustSchemata($unsafe) { From 368d2d1ddb1140545c1ebc4e852351ea5b445ab2 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 08:16:22 -0700 Subject: [PATCH 14/50] Improve robustness of cluster version bookkeeping Summary: Ref T4292. Small fixes: - There was a bug with the //first// write, where we'd write 1 but expect 0. Fix this. - Narrow the window where we hold the `isWriting` lock: we don't need to wait for the client to finish. - Release the lock even if something throws. - Use a more useful variable name. Test Plan: - Made new writes to a fresh cluster repository. - Made sequential writes. - Made concurrent writes. - Made good writes and bad writes. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15747 --- .../DiffusionGitReceivePackSSHWorkflow.php | 43 +++++++++++++------ .../storage/PhabricatorRepository.php | 3 -- ...habricatorRepositoryWorkingCopyVersion.php | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php index b138a2ef7d..9babe3ebb1 100644 --- a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php @@ -21,22 +21,30 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { if ($this->shouldProxy()) { $command = $this->getProxyCommand(); - $is_proxy = true; + $did_synchronize = false; } else { $command = csprintf('git-receive-pack %s', $repository->getLocalPath()); - $is_proxy = false; + $did_synchronize = true; $repository->synchronizeWorkingCopyBeforeWrite(); } - $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); - $future = id(new ExecFuture('%C', $command)) - ->setEnv($this->getEnvironment()); + $caught = null; + try { + $err = $this->executeRepositoryCommand($command); + } catch (Exception $ex) { + $caught = $ex; + } - $err = $this->newPassthruCommand() - ->setIOChannel($this->getIOChannel()) - ->setCommandChannelFromExecFuture($future) - ->execute(); + // We've committed the write (or rejected it), so we can release the lock + // without waiting for the client to receive the acknowledgement. + if ($did_synchronize) { + $repository->synchronizeWorkingCopyAfterWrite(); + } + + if ($caught) { + throw $caught; + } if (!$err) { $repository->writeStatusMessage( @@ -45,11 +53,20 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { $this->waitForGitClient(); } - if (!$is_proxy) { - $repository->synchronizeWorkingCopyAfterWrite(); - } - return $err; } + private function executeRepositoryCommand($command) { + $repository = $this->getRepository(); + $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); + + $future = id(new ExecFuture('%C', $command)) + ->setEnv($this->getEnvironment()); + + return $this->newPassthruCommand() + ->setIOChannel($this->getIOChannel()) + ->setCommandChannelFromExecFuture($future) + ->execute(); + } + } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 18e45359ce..ee7f6b6bb6 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -2410,9 +2410,6 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private function shouldEnableSynchronization() { - // TODO: This mostly works, but isn't stable enough for production yet. - return false; - $device = AlmanacKeys::getLiveDevice(); if (!$device) { return false; diff --git a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php index 00e74a3d61..1b1150f8cd 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php +++ b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php @@ -82,7 +82,7 @@ final class PhabricatorRepositoryWorkingCopyVersion $table, $repository_phid, $device_phid, - 1, + 0, 1); } From f424f9f2d206da118b8dec8c8b3a92d5687cf50f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 08:26:40 -0700 Subject: [PATCH 15/50] Record more details about where a write is taking place while holding a cluster lock Summary: Ref T4292. This will let the UI and future `bin/repository` tools give administrators more tools to understand problems when reporting or resolving them. Test Plan: - Pushed fully clean repository. - Pushed previously-pushed repository. - Forced write to abort, inspected useful information in the database. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15748 --- .../autopatches/20160418.repoversion.1.sql | 2 ++ .../DiffusionGitReceivePackSSHWorkflow.php | 3 ++- .../storage/PhabricatorRepository.php | 10 ++++++-- ...habricatorRepositoryWorkingCopyVersion.php | 23 ++++++++++++++----- 4 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 resources/sql/autopatches/20160418.repoversion.1.sql diff --git a/resources/sql/autopatches/20160418.repoversion.1.sql b/resources/sql/autopatches/20160418.repoversion.1.sql new file mode 100644 index 0000000000..e80e4322d0 --- /dev/null +++ b/resources/sql/autopatches/20160418.repoversion.1.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_repository.repository_workingcopyversion + ADD writeProperties LONGTEXT COLLATE {$COLLATE_TEXT}; diff --git a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php index 9babe3ebb1..f5e314f462 100644 --- a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php @@ -26,7 +26,8 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { $command = csprintf('git-receive-pack %s', $repository->getLocalPath()); $did_synchronize = true; - $repository->synchronizeWorkingCopyBeforeWrite(); + $viewer = $this->getUser(); + $repository->synchronizeWorkingCopyBeforeWrite($viewer); } $caught = null; diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index ee7f6b6bb6..fa13395e28 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -2482,7 +2482,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO /** * @task sync */ - public function synchronizeWorkingCopyBeforeWrite() { + public function synchronizeWorkingCopyBeforeWrite( + PhabricatorUser $actor) { if (!$this->shouldEnableSynchronization()) { return; } @@ -2516,7 +2517,12 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO PhabricatorRepositoryWorkingCopyVersion::willWrite( $repository_phid, - $device_phid); + $device_phid, + array( + 'userPHID' => $actor->getPHID(), + 'epoch' => PhabricatorTime::getNow(), + 'devicePHID' => $device_phid, + )); $this->clusterWriteVersion = $max_version; $this->clusterWriteLock = $write_lock; diff --git a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php index 1b1150f8cd..888301e807 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php +++ b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php @@ -7,6 +7,7 @@ final class PhabricatorRepositoryWorkingCopyVersion protected $devicePHID; protected $repositoryVersion; protected $isWriting; + protected $writeProperties; protected function getConfiguration() { return array( @@ -14,6 +15,7 @@ final class PhabricatorRepositoryWorkingCopyVersion self::CONFIG_COLUMN_SCHEMA => array( 'repositoryVersion' => 'uint32', 'isWriting' => 'bool', + 'writeProperties' => 'text?', ), self::CONFIG_KEY_SCHEMA => array( 'key_workingcopy' => array( @@ -66,7 +68,10 @@ final class PhabricatorRepositoryWorkingCopyVersion * lock is released by default. This is a durable lock which stays locked * by default. */ - public static function willWrite($repository_phid, $device_phid) { + public static function willWrite( + $repository_phid, + $device_phid, + array $write_properties) { $version = new self(); $conn_w = $version->establishConnection('w'); $table = $version->getTableName(); @@ -74,16 +79,19 @@ final class PhabricatorRepositoryWorkingCopyVersion queryfx( $conn_w, 'INSERT INTO %T - (repositoryPHID, devicePHID, repositoryVersion, isWriting) + (repositoryPHID, devicePHID, repositoryVersion, isWriting, + writeProperties) VALUES - (%s, %s, %d, %d) + (%s, %s, %d, %d, %s) ON DUPLICATE KEY UPDATE - isWriting = VALUES(isWriting)', + isWriting = VALUES(isWriting), + writeProperties = VALUES(writeProperties)', $table, $repository_phid, $device_phid, 0, - 1); + 1, + phutil_json_encode($write_properties)); } @@ -101,7 +109,10 @@ final class PhabricatorRepositoryWorkingCopyVersion queryfx( $conn_w, - 'UPDATE %T SET repositoryVersion = %d, isWriting = 0 + 'UPDATE %T SET + repositoryVersion = %d, + isWriting = 0, + writeProperties = null WHERE repositoryPHID = %s AND devicePHID = %s AND From 091a64e91bf6242cb7e76a671980fb8e31e5babd Mon Sep 17 00:00:00 2001 From: Aviv Eyal Date: Mon, 18 Apr 2016 19:33:40 +0000 Subject: [PATCH 16/50] Rename Differential field Projects to Tags Summary: Users can't find the "Tags" field in the Edit Menu; Added keyword "Tag". Test Plan: Looked in Edit page; I think this shouldn't change anything else? Reviewers: #blessed_reviewers, epriestley Reviewed By: #blessed_reviewers, epriestley Subscribers: Korvin Differential Revision: https://secure.phabricator.com/D15749 --- .../differential/customfield/DifferentialProjectsField.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/differential/customfield/DifferentialProjectsField.php b/src/applications/differential/customfield/DifferentialProjectsField.php index a784b98247..fd97bec9cb 100644 --- a/src/applications/differential/customfield/DifferentialProjectsField.php +++ b/src/applications/differential/customfield/DifferentialProjectsField.php @@ -8,7 +8,7 @@ final class DifferentialProjectsField } public function getFieldName() { - return pht('Projects'); + return pht('Tags'); } public function getFieldDescription() { From a3bb35e9d20a8eb5bc2c7128b03c4fbe1c954513 Mon Sep 17 00:00:00 2001 From: Aviv Eyal Date: Mon, 18 Apr 2016 13:43:32 -0700 Subject: [PATCH 17/50] make Trigger Daemon sleep correctly when one-time triggers exist Summary: Trigger daemon is trying to find the next event to invoke before sleeping, but the query includes already-elapsed triggers. It then tries to sleep for 0 seconds. Test Plan: On a new instance, schedule a single trigger of type `PhabricatorOneTimeTriggerClock` to a very near time. Use top to see trigger daemon not going to 100% CPU once the event has elapsed. Reviewers: #blessed_reviewers, epriestley Subscribers: Korvin Differential Revision: https://secure.phabricator.com/D15750 --- src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php index 1620c8f99a..a51280d6ab 100644 --- a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php +++ b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php @@ -261,7 +261,7 @@ final class PhabricatorTriggerDaemon * Get the number of seconds to sleep for before starting the next scheduling * phase. * - * If no events are scheduled soon, we'll sleep for 60 seconds. Otherwise, + * If no events are scheduled soon, we'll sleep briefly. Otherwise, * we'll sleep until the next scheduled event. * * @return int Number of seconds to sleep for. @@ -272,6 +272,7 @@ final class PhabricatorTriggerDaemon $next_triggers = id(new PhabricatorWorkerTriggerQuery()) ->setViewer($this->getViewer()) ->setOrder(PhabricatorWorkerTriggerQuery::ORDER_EXECUTION) + ->withNextEventBetween(0, null) ->setLimit(1) ->needEvents(true) ->execute(); From 575c01373ee77c12b06696b34cfdf43a2eb26c42 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 15:22:52 -0700 Subject: [PATCH 18/50] Extract repository command construction from Repositories Summary: Ref T4292. Ref T10366. Depends on D15751. Today, generating repository commands is purely a function of the repository, so they use protocols and credentials based on the repository configuration. For example, a repository with an SSH "remote URI" always generate SSH "remote commands". This needs to change in the future: - After T10366, repositories won't necessarily just have one type of remote URI. They can only have one at a time still, but the repository itself won't change based on which one is currently active. - For T4292, I need to generate intracluster commands, regardless of repository configuration. These will have different protocols and credentials. Prepare for these cases by separating out command construction, so they'll be able to generate commands in a more flexible way. Test Plan: - Added unit tests. - Browsed diffusion. - Ran `bin/phd debug pull` to pull a bunch of repos. - Ran daemons. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292, T10366 Differential Revision: https://secure.phabricator.com/D15752 --- src/__phutil_library_map__.php | 10 + .../DiffusionHistoryQueryConduitAPIMethod.php | 3 +- .../protocol/DiffusionCommandEngine.php | 173 ++++++++++++++ .../protocol/DiffusionGitCommandEngine.php | 37 +++ .../DiffusionMercurialCommandEngine.php | 70 ++++++ .../DiffusionSubversionCommandEngine.php | 54 +++++ .../DiffusionCommandEngineTestCase.php | 155 ++++++++++++ .../DiffusionLowLevelParentsQuery.php | 4 +- .../storage/PhabricatorRepository.php | 225 ++---------------- .../PhabricatorRepositoryTestCase.php | 3 +- 10 files changed, 524 insertions(+), 210 deletions(-) create mode 100644 src/applications/diffusion/protocol/DiffusionCommandEngine.php create mode 100644 src/applications/diffusion/protocol/DiffusionGitCommandEngine.php create mode 100644 src/applications/diffusion/protocol/DiffusionMercurialCommandEngine.php create mode 100644 src/applications/diffusion/protocol/DiffusionSubversionCommandEngine.php create mode 100644 src/applications/diffusion/protocol/__tests__/DiffusionCommandEngineTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 4361f12911..52fd00d833 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -570,6 +570,8 @@ phutil_register_library_map(array( 'DiffusionCachedResolveRefsQuery' => 'applications/diffusion/query/DiffusionCachedResolveRefsQuery.php', 'DiffusionChangeController' => 'applications/diffusion/controller/DiffusionChangeController.php', 'DiffusionChangeHeraldFieldGroup' => 'applications/diffusion/herald/DiffusionChangeHeraldFieldGroup.php', + 'DiffusionCommandEngine' => 'applications/diffusion/protocol/DiffusionCommandEngine.php', + 'DiffusionCommandEngineTestCase' => 'applications/diffusion/protocol/__tests__/DiffusionCommandEngineTestCase.php', 'DiffusionCommitAffectedFilesHeraldField' => 'applications/diffusion/herald/DiffusionCommitAffectedFilesHeraldField.php', 'DiffusionCommitAuthorHeraldField' => 'applications/diffusion/herald/DiffusionCommitAuthorHeraldField.php', 'DiffusionCommitAutocloseHeraldField' => 'applications/diffusion/herald/DiffusionCommitAutocloseHeraldField.php', @@ -634,6 +636,7 @@ phutil_register_library_map(array( 'DiffusionGitBlameQuery' => 'applications/diffusion/query/blame/DiffusionGitBlameQuery.php', 'DiffusionGitBranch' => 'applications/diffusion/data/DiffusionGitBranch.php', 'DiffusionGitBranchTestCase' => 'applications/diffusion/data/__tests__/DiffusionGitBranchTestCase.php', + 'DiffusionGitCommandEngine' => 'applications/diffusion/protocol/DiffusionGitCommandEngine.php', 'DiffusionGitFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionGitFileContentQuery.php', 'DiffusionGitLFSAuthenticateWorkflow' => 'applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php', 'DiffusionGitLFSResponse' => 'applications/diffusion/response/DiffusionGitLFSResponse.php', @@ -667,6 +670,7 @@ phutil_register_library_map(array( 'DiffusionLowLevelQuery' => 'applications/diffusion/query/lowlevel/DiffusionLowLevelQuery.php', 'DiffusionLowLevelResolveRefsQuery' => 'applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php', 'DiffusionMercurialBlameQuery' => 'applications/diffusion/query/blame/DiffusionMercurialBlameQuery.php', + 'DiffusionMercurialCommandEngine' => 'applications/diffusion/protocol/DiffusionMercurialCommandEngine.php', 'DiffusionMercurialFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionMercurialFileContentQuery.php', 'DiffusionMercurialRawDiffQuery' => 'applications/diffusion/query/rawdiff/DiffusionMercurialRawDiffQuery.php', 'DiffusionMercurialRequest' => 'applications/diffusion/request/DiffusionMercurialRequest.php', @@ -788,6 +792,7 @@ phutil_register_library_map(array( 'DiffusionServeController' => 'applications/diffusion/controller/DiffusionServeController.php', 'DiffusionSetPasswordSettingsPanel' => 'applications/diffusion/panel/DiffusionSetPasswordSettingsPanel.php', 'DiffusionSetupException' => 'applications/diffusion/exception/DiffusionSetupException.php', + 'DiffusionSubversionCommandEngine' => 'applications/diffusion/protocol/DiffusionSubversionCommandEngine.php', 'DiffusionSubversionSSHWorkflow' => 'applications/diffusion/ssh/DiffusionSubversionSSHWorkflow.php', 'DiffusionSubversionServeSSHWorkflow' => 'applications/diffusion/ssh/DiffusionSubversionServeSSHWorkflow.php', 'DiffusionSubversionWireProtocol' => 'applications/diffusion/protocol/DiffusionSubversionWireProtocol.php', @@ -4771,6 +4776,8 @@ phutil_register_library_map(array( 'DiffusionCachedResolveRefsQuery' => 'DiffusionLowLevelQuery', 'DiffusionChangeController' => 'DiffusionController', 'DiffusionChangeHeraldFieldGroup' => 'HeraldFieldGroup', + 'DiffusionCommandEngine' => 'Phobject', + 'DiffusionCommandEngineTestCase' => 'PhabricatorTestCase', 'DiffusionCommitAffectedFilesHeraldField' => 'DiffusionCommitHeraldField', 'DiffusionCommitAuthorHeraldField' => 'DiffusionCommitHeraldField', 'DiffusionCommitAutocloseHeraldField' => 'DiffusionCommitHeraldField', @@ -4835,6 +4842,7 @@ phutil_register_library_map(array( 'DiffusionGitBlameQuery' => 'DiffusionBlameQuery', 'DiffusionGitBranch' => 'Phobject', 'DiffusionGitBranchTestCase' => 'PhabricatorTestCase', + 'DiffusionGitCommandEngine' => 'DiffusionCommandEngine', 'DiffusionGitFileContentQuery' => 'DiffusionFileContentQuery', 'DiffusionGitLFSAuthenticateWorkflow' => 'DiffusionGitSSHWorkflow', 'DiffusionGitLFSResponse' => 'AphrontResponse', @@ -4868,6 +4876,7 @@ phutil_register_library_map(array( 'DiffusionLowLevelQuery' => 'Phobject', 'DiffusionLowLevelResolveRefsQuery' => 'DiffusionLowLevelQuery', 'DiffusionMercurialBlameQuery' => 'DiffusionBlameQuery', + 'DiffusionMercurialCommandEngine' => 'DiffusionCommandEngine', 'DiffusionMercurialFileContentQuery' => 'DiffusionFileContentQuery', 'DiffusionMercurialRawDiffQuery' => 'DiffusionRawDiffQuery', 'DiffusionMercurialRequest' => 'DiffusionRequest', @@ -4989,6 +4998,7 @@ phutil_register_library_map(array( 'DiffusionServeController' => 'DiffusionController', 'DiffusionSetPasswordSettingsPanel' => 'PhabricatorSettingsPanel', 'DiffusionSetupException' => 'Exception', + 'DiffusionSubversionCommandEngine' => 'DiffusionCommandEngine', 'DiffusionSubversionSSHWorkflow' => 'DiffusionSSHWorkflow', 'DiffusionSubversionServeSSHWorkflow' => 'DiffusionSubversionSSHWorkflow', 'DiffusionSubversionWireProtocol' => 'Phobject', diff --git a/src/applications/diffusion/conduit/DiffusionHistoryQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionHistoryQueryConduitAPIMethod.php index dcc8f56e8f..60985e364d 100644 --- a/src/applications/diffusion/conduit/DiffusionHistoryQueryConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionHistoryQueryConduitAPIMethod.php @@ -131,7 +131,8 @@ final class DiffusionHistoryQueryConduitAPIMethod hgsprintf('reverse(ancestors(%s))', $commit_hash), $path_arg); - $stdout = PhabricatorRepository::filterMercurialDebugOutput($stdout); + $stdout = DiffusionMercurialCommandEngine::filterMercurialDebugOutput( + $stdout); $lines = explode("\n", trim($stdout)); $lines = array_slice($lines, $offset); diff --git a/src/applications/diffusion/protocol/DiffusionCommandEngine.php b/src/applications/diffusion/protocol/DiffusionCommandEngine.php new file mode 100644 index 0000000000..e5d54f110b --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionCommandEngine.php @@ -0,0 +1,173 @@ +canBuildForRepository($repository)) { + return id(clone $engine) + ->setRepository($repository); + } + } + + throw new Exception( + pht( + 'No registered command engine can build commands for this '. + 'repository ("%s").', + $repository->getDisplayName())); + } + + private static function newCommandEngines() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->execute(); + } + + abstract protected function canBuildForRepository( + PhabricatorRepository $repository); + + abstract protected function newFormattedCommand($pattern, array $argv); + abstract protected function newCustomEnvironment(); + + public function setRepository(PhabricatorRepository $repository) { + $this->repository = $repository; + return $this; + } + + public function getRepository() { + return $this->repository; + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setCredentialPHID($credential_phid) { + $this->credentialPHID = $credential_phid; + return $this; + } + + public function getCredentialPHID() { + return $this->credentialPHID; + } + + public function setArgv(array $argv) { + $this->argv = $argv; + return $this; + } + + public function getArgv() { + return $this->argv; + } + + public function setPassthru($passthru) { + $this->passthru = $passthru; + return $this; + } + + public function getPassthru() { + return $this->passthru; + } + + public function newFuture() { + $argv = $this->newCommandArgv(); + $env = $this->newCommandEnvironment(); + + if ($this->getPassthru()) { + $future = newv('PhutilExecPassthru', $argv); + } else { + $future = newv('ExecFuture', $argv); + } + + $future->setEnv($env); + + return $future; + } + + private function newCommandArgv() { + $argv = $this->argv; + $pattern = $argv[0]; + $argv = array_slice($argv, 1); + + list($pattern, $argv) = $this->newFormattedCommand($pattern, $argv); + + return array_merge(array($pattern), $argv); + } + + private function newCommandEnvironment() { + $env = $this->newCommonEnvironment() + $this->newCustomEnvironment(); + foreach ($env as $key => $value) { + if ($value === null) { + unset($env[$key]); + } + } + return $env; + } + + private function newCommonEnvironment() { + $env = array(); + // NOTE: Force the language to "en_US.UTF-8", which overrides locale + // settings. This makes stuff print in English instead of, e.g., French, + // so we can parse the output of some commands, error messages, etc. + $env['LANG'] = 'en_US.UTF-8'; + + // Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155. + $env['PHABRICATOR_ENV'] = PhabricatorEnv::getSelectedEnvironmentName(); + + if ($this->isAnySSHProtocol()) { + $credential_phid = $this->getCredentialPHID(); + if ($credential_phid) { + $env['PHABRICATOR_CREDENTIAL'] = $credential_phid; + } + } + + return $env; + } + + protected function isSSHProtocol() { + return ($this->getProtocol() == 'ssh'); + } + + protected function isSVNProtocol() { + return ($this->getProtocol() == 'svn'); + } + + protected function isSVNSSHProtocol() { + return ($this->getProtocol() == 'svn+ssh'); + } + + protected function isHTTPProtocol() { + return ($this->getProtocol() == 'http'); + } + + protected function isHTTPSProtocol() { + return ($this->getProtocol() == 'https'); + } + + protected function isAnyHTTPProtocol() { + return ($this->isHTTPProtocol() || $this->isHTTPSProtocol()); + } + + protected function isAnySSHProtocol() { + return ($this->isSSHProtocol() || $this->isSVNSSHProtocol()); + } + + protected function getSSHWrapper() { + $root = dirname(phutil_get_library_root('phabricator')); + return $root.'/bin/ssh-connect'; + } + +} diff --git a/src/applications/diffusion/protocol/DiffusionGitCommandEngine.php b/src/applications/diffusion/protocol/DiffusionGitCommandEngine.php new file mode 100644 index 0000000000..3458d27ed6 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionGitCommandEngine.php @@ -0,0 +1,37 @@ +isGit(); + } + + protected function newFormattedCommand($pattern, array $argv) { + $pattern = "git {$pattern}"; + return array($pattern, $argv); + } + + protected function newCustomEnvironment() { + $env = array(); + + // NOTE: See T2965. Some time after Git 1.7.5.4, Git started fataling if + // it can not read $HOME. For many users, $HOME points at /root (this + // seems to be a default result of Apache setup). Instead, explicitly + // point $HOME at a readable, empty directory so that Git looks for the + // config file it's after, fails to locate it, and moves on. This is + // really silly, but seems like the least damaging approach to + // mitigating the issue. + + $root = dirname(phutil_get_library_root('phabricator')); + $env['HOME'] = $root.'/support/empty/'; + + if ($this->isAnySSHProtocol()) { + $env['GIT_SSH'] = $this->getSSHWrapper(); + } + + return $env; + } + +} diff --git a/src/applications/diffusion/protocol/DiffusionMercurialCommandEngine.php b/src/applications/diffusion/protocol/DiffusionMercurialCommandEngine.php new file mode 100644 index 0000000000..49c618b085 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionMercurialCommandEngine.php @@ -0,0 +1,70 @@ +isHg(); + } + + protected function newFormattedCommand($pattern, array $argv) { + $args = array(); + + if ($this->isAnySSHProtocol()) { + $pattern = "hg --config ui.ssh=%s {$pattern}"; + $args[] = $this->getSSHWrapper(); + } else { + $pattern = "hg {$pattern}"; + } + + return array($pattern, array_merge($args, $argv)); + } + + protected function newCustomEnvironment() { + $env = array(); + + // NOTE: This overrides certain configuration, extensions, and settings + // which make Mercurial commands do random unusual things. + $env['HGPLAIN'] = 1; + + return $env; + } + + /** + * Sanitize output of an `hg` command invoked with the `--debug` flag to make + * it usable. + * + * @param string Output from `hg --debug ...` + * @return string Usable output. + */ + public static function filterMercurialDebugOutput($stdout) { + // When hg commands are run with `--debug` and some config file isn't + // trusted, Mercurial prints out a warning to stdout, twice, after Feb 2011. + // + // http://selenic.com/pipermail/mercurial-devel/2011-February/028541.html + // + // After Jan 2015, it may also fail to write to a revision branch cache. + + $ignore = array( + 'ignoring untrusted configuration option', + "couldn't write revision branch cache:", + ); + + foreach ($ignore as $key => $pattern) { + $ignore[$key] = preg_quote($pattern, '/'); + } + + $ignore = '('.implode('|', $ignore).')'; + + $lines = preg_split('/(?<=\n)/', $stdout); + $regex = '/'.$ignore.'.*\n$/'; + + foreach ($lines as $key => $line) { + $lines[$key] = preg_replace($regex, '', $line); + } + + return implode('', $lines); + } + +} diff --git a/src/applications/diffusion/protocol/DiffusionSubversionCommandEngine.php b/src/applications/diffusion/protocol/DiffusionSubversionCommandEngine.php new file mode 100644 index 0000000000..d8e9e6ff17 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionSubversionCommandEngine.php @@ -0,0 +1,54 @@ +isSVN(); + } + + protected function newFormattedCommand($pattern, array $argv) { + $flags = array(); + $args = array(); + + $flags[] = '--non-interactive'; + + if ($this->isAnyHTTPProtocol() || $this->isSVNProtocol()) { + $flags[] = '--no-auth-cache'; + + if ($this->isAnyHTTPProtocol()) { + $flags[] = '--trust-server-cert'; + } + + $credential_phid = $this->getCredentialPHID(); + if ($credential_phid) { + $key = PassphrasePasswordKey::loadFromPHID( + $credential_phid, + PhabricatorUser::getOmnipotentUser()); + + $flags[] = '--username %P'; + $args[] = $key->getUsernameEnvelope(); + + $flags[] = '--password %P'; + $args[] = $key->getPasswordEnvelope(); + } + } + + $flags = implode(' ', $flags); + $pattern = "svn {$flags} {$pattern}"; + + return array($pattern, array_merge($args, $argv)); + } + + protected function newCustomEnvironment() { + $env = array(); + + if ($this->isAnySSHProtocol()) { + $env['SVN_SSH'] = $this->getSSHWrapper(); + } + + return $env; + } + +} diff --git a/src/applications/diffusion/protocol/__tests__/DiffusionCommandEngineTestCase.php b/src/applications/diffusion/protocol/__tests__/DiffusionCommandEngineTestCase.php new file mode 100644 index 0000000000..76d9ad9170 --- /dev/null +++ b/src/applications/diffusion/protocol/__tests__/DiffusionCommandEngineTestCase.php @@ -0,0 +1,155 @@ +assertCommandEngineFormat( + 'git xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'HOME' => $home, + ), + array( + 'vcs' => $type_git, + 'argv' => 'xyz', + )); + + $this->assertCommandEngineFormat( + 'hg xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'HGPLAIN' => '1', + ), + array( + 'vcs' => $type_hg, + 'argv' => 'xyz', + )); + + $this->assertCommandEngineFormat( + 'svn --non-interactive xyz', + array( + 'LANG' => 'en_US.UTF-8', + ), + array( + 'vcs' => $type_svn, + 'argv' => 'xyz', + )); + + + // Commands with SSH. + + $this->assertCommandEngineFormat( + 'git xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'HOME' => $home, + 'GIT_SSH' => $ssh_wrapper, + ), + array( + 'vcs' => $type_git, + 'argv' => 'xyz', + 'protocol' => 'ssh', + )); + + $this->assertCommandEngineFormat( + (string)csprintf('hg --config ui.ssh=%s xyz', $ssh_wrapper), + array( + 'LANG' => 'en_US.UTF-8', + 'HGPLAIN' => '1', + ), + array( + 'vcs' => $type_hg, + 'argv' => 'xyz', + 'protocol' => 'ssh', + )); + + $this->assertCommandEngineFormat( + 'svn --non-interactive xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'SVN_SSH' => $ssh_wrapper, + ), + array( + 'vcs' => $type_svn, + 'argv' => 'xyz', + 'protocol' => 'ssh', + )); + + + // Commands with HTTP. + + $this->assertCommandEngineFormat( + 'git xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'HOME' => $home, + ), + array( + 'vcs' => $type_git, + 'argv' => 'xyz', + 'protocol' => 'https', + )); + + $this->assertCommandEngineFormat( + 'hg xyz', + array( + 'LANG' => 'en_US.UTF-8', + 'HGPLAIN' => '1', + ), + array( + 'vcs' => $type_hg, + 'argv' => 'xyz', + 'protocol' => 'https', + )); + + $this->assertCommandEngineFormat( + 'svn --non-interactive --no-auth-cache --trust-server-cert xyz', + array( + 'LANG' => 'en_US.UTF-8', + ), + array( + 'vcs' => $type_svn, + 'argv' => 'xyz', + 'protocol' => 'https', + )); + } + + private function assertCommandEngineFormat( + $command, + array $env, + array $inputs) { + + $repository = id(new PhabricatorRepository()) + ->setVersionControlSystem($inputs['vcs']); + + $future = DiffusionCommandEngine::newCommandEngine($repository) + ->setArgv((array)$inputs['argv']) + ->setProtocol(idx($inputs, 'protocol')) + ->newFuture(); + + $command_string = $future->getCommand(); + + $actual_command = $command_string->getUnmaskedString(); + $this->assertEqual($command, $actual_command); + + $actual_environment = $future->getEnv(); + + $compare_environment = array_select_keys( + $actual_environment, + array_keys($env)); + + $this->assertEqual($env, $compare_environment); + } + +} diff --git a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelParentsQuery.php b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelParentsQuery.php index f49f828d79..a2673b0ce2 100644 --- a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelParentsQuery.php +++ b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelParentsQuery.php @@ -50,7 +50,9 @@ final class DiffusionLowLevelParentsQuery list($stdout) = $repository->execxLocalCommand( 'log --debug --limit 1 --template={parents} --rev %s', $this->identifier); - $stdout = PhabricatorRepository::filterMercurialDebugOutput($stdout); + + $stdout = DiffusionMercurialCommandEngine::filterMercurialDebugOutput( + $stdout); $hashes = preg_split('/\s+/', trim($stdout)); foreach ($hashes as $key => $value) { diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index fa13395e28..895d60fb8f 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -487,19 +487,22 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } private function newRemoteCommandFuture(array $argv) { - $argv = $this->formatRemoteCommand($argv); - $future = newv('ExecFuture', $argv); - $future->setEnv($this->getRemoteCommandEnvironment()); - return $future; + return $this->newRemoteCommandEngine($argv) + ->newFuture(); } private function newRemoteCommandPassthru(array $argv) { - $argv = $this->formatRemoteCommand($argv); - $passthru = newv('PhutilExecPassthru', $argv); - $passthru->setEnv($this->getRemoteCommandEnvironment()); - return $passthru; + return $this->newRemoteCommandEngine($argv) + ->setPassthru(true) + ->newFuture(); } + private function newRemoteCommandEngine(array $argv) { + return DiffusionCommandEngine::newCommandEngine($this) + ->setArgv($argv) + ->setCredentialPHID($this->getCredentialPHID()) + ->setProtocol($this->getRemoteProtocol()); + } /* -( Local Command Execution )-------------------------------------------- */ @@ -527,9 +530,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private function newLocalCommandFuture(array $argv) { $this->assertLocalExists(); - $argv = $this->formatLocalCommand($argv); - $future = newv('ExecFuture', $argv); - $future->setEnv($this->getLocalCommandEnvironment()); + $future = DiffusionCommandEngine::newCommandEngine($this) + ->setArgv($argv) + ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); @@ -541,9 +544,10 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private function newLocalCommandPassthru(array $argv) { $this->assertLocalExists(); - $argv = $this->formatLocalCommand($argv); - $future = newv('PhutilExecPassthru', $argv); - $future->setEnv($this->getLocalCommandEnvironment()); + $future = DiffusionCommandEngine::newCommandEngine($this) + ->setArgv($argv) + ->setPassthru(true) + ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); @@ -552,199 +556,6 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return $future; } - -/* -( Command Infrastructure )--------------------------------------------- */ - - - private function getSSHWrapper() { - $root = dirname(phutil_get_library_root('phabricator')); - return $root.'/bin/ssh-connect'; - } - - private function getCommonCommandEnvironment() { - $env = array( - // NOTE: Force the language to "en_US.UTF-8", which overrides locale - // settings. This makes stuff print in English instead of, e.g., French, - // so we can parse the output of some commands, error messages, etc. - 'LANG' => 'en_US.UTF-8', - - // Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155. - 'PHABRICATOR_ENV' => PhabricatorEnv::getSelectedEnvironmentName(), - ); - - switch ($this->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - // NOTE: See T2965. Some time after Git 1.7.5.4, Git started fataling if - // it can not read $HOME. For many users, $HOME points at /root (this - // seems to be a default result of Apache setup). Instead, explicitly - // point $HOME at a readable, empty directory so that Git looks for the - // config file it's after, fails to locate it, and moves on. This is - // really silly, but seems like the least damaging approach to - // mitigating the issue. - - $root = dirname(phutil_get_library_root('phabricator')); - $env['HOME'] = $root.'/support/empty/'; - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - // NOTE: This overrides certain configuration, extensions, and settings - // which make Mercurial commands do random unusual things. - $env['HGPLAIN'] = 1; - break; - default: - throw new Exception(pht('Unrecognized version control system.')); - } - - return $env; - } - - private function getLocalCommandEnvironment() { - return $this->getCommonCommandEnvironment(); - } - - private function getRemoteCommandEnvironment() { - $env = $this->getCommonCommandEnvironment(); - - if ($this->shouldUseSSH()) { - // NOTE: This is read by `bin/ssh-connect`, and tells it which credentials - // to use. - $env['PHABRICATOR_CREDENTIAL'] = $this->getCredentialPHID(); - switch ($this->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - // Force SVN to use `bin/ssh-connect`. - $env['SVN_SSH'] = $this->getSSHWrapper(); - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - // Force Git to use `bin/ssh-connect`. - $env['GIT_SSH'] = $this->getSSHWrapper(); - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - // We force Mercurial through `bin/ssh-connect` too, but it uses a - // command-line flag instead of an environmental variable. - break; - default: - throw new Exception(pht('Unrecognized version control system.')); - } - } - - return $env; - } - - private function formatRemoteCommand(array $args) { - $pattern = $args[0]; - $args = array_slice($args, 1); - - switch ($this->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - if ($this->shouldUseHTTP() || $this->shouldUseSVNProtocol()) { - $flags = array(); - $flag_args = array(); - $flags[] = '--non-interactive'; - $flags[] = '--no-auth-cache'; - if ($this->shouldUseHTTP()) { - $flags[] = '--trust-server-cert'; - } - - $credential_phid = $this->getCredentialPHID(); - if ($credential_phid) { - $key = PassphrasePasswordKey::loadFromPHID( - $credential_phid, - PhabricatorUser::getOmnipotentUser()); - $flags[] = '--username %P'; - $flags[] = '--password %P'; - $flag_args[] = $key->getUsernameEnvelope(); - $flag_args[] = $key->getPasswordEnvelope(); - } - - $flags = implode(' ', $flags); - $pattern = "svn {$flags} {$pattern}"; - $args = array_mergev(array($flag_args, $args)); - } else { - $pattern = "svn --non-interactive {$pattern}"; - } - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $pattern = "git {$pattern}"; - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - if ($this->shouldUseSSH()) { - $pattern = "hg --config ui.ssh=%s {$pattern}"; - array_unshift( - $args, - $this->getSSHWrapper()); - } else { - $pattern = "hg {$pattern}"; - } - break; - default: - throw new Exception(pht('Unrecognized version control system.')); - } - - array_unshift($args, $pattern); - - return $args; - } - - private function formatLocalCommand(array $args) { - $pattern = $args[0]; - $args = array_slice($args, 1); - - switch ($this->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - $pattern = "svn --non-interactive {$pattern}"; - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $pattern = "git {$pattern}"; - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - $pattern = "hg {$pattern}"; - break; - default: - throw new Exception(pht('Unrecognized version control system.')); - } - - array_unshift($args, $pattern); - - return $args; - } - - /** - * Sanitize output of an `hg` command invoked with the `--debug` flag to make - * it usable. - * - * @param string Output from `hg --debug ...` - * @return string Usable output. - */ - public static function filterMercurialDebugOutput($stdout) { - // When hg commands are run with `--debug` and some config file isn't - // trusted, Mercurial prints out a warning to stdout, twice, after Feb 2011. - // - // http://selenic.com/pipermail/mercurial-devel/2011-February/028541.html - // - // After Jan 2015, it may also fail to write to a revision branch cache. - - $ignore = array( - 'ignoring untrusted configuration option', - "couldn't write revision branch cache:", - ); - - foreach ($ignore as $key => $pattern) { - $ignore[$key] = preg_quote($pattern, '/'); - } - - $ignore = '('.implode('|', $ignore).')'; - - $lines = preg_split('/(?<=\n)/', $stdout); - $regex = '/'.$ignore.'.*\n$/'; - - foreach ($lines as $key => $line) { - $lines[$key] = preg_replace($regex, '', $line); - } - - return implode('', $lines); - } - public function getURI() { $callsign = $this->getCallsign(); if (strlen($callsign)) { diff --git a/src/applications/repository/storage/__tests__/PhabricatorRepositoryTestCase.php b/src/applications/repository/storage/__tests__/PhabricatorRepositoryTestCase.php index 84379feb49..c6d2278262 100644 --- a/src/applications/repository/storage/__tests__/PhabricatorRepositoryTestCase.php +++ b/src/applications/repository/storage/__tests__/PhabricatorRepositoryTestCase.php @@ -147,7 +147,8 @@ final class PhabricatorRepositoryTestCase ); foreach ($map as $input => $expect) { - $actual = PhabricatorRepository::filterMercurialDebugOutput($input); + $actual = DiffusionMercurialCommandEngine::filterMercurialDebugOutput( + $input); $this->assertEqual($expect, $actual, $input); } } From c30fe65ee9c8a4cad3fdbd09032af926384f847f Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 06:55:42 -0700 Subject: [PATCH 19/50] Remove the warning about the Git 2GB pathname issue Summary: Ref T10832. In practice, `git --version` is not a useful test for this issue: - Vendors like Debian have backported the patch into custom versions like `0.0.0.1-debian-lots-of-patches.3232`. - Vendors like Ubuntu distribute multiple different versions which report the same string from `git --version`, some of which are patched and some of which are not. In other cases, we can perform an empirical test for the vulnerability. Here, we can not, because we can't write a 2GB path in a reasonable amount of time. Since vendors (other than Apple) //generally// seem to be on top of this and any warning we try to raise based on `git --version` will frequently be incorrect, don't raise this warning. I'll note this in the changelog instead. Test Plan: Looked at setup issues, no more warning for vulnerable git version. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10832 Differential Revision: https://secure.phabricator.com/D15756 --- .../config/check/PhabricatorBinariesSetupCheck.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/applications/config/check/PhabricatorBinariesSetupCheck.php b/src/applications/config/check/PhabricatorBinariesSetupCheck.php index c3c0740cfa..0d577b5297 100644 --- a/src/applications/config/check/PhabricatorBinariesSetupCheck.php +++ b/src/applications/config/check/PhabricatorBinariesSetupCheck.php @@ -102,14 +102,7 @@ final class PhabricatorBinariesSetupCheck extends PhabricatorSetupCheck { $version = null; switch ($vcs['versionControlSystem']) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $bad_versions = array( - '< 2.7.4' => pht( - 'Prior to 2.7.4, Git contains two remote code execution '. - 'vulnerabilities which allow an attacker to take control of a '. - 'system by crafting a commit which affects very long paths, '. - 'then pushing it or tricking a victim into fetching it. This '. - 'is a severe security vulnerability.'), - ); + $bad_versions = array(); list($err, $stdout, $stderr) = exec_manual('git --version'); $version = trim(substr($stdout, strlen('git version '))); break; From c9daa2b0ad1e613e3074356f5dd5b964f8f2215d Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 19 Apr 2016 16:48:21 +0000 Subject: [PATCH 20/50] Consistently refer to 'Projects' as 'Tags' Summary: In calendar, dashboard, diffusion, diviner, feed, fund, maniphest, pholio, ponder, and slowvote use the term 'tags' if possible. This intenctionally skips diffusion, differential, and the projects application itself. Ref T10326 Ref T10349 Test Plan: inspection on a running, locally modified, system Reviewers: avivey, epriestley, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T10835, T10326, T10349 Differential Revision: https://secure.phabricator.com/D15753 --- .../controller/PhabricatorCalendarEventEditController.php | 2 +- .../dashboard/controller/PhabricatorDashboardEditController.php | 2 +- .../controller/PhabricatorDashboardPanelEditController.php | 2 +- .../differential/query/DifferentialRevisionSearchEngine.php | 2 +- .../diviner/controller/DivinerBookEditController.php | 2 +- src/applications/feed/query/PhabricatorFeedSearchEngine.php | 2 +- .../fund/controller/FundInitiativeEditController.php | 2 +- .../maniphest/export/ManiphestExcelDefaultFormat.php | 2 +- src/applications/pholio/controller/PholioMockEditController.php | 2 +- .../phurl/controller/PhabricatorPhurlURLEditController.php | 2 +- .../ponder/controller/PonderQuestionEditController.php | 2 +- .../slowvote/controller/PhabricatorSlowvoteEditController.php | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/applications/calendar/controller/PhabricatorCalendarEventEditController.php b/src/applications/calendar/controller/PhabricatorCalendarEventEditController.php index 10ed2b84e0..421084ceaf 100644 --- a/src/applications/calendar/controller/PhabricatorCalendarEventEditController.php +++ b/src/applications/calendar/controller/PhabricatorCalendarEventEditController.php @@ -459,7 +459,7 @@ final class PhabricatorCalendarEventEditController } $projects = id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($projects) ->setUser($viewer) diff --git a/src/applications/dashboard/controller/PhabricatorDashboardEditController.php b/src/applications/dashboard/controller/PhabricatorDashboardEditController.php index 6dc4a6b8b2..665f9735d8 100644 --- a/src/applications/dashboard/controller/PhabricatorDashboardEditController.php +++ b/src/applications/dashboard/controller/PhabricatorDashboardEditController.php @@ -161,7 +161,7 @@ final class PhabricatorDashboardEditController $form->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())); diff --git a/src/applications/dashboard/controller/PhabricatorDashboardPanelEditController.php b/src/applications/dashboard/controller/PhabricatorDashboardPanelEditController.php index 790b8d11da..76ce3389ad 100644 --- a/src/applications/dashboard/controller/PhabricatorDashboardPanelEditController.php +++ b/src/applications/dashboard/controller/PhabricatorDashboardPanelEditController.php @@ -240,7 +240,7 @@ final class PhabricatorDashboardPanelEditController $form->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())); diff --git a/src/applications/differential/query/DifferentialRevisionSearchEngine.php b/src/applications/differential/query/DifferentialRevisionSearchEngine.php index 002a5df180..5e4b76a326 100644 --- a/src/applications/differential/query/DifferentialRevisionSearchEngine.php +++ b/src/applications/differential/query/DifferentialRevisionSearchEngine.php @@ -179,7 +179,7 @@ final class DifferentialRevisionSearchEngine ->setValue($repository_phids)) ->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setDatasource(new PhabricatorProjectLogicalDatasource()) ->setValue($projects)) diff --git a/src/applications/diviner/controller/DivinerBookEditController.php b/src/applications/diviner/controller/DivinerBookEditController.php index 2b1f90579b..758cc190bb 100644 --- a/src/applications/diviner/controller/DivinerBookEditController.php +++ b/src/applications/diviner/controller/DivinerBookEditController.php @@ -75,7 +75,7 @@ final class DivinerBookEditController extends DivinerController { id(new AphrontFormTokenizerControl()) ->setDatasource(new PhabricatorProjectDatasource()) ->setName('projectPHIDs') - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setValue($book->getProjectPHIDs())) ->appendControl( id(new AphrontFormTokenizerControl()) diff --git a/src/applications/feed/query/PhabricatorFeedSearchEngine.php b/src/applications/feed/query/PhabricatorFeedSearchEngine.php index 94e82f8bfe..b8caf60ae7 100644 --- a/src/applications/feed/query/PhabricatorFeedSearchEngine.php +++ b/src/applications/feed/query/PhabricatorFeedSearchEngine.php @@ -102,7 +102,7 @@ final class PhabricatorFeedSearchEngine ); if ($this->requireViewer()->isLoggedIn()) { - $names['projects'] = pht('Projects'); + $names['projects'] = pht('Tags'); } return $names; diff --git a/src/applications/fund/controller/FundInitiativeEditController.php b/src/applications/fund/controller/FundInitiativeEditController.php index 354d686bd5..6e092d1e05 100644 --- a/src/applications/fund/controller/FundInitiativeEditController.php +++ b/src/applications/fund/controller/FundInitiativeEditController.php @@ -201,7 +201,7 @@ final class FundInitiativeEditController ->setValue($v_risk)) ->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())) diff --git a/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php b/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php index bd0c771361..1451ae504e 100644 --- a/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php +++ b/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php @@ -58,7 +58,7 @@ final class ManiphestExcelDefaultFormat extends ManiphestExcelFormat { pht('Date Created'), pht('Date Updated'), pht('Title'), - pht('Projects'), + pht('Tags'), pht('URI'), pht('Description'), ); diff --git a/src/applications/pholio/controller/PholioMockEditController.php b/src/applications/pholio/controller/PholioMockEditController.php index f3dedc9a33..362c5c41df 100644 --- a/src/applications/pholio/controller/PholioMockEditController.php +++ b/src/applications/pholio/controller/PholioMockEditController.php @@ -316,7 +316,7 @@ final class PholioMockEditController extends PholioController { ->setUser($viewer)) ->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())) diff --git a/src/applications/phurl/controller/PhabricatorPhurlURLEditController.php b/src/applications/phurl/controller/PhabricatorPhurlURLEditController.php index 3af53802e5..73ba3474fe 100644 --- a/src/applications/phurl/controller/PhabricatorPhurlURLEditController.php +++ b/src/applications/phurl/controller/PhabricatorPhurlURLEditController.php @@ -165,7 +165,7 @@ final class PhabricatorPhurlURLEditController ->setError($error_alias); $projects = id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($projects) ->setUser($viewer) diff --git a/src/applications/ponder/controller/PonderQuestionEditController.php b/src/applications/ponder/controller/PonderQuestionEditController.php index 907094c7e9..49ca5fe846 100644 --- a/src/applications/ponder/controller/PonderQuestionEditController.php +++ b/src/applications/ponder/controller/PonderQuestionEditController.php @@ -156,7 +156,7 @@ final class PonderQuestionEditController extends PonderController { $form->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())); diff --git a/src/applications/slowvote/controller/PhabricatorSlowvoteEditController.php b/src/applications/slowvote/controller/PhabricatorSlowvoteEditController.php index 7354466772..e9f3d48de4 100644 --- a/src/applications/slowvote/controller/PhabricatorSlowvoteEditController.php +++ b/src/applications/slowvote/controller/PhabricatorSlowvoteEditController.php @@ -152,7 +152,7 @@ final class PhabricatorSlowvoteEditController ->setValue($v_description)) ->appendControl( id(new AphrontFormTokenizerControl()) - ->setLabel(pht('Projects')) + ->setLabel(pht('Tags')) ->setName('projects') ->setValue($v_projects) ->setDatasource(new PhabricatorProjectDatasource())); From 0db6eaca417347d52d605a506717cd2232a5f286 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 05:50:19 -0700 Subject: [PATCH 21/50] Consolidate handling of SSH usernames Summary: Ref T4292. This consolidates code for figuring out which user we should connect to hosts with. Also narrows a lock window. Test Plan: Browsed Diffusion, pulled and pushed through an SSH proxy. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15754 --- src/applications/almanac/util/AlmanacKeys.php | 18 ++++++++++++++++++ .../diffusion/ssh/DiffusionSSHWorkflow.php | 15 ++++++--------- .../storage/PhabricatorRepository.php | 11 ++++++++--- .../storage/PhabricatorRepositoryURI.php | 2 +- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/applications/almanac/util/AlmanacKeys.php b/src/applications/almanac/util/AlmanacKeys.php index d15d3cb439..3f482416df 100644 --- a/src/applications/almanac/util/AlmanacKeys.php +++ b/src/applications/almanac/util/AlmanacKeys.php @@ -48,4 +48,22 @@ final class AlmanacKeys extends Phobject { return $device; } + public static function getClusterSSHUser() { + // NOTE: When instancing, we currently use the SSH username to figure out + // which instance you are connecting to. We can't use the host name because + // we have no way to tell which host you think you're reaching: the SSH + // protocol does not have a mechanism like a "Host" header. + $username = PhabricatorEnv::getEnvConfig('cluster.instance'); + if (strlen($username)) { + return $username; + } + + $username = PhabricatorEnv::getEnvConfig('diffusion.ssh-user'); + if (strlen($username)) { + return $username; + } + + return null; + } + } diff --git a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php index 15333ff360..9000a2d661 100644 --- a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php @@ -62,15 +62,12 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow { protected function getProxyCommand() { $uri = new PhutilURI($this->proxyURI); - $username = PhabricatorEnv::getEnvConfig('cluster.instance'); - if (!strlen($username)) { - $username = PhabricatorEnv::getEnvConfig('diffusion.ssh-user'); - if (!strlen($username)) { - throw new Exception( - pht( - 'Unable to determine the username to connect with when trying '. - 'to proxy an SSH request within the Phabricator cluster.')); - } + $username = AlmanacKeys::getClusterSSHUser(); + if ($username === null) { + throw new Exception( + pht( + 'Unable to determine the username to connect with when trying '. + 'to proxy an SSH request within the Phabricator cluster.')); } $port = $uri->getPort(); diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 895d60fb8f..18d80c1c5d 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1348,8 +1348,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $uri->setPath($uri->getPath().$this->getCloneName().'/'); } - $ssh_user = PhabricatorEnv::getEnvConfig('diffusion.ssh-user'); - if ($ssh_user) { + $ssh_user = AlmanacKeys::getClusterSSHUser(); + if ($ssh_user !== null) { $uri->setUser($ssh_user); } @@ -2324,7 +2324,12 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO 'refusing new writes.')); } - $max_version = $this->synchronizeWorkingCopyBeforeRead(); + try { + $max_version = $this->synchronizeWorkingCopyBeforeRead(); + } catch (Exception $ex) { + $write_lock->unlock(); + throw $ex; + } PhabricatorRepositoryWorkingCopyVersion::willWrite( $repository_phid, diff --git a/src/applications/repository/storage/PhabricatorRepositoryURI.php b/src/applications/repository/storage/PhabricatorRepositoryURI.php index 163eafa120..c2872c56ac 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryURI.php +++ b/src/applications/repository/storage/PhabricatorRepositoryURI.php @@ -227,7 +227,7 @@ final class PhabricatorRepositoryURI private function getForcedUser() { switch ($this->getBuiltinProtocol()) { case self::BUILTIN_PROTOCOL_SSH: - return PhabricatorEnv::getEnvConfig('diffusion.ssh-user'); + return AlmanacKeys::getClusterSSHUser(); default: return null; } From c70f4815a958b7ee5cc4ccbead92bd10242af336 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 06:19:39 -0700 Subject: [PATCH 22/50] Allow cluster devices to SSH to one another without acting as a user Summary: Ref T4292. When you run `git fetch` and connect to, say, `repo001.west.company.com`, we'll look at the current version of the repository in other nodes in the cluster. If `repo002.east.company.com` has a newer version of the repository, we'll fetch that version first, then respond to your request. To do this, we need to run `git fetch repo002.east.company.com ...` and have that connect to the other host and be able to fetch data. This change allows us to run `PHABRICATOR_AS_DEVICE=1 git fetch ...` to use device credentials to do this fetch. (Device credentials are already supported and used, they just always connect as a user right now, but these fetches should be doable without having a user. We will have a valid user when you run `git fetch` yourself, but we won't have one if the daemons notice that a repository is out of date and want to update it, so the update code should not depend on having a user.) Test Plan: ``` $ PHABRICATOR_AS_DEVICE=1 ./bin/ssh-connect local.phacility.com Warning: Permanently added 'local.phacility.com' (RSA) to the list of known hosts. PTY allocation request failed on channel 0 phabricator-ssh-exec: Welcome to Phabricator. You are logged in as device/daemon.phacility.net. You haven't specified a command to run. This means you're requesting an interactive shell, but Phabricator does not provide an interactive shell over SSH. Usually, you should run a command like `git clone` or `hg push` rather than connecting directly with SSH. Supported commands are: conduit, git-lfs-authenticate, git-receive-pack, git-upload-pack, hg, svnserve. Connection to local.phacility.com closed. ``` Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15755 --- scripts/ssh/ssh-connect.php | 28 +++++++++++ scripts/ssh/ssh-exec.php | 49 ++++++++++--------- .../protocol/DiffusionCommandEngine.php | 46 ++++++++++++++++- 3 files changed, 100 insertions(+), 23 deletions(-) diff --git a/scripts/ssh/ssh-connect.php b/scripts/ssh/ssh-connect.php index 8a142090c3..fac8d19130 100755 --- a/scripts/ssh/ssh-connect.php +++ b/scripts/ssh/ssh-connect.php @@ -34,7 +34,28 @@ $pattern[] = 'StrictHostKeyChecking=no'; $pattern[] = '-o'; $pattern[] = 'UserKnownHostsFile=/dev/null'; +$as_device = getenv('PHABRICATOR_AS_DEVICE'); $credential_phid = getenv('PHABRICATOR_CREDENTIAL'); + +if ($as_device) { + $device = AlmanacKeys::getLiveDevice(); + if (!$device) { + throw new Exception( + pht( + 'Attempting to create an SSH connection that authenticates with '. + 'the current device, but this host is not configured as a cluster '. + 'device.')); + } + + if ($credential_phid) { + throw new Exception( + pht( + 'Attempting to proxy an SSH connection that authenticates with '. + 'both the current device and a specific credential. These options '. + 'are mutually exclusive.')); + } +} + if ($credential_phid) { $viewer = PhabricatorUser::getOmnipotentUser(); $key = PassphraseSSHKey::loadFromPHID($credential_phid, $viewer); @@ -45,6 +66,13 @@ if ($credential_phid) { $arguments[] = $key->getKeyfileEnvelope(); } +if ($as_device) { + $pattern[] = '-l %R'; + $arguments[] = AlmanacKeys::getClusterSSHUser(); + $pattern[] = '-i %R'; + $arguments[] = AlmanacKeys::getKeyPath('device.key'); +} + $port = $args->getArg('port'); if ($port) { $pattern[] = '-p %d'; diff --git a/scripts/ssh/ssh-exec.php b/scripts/ssh/ssh-exec.php index 2238bab478..5d8ede2913 100755 --- a/scripts/ssh/ssh-exec.php +++ b/scripts/ssh/ssh-exec.php @@ -153,32 +153,37 @@ try { ->splitArguments($original_command); if ($device) { - $act_as_name = array_shift($original_argv); - if (!preg_match('/^@/', $act_as_name)) { - throw new Exception( - pht( - 'Commands executed by devices must identify an acting user in the '. - 'first command argument. This request was not constructed '. - 'properly.')); + // If we're authenticating as a device, the first argument may be a + // "@username" argument to act as a particular user. + $first_argument = head($original_argv); + if (preg_match('/^@/', $first_argument)) { + $act_as_name = array_shift($original_argv); + $act_as_name = substr($act_as_name, 1); + $user = id(new PhabricatorPeopleQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withUsernames(array($act_as_name)) + ->executeOne(); + if (!$user) { + throw new Exception( + pht( + 'Device request identifies an acting user with an invalid '. + 'username ("%s"). There is no user with this username.', + $act_as_name)); + } + } else { + $user = PhabricatorUser::getOmnipotentUser(); } + } - $act_as_name = substr($act_as_name, 1); - $user = id(new PhabricatorPeopleQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withUsernames(array($act_as_name)) - ->executeOne(); - if (!$user) { - throw new Exception( - pht( - 'Device request identifies an acting user with an invalid '. - 'username ("%s"). There is no user with this username.', - $act_as_name)); - } + if ($user->isOmnipotent()) { + $user_name = 'device/'.$device->getName(); + } else { + $user_name = $user->getUsername(); } $ssh_log->setData( array( - 'u' => $user->getUsername(), + 'u' => $user_name, 'P' => $user->getPHID(), )); @@ -187,7 +192,7 @@ try { pht( 'Your account ("%s") does not have permission to establish SSH '. 'sessions. Visit the web interface for more information.', - $user->getUsername())); + $user_name)); } $workflows = id(new PhutilClassMapQuery()) @@ -206,7 +211,7 @@ try { "Usually, you should run a command like `%s` or `%s` ". "rather than connecting directly with SSH.\n\n". "Supported commands are: %s.", - $user->getUsername(), + $user_name, 'git clone', 'hg push', implode(', ', array_keys($workflows)))); diff --git a/src/applications/diffusion/protocol/DiffusionCommandEngine.php b/src/applications/diffusion/protocol/DiffusionCommandEngine.php index e5d54f110b..6894c2c4e8 100644 --- a/src/applications/diffusion/protocol/DiffusionCommandEngine.php +++ b/src/applications/diffusion/protocol/DiffusionCommandEngine.php @@ -7,6 +7,7 @@ abstract class DiffusionCommandEngine extends Phobject { private $credentialPHID; private $argv; private $passthru; + private $connectAsDevice; public static function newCommandEngine(PhabricatorRepository $repository) { $engines = self::newCommandEngines(); @@ -82,6 +83,15 @@ abstract class DiffusionCommandEngine extends Phobject { return $this->passthru; } + public function setConnectAsDevice($connect_as_device) { + $this->connectAsDevice = $connect_as_device; + return $this; + } + + public function getConnectAsDevice() { + return $this->connectAsDevice; + } + public function newFuture() { $argv = $this->newCommandArgv(); $env = $this->newCommandEnvironment(); @@ -118,6 +128,8 @@ abstract class DiffusionCommandEngine extends Phobject { } private function newCommonEnvironment() { + $repository = $this->getRepository(); + $env = array(); // NOTE: Force the language to "en_US.UTF-8", which overrides locale // settings. This makes stuff print in English instead of, e.g., French, @@ -127,11 +139,43 @@ abstract class DiffusionCommandEngine extends Phobject { // Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155. $env['PHABRICATOR_ENV'] = PhabricatorEnv::getSelectedEnvironmentName(); + $as_device = $this->getConnectAsDevice(); + $credential_phid = $this->getCredentialPHID(); + + if ($as_device) { + $device = AlmanacKeys::getLiveDevice(); + if (!$device) { + throw new Exception( + pht( + 'Attempting to build a reposiory command (for repository "%s") '. + 'as device, but this host ("%s") is not configured as a cluster '. + 'device.', + $repository->getDisplayName(), + php_uname('n'))); + } + + if ($credential_phid) { + throw new Exception( + pht( + 'Attempting to build a repository command (for repository "%s"), '. + 'but the CommandEngine is configured to connect as both the '. + 'current cluster device ("%s") and with a specific credential '. + '("%s"). These options are mutually exclusive. Connections must '. + 'authenticate as one or the other, not both.', + $repository->getDisplayName(), + $device->getName(), + $credential_phid)); + } + } + + if ($this->isAnySSHProtocol()) { - $credential_phid = $this->getCredentialPHID(); if ($credential_phid) { $env['PHABRICATOR_CREDENTIAL'] = $credential_phid; } + if ($as_device) { + $env['PHABRICATOR_AS_DEVICE'] = 1; + } } return $env; From 31bc023eff763d48a0837f6792477ed10420b840 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 18 Apr 2016 14:32:45 -0700 Subject: [PATCH 23/50] Synchronize (hosted, git, clustered, SSH) repositories prior to reads Summary: Ref T4292. Before we write or read a hosted, clustered Git repository over SSH, check if another version of the repository exists on another node that is more up-to-date. If such a version does exist, fetch that version first. This allows reads and writes of any node to always act on the most up-to-date code. Test Plan: Faked my way through this and got a fetch via `bin/repository update`; this is difficult to test locally and needs more work before we can put it in production. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15757 --- .../AlmanacServiceViewController.php | 2 +- .../ssh/DiffusionGitUploadPackSSHWorkflow.php | 6 +- .../diffusion/ssh/DiffusionSSHWorkflow.php | 22 +++ .../PhabricatorRepositoryPullEngine.php | 2 + .../storage/PhabricatorRepository.php | 180 ++++++++++++++---- 5 files changed, 178 insertions(+), 34 deletions(-) diff --git a/src/applications/almanac/controller/AlmanacServiceViewController.php b/src/applications/almanac/controller/AlmanacServiceViewController.php index f4057ca32f..3cb7451463 100644 --- a/src/applications/almanac/controller/AlmanacServiceViewController.php +++ b/src/applications/almanac/controller/AlmanacServiceViewController.php @@ -132,7 +132,7 @@ final class AlmanacServiceViewController ->setHideServiceColumn(true); $header = id(new PHUIHeaderView()) - ->setHeader(pht('SERVICE BINDINGS')) + ->setHeader(pht('Service Bindings')) ->addActionLink( id(new PHUIButtonView()) ->setTag('a') diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php index 926a477627..1a83a1f30b 100644 --- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php @@ -16,11 +16,15 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow { protected function executeRepositoryOperations() { $repository = $this->getRepository(); + $skip_sync = $this->shouldSkipReadSynchronization(); + if ($this->shouldProxy()) { $command = $this->getProxyCommand(); } else { $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath()); - $repository->synchronizeWorkingCopyBeforeRead(); + if (!$skip_sync) { + $repository->synchronizeWorkingCopyBeforeRead(); + } } $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); diff --git a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php index 9000a2d661..b1694de814 100644 --- a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php @@ -201,6 +201,14 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow { $repository = $this->getRepository(); $viewer = $this->getUser(); + if ($viewer->isOmnipotent()) { + throw new Exception( + pht( + 'This request is authenticated as a cluster device, but is '. + 'performing a write. Writes must be performed with a real '. + 'user account.')); + } + switch ($repository->getServeOverSSH()) { case PhabricatorRepository::SERVE_READONLY: if ($protocol_command !== null) { @@ -236,4 +244,18 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow { return $this->hasWriteAccess; } + protected function shouldSkipReadSynchronization() { + $viewer = $this->getUser(); + + // Currently, the only case where devices interact over SSH without + // assuming user credentials is when synchronizing before a read. These + // synchronizing reads do not themselves need to be synchronized. + if ($viewer->isOmnipotent()) { + return true; + } + + return false; + } + + } diff --git a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php index 690f9425a4..f589f61b9d 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php @@ -96,6 +96,8 @@ final class PhabricatorRepositoryPullEngine } if ($repository->isHosted()) { + $repository->synchronizeWorkingCopyBeforeRead(); + if ($is_git) { $this->installGitHook(); } else if ($is_svn) { diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 18d80c1c5d..0ce38acdab 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1349,7 +1349,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } $ssh_user = AlmanacKeys::getClusterSSHUser(); - if ($ssh_user !== null) { + if (strlen($ssh_user)) { $uri->setUser($ssh_user); } @@ -1927,31 +1927,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $never_proxy, array $protocols) { - $service_phid = $this->getAlmanacServicePHID(); - if (!$service_phid) { - // No service, so this is a local repository. - return null; - } - - $service = id(new AlmanacServiceQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withPHIDs(array($service_phid)) - ->needBindings(true) - ->needProperties(true) - ->executeOne(); + $service = $this->loadAlmanacService(); if (!$service) { - throw new Exception( - pht( - 'The Almanac service for this repository is invalid or could not '. - 'be loaded.')); - } - - $service_type = $service->getServiceImplementation(); - if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) { - throw new Exception( - pht( - 'The Almanac service for this repository does not have the correct '. - 'service type.')); + return null; } $bindings = $service->getBindings(); @@ -1990,16 +1968,14 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } } - $protocol = $binding->getAlmanacPropertyValue('protocol'); - if ($protocol === null) { - $protocol = 'https'; - } + $uri = $this->getClusterRepositoryURIFromBinding($binding); + $protocol = $uri->getProtocol(); if (empty($protocol_map[$protocol])) { continue; } - $uris[] = $protocol.'://'.$iface->renderDisplayAddress().'/'; + $uris[] = $uri; } if (!$uris) { @@ -2226,6 +2202,16 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return false; } + $service_phid = $this->getAlmanacServicePHID(); + if (!$service_phid) { + return false; + } + + // TODO: For now, this is only supported for Git. + if (!$this->isGit()) { + return false; + } + return true; } @@ -2275,8 +2261,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } } - // TODO: Actualy fetch the newer version from one of the nodes which has - // it. + $this->synchronizeWorkingCopyFromDevices($fetchable); PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, @@ -2393,6 +2378,137 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } + /** + * @task sync + */ + private function synchronizeWorkingCopyFromDevices(array $device_phids) { + $service = $this->loadAlmanacService(); + if (!$service) { + throw new Exception(pht('Failed to load repository cluster service.')); + } + + $device_map = array_fuse($device_phids); + $bindings = $service->getBindings(); + + $fetchable = array(); + foreach ($bindings as $binding) { + // We can't fetch from disabled nodes. + if ($binding->getIsDisabled()) { + continue; + } + + // We can't fetch from nodes which don't have the newest version. + $device_phid = $binding->getDevicePHID(); + if (empty($device_map[$device_phid])) { + continue; + } + + // TODO: For now, only fetch over SSH. We could support fetching over + // HTTP eventually. + if ($binding->getAlmanacPropertyValue('protocol') != 'ssh') { + continue; + } + + $fetchable[] = $binding; + } + + if (!$fetchable) { + throw new Exception( + pht( + 'Leader lost: no up-to-date nodes in repository cluster are '. + 'fetchable.')); + } + + $caught = null; + foreach ($fetchable as $binding) { + try { + $this->synchronizeWorkingCopyFromBinding($binding); + $caught = null; + break; + } catch (Exception $ex) { + $caught = $ex; + } + } + + if ($caught) { + throw $caught; + } + } + + private function synchronizeWorkingCopyFromBinding($binding) { + $fetch_uri = $this->getClusterRepositoryURIFromBinding($binding); + + if ($this->isGit()) { + $argv = array( + 'fetch --prune -- %s %s', + $fetch_uri, + '+refs/*:refs/*', + ); + } else { + throw new Exception(pht('Binding sync only supported for git!')); + } + + $future = DiffusionCommandEngine::newCommandEngine($this) + ->setArgv($argv) + ->setConnectAsDevice(true) + ->setProtocol($fetch_uri->getProtocol()) + ->newFuture(); + + $future->setCWD($this->getLocalPath()); + + $future->resolvex(); + } + + private function getClusterRepositoryURIFromBinding( + AlmanacBinding $binding) { + $protocol = $binding->getAlmanacPropertyValue('protocol'); + if ($protocol === null) { + $protocol = 'https'; + } + + $iface = $binding->getInterface(); + $address = $iface->renderDisplayAddress(); + + $path = $this->getURI(); + + return id(new PhutilURI("{$protocol}://{$address}")) + ->setPath($path); + } + + private function loadAlmanacService() { + $service_phid = $this->getAlmanacServicePHID(); + if (!$service_phid) { + // No service, so this is a local repository. + return null; + } + + $service = id(new AlmanacServiceQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($service_phid)) + ->needBindings(true) + ->needProperties(true) + ->executeOne(); + if (!$service) { + throw new Exception( + pht( + 'The Almanac service for this repository is invalid or could not '. + 'be loaded.')); + } + + $service_type = $service->getServiceImplementation(); + if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) { + throw new Exception( + pht( + 'The Almanac service for this repository does not have the correct '. + 'service type.')); + } + + return $service; + } + + + + /* -( Symbols )-------------------------------------------------------------*/ public function getSymbolSources() { From d87c500002d7d3cb480618de4595976c00b8e6b0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 08:03:12 -0700 Subject: [PATCH 24/50] Synchronize (hosted, clustered, Git) repositories over Conduit + HTTP Summary: Ref T4292. We currently synchronize hosted, clustered, Git repositories when we receive an SSH pull or push. Additionally: - Synchronize before HTTP reads and writes. - Synchronize reads before Conduit requests. We could relax Conduit eventually and allow Diffusion to say "it's OK to give me stale data". We could also redirect some set of these actions to just go to the up-to-date host instead of connecting to a random host and synchronizing it. However, this potentially won't work as well at scale: if you have a larger number of servers, it sends all of the traffic to the leader immediately following a write. That can cause "thundering herd" issues, and isn't efficient if replicas are in different geographical regions and the write just went to the east coast but most clients are on the west coast. In large-scale cases, it's better to go to the local replica, wait for an update, then serve traffic from it -- particularly given that writes are relatively rare. But we can finesse this later once things are solid. Test Plan: - Pushed and pulled a Git repository over HTTP. - Browsed a Git repository from the web UI. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15758 --- .../DiffusionQueryConduitAPIMethod.php | 6 ++++ .../controller/DiffusionServeController.php | 33 ++++++++++++++++--- .../protocol/DiffusionCommandEngine.php | 16 +++++++++ .../storage/PhabricatorRepository.php | 22 ++++++++++--- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/applications/diffusion/conduit/DiffusionQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionQueryConduitAPIMethod.php index 21e8da485f..7a7d81f4e0 100644 --- a/src/applications/diffusion/conduit/DiffusionQueryConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionQueryConduitAPIMethod.php @@ -145,6 +145,12 @@ abstract class DiffusionQueryConduitAPIMethod $this->setDiffusionRequest($drequest); + // TODO: Allow web UI queries opt out of this if they don't care about + // fetching the most up-to-date data? Synchronization can be slow, and a + // lot of web reads are probably fine if they're a few seconds out of + // date. + $repository->synchronizeWorkingCopyBeforeRead(); + return $this->getResult($request); } } diff --git a/src/applications/diffusion/controller/DiffusionServeController.php b/src/applications/diffusion/controller/DiffusionServeController.php index cac05b46b9..32c4936484 100644 --- a/src/applications/diffusion/controller/DiffusionServeController.php +++ b/src/applications/diffusion/controller/DiffusionServeController.php @@ -538,10 +538,35 @@ final class DiffusionServeController extends DiffusionController { $command = csprintf('%s', $bin); $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); - list($err, $stdout, $stderr) = id(new ExecFuture('%C', $command)) - ->setEnv($env, true) - ->write($input) - ->resolve(); + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + + $did_write_lock = false; + if ($this->isReadOnlyRequest($repository)) { + $repository->synchronizeWorkingCopyBeforeRead(); + } else { + $did_write_lock = true; + $repository->synchronizeWorkingCopyBeforeWrite($viewer); + } + + $caught = null; + try { + list($err, $stdout, $stderr) = id(new ExecFuture('%C', $command)) + ->setEnv($env, true) + ->write($input) + ->resolve(); + } catch (Exception $ex) { + $caught = $ex; + } + + if ($did_write_lock) { + $repository->synchronizeWorkingCopyAfterWrite(); + } + + unset($unguarded); + + if ($caught) { + throw $caught; + } if ($err) { if ($this->isValidGitShallowCloneResponse($stdout, $stderr)) { diff --git a/src/applications/diffusion/protocol/DiffusionCommandEngine.php b/src/applications/diffusion/protocol/DiffusionCommandEngine.php index 6894c2c4e8..552ca813a2 100644 --- a/src/applications/diffusion/protocol/DiffusionCommandEngine.php +++ b/src/applications/diffusion/protocol/DiffusionCommandEngine.php @@ -8,6 +8,7 @@ abstract class DiffusionCommandEngine extends Phobject { private $argv; private $passthru; private $connectAsDevice; + private $sudoAsDaemon; public static function newCommandEngine(PhabricatorRepository $repository) { $engines = self::newCommandEngines(); @@ -92,10 +93,25 @@ abstract class DiffusionCommandEngine extends Phobject { return $this->connectAsDevice; } + public function setSudoAsDaemon($sudo_as_daemon) { + $this->sudoAsDaemon = $sudo_as_daemon; + return $this; + } + + public function getSudoAsDaemon() { + return $this->sudoAsDaemon; + } + public function newFuture() { $argv = $this->newCommandArgv(); $env = $this->newCommandEnvironment(); + if ($this->getSudoAsDaemon()) { + $command = call_user_func_array('csprintf', $argv); + $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); + $argv = array('%C', $command); + } + if ($this->getPassthru()) { $future = newv('PhutilExecPassthru', $argv); } else { diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 0ce38acdab..0719ef05a7 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1954,6 +1954,10 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $uris = array(); foreach ($bindings as $binding) { + if ($binding->getIsDisabled()) { + continue; + } + $iface = $binding->getInterface(); // If we're never proxying this and it's locally satisfiable, return @@ -2197,11 +2201,6 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private function shouldEnableSynchronization() { - $device = AlmanacKeys::getLiveDevice(); - if (!$device) { - return false; - } - $service_phid = $this->getAlmanacServicePHID(); if (!$service_phid) { return false; @@ -2212,6 +2211,18 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return false; } + // TODO: It may eventually make sense to try to version and synchronize + // observed repositories (so that daemons don't do reads against out-of + // date hosts), but don't bother for now. + if (!$this->isHosted()) { + return false; + } + + $device = AlmanacKeys::getLiveDevice(); + if (!$device) { + return false; + } + return true; } @@ -2451,6 +2462,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $future = DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->setConnectAsDevice(true) + ->setSudoAsDaemon(true) ->setProtocol($fetch_uri->getProtocol()) ->newFuture(); From 6edf181a7eff7e74cc3e22bc09cd4702d1e19a13 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 10:10:12 -0700 Subject: [PATCH 25/50] Record which cluster host received a push Summary: Ref T4292. When we write a push log, also log which node received the request. Test Plan: {F1230467} Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15759 --- .../sql/autopatches/20160419.pushlog.1.sql | 2 ++ .../engine/DiffusionCommitHookEngine.php | 8 +++++++ .../view/DiffusionPushLogListView.php | 21 +++++++++++++++++++ ...abricatorRepositoryPushLogSearchEngine.php | 7 ++++++- .../storage/PhabricatorRepositoryPushLog.php | 2 ++ 5 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 resources/sql/autopatches/20160419.pushlog.1.sql diff --git a/resources/sql/autopatches/20160419.pushlog.1.sql b/resources/sql/autopatches/20160419.pushlog.1.sql new file mode 100644 index 0000000000..3625f5860e --- /dev/null +++ b/resources/sql/autopatches/20160419.pushlog.1.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_repository.repository_pushlog + ADD devicePHID VARBINARY(64); diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index 857ae76da3..54fa32a6a3 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -1058,8 +1058,16 @@ final class DiffusionCommitHookEngine extends Phobject { // up. $phid = id(new PhabricatorRepositoryPushLog())->generatePHID(); + $device = AlmanacKeys::getLiveDevice(); + if ($device) { + $device_phid = $device->getPHID(); + } else { + $device_phid = null; + } + return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer()) ->setPHID($phid) + ->setDevicePHID($device_phid) ->setRepositoryPHID($this->getRepository()->getPHID()) ->attachRepository($this->getRepository()) ->setEpoch(time()); diff --git a/src/applications/diffusion/view/DiffusionPushLogListView.php b/src/applications/diffusion/view/DiffusionPushLogListView.php index 73a44794e8..b3d78f2369 100644 --- a/src/applications/diffusion/view/DiffusionPushLogListView.php +++ b/src/applications/diffusion/view/DiffusionPushLogListView.php @@ -38,6 +38,7 @@ final class DiffusionPushLogListView extends AphrontView { } $rows = array(); + $any_host = false; foreach ($logs as $log) { $repository = $log->getRepository(); @@ -59,6 +60,14 @@ final class DiffusionPushLogListView extends AphrontView { $log->getRefOldShort()); } + $device_phid = $log->getDevicePHID(); + if ($device_phid) { + $device = $handles[$device_phid]->renderLink(); + $any_host = true; + } else { + $device = null; + } + $rows[] = array( phutil_tag( 'a', @@ -75,6 +84,7 @@ final class DiffusionPushLogListView extends AphrontView { $handles[$log->getPusherPHID()]->renderLink(), $remote_address, $log->getPushEvent()->getRemoteProtocol(), + $device, $log->getRefType(), $log->getRefName(), $old_ref_link, @@ -100,6 +110,7 @@ final class DiffusionPushLogListView extends AphrontView { pht('Pusher'), pht('From'), pht('Via'), + pht('Host'), pht('Type'), pht('Name'), pht('Old'), @@ -116,10 +127,20 @@ final class DiffusionPushLogListView extends AphrontView { '', '', '', + '', 'wide', 'n', 'n', 'right', + )) + ->setColumnVisibility( + array( + true, + true, + true, + true, + true, + $any_host, )); return $table; diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php index b054423d26..320558de21 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php @@ -95,7 +95,12 @@ final class PhabricatorRepositoryPushLogSearchEngine protected function getRequiredHandlePHIDsForResultList( array $logs, PhabricatorSavedQuery $query) { - return mpull($logs, 'getPusherPHID'); + $phids = array(); + $phids[] = mpull($logs, 'getPusherPHID'); + $phids[] = mpull($logs, 'getDevicePHID'); + $phids = array_mergev($phids); + $phids = array_filter($phids); + return $phids; } protected function renderResultList( diff --git a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php index f16784bbf1..4c449b4a84 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php @@ -34,6 +34,7 @@ final class PhabricatorRepositoryPushLog protected $epoch; protected $pusherPHID; protected $pushEventPHID; + protected $devicePHID; protected $refType; protected $refNameHash; protected $refNameRaw; @@ -81,6 +82,7 @@ final class PhabricatorRepositoryPushLog 'refNew' => 'text40', 'mergeBase' => 'text40?', 'changeFlags' => 'uint32', + 'devicePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_repository' => array( From 287e761f199182c1839371cdee73cd72321d909f Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 12:21:50 -0700 Subject: [PATCH 26/50] Make repository synchronization safer when leaders are ambiguous Summary: Ref T4292. Right now, repository versions only get marked when a write happens. This potentially creates a problem: if I pushed all the sync code to `secure` and enabled `secure002` as a repository host, the daemons would create empty copies of all the repositories on that host. Usually, this would be fine. Most repositories have already received a write on `secure001`, so that working copy has a verison and is a leader. However, when a write happened to a rarely-used repository (say, rKEYSTORE) that hadn't received any write recently, it might be sent to `secure002` randomly. Now, we'd try to figure out if `secure002` has the most up-to-date copy of the repository or not. We wouldn't be able to, since we don't have any information about which node has the data on it, since we never got a write before. The old code could guess wrong and decide that `secure002` is a leader, then accept the write. Since this would bump the version on `secure002`, that would //make// it an authoritative leader, and `secure001` would synchronize from it passively (or on the next read or write), which would potentially destroy data. Instead: - Refuse to continue in situations like this. - When a repository is on exactly one device, mark it as a leader with version "0". - When a repository is created into a cluster service, mark its version as "0" on all devices (they're all leaders, since the repository is empty). This should mean that we won't lose data no matter how much weird stuff we run into. Test Plan: - In single-node mode, used `repository update` to verify that `0` was written properly. - With multiple nodes, used `repository update` to verify that we refuse to continue. - Created a new repository, verified versions were initialized correctly. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15761 --- .../editor/PhabricatorRepositoryEditor.php | 4 + .../storage/PhabricatorRepository.php | 127 ++++++++++++++---- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/src/applications/repository/editor/PhabricatorRepositoryEditor.php b/src/applications/repository/editor/PhabricatorRepositoryEditor.php index 5de7f068e7..70d904b112 100644 --- a/src/applications/repository/editor/PhabricatorRepositoryEditor.php +++ b/src/applications/repository/editor/PhabricatorRepositoryEditor.php @@ -683,6 +683,10 @@ final class PhabricatorRepositoryEditor $object->save(); } + if ($this->getIsNewObject()) { + $object->synchronizeWorkingCopyAfterCreation(); + } + return $xactions; } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 0719ef05a7..5269c4509c 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1932,7 +1932,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO return null; } - $bindings = $service->getBindings(); + $bindings = $service->getActiveBindings(); if (!$bindings) { throw new Exception( pht( @@ -1954,10 +1954,6 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $uris = array(); foreach ($bindings as $binding) { - if ($binding->getIsDisabled()) { - continue; - } - $iface = $binding->getInterface(); // If we're never proxying this and it's locally satisfiable, return @@ -2227,6 +2223,37 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } + /** + * Synchronize repository version information after creating a repository. + * + * This initializes working copy versions for all currently bound devices to + * 0, so that we don't get stuck making an ambiguous choice about which + * devices are leaders when we later synchronize before a read. + * + * @task sync + */ + public function synchronizeWorkingCopyAfterCreation() { + if (!$this->shouldEnableSynchronization()) { + return; + } + + $repository_phid = $this->getPHID(); + + $service = $this->loadAlmanacService(); + if (!$service) { + throw new Exception(pht('Failed to load repository cluster service.')); + } + + $bindings = $service->getActiveBindings(); + foreach ($bindings as $binding) { + PhabricatorRepositoryWorkingCopyVersion::updateVersion( + $repository_phid, + $binding->getDevicePHID(), + 0); + } + } + + /** * @task sync */ @@ -2255,34 +2282,91 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO if ($this_version) { $this_version = (int)$this_version->getRepositoryVersion(); } else { - $this_version = 0; + $this_version = -1; } if ($versions) { - $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); - } else { - $max_version = 0; - } + // This is the normal case, where we have some version information and + // can identify which nodes are leaders. If the current node is not a + // leader, we want to fetch from a leader and then update our version. - if ($max_version > $this_version) { - $fetchable = array(); - foreach ($versions as $version) { - if ($version->getRepositoryVersion() == $max_version) { - $fetchable[] = $version->getDevicePHID(); + $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); + if ($max_version > $this_version) { + $fetchable = array(); + foreach ($versions as $version) { + if ($version->getRepositoryVersion() == $max_version) { + $fetchable[] = $version->getDevicePHID(); + } } + + $this->synchronizeWorkingCopyFromDevices($fetchable); + + PhabricatorRepositoryWorkingCopyVersion::updateVersion( + $repository_phid, + $device_phid, + $max_version); } - $this->synchronizeWorkingCopyFromDevices($fetchable); + $result_version = $max_version; + } else { + // If no version records exist yet, we need to be careful, because we + // can not tell which nodes are leaders. + // There might be several nodes with arbitrary existing data, and we have + // no way to tell which one has the "right" data. If we pick wrong, we + // might erase some or all of the data in the repository. + + // Since this is dangeorus, we refuse to guess unless there is only one + // device. If we're the only device in the group, we obviously must be + // a leader. + + $service = $this->loadAlmanacService(); + if (!$service) { + throw new Exception(pht('Failed to load repository cluster service.')); + } + + $bindings = $service->getActiveBindings(); + $device_map = array(); + foreach ($bindings as $binding) { + $device_map[$binding->getDevicePHID()] = true; + } + + if (count($device_map) > 1) { + throw new Exception( + pht( + 'Repository "%s" exists on more than one device, but no device '. + 'has any repository version information. Phabricator can not '. + 'guess which copy of the existing data is authoritative. Remove '. + 'all but one device from service to mark the remaining device '. + 'as the authority.', + $this->getDisplayName())); + } + + if (empty($device_map[$device->getPHID()])) { + throw new Exception( + pht( + 'Repository "%s" is being synchronized on device "%s", but '. + 'this device is not bound to the corresponding cluster '. + 'service ("%s").', + $this->getDisplayName(), + $device->getName(), + $service->getName())); + } + + // The current device is the only device in service, so it must be a + // leader. We can safely have any future nodes which come online read + // from it. PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, - $max_version); + 0); + + $result_version = 0; } $read_lock->unlock(); - return $max_version; + return $result_version; } @@ -2399,15 +2483,10 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } $device_map = array_fuse($device_phids); - $bindings = $service->getBindings(); + $bindings = $service->getActiveBindings(); $fetchable = array(); foreach ($bindings as $binding) { - // We can't fetch from disabled nodes. - if ($binding->getIsDisabled()) { - continue; - } - // We can't fetch from nodes which don't have the newest version. $device_phid = $binding->getDevicePHID(); if (empty($device_map[$device_phid])) { From 1344dda75692d93490cad242920b3f967ffaefcd Mon Sep 17 00:00:00 2001 From: Aviv Eyal Date: Wed, 20 Apr 2016 01:46:17 +0000 Subject: [PATCH 27/50] Parse Tags in commits message for revisions Summary: This will stop breaking if you have subscribers and tags when updating a revision (`Error parsing field "Subscribers": The objects you have listed include objects which do not exist (Tags:)`), which I broke in D15749. Test Plan: run through arc-diff --update that failed earlier. Reviewers: chad, #blessed_reviewers, epriestley Reviewed By: #blessed_reviewers, epriestley Subscribers: Korvin Differential Revision: https://secure.phabricator.com/D15762 --- .../differential/customfield/DifferentialProjectsField.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/applications/differential/customfield/DifferentialProjectsField.php b/src/applications/differential/customfield/DifferentialProjectsField.php index fd97bec9cb..53bd7293fd 100644 --- a/src/applications/differential/customfield/DifferentialProjectsField.php +++ b/src/applications/differential/customfield/DifferentialProjectsField.php @@ -76,6 +76,7 @@ final class DifferentialProjectsField public function getCommitMessageLabels() { return array( + 'Tags', 'Project', 'Projects', ); From bab3690b547f8b5052a44bf29340a25bf424a959 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 19:31:47 -0700 Subject: [PATCH 28/50] Fill in missing cluster database documentation Summary: Ref T10751. Provide some guidance on replicas and promotion. I'm not trying to walk administrators through the gritty details of this. It's not too complex, they should understand it, and the MySQL documentation is pretty thorough. Test Plan: Read documentation. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10751 Differential Revision: https://secure.phabricator.com/D15763 --- .../user/cluster/cluster_databases.diviner | 96 +++++++++++++++++-- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/src/docs/user/cluster/cluster_databases.diviner b/src/docs/user/cluster/cluster_databases.diviner index 5192138257..82be860db7 100644 --- a/src/docs/user/cluster/cluster_databases.diviner +++ b/src/docs/user/cluster/cluster_databases.diviner @@ -6,31 +6,76 @@ Configuring Phabricator to use multiple database hosts. Overview ======== -WARNING: This feature is a very early prototype; the features this document -describes are mostly speculative fantasy. - You can deploy Phabricator with multiple database hosts, configured as a master and a set of replicas. The advantages of doing this are: - faster recovery from disasters by promoting a replica; - - graceful degradation if the master fails; - - reduced load on the master; and + - graceful degradation if the master fails; and - some tools to help monitor and manage replica health. This configuration is complex, and many installs do not need to pursue it. -Phabricator can not currently be configured into a multi-master mode, nor can -it be configured to automatically promote a replica to become the new master. - If you lose the master, Phabricator can degrade automatically into read-only mode and remain available, but can not fully recover without operational intervention unless the master recovers on its own. +Phabricator will not currently send read traffic to replicas unless the master +has failed, so configuring a replica will not currently spread any load away +from the master. Future versions of Phabricator are expected to be able to +distribute some read traffic to replicas. + +Phabricator can not currently be configured into a multi-master mode, nor can +it be configured to automatically promote a replica to become the new master. +There are no current plans to support multi-master mode or autonomous failover, +although this may change in the future. + Setting up MySQL Replication ============================ -TODO: Write this section. +To begin, set up a replica database server and configure MySQL replication. + +If you aren't sure how to do this, refer to the MySQL manual for instructions. +The MySQL documentation is comprehensive and walks through the steps and +options in good detail. You should understand MySQL replication before +deploying it in production: Phabricator layers on top of it, and does not +attempt to abstract it away. + +Some useful notes for configuring replication for Phabricator: + +**Binlog Format**: Phabricator issues some queries which MySQL will detect as +unsafe if you use the `STATEMENT` binlog format (the default). Instead, use +`MIXED` (recommended) or `ROW` as the `binlog_format`. + +**Grant `REPLICATION CLIENT` Privilege**: If you give the user that Phabricator +will use to connect to the replica database server the `REPLICATION CLIENT` +privilege, Phabricator's status console can give you more information about +replica health and state. + +**Copying Data to Replicas**: Phabricator currently uses a mixture of MyISAM +and InnoDB tables, so it can be difficult to guarantee that a dump is wholly +consistent and suitable for loading into a replica because MySQL uses different +consistency mechanisms for the different storage engines. + +An approach you may want to consider to limit downtime but still produce a +consistent dump is to leave Phabricator running but configured in read-only +mode while dumping: + + - Stop all the daemons. + - Set `cluster.read-only` to `true` and deploy the new configuration. The + web UI should now show that Phabricator is in "Read Only" mode. + - Dump the database. You can do this with `bin/storage dump --for-replica` + to add the `--master-data` flag to the underlying command and include a + `CHANGE MASTER ...` statement in the dump. + - Once the dump finishes, turn `cluster.read-only` off again to restore + service. Continue loading the dump into the replica normally. + +**Log Expiration**: You can configure MySQL to automatically clean up old +binary logs on startup with the `expire_logs_days` option. If you do not +configure this and do not explicitly purge old logs with `PURGE BINARY LOGS`, +the binary logs on disk will grow unboundedly and relatively quickly. + +Once you have a working replica, continue below to tell Phabricator about it. Configuring Replicas @@ -207,7 +252,38 @@ the new master. See the next section, "Promoting a Replica", for details. Promoting a Replica =================== -TODO: Write this section. +If you lose access to the master database, Phabricator will degrade into +read-only mode. This is described in greater detail below. + +The easiest way to get out of read-only mode is to restore the master database. +If the database recovers on its own or operations staff can revive it, +Phabricator will return to full working order after a few moments. + +If you can't restore the master or are unsure you will be able to restore the +master quickly, you can promote a replica to become the new master instead. + +Before doing this, you should first assess how far behind the master the +replica was when the link died. Any data which was not replicated will either +be lost or become very difficult to recover after you promote a replica. + +For example, if some `T1234` had been created on the master but had not yet +replicated and you promote the replica, a new `T1234` may be created on the +replica after promotion. Even if you can recover the master later, merging +the data will be difficult because each database may have conflicting changes +which can not be merged easily. + +If there was a significant replication delay at the time of the failure, you +may wait to try harder or spend more time attempting to recover the master +before choosing to promote. + +If you have made a choice to promote, disable replication on the replica and +mark it as the `master` in `cluster.databases`. Remove the original master and +deploy the configuration change to all surviving hosts. + +Once write service is restored, you should provision, deploy, and configure a +new replica by following the steps you took the first time around. You are +critically vulnerable to a second disruption until you have restored the +redundancy. Unreachable Masters From 48b015a3fa8320f87156c64e01eef2d6d5de05e7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 20:15:39 -0700 Subject: [PATCH 29/50] Add slightly more cluster repository documentation Summary: Ref T10751. There are still some missing support tools here, but explain some of this a little better. Test Plan: Read documentation. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10751 Differential Revision: https://secure.phabricator.com/D15764 --- .../user/cluster/cluster_repositories.diviner | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index c5179666a7..eb9a4f4ede 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -19,19 +19,19 @@ advantages of doing this are: This configuration is complex, and many installs do not need to pursue it. -This configuration is not currently supported with Subversion. +This configuration is not currently supported with Subversion or Mercurial. Repository Hosts ================ Repository hosts must run a complete, fully configured copy of Phabricator, -including a webserver. If you make repositories available over SSH, they must -also run a properly configured `sshd`. +including a webserver. They must also run a properly configured `sshd`. Generally, these hosts will run the same set of services and configuration that web hosts run. If you prefer, you can overlay these services and put web and -repository services on the same hosts. +repository services on the same hosts. See @{article:Clustering Introduction} +for some guidance on overlaying services. When a user requests information about a repository that can only be satisfied by examining a repository working copy, the webserver receiving the request @@ -57,6 +57,17 @@ If it isn't, they block the read until they can complete a fetch. Before responding to a write, replicas obtain a global lock, perform the same version check and fetch if necessary, then allow the write to continue. +Additionally, repositories passively check other nodes for updates and +replicate changes in the background. After you push a change to a repositroy, +it will usually spread passively to all other repository nodes within a few +minutes. + +Even if passive replication is slow, the active replication makes acknowledged +changes sequential to all observers: after a write is acknowledged, all +subsequent reads are guaranteed to see it. The system does not permit stale +reads, and you do not need to wait for a replication delay to see a consistent +view of the repository no matter which node you ask. + HTTP vs HTTPS ============= @@ -84,6 +95,81 @@ Other mitigations are possible, but securing a network against the NSA and similar agents of other rogue nations is beyond the scope of this document. +Monitoring Replication +====================== + +You can review the current status of a repository on cluster nodes in +{nav Diffusion > (Repository) > Manage Repository > Cluster Configuration}. + +This screen shows all the configured devices which are hosting the repository +and the available version. + +**Version**: When a repository is mutated by a push, Phabricator increases +an internal version number for the repository. This column shows which version +is on disk on the corresponding node. + +After a change is pushed, the node which received the change will have a larger +version number than the other nodes. The change should be passively replicated +to the remaining nodes after a brief period of time, although this can take +a while if the change was large or the network connection between nodes is +slow or unreliable. + +You can click the version number to see the corresponding push logs for that +change. The logs contain details about what was changed, and can help you +identify if replication is slow because a change is large or for some other +reason. + +**Writing**: This shows that the node is currently holding a write lock. This +normally means that it is actively receiving a push, but can also mean that +there was a write interruption. See "Write Interruptions" below for details. + + +Write Interruptions +=================== + +A repository cluster can be put into an inconsistent state by an interruption +in a brief window immediately after a write. + +Phabricator can not commit changes to a working copy (stored on disk) and to +the global state (stored in a database) atomically, so there is a narrow window +between committing these two different states when some tragedy (like a +lightning strike) can befall a server, leaving the global and local views of +the repository state divergent. + +In these cases, Phabricator fails into a "frozen" state where further writes +are not permitted until the failure is investigated and resolved. + +TODO: Complete the support tooling and provide recovery instructions. + + +Loss of Leaders +=============== + +A more straightforward failure condition is the loss of all servers in a +cluster which have the most up-to-date copy of a repository. This looks like +this: + + - There is a cluster setup with two nodes, X and Y. + - A new change is pushed to server X. + - Before the change can propagate to server Y, lightning strikes server X + and destroys it. + +Here, all of the "leader" nodes with the most up-to-date copy of the repository +have been lost. Phabricator will refuse to serve this repository because it +can not serve it consistently, and can not accept writes without data loss. + +The most straightforward way to resolve this issue is to restore any leader to +service. The change will be able to replicate to other nodes once a leader +comes back online. + +If you are unable to restore a leader or unsure that you can restore one +quickly, you can use the monitoring console to review which changes are +present on the leaders but not present on the followers by examining the +push logs. + +TODO: Complete the support tooling and provide recovery instructions. + + Backups ====== From b9cf9e6f0db9e774026f2418e6d0a33fb08058a4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 19 Apr 2016 20:25:21 -0700 Subject: [PATCH 30/50] Fix an issue with PHID/handle management in push logs Summary: Ref T10751. This cleans this up so it's a little more modern, and fixes a possible bad access on the log detail page. Test Plan: Viewed push log list, viewed push log detail. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10751 Differential Revision: https://secure.phabricator.com/D15765 --- .../DiffusionPushEventViewController.php | 3 +-- .../view/DiffusionPushLogListView.php | 24 +++++++++++-------- ...abricatorRepositoryPushLogSearchEngine.php | 14 +---------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/applications/diffusion/controller/DiffusionPushEventViewController.php b/src/applications/diffusion/controller/DiffusionPushEventViewController.php index 027cf16bbe..c5eb6368b4 100644 --- a/src/applications/diffusion/controller/DiffusionPushEventViewController.php +++ b/src/applications/diffusion/controller/DiffusionPushEventViewController.php @@ -50,8 +50,7 @@ final class DiffusionPushEventViewController $updates_table = id(new DiffusionPushLogListView()) ->setUser($viewer) - ->setLogs($logs) - ->setHandles($this->loadViewerHandles(mpull($logs, 'getPusherPHID'))); + ->setLogs($logs); $update_box = id(new PHUIObjectBoxView()) ->setHeaderText(pht('All Pushed Updates')) diff --git a/src/applications/diffusion/view/DiffusionPushLogListView.php b/src/applications/diffusion/view/DiffusionPushLogListView.php index b3d78f2369..303f0519f6 100644 --- a/src/applications/diffusion/view/DiffusionPushLogListView.php +++ b/src/applications/diffusion/view/DiffusionPushLogListView.php @@ -3,7 +3,6 @@ final class DiffusionPushLogListView extends AphrontView { private $logs; - private $handles; public function setLogs(array $logs) { assert_instances_of($logs, 'PhabricatorRepositoryPushLog'); @@ -11,15 +10,20 @@ final class DiffusionPushLogListView extends AphrontView { return $this; } - public function setHandles(array $handles) { - $this->handles = $handles; - return $this; - } - public function render() { $logs = $this->logs; - $viewer = $this->getUser(); - $handles = $this->handles; + $viewer = $this->getViewer(); + + $handle_phids = array(); + foreach ($logs as $log) { + $handle_phids[] = $log->getPusherPHID(); + $device_phid = $log->getDevicePHID(); + if ($device_phid) { + $handle_phids[] = $device_phid; + } + } + + $handles = $viewer->loadHandles($handle_phids); // Figure out which repositories are editable. We only let you see remote // IPs if you have edit capability on a repository. @@ -62,7 +66,7 @@ final class DiffusionPushLogListView extends AphrontView { $device_phid = $log->getDevicePHID(); if ($device_phid) { - $device = $handles[$device_phid]->renderLink(); + $device = $viewer->renderHandle($device_phid); $any_host = true; } else { $device = null; @@ -81,7 +85,7 @@ final class DiffusionPushLogListView extends AphrontView { 'href' => $repository->getURI(), ), $repository->getDisplayName()), - $handles[$log->getPusherPHID()]->renderLink(), + $viewer->renderHandle($log->getPusherPHID()), $remote_address, $log->getPushEvent()->getRemoteProtocol(), $device, diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php index 320558de21..8fd3baeb54 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php @@ -92,25 +92,13 @@ final class PhabricatorRepositoryPushLogSearchEngine return parent::buildSavedQueryFromBuiltin($query_key); } - protected function getRequiredHandlePHIDsForResultList( - array $logs, - PhabricatorSavedQuery $query) { - $phids = array(); - $phids[] = mpull($logs, 'getPusherPHID'); - $phids[] = mpull($logs, 'getDevicePHID'); - $phids = array_mergev($phids); - $phids = array_filter($phids); - return $phids; - } - protected function renderResultList( array $logs, PhabricatorSavedQuery $query, array $handles) { $table = id(new DiffusionPushLogListView()) - ->setUser($this->requireViewer()) - ->setHandles($handles) + ->setViewer($this->requireViewer()) ->setLogs($logs); return id(new PhabricatorApplicationSearchResultView()) From 11f8fffe5bb7da1cbcd0c71c27c129ebe3801629 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 06:00:10 -0700 Subject: [PATCH 31/50] Fix Phriction document linking in mail bodies Summary: Fixes T10840. When rendering mail, this rule wasn't falling through in quite the right way. Also adjust where the rules are for this so the special styles show up in Maniphest, etc. Test Plan: Made this comment: {F1238266} Which produced this HTML: {F1238267} ...and sent this mail: {F1238283} Reviewers: hach-que, chad Reviewed By: chad Maniphest Tasks: T10840 Differential Revision: https://secure.phabricator.com/D15767 --- resources/celerity/map.php | 10 +++++----- .../phriction/markup/PhrictionRemarkupRule.php | 16 ++++++++++++++-- .../phriction/phriction-document-css.css | 14 -------------- webroot/rsrc/css/core/remarkup.css | 12 ++++++++++++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 49f38b3c51..5cc7b4c250 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -7,7 +7,7 @@ */ return array( 'names' => array( - 'core.pkg.css' => 'ce06b6f6', + 'core.pkg.css' => '31417876', 'core.pkg.js' => '37344f3c', 'darkconsole.pkg.js' => 'e7393ebb', 'differential.pkg.css' => '7ba78475', @@ -88,7 +88,7 @@ return array( 'rsrc/css/application/phortune/phortune-credit-card-form.css' => '8391eb02', 'rsrc/css/application/phortune/phortune.css' => '9149f103', 'rsrc/css/application/phrequent/phrequent.css' => 'ffc185ad', - 'rsrc/css/application/phriction/phriction-document-css.css' => '55446c91', + 'rsrc/css/application/phriction/phriction-document-css.css' => '4282e4ad', 'rsrc/css/application/policy/policy-edit.css' => '815c66f7', 'rsrc/css/application/policy/policy-transaction-detail.css' => '82100a43', 'rsrc/css/application/policy/policy.css' => '957ea14c', @@ -104,7 +104,7 @@ return array( 'rsrc/css/application/tokens/tokens.css' => '3d0f239e', 'rsrc/css/application/uiexample/example.css' => '528b19de', 'rsrc/css/core/core.css' => 'd0801452', - 'rsrc/css/core/remarkup.css' => '2c9ed46f', + 'rsrc/css/core/remarkup.css' => '6aae5360', 'rsrc/css/core/syntax.css' => '9fd11da8', 'rsrc/css/core/z-index.css' => '5b6fcf3f', 'rsrc/css/diviner/diviner-shared.css' => 'aa3656aa', @@ -777,7 +777,7 @@ return array( 'phabricator-object-selector-css' => '85ee8ce6', 'phabricator-phtize' => 'd254d646', 'phabricator-prefab' => 'e67df814', - 'phabricator-remarkup-css' => '2c9ed46f', + 'phabricator-remarkup-css' => '6aae5360', 'phabricator-search-results-css' => '7dea472c', 'phabricator-shaped-request' => '7cbe244b', 'phabricator-side-menu-view-css' => '3a3d9f41', @@ -807,7 +807,7 @@ return array( 'phortune-credit-card-form-css' => '8391eb02', 'phortune-css' => '9149f103', 'phrequent-css' => 'ffc185ad', - 'phriction-document-css' => '55446c91', + 'phriction-document-css' => '4282e4ad', 'phui-action-panel-css' => '91c7b835', 'phui-badge-view-css' => '3baef8db', 'phui-big-info-view-css' => 'bd903741', diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index 2c9ab76284..5f5a91a280 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -107,6 +107,10 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { $name = $spec['explicitName']; $class = 'phriction-link'; + // If the name is something meaningful to humans, we'll render this + // in text as: "Title" . Otherwise, we'll just render: . + $is_interesting_name = (bool)strlen($name); + if (idx($existant_documents, $link) === null) { // The target document doesn't exist. if ($name === null) { @@ -127,6 +131,8 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { $name = $visible_documents[$link] ->getContent() ->getTitle(); + + $is_interesting_name = true; } } @@ -143,7 +149,12 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { if ($this->getEngine()->getState('toc')) { $text = $name; } else if ($text_mode || $mail_mode) { - return PhabricatorEnv::getProductionURI($href); + $href = PhabricatorEnv::getProductionURI($href); + if ($is_interesting_name) { + $text = pht('"%s" <%s>', $name, $href); + } else { + $text = pht('<%s>', $href); + } } else { if ($class === 'phriction-link-lock') { $name = array( @@ -164,8 +175,9 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { 'class' => $class, ), $name); - $this->getEngine()->overwriteStoredText($spec['token'], $text); } + + $this->getEngine()->overwriteStoredText($spec['token'], $text); } } diff --git a/webroot/rsrc/css/application/phriction/phriction-document-css.css b/webroot/rsrc/css/application/phriction/phriction-document-css.css index 7728c8dd77..228515b3b3 100644 --- a/webroot/rsrc/css/application/phriction/phriction-document-css.css +++ b/webroot/rsrc/css/application/phriction/phriction-document-css.css @@ -32,17 +32,3 @@ .phriction-history-nav-table td.nav-next { text-align: right; } - -.phui-document-content .phriction-link { - font-weight: bold; -} - -.phui-document-content .phriction-link-missing { - font-weight: bold; - color: {$red}; -} - -.phui-document-content .phriction-link-lock { - font-weight: bold; - color: {$greytext}; -} diff --git a/webroot/rsrc/css/core/remarkup.css b/webroot/rsrc/css/core/remarkup.css index 2313939f4c..8f48d7228b 100644 --- a/webroot/rsrc/css/core/remarkup.css +++ b/webroot/rsrc/css/core/remarkup.css @@ -234,6 +234,18 @@ background: #ffaaaa; } +.phabricator-remarkup .phriction-link { + font-weight: bold; +} + +.phabricator-remarkup .phriction-link-missing { + color: {$red}; +} + +.phabricator-remarkup .phriction-link-lock { + color: {$greytext}; +} + .phabricator-remarkup-mention-nopermission .phui-tag-core { background: {$lightgreybackground}; color: {$lightgreytext}; From 7f15e8fbe8d2d754d8422ce24b5253ba545bba52 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 08:48:11 -0700 Subject: [PATCH 32/50] Formally deprecate owners.query Conduit API method Summary: This is completely obsoleted by `owners.search`. See D15472. Test Plan: Viewed API method in UI console. Reviewers: avivey, chad Reviewed By: chad Differential Revision: https://secure.phabricator.com/D15769 --- .../PhabricatorConduitConsoleController.php | 52 +++++++++++++------ .../conduit/OwnersQueryConduitAPIMethod.php | 15 ++++-- .../PhabricatorSearchEngineAPIMethod.php | 5 +- .../PhabricatorEditEngineAPIMethod.php | 5 +- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/applications/conduit/controller/PhabricatorConduitConsoleController.php b/src/applications/conduit/controller/PhabricatorConduitConsoleController.php index 5ee76b44f7..b767ead4f0 100644 --- a/src/applications/conduit/controller/PhabricatorConduitConsoleController.php +++ b/src/applications/conduit/controller/PhabricatorConduitConsoleController.php @@ -23,25 +23,8 @@ final class PhabricatorConduitConsoleController $call_uri = '/api/'.$method->getAPIMethodName(); - $status = $method->getMethodStatus(); - $reason = $method->getMethodStatusDescription(); $errors = array(); - switch ($status) { - case ConduitAPIMethod::METHOD_STATUS_DEPRECATED: - $reason = nonempty($reason, pht('This method is deprecated.')); - $errors[] = pht('Deprecated Method: %s', $reason); - break; - case ConduitAPIMethod::METHOD_STATUS_UNSTABLE: - $reason = nonempty( - $reason, - pht( - 'This method is new and unstable. Its interface is subject '. - 'to change.')); - $errors[] = pht('Unstable Method: %s', $reason); - break; - } - $form = id(new AphrontFormView()) ->setAction($call_uri) ->setUser($request->getUser()) @@ -127,6 +110,41 @@ final class PhabricatorConduitConsoleController $view = id(new PHUIPropertyListView()); + $status = $method->getMethodStatus(); + $reason = $method->getMethodStatusDescription(); + + switch ($status) { + case ConduitAPIMethod::METHOD_STATUS_UNSTABLE: + $stability_icon = 'fa-exclamation-triangle yellow'; + $stability_label = pht('Unstable Method'); + $stability_info = nonempty( + $reason, + pht( + 'This method is new and unstable. Its interface is subject '. + 'to change.')); + break; + case ConduitAPIMethod::METHOD_STATUS_DEPRECATED: + $stability_icon = 'fa-exclamation-triangle red'; + $stability_label = pht('Deprecated Method'); + $stability_info = nonempty($reason, pht('This method is deprecated.')); + break; + default: + $stability_label = null; + break; + } + + if ($stability_label) { + $view->addProperty( + pht('Stability'), + array( + id(new PHUIIconView())->setIcon($stability_icon), + ' ', + phutil_tag('strong', array(), $stability_label.':'), + ' ', + $stability_info, + )); + } + $view->addProperty( pht('Returns'), $method->getReturnType()); diff --git a/src/applications/owners/conduit/OwnersQueryConduitAPIMethod.php b/src/applications/owners/conduit/OwnersQueryConduitAPIMethod.php index a3b7b52cab..c79f612545 100644 --- a/src/applications/owners/conduit/OwnersQueryConduitAPIMethod.php +++ b/src/applications/owners/conduit/OwnersQueryConduitAPIMethod.php @@ -6,12 +6,17 @@ final class OwnersQueryConduitAPIMethod extends OwnersConduitAPIMethod { return 'owners.query'; } + public function getMethodStatus() { + return self::METHOD_STATUS_DEPRECATED; + } + + public function getMethodStatusDescription() { + return pht('Obsolete; use "owners.search" instead.'); + } + + public function getMethodDescription() { - return pht( - 'Query for packages by one of the following: repository/path, '. - 'packages with a given user or project owner, or packages affiliated '. - 'with a user (owned by either the user or a project they are a member '. - 'of.) You should only provide at most one search query.'); + return pht('Query for Owners packages. Obsoleted by "owners.search".'); } protected function defineParamTypes() { diff --git a/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php b/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php index b5cd52471b..d1e8e5f3f8 100644 --- a/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php +++ b/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php @@ -37,7 +37,10 @@ abstract class PhabricatorSearchEngineAPIMethod } public function getMethodStatusDescription() { - return pht('ApplicationSearch methods are highly unstable.'); + return pht( + 'ApplicationSearch methods are fairly stable, but were introduced '. + 'relatively recently and may continue to evolve as more applications '. + 'adopt them.'); } final protected function defineParamTypes() { diff --git a/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php b/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php index b6f95ecd1b..e478de1e20 100644 --- a/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php +++ b/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php @@ -16,7 +16,10 @@ abstract class PhabricatorEditEngineAPIMethod } public function getMethodStatusDescription() { - return pht('ApplicationEditor methods are highly unstable.'); + return pht( + 'ApplicationEditor methods are fairly stable, but were introduced '. + 'relativelyr cently and may continue to evolve as more applications '. + 'adopt them.'); } final protected function defineParamTypes() { From eeccaf99b6cd886389e1f9bce7bcdc440f60c183 Mon Sep 17 00:00:00 2001 From: lkassianik Date: Fri, 15 Apr 2016 13:52:00 -0700 Subject: [PATCH 33/50] When scrolling forward a month in calendar date picker from 1/31, next chosen date should be 2/29, not 3/1. Summary: Fixes T9295 Test Plan: Create event, open datepicker for start date, choose 1/31/2016, open datepicker again, click right button to scroll month. New suggested date should be 2/29/2016 Reviewers: chad, epriestley, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T9295 Differential Revision: https://secure.phabricator.com/D15727 --- resources/celerity/map.php | 18 +++++++++--------- .../rsrc/js/core/behavior-fancy-datepicker.js | 9 +++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 5cc7b4c250..087103c912 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -479,7 +479,7 @@ return array( 'rsrc/js/core/behavior-device.js' => 'b5b36110', 'rsrc/js/core/behavior-drag-and-drop-textarea.js' => '4f6a4b4e', 'rsrc/js/core/behavior-error-log.js' => '6882e80a', - 'rsrc/js/core/behavior-fancy-datepicker.js' => '8ae55229', + 'rsrc/js/core/behavior-fancy-datepicker.js' => '568931f3', 'rsrc/js/core/behavior-file-tree.js' => '88236f00', 'rsrc/js/core/behavior-form.js' => '5c54cbf3', 'rsrc/js/core/behavior-gesture.js' => '3ab51e2c', @@ -622,7 +622,7 @@ return array( 'javelin-behavior-editengine-reorder-fields' => 'b59e1e96', 'javelin-behavior-error-log' => '6882e80a', 'javelin-behavior-event-all-day' => '38dcf3c8', - 'javelin-behavior-fancy-datepicker' => '8ae55229', + 'javelin-behavior-fancy-datepicker' => '568931f3', 'javelin-behavior-global-drag-and-drop' => 'c8e57404', 'javelin-behavior-herald-rule-editor' => '7ebaeed3', 'javelin-behavior-high-security-warning' => 'a464fe03', @@ -1301,6 +1301,13 @@ return array( 'phabricator-drag-and-drop-file-upload', 'javelin-workboard-board', ), + '568931f3' => array( + 'javelin-behavior', + 'javelin-util', + 'javelin-dom', + 'javelin-stratcom', + 'javelin-vector', + ), '56a1ca03' => array( 'javelin-behavior', 'javelin-behavior-device', @@ -1554,13 +1561,6 @@ return array( 'javelin-install', 'javelin-dom', ), - '8ae55229' => array( - 'javelin-behavior', - 'javelin-util', - 'javelin-dom', - 'javelin-stratcom', - 'javelin-vector', - ), '8bdb2835' => array( 'phui-fontkit-css', ), diff --git a/webroot/rsrc/js/core/behavior-fancy-datepicker.js b/webroot/rsrc/js/core/behavior-fancy-datepicker.js index 4c58974fc5..d3cdb1a8d3 100644 --- a/webroot/rsrc/js/core/behavior-fancy-datepicker.js +++ b/webroot/rsrc/js/core/behavior-fancy-datepicker.js @@ -264,6 +264,7 @@ JX.behavior('fancy-datepicker', function(config, statics) { function getValidDate() { var written_date = new Date(value_y, value_m-1, value_d); + if (isNaN(written_date.getTime())) { return new Date(); } else { @@ -272,6 +273,14 @@ JX.behavior('fancy-datepicker', function(config, statics) { value_y += 2000; written_date = new Date(value_y, value_m-1, value_d); } + + // adjust for a date like February 31 + var adjust = 1; + while (written_date.getMonth() !== value_m-1) { + written_date = new Date(value_y, value_m-1, value_d-adjust); + adjust++; + } + return written_date; } } From d9275da2d4bd9c0bc54490736c7f56e70d7dd810 Mon Sep 17 00:00:00 2001 From: lkassianik Date: Wed, 20 Apr 2016 09:52:15 -0700 Subject: [PATCH 34/50] Better wording for cancelling/reinstating recurring events Summary: Fixes T10744 Test Plan: Create recurring event, cancel one instance, cancel the parent event, reinstate event. Wording in the reinstating dialog should be clear about reinstating only instances that haven't been individually cancelled. Reviewers: chad, epriestley, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T10744 Differential Revision: https://secure.phabricator.com/D15770 --- .../controller/PhabricatorCalendarEventCancelController.php | 3 ++- .../controller/PhabricatorCalendarEventViewController.php | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/applications/calendar/controller/PhabricatorCalendarEventCancelController.php b/src/applications/calendar/controller/PhabricatorCalendarEventCancelController.php index 4be5a77d11..81736436a0 100644 --- a/src/applications/calendar/controller/PhabricatorCalendarEventCancelController.php +++ b/src/applications/calendar/controller/PhabricatorCalendarEventCancelController.php @@ -82,7 +82,8 @@ final class PhabricatorCalendarEventCancelController } else if ($is_parent) { $title = pht('Reinstate Recurrence'); $paragraph = pht( - 'Reinstate the entire series of recurring events?'); + 'Reinstate all instances of this recurrence + that have not been individually cancelled?'); $cancel = pht("Don't Reinstate Recurrence"); $submit = pht('Reinstate Recurrence'); } else { diff --git a/src/applications/calendar/controller/PhabricatorCalendarEventViewController.php b/src/applications/calendar/controller/PhabricatorCalendarEventViewController.php index 7c3b34bc09..98d3a2fe74 100644 --- a/src/applications/calendar/controller/PhabricatorCalendarEventViewController.php +++ b/src/applications/calendar/controller/PhabricatorCalendarEventViewController.php @@ -219,8 +219,8 @@ final class PhabricatorCalendarEventViewController $reinstate_label = pht('Reinstate This Instance'); $cancel_disabled = (!$can_edit || $can_reinstate); } else if ($event->getIsRecurrenceParent()) { - $cancel_label = pht('Cancel Recurrence'); - $reinstate_label = pht('Reinstate Recurrence'); + $cancel_label = pht('Cancel All'); + $reinstate_label = pht('Reinstate All'); $cancel_disabled = !$can_edit; } else { $cancel_label = pht('Cancel Event'); From 11aa902bd1e732068ed42ded635e0c524416791d Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 05:05:49 -0700 Subject: [PATCH 35/50] Show "Last Writer" and "Last Write At" in the UI, add more documentation Summary: Ref T10751. Make the UI more useful and explain what failure states mean and how to get out of them. The `bin/repository thaw` command does not exist yet, I'll write that soon. Test Plan: {F1238241} Reviewers: chad Reviewed By: chad Maniphest Tasks: T10751 Differential Revision: https://secure.phabricator.com/D15766 --- ...fusionRepositoryClusterManagementPanel.php | 29 +++++++ ...habricatorRepositoryWorkingCopyVersion.php | 3 +- .../user/cluster/cluster_repositories.diviner | 87 +++++++++++++++++-- 3 files changed, 112 insertions(+), 7 deletions(-) diff --git a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php index 2017871453..22ecb3db3f 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php @@ -104,6 +104,29 @@ final class DiffusionRepositoryClusterManagementPanel ->setIcon('fa-pencil grey'); } + $write_properties = null; + if ($version) { + $write_properties = $version->getWriteProperties(); + if ($write_properties) { + try { + $write_properties = phutil_json_decode($write_properties); + } catch (Exception $ex) { + $write_properties = null; + } + } + } + + if ($write_properties) { + $writer_phid = idx($write_properties, 'userPHID'); + $last_writer = $viewer->renderHandle($writer_phid); + + $writer_epoch = idx($write_properties, 'epoch'); + $writer_epoch = phabricator_datetime($writer_epoch, $viewer); + } else { + $last_writer = null; + $writer_epoch = null; + } + $rows[] = array( $binding_icon, phutil_tag( @@ -114,6 +137,8 @@ final class DiffusionRepositoryClusterManagementPanel $device->getName()), $version_number, $is_writing, + $last_writer, + $writer_epoch, ); } } @@ -126,6 +151,8 @@ final class DiffusionRepositoryClusterManagementPanel pht('Device'), pht('Version'), pht('Writing'), + pht('Last Writer'), + pht('Last Write At'), )) ->setColumnClasses( array( @@ -133,6 +160,8 @@ final class DiffusionRepositoryClusterManagementPanel null, null, 'right wide', + null, + 'date', )); $doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories'); diff --git a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php index 888301e807..c8ad477a3d 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php +++ b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php @@ -111,8 +111,7 @@ final class PhabricatorRepositoryWorkingCopyVersion $conn_w, 'UPDATE %T SET repositoryVersion = %d, - isWriting = 0, - writeProperties = null + isWriting = 0 WHERE repositoryPHID = %s AND devicePHID = %s AND diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index eb9a4f4ede..fc35b3b619 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -123,23 +123,55 @@ reason. normally means that it is actively receiving a push, but can also mean that there was a write interruption. See "Write Interruptions" below for details. +**Last Writer**: This column identifies the user who most recently pushed a +change to this device. If the write lock is currently held, this user is +the user whose change is holding the lock. + +**Last Write At**: When the most recent write started. If the write lock is +currently held, this shows when the lock was acquired. + Write Interruptions =================== A repository cluster can be put into an inconsistent state by an interruption -in a brief window immediately after a write. +in a brief window during and immediately after a write. Phabricator can not commit changes to a working copy (stored on disk) and to the global state (stored in a database) atomically, so there is a narrow window between committing these two different states when some tragedy (like a lightning strike) can befall a server, leaving the global and local views of -the repository state divergent. +the repository state possibly divergent. -In these cases, Phabricator fails into a "frozen" state where further writes +In these cases, Phabricator fails into a frozen state where further writes are not permitted until the failure is investigated and resolved. -TODO: Complete the support tooling and provide recovery instructions. +You can use the monitoring console to review the state of a frozen repository +with a held write lock. The **Writing** column will show which node is holding +the lock, and whoever is named in the **Last Writer** column may be able to +help you figure out what happened by providing more information about what they +were doing and what they observed. + +Because the push was not acknowledged, it is normally safe to demote the node: +the user should have received an error anyway, and should not expect their push +to have worked. However, data is technically at risk and you may want to +investigate further and try to understand the issue in more detail before +continuing. + +There is no way to explicitly keep the write, but if it was committed to disk +you can recover it manually from the working copy on the device and then push +it again. + +If you demote the node, the in-process write will be thrown away, even if it +was complete on disk. To demote the node and release the write lock, run this +command: + +``` +phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net +``` + +{icon exclamation-triangle, color="yellow"} Any committed but unacknowledged +data on the device will be lost. Loss of Leaders @@ -167,7 +199,52 @@ quickly, you can use the monitoring console to review which changes are present on the leaders but not present on the followers by examining the push logs. -TODO: Complete the support tooling and provide recovery instructions. +If you are comfortable discarding these changes, you can instruct Phabricator +that it can forget about the leaders in two ways: disable the service bindings +to all of the leader nodes so they are no longer part of the cluster, or +use `bin/repository thaw` to `--demote` the leaders explicitly. + +If you do this, **you will lose data**. Either action will discard any changes +on the affected leaders which have not replicated to other nodes in the cluster. + +To demote a device, run this command: + +``` +phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net +``` + +{icon exclamation-triangle, color="red"} Any data which is only present on +**this** device will be lost. + + +Ambiguous Leaders +================= + +Repository clusters can also freeze if the leader nodes are ambiguous. This +can happen if you replace an entire cluster with new devices suddenly, or +make a mistake with the `--demote` flag. + +When Phabricator can not tell which node in a cluster is a leader, it freezes +the cluster because it is possible that some nodes have less data and others +have more, and if it choses a leader arbitrarily it may destroy some data +which you would prefer to retain. + +To resolve this, you need to tell Phabricator which node has the most +up-to-date data and promote that node to become a leader. If you do this, +**you may lose data** if you promote the wrong node, and some other node +really had more up-to-date data. If you want to double check, you can examine +the working copies on disk before promoting, by connecting to the machines and +using commands like `git log` to inspect state. + +Once you have identified a node which has data you're happy with, use +`bin/repository thaw` to `--promote` the device: + +``` +phabricator/ $ ./bin/repository thaw rXYZ --promote repo002.corp.net +``` + +{icon exclamation-triangle, color="red"} Any data which is only present on +**other** devices will be lost. Backups From bd4fb3c9fac0274e21dcfab9c42860e419d3f253 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 06:54:53 -0700 Subject: [PATCH 36/50] Implement `bin/repository thaw` for unfreezing cluster repositories Summary: Ref T10751. Add support tooling for manually prying your way out of trouble if disaster strikes. Refine documentation, try to refer to devices as "devices" more consistently instead of sometimes calling them "nodes". Test Plan: Promoted and demoted repository devices with `bin/repository thaw`. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10751 Differential Revision: https://secure.phabricator.com/D15768 --- src/__phutil_library_map__.php | 2 + ...icatorRepositoryManagementThawWorkflow.php | 186 ++++++++++++++++++ .../storage/PhabricatorRepository.php | 9 +- ...habricatorRepositoryWorkingCopyVersion.php | 20 ++ .../user/cluster/cluster_repositories.diviner | 146 +++++++++----- 5 files changed, 311 insertions(+), 52 deletions(-) create mode 100644 src/applications/repository/management/PhabricatorRepositoryManagementThawWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 52fd00d833..95cee8daba 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3183,6 +3183,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php', 'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php', 'PhabricatorRepositoryManagementReparseWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementReparseWorkflow.php', + 'PhabricatorRepositoryManagementThawWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementThawWorkflow.php', 'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php', 'PhabricatorRepositoryManagementWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementWorkflow.php', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php', @@ -7834,6 +7835,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementReparseWorkflow' => 'PhabricatorRepositoryManagementWorkflow', + 'PhabricatorRepositoryManagementThawWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementThawWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementThawWorkflow.php new file mode 100644 index 0000000000..2f09ffd9be --- /dev/null +++ b/src/applications/repository/management/PhabricatorRepositoryManagementThawWorkflow.php @@ -0,0 +1,186 @@ +setName('thaw') + ->setExamples('**thaw** [options] __repository__ ...') + ->setSynopsis( + pht( + 'Resolve issues with frozen cluster repositories. Very advanced '. + 'and dangerous.')) + ->setArguments( + array( + array( + 'name' => 'demote', + 'param' => 'device', + 'help' => pht( + 'Demote a device, discarding local changes. Clears stuck '. + 'write locks and recovers from lost leaders.'), + ), + array( + 'name' => 'promote', + 'param' => 'device', + 'help' => pht( + 'Promote a device, discarding changes on other devices. '. + 'Resolves ambiguous leadership and recovers from demotion '. + 'mistakes.'), + ), + array( + 'name' => 'force', + 'help' => pht('Run operations without asking for confirmation.'), + ), + array( + 'name' => 'repositories', + 'wildcard' => true, + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $repositories = $this->loadRepositories($args, 'repositories'); + if (!$repositories) { + throw new PhutilArgumentUsageException( + pht('Specify one or more repositories to thaw.')); + } + + $promote = $args->getArg('promote'); + $demote = $args->getArg('demote'); + + if (!$promote && !$demote) { + throw new PhutilArgumentUsageException( + pht('You must choose a device to --promote or --demote.')); + } + + if ($promote && $demote) { + throw new PhutilArgumentUsageException( + pht('Specify either --promote or --demote, but not both.')); + } + + $device_name = nonempty($promote, $demote); + + $device = id(new AlmanacDeviceQuery()) + ->setViewer($viewer) + ->withNames(array($device_name)) + ->executeOne(); + if (!$device) { + throw new PhutilArgumentUsageException( + pht('No device "%s" exists.', $device_name)); + } + + if ($promote) { + $risk_message = pht( + 'Promoting a device can cause the loss of any repository data which '. + 'only exists on other devices. The version of the repository on the '. + 'promoted device will become authoritative.'); + } else { + $risk_message = pht( + 'Demoting a device can cause the loss of any repository data which '. + 'only exists on the demoted device. The version of the repository '. + 'on some other device will become authoritative.'); + } + + echo tsprintf( + "** %s ** %s\n", + pht('DATA AT RISK'), + $risk_message); + + $is_force = $args->getArg('force'); + $prompt = pht('Accept the possibilty of permanent data loss?'); + if (!$is_force && !phutil_console_confirm($prompt)) { + throw new PhutilArgumentUsageException( + pht('User aborted the workflow.')); + } + + foreach ($repositories as $repository) { + $repository_phid = $repository->getPHID(); + + $write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock( + $repository_phid); + + echo tsprintf( + "%s\n", + pht( + 'Waiting to acquire write lock for "%s"...', + $repository->getDisplayName())); + + $write_lock->lock(phutil_units('5 minutes in seconds')); + try { + + $service = $repository->loadAlmanacService(); + if (!$service) { + throw new PhutilArgumentUsageException( + pht( + 'Repository "%s" is not a cluster repository: it is not '. + 'bound to an Almanac service.', + $repository->getDisplayName())); + } + + $bindings = $service->getActiveBindings(); + $bindings = mpull($bindings, null, 'getDevicePHID'); + if (empty($bindings[$device->getPHID()])) { + throw new PhutilArgumentUsageException( + pht( + 'Repository "%s" has no active binding to device "%s". Only '. + 'actively bound devices can be promoted or demoted.', + $repository->getDisplayName(), + $device->getName())); + } + + $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( + $repository->getPHID()); + + $versions = mpull($versions, null, 'getDevicePHID'); + $versions = array_select_keys($versions, array_keys($bindings)); + + if ($versions && $promote) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to promote "%s" for repository "%s": the leaders for '. + 'this cluster are not ambiguous.', + $device->getName(), + $repository->getDisplayName())); + } + + if ($promote) { + PhabricatorRepositoryWorkingCopyVersion::updateVersion( + $repository->getPHID(), + $device->getPHID(), + 0); + + echo tsprintf( + "%s\n", + pht( + 'Promoted "%s" to become a leader for "%s".', + $device->getName(), + $repository->getDisplayName())); + } + + if ($demote) { + PhabricatorRepositoryWorkingCopyVersion::demoteDevice( + $repository->getPHID(), + $device->getPHID()); + + echo tsprintf( + "%s\n", + pht( + 'Demoted "%s" from leadership of repository "%s".', + $device->getName(), + $repository->getDisplayName())); + } + } catch (Exception $ex) { + $write_lock->unlock(); + throw $ex; + } + + $write_lock->unlock(); + } + + return 0; + } + +} diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 5269c4509c..088b33ae62 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -2397,11 +2397,12 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO continue; } - // TODO: This should provide more help so users can resolve the issue. throw new Exception( pht( - 'An incomplete write was previously performed to this repository; '. - 'refusing new writes.')); + 'An previous write to this repository was interrupted; refusing '. + 'new writes. This issue resolves operator intervention to resolve, '. + 'see "Write Interruptions" in the "Cluster: Repositories" in the '. + 'documentation for instructions.')); } try { @@ -2566,7 +2567,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO ->setPath($path); } - private function loadAlmanacService() { + public function loadAlmanacService() { $service_phid = $this->getAlmanacServicePHID(); if (!$service_phid) { // No service, so this is a local repository. diff --git a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php index c8ad477a3d..0feeec759f 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php +++ b/src/applications/repository/storage/PhabricatorRepositoryWorkingCopyVersion.php @@ -132,6 +132,7 @@ final class PhabricatorRepositoryWorkingCopyVersion $repository_phid, $device_phid, $new_version) { + $version = new self(); $conn_w = $version->establishConnection('w'); $table = $version->getTableName(); @@ -152,4 +153,23 @@ final class PhabricatorRepositoryWorkingCopyVersion } + /** + * Explicitly demote a device. + */ + public static function demoteDevice( + $repository_phid, + $device_phid) { + + $version = new self(); + $conn_w = $version->establishConnection('w'); + $table = $version->getTableName(); + + queryfx( + $conn_w, + 'DELETE FROM %T WHERE repositoryPHID = %s AND devicePHID = %s', + $table, + $repository_phid, + $device_phid); + } + } diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index fc35b3b619..ac9531c241 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -98,7 +98,7 @@ similar agents of other rogue nations is beyond the scope of this document. Monitoring Replication ====================== -You can review the current status of a repository on cluster nodes in +You can review the current status of a repository on cluster devices in {nav Diffusion > (Repository) > Manage Repository > Cluster Configuration}. This screen shows all the configured devices which are hosting the repository @@ -106,20 +106,20 @@ and the available version. **Version**: When a repository is mutated by a push, Phabricator increases an internal version number for the repository. This column shows which version -is on disk on the corresponding node. +is on disk on the corresponding device. -After a change is pushed, the node which received the change will have a larger -version number than the other nodes. The change should be passively replicated -to the remaining nodes after a brief period of time, although this can take -a while if the change was large or the network connection between nodes is -slow or unreliable. +After a change is pushed, the device which received the change will have a +larger version number than the other devices. The change should be passively +replicated to the remaining devices after a brief period of time, although this +can take a while if the change was large or the network connection between +devices is slow or unreliable. You can click the version number to see the corresponding push logs for that change. The logs contain details about what was changed, and can help you identify if replication is slow because a change is large or for some other reason. -**Writing**: This shows that the node is currently holding a write lock. This +**Writing**: This shows that the device is currently holding a write lock. This normally means that it is actively receiving a push, but can also mean that there was a write interruption. See "Write Interruptions" below for details. @@ -131,43 +131,74 @@ the user whose change is holding the lock. currently held, this shows when the lock was acquired. +Cluster Failure Modes +===================== + +There are three major cluster failure modes: + + - **Write Interruptions**: A write started but did not complete, leaving + the disk state and cluster state out of sync. + - **Loss of Leaders**: None of the devices with the most up-to-date data + are reachable. + - **Ambiguous Leaders**: The internal state of the repository is unclear. + +Phabricator can detect these issues, and responds by freezing the repository +(usually preventing all reads and writes) until the issue is resolved. These +conditions are normally rare and very little data is at risk, but Phabricator +errs on the side of caution and requires decisions which may result in data +loss to be confirmed by a human. + +The next sections cover these failure modes and appropriate responses in +more detail. In general, you will respond to these issues by assessing the +situation and then possibly choosing to discard some data. + + Write Interruptions =================== A repository cluster can be put into an inconsistent state by an interruption -in a brief window during and immediately after a write. +in a brief window during and immediately after a write. This looks like this: + + - A change is pushed to a server. + - The server acquires a write lock and begins writing the change. + - During or immediately after the write, lightning strikes the server + and destroys it. Phabricator can not commit changes to a working copy (stored on disk) and to -the global state (stored in a database) atomically, so there is a narrow window -between committing these two different states when some tragedy (like a -lightning strike) can befall a server, leaving the global and local views of -the repository state possibly divergent. +the global state (stored in a database) atomically, so there is necessarily a +narrow window between committing these two different states when some tragedy +can befall a server, leaving the global and local views of the repository state +possibly divergent. In these cases, Phabricator fails into a frozen state where further writes -are not permitted until the failure is investigated and resolved. +are not permitted until the failure is investigated and resolved. When a +repository is frozen in this way it remains readable. You can use the monitoring console to review the state of a frozen repository -with a held write lock. The **Writing** column will show which node is holding -the lock, and whoever is named in the **Last Writer** column may be able to -help you figure out what happened by providing more information about what they -were doing and what they observed. +with a held write lock. The **Writing** column will show which device is +holding the lock, and whoever is named in the **Last Writer** column may be +able to help you figure out what happened by providing more information about +what they were doing and what they observed. -Because the push was not acknowledged, it is normally safe to demote the node: -the user should have received an error anyway, and should not expect their push -to have worked. However, data is technically at risk and you may want to -investigate further and try to understand the issue in more detail before +Because the push was not acknowledged, it is normally safe to resolve this +issue by demoting the device. Demoting the device will undo any changes +committed by the push, and they will be lost forever. + +However, the user should have received an error anyway, and should not expect +their push to have worked. Still, data is technically at risk and you may want +to investigate further and try to understand the issue in more detail before continuing. There is no way to explicitly keep the write, but if it was committed to disk -you can recover it manually from the working copy on the device and then push -it again. +you can recover it manually from the working copy on the device (for example, +by using `git format-patch`) and then push it again after recovering. -If you demote the node, the in-process write will be thrown away, even if it -was complete on disk. To demote the node and release the write lock, run this +If you demote the device, the in-process write will be thrown away, even if it +was complete on disk. To demote the device and release the write lock, run this command: ``` -phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net +phabricator/ $ ./bin/repository thaw --demote ``` {icon exclamation-triangle, color="yellow"} Any committed but unacknowledged @@ -181,17 +212,18 @@ A more straightforward failure condition is the loss of all servers in a cluster which have the most up-to-date copy of a repository. This looks like this: - - There is a cluster setup with two nodes, X and Y. + - There is a cluster setup with two devices, X and Y. - A new change is pushed to server X. - Before the change can propagate to server Y, lightning strikes server X and destroys it. -Here, all of the "leader" nodes with the most up-to-date copy of the repository -have been lost. Phabricator will refuse to serve this repository because it -can not serve it consistently, and can not accept writes without data loss. +Here, all of the "leader" devices with the most up-to-date copy of the +repository have been lost. Phabricator will freeze the repository refuse to +serve requests because it can not serve it consistently, and can not accept new +writes without data loss. The most straightforward way to resolve this issue is to restore any leader to -service. The change will be able to replicate to other nodes once a leader +service. The change will be able to replicate to other devices once a leader comes back online. If you are unable to restore a leader or unsure that you can restore one @@ -201,13 +233,20 @@ push logs. If you are comfortable discarding these changes, you can instruct Phabricator that it can forget about the leaders in two ways: disable the service bindings -to all of the leader nodes so they are no longer part of the cluster, or -use `bin/repository thaw` to `--demote` the leaders explicitly. +to all of the leader devices so they are no longer part of the cluster, or use +`bin/repository thaw` to `--demote` the leaders explicitly. If you do this, **you will lose data**. Either action will discard any changes -on the affected leaders which have not replicated to other nodes in the cluster. +on the affected leaders which have not replicated to other devices in the +cluster. -To demote a device, run this command: +To remove a device from the cluster, disable all of the bindings to it +in Almanac, using the web UI. + +{icon exclamation-triangle, color="red"} Any data which is only present on +the disabled device will be lost. + +To demote a device without removing it from the cluster, run this command: ``` phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net @@ -220,24 +259,35 @@ phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net Ambiguous Leaders ================= -Repository clusters can also freeze if the leader nodes are ambiguous. This +Repository clusters can also freeze if the leader devices are ambiguous. This can happen if you replace an entire cluster with new devices suddenly, or -make a mistake with the `--demote` flag. +make a mistake with the `--demote` flag. This generally arises from some kind +of operator error, like this: -When Phabricator can not tell which node in a cluster is a leader, it freezes -the cluster because it is possible that some nodes have less data and others + - Someone accidentally uses `bin/repository thaw ... --demote` to demote + every device in a cluster. + - Someone accidentally deletes all the version information for a repository + from the database by making a mistake with a `DELETE` or `UPDATE` query. + - Someone accidentally disable all of the devices in a cluster, then add + entirely new ones before repositories can propagate. + +When Phabricator can not tell which device in a cluster is a leader, it freezes +the cluster because it is possible that some devices have less data and others have more, and if it choses a leader arbitrarily it may destroy some data which you would prefer to retain. -To resolve this, you need to tell Phabricator which node has the most -up-to-date data and promote that node to become a leader. If you do this, -**you may lose data** if you promote the wrong node, and some other node -really had more up-to-date data. If you want to double check, you can examine -the working copies on disk before promoting, by connecting to the machines and -using commands like `git log` to inspect state. +To resolve this, you need to tell Phabricator which device has the most +up-to-date data and promote that device to become a leader. If you know all +devices have the same data, you are free to promote any device. -Once you have identified a node which has data you're happy with, use -`bin/repository thaw` to `--promote` the device: +If you promote a device, **you may lose data** if you promote the wrong device +and some other device really had more up-to-date data. If you want to double +check, you can examine the working copies on disk before promoting by +connecting to the machines and using commands like `git log` to inspect state. + +Once you have identified a device which has data you're happy with, use +`bin/repository thaw` to `--promote` the device. The data on the chosen +device will become authoritative: ``` phabricator/ $ ./bin/repository thaw rXYZ --promote repo002.corp.net From 3b154a34c7e6f2827fdc542f5f5d77b4e9379dcf Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 10:52:23 -0700 Subject: [PATCH 37/50] Use less hip lingo Summary: Woah man. Test Plan: spellcheck Reviewers: chad, eadler Reviewed By: chad, eadler Differential Revision: https://secure.phabricator.com/D15771 --- .../transactions/editengine/PhabricatorEditEngineAPIMethod.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php b/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php index e478de1e20..7bd09df18a 100644 --- a/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php +++ b/src/applications/transactions/editengine/PhabricatorEditEngineAPIMethod.php @@ -18,7 +18,7 @@ abstract class PhabricatorEditEngineAPIMethod public function getMethodStatusDescription() { return pht( 'ApplicationEditor methods are fairly stable, but were introduced '. - 'relativelyr cently and may continue to evolve as more applications '. + 'relatively recently and may continue to evolve as more applications '. 'adopt them.'); } From df8c3c4fa596e0d18719078dbe2245db9c420a73 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 14:24:24 -0700 Subject: [PATCH 38/50] Give application actors in feed reasonable icons Summary: Ref T8952. Currently, when an application (most commonly Herald, but sometimes Drydock, Diffusion, etc) publishes a feed story, we get an empty grey box for it in feed. Instead, give the story a little application icon kind of "profile picture"-like thing. Test Plan: Here's how it looks: {F1239003} Feel free to tweak/counter-diff. Reviewers: chad Reviewed By: chad Maniphest Tasks: T8952 Differential Revision: https://secure.phabricator.com/D15773 --- resources/celerity/map.php | 6 ++--- ...bricatorApplicationApplicationPHIDType.php | 6 +++-- ...ricatorApplicationTransactionFeedStory.php | 11 +++++++-- src/view/phui/PHUIFeedStoryView.php | 24 +++++++++++++++++-- webroot/rsrc/css/phui/phui-feed-story.css | 19 ++++++++++++--- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 087103c912..b0731d69eb 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -7,7 +7,7 @@ */ return array( 'names' => array( - 'core.pkg.css' => '31417876', + 'core.pkg.css' => '04a95108', 'core.pkg.js' => '37344f3c', 'darkconsole.pkg.js' => 'e7393ebb', 'differential.pkg.css' => '7ba78475', @@ -131,7 +131,7 @@ return array( 'rsrc/css/phui/phui-document-pro.css' => '73e45fd2', 'rsrc/css/phui/phui-document-summary.css' => '9ca48bdf', 'rsrc/css/phui/phui-document.css' => '9c71d2bf', - 'rsrc/css/phui/phui-feed-story.css' => '04aec08f', + 'rsrc/css/phui/phui-feed-story.css' => 'd8440402', 'rsrc/css/phui/phui-fontkit.css' => '9cda225e', 'rsrc/css/phui/phui-form-view.css' => '6a51768e', 'rsrc/css/phui/phui-form.css' => 'aac1d51d', @@ -823,7 +823,7 @@ return array( 'phui-document-summary-view-css' => '9ca48bdf', 'phui-document-view-css' => '9c71d2bf', 'phui-document-view-pro-css' => '73e45fd2', - 'phui-feed-story-css' => '04aec08f', + 'phui-feed-story-css' => 'd8440402', 'phui-font-icon-base-css' => '6449bce8', 'phui-fontkit-css' => '9cda225e', 'phui-form-css' => 'aac1d51d', diff --git a/src/applications/meta/phid/PhabricatorApplicationApplicationPHIDType.php b/src/applications/meta/phid/PhabricatorApplicationApplicationPHIDType.php index d0b1f763a4..946f56616d 100644 --- a/src/applications/meta/phid/PhabricatorApplicationApplicationPHIDType.php +++ b/src/applications/meta/phid/PhabricatorApplicationApplicationPHIDType.php @@ -37,8 +37,10 @@ final class PhabricatorApplicationApplicationPHIDType foreach ($handles as $phid => $handle) { $application = $objects[$phid]; - $handle->setName($application->getName()); - $handle->setURI($application->getApplicationURI()); + $handle + ->setName($application->getName()) + ->setURI($application->getApplicationURI()) + ->setIcon($application->getIcon()); } } diff --git a/src/applications/transactions/feed/PhabricatorApplicationTransactionFeedStory.php b/src/applications/transactions/feed/PhabricatorApplicationTransactionFeedStory.php index 5e9d7c40b3..4e5c0e3ba1 100644 --- a/src/applications/transactions/feed/PhabricatorApplicationTransactionFeedStory.php +++ b/src/applications/transactions/feed/PhabricatorApplicationTransactionFeedStory.php @@ -99,8 +99,15 @@ class PhabricatorApplicationTransactionFeedStory } } - $view->setImage( - $this->getHandle($xaction->getAuthorPHID())->getImageURI()); + $author_phid = $xaction->getAuthorPHID(); + $author_handle = $this->getHandle($author_phid); + $author_image = $author_handle->getImageURI(); + + if ($author_image) { + $view->setImage($author_image); + } else { + $view->setAuthorIcon($author_handle->getIcon()); + } return $view; } diff --git a/src/view/phui/PHUIFeedStoryView.php b/src/view/phui/PHUIFeedStoryView.php index 978e25062f..8267fa0197 100644 --- a/src/view/phui/PHUIFeedStoryView.php +++ b/src/view/phui/PHUIFeedStoryView.php @@ -16,6 +16,7 @@ final class PHUIFeedStoryView extends AphrontView { private $actions = array(); private $chronologicalKey; private $tags; + private $authorIcon; public function setTags($tags) { $this->tags = $tags; @@ -82,6 +83,15 @@ final class PHUIFeedStoryView extends AphrontView { return $this; } + public function setAuthorIcon($author_icon) { + $this->authorIcon = $author_icon; + return $this; + } + + public function getAuthorIcon() { + return $this->authorIcon; + } + public function setTokenBar(array $tokens) { $this->tokenBar = $tokens; return $this; @@ -163,8 +173,18 @@ final class PHUIFeedStoryView extends AphrontView { $foot = null; $actor = new PHUIIconView(); - $actor->setImage($this->image); - $actor->addClass('phui-feed-story-actor-image'); + $actor->addClass('phui-feed-story-actor'); + + $author_icon = $this->getAuthorIcon(); + + if ($this->image) { + $actor->addClass('phui-feed-story-actor-image'); + $actor->setImage($this->image); + } else if ($author_icon) { + $actor->addClass('phui-feed-story-actor-icon'); + $actor->setIcon($author_icon); + } + if ($this->imageHref) { $actor->setHref($this->imageHref); } diff --git a/webroot/rsrc/css/phui/phui-feed-story.css b/webroot/rsrc/css/phui/phui-feed-story.css index cc0bf279fd..0d4b98a461 100644 --- a/webroot/rsrc/css/phui/phui-feed-story.css +++ b/webroot/rsrc/css/phui/phui-feed-story.css @@ -10,16 +10,29 @@ border: none; } -.phui-feed-story-head .phui-feed-story-actor-image { +.phui-feed-story-head .phui-feed-story-actor { width: 35px; height: 35px; - background-size: 35px; float: left; margin-right: 8px; - box-shadow: {$borderinset}; border-radius: 3px; + box-shadow: {$borderinset}; } +.phui-feed-story-head .phui-feed-story-actor-image { + background-size: 35px; +} + +.phui-feed-story-head .phui-feed-story-actor-icon { + text-align: center; + vertical-align: middle; + font-size: 24px; + line-height: 35px; + color: {$lightgreytext}; + background: {$greybackground}; +} + + .phui-feed-story-head { padding: 12px 4px; overflow: hidden; From 9419e4f13a5372c06371769630c0a6936af754ba Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 14:35:20 -0700 Subject: [PATCH 39/50] Reduce strength of Herald and user subscription stories Summary: Fixes T8952. When Herald changes subscribers, it is zzzzz very boring. When users change subscribers, it is still super boring (more boring than a merge, for example). Test Plan: Viewed feed, saw fewer Herald stories. Reviewers: chad Reviewed By: chad Maniphest Tasks: T8952 Differential Revision: https://secure.phabricator.com/D15774 --- .../PhabricatorApplicationTransaction.php | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index e3fe7707f6..ab22111647 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -1227,7 +1227,17 @@ abstract class PhabricatorApplicationTransaction // Make this weaker than TYPE_COMMENT. return 0.25; } - break; + + if ($this->isApplicationAuthor()) { + // When applications (most often: Herald) change subscriptions it + // is very uninteresting. + return 0.000000001; + } + + // In other cases, subscriptions are more interesting than comments + // (which are shown anyway) but less interesting than any other type of + // transaction. + return 0.75; } return 1.0; @@ -1462,6 +1472,14 @@ abstract class PhabricatorApplicationTransaction return true; } + private function isApplicationAuthor() { + $author_phid = $this->getAuthorPHID(); + $author_type = phid_get_type($author_phid); + $application_type = PhabricatorApplicationApplicationPHIDType::TYPECONST; + return ($author_type == $application_type); + } + + private function getInterestingMoves(array $moves) { // Remove moves which only shift the position of a task within a column. foreach ($moves as $key => $move) { From 93e341fbdad3b73bcff2de76ec9ac084cfeb30f0 Mon Sep 17 00:00:00 2001 From: Joshua Spence Date: Thu, 21 Apr 2016 09:25:39 +1000 Subject: [PATCH 40/50] Fix `./bin/aphlict status` Summary: Fixes T10844. After recent changes to Aphlict (T6915 and T10697), `./bin/status` needs to be aware of the configuration file. As such, it is now necessary to run `./bin/aphlict status --config /path/to/config.json` rather than `./bin/aphlict status`. Test Plan: Ran `./bin/aphlict start ...` and `./bin/aphlict status` and saw "Aphlict (`$PID`) is running". Reviewers: epriestley, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T10844 Differential Revision: https://secure.phabricator.com/D15776 --- .../management/PhabricatorAphlictManagementStatusWorkflow.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/applications/aphlict/management/PhabricatorAphlictManagementStatusWorkflow.php b/src/applications/aphlict/management/PhabricatorAphlictManagementStatusWorkflow.php index c9a0768bf2..bfc5ac32a6 100644 --- a/src/applications/aphlict/management/PhabricatorAphlictManagementStatusWorkflow.php +++ b/src/applications/aphlict/management/PhabricatorAphlictManagementStatusWorkflow.php @@ -7,10 +7,11 @@ final class PhabricatorAphlictManagementStatusWorkflow $this ->setName('status') ->setSynopsis(pht('Show the status of the notification server.')) - ->setArguments(array()); + ->setArguments($this->getLaunchArguments()); } public function execute(PhutilArgumentParser $args) { + $this->parseLaunchArguments($args); $console = PhutilConsole::getConsole(); $pid = $this->getPID(); From 34c488e1651ee34edabd3af4231fef385d52b7d5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 19:04:22 -0700 Subject: [PATCH 41/50] Normalize Phriction links when looking them up in remarkup Summary: Fixes T10845. Test Plan: Verified that `[[ quack ]]` and `[[ QUACK ]]` both work. Previously, the link had to exactly match the capitalization of the target. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10845 Differential Revision: https://secure.phabricator.com/D15777 --- .../phriction/markup/PhrictionRemarkupRule.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index 5f5a91a280..79e3788d11 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -104,6 +104,7 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { foreach ($metadata as $spec) { $link = $spec['link']; + $slug = PhabricatorSlug::normalize($link); $name = $spec['explicitName']; $class = 'phriction-link'; @@ -111,24 +112,24 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { // in text as: "Title" . Otherwise, we'll just render: . $is_interesting_name = (bool)strlen($name); - if (idx($existant_documents, $link) === null) { + if (idx($existant_documents, $slug) === null) { // The target document doesn't exist. if ($name === null) { - $name = explode('/', trim($link, '/')); + $name = explode('/', trim($slug, '/')); $name = end($name); } $class = 'phriction-link-missing'; - } else if (idx($visible_documents, $link) === null) { + } else if (idx($visible_documents, $slug) === null) { // The document exists, but the user can't see it. if ($name === null) { - $name = explode('/', trim($link, '/')); + $name = explode('/', trim($slug, '/')); $name = end($name); } $class = 'phriction-link-lock'; } else { if ($name === null) { // Use the title of the document if no name is set. - $name = $visible_documents[$link] + $name = $visible_documents[$slug] ->getContent() ->getTitle(); From c986caebb21f7f7119fd45e07a4427ab7f58632b Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 20:21:36 -0700 Subject: [PATCH 42/50] Put all cluster docs in the right documentation group Summary: Some of these had the wrong `@group` header. Test Plan: `grep` Reviewers: chad Reviewed By: chad Differential Revision: https://secure.phabricator.com/D15778 --- src/docs/user/cluster/cluster_daemons.diviner | 2 +- src/docs/user/cluster/cluster_databases.diviner | 2 +- src/docs/user/cluster/cluster_notifications.diviner | 2 +- src/docs/user/cluster/cluster_repositories.diviner | 2 +- src/docs/user/cluster/cluster_webservers.diviner | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/docs/user/cluster/cluster_daemons.diviner b/src/docs/user/cluster/cluster_daemons.diviner index f6aa1cbe74..61cedd464d 100644 --- a/src/docs/user/cluster/cluster_daemons.diviner +++ b/src/docs/user/cluster/cluster_daemons.diviner @@ -1,5 +1,5 @@ @title Cluster: Daemons -@group intro +@group cluster Configuring Phabricator to use multiple daemon hosts. diff --git a/src/docs/user/cluster/cluster_databases.diviner b/src/docs/user/cluster/cluster_databases.diviner index 82be860db7..1a3ea1c86d 100644 --- a/src/docs/user/cluster/cluster_databases.diviner +++ b/src/docs/user/cluster/cluster_databases.diviner @@ -1,5 +1,5 @@ @title Cluster: Databases -@group intro +@group cluster Configuring Phabricator to use multiple database hosts. diff --git a/src/docs/user/cluster/cluster_notifications.diviner b/src/docs/user/cluster/cluster_notifications.diviner index f3837c869e..3dd4e9d903 100644 --- a/src/docs/user/cluster/cluster_notifications.diviner +++ b/src/docs/user/cluster/cluster_notifications.diviner @@ -1,5 +1,5 @@ @title Cluster: Notifications -@group intro +@group cluster Configuring Phabricator to use multiple notification servers. diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index ac9531c241..e9dd066783 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -1,5 +1,5 @@ @title Cluster: Repositories -@group intro +@group cluster Configuring Phabricator to use multiple repository hosts. diff --git a/src/docs/user/cluster/cluster_webservers.diviner b/src/docs/user/cluster/cluster_webservers.diviner index a1ebc9491b..95699a8291 100644 --- a/src/docs/user/cluster/cluster_webservers.diviner +++ b/src/docs/user/cluster/cluster_webservers.diviner @@ -1,5 +1,5 @@ @title Cluster: Web Servers -@group intro +@group cluster Configuring Phabricator to use multiple web servers. From fb2b88a4a81f12d8e338e466a08f6d4e0014e83a Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 21 Apr 2016 10:27:12 -0700 Subject: [PATCH 43/50] Fix Phriction link syntax a little more Summary: This still wasn't quite right -- a link like `[[ Porcupine Facts ]]` with a space would not lookup correctly, and would render as `porcupine_facts`. Test Plan: Verified that `[[ Porcupine Facts ]]` now works correctly. Reviewers: chad Reviewed By: chad Differential Revision: https://secure.phabricator.com/D15780 --- .../phriction/markup/PhrictionRemarkupRule.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/applications/phriction/markup/PhrictionRemarkupRule.php b/src/applications/phriction/markup/PhrictionRemarkupRule.php index 79e3788d11..c3a795a524 100644 --- a/src/applications/phriction/markup/PhrictionRemarkupRule.php +++ b/src/applications/phriction/markup/PhrictionRemarkupRule.php @@ -85,6 +85,9 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { } $slugs = ipull($metadata, 'link'); + foreach ($slugs as $key => $slug) { + $slugs[$key] = PhabricatorSlug::normalize($slug); + } // We have to make two queries here to distinguish between // documents the user can't see, and documents that don't @@ -115,14 +118,14 @@ final class PhrictionRemarkupRule extends PhutilRemarkupRule { if (idx($existant_documents, $slug) === null) { // The target document doesn't exist. if ($name === null) { - $name = explode('/', trim($slug, '/')); + $name = explode('/', trim($link, '/')); $name = end($name); } $class = 'phriction-link-missing'; } else if (idx($visible_documents, $slug) === null) { // The document exists, but the user can't see it. if ($name === null) { - $name = explode('/', trim($slug, '/')); + $name = explode('/', trim($link, '/')); $name = end($name); } $class = 'phriction-link-lock'; From bd8969a23cd93f26fdbad1b0d8faa333db53c1ae Mon Sep 17 00:00:00 2001 From: lkassianik Date: Thu, 21 Apr 2016 10:19:19 -0700 Subject: [PATCH 44/50] Calendar event list items 'Attending:' field should only show users who have confirmed attendance Summary: Fixes T8897 Test Plan: Open any list view of Calendar events, every event should only show "Attending: ..." with users who are attending event. Reviewers: chad, epriestley, #blessed_reviewers Reviewed By: epriestley, #blessed_reviewers Subscribers: Korvin Maniphest Tasks: T8897 Differential Revision: https://secure.phabricator.com/D15779 --- .../calendar/query/PhabricatorCalendarEventSearchEngine.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/applications/calendar/query/PhabricatorCalendarEventSearchEngine.php b/src/applications/calendar/query/PhabricatorCalendarEventSearchEngine.php index c23996470d..8b321ef73e 100644 --- a/src/applications/calendar/query/PhabricatorCalendarEventSearchEngine.php +++ b/src/applications/calendar/query/PhabricatorCalendarEventSearchEngine.php @@ -271,7 +271,10 @@ final class PhabricatorCalendarEventSearchEngine $attendees = array(); foreach ($event->getInvitees() as $invitee) { - $attendees[] = $invitee->getInviteePHID(); + $status_attending = PhabricatorCalendarEventInvitee::STATUS_ATTENDING; + if ($invitee->getStatus() === $status_attending) { + $attendees[] = $invitee->getInviteePHID(); + } } if ($event->getIsGhostEvent()) { From 9656fe48bcfead5d315692518c52eed6c8e4d15d Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 11:06:13 -0700 Subject: [PATCH 45/50] Add a "Repository Servers" cluster administration panel Summary: Ref T4292. This adds a new high-level overview panel. Test Plan: {F1238854} Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15772 --- src/__phutil_library_map__.php | 2 + .../PhabricatorConfigApplication.php | 1 + ...torConfigClusterRepositoriesController.php | 343 ++++++++++++++++++ .../PhabricatorConfigController.php | 1 + .../user/cluster/cluster_repositories.diviner | 39 +- 5 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 src/applications/config/controller/PhabricatorConfigClusterRepositoriesController.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 95cee8daba..65eb7f6c20 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2055,6 +2055,7 @@ phutil_register_library_map(array( 'PhabricatorConfigCacheController' => 'applications/config/controller/PhabricatorConfigCacheController.php', 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php', + 'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -6502,6 +6503,7 @@ phutil_register_library_map(array( 'PhabricatorConfigCacheController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php index bdaf0b2cc0..62fe79f4ce 100644 --- a/src/applications/config/application/PhabricatorConfigApplication.php +++ b/src/applications/config/application/PhabricatorConfigApplication.php @@ -65,6 +65,7 @@ final class PhabricatorConfigApplication extends PhabricatorApplication { 'cluster/' => array( 'databases/' => 'PhabricatorConfigClusterDatabasesController', 'notifications/' => 'PhabricatorConfigClusterNotificationsController', + 'repositories/' => 'PhabricatorConfigClusterRepositoriesController', ), ), ); diff --git a/src/applications/config/controller/PhabricatorConfigClusterRepositoriesController.php b/src/applications/config/controller/PhabricatorConfigClusterRepositoriesController.php new file mode 100644 index 0000000000..d03d58a954 --- /dev/null +++ b/src/applications/config/controller/PhabricatorConfigClusterRepositoriesController.php @@ -0,0 +1,343 @@ +buildSideNavView(); + $nav->selectFilter('cluster/repositories/'); + + $title = pht('Repository Servers'); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb(pht('Repository Servers')); + + $repository_status = $this->buildClusterRepositoryStatus(); + + $view = id(new PHUITwoColumnView()) + ->setNavigation($nav) + ->setMainColumn($repository_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->appendChild($view); + } + + private function buildClusterRepositoryStatus() { + $viewer = $this->getViewer(); + + Javelin::initBehavior('phabricator-tooltips'); + + $all_services = id(new AlmanacServiceQuery()) + ->setViewer($viewer) + ->withServiceTypes( + array( + AlmanacClusterRepositoryServiceType::SERVICETYPE, + )) + ->needBindings(true) + ->needProperties(true) + ->execute(); + $all_services = mpull($all_services, null, 'getPHID'); + + $all_repositories = id(new PhabricatorRepositoryQuery()) + ->setViewer($viewer) + ->withHosted(PhabricatorRepositoryQuery::HOSTED_PHABRICATOR) + ->withTypes( + array( + PhabricatorRepositoryType::REPOSITORY_TYPE_GIT, + )) + ->execute(); + $all_repositories = mpull($all_repositories, null, 'getPHID'); + + $all_versions = id(new PhabricatorRepositoryWorkingCopyVersion()) + ->loadAll(); + + $all_devices = $this->getDevices($all_services, false); + $all_active_devices = $this->getDevices($all_services, true); + + $leader_versions = $this->getLeaderVersionsByRepository( + $all_repositories, + $all_versions, + $all_active_devices); + + $push_times = $this->loadLeaderPushTimes($leader_versions); + + $repository_groups = mgroup($all_repositories, 'getAlmanacServicePHID'); + $repository_versions = mgroup($all_versions, 'getRepositoryPHID'); + + $rows = array(); + foreach ($all_services as $service) { + $service_phid = $service->getPHID(); + + if ($service->getAlmanacPropertyValue('closed')) { + $status_icon = 'fa-folder'; + $status_tip = pht('Closed'); + } else { + $status_icon = 'fa-folder-open green'; + $status_tip = pht('Open'); + } + + $status_icon = id(new PHUIIconView()) + ->setIcon($status_icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $status_tip, + )); + + $devices = idx($all_devices, $service_phid, array()); + $active_devices = idx($all_active_devices, $service_phid, array()); + + $device_icon = 'fa-server green'; + + $device_label = pht( + '%s Active', + phutil_count($active_devices)); + + $device_status = array( + id(new PHUIIconView())->setIcon($device_icon), + ' ', + $device_label, + ); + + $repositories = idx($repository_groups, $service_phid, array()); + + $repository_status = pht( + '%s', + phutil_count($repositories)); + + $no_leader = array(); + $full_sync = array(); + $partial_sync = array(); + $no_sync = array(); + $lag = array(); + + // Threshold in seconds before we start complaining that repositories + // are not synchronized when there is only one leader. + $threshold = phutil_units('5 minutes in seconds'); + + $messages = array(); + + foreach ($repositories as $repository) { + $repository_phid = $repository->getPHID(); + + $leader_version = idx($leader_versions, $repository_phid); + if ($leader_version === null) { + $no_leader[] = $repository; + $messages[] = pht( + 'Repository %s has an ambiguous leader.', + $viewer->renderHandle($repository_phid)->render()); + continue; + } + + $versions = idx($repository_versions, $repository_phid, array()); + + $leaders = 0; + foreach ($versions as $version) { + if ($version->getRepositoryVersion() == $leader_version) { + $leaders++; + } + } + + if ($leaders == count($active_devices)) { + $full_sync[] = $repository; + } else { + $push_epoch = idx($push_times, $repository_phid); + if ($push_epoch) { + $duration = (PhabricatorTime::getNow() - $push_epoch); + $lag[] = $duration; + } else { + $duration = null; + } + + if ($leaders >= 2 || ($duration && ($duration < $threshold))) { + $partial_sync[] = $repository; + } else { + $no_sync[] = $repository; + if ($push_epoch) { + $messages[] = pht( + 'Repository %s has unreplicated changes (for %s).', + $viewer->renderHandle($repository_phid)->render(), + phutil_format_relative_time($duration)); + } else { + $messages[] = pht( + 'Repository %s has unreplicated changes.', + $viewer->renderHandle($repository_phid)->render()); + } + } + + } + } + + $with_lag = false; + + if ($no_leader) { + $replication_icon = 'fa-times red'; + $replication_label = pht('Ambiguous Leader'); + } else if ($no_sync) { + $replication_icon = 'fa-refresh yellow'; + $replication_label = pht('Unsynchronized'); + $with_lag = true; + } else if ($partial_sync) { + $replication_icon = 'fa-refresh green'; + $replication_label = pht('Partial'); + $with_lag = true; + } else if ($full_sync) { + $replication_icon = 'fa-check green'; + $replication_label = pht('Synchronized'); + } else { + $replication_icon = 'fa-times grey'; + $replication_label = pht('No Repositories'); + } + + if ($with_lag && $lag) { + $lag_status = phutil_format_relative_time(max($lag)); + $lag_status = pht(' (%s)', $lag_status); + } else { + $lag_status = null; + } + + $replication_status = array( + id(new PHUIIconView())->setIcon($replication_icon), + ' ', + $replication_label, + $lag_status, + ); + + $messages = phutil_implode_html(phutil_tag('br'), $messages); + + $rows[] = array( + $status_icon, + $viewer->renderHandle($service->getPHID()), + $device_status, + $repository_status, + $replication_status, + $messages, + ); + } + + + $table = id(new AphrontTableView($rows)) + ->setNoDataString( + pht('No repository cluster services are configured.')) + ->setHeaders( + array( + null, + pht('Service'), + pht('Devices'), + pht('Repos'), + pht('Sync'), + pht('Messages'), + )) + ->setColumnClasses( + array( + null, + 'pri', + null, + null, + null, + 'wide', + )); + + $doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories'); + + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Cluster Repository Status')) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + return id(new PHUIObjectBoxView()) + ->setHeader($header) + ->setTable($table); + } + + private function getDevices( + array $all_services, + $only_active) { + + $devices = array(); + foreach ($all_services as $service) { + $map = array(); + foreach ($service->getBindings() as $binding) { + if ($only_active && $binding->getIsDisabled()) { + continue; + } + + $device = $binding->getDevice(); + $device_phid = $device->getPHID(); + + $map[$device_phid] = $device; + } + $devices[$service->getPHID()] = $map; + } + + return $devices; + } + + private function getLeaderVersionsByRepository( + array $all_repositories, + array $all_versions, + array $active_devices) { + + $version_map = mgroup($all_versions, 'getRepositoryPHID'); + + $result = array(); + foreach ($all_repositories as $repository_phid => $repository) { + $service_phid = $repository->getAlmanacServicePHID(); + if (!$service_phid) { + continue; + } + + $devices = idx($active_devices, $service_phid); + if (!$devices) { + continue; + } + + $versions = idx($version_map, $repository_phid, array()); + $versions = mpull($versions, null, 'getDevicePHID'); + $versions = array_select_keys($versions, array_keys($devices)); + if (!$versions) { + continue; + } + + $leader = (int)max(mpull($versions, 'getRepositoryVersion')); + $result[$repository_phid] = $leader; + } + + return $result; + } + + private function loadLeaderPushTimes(array $leader_versions) { + $viewer = $this->getViewer(); + + if (!$leader_versions) { + return array(); + } + + $events = id(new PhabricatorRepositoryPushEventQuery()) + ->setViewer($viewer) + ->withIDs($leader_versions) + ->execute(); + $events = mpull($events, null, 'getID'); + + $result = array(); + foreach ($leader_versions as $key => $version) { + $event = idx($events, $version); + if (!$event) { + continue; + } + + $result[$key] = $event->getEpoch(); + } + + return $result; + } + + +} diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php index c390688930..ac629f85b5 100644 --- a/src/applications/config/controller/PhabricatorConfigController.php +++ b/src/applications/config/controller/PhabricatorConfigController.php @@ -25,6 +25,7 @@ abstract class PhabricatorConfigController extends PhabricatorController { $nav->addLabel(pht('Cluster')); $nav->addFilter('cluster/databases/', pht('Database Servers')); $nav->addFilter('cluster/notifications/', pht('Notification Servers')); + $nav->addFilter('cluster/repositories/', pht('Repository Servers')); $nav->addLabel(pht('Welcome')); $nav->addFilter('welcome/', pht('Welcome Screen')); $nav->addLabel(pht('Modules')); diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index e9dd066783..bb6019ee4d 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -95,14 +95,41 @@ Other mitigations are possible, but securing a network against the NSA and similar agents of other rogue nations is beyond the scope of this document. -Monitoring Replication -====================== +Monitoring Services +=================== -You can review the current status of a repository on cluster devices in -{nav Diffusion > (Repository) > Manage Repository > Cluster Configuration}. +You can get an overview of repository cluster status from the +{nav Config > Repository Servers} screen. This table shows a high-level +overview of all active repository services. + +**Repos**: The number of repositories hosted on this service. + +**Sync**: Synchronization status of repositories on this service. This is an +at-a-glance view of service health, and can show these values: + + - **Synchronized**: All nodes are fully synchronized and have the latest + version of all repositories. + - **Partial**: All repositories either have at least two leaders, or have + a very recent write which is not expected to have propagated yet. + - **Unsynchronized**: At least one repository has changes which are + only available on one node and were not pushed very recently. Data may + be at risk. + - **No Repositories**: This service has no repositories. + - **Ambiguous Leader**: At least one repository has an ambiguous leader. + +If this screen identifies problems, you can drill down into repository details +to get more information about them. See the next section for details. + + +Monitoring Repositories +======================= + +You can get a more detailed view the current status of a specific repository on +cluster devices in {nav Diffusion > (Repository) > Manage Repository > Cluster +Configuration}. This screen shows all the configured devices which are hosting the repository -and the available version. +and the available version on that device. **Version**: When a repository is mutated by a push, Phabricator increases an internal version number for the repository. This column shows which version @@ -131,6 +158,8 @@ the user whose change is holding the lock. currently held, this shows when the lock was acquired. + + Cluster Failure Modes ===================== From 43935d5916f25756f620ca77056a648e5a65176c Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 20 Apr 2016 15:14:35 -0700 Subject: [PATCH 46/50] Don't cache resources we can't generate properly Summary: Fixes T10843. In a multi-server setup, we can do this: - Two servers, A and B. - You push an update. - A gets pushed first. - After A has been pushed, but before B has been pushed, a user loads a page from A. - It generates resource URIs like `/stuff/new/package.css`. - Those requests hit B. - B doesn't have the new resources yet. - It responds with old resources. - Your CDN caches things. You now have a poisoned CDN: old data is saved in a new URL. To try to avoid this with as little work as possible and generally make it hard to get wrong, check the URL hash against the hash we would generate. If they don't match, serve our best guess at the resource, but don't cache it. This should make things mostly keep working during the push, but prevent caches from becoming poisoned, and everyone should get a working version of everything after the push finishes. Test Plan: - `curl`'d a resource, got a cacheable one. - Changed the hash a little, `curl`'d again. This time: valid resource, but not cacheable. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10843 Differential Revision: https://secure.phabricator.com/D15775 --- .../celerity/CelerityResourceMap.php | 5 ++++ .../CelerityPhabricatorResourceController.php | 6 +++- .../controller/CelerityResourceController.php | 29 +++++++++++++------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/applications/celerity/CelerityResourceMap.php b/src/applications/celerity/CelerityResourceMap.php index 03ccfe607d..e53b656aaf 100644 --- a/src/applications/celerity/CelerityResourceMap.php +++ b/src/applications/celerity/CelerityResourceMap.php @@ -208,6 +208,11 @@ final class CelerityResourceMap extends Phobject { } + public function getHashForName($name) { + return idx($this->nameMap, $name); + } + + /** * Return the absolute URI for a resource, identified by hash. * This method is fairly low-level and ignores packaging. diff --git a/src/applications/celerity/controller/CelerityPhabricatorResourceController.php b/src/applications/celerity/controller/CelerityPhabricatorResourceController.php index c5f878b61e..5751996db8 100644 --- a/src/applications/celerity/controller/CelerityPhabricatorResourceController.php +++ b/src/applications/celerity/controller/CelerityPhabricatorResourceController.php @@ -31,7 +31,11 @@ final class CelerityPhabricatorResourceController return new Aphront400Response(); } - return $this->serveResource($this->path); + return $this->serveResource( + array( + 'path' => $this->path, + 'hash' => $this->hash, + )); } protected function buildResourceTransformer() { diff --git a/src/applications/celerity/controller/CelerityResourceController.php b/src/applications/celerity/controller/CelerityResourceController.php index 4e6feeac6c..debb37c1ca 100644 --- a/src/applications/celerity/controller/CelerityResourceController.php +++ b/src/applications/celerity/controller/CelerityResourceController.php @@ -24,7 +24,10 @@ abstract class CelerityResourceController extends PhabricatorController { abstract public function getCelerityResourceMap(); - protected function serveResource($path, $package_hash = null) { + protected function serveResource(array $spec) { + $path = $spec['path']; + $hash = idx($spec, 'hash'); + // Sanity checking to keep this from exposing anything sensitive, since it // ultimately boils down to disk reads. if (preg_match('@(//|\.\.)@', $path)) { @@ -40,18 +43,24 @@ abstract class CelerityResourceController extends PhabricatorController { $dev_mode = PhabricatorEnv::getEnvConfig('phabricator.developer-mode'); - if (AphrontRequest::getHTTPHeader('If-Modified-Since') && !$dev_mode) { + $map = $this->getCelerityResourceMap(); + $expect_hash = $map->getHashForName($path); + + // Test if the URI hash is correct for our current resource map. If it + // is not, refuse to cache this resource. This avoids poisoning caches + // and CDNs if we're getting a request for a new resource to an old node + // shortly after a push. + $is_cacheable = ($hash === $expect_hash) && + $this->isCacheableResourceType($type); + if (AphrontRequest::getHTTPHeader('If-Modified-Since') && $is_cacheable) { // Return a "304 Not Modified". We don't care about the value of this // field since we never change what resource is served by a given URI. return $this->makeResponseCacheable(new Aphront304Response()); } - $is_cacheable = (!$dev_mode) && - $this->isCacheableResourceType($type); - $cache = null; $data = null; - if ($is_cacheable) { + if ($is_cacheable && !$dev_mode) { $cache = PhabricatorCaches::getImmutableCache(); $request_path = $this->getRequest()->getPath(); @@ -61,8 +70,6 @@ abstract class CelerityResourceController extends PhabricatorController { } if ($data === null) { - $map = $this->getCelerityResourceMap(); - if ($map->isPackageResource($path)) { $resource_names = $map->getResourceNamesForPackageName($path); if (!$resource_names) { @@ -117,7 +124,11 @@ abstract class CelerityResourceController extends PhabricatorController { $response->addAllowOrigin('*'); } - return $this->makeResponseCacheable($response); + if ($is_cacheable) { + $response = $this->makeResponseCacheable($response); + } + + return $response; } public static function getSupportedResourceTypes() { From 0f0105e783efb63fb9aeb10813ff8e9bd63055f4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 21 Apr 2016 16:58:25 -0700 Subject: [PATCH 47/50] Send the `aphlict` process log to the `node` log Summary: I've possibly seen a couple of `aphlict` processes exit under suspicious circumstances (maybe?). Make sure any PHP errors get captured into the log. Test Plan: - Added an exception after forking. - Before change: vanished into thin air. - After change: visible in the log. Reviewers: chad Reviewed By: chad Differential Revision: https://secure.phabricator.com/D15782 --- .../PhabricatorAphlictManagementWorkflow.php | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php b/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php index d45c9dd50a..3f9620d3be 100644 --- a/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php +++ b/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php @@ -301,7 +301,7 @@ abstract class PhabricatorAphlictManagementWorkflow return $pid; } - final public function cleanup($signo = '?') { + final public function cleanup($signo = null) { global $g_future; if ($g_future) { $g_future->resolveKill(); @@ -310,6 +310,11 @@ abstract class PhabricatorAphlictManagementWorkflow Filesystem::remove($this->getPIDPath()); + if ($signo !== null) { + $signame = phutil_get_signal_name($signo); + error_log("Caught signal {$signame}, exiting."); + } + exit(1); } @@ -428,6 +433,15 @@ abstract class PhabricatorAphlictManagementWorkflow $console = PhutilConsole::getConsole(); $this->willLaunch(); + $log = $this->getOverseerLogPath(); + if ($log !== null) { + echo tsprintf( + "%s\n", + pht( + 'Writing logs to: %s', + $log)); + } + $pid = pcntl_fork(); if ($pid < 0) { throw new Exception( @@ -439,6 +453,12 @@ abstract class PhabricatorAphlictManagementWorkflow exit(0); } + // Redirect process errors to the error log. If we do not do this, any + // error the `aphlict` process itself encounters vanishes into thin air. + if ($log !== null) { + ini_set('error_log', $log); + } + // When we fork, the child process will inherit its parent's set of open // file descriptors. If the parent process of bin/aphlict is waiting for // bin/aphlict's file descriptors to close, it will be stuck waiting on @@ -529,4 +549,15 @@ abstract class PhabricatorAphlictManagementWorkflow $server_argv); } + private function getOverseerLogPath() { + // For now, just return the first log. We could refine this eventually. + $logs = idx($this->configData, 'logs', array()); + + foreach ($logs as $log) { + return $log['path']; + } + + return null; + } + } From 711f13660e547d83aa5a6fa9580bb07ed49d62fd Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 22 Apr 2016 03:58:54 -0700 Subject: [PATCH 48/50] Synchronize working copies before doing a "bypassCache" commit read Summary: Ref T4292. When the daemons make a query for repository information, we need to make sure the working copy on disk is up to date before we serve the response, since we might not have the inforamtion we need to respond otherwise. We do this automatically for almost all Diffusion methods, but this particular method is a little unusual and does not get this check for free. Add this check. Test Plan: - Made this code throw. - Ran `bin/repository reparse --message ...`, saw the code get hit. - Ran `bin/repository lookup-user ...`, saw this code get hit. - Made this code not throw. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15783 --- .../conduit/DiffusionQueryCommitsConduitAPIMethod.php | 3 +++ .../PhabricatorRepositoryManagementLookupUsersWorkflow.php | 1 + .../PhabricatorRepositoryCommitMessageParserWorker.php | 1 + 3 files changed, 5 insertions(+) diff --git a/src/applications/diffusion/conduit/DiffusionQueryCommitsConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionQueryCommitsConduitAPIMethod.php index a71b21a724..bb205aad32 100644 --- a/src/applications/diffusion/conduit/DiffusionQueryCommitsConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionQueryCommitsConduitAPIMethod.php @@ -42,6 +42,9 @@ final class DiffusionQueryCommitsConduitAPIMethod ->executeOne(); if ($repository) { $query->withRepository($repository); + if ($bypass_cache) { + $repository->synchronizeWorkingCopyBeforeRead(); + } } } diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php index 949fd96141..e02a8dc05c 100644 --- a/src/applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php +++ b/src/applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php @@ -43,6 +43,7 @@ final class PhabricatorRepositoryManagementLookupUsersWorkflow )), 'diffusion.querycommits', array( + 'repositoryPHID' => $repo->getPHID(), 'phids' => array($commit->getPHID()), 'bypassCache' => true, )); diff --git a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php index 25eedf0be2..4e92a0f32e 100644 --- a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php +++ b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php @@ -23,6 +23,7 @@ abstract class PhabricatorRepositoryCommitMessageParserWorker )), 'diffusion.querycommits', array( + 'repositoryPHID' => $repository->getPHID(), 'phids' => array($commit->getPHID()), 'bypassCache' => true, 'needMessages' => true, From ab20f243b34183cf55fb1c2a84739b5c5c17c0d0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 22 Apr 2016 04:40:17 -0700 Subject: [PATCH 49/50] Improve consistency of file access policies, particularly for LFS Summary: Ref T7789. Currently, we use different viewers if you have `security.alternate-file-domain` configured vs if you do not. This is largely residual from the days of one-time-tokens, and can cause messy configuration-dependent bugs like the one in T7789#172057. Instead, always use the omnipotent viewer. Knowledge of the secret key alone is sufficient to access a file. Test Plan: - Disabled `security.alternate-file-domain`. - Reproduced an issue similar to the one described on T7789. - Applied change. - Clean LFS interaction. Reviewers: chad Reviewed By: chad Maniphest Tasks: T7789 Differential Revision: https://secure.phabricator.com/D15784 --- .../PhabricatorFileDataController.php | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/applications/files/controller/PhabricatorFileDataController.php b/src/applications/files/controller/PhabricatorFileDataController.php index dfae99d201..5c2fc7bbda 100644 --- a/src/applications/files/controller/PhabricatorFileDataController.php +++ b/src/applications/files/controller/PhabricatorFileDataController.php @@ -22,25 +22,21 @@ final class PhabricatorFileDataController extends PhabricatorFileController { $req_domain = $request->getHost(); $main_domain = id(new PhutilURI($base_uri))->getDomain(); - if (!strlen($alt) || $main_domain == $alt_domain) { // No alternate domain. $should_redirect = false; - $use_viewer = $viewer; $is_alternate_domain = false; } else if ($req_domain != $alt_domain) { // Alternate domain, but this request is on the main domain. $should_redirect = true; - $use_viewer = $viewer; $is_alternate_domain = false; } else { // Alternate domain, and on the alternate domain. $should_redirect = false; - $use_viewer = PhabricatorUser::getOmnipotentUser(); $is_alternate_domain = true; } - $response = $this->loadFile($use_viewer); + $response = $this->loadFile(); if ($response) { return $response; } @@ -112,7 +108,21 @@ final class PhabricatorFileDataController extends PhabricatorFileController { return $response; } - private function loadFile(PhabricatorUser $viewer) { + private function loadFile() { + // Access to files is provided by knowledge of a per-file secret key in + // the URI. Knowledge of this secret is sufficient to retrieve the file. + + // For some requests, we also have a valid viewer. However, for many + // requests (like alternate domain requests or Git LFS requests) we will + // not. Even if we do have a valid viewer, use the omnipotent viewer to + // make this logic simpler and more consistent. + + // Beyond making the policy check itself more consistent, this also makes + // sure we're consitent about returning HTTP 404 on bad requests instead + // of serving HTTP 200 with a login page, which can mislead some clients. + + $viewer = PhabricatorUser::getOmnipotentUser(); + $file = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withPHIDs(array($this->phid)) From 00885edc47d46caa9ff10ec133dfe503c07564f0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 22 Apr 2016 05:32:57 -0700 Subject: [PATCH 50/50] Don't try to synchronize repositories with no working copy Summary: Ref T4292. Sometimes, we may not have a working copy for a repository. The easiest way to get into this condition is to deactivate a repository. We could try to clone + fetch in this case, but that's kind of complex, and there's an easy command that administrators can run manually. For now, just tell them to do that. This affects the inactive repositories on `secure`, like rGITCOINS. Test Plan: Removed working copy, got message. Reviewers: chad Reviewed By: chad Maniphest Tasks: T4292 Differential Revision: https://secure.phabricator.com/D15786 --- .../repository/storage/PhabricatorRepository.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 088b33ae62..4a217d7e80 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -2528,8 +2528,22 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO private function synchronizeWorkingCopyFromBinding($binding) { $fetch_uri = $this->getClusterRepositoryURIFromBinding($binding); + $local_path = $this->getLocalPath(); if ($this->isGit()) { + if (!Filesystem::pathExists($local_path)) { + $device = AlmanacKeys::getLiveDevice(); + throw new Exception( + pht( + 'Repository "%s" does not have a working copy on this device '. + 'yet, so it can not be synchronized. Wait for the daemons to '. + 'construct one or run `bin/repository update %s` on this host '. + '("%s") to build it explicitly.', + $this->getDisplayName(), + $this->getMonogram(), + $device->getName())); + } + $argv = array( 'fetch --prune -- %s %s', $fetch_uri, @@ -2546,7 +2560,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO ->setProtocol($fetch_uri->getProtocol()) ->newFuture(); - $future->setCWD($this->getLocalPath()); + $future->setCWD($local_path); $future->resolvex(); }