From b2b17485b9571b118ff5e9e5fb80af3dbae64ea6 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 31 Aug 2019 09:36:23 -0700 Subject: [PATCH 01/16] Clean up two straggling UI issues in Phortune Ref T13401. The checkout UI didn't get fully updated to the new View objects, and account handles are still manually building a URI that goes to the wrong place. --- .../controller/cart/PhortuneCartCheckoutController.php | 10 +++------- .../phortune/phid/PhortuneAccountPHIDType.php | 7 +++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/applications/phortune/controller/cart/PhortuneCartCheckoutController.php b/src/applications/phortune/controller/cart/PhortuneCartCheckoutController.php index 793187c404..cbd434c900 100644 --- a/src/applications/phortune/controller/cart/PhortuneCartCheckoutController.php +++ b/src/applications/phortune/controller/cart/PhortuneCartCheckoutController.php @@ -101,13 +101,9 @@ final class PhortuneCartCheckoutController } } - $cart_table = $this->buildCartContentTable($cart); - - $cart_box = id(new PHUIObjectBoxView()) - ->setFormErrors($errors) - ->setHeaderText(pht('Cart Contents')) - ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) - ->setTable($cart_table); + $cart_box = id(new PhortuneOrderItemsView()) + ->setViewer($viewer) + ->setOrder($cart); $title = $cart->getName(); diff --git a/src/applications/phortune/phid/PhortuneAccountPHIDType.php b/src/applications/phortune/phid/PhortuneAccountPHIDType.php index cf5f5d06f2..90632a980d 100644 --- a/src/applications/phortune/phid/PhortuneAccountPHIDType.php +++ b/src/applications/phortune/phid/PhortuneAccountPHIDType.php @@ -32,10 +32,9 @@ final class PhortuneAccountPHIDType extends PhabricatorPHIDType { foreach ($handles as $phid => $handle) { $account = $objects[$phid]; - $id = $account->getID(); - - $handle->setName($account->getName()); - $handle->setURI("/phortune/{$id}/"); + $handle + ->setName($account->getName()) + ->setURI($account->getURI()); } } From 9316cbf7fd274a46bc5b8d5eca7b87e208f74faf Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 2 Sep 2019 06:20:20 -0700 Subject: [PATCH 02/16] Move web application classes into "phabricator/" Summary: Ref T13395. Companion change to D20773. Test Plan: See D20773. Maniphest Tasks: T13395 Differential Revision: https://secure.phabricator.com/D20774 --- .arclint | 3 +- src/__phutil_library_map__.php | 236 +++ .../auth/adapter/PhutilAmazonAuthAdapter.php | 80 + .../auth/adapter/PhutilAsanaAuthAdapter.php | 86 + .../auth/adapter/PhutilAuthAdapter.php | 123 ++ .../adapter/PhutilBitbucketAuthAdapter.php | 73 + .../auth/adapter/PhutilDisqusAuthAdapter.php | 84 + .../auth/adapter/PhutilEmptyAuthAdapter.php | 42 + .../adapter/PhutilFacebookAuthAdapter.php | 114 ++ .../auth/adapter/PhutilGitHubAuthAdapter.php | 72 + .../auth/adapter/PhutilGoogleAuthAdapter.php | 105 + .../auth/adapter/PhutilJIRAAuthAdapter.php | 164 ++ .../auth/adapter/PhutilLDAPAuthAdapter.php | 505 +++++ .../auth/adapter/PhutilOAuth1AuthAdapter.php | 211 ++ .../auth/adapter/PhutilOAuthAuthAdapter.php | 228 +++ .../adapter/PhutilPhabricatorAuthAdapter.php | 102 + .../auth/adapter/PhutilSlackAuthAdapter.php | 61 + .../auth/adapter/PhutilTwitchAuthAdapter.php | 76 + .../auth/adapter/PhutilTwitterAuthAdapter.php | 75 + .../adapter/PhutilWordPressAuthAdapter.php | 73 + .../PhutilAuthConfigurationException.php | 6 + .../PhutilAuthCredentialException.php | 6 + .../auth/exception/PhutilAuthException.php | 7 + .../PhutilAuthUserAbortedException.php | 14 + .../data/PhutilCalendarAbsoluteDateTime.php | 287 +++ .../data/PhutilCalendarContainerNode.php | 30 + .../parser/data/PhutilCalendarDateTime.php | 54 + .../data/PhutilCalendarDocumentNode.php | 12 + .../parser/data/PhutilCalendarDuration.php | 181 ++ .../parser/data/PhutilCalendarEventNode.php | 172 ++ .../parser/data/PhutilCalendarNode.php | 20 + .../data/PhutilCalendarProxyDateTime.php | 51 + .../parser/data/PhutilCalendarRawNode.php | 8 + .../data/PhutilCalendarRecurrenceList.php | 43 + .../data/PhutilCalendarRecurrenceRule.php | 1820 +++++++++++++++++ .../data/PhutilCalendarRecurrenceSet.php | 162 ++ .../data/PhutilCalendarRecurrenceSource.php | 34 + .../data/PhutilCalendarRelativeDateTime.php | 74 + .../parser/data/PhutilCalendarRootNode.php | 12 + .../parser/data/PhutilCalendarUserNode.php | 40 + .../PhutilCalendarDateTimeTestCase.php | 49 + .../PhutilCalendarRecurrenceRuleTestCase.php | 1750 ++++++++++++++++ .../PhutilCalendarRecurrenceTestCase.php | 196 ++ .../calendar/parser/ics/PhutilICSParser.php | 919 +++++++++ .../parser/ics/PhutilICSParserException.php | 16 + .../calendar/parser/ics/PhutilICSWriter.php | 387 ++++ .../ics/__tests__/PhutilICSParserTestCase.php | 341 +++ .../ics/__tests__/PhutilICSWriterTestCase.php | 144 ++ .../parser/ics/__tests__/data/duration.ics | 8 + .../ics/__tests__/data/err-bad-base64.ics | 5 + .../ics/__tests__/data/err-bad-boolean.ics | 5 + .../ics/__tests__/data/err-bad-datetime.ics | 5 + .../ics/__tests__/data/err-bad-duration.ics | 5 + .../ics/__tests__/data/err-empty-datetime.ics | 5 + .../ics/__tests__/data/err-empty-duration.ics | 5 + .../ics/__tests__/data/err-extra-end.ics | 1 + .../ics/__tests__/data/err-initial-unfold.ics | 2 + .../data/err-malformed-double-quote.ics | 5 + .../data/err-malformed-parameter.ics | 5 + .../__tests__/data/err-malformed-property.ics | 5 + .../ics/__tests__/data/err-many-datetime.ics | 5 + .../ics/__tests__/data/err-many-duration.ics | 5 + .../ics/__tests__/data/err-missing-end.ics | 2 + .../ics/__tests__/data/err-missing-value.ics | 5 + .../data/err-mixmatched-sections.ics | 4 + .../data/err-multiple-parameters.ics | 5 + .../ics/__tests__/data/err-root-property.ics | 1 + .../data/err-unescaped-backslash.ics | 5 + .../__tests__/data/err-unexpected-text.ics | 5 + .../parser/ics/__tests__/data/floating.ics | 8 + .../ics/__tests__/data/good-boolean.ics | 5 + .../__tests__/data/multiple-vcalendars.ics | 4 + .../parser/ics/__tests__/data/simple.ics | 12 + .../parser/ics/__tests__/data/valarm.ics | 16 + .../parser/ics/__tests__/data/weekly.ics | 14 + .../ics/__tests__/data/writer-christmas.ics | 13 + .../__tests__/data/writer-office-party.ics | 15 + .../data/writer-recurring-christmas.ics | 16 + .../ics/__tests__/data/writer-tea-time.ics | 16 + .../ics/__tests__/data/zimbra-timezone.ics | 12 + .../lipsum/PhutilLipsumContextFreeGrammar.php | 107 + .../PhutilRealNameContextFreeGrammar.php | 155 ++ ...utilCLikeCodeSnippetContextFreeGrammar.php | 254 +++ .../PhutilCodeSnippetContextFreeGrammar.php | 205 ++ ...hutilJavaCodeSnippetContextFreeGrammar.php | 184 ++ ...PhutilPHPCodeSnippetContextFreeGrammar.php | 57 + .../markup/PhutilRemarkupBlockStorage.php | 176 ++ .../PhutilRemarkupBlockInterpreter.php | 36 + .../blockrule/PhutilRemarkupBlockRule.php | 170 ++ .../blockrule/PhutilRemarkupCodeBlockRule.php | 252 +++ .../PhutilRemarkupDefaultBlockRule.php | 44 + .../PhutilRemarkupHeaderBlockRule.php | 162 ++ .../PhutilRemarkupHorizontalRuleBlockRule.php | 37 + .../PhutilRemarkupInlineBlockRule.php | 13 + .../PhutilRemarkupInterpreterBlockRule.php | 89 + .../blockrule/PhutilRemarkupListBlockRule.php | 567 +++++ .../PhutilRemarkupLiteralBlockRule.php | 93 + .../blockrule/PhutilRemarkupNoteBlockRule.php | 121 ++ .../PhutilRemarkupQuotedBlockRule.php | 108 + .../PhutilRemarkupQuotesBlockRule.php | 47 + .../PhutilRemarkupReplyBlockRule.php | 91 + .../PhutilRemarkupSimpleTableBlockRule.php | 96 + .../PhutilRemarkupTableBlockRule.php | 142 ++ .../PhutilRemarkupTestInterpreterRule.php | 17 + .../markuprule/PhutilRemarkupBoldRule.php | 24 + .../markuprule/PhutilRemarkupDelRule.php | 24 + .../PhutilRemarkupDocumentLinkRule.php | 175 ++ .../PhutilRemarkupEscapeRemarkupRule.php | 19 + .../PhutilRemarkupHighlightRule.php | 37 + ...PhutilRemarkupHyperlinkEngineExtension.php | 30 + .../markuprule/PhutilRemarkupHyperlinkRef.php | 38 + .../PhutilRemarkupHyperlinkRule.php | 234 +++ .../markuprule/PhutilRemarkupItalicRule.php | 24 + .../PhutilRemarkupLinebreaksRule.php | 13 + .../PhutilRemarkupMonospaceRule.php | 49 + .../markup/markuprule/PhutilRemarkupRule.php | 109 + .../PhutilRemarkupUnderlineRule.php | 24 + .../markup/remarkup/PhutilRemarkupEngine.php | 302 +++ .../PhutilRemarkupEngineTestCase.php | 132 ++ .../__tests__/remarkup/across-newlines.txt | 7 + .../remarkup/backticks-whitespace.txt | 17 + .../__tests__/remarkup/block-then-list.txt | 12 + .../remarkup/code-block-whitespace.txt | 9 + .../remarkup/__tests__/remarkup/del.txt | 11 + .../remarkup/__tests__/remarkup/diff.txt | 36 + .../__tests__/remarkup/disallowed-link.txt | 5 + .../remarkup/__tests__/remarkup/entities.txt | 5 + .../__tests__/remarkup/header-skip.txt | 11 + .../remarkup/__tests__/remarkup/headers.txt | 57 + .../remarkup/__tests__/remarkup/highlight.txt | 9 + .../__tests__/remarkup/horizonal-rule.txt | 41 + .../remarkup/__tests__/remarkup/important.txt | 15 + .../__tests__/remarkup/interpreter-test.txt | 58 + .../__tests__/remarkup/just-backticks.txt | 5 + .../__tests__/remarkup/leading-newline.txt | 6 + .../__tests__/remarkup/link-alternate.txt | 12 + .../__tests__/remarkup/link-brackets.txt | 5 + .../__tests__/remarkup/link-edge-cases.txt | 35 + .../__tests__/remarkup/link-mailto.txt | 18 + .../__tests__/remarkup/link-mixed.txt | 18 + .../__tests__/remarkup/link-noreferrer.txt | 16 + .../__tests__/remarkup/link-same-window.txt | 11 + .../__tests__/remarkup/link-square.txt | 29 + .../remarkup/__tests__/remarkup/link-tel.txt | 18 + .../remarkup/link-with-angle-brackets.txt | 5 + .../remarkup/link-with-angle-link-anchor.txt | 5 + .../remarkup/link-with-link-anchor.txt | 5 + .../remarkup/link-with-punctuation.txt | 9 + .../__tests__/remarkup/link-with-tilde.txt | 5 + .../remarkup/__tests__/remarkup/link.txt | 5 + .../remarkup/list-alternate-style.txt | 15 + .../__tests__/remarkup/list-blow-stack.txt | 138 ++ .../__tests__/remarkup/list-checkboxes.txt | 41 + .../__tests__/remarkup/list-crazystairs.txt | 15 + .../remarkup/list-first-style-wins.txt | 19 + .../remarkup/__tests__/remarkup/list-hash.txt | 19 + .../__tests__/remarkup/list-header-last.txt | 7 + .../__tests__/remarkup/list-header.txt | 12 + .../__tests__/remarkup/list-mixed-styles.txt | 15 + .../__tests__/remarkup/list-multi.txt | 14 + .../__tests__/remarkup/list-multiline.txt | 16 + .../remarkup/__tests__/remarkup/list-nest.txt | 30 + .../__tests__/remarkup/list-paragraphs.txt | 27 + .../__tests__/remarkup/list-staircase.txt | 23 + .../remarkup/__tests__/remarkup/list-star.txt | 19 + .../__tests__/remarkup/list-then-a-list.txt | 15 + .../__tests__/remarkup/list-vs-codeblock.txt | 17 + .../remarkup/__tests__/remarkup/list.txt | 13 + .../remarkup/monospaced-in-monospaced.txt | 18 + .../__tests__/remarkup/monospaced-plural.txt | 11 + .../__tests__/remarkup/monospaced.txt | 5 + .../__tests__/remarkup/newline-then-block.txt | 30 + .../__tests__/remarkup/note-multiline.txt | 14 + .../remarkup/__tests__/remarkup/note.txt | 15 + .../remarkup/ordered-list-with-numbers.txt | 64 + .../remarkup/percent-block-adjacent.txt | 29 + .../remarkup/percent-block-multiline.txt | 21 + .../remarkup/percent-block-oneline.txt | 11 + .../__tests__/remarkup/percent-block-solo.txt | 8 + .../__tests__/remarkup/quoted-angry.txt | 5 + .../__tests__/remarkup/quoted-code-block.txt | 16 + .../remarkup/quoted-indent-block.txt | 5 + .../__tests__/remarkup/quoted-lists.txt | 24 + .../__tests__/remarkup/quoted-quote.txt | 19 + .../remarkup/__tests__/remarkup/quotes.txt | 9 + .../__tests__/remarkup/raw-escape.txt | 17 + .../__tests__/remarkup/reply-basic.txt | 11 + .../__tests__/remarkup/reply-nested.txt | 48 + .../remarkup/simple-table-with-empty-row.txt | 13 + .../simple-table-with-leading-space.txt | 7 + .../remarkup/simple-table-with-link.txt | 7 + .../__tests__/remarkup/simple-table.txt | 24 + .../remarkup/__tests__/remarkup/simple.txt | 5 + .../remarkup/table-with-direct-content.txt | 5 + .../remarkup/table-with-leading-space.txt | 7 + .../remarkup/table-with-long-header.txt | 8 + .../remarkup/__tests__/remarkup/table.txt | 16 + .../__tests__/remarkup/tick-block-multi.txt | 18 + .../__tests__/remarkup/tick-block.txt | 5 + .../remarkup/__tests__/remarkup/toc.txt | 29 + .../trailing-whitespace-codeblock.txt | 39 + .../remarkup/__tests__/remarkup/underline.txt | 13 + .../remarkup/__tests__/remarkup/warning.txt | 15 + .../connection/AphrontDatabaseConnection.php | 305 +++ .../AphrontDatabaseTransactionState.php | 105 + .../AphrontIsolatedDatabaseConnection.php | 132 ++ .../AphrontBaseMySQLDatabaseConnection.php | 405 ++++ .../mysql/AphrontMySQLDatabaseConnection.php | 233 +++ .../mysql/AphrontMySQLiDatabaseConnection.php | 244 +++ .../AphrontAccessDeniedQueryException.php | 4 + .../AphrontCharacterSetQueryException.php | 3 + .../AphrontConnectionLostQueryException.php | 4 + .../AphrontConnectionQueryException.php | 3 + .../exception/AphrontCountQueryException.php | 3 + .../AphrontDeadlockQueryException.php | 4 + .../AphrontDuplicateKeyQueryException.php | 3 + ...phrontInvalidCredentialsQueryException.php | 4 + .../AphrontLockTimeoutQueryException.php | 4 + .../AphrontNotSupportedQueryException.php | 3 + .../AphrontObjectMissingQueryException.php | 3 + .../AphrontParameterQueryException.php | 16 + .../exception/AphrontQueryException.php | 6 + .../AphrontQueryTimeoutQueryException.php | 4 + .../AphrontRecoverableQueryException.php | 3 + .../exception/AphrontSchemaQueryException.php | 3 + .../storage/future/QueryFuture.php | 129 ++ .../xsprintf/AphrontDatabaseTableRef.php | 23 + .../AphrontDatabaseTableRefInterface.php | 8 + .../xsprintf/PhutilQsprintfInterface.php | 9 + .../storage/xsprintf/PhutilQueryString.php | 57 + .../storage/xsprintf/qsprintf.php | 516 +++++ .../storage/xsprintf/queryfx.php | 27 + 232 files changed, 17837 insertions(+), 1 deletion(-) create mode 100644 src/applications/auth/adapter/PhutilAmazonAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilAsanaAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilDisqusAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilEmptyAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilFacebookAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilGitHubAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilGoogleAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilJIRAAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilLDAPAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilOAuthAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilSlackAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilTwitchAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilTwitterAuthAdapter.php create mode 100644 src/applications/auth/adapter/PhutilWordPressAuthAdapter.php create mode 100644 src/applications/auth/exception/PhutilAuthConfigurationException.php create mode 100644 src/applications/auth/exception/PhutilAuthCredentialException.php create mode 100644 src/applications/auth/exception/PhutilAuthException.php create mode 100644 src/applications/auth/exception/PhutilAuthUserAbortedException.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarAbsoluteDateTime.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarContainerNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarDateTime.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarDuration.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarEventNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRawNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRecurrenceList.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarRootNode.php create mode 100644 src/applications/calendar/parser/data/PhutilCalendarUserNode.php create mode 100644 src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php create mode 100644 src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php create mode 100644 src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php create mode 100644 src/applications/calendar/parser/ics/PhutilICSParser.php create mode 100644 src/applications/calendar/parser/ics/PhutilICSParserException.php create mode 100644 src/applications/calendar/parser/ics/PhutilICSWriter.php create mode 100644 src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php create mode 100644 src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php create mode 100644 src/applications/calendar/parser/ics/__tests__/data/duration.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/floating.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/simple.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/valarm.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/weekly.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics create mode 100644 src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics create mode 100644 src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php create mode 100644 src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php create mode 100644 src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php create mode 100644 src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php create mode 100644 src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php create mode 100644 src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php create mode 100644 src/infrastructure/markup/PhutilRemarkupBlockStorage.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php create mode 100644 src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupBoldRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupRule.php create mode 100644 src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php create mode 100644 src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php create mode 100644 src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt create mode 100644 src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt create mode 100644 src/infrastructure/storage/connection/AphrontDatabaseConnection.php create mode 100644 src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php create mode 100644 src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php create mode 100644 src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php create mode 100644 src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php create mode 100644 src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php create mode 100644 src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontCharacterSetQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontConnectionLostQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontConnectionQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontCountQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontDeadlockQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontDuplicateKeyQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontInvalidCredentialsQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontLockTimeoutQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontNotSupportedQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontObjectMissingQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontParameterQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontQueryTimeoutQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontRecoverableQueryException.php create mode 100644 src/infrastructure/storage/exception/AphrontSchemaQueryException.php create mode 100644 src/infrastructure/storage/future/QueryFuture.php create mode 100644 src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php create mode 100644 src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php create mode 100644 src/infrastructure/storage/xsprintf/PhutilQsprintfInterface.php create mode 100644 src/infrastructure/storage/xsprintf/PhutilQueryString.php create mode 100644 src/infrastructure/storage/xsprintf/qsprintf.php create mode 100644 src/infrastructure/storage/xsprintf/queryfx.php diff --git a/.arclint b/.arclint index 6047e2ccb6..d065c3d12e 100644 --- a/.arclint +++ b/.arclint @@ -1,7 +1,8 @@ { "exclude": [ "(^externals/)", - "(^webroot/rsrc/externals/(?!javelin/))" + "(^webroot/rsrc/externals/(?!javelin/))", + "(/__tests__/data/)" ], "linters": { "chmod": { diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 5d9a0aed3c..96d3713b48 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -176,15 +176,27 @@ phutil_register_library_map(array( 'Aphront400Response' => 'aphront/response/Aphront400Response.php', 'Aphront403Response' => 'aphront/response/Aphront403Response.php', 'Aphront404Response' => 'aphront/response/Aphront404Response.php', + 'AphrontAccessDeniedQueryException' => 'infrastructure/storage/exception/AphrontAccessDeniedQueryException.php', 'AphrontAjaxResponse' => 'aphront/response/AphrontAjaxResponse.php', 'AphrontApplicationConfiguration' => 'aphront/configuration/AphrontApplicationConfiguration.php', 'AphrontBarView' => 'view/widget/bars/AphrontBarView.php', + 'AphrontBaseMySQLDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php', 'AphrontBoolHTTPParameterType' => 'aphront/httpparametertype/AphrontBoolHTTPParameterType.php', 'AphrontCalendarEventView' => 'applications/calendar/view/AphrontCalendarEventView.php', + 'AphrontCharacterSetQueryException' => 'infrastructure/storage/exception/AphrontCharacterSetQueryException.php', + 'AphrontConnectionLostQueryException' => 'infrastructure/storage/exception/AphrontConnectionLostQueryException.php', + 'AphrontConnectionQueryException' => 'infrastructure/storage/exception/AphrontConnectionQueryException.php', 'AphrontController' => 'aphront/AphrontController.php', + 'AphrontCountQueryException' => 'infrastructure/storage/exception/AphrontCountQueryException.php', 'AphrontCursorPagerView' => 'view/control/AphrontCursorPagerView.php', + 'AphrontDatabaseConnection' => 'infrastructure/storage/connection/AphrontDatabaseConnection.php', + 'AphrontDatabaseTableRef' => 'infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php', + 'AphrontDatabaseTableRefInterface' => 'infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php', + 'AphrontDatabaseTransactionState' => 'infrastructure/storage/connection/AphrontDatabaseTransactionState.php', + 'AphrontDeadlockQueryException' => 'infrastructure/storage/exception/AphrontDeadlockQueryException.php', 'AphrontDialogResponse' => 'aphront/response/AphrontDialogResponse.php', 'AphrontDialogView' => 'view/AphrontDialogView.php', + 'AphrontDuplicateKeyQueryException' => 'infrastructure/storage/exception/AphrontDuplicateKeyQueryException.php', 'AphrontEpochHTTPParameterType' => 'aphront/httpparametertype/AphrontEpochHTTPParameterType.php', 'AphrontException' => 'aphront/exception/AphrontException.php', 'AphrontFileHTTPParameterType' => 'aphront/httpparametertype/AphrontFileHTTPParameterType.php', @@ -217,6 +229,8 @@ phutil_register_library_map(array( 'AphrontHTTPSink' => 'aphront/sink/AphrontHTTPSink.php', 'AphrontHTTPSinkTestCase' => 'aphront/sink/__tests__/AphrontHTTPSinkTestCase.php', 'AphrontIntHTTPParameterType' => 'aphront/httpparametertype/AphrontIntHTTPParameterType.php', + 'AphrontInvalidCredentialsQueryException' => 'infrastructure/storage/exception/AphrontInvalidCredentialsQueryException.php', + 'AphrontIsolatedDatabaseConnection' => 'infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php', 'AphrontIsolatedDatabaseConnectionTestCase' => 'infrastructure/storage/__tests__/AphrontIsolatedDatabaseConnectionTestCase.php', 'AphrontIsolatedHTTPSink' => 'aphront/sink/AphrontIsolatedHTTPSink.php', 'AphrontJSONResponse' => 'aphront/response/AphrontJSONResponse.php', @@ -224,19 +238,28 @@ phutil_register_library_map(array( 'AphrontKeyboardShortcutsAvailableView' => 'view/widget/AphrontKeyboardShortcutsAvailableView.php', 'AphrontListFilterView' => 'view/layout/AphrontListFilterView.php', 'AphrontListHTTPParameterType' => 'aphront/httpparametertype/AphrontListHTTPParameterType.php', + 'AphrontLockTimeoutQueryException' => 'infrastructure/storage/exception/AphrontLockTimeoutQueryException.php', 'AphrontMalformedRequestException' => 'aphront/exception/AphrontMalformedRequestException.php', 'AphrontMoreView' => 'view/layout/AphrontMoreView.php', 'AphrontMultiColumnView' => 'view/layout/AphrontMultiColumnView.php', + 'AphrontMySQLDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php', 'AphrontMySQLDatabaseConnectionTestCase' => 'infrastructure/storage/__tests__/AphrontMySQLDatabaseConnectionTestCase.php', + 'AphrontMySQLiDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php', + 'AphrontNotSupportedQueryException' => 'infrastructure/storage/exception/AphrontNotSupportedQueryException.php', 'AphrontNullView' => 'view/AphrontNullView.php', + 'AphrontObjectMissingQueryException' => 'infrastructure/storage/exception/AphrontObjectMissingQueryException.php', 'AphrontPHIDHTTPParameterType' => 'aphront/httpparametertype/AphrontPHIDHTTPParameterType.php', 'AphrontPHIDListHTTPParameterType' => 'aphront/httpparametertype/AphrontPHIDListHTTPParameterType.php', 'AphrontPHPHTTPSink' => 'aphront/sink/AphrontPHPHTTPSink.php', 'AphrontPageView' => 'view/page/AphrontPageView.php', + 'AphrontParameterQueryException' => 'infrastructure/storage/exception/AphrontParameterQueryException.php', 'AphrontPlainTextResponse' => 'aphront/response/AphrontPlainTextResponse.php', 'AphrontProgressBarView' => 'view/widget/bars/AphrontProgressBarView.php', 'AphrontProjectListHTTPParameterType' => 'aphront/httpparametertype/AphrontProjectListHTTPParameterType.php', 'AphrontProxyResponse' => 'aphront/response/AphrontProxyResponse.php', + 'AphrontQueryException' => 'infrastructure/storage/exception/AphrontQueryException.php', + 'AphrontQueryTimeoutQueryException' => 'infrastructure/storage/exception/AphrontQueryTimeoutQueryException.php', + 'AphrontRecoverableQueryException' => 'infrastructure/storage/exception/AphrontRecoverableQueryException.php', 'AphrontRedirectResponse' => 'aphront/response/AphrontRedirectResponse.php', 'AphrontRedirectResponseTestCase' => 'aphront/response/__tests__/AphrontRedirectResponseTestCase.php', 'AphrontReloadResponse' => 'aphront/response/AphrontReloadResponse.php', @@ -247,6 +270,7 @@ phutil_register_library_map(array( 'AphrontResponseProducerInterface' => 'aphront/interface/AphrontResponseProducerInterface.php', 'AphrontRoutingMap' => 'aphront/site/AphrontRoutingMap.php', 'AphrontRoutingResult' => 'aphront/site/AphrontRoutingResult.php', + 'AphrontSchemaQueryException' => 'infrastructure/storage/exception/AphrontSchemaQueryException.php', 'AphrontSelectHTTPParameterType' => 'aphront/httpparametertype/AphrontSelectHTTPParameterType.php', 'AphrontSideNavFilterView' => 'view/layout/AphrontSideNavFilterView.php', 'AphrontSite' => 'aphront/site/AphrontSite.php', @@ -5512,6 +5536,93 @@ phutil_register_library_map(array( 'PhrictionTransactionComment' => 'applications/phriction/storage/PhrictionTransactionComment.php', 'PhrictionTransactionEditor' => 'applications/phriction/editor/PhrictionTransactionEditor.php', 'PhrictionTransactionQuery' => 'applications/phriction/query/PhrictionTransactionQuery.php', + 'PhutilAmazonAuthAdapter' => 'applications/auth/adapter/PhutilAmazonAuthAdapter.php', + 'PhutilAsanaAuthAdapter' => 'applications/auth/adapter/PhutilAsanaAuthAdapter.php', + 'PhutilAuthAdapter' => 'applications/auth/adapter/PhutilAuthAdapter.php', + 'PhutilAuthConfigurationException' => 'applications/auth/exception/PhutilAuthConfigurationException.php', + 'PhutilAuthCredentialException' => 'applications/auth/exception/PhutilAuthCredentialException.php', + 'PhutilAuthException' => 'applications/auth/exception/PhutilAuthException.php', + 'PhutilAuthUserAbortedException' => 'applications/auth/exception/PhutilAuthUserAbortedException.php', + 'PhutilBitbucketAuthAdapter' => 'applications/auth/adapter/PhutilBitbucketAuthAdapter.php', + 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php', + 'PhutilCalendarAbsoluteDateTime' => 'applications/calendar/parser/data/PhutilCalendarAbsoluteDateTime.php', + 'PhutilCalendarContainerNode' => 'applications/calendar/parser/data/PhutilCalendarContainerNode.php', + 'PhutilCalendarDateTime' => 'applications/calendar/parser/data/PhutilCalendarDateTime.php', + 'PhutilCalendarDateTimeTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php', + 'PhutilCalendarDocumentNode' => 'applications/calendar/parser/data/PhutilCalendarDocumentNode.php', + 'PhutilCalendarDuration' => 'applications/calendar/parser/data/PhutilCalendarDuration.php', + 'PhutilCalendarEventNode' => 'applications/calendar/parser/data/PhutilCalendarEventNode.php', + 'PhutilCalendarNode' => 'applications/calendar/parser/data/PhutilCalendarNode.php', + 'PhutilCalendarProxyDateTime' => 'applications/calendar/parser/data/PhutilCalendarProxyDateTime.php', + 'PhutilCalendarRawNode' => 'applications/calendar/parser/data/PhutilCalendarRawNode.php', + 'PhutilCalendarRecurrenceList' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceList.php', + 'PhutilCalendarRecurrenceRule' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php', + 'PhutilCalendarRecurrenceRuleTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php', + 'PhutilCalendarRecurrenceSet' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php', + 'PhutilCalendarRecurrenceSource' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php', + 'PhutilCalendarRecurrenceTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php', + 'PhutilCalendarRelativeDateTime' => 'applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php', + 'PhutilCalendarRootNode' => 'applications/calendar/parser/data/PhutilCalendarRootNode.php', + 'PhutilCalendarUserNode' => 'applications/calendar/parser/data/PhutilCalendarUserNode.php', + 'PhutilCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php', + 'PhutilDisqusAuthAdapter' => 'applications/auth/adapter/PhutilDisqusAuthAdapter.php', + 'PhutilEmptyAuthAdapter' => 'applications/auth/adapter/PhutilEmptyAuthAdapter.php', + 'PhutilFacebookAuthAdapter' => 'applications/auth/adapter/PhutilFacebookAuthAdapter.php', + 'PhutilGitHubAuthAdapter' => 'applications/auth/adapter/PhutilGitHubAuthAdapter.php', + 'PhutilGoogleAuthAdapter' => 'applications/auth/adapter/PhutilGoogleAuthAdapter.php', + 'PhutilICSParser' => 'applications/calendar/parser/ics/PhutilICSParser.php', + 'PhutilICSParserException' => 'applications/calendar/parser/ics/PhutilICSParserException.php', + 'PhutilICSParserTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php', + 'PhutilICSWriter' => 'applications/calendar/parser/ics/PhutilICSWriter.php', + 'PhutilICSWriterTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php', + 'PhutilJIRAAuthAdapter' => 'applications/auth/adapter/PhutilJIRAAuthAdapter.php', + 'PhutilJavaCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php', + 'PhutilLDAPAuthAdapter' => 'applications/auth/adapter/PhutilLDAPAuthAdapter.php', + 'PhutilLipsumContextFreeGrammar' => 'infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php', + 'PhutilOAuth1AuthAdapter' => 'applications/auth/adapter/PhutilOAuth1AuthAdapter.php', + 'PhutilOAuthAuthAdapter' => 'applications/auth/adapter/PhutilOAuthAuthAdapter.php', + 'PhutilPHPCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php', + 'PhutilPhabricatorAuthAdapter' => 'applications/auth/adapter/PhutilPhabricatorAuthAdapter.php', + 'PhutilQsprintfInterface' => 'infrastructure/storage/xsprintf/PhutilQsprintfInterface.php', + 'PhutilQueryString' => 'infrastructure/storage/xsprintf/PhutilQueryString.php', + 'PhutilRealNameContextFreeGrammar' => 'infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php', + 'PhutilRemarkupBlockInterpreter' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php', + 'PhutilRemarkupBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php', + 'PhutilRemarkupBlockStorage' => 'infrastructure/markup/PhutilRemarkupBlockStorage.php', + 'PhutilRemarkupBoldRule' => 'infrastructure/markup/markuprule/PhutilRemarkupBoldRule.php', + 'PhutilRemarkupCodeBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php', + 'PhutilRemarkupDefaultBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php', + 'PhutilRemarkupDelRule' => 'infrastructure/markup/markuprule/PhutilRemarkupDelRule.php', + 'PhutilRemarkupDocumentLinkRule' => 'infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php', + 'PhutilRemarkupEngine' => 'infrastructure/markup/remarkup/PhutilRemarkupEngine.php', + 'PhutilRemarkupEngineTestCase' => 'infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php', + 'PhutilRemarkupEscapeRemarkupRule' => 'infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php', + 'PhutilRemarkupHeaderBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php', + 'PhutilRemarkupHighlightRule' => 'infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php', + 'PhutilRemarkupHorizontalRuleBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php', + 'PhutilRemarkupHyperlinkEngineExtension' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php', + 'PhutilRemarkupHyperlinkRef' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php', + 'PhutilRemarkupHyperlinkRule' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php', + 'PhutilRemarkupInlineBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php', + 'PhutilRemarkupInterpreterBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php', + 'PhutilRemarkupItalicRule' => 'infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php', + 'PhutilRemarkupLinebreaksRule' => 'infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php', + 'PhutilRemarkupListBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php', + 'PhutilRemarkupLiteralBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php', + 'PhutilRemarkupMonospaceRule' => 'infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php', + 'PhutilRemarkupNoteBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php', + 'PhutilRemarkupQuotedBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php', + 'PhutilRemarkupQuotesBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php', + 'PhutilRemarkupReplyBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php', + 'PhutilRemarkupRule' => 'infrastructure/markup/markuprule/PhutilRemarkupRule.php', + 'PhutilRemarkupSimpleTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php', + 'PhutilRemarkupTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php', + 'PhutilRemarkupTestInterpreterRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php', + 'PhutilRemarkupUnderlineRule' => 'infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php', + 'PhutilSlackAuthAdapter' => 'applications/auth/adapter/PhutilSlackAuthAdapter.php', + 'PhutilTwitchAuthAdapter' => 'applications/auth/adapter/PhutilTwitchAuthAdapter.php', + 'PhutilTwitterAuthAdapter' => 'applications/auth/adapter/PhutilTwitterAuthAdapter.php', + 'PhutilWordPressAuthAdapter' => 'applications/auth/adapter/PhutilWordPressAuthAdapter.php', 'PolicyLockOptionType' => 'applications/policy/config/PolicyLockOptionType.php', 'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php', 'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php', @@ -5587,6 +5698,7 @@ phutil_register_library_map(array( 'ProjectReplyHandler' => 'applications/project/mail/ProjectReplyHandler.php', 'ProjectSearchConduitAPIMethod' => 'applications/project/conduit/ProjectSearchConduitAPIMethod.php', 'QueryFormattingTestCase' => 'infrastructure/storage/__tests__/QueryFormattingTestCase.php', + 'QueryFuture' => 'infrastructure/storage/future/QueryFuture.php', 'ReleephAuthorFieldSpecification' => 'applications/releeph/field/specification/ReleephAuthorFieldSpecification.php', 'ReleephBranch' => 'applications/releeph/storage/ReleephBranch.php', 'ReleephBranchAccessController' => 'applications/releeph/controller/branch/ReleephBranchAccessController.php', @@ -5713,7 +5825,15 @@ phutil_register_library_map(array( 'phid_get_subtype' => 'applications/phid/utils.php', 'phid_get_type' => 'applications/phid/utils.php', 'phid_group_by_type' => 'applications/phid/utils.php', + 'qsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'qsprintf_check_scalar_type' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'qsprintf_check_type' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'queryfx' => 'infrastructure/storage/xsprintf/queryfx.php', + 'queryfx_all' => 'infrastructure/storage/xsprintf/queryfx.php', + 'queryfx_one' => 'infrastructure/storage/xsprintf/queryfx.php', 'require_celerity_resource' => 'applications/celerity/api.php', + 'vqsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'xsprintf_query' => 'infrastructure/storage/xsprintf/qsprintf.php', ), 'xmap' => array( 'AlmanacAddress' => 'Phobject', @@ -5937,18 +6057,35 @@ phutil_register_library_map(array( 'Aphront400Response' => 'AphrontResponse', 'Aphront403Response' => 'AphrontHTMLResponse', 'Aphront404Response' => 'AphrontHTMLResponse', + 'AphrontAccessDeniedQueryException' => 'AphrontQueryException', 'AphrontAjaxResponse' => 'AphrontResponse', 'AphrontApplicationConfiguration' => 'Phobject', 'AphrontBarView' => 'AphrontView', + 'AphrontBaseMySQLDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontBoolHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontCalendarEventView' => 'AphrontView', + 'AphrontCharacterSetQueryException' => 'AphrontQueryException', + 'AphrontConnectionLostQueryException' => 'AphrontRecoverableQueryException', + 'AphrontConnectionQueryException' => 'AphrontQueryException', 'AphrontController' => 'Phobject', + 'AphrontCountQueryException' => 'AphrontQueryException', 'AphrontCursorPagerView' => 'AphrontView', + 'AphrontDatabaseConnection' => array( + 'Phobject', + 'PhutilQsprintfInterface', + ), + 'AphrontDatabaseTableRef' => array( + 'Phobject', + 'AphrontDatabaseTableRefInterface', + ), + 'AphrontDatabaseTransactionState' => 'Phobject', + 'AphrontDeadlockQueryException' => 'AphrontRecoverableQueryException', 'AphrontDialogResponse' => 'AphrontResponse', 'AphrontDialogView' => array( 'AphrontView', 'AphrontResponseProducerInterface', ), + 'AphrontDuplicateKeyQueryException' => 'AphrontQueryException', 'AphrontEpochHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontException' => 'Exception', 'AphrontFileHTTPParameterType' => 'AphrontHTTPParameterType', @@ -5981,6 +6118,8 @@ phutil_register_library_map(array( 'AphrontHTTPSink' => 'Phobject', 'AphrontHTTPSinkTestCase' => 'PhabricatorTestCase', 'AphrontIntHTTPParameterType' => 'AphrontHTTPParameterType', + 'AphrontInvalidCredentialsQueryException' => 'AphrontQueryException', + 'AphrontIsolatedDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontIsolatedDatabaseConnectionTestCase' => 'PhabricatorTestCase', 'AphrontIsolatedHTTPSink' => 'AphrontHTTPSink', 'AphrontJSONResponse' => 'AphrontResponse', @@ -5988,15 +6127,21 @@ phutil_register_library_map(array( 'AphrontKeyboardShortcutsAvailableView' => 'AphrontView', 'AphrontListFilterView' => 'AphrontView', 'AphrontListHTTPParameterType' => 'AphrontHTTPParameterType', + 'AphrontLockTimeoutQueryException' => 'AphrontRecoverableQueryException', 'AphrontMalformedRequestException' => 'AphrontException', 'AphrontMoreView' => 'AphrontView', 'AphrontMultiColumnView' => 'AphrontView', + 'AphrontMySQLDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', 'AphrontMySQLDatabaseConnectionTestCase' => 'PhabricatorTestCase', + 'AphrontMySQLiDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', + 'AphrontNotSupportedQueryException' => 'AphrontQueryException', 'AphrontNullView' => 'AphrontView', + 'AphrontObjectMissingQueryException' => 'AphrontQueryException', 'AphrontPHIDHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontPHIDListHTTPParameterType' => 'AphrontListHTTPParameterType', 'AphrontPHPHTTPSink' => 'AphrontHTTPSink', 'AphrontPageView' => 'AphrontView', + 'AphrontParameterQueryException' => 'AphrontQueryException', 'AphrontPlainTextResponse' => 'AphrontResponse', 'AphrontProgressBarView' => 'AphrontBarView', 'AphrontProjectListHTTPParameterType' => 'AphrontListHTTPParameterType', @@ -6004,6 +6149,9 @@ phutil_register_library_map(array( 'AphrontResponse', 'AphrontResponseProducerInterface', ), + 'AphrontQueryException' => 'Exception', + 'AphrontQueryTimeoutQueryException' => 'AphrontRecoverableQueryException', + 'AphrontRecoverableQueryException' => 'AphrontQueryException', 'AphrontRedirectResponse' => 'AphrontResponse', 'AphrontRedirectResponseTestCase' => 'PhabricatorTestCase', 'AphrontReloadResponse' => 'AphrontRedirectResponse', @@ -6013,6 +6161,7 @@ phutil_register_library_map(array( 'AphrontResponse' => 'Phobject', 'AphrontRoutingMap' => 'Phobject', 'AphrontRoutingResult' => 'Phobject', + 'AphrontSchemaQueryException' => 'AphrontQueryException', 'AphrontSelectHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontSideNavFilterView' => 'AphrontView', 'AphrontSite' => 'Phobject', @@ -12169,6 +12318,92 @@ phutil_register_library_map(array( 'PhrictionTransactionComment' => 'PhabricatorApplicationTransactionComment', 'PhrictionTransactionEditor' => 'PhabricatorApplicationTransactionEditor', 'PhrictionTransactionQuery' => 'PhabricatorApplicationTransactionQuery', + 'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilAuthAdapter' => 'Phobject', + 'PhutilAuthConfigurationException' => 'PhutilAuthException', + 'PhutilAuthCredentialException' => 'PhutilAuthException', + 'PhutilAuthException' => 'Exception', + 'PhutilAuthUserAbortedException' => 'PhutilAuthException', + 'PhutilBitbucketAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'PhutilCodeSnippetContextFreeGrammar', + 'PhutilCalendarAbsoluteDateTime' => 'PhutilCalendarDateTime', + 'PhutilCalendarContainerNode' => 'PhutilCalendarNode', + 'PhutilCalendarDateTime' => 'Phobject', + 'PhutilCalendarDateTimeTestCase' => 'PhutilTestCase', + 'PhutilCalendarDocumentNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarDuration' => 'Phobject', + 'PhutilCalendarEventNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarNode' => 'Phobject', + 'PhutilCalendarProxyDateTime' => 'PhutilCalendarDateTime', + 'PhutilCalendarRawNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarRecurrenceList' => 'PhutilCalendarRecurrenceSource', + 'PhutilCalendarRecurrenceRule' => 'PhutilCalendarRecurrenceSource', + 'PhutilCalendarRecurrenceRuleTestCase' => 'PhutilTestCase', + 'PhutilCalendarRecurrenceSet' => 'Phobject', + 'PhutilCalendarRecurrenceSource' => 'Phobject', + 'PhutilCalendarRecurrenceTestCase' => 'PhutilTestCase', + 'PhutilCalendarRelativeDateTime' => 'PhutilCalendarProxyDateTime', + 'PhutilCalendarRootNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarUserNode' => 'PhutilCalendarNode', + 'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilGoogleAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilICSParser' => 'Phobject', + 'PhutilICSParserException' => 'Exception', + 'PhutilICSParserTestCase' => 'PhutilTestCase', + 'PhutilICSWriter' => 'Phobject', + 'PhutilICSWriterTestCase' => 'PhutilTestCase', + 'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter', + 'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilQueryString' => 'Phobject', + 'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilRemarkupBlockInterpreter' => 'Phobject', + 'PhutilRemarkupBlockRule' => 'Phobject', + 'PhutilRemarkupBlockStorage' => 'Phobject', + 'PhutilRemarkupBoldRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupCodeBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupDefaultBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupDelRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupDocumentLinkRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupEngine' => 'PhutilMarkupEngine', + 'PhutilRemarkupEngineTestCase' => 'PhutilTestCase', + 'PhutilRemarkupEscapeRemarkupRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupHeaderBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupHighlightRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupHorizontalRuleBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupHyperlinkEngineExtension' => 'Phobject', + 'PhutilRemarkupHyperlinkRef' => 'Phobject', + 'PhutilRemarkupHyperlinkRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupInlineBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupInterpreterBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupItalicRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupLinebreaksRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupListBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupLiteralBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupMonospaceRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupNoteBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupQuotedBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupQuotesBlockRule' => 'PhutilRemarkupQuotedBlockRule', + 'PhutilRemarkupReplyBlockRule' => 'PhutilRemarkupQuotedBlockRule', + 'PhutilRemarkupRule' => 'Phobject', + 'PhutilRemarkupSimpleTableBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter', + 'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule', + 'PhutilSlackAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PolicyLockOptionType' => 'PhabricatorConfigJSONOptionType', 'PonderAddAnswerView' => 'AphrontView', 'PonderAnswer' => array( @@ -12265,6 +12500,7 @@ phutil_register_library_map(array( 'ProjectReplyHandler' => 'PhabricatorApplicationTransactionReplyHandler', 'ProjectSearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', 'QueryFormattingTestCase' => 'PhabricatorTestCase', + 'QueryFuture' => 'Future', 'ReleephAuthorFieldSpecification' => 'ReleephFieldSpecification', 'ReleephBranch' => array( 'ReleephDAO', diff --git a/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php b/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php new file mode 100644 index 0000000000..94c529ae2d --- /dev/null +++ b/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php @@ -0,0 +1,80 @@ +getOAuthAccountData('user_id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return null; + } + + public function getAccountImageURI() { + return null; + } + + public function getAccountURI() { + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://www.amazon.com/ap/oa'; + } + + protected function getTokenBaseURI() { + return 'https://api.amazon.com/auth/o2/token'; + } + + public function getScope() { + return 'profile'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://api.amazon.com/user/profile'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + list($body) = $future->resolvex(); + + try { + return phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Amazon account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php b/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php new file mode 100644 index 0000000000..5d9a9ec478 --- /dev/null +++ b/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php @@ -0,0 +1,86 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return null; + } + + public function getAccountImageURI() { + $photo = $this->getOAuthAccountData('photo', array()); + if (is_array($photo)) { + return idx($photo, 'image_128x128'); + } else { + return null; + } + } + + public function getAccountURI() { + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://app.asana.com/-/oauth_authorize'; + } + + protected function getTokenBaseURI() { + return 'https://app.asana.com/-/oauth_token'; + } + + public function getScope() { + return null; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + public function getExtraRefreshParameters() { + return array( + 'grant_type' => 'refresh_token', + ); + } + + public function supportsTokenRefresh() { + return true; + } + + protected function loadOAuthAccountData() { + return id(new PhutilAsanaFuture()) + ->setAccessToken($this->getAccessToken()) + ->setRawAsanaQuery('users/me') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilAuthAdapter.php b/src/applications/auth/adapter/PhutilAuthAdapter.php new file mode 100644 index 0000000000..73ef9e4834 --- /dev/null +++ b/src/applications/auth/adapter/PhutilAuthAdapter.php @@ -0,0 +1,123 @@ +` is globally unique and always identifies the + * same identity. + * + * If the adapter was unable to authenticate an identity, it should return + * `null`. + * + * @return string|null Unique account identifier, or `null` if authentication + * failed. + */ + abstract public function getAccountID(); + + + /** + * Get a string identifying this adapter, like "ldap". This string should be + * unique to the adapter class. + * + * @return string Unique adapter identifier. + */ + abstract public function getAdapterType(); + + + /** + * Get a string identifying the domain this adapter is acting on. This allows + * an adapter (like LDAP) to act against different identity domains without + * conflating credentials. For providers like Facebook or Google, the adapters + * just return the relevant domain name. + * + * @return string Domain the adapter is associated with. + */ + abstract public function getAdapterDomain(); + + + /** + * Generate a string uniquely identifying this adapter configuration. Within + * the scope of a given key, all account IDs must uniquely identify exactly + * one identity. + * + * @return string Unique identifier for this adapter configuration. + */ + public function getAdapterKey() { + return $this->getAdapterType().':'.$this->getAdapterDomain(); + } + + + /** + * Optionally, return an email address associated with this account. + * + * @return string|null An email address associated with the account, or + * `null` if data is not available. + */ + public function getAccountEmail() { + return null; + } + + + /** + * Optionally, return a human readable username associated with this account. + * + * @return string|null Account username, or `null` if data isn't available. + */ + public function getAccountName() { + return null; + } + + + /** + * Optionally, return a URI corresponding to a human-viewable profile for + * this account. + * + * @return string|null A profile URI associated with this account, or + * `null` if the data isn't available. + */ + public function getAccountURI() { + return null; + } + + + /** + * Optionally, return a profile image URI associated with this account. + * + * @return string|null URI for an account profile image, or `null` if one is + * not available. + */ + public function getAccountImageURI() { + return null; + } + + + /** + * Optionally, return a real name associated with this account. + * + * @return string|null A human real name, or `null` if this data is not + * available. + */ + public function getAccountRealName() { + return null; + } + +} diff --git a/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php b/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php new file mode 100644 index 0000000000..1384548d5e --- /dev/null +++ b/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php @@ -0,0 +1,73 @@ +getUserInfo(), 'username'); + } + + public function getAccountName() { + return idx($this->getUserInfo(), 'display_name'); + } + + public function getAccountURI() { + $name = $this->getAccountID(); + if (strlen($name)) { + return 'https://bitbucket.org/'.$name; + } + return null; + } + + public function getAccountImageURI() { + return idx($this->getUserInfo(), 'avatar'); + } + + public function getAccountRealName() { + $parts = array( + idx($this->getUserInfo(), 'first_name'), + idx($this->getUserInfo(), 'last_name'), + ); + $parts = array_filter($parts); + return implode(' ', $parts); + } + + public function getAdapterType() { + return 'bitbucket'; + } + + public function getAdapterDomain() { + return 'bitbucket.org'; + } + + protected function getRequestTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/request_token'; + } + + protected function getAuthorizeTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/authenticate'; + } + + protected function getValidateTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/access_token'; + } + + private function getUserInfo() { + if ($this->userInfo === null) { + // We don't need any of the data in the handshake, but do need to + // finish the process. This makes sure we've completed the handshake. + $this->getHandshakeData(); + + $uri = new PhutilURI('https://bitbucket.org/api/1.0/user'); + + $data = $this->newOAuth1Future($uri) + ->setMethod('GET') + ->resolveJSON(); + + $this->userInfo = idx($data, 'user', array()); + } + return $this->userInfo; + } + +} diff --git a/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php b/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php new file mode 100644 index 0000000000..b9a33b293a --- /dev/null +++ b/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php @@ -0,0 +1,84 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('username'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar', 'permalink'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('profileUrl'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://disqus.com/api/oauth/2.0/authorize/'; + } + + protected function getTokenBaseURI() { + return 'https://disqus.com/api/oauth/2.0/access_token/'; + } + + public function getScope() { + return 'read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://disqus.com/api/3.0/users/details.json'); + $uri->replaceQueryParam('api_key', $this->getClientID()); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + $uri = (string)$uri; + + $future = new HTTPSFuture($uri); + $future->setMethod('GET'); + list($body) = $future->resolvex(); + + try { + $data = phutil_json_decode($body); + return $data['response']; + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Disqus account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php b/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php new file mode 100644 index 0000000000..a59dddd262 --- /dev/null +++ b/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php @@ -0,0 +1,42 @@ +adapterDomain = $adapter_domain; + return $this; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterType($adapter_type) { + $this->adapterType = $adapter_type; + return $this; + } + + public function getAdapterType() { + return $this->adapterType; + } + + public function setAccountID($account_id) { + $this->accountID = $account_id; + return $this; + } + + public function getAccountID() { + return $this->accountID; + } + +} diff --git a/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php b/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php new file mode 100644 index 0000000000..ab569a14a0 --- /dev/null +++ b/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php @@ -0,0 +1,114 @@ +requireSecureBrowsing = $require_secure_browsing; + return $this; + } + + public function getAdapterType() { + return 'facebook'; + } + + public function getAdapterDomain() { + return 'facebook.com'; + } + + public function getAccountID() { + return $this->getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + $link = $this->getOAuthAccountData('link'); + if (!$link) { + return null; + } + + $matches = null; + if (!preg_match('@/([^/]+)$@', $link, $matches)) { + return null; + } + + return $matches[1]; + } + + public function getAccountImageURI() { + $picture = $this->getOAuthAccountData('picture'); + if ($picture) { + $picture_data = idx($picture, 'data'); + if ($picture_data) { + return idx($picture_data, 'url'); + } + } + return null; + } + + public function getAccountURI() { + return $this->getOAuthAccountData('link'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + public function getAccountSecuritySettings() { + return $this->getOAuthAccountData('security_settings'); + } + + protected function getAuthenticateBaseURI() { + return 'https://www.facebook.com/dialog/oauth'; + } + + protected function getTokenBaseURI() { + return 'https://graph.facebook.com/oauth/access_token'; + } + + protected function loadOAuthAccountData() { + $fields = array( + 'id', + 'name', + 'email', + 'link', + 'security_settings', + 'picture', + ); + + $uri = new PhutilURI('https://graph.facebook.com/me'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + $uri->replaceQueryParam('fields', implode(',', $fields)); + list($body) = id(new HTTPSFuture($uri))->resolvex(); + + $data = null; + try { + $data = phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Facebook account data request.'), + $ex); + } + + if ($this->requireSecureBrowsing) { + if (empty($data['security_settings']['secure_browsing']['enabled'])) { + throw new Exception( + pht( + 'This Phabricator install requires you to enable Secure Browsing '. + 'on your Facebook account in order to use it to log in to '. + 'Phabricator. For more information, see %s', + 'https://www.facebook.com/help/156201551113407/')); + } + } + + return $data; + } + +} diff --git a/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php b/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php new file mode 100644 index 0000000000..7fa8a660fa --- /dev/null +++ b/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php @@ -0,0 +1,72 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('login'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar_url'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if (strlen($name)) { + return 'https://github.com/'.$name; + } + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://github.com/login/oauth/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://github.com/login/oauth/access_token'; + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://api.github.com/user'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + + // NOTE: GitHub requires a User-Agent string. + $future->addHeader('User-Agent', __CLASS__); + + list($body) = $future->resolvex(); + + try { + return phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from GitHub account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php b/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php new file mode 100644 index 0000000000..11c10087ba --- /dev/null +++ b/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php @@ -0,0 +1,105 @@ +getAccountEmail(); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + // Guess account name from email address, this is just a hint anyway. + $email = $this->getAccountEmail(); + $email = explode('@', $email); + $email = head($email); + return $email; + } + + public function getAccountImageURI() { + $uri = $this->getOAuthAccountData('picture'); + + // Change the "sz" parameter ("size") from the default to 100 to ask for + // a 100x100px image. + if ($uri !== null) { + $uri = new PhutilURI($uri); + $uri->replaceQueryParam('sz', 100); + $uri = (string)$uri; + } + + return $uri; + } + + public function getAccountURI() { + return $this->getOAuthAccountData('link'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://accounts.google.com/o/oauth2/auth'; + } + + protected function getTokenBaseURI() { + return 'https://accounts.google.com/o/oauth2/token'; + } + + public function getScope() { + $scopes = array( + 'email', + 'profile', + ); + + return implode(' ', $scopes); + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://www.googleapis.com/userinfo/v2/me'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + list($status, $body) = $future->resolve(); + + if ($status->isError()) { + throw $status; + } + + try { + $result = phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Google account data request.'), + $ex); + } + + return $result; + } + +} diff --git a/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php b/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php new file mode 100644 index 0000000000..a045065590 --- /dev/null +++ b/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php @@ -0,0 +1,164 @@ +jiraBaseURI = $jira_base_uri; + return $this; + } + + public function getJIRABaseURI() { + return $this->jiraBaseURI; + } + + public function getAccountID() { + // Make sure the handshake is finished; this method is used for its + // side effect by Auth providers. + $this->getHandshakeData(); + + return idx($this->getUserInfo(), 'key'); + } + + public function getAccountName() { + return idx($this->getUserInfo(), 'name'); + } + + public function getAccountImageURI() { + $avatars = idx($this->getUserInfo(), 'avatarUrls'); + if ($avatars) { + return idx($avatars, '48x48'); + } + return null; + } + + public function getAccountRealName() { + return idx($this->getUserInfo(), 'displayName'); + } + + public function getAccountEmail() { + return idx($this->getUserInfo(), 'emailAddress'); + } + + public function getAdapterType() { + return 'jira'; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterDomain($domain) { + $this->adapterDomain = $domain; + return $this; + } + + protected function getSignatureMethod() { + return 'RSA-SHA1'; + } + + protected function getRequestTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/request-token'); + } + + protected function getAuthorizeTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/authorize'); + } + + protected function getValidateTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/access-token'); + } + + private function getJIRAURI($path) { + return rtrim($this->jiraBaseURI, '/').'/'.ltrim($path, '/'); + } + + private function getUserInfo() { + if ($this->userInfo === null) { + $this->currentSession = $this->newJIRAFuture('rest/auth/1/session', 'GET') + ->resolveJSON(); + + // The session call gives us the username, but not the user key or other + // information. Make a second call to get additional information. + + $params = array( + 'username' => $this->currentSession['name'], + ); + + $this->userInfo = $this->newJIRAFuture('rest/api/2/user', 'GET', $params) + ->resolveJSON(); + } + + return $this->userInfo; + } + + public static function newJIRAKeypair() { + $config = array( + 'digest_alg' => 'sha512', + 'private_key_bits' => 4096, + 'private_key_type' => OPENSSL_KEYTYPE_RSA, + ); + + $res = openssl_pkey_new($config); + if (!$res) { + throw new Exception(pht('%s failed!', 'openssl_pkey_new()')); + } + + $private_key = null; + $ok = openssl_pkey_export($res, $private_key); + if (!$ok) { + throw new Exception(pht('%s failed!', 'openssl_pkey_export()')); + } + + $public_key = openssl_pkey_get_details($res); + if (!$ok || empty($public_key['key'])) { + throw new Exception(pht('%s failed!', 'openssl_pkey_get_details()')); + } + $public_key = $public_key['key']; + + return array($public_key, $private_key); + } + + + /** + * JIRA indicates that the user has clicked the "Deny" button by passing a + * well known `oauth_verifier` value ("denied"), which we check for here. + */ + protected function willFinishOAuthHandshake() { + $jira_magic_word = 'denied'; + if ($this->getVerifier() == $jira_magic_word) { + throw new PhutilAuthUserAbortedException(); + } + } + + public function newJIRAFuture($path, $method, $params = array()) { + if ($method == 'GET') { + $uri_params = $params; + $body_params = array(); + } else { + // For other types of requests, JIRA expects the request body to be + // JSON encoded. + $uri_params = array(); + $body_params = phutil_json_encode($params); + } + + $uri = new PhutilURI($this->getJIRAURI($path), $uri_params); + + // JIRA returns a 415 error if we don't provide a Content-Type header. + + return $this->newOAuth1Future($uri, $body_params) + ->setMethod($method) + ->addHeader('Content-Type', 'application/json'); + } + +} diff --git a/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php b/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php new file mode 100644 index 0000000000..14047c1761 --- /dev/null +++ b/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php @@ -0,0 +1,505 @@ +hostname = $host; + return $this; + } + + public function setPort($port) { + $this->port = $port; + return $this; + } + + public function getAdapterDomain() { + return 'self'; + } + + public function setBaseDistinguishedName($base_distinguished_name) { + $this->baseDistinguishedName = $base_distinguished_name; + return $this; + } + + public function setSearchAttributes(array $search_attributes) { + $this->searchAttributes = $search_attributes; + return $this; + } + + public function setUsernameAttribute($username_attribute) { + $this->usernameAttribute = $username_attribute; + return $this; + } + + public function setRealNameAttributes(array $attributes) { + $this->realNameAttributes = $attributes; + return $this; + } + + public function setLDAPVersion($ldap_version) { + $this->ldapVersion = $ldap_version; + return $this; + } + + public function setLDAPReferrals($ldap_referrals) { + $this->ldapReferrals = $ldap_referrals; + return $this; + } + + public function setLDAPStartTLS($ldap_start_tls) { + $this->ldapStartTLS = $ldap_start_tls; + return $this; + } + + public function setAnonymousUsername($anonymous_username) { + $this->anonymousUsername = $anonymous_username; + return $this; + } + + public function setAnonymousPassword( + PhutilOpaqueEnvelope $anonymous_password) { + $this->anonymousPassword = $anonymous_password; + return $this; + } + + public function setLoginUsername($login_username) { + $this->loginUsername = $login_username; + return $this; + } + + public function setLoginPassword(PhutilOpaqueEnvelope $login_password) { + $this->loginPassword = $login_password; + return $this; + } + + public function setActiveDirectoryDomain($domain) { + $this->activeDirectoryDomain = $domain; + return $this; + } + + public function setAlwaysSearch($always_search) { + $this->alwaysSearch = $always_search; + return $this; + } + + public function getAccountID() { + return $this->readLDAPRecordAccountID($this->getLDAPUserData()); + } + + public function getAccountName() { + return $this->readLDAPRecordAccountName($this->getLDAPUserData()); + } + + public function getAccountRealName() { + return $this->readLDAPRecordRealName($this->getLDAPUserData()); + } + + public function getAccountEmail() { + return $this->readLDAPRecordEmail($this->getLDAPUserData()); + } + + public function readLDAPRecordAccountID(array $record) { + $key = $this->usernameAttribute; + if (!strlen($key)) { + $key = head($this->searchAttributes); + } + return $this->readLDAPData($record, $key); + } + + public function readLDAPRecordAccountName(array $record) { + return $this->readLDAPRecordAccountID($record); + } + + public function readLDAPRecordRealName(array $record) { + $parts = array(); + foreach ($this->realNameAttributes as $attribute) { + $parts[] = $this->readLDAPData($record, $attribute); + } + $parts = array_filter($parts); + + if ($parts) { + return implode(' ', $parts); + } + + return null; + } + + public function readLDAPRecordEmail(array $record) { + return $this->readLDAPData($record, 'mail'); + } + + private function getLDAPUserData() { + if ($this->ldapUserData === null) { + $this->ldapUserData = $this->loadLDAPUserData(); + } + + return $this->ldapUserData; + } + + private function readLDAPData(array $data, $key, $default = null) { + $list = idx($data, $key); + if ($list === null) { + // At least in some cases (and maybe in all cases) the results from + // ldap_search() are keyed in lowercase. If we missed on the first + // try, retry with a lowercase key. + $list = idx($data, phutil_utf8_strtolower($key)); + } + + // NOTE: In most cases, the property is an array, like: + // + // array( + // 'count' => 1, + // 0 => 'actual-value-we-want', + // ) + // + // However, in at least the case of 'dn', the property is a bare string. + + if (is_scalar($list) && strlen($list)) { + return $list; + } else if (is_array($list)) { + return $list[0]; + } else { + return $default; + } + } + + private function formatLDAPAttributeSearch($attribute, $login_user) { + // If the attribute contains the literal token "${login}", treat it as a + // query and substitute the user's login name for the token. + + if (strpos($attribute, '${login}') !== false) { + $escaped_user = ldap_sprintf('%S', $login_user); + $attribute = str_replace('${login}', $escaped_user, $attribute); + return $attribute; + } + + // Otherwise, treat it as a simple attribute search. + + return ldap_sprintf( + '%Q=%S', + $attribute, + $login_user); + } + + private function loadLDAPUserData() { + $conn = $this->establishConnection(); + + $login_user = $this->loginUsername; + $login_pass = $this->loginPassword; + + if ($this->shouldBindWithoutIdentity()) { + $distinguished_name = null; + $search_query = null; + foreach ($this->searchAttributes as $attribute) { + $search_query = $this->formatLDAPAttributeSearch( + $attribute, + $login_user); + $record = $this->searchLDAPForRecord($search_query); + if ($record) { + $distinguished_name = $this->readLDAPData($record, 'dn'); + break; + } + } + if ($distinguished_name === null) { + throw new PhutilAuthCredentialException(); + } + } else { + $search_query = $this->formatLDAPAttributeSearch( + head($this->searchAttributes), + $login_user); + if ($this->activeDirectoryDomain) { + $distinguished_name = ldap_sprintf( + '%s@%Q', + $login_user, + $this->activeDirectoryDomain); + } else { + $distinguished_name = ldap_sprintf( + '%Q,%Q', + $search_query, + $this->baseDistinguishedName); + } + } + + $this->bindLDAP($conn, $distinguished_name, $login_pass); + + $result = $this->searchLDAPForRecord($search_query); + if (!$result) { + // This is unusual (since the bind succeeded) but we've seen it at least + // once in the wild, where the anonymous user is allowed to search but + // the credentialed user is not. + + // If we don't have anonymous credentials, raise an explicit exception + // here since we'll fail a typehint if we don't return an array anyway + // and this is a more useful error. + + // If we do have anonymous credentials, we'll rebind and try the search + // again below. Doing this automatically means things work correctly more + // often without requiring additional configuration. + if (!$this->shouldBindWithoutIdentity()) { + // No anonymous credentials, so we just fail here. + throw new Exception( + pht( + 'LDAP: Failed to retrieve record for user "%s" when searching. '. + 'Credentialed users may not be able to search your LDAP server. '. + 'Try configuring anonymous credentials or fully anonymous binds.', + $login_user)); + } else { + // Rebind as anonymous and try the search again. + $user = $this->anonymousUsername; + $pass = $this->anonymousPassword; + $this->bindLDAP($conn, $user, $pass); + + $result = $this->searchLDAPForRecord($search_query); + if (!$result) { + throw new Exception( + pht( + 'LDAP: Failed to retrieve record for user "%s" when searching '. + 'with both user and anonymous credentials.', + $login_user)); + } + } + } + + return $result; + } + + private function establishConnection() { + if (!$this->ldapConnection) { + $host = $this->hostname; + $port = $this->port; + + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'connect', + 'host' => $host, + 'port' => $this->port, + )); + + $conn = @ldap_connect($host, $this->port); + + $profiler->endServiceCall( + $call_id, + array( + 'ok' => (bool)$conn, + )); + + if (!$conn) { + throw new Exception( + pht('Unable to connect to LDAP server (%s:%d).', $host, $port)); + } + + $options = array( + LDAP_OPT_PROTOCOL_VERSION => (int)$this->ldapVersion, + LDAP_OPT_REFERRALS => (int)$this->ldapReferrals, + ); + + foreach ($options as $name => $value) { + $ok = @ldap_set_option($conn, $name, $value); + if (!$ok) { + $this->raiseConnectionException( + $conn, + pht( + "Unable to set LDAP option '%s' to value '%s'!", + $name, + $value)); + } + } + + if ($this->ldapStartTLS) { + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'start-tls', + )); + + // NOTE: This boils down to a function call to ldap_start_tls_s() in + // C, which is a service call. + $ok = @ldap_start_tls($conn); + + $profiler->endServiceCall( + $call_id, + array()); + + if (!$ok) { + $this->raiseConnectionException( + $conn, + pht('Unable to start TLS connection when connecting to LDAP.')); + } + } + + if ($this->shouldBindWithoutIdentity()) { + $user = $this->anonymousUsername; + $pass = $this->anonymousPassword; + $this->bindLDAP($conn, $user, $pass); + } + + $this->ldapConnection = $conn; + } + + return $this->ldapConnection; + } + + + private function searchLDAPForRecord($dn) { + $conn = $this->establishConnection(); + + $results = $this->searchLDAP('%Q', $dn); + + if (!$results) { + return null; + } + + if (count($results) > 1) { + throw new Exception( + pht( + 'LDAP record query returned more than one result. The query must '. + 'uniquely identify a record.')); + } + + return head($results); + } + + public function searchLDAP($pattern /* ... */) { + $args = func_get_args(); + $query = call_user_func_array('ldap_sprintf', $args); + + $conn = $this->establishConnection(); + + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'search', + 'dn' => $this->baseDistinguishedName, + 'query' => $query, + )); + + $result = @ldap_search($conn, $this->baseDistinguishedName, $query); + + $profiler->endServiceCall($call_id, array()); + + if (!$result) { + $this->raiseConnectionException( + $conn, + pht('LDAP search failed.')); + } + + $entries = @ldap_get_entries($conn, $result); + + if (!$entries) { + $this->raiseConnectionException( + $conn, + pht('Failed to get LDAP entries from search result.')); + } + + $results = array(); + for ($ii = 0; $ii < $entries['count']; $ii++) { + $results[] = $entries[$ii]; + } + + return $results; + } + + private function raiseConnectionException($conn, $message) { + $errno = @ldap_errno($conn); + $error = @ldap_error($conn); + + // This is `LDAP_INVALID_CREDENTIALS`. + if ($errno == 49) { + throw new PhutilAuthCredentialException(); + } + + if ($errno || $error) { + $full_message = pht( + "LDAP Exception: %s\nLDAP Error #%d: %s", + $message, + $errno, + $error); + } else { + $full_message = pht( + 'LDAP Exception: %s', + $message); + } + + throw new Exception($full_message); + } + + private function bindLDAP($conn, $user, PhutilOpaqueEnvelope $pass) { + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'bind', + 'user' => $user, + )); + + // NOTE: ldap_bind() dumps cleartext passwords into logs by default. Keep + // it quiet. + if (strlen($user)) { + $ok = @ldap_bind($conn, $user, $pass->openEnvelope()); + } else { + $ok = @ldap_bind($conn); + } + + $profiler->endServiceCall($call_id, array()); + + if (!$ok) { + if (strlen($user)) { + $this->raiseConnectionException( + $conn, + pht('Failed to bind to LDAP server (as user "%s").', $user)); + } else { + $this->raiseConnectionException( + $conn, + pht('Failed to bind to LDAP server (without username).')); + } + } + } + + + /** + * Determine if this adapter should attempt to bind to the LDAP server + * without a user identity. + * + * Generally, we can bind directly if we have a username/password, or if the + * "Always Search" flag is set, indicating that the empty username and + * password are sufficient. + * + * @return bool True if the adapter should perform binds without identity. + */ + private function shouldBindWithoutIdentity() { + return $this->alwaysSearch || strlen($this->anonymousUsername); + } + +} diff --git a/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php b/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php new file mode 100644 index 0000000000..aad5f0649d --- /dev/null +++ b/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php @@ -0,0 +1,211 @@ +privateKey = $private_key; + return $this; + } + + public function getPrivateKey() { + return $this->privateKey; + } + + public function setCallbackURI($callback_uri) { + $this->callbackURI = $callback_uri; + return $this; + } + + public function getCallbackURI() { + return $this->callbackURI; + } + + public function setVerifier($verifier) { + $this->verifier = $verifier; + return $this; + } + + public function getVerifier() { + return $this->verifier; + } + + public function setConsumerSecret(PhutilOpaqueEnvelope $consumer_secret) { + $this->consumerSecret = $consumer_secret; + return $this; + } + + public function getConsumerSecret() { + return $this->consumerSecret; + } + + public function setConsumerKey($consumer_key) { + $this->consumerKey = $consumer_key; + return $this; + } + + public function getConsumerKey() { + return $this->consumerKey; + } + + public function setTokenSecret($token_secret) { + $this->tokenSecret = $token_secret; + return $this; + } + + public function getTokenSecret() { + return $this->tokenSecret; + } + + public function setToken($token) { + $this->token = $token; + return $this; + } + + public function getToken() { + return $this->token; + } + + protected function getHandshakeData() { + if ($this->handshakeData === null) { + $this->finishOAuthHandshake(); + } + return $this->handshakeData; + } + + abstract protected function getRequestTokenURI(); + abstract protected function getAuthorizeTokenURI(); + abstract protected function getValidateTokenURI(); + + protected function getSignatureMethod() { + return 'HMAC-SHA1'; + } + + public function getContentSecurityPolicyFormActions() { + return array( + $this->getAuthorizeTokenURI(), + ); + } + + protected function newOAuth1Future($uri, $data = array()) { + $future = id(new PhutilOAuth1Future($uri, $data)) + ->setMethod('POST') + ->setSignatureMethod($this->getSignatureMethod()); + + $consumer_key = $this->getConsumerKey(); + if (strlen($consumer_key)) { + $future->setConsumerKey($consumer_key); + } else { + throw new Exception( + pht( + '%s is required!', + 'setConsumerKey()')); + } + + $consumer_secret = $this->getConsumerSecret(); + if ($consumer_secret) { + $future->setConsumerSecret($consumer_secret); + } + + if (strlen($this->getToken())) { + $future->setToken($this->getToken()); + } + + if (strlen($this->getTokenSecret())) { + $future->setTokenSecret($this->getTokenSecret()); + } + + if ($this->getPrivateKey()) { + $future->setPrivateKey($this->getPrivateKey()); + } + + return $future; + } + + public function getClientRedirectURI() { + $request_token_uri = $this->getRequestTokenURI(); + + $future = $this->newOAuth1Future($request_token_uri); + if (strlen($this->getCallbackURI())) { + $future->setCallbackURI($this->getCallbackURI()); + } + + list($body) = $future->resolvex(); + $data = id(new PhutilQueryStringParser())->parseQueryString($body); + + // NOTE: Per the spec, this value MUST be the string 'true'. + $confirmed = idx($data, 'oauth_callback_confirmed'); + if ($confirmed !== 'true') { + throw new Exception( + pht("Expected '%s' to be '%s'!", 'oauth_callback_confirmed', 'true')); + } + + $this->readTokenAndTokenSecret($data); + + $authorize_token_uri = new PhutilURI($this->getAuthorizeTokenURI()); + $authorize_token_uri->replaceQueryParam('oauth_token', $this->getToken()); + + return (string)$authorize_token_uri; + } + + protected function finishOAuthHandshake() { + $this->willFinishOAuthHandshake(); + + if (!$this->getToken()) { + throw new Exception(pht('Expected token to finish OAuth handshake!')); + } + if (!$this->getVerifier()) { + throw new Exception(pht('Expected verifier to finish OAuth handshake!')); + } + + $validate_uri = $this->getValidateTokenURI(); + $params = array( + 'oauth_verifier' => $this->getVerifier(), + ); + + list($body) = $this->newOAuth1Future($validate_uri, $params)->resolvex(); + $data = id(new PhutilQueryStringParser())->parseQueryString($body); + + $this->readTokenAndTokenSecret($data); + + $this->handshakeData = $data; + } + + private function readTokenAndTokenSecret(array $data) { + $token = idx($data, 'oauth_token'); + if (!$token) { + throw new Exception(pht("Expected '%s' in response!", 'oauth_token')); + } + + $token_secret = idx($data, 'oauth_token_secret'); + if (!$token_secret) { + throw new Exception( + pht("Expected '%s' in response!", 'oauth_token_secret')); + } + + $this->setToken($token); + $this->setTokenSecret($token_secret); + + return $this; + } + + /** + * Hook that allows subclasses to take actions before the OAuth handshake + * is completed. + */ + protected function willFinishOAuthHandshake() { + return; + } + +} diff --git a/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php b/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php new file mode 100644 index 0000000000..47d299ee36 --- /dev/null +++ b/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php @@ -0,0 +1,228 @@ + $this->getClientID(), + 'scope' => $this->getScope(), + 'redirect_uri' => $this->getRedirectURI(), + 'state' => $this->getState(), + ) + $this->getExtraAuthenticateParameters(); + + $uri = new PhutilURI($this->getAuthenticateBaseURI(), $params); + + return phutil_string_cast($uri); + } + + public function getAdapterType() { + $this_class = get_class($this); + $type_name = str_replace('PhutilAuthAdapterOAuth', '', $this_class); + return strtolower($type_name); + } + + public function setState($state) { + $this->state = $state; + return $this; + } + + public function getState() { + return $this->state; + } + + public function setCode($code) { + $this->code = $code; + return $this; + } + + public function getCode() { + return $this->code; + } + + public function setRedirectURI($redirect_uri) { + $this->redirectURI = $redirect_uri; + return $this; + } + + public function getRedirectURI() { + return $this->redirectURI; + } + + public function getExtraAuthenticateParameters() { + return array(); + } + + public function getExtraTokenParameters() { + return array(); + } + + public function getExtraRefreshParameters() { + return array(); + } + + public function setScope($scope) { + $this->scope = $scope; + return $this; + } + + public function getScope() { + return $this->scope; + } + + public function setClientSecret(PhutilOpaqueEnvelope $client_secret) { + $this->clientSecret = $client_secret; + return $this; + } + + public function getClientSecret() { + return $this->clientSecret; + } + + public function setClientID($client_id) { + $this->clientID = $client_id; + return $this; + } + + public function getClientID() { + return $this->clientID; + } + + public function getAccessToken() { + return $this->getAccessTokenData('access_token'); + } + + public function getAccessTokenExpires() { + return $this->getAccessTokenData('expires_epoch'); + } + + public function getRefreshToken() { + return $this->getAccessTokenData('refresh_token'); + } + + protected function getAccessTokenData($key, $default = null) { + if ($this->accessTokenData === null) { + $this->accessTokenData = $this->loadAccessTokenData(); + } + + return idx($this->accessTokenData, $key, $default); + } + + public function supportsTokenRefresh() { + return false; + } + + public function refreshAccessToken($refresh_token) { + $this->accessTokenData = $this->loadRefreshTokenData($refresh_token); + return $this; + } + + protected function loadRefreshTokenData($refresh_token) { + $params = array( + 'refresh_token' => $refresh_token, + ) + $this->getExtraRefreshParameters(); + + // NOTE: Make sure we return the refresh_token so that subsequent + // calls to getRefreshToken() return it; providers normally do not echo + // it back for token refresh requests. + + return $this->makeTokenRequest($params) + array( + 'refresh_token' => $refresh_token, + ); + } + + protected function loadAccessTokenData() { + $code = $this->getCode(); + if (!$code) { + throw new PhutilInvalidStateException('setCode'); + } + + $params = array( + 'code' => $this->getCode(), + ) + $this->getExtraTokenParameters(); + + return $this->makeTokenRequest($params); + } + + private function makeTokenRequest(array $params) { + $uri = $this->getTokenBaseURI(); + $query_data = array( + 'client_id' => $this->getClientID(), + 'client_secret' => $this->getClientSecret()->openEnvelope(), + 'redirect_uri' => $this->getRedirectURI(), + ) + $params; + + $future = new HTTPSFuture($uri, $query_data); + $future->setMethod('POST'); + list($body) = $future->resolvex(); + + $data = $this->readAccessTokenResponse($body); + + if (isset($data['expires_in'])) { + $data['expires_epoch'] = $data['expires_in']; + } else if (isset($data['expires'])) { + $data['expires_epoch'] = $data['expires']; + } + + // If we got some "expires" value back, interpret it as an epoch timestamp + // if it's after the year 2010 and as a relative number of seconds + // otherwise. + if (isset($data['expires_epoch'])) { + if ($data['expires_epoch'] < (60 * 60 * 24 * 365 * 40)) { + $data['expires_epoch'] += time(); + } + } + + if (isset($data['error'])) { + throw new Exception(pht('Access token error: %s', $data['error'])); + } + + return $data; + } + + protected function readAccessTokenResponse($body) { + // NOTE: Most providers either return JSON or HTTP query strings, so try + // both mechanisms. If your provider does something else, override this + // method. + + $data = json_decode($body, true); + + if (!is_array($data)) { + $data = array(); + parse_str($body, $data); + } + + if (empty($data['access_token']) && + empty($data['error'])) { + throw new Exception( + pht('Failed to decode OAuth access token response: %s', $body)); + } + + return $data; + } + + protected function getOAuthAccountData($key, $default = null) { + if ($this->oauthAccountData === null) { + $this->oauthAccountData = $this->loadOAuthAccountData(); + } + + return idx($this->oauthAccountData, $key, $default); + } + +} diff --git a/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php b/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php new file mode 100644 index 0000000000..e66ba32f94 --- /dev/null +++ b/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php @@ -0,0 +1,102 @@ +phabricatorBaseURI = $uri; + return $this; + } + + public function getPhabricatorBaseURI() { + return $this->phabricatorBaseURI; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterDomain($domain) { + $this->adapterDomain = $domain; + return $this; + } + + public function getAdapterType() { + return 'phabricator'; + } + + public function getAccountID() { + return $this->getOAuthAccountData('phid'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('primaryEmail'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('userName'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('image'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('uri'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('realName'); + } + + protected function getAuthenticateBaseURI() { + return $this->getPhabricatorURI('oauthserver/auth/'); + } + + protected function getTokenBaseURI() { + return $this->getPhabricatorURI('oauthserver/token/'); + } + + public function getScope() { + return ''; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = id(new PhutilURI($this->getPhabricatorURI('api/user.whoami'))) + ->replaceQueryParam('access_token', $this->getAccessToken()); + list($body) = id(new HTTPSFuture($uri))->resolvex(); + + try { + $data = phutil_json_decode($body); + return $data['result']; + } catch (PhutilJSONParserException $ex) { + throw new Exception( + pht( + 'Expected valid JSON response from Phabricator %s request.', + 'user.whoami'), + $ex); + } + } + + private function getPhabricatorURI($path) { + return rtrim($this->phabricatorBaseURI, '/').'/'.ltrim($path, '/'); + } + +} diff --git a/src/applications/auth/adapter/PhutilSlackAuthAdapter.php b/src/applications/auth/adapter/PhutilSlackAuthAdapter.php new file mode 100644 index 0000000000..6578a9af7a --- /dev/null +++ b/src/applications/auth/adapter/PhutilSlackAuthAdapter.php @@ -0,0 +1,61 @@ +getOAuthAccountData('user'); + return idx($user, 'id'); + } + + public function getAccountEmail() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'email'); + } + + public function getAccountImageURI() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'image_512'); + } + + public function getAccountRealName() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://slack.com/oauth/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://slack.com/api/oauth.access'; + } + + public function getScope() { + return 'identity.basic,identity.team,identity.avatar'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilSlackFuture()) + ->setAccessToken($this->getAccessToken()) + ->setRawSlackQuery('users.identity') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php b/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php new file mode 100644 index 0000000000..dce2c7e2f0 --- /dev/null +++ b/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php @@ -0,0 +1,76 @@ +getOAuthAccountData('_id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('name'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('logo'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if ($name) { + return 'http://www.twitch.tv/'.$name; + } + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('display_name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://api.twitch.tv/kraken/oauth2/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://api.twitch.tv/kraken/oauth2/token'; + } + + public function getScope() { + return 'user_read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilTwitchFuture()) + ->setClientID($this->getClientID()) + ->setAccessToken($this->getAccessToken()) + ->setRawTwitchQuery('user') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php b/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php new file mode 100644 index 0000000000..6f738c75f6 --- /dev/null +++ b/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php @@ -0,0 +1,75 @@ +getHandshakeData(), 'user_id'); + } + + public function getAccountName() { + return idx($this->getHandshakeData(), 'screen_name'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if (strlen($name)) { + return 'https://twitter.com/'.$name; + } + return null; + } + + public function getAccountImageURI() { + $info = $this->getUserInfo(); + return idx($info, 'profile_image_url'); + } + + public function getAccountRealName() { + $info = $this->getUserInfo(); + return idx($info, 'name'); + } + + public function getAdapterType() { + return 'twitter'; + } + + public function getAdapterDomain() { + return 'twitter.com'; + } + + protected function getRequestTokenURI() { + return 'https://api.twitter.com/oauth/request_token'; + } + + protected function getAuthorizeTokenURI() { + return 'https://api.twitter.com/oauth/authorize'; + } + + protected function getValidateTokenURI() { + return 'https://api.twitter.com/oauth/access_token'; + } + + private function getUserInfo() { + if ($this->userInfo === null) { + $params = array( + 'user_id' => $this->getAccountID(), + ); + + $uri = new PhutilURI( + 'https://api.twitter.com/1.1/users/show.json', + $params); + + $data = $this->newOAuth1Future($uri) + ->setMethod('GET') + ->resolveJSON(); + + $this->userInfo = $data; + } + return $this->userInfo; + } + +} diff --git a/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php b/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php new file mode 100644 index 0000000000..c09c7cffab --- /dev/null +++ b/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php @@ -0,0 +1,73 @@ +getOAuthAccountData('ID'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('username'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar_URL'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('profile_URL'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('display_name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://public-api.wordpress.com/oauth2/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://public-api.wordpress.com/oauth2/token'; + } + + public function getScope() { + return 'user_read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + 'blog_id' => 0, + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilWordPressFuture()) + ->setClientID($this->getClientID()) + ->setAccessToken($this->getAccessToken()) + ->setRawWordPressQuery('/me/') + ->resolve(); + } + +} diff --git a/src/applications/auth/exception/PhutilAuthConfigurationException.php b/src/applications/auth/exception/PhutilAuthConfigurationException.php new file mode 100644 index 0000000000..e684076d84 --- /dev/null +++ b/src/applications/auth/exception/PhutilAuthConfigurationException.php @@ -0,0 +1,6 @@ +\d{4})(?P\d{2})(?P\d{2})'. + '(?:'. + 'T(?P\d{2})(?P\d{2})(?P\d{2})(?Z)?'. + ')?'. + '\z/'; + + $matches = null; + $ok = preg_match($pattern, $value, $matches); + if (!$ok) { + throw new Exception( + pht( + 'Expected ISO8601 datetime in the format "19990105T112233Z", '. + 'found "%s".', + $value)); + } + + if (isset($matches['z'])) { + if ($timezone != 'UTC') { + throw new Exception( + pht( + 'ISO8601 date ends in "Z" indicating UTC, but a timezone other '. + 'than UTC ("%s") was specified.', + $timezone)); + } + } + + $datetime = id(new self()) + ->setYear((int)$matches['y']) + ->setMonth((int)$matches['m']) + ->setDay((int)$matches['d']) + ->setTimezone($timezone); + + if (isset($matches['h'])) { + $datetime + ->setHour((int)$matches['h']) + ->setMinute((int)$matches['i']) + ->setSecond((int)$matches['s']); + } else { + $datetime + ->setIsAllDay(true); + } + + return $datetime; + } + + public static function newFromEpoch($epoch, $timezone = 'UTC') { + $date = new DateTime('@'.$epoch); + + $zone = new DateTimeZone($timezone); + $date->setTimezone($zone); + + return id(new self()) + ->setYear((int)$date->format('Y')) + ->setMonth((int)$date->format('m')) + ->setDay((int)$date->format('d')) + ->setHour((int)$date->format('H')) + ->setMinute((int)$date->format('i')) + ->setSecond((int)$date->format('s')) + ->setTimezone($timezone); + } + + public static function newFromDictionary(array $dict) { + static $keys; + if ($keys === null) { + $keys = array_fuse( + array( + 'kind', + 'year', + 'month', + 'day', + 'hour', + 'minute', + 'second', + 'timezone', + 'isAllDay', + )); + } + + foreach ($dict as $key => $value) { + if (!isset($keys[$key])) { + throw new Exception( + pht( + 'Unexpected key "%s" in datetime dictionary, expected keys: %s.', + $key, + implode(', ', array_keys($keys)))); + } + } + + if (idx($dict, 'kind') !== 'absolute') { + throw new Exception( + pht( + 'Expected key "%s" with value "%s" in datetime dictionary.', + 'kind', + 'absolute')); + } + + if (!isset($dict['year'])) { + throw new Exception( + pht( + 'Expected key "%s" in datetime dictionary.', + 'year')); + } + + $datetime = id(new self()) + ->setYear(idx($dict, 'year')) + ->setMonth(idx($dict, 'month', 1)) + ->setDay(idx($dict, 'day', 1)) + ->setHour(idx($dict, 'hour', 0)) + ->setMinute(idx($dict, 'minute', 0)) + ->setSecond(idx($dict, 'second', 0)) + ->setTimezone(idx($dict, 'timezone')) + ->setIsAllDay((bool)idx($dict, 'isAllDay', false)); + + return $datetime; + } + + public function newRelativeDateTime($duration) { + if (is_string($duration)) { + $duration = PhutilCalendarDuration::newFromISO8601($duration); + } + + if (!($duration instanceof PhutilCalendarDuration)) { + throw new Exception( + pht( + 'Expected "PhutilCalendarDuration" object or ISO8601 duration '. + 'string.')); + } + + return id(new PhutilCalendarRelativeDateTime()) + ->setOrigin($this) + ->setDuration($duration); + } + + public function toDictionary() { + return array( + 'kind' => 'absolute', + 'year' => (int)$this->getYear(), + 'month' => (int)$this->getMonth(), + 'day' => (int)$this->getDay(), + 'hour' => (int)$this->getHour(), + 'minute' => (int)$this->getMinute(), + 'second' => (int)$this->getSecond(), + 'timezone' => $this->getTimezone(), + 'isAllDay' => (bool)$this->getIsAllDay(), + ); + } + + public function setYear($year) { + $this->year = $year; + return $this; + } + + public function getYear() { + return $this->year; + } + + public function setMonth($month) { + $this->month = $month; + return $this; + } + + public function getMonth() { + return $this->month; + } + + public function setDay($day) { + $this->day = $day; + return $this; + } + + public function getDay() { + return $this->day; + } + + public function setHour($hour) { + $this->hour = $hour; + return $this; + } + + public function getHour() { + return $this->hour; + } + + public function setMinute($minute) { + $this->minute = $minute; + return $this; + } + + public function getMinute() { + return $this->minute; + } + + public function setSecond($second) { + $this->second = $second; + return $this; + } + + public function getSecond() { + return $this->second; + } + + public function setTimezone($timezone) { + $this->timezone = $timezone; + return $this; + } + + public function getTimezone() { + return $this->timezone; + } + + private function getEffectiveTimezone() { + $date_timezone = $this->getTimezone(); + $viewer_timezone = $this->getViewerTimezone(); + + // Because all-day events are always "floating", the effective timezone + // is the viewer timezone if it is available. Otherwise, we'll return a + // DateTime object with the correct values, but it will be incorrectly + // adjusted forward or backward to the viewer's zone later. + + $zones = array(); + if ($this->getIsAllDay()) { + $zones[] = $viewer_timezone; + $zones[] = $date_timezone; + } else { + $zones[] = $date_timezone; + $zones[] = $viewer_timezone; + } + $zones = array_filter($zones); + + if (!$zones) { + throw new Exception( + pht( + 'Datetime has no timezone or viewer timezone.')); + } + + return head($zones); + } + + public function newPHPDateTimeZone() { + $zone = $this->getEffectiveTimezone(); + return new DateTimeZone($zone); + } + + public function newPHPDateTime() { + $zone = $this->newPHPDateTimeZone(); + + $y = $this->getYear(); + $m = $this->getMonth(); + $d = $this->getDay(); + + if ($this->getIsAllDay()) { + $h = 0; + $i = 0; + $s = 0; + } else { + $h = $this->getHour(); + $i = $this->getMinute(); + $s = $this->getSecond(); + } + + $format = sprintf('%04d-%02d-%02d %02d:%02d:%02d', $y, $m, $d, $h, $i, $s); + + return new DateTime($format, $zone); + } + + + public function newAbsoluteDateTime() { + return clone $this; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php b/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php new file mode 100644 index 0000000000..5beebb784e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php @@ -0,0 +1,30 @@ +children; + } + + final public function getChildrenOfType($type) { + $result = array(); + + foreach ($this->getChildren() as $key => $child) { + if ($child->getNodeType() != $type) { + continue; + } + $result[$key] = $child; + } + + return $result; + } + + final public function appendChild(PhutilCalendarNode $node) { + $this->children[] = $node; + return $this; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarDateTime.php new file mode 100644 index 0000000000..f9bf5a07a4 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDateTime.php @@ -0,0 +1,54 @@ +viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function setIsAllDay($is_all_day) { + $this->isAllDay = $is_all_day; + return $this; + } + + public function getIsAllDay() { + return $this->isAllDay; + } + + public function getEpoch() { + $datetime = $this->newPHPDateTime(); + return (int)$datetime->format('U'); + } + + public function getISO8601() { + $datetime = $this->newPHPDateTime(); + + if ($this->getIsAllDay()) { + return $datetime->format('Ymd'); + } else if ($this->getTimezone()) { + // With a timezone, the event occurs at a specific second universally. + // We return the UTC representation of that point in time. + $datetime->setTimezone(new DateTimeZone('UTC')); + return $datetime->format('Ymd\\THis\\Z'); + } else { + // With no timezone, events are "floating" and occur at local time. + // We return a representation without the "Z". + return $datetime->format('Ymd\\THis'); + } + } + + abstract public function newPHPDateTimeZone(); + abstract public function newPHPDateTime(); + abstract public function newAbsoluteDateTime(); + + abstract public function getTimezone(); +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php b/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php new file mode 100644 index 0000000000..b2e92dd432 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php @@ -0,0 +1,12 @@ +getChildrenOfType(PhutilCalendarEventNode::NODETYPE); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDuration.php b/src/applications/calendar/parser/data/PhutilCalendarDuration.php new file mode 100644 index 0000000000..863150e17e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDuration.php @@ -0,0 +1,181 @@ + $value) { + if (!isset($keys[$key])) { + throw new Exception( + pht( + 'Unexpected key "%s" in duration dictionary, expected keys: %s.', + $key, + implode(', ', array_keys($keys)))); + } + } + + $duration = id(new self()) + ->setIsNegative(idx($dict, 'isNegative', false)) + ->setWeeks(idx($dict, 'weeks', 0)) + ->setDays(idx($dict, 'days', 0)) + ->setHours(idx($dict, 'hours', 0)) + ->setMinutes(idx($dict, 'minutes', 0)) + ->setSeconds(idx($dict, 'seconds', 0)); + + return $duration; + } + + public function toDictionary() { + return array( + 'isNegative' => $this->getIsNegative(), + 'weeks' => $this->getWeeks(), + 'days' => $this->getDays(), + 'hours' => $this->getHours(), + 'minutes' => $this->getMinutes(), + 'seconds' => $this->getSeconds(), + ); + } + + public static function newFromISO8601($value) { + $pattern = + '/^'. + '(?P[+-])?'. + 'P'. + '(?:'. + '(?P\d+)W'. + '|'. + '(?:(?:(?P\d+)D)?'. + '(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?(?:(?P\d+)S)?)?'. + ')'. + ')'. + '\z/'; + + $matches = null; + $ok = preg_match($pattern, $value, $matches); + if (!$ok) { + throw new Exception( + pht( + 'Expected ISO8601 duration in the format "P12DT3H4M5S", found '. + '"%s".', + $value)); + } + + $is_negative = (idx($matches, 'sign') == '-'); + + return id(new self()) + ->setIsNegative($is_negative) + ->setWeeks((int)idx($matches, 'W', 0)) + ->setDays((int)idx($matches, 'D', 0)) + ->setHours((int)idx($matches, 'H', 0)) + ->setMinutes((int)idx($matches, 'M', 0)) + ->setSeconds((int)idx($matches, 'S', 0)); + } + + public function toISO8601() { + $parts = array(); + $parts[] = 'P'; + + $weeks = $this->getWeeks(); + if ($weeks) { + $parts[] = $weeks.'W'; + } else { + $days = $this->getDays(); + if ($days) { + $parts[] = $days.'D'; + } + + $parts[] = 'T'; + + $hours = $this->getHours(); + if ($hours) { + $parts[] = $hours.'H'; + } + + $minutes = $this->getMinutes(); + if ($minutes) { + $parts[] = $minutes.'M'; + } + + $seconds = $this->getSeconds(); + if ($seconds) { + $parts[] = $seconds.'S'; + } + } + + return implode('', $parts); + } + + public function setIsNegative($is_negative) { + $this->isNegative = $is_negative; + return $this; + } + + public function getIsNegative() { + return $this->isNegative; + } + + public function setWeeks($weeks) { + $this->weeks = $weeks; + return $this; + } + + public function getWeeks() { + return $this->weeks; + } + + public function setDays($days) { + $this->days = $days; + return $this; + } + + public function getDays() { + return $this->days; + } + + public function setHours($hours) { + $this->hours = $hours; + return $this; + } + + public function getHours() { + return $this->hours; + } + + public function setMinutes($minutes) { + $this->minutes = $minutes; + return $this; + } + + public function getMinutes() { + return $this->minutes; + } + + public function setSeconds($seconds) { + $this->seconds = $seconds; + return $this; + } + + public function getSeconds() { + return $this->seconds; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarEventNode.php b/src/applications/calendar/parser/data/PhutilCalendarEventNode.php new file mode 100644 index 0000000000..d3d33aa77e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarEventNode.php @@ -0,0 +1,172 @@ +uid = $uid; + return $this; + } + + public function getUID() { + return $this->uid; + } + + public function setName($name) { + $this->name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setDescription($description) { + $this->description = $description; + return $this; + } + + public function getDescription() { + return $this->description; + } + + public function setStartDateTime(PhutilCalendarDateTime $start) { + $this->startDateTime = $start; + return $this; + } + + public function getStartDateTime() { + return $this->startDateTime; + } + + public function setEndDateTime(PhutilCalendarDateTime $end) { + $this->endDateTime = $end; + return $this; + } + + public function getEndDateTime() { + $end = $this->endDateTime; + if ($end) { + return $end; + } + + $start = $this->getStartDateTime(); + $duration = $this->getDuration(); + if ($start && $duration) { + return id(new PhutilCalendarRelativeDateTime()) + ->setOrigin($start) + ->setDuration($duration); + } + + // If no end date or duration are specified, the event is instantaneous. + return $start; + } + + public function setDuration(PhutilCalendarDuration $duration) { + $this->duration = $duration; + return $this; + } + + public function getDuration() { + return $this->duration; + } + + public function setCreatedDateTime(PhutilCalendarDateTime $created) { + $this->createdDateTime = $created; + return $this; + } + + public function getCreatedDateTime() { + return $this->createdDateTime; + } + + public function setModifiedDateTime(PhutilCalendarDateTime $modified) { + $this->modifiedDateTime = $modified; + return $this; + } + + public function getModifiedDateTime() { + return $this->modifiedDateTime; + } + + public function setOrganizer(PhutilCalendarUserNode $organizer) { + $this->organizer = $organizer; + return $this; + } + + public function getOrganizer() { + return $this->organizer; + } + + public function setAttendees(array $attendees) { + assert_instances_of($attendees, 'PhutilCalendarUserNode'); + $this->attendees = $attendees; + return $this; + } + + public function getAttendees() { + return $this->attendees; + } + + public function addAttendee(PhutilCalendarUserNode $attendee) { + $this->attendees[] = $attendee; + return $this; + } + + public function setRecurrenceRule( + PhutilCalendarRecurrenceRule $recurrence_rule) { + $this->recurrenceRule = $recurrence_rule; + return $this; + } + + public function getRecurrenceRule() { + return $this->recurrenceRule; + } + + public function setRecurrenceExceptions(array $recurrence_exceptions) { + assert_instances_of($recurrence_exceptions, 'PhutilCalendarDateTime'); + $this->recurrenceExceptions = $recurrence_exceptions; + return $this; + } + + public function getRecurrenceExceptions() { + return $this->recurrenceExceptions; + } + + public function setRecurrenceDates(array $recurrence_dates) { + assert_instances_of($recurrence_dates, 'PhutilCalendarDateTime'); + $this->recurrenceDates = $recurrence_dates; + return $this; + } + + public function getRecurrenceDates() { + return $this->recurrenceDates; + } + + public function setRecurrenceID($recurrence_id) { + $this->recurrenceID = $recurrence_id; + return $this; + } + + public function getRecurrenceID() { + return $this->recurrenceID; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarNode.php b/src/applications/calendar/parser/data/PhutilCalendarNode.php new file mode 100644 index 0000000000..e4d7905bc3 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarNode.php @@ -0,0 +1,20 @@ +getPhobjectClassConstant('NODETYPE'); + } + + final public function setAttribute($key, $value) { + $this->attributes[$key] = $value; + return $this; + } + + final public function getAttribute($key, $default = null) { + return idx($this->attributes, $key, $default); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php new file mode 100644 index 0000000000..1293c484ab --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php @@ -0,0 +1,51 @@ +proxy = $proxy; + return $this; + } + + final protected function getProxy() { + return $this->proxy; + } + + public function __clone() { + $this->proxy = clone $this->proxy; + } + + public function setViewerTimezone($timezone) { + $this->getProxy()->setViewerTimezone($timezone); + return $this; + } + + public function getViewerTimezone() { + return $this->getProxy()->getViewerTimezone(); + } + + public function setIsAllDay($is_all_day) { + $this->getProxy()->setIsAllDay($is_all_day); + return $this; + } + + public function getIsAllDay() { + return $this->getProxy()->getIsAllDay(); + } + + public function newPHPDateTimezone() { + return $this->getProxy()->newPHPDateTimezone(); + } + + public function newPHPDateTime() { + return $this->getProxy()->newPHPDateTime(); + } + + public function getTimezone() { + return $this->getProxy()->getTimezone(); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRawNode.php b/src/applications/calendar/parser/data/PhutilCalendarRawNode.php new file mode 100644 index 0000000000..228b3208fb --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRawNode.php @@ -0,0 +1,8 @@ +dates = $dates; + return $this; + } + + public function getDates() { + return $this->dates; + } + + public function resetSource() { + foreach ($this->getDates() as $date) { + $date->setViewerTimezone($this->getViewerTimezone()); + } + + $order = msort($this->getDates(), 'getEpoch'); + $order = array_reverse($order); + $this->order = $order; + + return $this; + } + + public function getNextEvent($cursor) { + while ($this->order) { + $next = array_pop($this->order); + if ($next->getEpoch() >= $cursor) { + return $next; + } + } + + return null; + } + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php new file mode 100644 index 0000000000..504f7d8e9e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php @@ -0,0 +1,1820 @@ +getFrequency(); + + $interval = $this->getInterval(); + if ($interval != 1) { + $parts['INTERVAL'] = $interval; + } + + $by_second = $this->getBySecond(); + if ($by_second) { + $parts['BYSECOND'] = $by_second; + } + + $by_minute = $this->getByMinute(); + if ($by_minute) { + $parts['BYMINUTE'] = $by_minute; + } + + $by_hour = $this->getByHour(); + if ($by_hour) { + $parts['BYHOUR'] = $by_hour; + } + + $by_day = $this->getByDay(); + if ($by_day) { + $parts['BYDAY'] = $by_day; + } + + $by_month = $this->getByMonth(); + if ($by_month) { + $parts['BYMONTH'] = $by_month; + } + + $by_monthday = $this->getByMonthDay(); + if ($by_monthday) { + $parts['BYMONTHDAY'] = $by_monthday; + } + + $by_yearday = $this->getByYearDay(); + if ($by_yearday) { + $parts['BYYEARDAY'] = $by_yearday; + } + + $by_weekno = $this->getByWeekNumber(); + if ($by_weekno) { + $parts['BYWEEKNO'] = $by_weekno; + } + + $by_setpos = $this->getBySetPosition(); + if ($by_setpos) { + $parts['BYSETPOS'] = $by_setpos; + } + + $wkst = $this->getWeekStart(); + if ($wkst != self::WEEKDAY_MONDAY) { + $parts['WKST'] = $wkst; + } + + $count = $this->getCount(); + if ($count) { + $parts['COUNT'] = $count; + } + + $until = $this->getUntil(); + if ($until) { + $parts['UNTIL'] = $until->getISO8601(); + } + + return $parts; + } + + public static function newFromDictionary(array $dict) { + static $expect; + if ($expect === null) { + $expect = array_fuse( + array( + 'FREQ', + 'INTERVAL', + 'BYSECOND', + 'BYMINUTE', + 'BYHOUR', + 'BYDAY', + 'BYMONTH', + 'BYMONTHDAY', + 'BYYEARDAY', + 'BYWEEKNO', + 'BYSETPOS', + 'WKST', + 'UNTIL', + 'COUNT', + )); + } + + foreach ($dict as $key => $value) { + if (empty($expect[$key])) { + throw new Exception( + pht( + 'RRULE dictionary includes unknown key "%s". Expected keys '. + 'are: %s.', + $key, + implode(', ', array_keys($expect)))); + } + } + + $rrule = id(new self()) + ->setFrequency(idx($dict, 'FREQ')) + ->setInterval(idx($dict, 'INTERVAL', 1)) + ->setBySecond(idx($dict, 'BYSECOND', array())) + ->setByMinute(idx($dict, 'BYMINUTE', array())) + ->setByHour(idx($dict, 'BYHOUR', array())) + ->setByDay(idx($dict, 'BYDAY', array())) + ->setByMonth(idx($dict, 'BYMONTH', array())) + ->setByMonthDay(idx($dict, 'BYMONTHDAY', array())) + ->setByYearDay(idx($dict, 'BYYEARDAY', array())) + ->setByWeekNumber(idx($dict, 'BYWEEKNO', array())) + ->setBySetPosition(idx($dict, 'BYSETPOS', array())) + ->setWeekStart(idx($dict, 'WKST', self::WEEKDAY_MONDAY)); + + $count = idx($dict, 'COUNT'); + if ($count) { + $rrule->setCount($count); + } + + $until = idx($dict, 'UNTIL'); + if ($until) { + $until = PhutilCalendarAbsoluteDateTime::newFromISO8601($until); + $rrule->setUntil($until); + } + + return $rrule; + } + + public function toRRULE() { + $dict = $this->toDictionary(); + + $parts = array(); + foreach ($dict as $key => $value) { + if (is_array($value)) { + $value = implode(',', $value); + } + $parts[] = "{$key}={$value}"; + } + + return implode(';', $parts); + } + + public static function newFromRRULE($rrule) { + $parts = explode(';', $rrule); + + $dict = array(); + foreach ($parts as $part) { + list($key, $value) = explode('=', $part, 2); + switch ($key) { + case 'FREQ': + case 'INTERVAL': + case 'WKST': + case 'COUNT': + case 'UNTIL'; + break; + default: + $value = explode(',', $value); + break; + } + $dict[$key] = $value; + } + + $int_lists = array_fuse( + array( + // NOTE: "BYDAY" is absent, and takes a list like "MO, TU, WE". + 'BYSECOND', + 'BYMINUTE', + 'BYHOUR', + 'BYMONTH', + 'BYMONTHDAY', + 'BYYEARDAY', + 'BYWEEKNO', + 'BYSETPOS', + )); + + $int_values = array_fuse( + array( + 'COUNT', + 'INTERVAL', + )); + + foreach ($dict as $key => $value) { + if (isset($int_values[$key])) { + // None of these values may be negative. + if (!preg_match('/^\d+\z/', $value)) { + throw new Exception( + pht( + 'Unexpected value "%s" in "%s" RULE property: expected an '. + 'integer.', + $value, + $key)); + } + $dict[$key] = (int)$value; + } + + if (isset($int_lists[$key])) { + foreach ($value as $k => $v) { + if (!preg_match('/^-?\d+\z/', $v)) { + throw new Exception( + pht( + 'Unexpected value "%s" in "%s" RRULE property: expected '. + 'only integers.', + $v, + $key)); + } + $value[$k] = (int)$v; + } + $dict[$key] = $value; + } + } + + return self::newFromDictionary($dict); + } + + private static function getAllWeekdayConstants() { + return array_keys(self::getWeekdayIndexMap()); + } + + private static function getWeekdayIndexMap() { + static $map = array( + self::WEEKDAY_SUNDAY => self::WEEKINDEX_SUNDAY, + self::WEEKDAY_MONDAY => self::WEEKINDEX_MONDAY, + self::WEEKDAY_TUESDAY => self::WEEKINDEX_TUESDAY, + self::WEEKDAY_WEDNESDAY => self::WEEKINDEX_WEDNESDAY, + self::WEEKDAY_THURSDAY => self::WEEKINDEX_THURSDAY, + self::WEEKDAY_FRIDAY => self::WEEKINDEX_FRIDAY, + self::WEEKDAY_SATURDAY => self::WEEKINDEX_SATURDAY, + ); + + return $map; + } + + private static function getWeekdayIndex($weekday) { + $map = self::getWeekdayIndexMap(); + if (!isset($map[$weekday])) { + $constants = array_keys($map); + throw new Exception( + pht( + 'Weekday "%s" is not a valid weekday constant. Valid constants '. + 'are: %s.', + $weekday, + implode(', ', $constants))); + } + + return $map[$weekday]; + } + + public function setStartDateTime(PhutilCalendarDateTime $start) { + $this->startDateTime = $start; + return $this; + } + + public function getStartDateTime() { + return $this->startDateTime; + } + + public function setCount($count) { + if ($count < 1) { + throw new Exception( + pht( + 'RRULE COUNT value "%s" is invalid: count must be at least 1.', + $count)); + } + + $this->count = $count; + return $this; + } + + public function getCount() { + return $this->count; + } + + public function setUntil(PhutilCalendarDateTime $until) { + $this->until = $until; + return $this; + } + + public function getUntil() { + return $this->until; + } + + public function setFrequency($frequency) { + static $map = array( + self::FREQUENCY_SECONDLY => self::SCALE_SECONDLY, + self::FREQUENCY_MINUTELY => self::SCALE_MINUTELY, + self::FREQUENCY_HOURLY => self::SCALE_HOURLY, + self::FREQUENCY_DAILY => self::SCALE_DAILY, + self::FREQUENCY_WEEKLY => self::SCALE_WEEKLY, + self::FREQUENCY_MONTHLY => self::SCALE_MONTHLY, + self::FREQUENCY_YEARLY => self::SCALE_YEARLY, + ); + + if (empty($map[$frequency])) { + throw new Exception( + pht( + 'RRULE FREQ "%s" is invalid. Valid frequencies are: %s.', + $frequency, + implode(', ', array_keys($map)))); + } + + $this->frequency = $frequency; + $this->frequencyScale = $map[$frequency]; + + return $this; + } + + public function getFrequency() { + return $this->frequency; + } + + public function getFrequencyScale() { + return $this->frequencyScale; + } + + public function setInterval($interval) { + if (!is_int($interval)) { + throw new Exception( + pht( + 'RRULE INTERVAL "%s" is invalid: interval must be an integer.', + $interval)); + } + + if ($interval < 1) { + throw new Exception( + pht( + 'RRULE INTERVAL "%s" is invalid: interval must be 1 or more.', + $interval)); + } + + $this->interval = $interval; + return $this; + } + + public function getInterval() { + return $this->interval; + } + + public function setBySecond(array $by_second) { + $this->assertByRange('BYSECOND', $by_second, 0, 60); + $this->bySecond = array_fuse($by_second); + return $this; + } + + public function getBySecond() { + return $this->bySecond; + } + + public function setByMinute(array $by_minute) { + $this->assertByRange('BYMINUTE', $by_minute, 0, 59); + $this->byMinute = array_fuse($by_minute); + return $this; + } + + public function getByMinute() { + return $this->byMinute; + } + + public function setByHour(array $by_hour) { + $this->assertByRange('BYHOUR', $by_hour, 0, 23); + $this->byHour = array_fuse($by_hour); + return $this; + } + + public function getByHour() { + return $this->byHour; + } + + public function setByDay(array $by_day) { + $constants = self::getAllWeekdayConstants(); + $constants = implode('|', $constants); + + $pattern = '/^(?:[+-]?([1-9]\d?))?('.$constants.')\z/'; + foreach ($by_day as $key => $value) { + $matches = null; + if (!preg_match($pattern, $value, $matches)) { + throw new Exception( + pht( + 'RRULE BYDAY value "%s" is invalid: rule part must be in the '. + 'expected form (like "MO", "-3TH", or "+2SU").', + $value)); + } + + // The maximum allowed value is 53, which corresponds to "the 53rd + // Monday every year" or similar when evaluated against a YEARLY rule. + + $maximum = 53; + $magnitude = (int)$matches[1]; + if ($magnitude > $maximum) { + throw new Exception( + pht( + 'RRULE BYDAY value "%s" has an offset with magnitude "%s", but '. + 'the maximum permitted value is "%s".', + $value, + $magnitude, + $maximum)); + } + + // Normalize "+3FR" into "3FR". + $by_day[$key] = ltrim($value, '+'); + } + + $this->byDay = array_fuse($by_day); + return $this; + } + + public function getByDay() { + return $this->byDay; + } + + public function setByMonthDay(array $by_month_day) { + $this->assertByRange('BYMONTHDAY', $by_month_day, -31, 31, false); + $this->byMonthDay = array_fuse($by_month_day); + return $this; + } + + public function getByMonthDay() { + return $this->byMonthDay; + } + + public function setByYearDay($by_year_day) { + $this->assertByRange('BYYEARDAY', $by_year_day, -366, 366, false); + $this->byYearDay = array_fuse($by_year_day); + return $this; + } + + public function getByYearDay() { + return $this->byYearDay; + } + + public function setByMonth(array $by_month) { + $this->assertByRange('BYMONTH', $by_month, 1, 12); + $this->byMonth = array_fuse($by_month); + return $this; + } + + public function getByMonth() { + return $this->byMonth; + } + + public function setByWeekNumber(array $by_week_number) { + $this->assertByRange('BYWEEKNO', $by_week_number, -53, 53, false); + $this->byWeekNumber = array_fuse($by_week_number); + return $this; + } + + public function getByWeekNumber() { + return $this->byWeekNumber; + } + + public function setBySetPosition(array $by_set_position) { + $this->assertByRange('BYSETPOS', $by_set_position, -366, 366, false); + $this->bySetPosition = $by_set_position; + return $this; + } + + public function getBySetPosition() { + return $this->bySetPosition; + } + + public function setWeekStart($week_start) { + // Make sure this is a valid weekday constant. + self::getWeekdayIndex($week_start); + + $this->weekStart = $week_start; + return $this; + } + + public function getWeekStart() { + return $this->weekStart; + } + + public function resetSource() { + $frequency = $this->getFrequency(); + + if ($this->getByMonthDay()) { + switch ($frequency) { + case self::FREQUENCY_WEEKLY: + // RFC5545: "The BYMONTHDAY rule part MUST NOT be specified when the + // FREQ rule part is set to WEEKLY." + throw new Exception( + pht( + 'RRULE specifies BYMONTHDAY with FREQ set to WEEKLY, which '. + 'violates RFC5545.')); + break; + default: + break; + } + + } + + if ($this->getByYearDay()) { + switch ($frequency) { + case self::FREQUENCY_DAILY: + case self::FREQUENCY_WEEKLY: + case self::FREQUENCY_MONTHLY: + // RFC5545: "The BYYEARDAY rule part MUST NOT be specified when the + // FREQ rule part is set to DAILY, WEEKLY, or MONTHLY." + throw new Exception( + pht( + 'RRULE specifies BYYEARDAY with FREQ of DAILY, WEEKLY or '. + 'MONTHLY, which violates RFC5545.')); + default: + break; + } + } + + // TODO + // RFC5545: "The BYDAY rule part MUST NOT be specified with a numeric + // value when the FREQ rule part is not set to MONTHLY or YEARLY." + // RFC5545: "Furthermore, the BYDAY rule part MUST NOT be specified with a + // numeric value with the FREQ rule part set to YEARLY when the BYWEEKNO + // rule part is specified." + + + $date = $this->getStartDateTime(); + + $this->cursorSecond = $date->getSecond(); + $this->cursorMinute = $date->getMinute(); + $this->cursorHour = $date->getHour(); + + $this->cursorDay = $date->getDay(); + $this->cursorMonth = $date->getMonth(); + $this->cursorYear = $date->getYear(); + + $year_map = $this->getYearMap($this->cursorYear, $this->getWeekStart()); + $key = $this->cursorMonth.'M'.$this->cursorDay.'D'; + $this->cursorWeek = $year_map['info'][$key]['week']; + $this->cursorWeekday = $year_map['info'][$key]['weekday']; + + $this->setSeconds = array(); + $this->setMinutes = array(); + $this->setHours = array(); + $this->setDays = array(); + $this->setMonths = array(); + $this->setYears = array(); + + $this->stateSecond = null; + $this->stateMinute = null; + $this->stateHour = null; + $this->stateDay = null; + $this->stateWeek = null; + $this->stateMonth = null; + $this->stateYear = null; + + // If we have a BYSETPOS, we need to generate the entire set before we + // can filter it and return results. Normally, we start generating at + // the start date, but we need to go back one interval to generate + // BYSETPOS events so we can make sure the entire set is generated. + if ($this->getBySetPosition()) { + $interval = $this->getInterval(); + switch ($frequency) { + case self::FREQUENCY_YEARLY: + $this->cursorYear -= $interval; + break; + case self::FREQUENCY_MONTHLY: + $this->cursorMonth -= $interval; + $this->rewindMonth(); + break; + case self::FREQUENCY_WEEKLY: + $this->cursorWeek -= $interval; + $this->rewindWeek(); + break; + case self::FREQUENCY_DAILY: + $this->cursorDay -= $interval; + $this->rewindDay(); + break; + case self::FREQUENCY_HOURLY: + $this->cursorHour -= $interval; + $this->rewindHour(); + break; + case self::FREQUENCY_MINUTELY: + $this->cursorMinute -= $interval; + $this->rewindMinute(); + break; + case self::FREQUENCY_SECONDLY: + default: + throw new Exception( + pht( + 'RRULE specifies BYSETPOS with FREQ "%s", but this is invalid.', + $frequency)); + } + } + + // We can generate events from before the cursor when evaluating rules + // with BYSETPOS or FREQ=WEEKLY. + $this->minimumEpoch = $this->getStartDateTime()->getEpoch(); + + $cursor_state = array( + 'year' => $this->cursorYear, + 'month' => $this->cursorMonth, + 'week' => $this->cursorWeek, + 'day' => $this->cursorDay, + 'hour' => $this->cursorHour, + ); + + $this->cursorDayState = $cursor_state; + $this->cursorWeekState = $cursor_state; + $this->cursorHourState = $cursor_state; + + $by_hour = $this->getByHour(); + $by_minute = $this->getByMinute(); + $by_second = $this->getBySecond(); + + $scale = $this->getFrequencyScale(); + + // We return all-day events if the start date is an all-day event and we + // don't have more granular selectors or a more granular frequency. + $this->isAllDay = $date->getIsAllDay() + && !$by_hour + && !$by_minute + && !$by_second + && ($scale > self::SCALE_HOURLY); + } + + public function getNextEvent($cursor) { + while (true) { + $event = $this->generateNextEvent(); + if (!$event) { + break; + } + + $epoch = $event->getEpoch(); + if ($this->minimumEpoch) { + if ($epoch < $this->minimumEpoch) { + continue; + } + } + + if ($epoch < $cursor) { + continue; + } + + break; + } + + return $event; + } + + private function generateNextEvent() { + if ($this->activeSet) { + return array_pop($this->activeSet); + } + + $this->baseYear = $this->cursorYear; + + $by_setpos = $this->getBySetPosition(); + if ($by_setpos) { + $old_state = $this->getSetPositionState(); + } + + while (!$this->activeSet) { + $this->activeSet = $this->nextSet; + $this->nextSet = array(); + + while (true) { + if ($this->isAllDay) { + $this->nextDay(); + } else { + $this->nextSecond(); + } + + $result = id(new PhutilCalendarAbsoluteDateTime()) + ->setTimezone($this->getStartDateTime()->getTimezone()) + ->setViewerTimezone($this->getViewerTimezone()) + ->setYear($this->stateYear) + ->setMonth($this->stateMonth) + ->setDay($this->stateDay); + + if ($this->isAllDay) { + $result->setIsAllDay(true); + } else { + $result + ->setHour($this->stateHour) + ->setMinute($this->stateMinute) + ->setSecond($this->stateSecond); + } + + // If we don't have BYSETPOS, we're all done. We put this into the + // set and will immediately return it. + if (!$by_setpos) { + $this->activeSet[] = $result; + break; + } + + // Otherwise, check if we've completed a set. The set is complete if + // the state has moved past the span we were examining (for example, + // with a YEARLY event, if the state is now in the next year). + $new_state = $this->getSetPositionState(); + if ($new_state == $old_state) { + $this->activeSet[] = $result; + continue; + } + + $this->activeSet = $this->applySetPos($this->activeSet, $by_setpos); + $this->activeSet = array_reverse($this->activeSet); + $this->nextSet[] = $result; + $old_state = $new_state; + break; + } + } + + return array_pop($this->activeSet); + } + + + protected function nextSecond() { + if ($this->setSeconds) { + $this->stateSecond = array_pop($this->setSeconds); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $is_secondly = ($frequency == self::FREQUENCY_SECONDLY); + $by_second = $this->getBySecond(); + + while (!$this->setSeconds) { + $this->nextMinute(); + + if ($is_secondly || $by_second) { + $seconds = $this->newSecondsSet( + ($is_secondly ? $interval : 1), + $by_second); + } else { + $seconds = array( + $this->cursorSecond, + ); + } + + $this->setSeconds = array_reverse($seconds); + } + + $this->stateSecond = array_pop($this->setSeconds); + } + + protected function nextMinute() { + if ($this->setMinutes) { + $this->stateMinute = array_pop($this->setMinutes); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_minutely = ($frequency === self::FREQUENCY_MINUTELY); + $by_minute = $this->getByMinute(); + + while (!$this->setMinutes) { + $this->nextHour(); + + if ($is_minutely || $by_minute) { + $minutes = $this->newMinutesSet( + ($is_minutely ? $interval : 1), + $by_minute); + } else if ($scale < self::SCALE_MINUTELY) { + $minutes = $this->newMinutesSet( + 1, + array()); + } else { + $minutes = array( + $this->cursorMinute, + ); + } + + $this->setMinutes = array_reverse($minutes); + } + + $this->stateMinute = array_pop($this->setMinutes); + } + + protected function nextHour() { + if ($this->setHours) { + $this->stateHour = array_pop($this->setHours); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_hourly = ($frequency === self::FREQUENCY_HOURLY); + $by_hour = $this->getByHour(); + + while (!$this->setHours) { + $this->nextDay(); + + $is_dynamic = $is_hourly + || $by_hour + || ($scale < self::SCALE_HOURLY); + + if ($is_dynamic) { + $hours = $this->newHoursSet( + ($is_hourly ? $interval : 1), + $by_hour); + } else { + $hours = array( + $this->cursorHour, + ); + } + + $this->setHours = array_reverse($hours); + } + + $this->stateHour = array_pop($this->setHours); + } + + protected function nextDay() { + if ($this->setDays) { + $info = array_pop($this->setDays); + $this->setDayState($info); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_daily = ($frequency === self::FREQUENCY_DAILY); + $is_weekly = ($frequency === self::FREQUENCY_WEEKLY); + + $by_day = $this->getByDay(); + $by_monthday = $this->getByMonthDay(); + $by_yearday = $this->getByYearDay(); + $by_weekno = $this->getByWeekNumber(); + $by_month = $this->getByMonth(); + $week_start = $this->getWeekStart(); + + while (!$this->setDays) { + if ($is_weekly) { + $this->nextWeek(); + } else { + $this->nextMonth(); + } + + // NOTE: We normally handle BYMONTH when iterating months, but it acts + // like a filter if FREQ=WEEKLY. + + $is_dynamic = $is_daily + || $is_weekly + || $by_day + || $by_monthday + || $by_yearday + || $by_weekno + || ($by_month && $is_weekly) + || ($scale < self::SCALE_DAILY); + + if ($is_dynamic) { + $weeks = $this->newDaysSet( + ($is_daily ? $interval : 1), + $by_day, + $by_monthday, + $by_yearday, + $by_weekno, + $by_month, + $week_start); + } else { + // The cursor day may not actually exist in the current month, so + // make sure the day is valid before we generate a set which contains + // it. + $year_map = $this->getYearMap($this->stateYear, $week_start); + if ($this->cursorDay > $year_map['monthDays'][$this->stateMonth]) { + $weeks = array( + array(), + ); + } else { + $key = $this->stateMonth.'M'.$this->cursorDay.'D'; + $weeks = array( + array($year_map['info'][$key]), + ); + } + } + + // Unpack the weeks into days. + $days = array_mergev($weeks); + + $this->setDays = array_reverse($days); + } + + $info = array_pop($this->setDays); + $this->setDayState($info); + } + + private function setDayState(array $info) { + $this->stateDay = $info['monthday']; + $this->stateWeek = $info['week']; + $this->stateMonth = $info['month']; + } + + protected function nextMonth() { + if ($this->setMonths) { + $this->stateMonth = array_pop($this->setMonths); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_monthly = ($frequency === self::FREQUENCY_MONTHLY); + + $by_month = $this->getByMonth(); + + // If we have a BYMONTHDAY, we consider that set of days in every month. + // For example, "FREQ=YEARLY;BYMONTHDAY=3" means "the third day of every + // month", so we need to expand the month set if the constraint is present. + $by_monthday = $this->getByMonthDay(); + + // Likewise, we need to generate all months if we have BYYEARDAY or + // BYWEEKNO or BYDAY. + $by_yearday = $this->getByYearDay(); + $by_weekno = $this->getByWeekNumber(); + $by_day = $this->getByDay(); + + while (!$this->setMonths) { + $this->nextYear(); + + $is_dynamic = $is_monthly + || $by_month + || $by_monthday + || $by_yearday + || $by_weekno + || $by_day + || ($scale < self::SCALE_MONTHLY); + + if ($is_dynamic) { + $months = $this->newMonthsSet( + ($is_monthly ? $interval : 1), + $by_month); + } else { + $months = array( + $this->cursorMonth, + ); + } + + $this->setMonths = array_reverse($months); + } + + $this->stateMonth = array_pop($this->setMonths); + } + + protected function nextWeek() { + if ($this->setWeeks) { + $this->stateWeek = array_pop($this->setWeeks); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $by_weekno = $this->getByWeekNumber(); + + while (!$this->setWeeks) { + $this->nextYear(); + + $weeks = $this->newWeeksSet( + $interval, + $by_weekno); + + $this->setWeeks = array_reverse($weeks); + } + + $this->stateWeek = array_pop($this->setWeeks); + } + + protected function nextYear() { + $this->stateYear = $this->cursorYear; + + $frequency = $this->getFrequency(); + $is_yearly = ($frequency === self::FREQUENCY_YEARLY); + + if ($is_yearly) { + $interval = $this->getInterval(); + } else { + $interval = 1; + } + + $this->cursorYear = $this->cursorYear + $interval; + + if ($this->cursorYear > ($this->baseYear + 100)) { + throw new Exception( + pht( + 'RRULE evaluation failed to generate more events in the next 100 '. + 'years. This RRULE is likely invalid or degenerate.')); + } + + } + + private function newSecondsSet($interval, $set) { + // TODO: This doesn't account for leap seconds. In theory, it probably + // should, although this shouldn't impact any real events. + $seconds_in_minute = 60; + + if ($this->cursorSecond >= $seconds_in_minute) { + $this->cursorSecond -= $seconds_in_minute; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorSecond, + $interval, + $set, + $seconds_in_minute); + + $this->cursorSecond = ($cursor - $seconds_in_minute); + + return $result; + } + + private function newMinutesSet($interval, $set) { + // NOTE: This value is legitimately a constant! Amazing! + $minutes_in_hour = 60; + + if ($this->cursorMinute >= $minutes_in_hour) { + $this->cursorMinute -= $minutes_in_hour; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorMinute, + $interval, + $set, + $minutes_in_hour); + + $this->cursorMinute = ($cursor - $minutes_in_hour); + + return $result; + } + + private function newHoursSet($interval, $set) { + // TODO: This doesn't account for hours caused by daylight savings time. + // It probably should, although this seems unlikely to impact any real + // events. + $hours_in_day = 24; + + // If the hour cursor is behind the current time, we need to forward it in + // INTERVAL increments so we end up with the right offset. + list($skip, $this->cursorHourState) = $this->advanceCursorState( + $this->cursorHourState, + self::SCALE_HOURLY, + $interval, + $this->getWeekStart()); + + if ($skip) { + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorHour, + $interval, + $set, + $hours_in_day); + + $this->cursorHour = ($cursor - $hours_in_day); + + return $result; + } + + private function newWeeksSet($interval, $set) { + $week_start = $this->getWeekStart(); + + list($skip, $this->cursorWeekState) = $this->advanceCursorState( + $this->cursorWeekState, + self::SCALE_WEEKLY, + $interval, + $week_start); + + if ($skip) { + return array(); + } + + $year_map = $this->getYearMap($this->stateYear, $week_start); + + $result = array(); + while (true) { + if (!isset($year_map['weekMap'][$this->cursorWeek])) { + break; + } + $result[] = $this->cursorWeek; + $this->cursorWeek += $interval; + } + + $this->cursorWeek -= $year_map['weekCount']; + + return $result; + } + + private function newDaysSet( + $interval_day, + $by_day, + $by_monthday, + $by_yearday, + $by_weekno, + $by_month, + $week_start) { + + $frequency = $this->getFrequency(); + $is_yearly = ($frequency == self::FREQUENCY_YEARLY); + $is_monthly = ($frequency == self::FREQUENCY_MONTHLY); + $is_weekly = ($frequency == self::FREQUENCY_WEEKLY); + + $selection = array(); + if ($is_weekly) { + $year_map = $this->getYearMap($this->stateYear, $week_start); + + if (isset($year_map['weekMap'][$this->stateWeek])) { + foreach ($year_map['weekMap'][$this->stateWeek] as $key) { + $selection[] = $year_map['info'][$key]; + } + } + } else { + // If the day cursor is behind the current year and month, we need to + // forward it in INTERVAL increments so we end up with the right offset + // in the current month. + list($skip, $this->cursorDayState) = $this->advanceCursorState( + $this->cursorDayState, + self::SCALE_DAILY, + $interval_day, + $week_start); + + if (!$skip) { + $year_map = $this->getYearMap($this->stateYear, $week_start); + while (true) { + $month_idx = $this->stateMonth; + $month_days = $year_map['monthDays'][$month_idx]; + if ($this->cursorDay > $month_days) { + // NOTE: The year map is now out of date, but we're about to break + // out of the loop anyway so it doesn't matter. + break; + } + + $day_idx = $this->cursorDay; + + $key = "{$month_idx}M{$day_idx}D"; + $selection[] = $year_map['info'][$key]; + + $this->cursorDay += $interval_day; + } + } + } + + // As a special case, BYDAY applies to relative month offsets if BYMONTH + // is present in a YEARLY rule. + if ($is_yearly) { + if ($this->getByMonth()) { + $is_yearly = false; + $is_monthly = true; + } + } + + // As a special case, BYDAY makes us examine all week days. This doesn't + // check BYMONTHDAY or BYYEARDAY because they are not valid with WEEKLY. + $filter_weekday = true; + if ($is_weekly) { + if ($by_day) { + $filter_weekday = false; + } + } + + $weeks = array(); + foreach ($selection as $key => $info) { + if ($is_weekly) { + if ($filter_weekday) { + if ($info['weekday'] != $this->cursorWeekday) { + continue; + } + } + } else { + if ($info['month'] != $this->stateMonth) { + continue; + } + } + + if ($by_day) { + if (empty($by_day[$info['weekday']])) { + if ($is_yearly) { + if (empty($by_day[$info['weekday.yearly']]) && + empty($by_day[$info['-weekday.yearly']])) { + continue; + } + } else if ($is_monthly) { + if (empty($by_day[$info['weekday.monthly']]) && + empty($by_day[$info['-weekday.monthly']])) { + continue; + } + } else { + continue; + } + } + } + + if ($by_monthday) { + if (empty($by_monthday[$info['monthday']]) && + empty($by_monthday[$info['-monthday']])) { + continue; + } + } + + if ($by_yearday) { + if (empty($by_yearday[$info['yearday']]) && + empty($by_yearday[$info['-yearday']])) { + continue; + } + } + + if ($by_weekno) { + if (empty($by_weekno[$info['week']]) && + empty($by_weekno[$info['-week']])) { + continue; + } + } + + if ($by_month) { + if (empty($by_month[$info['month']])) { + continue; + } + } + + $weeks[$info['week']][] = $info; + } + + return array_values($weeks); + } + + private function newMonthsSet($interval, $set) { + // NOTE: This value is also a real constant! Wow! + $months_in_year = 12; + + if ($this->cursorMonth > $months_in_year) { + $this->cursorMonth -= $months_in_year; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorMonth, + $interval, + $set, + $months_in_year + 1); + + $this->cursorMonth = ($cursor - $months_in_year); + + return $result; + } + + public static function getYearMap($year, $week_start) { + static $maps = array(); + + $key = "{$year}/{$week_start}"; + if (isset($maps[$key])) { + return $maps[$key]; + } + + $map = self::newYearMap($year, $week_start); + $maps[$key] = $map; + + return $maps[$key]; + } + + private static function newYearMap($year, $weekday_start) { + $weekday_index = self::getWeekdayIndex($weekday_start); + + $is_leap = (($year % 4 === 0) && ($year % 100 !== 0)) || + ($year % 400 === 0); + + // There may be some clever way to figure out which day of the week a given + // year starts on and avoid the cost of a DateTime construction, but I + // wasn't able to turn it up and we only need to do this once per year. + $datetime = new DateTime("{$year}-01-01", new DateTimeZone('UTC')); + $weekday = (int)$datetime->format('w'); + + if ($is_leap) { + $max_day = 366; + } else { + $max_day = 365; + } + + $month_days = array( + 1 => 31, + 2 => $is_leap ? 29 : 28, + 3 => 31, + 4 => 30, + 5 => 31, + 6 => 30, + 7 => 31, + 8 => 31, + 9 => 30, + 10 => 31, + 11 => 30, + 12 => 31, + ); + + // Per the spec, the first week of the year must contain at least four + // days. If the week starts on a Monday but the year starts on a Saturday, + // the first couple of days don't count as a week. In this case, the first + // week will begin on January 3. + $first_week_size = 0; + $first_weekday = $weekday; + for ($year_day = 1; $year_day <= $max_day; $year_day++) { + $first_weekday = ($first_weekday + 1) % 7; + $first_week_size++; + if ($first_weekday === $weekday_index) { + break; + } + } + + if ($first_week_size >= 4) { + $week_number = 1; + } else { + $week_number = 0; + } + + $info_map = array(); + + $weekday_map = self::getWeekdayIndexMap(); + $weekday_map = array_flip($weekday_map); + + $yearly_counts = array(); + $monthly_counts = array(); + + $month_number = 1; + $month_day = 1; + for ($year_day = 1; $year_day <= $max_day; $year_day++) { + $key = "{$month_number}M{$month_day}D"; + + $short_day = $weekday_map[$weekday]; + if (empty($yearly_counts[$short_day])) { + $yearly_counts[$short_day] = 0; + } + $yearly_counts[$short_day]++; + + if (empty($monthly_counts[$month_number][$short_day])) { + $monthly_counts[$month_number][$short_day] = 0; + } + $monthly_counts[$month_number][$short_day]++; + + $info = array( + 'year' => $year, + 'key' => $key, + 'month' => $month_number, + 'monthday' => $month_day, + '-monthday' => -$month_days[$month_number] + $month_day - 1, + 'yearday' => $year_day, + '-yearday' => -$max_day + $year_day - 1, + 'week' => $week_number, + 'weekday' => $short_day, + 'weekday.yearly' => $yearly_counts[$short_day], + 'weekday.monthly' => $monthly_counts[$month_number][$short_day], + ); + + $info_map[$key] = $info; + + $weekday = ($weekday + 1) % 7; + if ($weekday === $weekday_index) { + $week_number++; + } + + $month_day = ($month_day + 1); + if ($month_day > $month_days[$month_number]) { + $month_day = 1; + $month_number++; + } + } + + // Check how long the final week is. If it doesn't have four days, this + // is really the first week of the next year. + $final_week = array(); + foreach ($info_map as $key => $info) { + if ($info['week'] == $week_number) { + $final_week[] = $key; + } + } + + if (count($final_week) < 4) { + $week_number = $week_number - 1; + $next_year = self::getYearMap($year + 1, $weekday_start); + $next_year_weeks = $next_year['weekCount']; + } else { + $next_year_weeks = null; + } + + if ($first_week_size < 4) { + $last_year = self::getYearMap($year - 1, $weekday_start); + $last_year_weeks = $last_year['weekCount']; + } else { + $last_year_weeks = null; + } + + // Now that we know how many weeks the year has, we can compute the + // negative offsets. + foreach ($info_map as $key => $info) { + $week = $info['week']; + + if ($week === 0) { + // If this day is part of the first partial week of the year, give + // it the week number of the last week of the prior year instead. + $info['week'] = $last_year_weeks; + $info['-week'] = -1; + } else if ($week > $week_number) { + // If this day is part of the last partial week of the year, give + // it week numbers from the next year. + $info['week'] = 1; + $info['-week'] = -$next_year_weeks; + } else { + $info['-week'] = -$week_number + $week - 1; + } + + // Do all the arithmetic to figure out if this is the -19th Thursday + // in the year and such. + $month_number = $info['month']; + $short_day = $info['weekday']; + $monthly_count = $monthly_counts[$month_number][$short_day]; + $monthly_index = $info['weekday.monthly']; + $info['-weekday.monthly'] = -$monthly_count + $monthly_index - 1; + $info['-weekday.monthly'] .= $short_day; + $info['weekday.monthly'] .= $short_day; + + $yearly_count = $yearly_counts[$short_day]; + $yearly_index = $info['weekday.yearly']; + $info['-weekday.yearly'] = -$yearly_count + $yearly_index - 1; + $info['-weekday.yearly'] .= $short_day; + $info['weekday.yearly'] .= $short_day; + + $info_map[$key] = $info; + } + + $week_map = array(); + foreach ($info_map as $key => $info) { + $week_map[$info['week']][] = $key; + } + + return array( + 'info' => $info_map, + 'weekCount' => $week_number, + 'dayCount' => $max_day, + 'monthDays' => $month_days, + 'weekMap' => $week_map, + ); + } + + private function newIteratorSet($cursor, $interval, $set, $limit) { + if ($interval < 1) { + throw new Exception( + pht( + 'Invalid iteration interval ("%d"), must be at least 1.', + $interval)); + } + + $result = array(); + $seen = array(); + + $ii = $cursor; + while (true) { + if (!$set || isset($set[$ii])) { + $result[] = $ii; + } + + $ii = ($ii + $interval); + + if ($ii >= $limit) { + break; + } + } + + sort($result); + $result = array_values($result); + + return array($ii, $result); + } + + private function applySetPos(array $values, array $setpos) { + $select = array(); + + $count = count($values); + foreach ($setpos as $pos) { + if ($pos > 0 && $pos <= $count) { + $select[] = ($pos - 1); + } else if ($pos < 0 && $pos >= -$count) { + $select[] = ($count + $pos); + } + } + + sort($select); + $select = array_unique($select); + + return array_select_keys($values, $select); + } + + private function assertByRange( + $source, + array $values, + $min, + $max, + $allow_zero = true) { + + foreach ($values as $value) { + if (!is_int($value)) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: values must be '. + 'integers.', + $value, + $source)); + } + + if ($value < $min || $value > $max) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: it must be '. + 'between %s and %s.', + $value, + $source, + $min, + $max)); + } + + if (!$value && !$allow_zero) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: it must not '. + 'be zero.', + $value, + $source)); + } + } + } + + private function getSetPositionState() { + $scale = $this->getFrequencyScale(); + + $parts = array(); + $parts[] = $this->stateYear; + + if ($scale == self::SCALE_WEEKLY) { + $parts[] = $this->stateWeek; + } else { + if ($scale < self::SCALE_YEARLY) { + $parts[] = $this->stateMonth; + } + if ($scale < self::SCALE_MONTHLY) { + $parts[] = $this->stateDay; + } + if ($scale < self::SCALE_DAILY) { + $parts[] = $this->stateHour; + } + if ($scale < self::SCALE_HOURLY) { + $parts[] = $this->stateMinute; + } + } + + return implode('/', $parts); + } + + private function rewindMonth() { + while ($this->cursorMonth < 1) { + $this->cursorYear--; + $this->cursorMonth += 12; + } + } + + private function rewindWeek() { + $week_start = $this->getWeekStart(); + while ($this->cursorWeek < 1) { + $this->cursorYear--; + $year_map = $this->getYearMap($this->cursorYear, $week_start); + $this->cursorWeek += $year_map['weekCount']; + } + } + + private function rewindDay() { + $week_start = $this->getWeekStart(); + while ($this->cursorDay < 1) { + $year_map = $this->getYearMap($this->cursorYear, $week_start); + $this->cursorDay += $year_map['monthDays'][$this->cursorMonth]; + $this->cursorMonth--; + $this->rewindMonth(); + } + } + + private function rewindHour() { + while ($this->cursorHour < 0) { + $this->cursorHour += 24; + $this->cursorDay--; + $this->rewindDay(); + } + } + + private function rewindMinute() { + while ($this->cursorMinute < 0) { + $this->cursorMinute += 60; + $this->cursorHour--; + $this->rewindHour(); + } + } + + private function advanceCursorState( + array $cursor, + $scale, + $interval, + $week_start) { + + $state = array( + 'year' => $this->stateYear, + 'month' => $this->stateMonth, + 'week' => $this->stateWeek, + 'day' => $this->stateDay, + 'hour' => $this->stateHour, + ); + + // In the common case when the interval is 1, we'll visit every possible + // value so we don't need to do any math and can just jump to the first + // hour, day, etc. + if ($interval == 1) { + if ($this->isCursorBehind($cursor, $state, $scale)) { + switch ($scale) { + case self::SCALE_DAILY: + $this->cursorDay = 1; + break; + case self::SCALE_HOURLY: + $this->cursorHour = 0; + break; + case self::SCALE_WEEKLY: + $this->cursorWeek = 1; + break; + } + } + + return array(false, $state); + } + + $year_map = $this->getYearMap($cursor['year'], $week_start); + while ($this->isCursorBehind($cursor, $state, $scale)) { + switch ($scale) { + case self::SCALE_DAILY: + $cursor['day'] += $interval; + break; + case self::SCALE_HOURLY: + $cursor['hour'] += $interval; + break; + case self::SCALE_WEEKLY: + $cursor['week'] += $interval; + break; + } + + if ($scale <= self::SCALE_HOURLY) { + while ($cursor['hour'] >= 24) { + $cursor['hour'] -= 24; + $cursor['day']++; + } + } + + if ($scale == self::SCALE_WEEKLY) { + while ($cursor['week'] > $year_map['weekCount']) { + $cursor['week'] -= $year_map['weekCount']; + $cursor['year']++; + $year_map = $this->getYearMap($cursor['year'], $week_start); + } + } + + if ($scale <= self::SCALE_DAILY) { + while ($cursor['day'] > $year_map['monthDays'][$cursor['month']]) { + $cursor['day'] -= $year_map['monthDays'][$cursor['month']]; + $cursor['month']++; + if ($cursor['month'] > 12) { + $cursor['month'] -= 12; + $cursor['year']++; + $year_map = $this->getYearMap($cursor['year'], $week_start); + } + } + } + } + + switch ($scale) { + case self::SCALE_DAILY: + $this->cursorDay = $cursor['day']; + break; + case self::SCALE_HOURLY: + $this->cursorHour = $cursor['hour']; + break; + case self::SCALE_WEEKLY: + $this->cursorWeek = $cursor['week']; + break; + } + + $skip = $this->isCursorBehind($state, $cursor, $scale); + + return array($skip, $cursor); + } + + private function isCursorBehind(array $cursor, array $state, $scale) { + if ($cursor['year'] < $state['year']) { + return true; + } else if ($cursor['year'] > $state['year']) { + return false; + } + + if ($scale == self::SCALE_WEEKLY) { + return false; + } + + if ($cursor['month'] < $state['month']) { + return true; + } else if ($cursor['month'] > $state['month']) { + return false; + } + + if ($scale >= self::SCALE_DAILY) { + return false; + } + + if ($cursor['day'] < $state['day']) { + return true; + } else if ($cursor['day'] > $state['day']) { + return false; + } + + if ($scale >= self::SCALE_HOURLY) { + return false; + } + + if ($cursor['hour'] < $state['hour']) { + return true; + } else if ($cursor['hour'] > $state['hour']) { + return false; + } + + return false; + } + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php new file mode 100644 index 0000000000..6ebcd4a58a --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php @@ -0,0 +1,162 @@ +sources[] = $source; + return $this; + } + + public function setViewerTimezone($viewer_timezone) { + $this->viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function getEventsBetween( + PhutilCalendarDateTime $start = null, + PhutilCalendarDateTime $end = null, + $limit = null) { + + if ($end === null && $limit === null) { + throw new Exception( + pht( + 'Recurring event range queries must have an end date, a limit, or '. + 'both.')); + } + + $timezone = $this->getViewerTimezone(); + + $sources = array(); + foreach ($this->sources as $source) { + $source = clone $source; + $source->setViewerTimezone($timezone); + $source->resetSource(); + + $sources[] = array( + 'source' => $source, + 'state' => null, + 'epoch' => null, + ); + } + + if ($start) { + $start = clone $start; + $start->setViewerTimezone($timezone); + $min_epoch = $start->getEpoch(); + } else { + $min_epoch = 0; + } + + if ($end) { + $end = clone $end; + $end->setViewerTimezone($timezone); + $end_epoch = $end->getEpoch(); + } else { + $end_epoch = null; + } + + $results = array(); + $index = 0; + $cursor = 0; + while (true) { + // Get the next event for each source which we don't have a future + // event for. + foreach ($sources as $key => $source) { + $state = $source['state']; + $epoch = $source['epoch']; + + if ($state !== null && $epoch >= $cursor) { + // We have an event for this source, and it's a future event, so + // we don't need to do anything. + continue; + } + + $next = $source['source']->getNextEvent($cursor); + if ($next === null) { + // This source doesn't have any more events, so we're all done. + unset($sources[$key]); + continue; + } + + $next_epoch = $next->getEpoch(); + + if ($end_epoch !== null && $next_epoch > $end_epoch) { + // We have an end time and the next event from this source is + // past that end, so we know there are no more relevant events + // coming from this source. + unset($sources[$key]); + continue; + } + + $sources[$key]['state'] = $next; + $sources[$key]['epoch'] = $next_epoch; + } + + if (!$sources) { + // We've run out of sources which can produce valid events in the + // window, so we're all done. + break; + } + + // Find the minimum event time across all sources. + $next_epoch = null; + foreach ($sources as $source) { + if ($next_epoch === null) { + $next_epoch = $source['epoch']; + } else { + $next_epoch = min($next_epoch, $source['epoch']); + } + } + + $is_exception = false; + $next_source = null; + foreach ($sources as $source) { + if ($source['epoch'] == $next_epoch) { + if ($source['source']->getIsExceptionSource()) { + $is_exception = true; + } else { + $next_source = $source; + } + } + } + + // If this is an exception, it means the event does NOT occur. We + // skip it and move on. If it's not an exception, it does occur, so + // we record it. + if (!$is_exception) { + + // Only actually include this event in the results if it starts after + // any specified start time. We increment the index regardless, so we + // return results with proper offsets. + if ($next_source['epoch'] >= $min_epoch) { + $results[$index] = $next_source['state']; + } + $index++; + + if ($limit !== null && (count($results) >= $limit)) { + break; + } + } + + $cursor = $next_epoch + 1; + + // If we have an end of the window and we've reached it, we're done. + if ($end_epoch) { + if ($cursor > $end_epoch) { + break; + } + } + } + + return $results; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php new file mode 100644 index 0000000000..214fd82933 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php @@ -0,0 +1,34 @@ +isExceptionSource = $is_exception_source; + return $this; + } + + public function getIsExceptionSource() { + return $this->isExceptionSource; + } + + public function setViewerTimezone($viewer_timezone) { + $this->viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function resetSource() { + return; + } + + abstract public function getNextEvent($cursor); + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php new file mode 100644 index 0000000000..15c5dc231c --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php @@ -0,0 +1,74 @@ +setProxy($origin); + } + + public function getOrigin() { + return $this->getProxy(); + } + + public function setDuration(PhutilCalendarDuration $duration) { + $this->duration = $duration; + return $this; + } + + public function getDuration() { + return $this->duration; + } + + public function newPHPDateTime() { + $datetime = parent::newPHPDateTime(); + $duration = $this->getDuration(); + + if ($duration->getIsNegative()) { + $sign = '-'; + } else { + $sign = '+'; + } + + $map = array( + 'weeks' => $duration->getWeeks(), + 'days' => $duration->getDays(), + 'hours' => $duration->getHours(), + 'minutes' => $duration->getMinutes(), + 'seconds' => $duration->getSeconds(), + ); + + foreach ($map as $unit => $value) { + if (!$value) { + continue; + } + $datetime->modify("{$sign}{$value} {$unit}"); + } + + return $datetime; + } + + public function newAbsoluteDateTime() { + $clone = clone $this; + + if ($clone->getTimezone()) { + $clone->setViewerTimezone(null); + } + + $datetime = $clone->newPHPDateTime(); + + return id(new PhutilCalendarAbsoluteDateTime()) + ->setYear((int)$datetime->format('Y')) + ->setMonth((int)$datetime->format('m')) + ->setDay((int)$datetime->format('d')) + ->setHour((int)$datetime->format('H')) + ->setMinute((int)$datetime->format('i')) + ->setSecond((int)$datetime->format('s')) + ->setIsAllDay($clone->getIsAllDay()) + ->setTimezone($clone->getTimezone()) + ->setViewerTimezone($this->getViewerTimezone()); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRootNode.php b/src/applications/calendar/parser/data/PhutilCalendarRootNode.php new file mode 100644 index 0000000000..de22587f0b --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRootNode.php @@ -0,0 +1,12 @@ +getChildrenOfType(PhutilCalendarDocumentNode::NODETYPE); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarUserNode.php b/src/applications/calendar/parser/data/PhutilCalendarUserNode.php new file mode 100644 index 0000000000..ea81e0d9d8 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarUserNode.php @@ -0,0 +1,40 @@ +name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setURI($uri) { + $this->uri = $uri; + return $this; + } + + public function getURI() { + return $this->uri; + } + + public function setStatus($status) { + $this->status = $status; + return $this; + } + + public function getStatus() { + return $this->status; + } + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php new file mode 100644 index 0000000000..d80aef92fb --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php @@ -0,0 +1,49 @@ +setTimezone('America/Los_Angeles') + ->setViewerTimezone('America/Chicago') + ->setIsAllDay(true); + + $this->assertEqual( + '20161128', + $start->getISO8601()); + + $end = $start + ->newAbsoluteDateTime() + ->setHour(0) + ->setMinute(0) + ->setSecond(0) + ->newRelativeDateTime('P1D') + ->newAbsoluteDateTime(); + + $this->assertEqual( + '20161129', + $end->getISO8601()); + + // This is a date which explicitly has no specified timezone. + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601('20161128', null) + ->setViewerTimezone('UTC'); + + $this->assertEqual( + '20161128', + $start->getISO8601()); + + $end = $start + ->newAbsoluteDateTime() + ->setHour(0) + ->setMinute(0) + ->setSecond(0) + ->newRelativeDateTime('P1D') + ->newAbsoluteDateTime(); + + $this->assertEqual( + '20161129', + $end->getISO8601()); + } + + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php new file mode 100644 index 0000000000..228a921b87 --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php @@ -0,0 +1,1750 @@ +setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_DAILY); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Simple daily event.')); + + + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_HOURLY) + ->setByHour(array(12, 13)); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 5); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T130000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T130000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Hourly event with BYHOUR.')); + + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_YEARLY); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 2); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20170101T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Yearly event.')); + + + // This is an efficiency test for bizarre rules: it defines a secondly + // event which only occurs one a year, and generates 3 instances of it. + // This implementation should be fast enough that this test doesn't take + // a significant amount of time. + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_SECONDLY) + ->setByMonth(array(1)) + ->setByMonthDay(array(1)) + ->setByHour(array(12)) + ->setByMinute(array(0)) + ->setBySecond(array(0)); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20170101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20180101T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Secondly event with many constraints.')); + } + + public function testYearlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19980902', + '19990902', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19990902', + '20010902', + ); + + $tests[] = array( + 'DTSTART' => '20000229', + ); + $expect[] = array( + '20000229', + '20040229', + '20080229', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980102', + '19980302', + '19990102', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19970907', + '19970914', + '19970921', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19971225', + '19980106', + '19981231', + ); + + // Same test as above, just making sure the optional "+" syntax works. + $tests[] = array( + 'BYDAY' => array('+1TU', '-1TH'), + ); + $expect[] = array( + '19971225', + '19980106', + '19981231', + ); + + $tests[] = array( + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19971211', + '19980120', + '19981217', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19980106', + '19980129', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19980115', + '19980120', + '19980312', + ); + + $tests[] = array( + 'BYYEARDAY' => array(1, 100, 200, 365), + 'COUNT' => 4, + ); + $expect[] = array( + '19971231', + '19980101', + '19980410', + '19980719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(-365, -266, -166, -1), + 'COUNT' => 4, + ); + $expect[] = array( + '19971231', + '19980101', + '19980410', + '19980719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(1, 100, 200, 365), + 'BYMONTH' => array(4, 7), + 'COUNT' => 4, + ); + $expect[] = array( + '19980410', + '19980719', + '19990410', + '19990719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(-365, -266, -166, -1), + 'BYMONTH' => array(4, 7), + 'COUNT' => 4, + ); + $expect[] = array( + '19980410', + '19980719', + '19990410', + '19990719', + ); + + $tests[] = array( + 'BYWEEKNO' => array(20), + ); + $expect[] = array( + '19980511', + '19980512', + '19980513', + ); + + $tests[] = array( + 'BYWEEKNO' => array(1), + 'BYDAY' => array('MO'), + ); + $expect[] = array( + '19971229', + '19990104', + '20000103', + ); + + $tests[] = array( + 'BYWEEKNO' => array(52), + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19971228', + '19981227', + '20000102', + ); + + $tests[] = array( + 'BYWEEKNO' => array(-1), + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19971228', + '19990103', + '20000102', + ); + + $tests[] = array( + 'BYWEEKNO' => array(53), + 'BYDAY' => array('MO'), + ); + $expect[] = array( + '19981228', + '20041227', + '20091228', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19980902T060000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 30), + ); + $expect[] = array( + '19970902T001500Z', + '19970902T003000Z', + '19980902T001500Z', + ); + + $tests[] = array( + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T000010Z', + '19970902T000020Z', + '19980902T000010Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 30), + ); + $expect[] = array( + '19970902T061500Z', + '19970902T063000Z', + '19970902T181500Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T060010Z', + '19970902T060020Z', + '19970902T180010Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 30), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T001510Z', + '19970902T001520Z', + '19970902T003010Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 30), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T061510Z', + '19970902T061520Z', + '19970902T063010Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(15), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19971115T180000Z', + '19980215T060000Z', + '19981115T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testMonthlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19971002', + '19971102', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19971102', + '19980102', + ); + + $tests[] = array( + 'INTERVAL' => 18, + ); + $expect[] = array( + '19970902', + '19990302', + '20000902', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980102', + '19980302', + '19990102', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(5, 7), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYDAY' => array('3MO'), + ); + $expect[] = array( + '19970915', + '19971020', + '19971117', + ); + + $tests[] = array( + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19970902', + '19970925', + '19971007', + ); + + $tests[] = array( + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19970911', + '19970916', + '19971016', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19980106', + '19980129', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19980115', + '19980120', + '19980312', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR'), + 'BYSETPOS' => array(-1), + ); + $expect[] = array( + '19970930', + '19971031', + '19971128', + ); + + $tests[] = array( + 'BYDAY' => array('1MO', '1TU', '1WE', '1TH', '1FR', '-1FR'), + 'BYMONTHDAY' => array(1, -1, -2), + ); + $expect[] = array( + '19971001', + '19971031', + '19971201', + ); + + $tests[] = array( + 'BYDAY' => array('1MO', '1TU', '1WE', '1TH', 'FR'), + 'BYMONTHDAY' => array(1, -1, -2), + ); + $expect[] = array( + '19971001', + '19971031', + '19971201', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19971002T060000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(6, 18), + ); + $expect[] = array( + '19970902T000600Z', + '19970902T001800Z', + '19971002T000600Z', + ); + + $tests[] = array( + 'BYSECOND' => array(6, 18), + ); + $expect[] = array( + '19970902T000006Z', + '19970902T000018Z', + '19971002T000006Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970913T180000Z', + '19970917T060000Z', + '19971013T180000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, 3, -3), + ); + $expect[] = array( + '19970913T180000Z', + '19970917T060000Z', + '19971013T180000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(4, -1), + ); + $expect[] = array( + '19970917T180000Z', + '19971017T180000Z', + '19971117T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testWeeklyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19970909', + '19970916', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19970916', + '19970930', + ); + + $tests[] = array( + 'INTERVAL' => 20, + ); + $expect[] = array( + '19970902', + '19980120', + '19980609', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980106', + '19980113', + '19980120', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19970909T060000Z', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T180000Z', + '19970904T060000Z', + '19970909T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testDailyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19970903', + '19970904', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19970904', + '19970906', + ); + + $tests[] = array( + 'INTERVAL' => 92, + ); + $expect[] = array( + '19970902', + '19971203', + '19980305', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980102', + '19980103', + ); + + // This is testing that INTERVAL is respected in the presence of a BYMONTH + // filter which skips some months. + $tests[] = array( + 'BYMONTH' => array(12), + 'INTERVAL' => 17, + ); + $expect[] = array( + '19971213', + '19971230', + '19981205', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 45), + 'BYSETPOS' => array(3, -3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T181500Z', + '19970903T064500Z', + '19970903T181500Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'DAILY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testHourlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902T090000Z', + '19970902T100000Z', + '19970902T110000Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T110000Z', + '19970902T130000Z', + ); + + $tests[] = array( + 'INTERVAL' => 769, + ); + $expect[] = array( + '19970902T090000Z', + '19971004T100000Z', + '19971105T110000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903T000000Z', + '19970903T010000Z', + '19970903T020000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105T000000Z', + '19980105T010000Z', + '19980105T020000Z', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902T090000Z', + '19970902T100000Z', + '19970902T110000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYYEARDAY' => array(1, 100, 200, 365), + ); + $expect[] = array( + '19971231T000000Z', + '19971231T010000Z', + '19971231T020000Z', + '19971231T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYYEARDAY' => array(-365, -266, -166, -1), + ); + $expect[] = array( + '19971231T000000Z', + '19971231T010000Z', + '19971231T020000Z', + '19971231T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYMONTH' => array(4, 7), + 'BYYEARDAY' => array(1, 100, 200, 365), + ); + $expect[] = array( + '19980410T000000Z', + '19980410T010000Z', + '19980410T020000Z', + '19980410T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYMONTH' => array(4, 7), + 'BYYEARDAY' => array(-365, -266, -166, -1), + ); + $expect[] = array( + '19980410T000000Z', + '19980410T010000Z', + '19980410T020000Z', + '19980410T030000Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T180000Z', + '19970903T060000Z', + '19970903T180000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 45), + 'BYSECOND' => array(15, 45), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970902T091545Z', + '19970902T094515Z', + '19970902T101545Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'HOURLY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testMinutelyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array( + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090100Z', + '19970902T090200Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090200Z', + '19970902T090400Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(6, 18), + 'BYSECOND' => array(6, 18), + ); + $expect[] = array( + '19970902T180606Z', + '19970902T180618Z', + '19970902T181806Z', + ); + + $tests[] = array( + 'BYSECOND' => array(15, 30, 45), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970902T090015Z', + '19970902T090045Z', + '19970902T090115Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'MINUTELY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testSecondlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902T090000Z', + '19970902T090001Z', + '19970902T090002Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090002Z', + '19970902T090004Z', + ); + + $tests[] = array( + 'INTERVAL' => 90061, + ); + $expect[] = array( + '19970902T090000Z', + '19970903T100101Z', + '19970904T110202Z', + ); + + $tests[] = array( + 'BYSECOND' => array(0), + 'BYMINUTE' => array(1), + 'DTSTART' => '20100322T120100Z', + ); + $expect[] = array( + '20100322T120100Z', + '20100322T130100Z', + '20100322T140100Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'SECONDLY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testRFC5545RecurrenceRules() { + // These tests are derived from the examples in RFC5545. + $tests = array(); + $expect = array(); + + $tests[] = array( + 'FREQ' => 'DAILY', + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970903T090000Z', + '19970904T090000Z', + '19970905T090000Z', + '19970906T090000Z', + '19970907T090000Z', + '19970908T090000Z', + '19970909T090000Z', + '19970910T090000Z', + '19970911T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'DAILY', + 'INTERVAL' => 2, + 'DTSTART' => '19970902T090000Z', + 'COUNT' => 5, + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970906T090000Z', + '19970908T090000Z', + '19970910T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYMONTH' => array(1), + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR', 'SA', 'SU'), + 'DTSTART' => '19970902T090000Z', + 'COUNT' => 3, + ); + $expect[] = array( + '19980101T090000Z', + '19980102T090000Z', + '19980103T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 3, + 'BYDAY' => array('1FR'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970905T090000Z', + '19971003T090000Z', + '19971107T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'INTERVAL' => 2, + 'COUNT' => 5, + 'BYDAY' => array('1SU', '-1SU'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970907T090000Z', + '19970928T090000Z', + '19971102T090000Z', + '19971130T090000Z', + '19980104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'BYDAY' => array('-2MO'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970922T090000Z', + '19971020T090000Z', + '19971117T090000Z', + '19971222T090000Z', + '19980119T090000Z', + '19980216T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'BYMONTHDAY' => array(-3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970928T090000Z', + '19971029T090000Z', + '19971128T090000Z', + '19971229T090000Z', + '19980129T090000Z', + '19980226T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 5, + 'BYMONTHDAY' => array(2, 15), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970915T090000Z', + '19971002T090000Z', + '19971015T090000Z', + '19971102T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 5, + 'BYMONTHDAY' => array(-1, 1), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970930T090000Z', + '19971001T090000Z', + '19971031T090000Z', + '19971101T090000Z', + '19971130T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 7, + 'INTERVAL' => 18, + 'BYMONTHDAY' => array(10, 11, 12, 13, 14, 15), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970910T090000Z', + '19970911T090000Z', + '19970912T090000Z', + '19970913T090000Z', + '19970914T090000Z', + '19970915T090000Z', + '19990310T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'INTERVAL' => 2, + 'BYDAY' => array('TU'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970909T090000Z', + '19970916T090000Z', + '19970923T090000Z', + '19970930T090000Z', + '19971104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 10, + 'BYMONTH' => array(6, 7), + 'DTSTART' => '19970610T090000Z', + ); + $expect[] = array( + '19970610T090000Z', + '19970710T090000Z', + '19980610T090000Z', + '19980710T090000Z', + '19990610T090000Z', + '19990710T090000Z', + '20000610T090000Z', + '20000710T090000Z', + '20010610T090000Z', + '20010710T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 4, + 'INTERVAL' => 3, + 'BYYEARDAY' => array(1, 100, 200), + 'DTSTART' => '19970101T090000Z', + ); + $expect[] = array( + '19970101T090000Z', + '19970410T090000Z', + '19970719T090000Z', + '20000101T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'BYDAY' => array('20MO'), + 'DTSTART' => '19970519T090000Z', + ); + $expect[] = array( + '19970519T090000Z', + '19980518T090000Z', + '19990517T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'BYWEEKNO' => array(20), + 'BYDAY' => array('MO'), + 'DTSTART' => '19970512T090000Z', + ); + $expect[] = array( + '19970512T090000Z', + '19980511T090000Z', + '19990517T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('TH'), + 'BYMONTH' => array(3), + 'DTSTART' => '19970313T090000Z', + 'COUNT' => 5, + ); + $expect[] = array( + '19970313T090000Z', + '19970320T090000Z', + '19970327T090000Z', + '19980305T090000Z', + '19980312T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('TH'), + 'BYMONTH' => array(6, 7, 8), + 'DTSTART' => '19970101T090000Z', + 'COUNT' => 15, + ); + $expect[] = array( + '19970605T090000Z', + '19970612T090000Z', + '19970619T090000Z', + '19970626T090000Z', + '19970703T090000Z', + '19970710T090000Z', + '19970717T090000Z', + '19970724T090000Z', + '19970731T090000Z', + '19970807T090000Z', + '19970814T090000Z', + '19970821T090000Z', + '19970828T090000Z', + '19980604T090000Z', + '19980611T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('FR'), + 'BYMONTHDAY' => array(13), + 'COUNT' => 4, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19980213T090000Z', + '19980313T090000Z', + '19981113T090000Z', + '19990813T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('SA'), + 'BYMONTHDAY' => array(7, 8, 9, 10, 11, 12, 13), + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970913T090000Z', + '19971011T090000Z', + '19971108T090000Z', + '19971213T090000Z', + '19980110T090000Z', + '19980207T090000Z', + '19980307T090000Z', + '19980411T090000Z', + '19980509T090000Z', + '19980613T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'INTERVAL' => 4, + 'BYMONTH' => array(11), + 'BYDAY' => array('TU'), + 'BYMONTHDAY' => array(2, 3, 4, 5, 6, 7, 8), + 'COUNT' => 6, + 'DTSTART' => '19961105T090000Z', + ); + $expect[] = array( + '19961105T090000Z', + '20001107T090000Z', + '20041102T090000Z', + '20081104T090000Z', + '20121106T090000Z', + '20161108T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('TU', 'WE', 'TH'), + 'BYSETPOS' => array(3), + 'COUNT' => 3, + 'DTSTART' => '19970904T090000Z', + ); + $expect[] = array( + '19970904T090000Z', + '19971007T090000Z', + '19971106T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR'), + 'BYSETPOS' => array(-2), + 'COUNT' => 3, + 'DTSTART' => '19970929T090000Z', + ); + $expect[] = array( + '19970929T090000Z', + '19971030T090000Z', + '19971127T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'HOURLY', + 'INTERVAL' => 3, + 'DTSTART' => '19970929T090000Z', + 'COUNT' => 3, + ); + $expect[] = array( + '19970929T090000Z', + '19970929T120000Z', + '19970929T150000Z', + ); + + $tests[] = array( + 'FREQ' => 'MINUTELY', + 'INTERVAL' => 15, + 'COUNT' => 6, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970902T091500Z', + '19970902T093000Z', + '19970902T094500Z', + '19970902T100000Z', + '19970902T101500Z', + ); + + $tests[] = array( + 'FREQ' => 'MINUTELY', + 'INTERVAL' => 90, + 'COUNT' => 4, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970902T103000Z', + '19970902T120000Z', + '19970902T133000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970909T090000Z', + '19970916T090000Z', + '19970923T090000Z', + '19970930T090000Z', + '19971007T090000Z', + '19971014T090000Z', + '19971021T090000Z', + '19971028T090000Z', + '19971104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 6, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970916T090000Z', + '19970930T090000Z', + '19971014T090000Z', + '19971028T090000Z', + '19971111T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 10, + 'WKST' => 'SU', + 'BYDAY' => array('TU', 'TH'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970909T090000Z', + '19970911T090000Z', + '19970916T090000Z', + '19970918T090000Z', + '19970923T090000Z', + '19970925T090000Z', + '19970930T090000Z', + '19971002T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 8, + 'WKST' => 'SU', + 'BYDAY' => array('TU', 'TH'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970916T090000Z', + '19970918T090000Z', + '19970930T090000Z', + '19971002T090000Z', + '19971014T090000Z', + '19971016T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 4, + 'BYDAY' => array('TU', 'SU'), + 'WKST' => 'MO', + 'DTSTART' => '19970805T090000Z', + ); + $expect[] = array( + '19970805T090000Z', + '19970810T090000Z', + '19970819T090000Z', + '19970824T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 4, + 'BYDAY' => array('TU', 'SU'), + 'WKST' => 'SU', + 'DTSTART' => '19970805T090000Z', + ); + $expect[] = array( + '19970805T090000Z', + '19970817T090000Z', + '19970819T090000Z', + '19970831T090000Z', + ); + + + $this->assertRules(array(), $tests, $expect); + } + + + private function assertRules(array $defaults, array $tests, array $expect) { + foreach ($tests as $key => $test) { + $options = $test + $defaults; + + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601( + $options['DTSTART']); + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency($options['FREQ']); + + $interval = idx($options, 'INTERVAL'); + if ($interval) { + $rrule->setInterval($interval); + } + + $by_day = idx($options, 'BYDAY'); + if ($by_day) { + $rrule->setByDay($by_day); + } + + $by_month = idx($options, 'BYMONTH'); + if ($by_month) { + $rrule->setByMonth($by_month); + } + + $by_monthday = idx($options, 'BYMONTHDAY'); + if ($by_monthday) { + $rrule->setByMonthDay($by_monthday); + } + + $by_yearday = idx($options, 'BYYEARDAY'); + if ($by_yearday) { + $rrule->setByYearDay($by_yearday); + } + + $by_weekno = idx($options, 'BYWEEKNO'); + if ($by_weekno) { + $rrule->setByWeekNumber($by_weekno); + } + + $by_hour = idx($options, 'BYHOUR'); + if ($by_hour) { + $rrule->setByHour($by_hour); + } + + $by_minute = idx($options, 'BYMINUTE'); + if ($by_minute) { + $rrule->setByMinute($by_minute); + } + + $by_second = idx($options, 'BYSECOND'); + if ($by_second) { + $rrule->setBySecond($by_second); + } + + $by_setpos = idx($options, 'BYSETPOS'); + if ($by_setpos) { + $rrule->setBySetPosition($by_setpos); + } + + $week_start = idx($options, 'WKST'); + if ($week_start) { + $rrule->setWeekStart($week_start); + } + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, $options['COUNT']); + + $this->assertEqual( + $expect[$key], + mpull($result, 'getISO8601')); + } + } + + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php new file mode 100644 index 0000000000..989975ee59 --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php @@ -0,0 +1,196 @@ +getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + array(), + $result, + pht('Set with no sources.')); + + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource(new PhutilCalendarRecurrenceList()); + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + array(), + $result, + pht('Set with empty list source.')); + + + $list = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source = id(new PhutilCalendarRecurrenceList()) + ->setDates($list); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Simple date list.')); + + $list_a = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + ); + + $list_b = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source_a = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_a); + + $source_b = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_b); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source_a) + ->addSource($source_b); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple date lists.')); + + $list_a = array( + // This is Jan 1, 3, 5, 7, 8 and 10, but listed out-of-order. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160110T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160107T120000Z'), + ); + + $list_b = array( + // This is Jan 2, 4, 5, 8, but listed out of order. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + ); + + $list_c = array( + // We're going to use this as an exception list. + + // This is Jan 7 (listed in one other source), 8 (listed in two) + // and 9 (listed in none). + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160107T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160109T120000Z'), + ); + + $expect = array( + // From source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + // From source B. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + // From source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + // From source B. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + // From source A and B. Should appear only once. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + // The 6th appears in no source. + // The 7th, 8th and 9th are excluded. + // The 10th is from source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160110T120000Z'), + ); + + $list_a = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_a); + + $list_b = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_b); + + $list_c = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_c) + ->setIsExceptionSource(true); + + $date_set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($list_b) + ->addSource($list_c) + ->addSource($list_a); + + $date_set->setViewerTimezone('UTC'); + + $result = $date_set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Set of all results in multiple lists with exclusions.')); + + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + ); + $result = $date_set->getEventsBetween(null, null, 1); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple lists, one result.')); + + $expect = array( + 2 => PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + 3 => PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + ); + $result = $date_set->getEventsBetween( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z')); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple lists, time window.')); + } + + public function testCalendarRecurrenceOffsets() { + $list = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source = id(new PhutilCalendarRecurrenceList()) + ->setDates($list); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source); + + $t1 = PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120001Z'); + $t2 = PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'); + + $expect = array( + 2 => $t2, + ); + + $result = $set->getEventsBetween($t1, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Correct event indexes with start date.')); + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSParser.php b/src/applications/calendar/parser/ics/PhutilICSParser.php new file mode 100644 index 0000000000..015bea601c --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSParser.php @@ -0,0 +1,919 @@ +stack = array(); + $this->node = null; + $this->cursor = null; + $this->warnings = array(); + + $lines = $this->unfoldICSLines($data); + $this->lines = $lines; + + $root = $this->newICSNode(''); + $this->stack[] = $root; + $this->node = $root; + + foreach ($lines as $key => $line) { + $this->cursor = $key; + $matches = null; + if (preg_match('(^BEGIN:(.*)\z)', $line, $matches)) { + $this->beginParsingNode($matches[1]); + } else if (preg_match('(^END:(.*)\z)', $line, $matches)) { + $this->endParsingNode($matches[1]); + } else { + if (count($this->stack) < 2) { + $this->raiseParseFailure( + self::PARSE_ROOT_PROPERTY, + pht( + 'Found unexpected property at ICS document root.')); + } + $this->parseICSProperty($line); + } + } + + if (count($this->stack) > 1) { + $this->raiseParseFailure( + self::PARSE_MISSING_END, + pht( + 'Expected all "BEGIN:" sections in ICS document to have '. + 'corresponding "END:" sections.')); + } + + $this->node = null; + $this->lines = null; + $this->cursor = null; + + return $root; + } + + private function getNode() { + return $this->node; + } + + private function unfoldICSLines($data) { + $lines = phutil_split_lines($data, $retain_endings = false); + $this->lines = $lines; + + // ICS files are wrapped at 75 characters, with overlong lines continued + // on the following line with an initial space or tab. Unwrap all of the + // lines in the file. + + // This unwrapping is specifically byte-oriented, not character oriented, + // and RFC5545 anticipates that simple implementations may even split UTF8 + // characters in the middle. + + $last = null; + foreach ($lines as $idx => $line) { + $this->cursor = $idx; + if (!preg_match('/^[ \t]/', $line)) { + $last = $idx; + continue; + } + + if ($last === null) { + $this->raiseParseFailure( + self::PARSE_INITIAL_UNFOLD, + pht( + 'First line of ICS file begins with a space or tab, but this '. + 'marks a line which should be unfolded.')); + } + + $lines[$last] = $lines[$last].substr($line, 1); + unset($lines[$idx]); + } + + return $lines; + } + + private function beginParsingNode($type) { + $node = $this->getNode(); + $new_node = $this->newICSNode($type); + + if ($node instanceof PhutilCalendarContainerNode) { + $node->appendChild($new_node); + } else { + $this->raiseParseFailure( + self::PARSE_UNEXPECTED_CHILD, + pht( + 'Found unexpected node "%s" inside node "%s".', + $new_node->getAttribute('ics.type'), + $node->getAttribute('ics.type'))); + } + + $this->stack[] = $new_node; + $this->node = $new_node; + + return $this; + } + + private function newICSNode($type) { + switch ($type) { + case '': + $node = new PhutilCalendarRootNode(); + break; + case 'VCALENDAR': + $node = new PhutilCalendarDocumentNode(); + break; + case 'VEVENT': + $node = new PhutilCalendarEventNode(); + break; + default: + $node = new PhutilCalendarRawNode(); + break; + } + + $node->setAttribute('ics.type', $type); + + return $node; + } + + private function endParsingNode($type) { + $node = $this->getNode(); + if ($node instanceof PhutilCalendarRootNode) { + $this->raiseParseFailure( + self::PARSE_EXTRA_END, + pht( + 'Found unexpected "END" without a "BEGIN".')); + } + + $old_type = $node->getAttribute('ics.type'); + if ($old_type != $type) { + $this->raiseParseFailure( + self::PARSE_MISMATCHED_SECTIONS, + pht( + 'Found mismatched "BEGIN" ("%s") and "END" ("%s") sections.', + $old_type, + $type)); + } + + array_pop($this->stack); + $this->node = last($this->stack); + + return $this; + } + + private function parseICSProperty($line) { + $matches = null; + + // Properties begin with an alphanumeric name with no escaping, followed + // by either a ";" (to begin a list of parameters) or a ":" (to begin + // the actual field body). + + $ok = preg_match('(^([A-Za-z0-9-]+)([;:])(.*)\z)', $line, $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_PROPERTY, + pht( + 'Found malformed property in ICS document.')); + } + + $name = $matches[1]; + $body = $matches[3]; + $has_parameters = ($matches[2] == ';'); + + $parameters = array(); + if ($has_parameters) { + // Parameters are a sensible name, a literal "=", a pile of magic, + // and then maybe a comma and another parameter. + + while (true) { + // We're going to get the first couple of parts first. + $ok = preg_match('(^([^=]+)=)', $body, $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_PARAMETER_NAME, + pht( + 'Found malformed property in ICS document: %s', + $body)); + } + + $param_name = $matches[1]; + $body = substr($body, strlen($matches[0])); + + // Now we're going to match zero or more values. + $param_values = array(); + while (true) { + // The value can either be a double-quoted string or an unquoted + // string, with some characters forbidden. + if (strlen($body) && $body[0] == '"') { + $is_quoted = true; + $ok = preg_match( + '(^"([^\x00-\x08\x10-\x19"]*)")', + $body, + $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_DOUBLE_QUOTE, + pht( + 'Found malformed double-quoted string in ICS document '. + 'parameter value.')); + } + } else { + $is_quoted = false; + + // It's impossible for this not to match since it can match + // nothing, and it's valid for it to match nothing. + preg_match('(^([^\x00-\x08\x10-\x19";:,]*))', $body, $matches); + } + + // NOTE: RFC5545 says "Property parameter values that are not in + // quoted-strings are case-insensitive." -- that is, the quoted and + // unquoted representations are not equivalent. Thus, preserve the + // original formatting in case we ever need to respect this. + + $param_values[] = array( + 'value' => $this->unescapeParameterValue($matches[1]), + 'quoted' => $is_quoted, + ); + + $body = substr($body, strlen($matches[0])); + if (!strlen($body)) { + $this->raiseParseFailure( + self::PARSE_MISSING_VALUE, + pht( + 'Expected ":" after parameters in ICS document property.')); + } + + // If we have a comma now, we're going to read another value. Strip + // it off and keep going. + if ($body[0] == ',') { + $body = substr($body, 1); + continue; + } + + // If we have a semicolon, we're going to read another parameter. + if ($body[0] == ';') { + break; + } + + // If we have a colon, this is the last value and also the last + // property. Break, then handle the colon below. + if ($body[0] == ':') { + break; + } + + $short_body = id(new PhutilUTF8StringTruncator()) + ->setMaximumGlyphs(32) + ->truncateString($body); + + // We aren't expecting anything else. + $this->raiseParseFailure( + self::PARSE_UNEXPECTED_TEXT, + pht( + 'Found unexpected text ("%s") after reading parameter value.', + $short_body)); + } + + $parameters[] = array( + 'name' => $param_name, + 'values' => $param_values, + ); + + if ($body[0] == ';') { + $body = substr($body, 1); + continue; + } + + if ($body[0] == ':') { + $body = substr($body, 1); + break; + } + } + } + + $value = $this->unescapeFieldValue($name, $parameters, $body); + + $node = $this->getNode(); + + + $raw = $node->getAttribute('ics.properties', array()); + $raw[] = array( + 'name' => $name, + 'parameters' => $parameters, + 'value' => $value, + ); + $node->setAttribute('ics.properties', $raw); + + switch ($node->getAttribute('ics.type')) { + case 'VEVENT': + $this->didParseEventProperty($node, $name, $parameters, $value); + break; + } + } + + private function unescapeParameterValue($data) { + // The parameter grammar is adjusted by RFC6868 to permit escaping with + // carets. Remove that escaping. + + // This escaping is a bit weird because it's trying to be backwards + // compatible and the original spec didn't think about this and didn't + // provide much room to fix things. + + $out = ''; + $esc = false; + foreach (phutil_utf8v($data) as $c) { + if (!$esc) { + if ($c != '^') { + $out .= $c; + } else { + $esc = true; + } + } else { + switch ($c) { + case 'n': + $out .= "\n"; + break; + case '^': + $out .= '^'; + break; + case "'": + // NOTE: This is " " being decoded into a + // double quote! + $out .= '"'; + break; + default: + // NOTE: The caret is NOT an escape for any other characters. + // This is a "MUST" requirement of RFC6868. + $out .= '^'.$c; + break; + } + } + } + + // NOTE: Because caret on its own just means "caret" for backward + // compatibility, we don't warn if we're still in escaped mode once we + // reach the end of the string. + + return $out; + } + + private function unescapeFieldValue($name, array $parameters, $data) { + // NOTE: The encoding of the field value data is dependent on the field + // name (which defines a default encoding) and the parameters (which may + // include "VALUE", specifying a type of the data. + + $default_types = array( + 'CALSCALE' => 'TEXT', + 'METHOD' => 'TEXT', + 'PRODID' => 'TEXT', + 'VERSION' => 'TEXT', + + 'ATTACH' => 'URI', + 'CATEGORIES' => 'TEXT', + 'CLASS' => 'TEXT', + 'COMMENT' => 'TEXT', + 'DESCRIPTION' => 'TEXT', + + // TODO: The spec appears to contradict itself: it says that the value + // type is FLOAT, but it also says that this property value is actually + // two semicolon-separated values, which is not what FLOAT is defined as. + 'GEO' => 'TEXT', + + 'LOCATION' => 'TEXT', + 'PERCENT-COMPLETE' => 'INTEGER', + 'PRIORITY' => 'INTEGER', + 'RESOURCES' => 'TEXT', + 'STATUS' => 'TEXT', + 'SUMMARY' => 'TEXT', + + 'COMPLETED' => 'DATE-TIME', + 'DTEND' => 'DATE-TIME', + 'DUE' => 'DATE-TIME', + 'DTSTART' => 'DATE-TIME', + 'DURATION' => 'DURATION', + 'FREEBUSY' => 'PERIOD', + 'TRANSP' => 'TEXT', + + 'TZID' => 'TEXT', + 'TZNAME' => 'TEXT', + 'TZOFFSETFROM' => 'UTC-OFFSET', + 'TZOFFSETTO' => 'UTC-OFFSET', + 'TZURL' => 'URI', + + 'ATTENDEE' => 'CAL-ADDRESS', + 'CONTACT' => 'TEXT', + 'ORGANIZER' => 'CAL-ADDRESS', + 'RECURRENCE-ID' => 'DATE-TIME', + 'RELATED-TO' => 'TEXT', + 'URL' => 'URI', + 'UID' => 'TEXT', + 'EXDATE' => 'DATE-TIME', + 'RDATE' => 'DATE-TIME', + 'RRULE' => 'RECUR', + + 'ACTION' => 'TEXT', + 'REPEAT' => 'INTEGER', + 'TRIGGER' => 'DURATION', + + 'CREATED' => 'DATE-TIME', + 'DTSTAMP' => 'DATE-TIME', + 'LAST-MODIFIED' => 'DATE-TIME', + 'SEQUENCE' => 'INTEGER', + + 'REQUEST-STATUS' => 'TEXT', + ); + + $value_type = idx($default_types, $name, 'TEXT'); + + foreach ($parameters as $parameter) { + if ($parameter['name'] == 'VALUE') { + $value_type = idx(head($parameter['values']), 'value'); + } + } + + switch ($value_type) { + case 'BINARY': + $result = base64_decode($data, true); + if ($result === false) { + $this->raiseParseFailure( + self::PARSE_BAD_BASE64, + pht( + 'Unable to decode base64 data: %s', + $data)); + } + break; + case 'BOOLEAN': + $map = array( + 'true' => true, + 'false' => false, + ); + $result = phutil_utf8_strtolower($data); + if (!isset($map[$result])) { + $this->raiseParseFailure( + self::PARSE_BAD_BOOLEAN, + pht( + 'Unexpected BOOLEAN value "%s".', + $data)); + } + $result = $map[$result]; + break; + case 'CAL-ADDRESS': + $result = $data; + break; + case 'DATE': + // This is a comma-separated list of "YYYYMMDD" values. + $result = explode(',', $data); + break; + case 'DATE-TIME': + if (!strlen($data)) { + $result = array(); + } else { + $result = explode(',', $data); + } + break; + case 'DURATION': + if (!strlen($data)) { + $result = array(); + } else { + $result = explode(',', $data); + } + break; + case 'FLOAT': + $result = explode(',', $data); + foreach ($result as $k => $v) { + $result[$k] = (float)$v; + } + break; + case 'INTEGER': + $result = explode(',', $data); + foreach ($result as $k => $v) { + $result[$k] = (int)$v; + } + break; + case 'PERIOD': + $result = explode(',', $data); + break; + case 'RECUR': + $result = $data; + break; + case 'TEXT': + $result = $this->unescapeTextValue($data); + break; + case 'TIME': + $result = explode(',', $data); + break; + case 'URI': + $result = $data; + break; + case 'UTC-OFFSET': + $result = $data; + break; + default: + // RFC5545 says we MUST preserve the data for any types we don't + // recognize. + $result = $data; + break; + } + + return array( + 'type' => $value_type, + 'value' => $result, + 'raw' => $data, + ); + } + + private function unescapeTextValue($data) { + $result = array(); + + $buf = ''; + $esc = false; + foreach (phutil_utf8v($data) as $c) { + if (!$esc) { + if ($c == '\\') { + $esc = true; + } else if ($c == ',') { + $result[] = $buf; + $buf = ''; + } else { + $buf .= $c; + } + } else { + switch ($c) { + case 'n': + case 'N': + $buf .= "\n"; + break; + default: + $buf .= $c; + break; + } + $esc = false; + } + } + + if ($esc) { + $this->raiseParseFailure( + self::PARSE_UNESCAPED_BACKSLASH, + pht( + 'ICS document contains TEXT value ending with unescaped '. + 'backslash.')); + } + + $result[] = $buf; + + return $result; + } + + private function raiseParseFailure($code, $message) { + if ($this->lines && isset($this->lines[$this->cursor])) { + $message = pht( + "ICS Parse Error near line %s:\n\n>>> %s\n\n%s", + $this->cursor + 1, + $this->lines[$this->cursor], + $message); + } else { + $message = pht( + 'ICS Parse Error: %s', + $message); + } + + throw id(new PhutilICSParserException($message)) + ->setParserFailureCode($code); + } + + private function raiseWarning($code, $message) { + $this->warnings[] = array( + 'code' => $code, + 'line' => $this->cursor, + 'text' => $this->lines[$this->cursor], + 'message' => $message, + ); + + return $this; + } + + public function getWarnings() { + return $this->warnings; + } + + private function didParseEventProperty( + PhutilCalendarEventNode $node, + $name, + array $parameters, + array $value) { + + switch ($name) { + case 'UID': + $text = $this->newTextFromProperty($parameters, $value); + $node->setUID($text); + break; + case 'CREATED': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setCreatedDateTime($datetime); + break; + case 'DTSTAMP': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setModifiedDateTime($datetime); + break; + case 'SUMMARY': + $text = $this->newTextFromProperty($parameters, $value); + $node->setName($text); + break; + case 'DESCRIPTION': + $text = $this->newTextFromProperty($parameters, $value); + $node->setDescription($text); + break; + case 'DTSTART': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setStartDateTime($datetime); + break; + case 'DTEND': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setEndDateTime($datetime); + break; + case 'DURATION': + $duration = $this->newDurationFromProperty($parameters, $value); + $node->setDuration($duration); + break; + case 'RRULE': + $rrule = $this->newRecurrenceRuleFromProperty($parameters, $value); + $node->setRecurrenceRule($rrule); + break; + case 'RECURRENCE-ID': + $text = $this->newTextFromProperty($parameters, $value); + $node->setRecurrenceID($text); + break; + case 'ATTENDEE': + $attendee = $this->newAttendeeFromProperty($parameters, $value); + $node->addAttendee($attendee); + break; + } + + } + + private function newTextFromProperty(array $parameters, array $value) { + $value = $value['value']; + return implode("\n\n", $value); + } + + private function newAttendeeFromProperty(array $parameters, array $value) { + $uri = $value['value']; + + switch (idx($parameters, 'PARTSTAT')) { + case 'ACCEPTED': + $status = PhutilCalendarUserNode::STATUS_ACCEPTED; + break; + case 'DECLINED': + $status = PhutilCalendarUserNode::STATUS_DECLINED; + break; + case 'NEEDS-ACTION': + default: + $status = PhutilCalendarUserNode::STATUS_INVITED; + break; + } + + $name = $this->getScalarParameterValue($parameters, 'CN'); + + return id(new PhutilCalendarUserNode()) + ->setURI($uri) + ->setName($name) + ->setStatus($status); + } + + private function newDateTimeFromProperty(array $parameters, array $value) { + $value = $value['value']; + + if (!$value) { + $this->raiseParseFailure( + self::PARSE_EMPTY_DATETIME, + pht( + 'Expected DATE-TIME to have exactly one value, found none.')); + + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MANY_DATETIME, + pht( + 'Expected DATE-TIME to have exactly one value, found more than '. + 'one.')); + } + + $value = head($value); + $tzid = $this->getScalarParameterValue($parameters, 'TZID'); + + if (preg_match('/Z\z/', $value)) { + if ($tzid) { + $this->raiseWarning( + self::WARN_TZID_UTC, + pht( + 'DATE-TIME "%s" uses "Z" to specify UTC, but also has a TZID '. + 'parameter with value "%s". This violates RFC5545. The TZID '. + 'will be ignored, and the value will be interpreted as UTC.', + $value, + $tzid)); + } + $tzid = 'UTC'; + } else if ($tzid !== null) { + $tzid = $this->guessTimezone($tzid); + } + + try { + $datetime = PhutilCalendarAbsoluteDateTime::newFromISO8601( + $value, + $tzid); + } catch (Exception $ex) { + $this->raiseParseFailure( + self::PARSE_BAD_DATETIME, + pht( + 'Error parsing DATE-TIME: %s', + $ex->getMessage())); + } + + return $datetime; + } + + private function newDurationFromProperty(array $parameters, array $value) { + $value = $value['value']; + + if (!$value) { + $this->raiseParseFailure( + self::PARSE_EMPTY_DURATION, + pht( + 'Expected DURATION to have exactly one value, found none.')); + + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MANY_DURATION, + pht( + 'Expected DURATION to have exactly one value, found more than '. + 'one.')); + } + + $value = head($value); + + try { + $duration = PhutilCalendarDuration::newFromISO8601($value); + } catch (Exception $ex) { + $this->raiseParseFailure( + self::PARSE_BAD_DURATION, + pht( + 'Invalid DURATION: %s', + $ex->getMessage())); + } + + return $duration; + } + + private function newRecurrenceRuleFromProperty(array $parameters, $value) { + return PhutilCalendarRecurrenceRule::newFromRRULE($value['value']); + } + + private function getScalarParameterValue( + array $parameters, + $name, + $default = null) { + + $match = null; + foreach ($parameters as $parameter) { + if ($parameter['name'] == $name) { + $match = $parameter; + } + } + + if ($match === null) { + return $default; + } + + $value = $match['values']; + if (!$value) { + // Parameter is specified, but with no value, like "KEY=". Just return + // the default, as though the parameter was not specified. + return $default; + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MULTIPLE_PARAMETERS, + pht( + 'Expected parameter "%s" to have at most one value, but found '. + 'more than one.', + $name)); + } + + return idx(head($value), 'value'); + } + + private function guessTimezone($tzid) { + $map = DateTimeZone::listIdentifiers(); + $map = array_fuse($map); + if (isset($map[$tzid])) { + // This is a real timezone we recognize, so just use it as provided. + return $tzid; + } + + // These are alternate names for timezones. + static $aliases; + + if ($aliases === null) { + $aliases = array( + 'Etc/GMT' => 'UTC', + ); + + // Load the map of Windows timezones. + $root_path = dirname(phutil_get_library_root('phutil')); + $windows_path = $root_path.'/resources/timezones/windows_timezones.json'; + $windows_data = Filesystem::readFile($windows_path); + $windows_zones = phutil_json_decode($windows_data); + + $aliases = $aliases + $windows_zones; + } + + if (isset($aliases[$tzid])) { + return $aliases[$tzid]; + } + + // Look for something that looks like "UTC+3" or "GMT -05.00". If we find + // anything, pick a timezone with that offset. + $offset_pattern = + '/'. + '(?:UTC|GMT)'. + '\s*'. + '(?P[+-])'. + '\s*'. + '(?P\d+)'. + '(?:'. + '[:.](?P\d+)'. + ')?'. + '/i'; + + $matches = null; + if (preg_match($offset_pattern, $tzid, $matches)) { + $hours = (int)$matches['h']; + $minutes = (int)idx($matches, 'm'); + $offset = ($hours * 60 * 60) + ($minutes * 60); + + if (idx($matches, 'sign') == '-') { + $offset = -$offset; + } + + // NOTE: We could possibly do better than this, by using the event start + // time to guess a timezone. However, that won't work for recurring + // events and would require us to do this work after finishing initial + // parsing. Since these unusual offset-based timezones appear to be rare, + // the benefit may not be worth the complexity. + $now = new DateTime('@'.time()); + + foreach ($map as $identifier) { + $zone = new DateTimeZone($identifier); + if ($zone->getOffset($now) == $offset) { + $this->raiseWarning( + self::WARN_TZID_GUESS, + pht( + 'TZID "%s" is unknown, guessing "%s" based on pattern "%s".', + $tzid, + $identifier, + $matches[0])); + return $identifier; + } + } + } + + $this->raiseWarning( + self::WARN_TZID_IGNORED, + pht( + 'TZID "%s" is unknown, using UTC instead.', + $tzid)); + + return 'UTC'; + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSParserException.php b/src/applications/calendar/parser/ics/PhutilICSParserException.php new file mode 100644 index 0000000000..09563ffaef --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSParserException.php @@ -0,0 +1,16 @@ +parserFailureCode = $code; + return $this; + } + + public function getParserFailureCode() { + return $this->parserFailureCode; + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSWriter.php b/src/applications/calendar/parser/ics/PhutilICSWriter.php new file mode 100644 index 0000000000..c5baa791de --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSWriter.php @@ -0,0 +1,387 @@ +getChildren() as $child) { + $out[] = $this->writeNode($child); + } + + return implode('', $out); + } + + private function writeNode(PhutilCalendarNode $node) { + if (!$this->getICSNodeType($node)) { + return null; + } + + $out = array(); + + $out[] = $this->writeBeginNode($node); + $out[] = $this->writeNodeProperties($node); + + if ($node instanceof PhutilCalendarContainerNode) { + foreach ($node->getChildren() as $child) { + $out[] = $this->writeNode($child); + } + } + + $out[] = $this->writeEndNode($node); + + return implode('', $out); + } + + private function writeBeginNode(PhutilCalendarNode $node) { + $type = $this->getICSNodeType($node); + return $this->wrapICSLine("BEGIN:{$type}"); + } + + private function writeEndNode(PhutilCalendarNode $node) { + $type = $this->getICSNodeType($node); + return $this->wrapICSLine("END:{$type}"); + } + + private function writeNodeProperties(PhutilCalendarNode $node) { + $properties = $this->getNodeProperties($node); + + $out = array(); + foreach ($properties as $property) { + $propname = $property['name']; + $propvalue = $property['value']; + + $propline = array(); + $propline[] = $propname; + + foreach ($property['parameters'] as $parameter) { + $paramname = $parameter['name']; + $paramvalue = $parameter['value']; + $propline[] = ";{$paramname}={$paramvalue}"; + } + + $propline[] = ":{$propvalue}"; + $propline = implode('', $propline); + + $out[] = $this->wrapICSLine($propline); + } + + return implode('', $out); + } + + private function getICSNodeType(PhutilCalendarNode $node) { + switch ($node->getNodeType()) { + case PhutilCalendarDocumentNode::NODETYPE: + return 'VCALENDAR'; + case PhutilCalendarEventNode::NODETYPE: + return 'VEVENT'; + default: + return null; + } + } + + private function wrapICSLine($line) { + $out = array(); + $buf = ''; + + // NOTE: The line may contain sequences of combining characters which are + // more than 80 bytes in length. If it does, we'll split them in the + // middle of the sequence. This is okay and generally anticipated by + // RFC5545, which even allows implementations to split multibyte + // characters. The sequence will be stitched back together properly by + // whatever is parsing things. + + foreach (phutil_utf8v($line) as $character) { + // If adding this character would bring the line over 75 bytes, start + // a new line. + if (strlen($buf) + strlen($character) > 75) { + $out[] = $buf."\r\n"; + $buf = ' '; + } + + $buf .= $character; + } + + $out[] = $buf."\r\n"; + + return implode('', $out); + } + + private function getNodeProperties(PhutilCalendarNode $node) { + switch ($node->getNodeType()) { + case PhutilCalendarDocumentNode::NODETYPE: + return $this->getDocumentNodeProperties($node); + case PhutilCalendarEventNode::NODETYPE: + return $this->getEventNodeProperties($node); + default: + return array(); + } + } + + private function getDocumentNodeProperties( + PhutilCalendarDocumentNode $event) { + $properties = array(); + + $properties[] = $this->newTextProperty( + 'VERSION', + '2.0'); + + $properties[] = $this->newTextProperty( + 'PRODID', + '-//Phacility//Phabricator//EN'); + + return $properties; + } + + private function getEventNodeProperties(PhutilCalendarEventNode $event) { + $properties = array(); + + $uid = $event->getUID(); + if (!strlen($uid)) { + throw new Exception( + pht( + 'Unable to write ICS document: event has no UID, but each event '. + 'MUST have a UID.')); + } + $properties[] = $this->newTextProperty( + 'UID', + $uid); + + $created = $event->getCreatedDateTime(); + if ($created) { + $properties[] = $this->newDateTimeProperty( + 'CREATED', + $event->getCreatedDateTime()); + } + + $dtstamp = $event->getModifiedDateTime(); + if (!$dtstamp) { + throw new Exception( + pht( + 'Unable to write ICS document: event has no modified time, but '. + 'each event MUST have a modified time.')); + } + $properties[] = $this->newDateTimeProperty( + 'DTSTAMP', + $dtstamp); + + $dtstart = $event->getStartDateTime(); + if ($dtstart) { + $properties[] = $this->newDateTimeProperty( + 'DTSTART', + $dtstart); + } + + $dtend = $event->getEndDateTime(); + if ($dtend) { + $properties[] = $this->newDateTimeProperty( + 'DTEND', + $event->getEndDateTime()); + } + + $name = $event->getName(); + if (strlen($name)) { + $properties[] = $this->newTextProperty( + 'SUMMARY', + $name); + } + + $description = $event->getDescription(); + if (strlen($description)) { + $properties[] = $this->newTextProperty( + 'DESCRIPTION', + $description); + } + + $organizer = $event->getOrganizer(); + if ($organizer) { + $properties[] = $this->newUserProperty( + 'ORGANIZER', + $organizer); + } + + $attendees = $event->getAttendees(); + if ($attendees) { + foreach ($attendees as $attendee) { + $properties[] = $this->newUserProperty( + 'ATTENDEE', + $attendee); + } + } + + $rrule = $event->getRecurrenceRule(); + if ($rrule) { + $properties[] = $this->newRRULEProperty( + 'RRULE', + $rrule); + } + + $recurrence_id = $event->getRecurrenceID(); + if ($recurrence_id) { + $properties[] = $this->newTextProperty( + 'RECURRENCE-ID', + $recurrence_id); + } + + $exdates = $event->getRecurrenceExceptions(); + if ($exdates) { + $properties[] = $this->newDateTimesProperty( + 'EXDATE', + $exdates); + } + + $rdates = $event->getRecurrenceDates(); + if ($rdates) { + $properties[] = $this->newDateTimesProperty( + 'RDATE', + $rdates); + } + + return $properties; + } + + private function newTextProperty( + $name, + $value, + array $parameters = array()) { + + $map = array( + '\\' => '\\\\', + ',' => '\\,', + "\n" => '\\n', + ); + + $value = (array)$value; + foreach ($value as $k => $v) { + $v = str_replace(array_keys($map), array_values($map), $v); + $value[$k] = $v; + } + + $value = implode(',', $value); + + return $this->newProperty($name, $value, $parameters); + } + + private function newDateTimeProperty( + $name, + PhutilCalendarDateTime $value, + array $parameters = array()) { + + return $this->newDateTimesProperty($name, array($value), $parameters); + } + + private function newDateTimesProperty( + $name, + array $values, + array $parameters = array()) { + assert_instances_of($values, 'PhutilCalendarDateTime'); + + if (head($values)->getIsAllDay()) { + $parameters[] = array( + 'name' => 'VALUE', + 'values' => array( + 'DATE', + ), + ); + } + + $datetimes = array(); + foreach ($values as $value) { + $datetimes[] = $value->getISO8601(); + } + $datetimes = implode(';', $datetimes); + + return $this->newProperty($name, $datetimes, $parameters); + } + + private function newUserProperty( + $name, + PhutilCalendarUserNode $value, + array $parameters = array()) { + + $parameters[] = array( + 'name' => 'CN', + 'values' => array( + $value->getName(), + ), + ); + + $partstat = null; + switch ($value->getStatus()) { + case PhutilCalendarUserNode::STATUS_INVITED: + $partstat = 'NEEDS-ACTION'; + break; + case PhutilCalendarUserNode::STATUS_ACCEPTED: + $partstat = 'ACCEPTED'; + break; + case PhutilCalendarUserNode::STATUS_DECLINED: + $partstat = 'DECLINED'; + break; + } + + if ($partstat !== null) { + $parameters[] = array( + 'name' => 'PARTSTAT', + 'values' => array( + $partstat, + ), + ); + } + + // TODO: We could reasonably fill in "ROLE" and "RSVP" here too, but it + // isn't clear if these are important to external programs or not. + + return $this->newProperty($name, $value->getURI(), $parameters); + } + + private function newRRULEProperty( + $name, + PhutilCalendarRecurrenceRule $rule, + array $parameters = array()) { + + $value = $rule->toRRULE(); + return $this->newProperty($name, $value, $parameters); + } + + private function newProperty( + $name, + $value, + array $parameters = array()) { + + $map = array( + '^' => '^^', + "\n" => '^n', + '"' => "^'", + ); + + $writable_params = array(); + foreach ($parameters as $k => $parameter) { + $value_list = array(); + foreach ($parameter['values'] as $v) { + $v = str_replace(array_keys($map), array_values($map), $v); + + // If the parameter value isn't a very simple one, quote it. + + // RFC5545 says that we MUST quote it if it has a colon, a semicolon, + // or a comma, and that we MUST quote it if it's a URI. + if (!preg_match('/^[A-Za-z0-9-]*\z/', $v)) { + $v = '"'.$v.'"'; + } + + $value_list[] = $v; + } + + $writable_params[] = array( + 'name' => $parameter['name'], + 'value' => implode(',', $value_list), + ); + } + + return array( + 'name' => $name, + 'value' => $value, + 'parameters' => $writable_params, + ); + } + +} diff --git a/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php b/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php new file mode 100644 index 0000000000..e99acaf8ab --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php @@ -0,0 +1,341 @@ +parseICSSingleEvent('simple.ics'); + + $this->assertEqual( + array( + array( + 'name' => 'CREATED', + 'parameters' => array(), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160908T172702Z', + ), + 'raw' => '20160908T172702Z', + ), + ), + array( + 'name' => 'UID', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + '1CEB57AF-0C9C-402D-B3BD-D75BD4843F68', + ), + 'raw' => '1CEB57AF-0C9C-402D-B3BD-D75BD4843F68', + ), + ), + array( + 'name' => 'DTSTART', + 'parameters' => array( + array( + 'name' => 'TZID', + 'values' => array( + array( + 'value' => 'America/Los_Angeles', + 'quoted' => false, + ), + ), + ), + ), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160915T090000', + ), + 'raw' => '20160915T090000', + ), + ), + array( + 'name' => 'DTEND', + 'parameters' => array( + array( + 'name' => 'TZID', + 'values' => array( + array( + 'value' => 'America/Los_Angeles', + 'quoted' => false, + ), + ), + ), + ), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160915T100000', + ), + 'raw' => '20160915T100000', + ), + ), + array( + 'name' => 'SUMMARY', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + 'Simple Event', + ), + 'raw' => 'Simple Event', + ), + ), + array( + 'name' => 'DESCRIPTION', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + 'This is a simple event.', + ), + 'raw' => 'This is a simple event.', + ), + ), + ), + $event->getAttribute('ics.properties')); + + $this->assertEqual( + 'Simple Event', + $event->getName()); + + $this->assertEqual( + 'This is a simple event.', + $event->getDescription()); + + $this->assertEqual( + 1473955200, + $event->getStartDateTime()->getEpoch()); + + $this->assertEqual( + 1473955200 + phutil_units('1 hour in seconds'), + $event->getEndDateTime()->getEpoch()); + } + + public function testICSOddTimezone() { + $event = $this->parseICSSingleEvent('zimbra-timezone.ics'); + + $start = $event->getStartDateTime(); + + $this->assertEqual( + '20170303T140000Z', + $start->getISO8601()); + } + + public function testICSFloatingTime() { + // This tests "floating" event times, which have no absolute time and are + // supposed to be interpreted using the viewer's timezone. It also uses + // a duration, and the duration needs to float along with the viewer + // timezone. + + $event = $this->parseICSSingleEvent('floating.ics'); + + $start = $event->getStartDateTime(); + + $caught = null; + try { + $start->getEpoch(); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue( + ($caught instanceof Exception), + pht('Expected exception for floating time with no viewer timezone.')); + + $newyears_utc = strtotime('2015-01-01 00:00:00 UTC'); + $this->assertEqual(1420070400, $newyears_utc); + + $start->setViewerTimezone('UTC'); + $this->assertEqual( + $newyears_utc, + $start->getEpoch()); + + $start->setViewerTimezone('America/Los_Angeles'); + $this->assertEqual( + $newyears_utc + phutil_units('8 hours in seconds'), + $start->getEpoch()); + + $start->setViewerTimezone('America/New_York'); + $this->assertEqual( + $newyears_utc + phutil_units('5 hours in seconds'), + $start->getEpoch()); + + $end = $event->getEndDateTime(); + $end->setViewerTimezone('UTC'); + $this->assertEqual( + $newyears_utc + phutil_units('24 hours in seconds'), + $end->getEpoch()); + + $end->setViewerTimezone('America/Los_Angeles'); + $this->assertEqual( + $newyears_utc + phutil_units('32 hours in seconds'), + $end->getEpoch()); + + $end->setViewerTimezone('America/New_York'); + $this->assertEqual( + $newyears_utc + phutil_units('29 hours in seconds'), + $end->getEpoch()); + } + + public function testICSVALARM() { + $event = $this->parseICSSingleEvent('valarm.ics'); + + // For now, we parse but ignore VALARM sections. This test just makes + // sure they survive parsing. + + $start_epoch = strtotime('2016-10-19 22:00:00 UTC'); + $this->assertEqual(1476914400, $start_epoch); + + $this->assertEqual( + $start_epoch, + $event->getStartDateTime()->getEpoch()); + } + + public function testICSDuration() { + $event = $this->parseICSSingleEvent('duration.ics'); + + // Raw value is "20160719T095722Z". + $start_epoch = strtotime('2016-07-19 09:57:22 UTC'); + $this->assertEqual(1468922242, $start_epoch); + + // Raw value is "P1DT17H4M23S". + $duration = + phutil_units('1 day in seconds') + + phutil_units('17 hours in seconds') + + phutil_units('4 minutes in seconds') + + phutil_units('23 seconds in seconds'); + + $this->assertEqual( + $start_epoch, + $event->getStartDateTime()->getEpoch()); + + $this->assertEqual( + $start_epoch + $duration, + $event->getEndDateTime()->getEpoch()); + } + + public function testICSWeeklyEvent() { + $event = $this->parseICSSingleEvent('weekly.ics'); + + $start = $event->getStartDateTime(); + $start->setViewerTimezone('UTC'); + + $rrule = $event->getRecurrenceRule() + ->setStartDateTime($start); + + $rset = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $rset->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150811'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150818'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150825'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Weekly recurring event.')); + } + + public function testICSParserErrors() { + $map = array( + 'err-missing-end.ics' => PhutilICSParser::PARSE_MISSING_END, + 'err-bad-base64.ics' => PhutilICSParser::PARSE_BAD_BASE64, + 'err-bad-boolean.ics' => PhutilICSParser::PARSE_BAD_BOOLEAN, + 'err-extra-end.ics' => PhutilICSParser::PARSE_EXTRA_END, + 'err-initial-unfold.ics' => PhutilICSParser::PARSE_INITIAL_UNFOLD, + 'err-malformed-double-quote.ics' => + PhutilICSParser::PARSE_MALFORMED_DOUBLE_QUOTE, + 'err-malformed-parameter.ics' => + PhutilICSParser::PARSE_MALFORMED_PARAMETER_NAME, + 'err-malformed-property.ics' => + PhutilICSParser::PARSE_MALFORMED_PROPERTY, + 'err-missing-value.ics' => PhutilICSParser::PARSE_MISSING_VALUE, + 'err-mixmatched-sections.ics' => + PhutilICSParser::PARSE_MISMATCHED_SECTIONS, + 'err-root-property.ics' => PhutilICSParser::PARSE_ROOT_PROPERTY, + 'err-unescaped-backslash.ics' => + PhutilICSParser::PARSE_UNESCAPED_BACKSLASH, + 'err-unexpected-text.ics' => PhutilICSParser::PARSE_UNEXPECTED_TEXT, + 'err-multiple-parameters.ics' => + PhutilICSParser::PARSE_MULTIPLE_PARAMETERS, + 'err-empty-datetime.ics' => + PhutilICSParser::PARSE_EMPTY_DATETIME, + 'err-many-datetime.ics' => + PhutilICSParser::PARSE_MANY_DATETIME, + 'err-bad-datetime.ics' => + PhutilICSParser::PARSE_BAD_DATETIME, + 'err-empty-duration.ics' => + PhutilICSParser::PARSE_EMPTY_DURATION, + 'err-many-duration.ics' => + PhutilICSParser::PARSE_MANY_DURATION, + 'err-bad-duration.ics' => + PhutilICSParser::PARSE_BAD_DURATION, + + 'simple.ics' => null, + 'good-boolean.ics' => null, + 'multiple-vcalendars.ics' => null, + ); + + foreach ($map as $test_file => $expect) { + $caught = null; + try { + $this->parseICSDocument($test_file); + } catch (PhutilICSParserException $ex) { + $caught = $ex; + } + + if ($expect === null) { + $this->assertTrue( + ($caught === null), + pht( + 'Expected no exception parsing "%s", got: %s', + $test_file, + (string)$ex)); + } else { + if ($caught) { + $code = $ex->getParserFailureCode(); + $explain = pht( + 'Expected one exception parsing "%s", got a different '. + 'one: %s', + $test_file, + (string)$ex); + } else { + $code = null; + $explain = pht( + 'Expected exception parsing "%s", got none.', + $test_file); + } + + $this->assertEqual($expect, $code, $explain); + } + } + } + + private function parseICSSingleEvent($name) { + $root = $this->parseICSDocument($name); + + $documents = $root->getDocuments(); + $this->assertEqual(1, count($documents)); + $document = head($documents); + + $events = $document->getEvents(); + $this->assertEqual(1, count($events)); + + return head($events); + } + + private function parseICSDocument($name) { + $path = dirname(__FILE__).'/data/'.$name; + $data = Filesystem::readFile($path); + return id(new PhutilICSParser()) + ->parseICSData($data); + } + + +} diff --git a/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php b/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php new file mode 100644 index 0000000000..dec1bf27b0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php @@ -0,0 +1,144 @@ +setUID('tea-time') + ->setName('Tea Time') + ->setDescription( + "Tea and, perhaps, crumpets.\n". + "Your presence is requested!\n". + "This is a long list of types of tea to test line wrapping: {$teas}.") + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160915T070000Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160915T070000Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160916T150000Z')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160916T160000Z')); + + $ics_data = $this->writeICSSingleEvent($event); + + $this->assertICS('writer-tea-time.ics', $ics_data); + } + + public function testICSWriterChristmas() { + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601('20001225T000000Z'); + $end = PhutilCalendarAbsoluteDateTime::newFromISO8601('20001226T000000Z'); + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_YEARLY) + ->setByMonth(array(12)) + ->setByMonthDay(array(25)); + + $event = id(new PhutilCalendarEventNode()) + ->setUID('recurring-christmas') + ->setName('Christmas') + ->setDescription('Festival holiday first occurring in the year 2000.') + ->setStartDateTime($start) + ->setEndDateTime($end) + ->setCreatedDateTime($start) + ->setModifiedDateTime($start) + ->setRecurrenceRule($rrule) + ->setRecurrenceExceptions( + array( + // In 2007, Christmas was cancelled. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20071225T000000Z'), + )) + ->setRecurrenceDates( + array( + // We had an extra early Christmas in 2009. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20091125T000000Z'), + )); + + $ics_data = $this->writeICSSingleEvent($event); + $this->assertICS('writer-recurring-christmas.ics', $ics_data); + } + + public function testICSWriterAllDay() { + $event = id(new PhutilCalendarEventNode()) + ->setUID('christmas-day') + ->setName('Christmas 2016') + ->setDescription('A minor religious holiday.') + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160901T232425Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160901T232425Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161225')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161226')); + + $ics_data = $this->writeICSSingleEvent($event); + + $this->assertICS('writer-christmas.ics', $ics_data); + } + + public function testICSWriterUsers() { + $event = id(new PhutilCalendarEventNode()) + ->setUID('office-party') + ->setName('Office Party') + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161001T120000Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161001T120000Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161215T200000Z')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161215T230000Z')) + ->setOrganizer( + id(new PhutilCalendarUserNode()) + ->setName('Big Boss') + ->setURI('mailto:big.boss@example.com')) + ->addAttendee( + id(new PhutilCalendarUserNode()) + ->setName('Milton') + ->setStatus(PhutilCalendarUserNode::STATUS_INVITED) + ->setURI('mailto:milton@example.com')) + ->addAttendee( + id(new PhutilCalendarUserNode()) + ->setName('Nancy') + ->setStatus(PhutilCalendarUserNode::STATUS_ACCEPTED) + ->setURI('mailto:nancy@example.com')); + + $ics_data = $this->writeICSSingleEvent($event); + $this->assertICS('writer-office-party.ics', $ics_data); + } + + private function writeICSSingleEvent(PhutilCalendarEventNode $event) { + $calendar = id(new PhutilCalendarDocumentNode()) + ->appendChild($event); + + $root = id(new PhutilCalendarRootNode()) + ->appendChild($calendar); + + return $this->writeICS($root); + } + + private function writeICS(PhutilCalendarRootNode $root) { + return id(new PhutilICSWriter()) + ->writeICSDocument($root); + } + + private function assertICS($name, $actual) { + $path = dirname(__FILE__).'/data/'.$name; + $data = Filesystem::readFile($path); + $this->assertEqual($data, $actual, pht('ICS: %s', $name)); + } + +} diff --git a/src/applications/calendar/parser/ics/__tests__/data/duration.ics b/src/applications/calendar/parser/ics/__tests__/data/duration.ics new file mode 100644 index 0000000000..8f2a8b5c6e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/duration.ics @@ -0,0 +1,8 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20160719T095722Z +DURATION:P1DT17H4M23S +SUMMARY:Duration Event +DESCRIPTION:This is an event with a complex duration. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics new file mode 100644 index 0000000000..ff1997e9e5 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DATA;VALUE=BINARY;ENCODING=BASE64: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics new file mode 100644 index 0000000000..f5cd06ca83 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DUCK;VALUE=BOOLEAN:QUACK +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics new file mode 100644 index 0000000000..b6dada8812 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:quack +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics new file mode 100644 index 0000000000..3d0eb7b04e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION:quack +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics new file mode 100644 index 0000000000..554fb2393f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics new file mode 100644 index 0000000000..33b4d78796 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics b/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics new file mode 100644 index 0000000000..b0bb4bc920 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics @@ -0,0 +1 @@ +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics b/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics new file mode 100644 index 0000000000..b57778242a --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics @@ -0,0 +1,2 @@ + BEGIN:VCALENDAR +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics new file mode 100644 index 0000000000..cd7623e51b --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +A;B="C:D +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics new file mode 100644 index 0000000000..8968cfba70 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +A;B:C +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics new file mode 100644 index 0000000000..2121531d2d --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +PEANUTBUTTER&JELLY:sandwich +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics new file mode 100644 index 0000000000..5e617a466e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20130101,20130101 +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics new file mode 100644 index 0000000000..d43d9efd8f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION:P1W,P2W +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics b/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics new file mode 100644 index 0000000000..d33f05bad7 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics @@ -0,0 +1,2 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics b/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics new file mode 100644 index 0000000000..39d83aa24f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +TRIANGLE;color=red +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics b/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics new file mode 100644 index 0000000000..e3a1ab0009 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics @@ -0,0 +1,4 @@ +BEGIN:A +BEGIN:B +END:A +END:B diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics b/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics new file mode 100644 index 0000000000..38da44f3ff --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART;TZID=A,B:20160915T090000 +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics b/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics new file mode 100644 index 0000000000..a36bd4c0b3 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics @@ -0,0 +1 @@ +NAME:value diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics b/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics new file mode 100644 index 0000000000..a1ba94b67a --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +STORY:The duck coughed up an unescaped backslash: \ +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics b/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics new file mode 100644 index 0000000000..30873a2f60 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +SQUARE;color=red" +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/floating.ics b/src/applications/calendar/parser/ics/__tests__/data/floating.ics new file mode 100644 index 0000000000..eecfe23f3e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/floating.ics @@ -0,0 +1,8 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20150101T000000 +DURATION:P1D +SUMMARY:New Year's 2015 +DESCRIPTION:This is an event with a floating start time. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics b/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics new file mode 100644 index 0000000000..d281b17fb2 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DUCK;VALUE=BOOLEAN:TRUE +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics b/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics new file mode 100644 index 0000000000..68e99a6e18 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics @@ -0,0 +1,4 @@ +BEGIN:VCALENDAR +END:VCALENDAR +BEGIN:VCALENDAR +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/simple.ics b/src/applications/calendar/parser/ics/__tests__/data/simple.ics new file mode 100644 index 0000000000..8181a24ff0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/simple.ics @@ -0,0 +1,12 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +BEGIN:VEVENT +CREATED:20160908T172702Z +UID:1CEB57AF-0C9C-402D-B3BD-D75BD4843F68 +DTSTART;TZID=America/Los_Angeles:20160915T090000 +DTEND;TZID=America/Los_Angeles:20160915T100000 +SUMMARY:Simple Event +DESCRIPTION:This is a simple event. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/valarm.ics b/src/applications/calendar/parser/ics/__tests__/data/valarm.ics new file mode 100644 index 0000000000..060f5ef85f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/valarm.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +CREATED:20161027T173727 +DTSTAMP:20161027T173727 +LAST-MODIFIED:20161027T173727 +UID:aic4zm86mg +SUMMARY:alarm event +DTSTART;TZID=Europe/Berlin:20161020T000000 +DTEND;TZID=Europe/Berlin:20161020T010000 +BEGIN:VALARM +ACTION:AUDIO +TRIGGER:-PT15M +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/weekly.ics b/src/applications/calendar/parser/ics/__tests__/data/weekly.ics new file mode 100644 index 0000000000..8e067bf8ea --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/weekly.ics @@ -0,0 +1,14 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND;VALUE=DATE:20150812 +LAST-MODIFIED:20160822T130015Z +UID:4AE69E91-4A51-4B77-8849-85981E037A83 +DTSTAMP:20161129T152151Z +SUMMARY:Weekly Event +DTSTART;VALUE=DATE:20150811 +CREATED:20141109T163445Z +RRULE:FREQ=WEEKLY +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics new file mode 100644 index 0000000000..4624d151d0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics @@ -0,0 +1,13 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:christmas-day +CREATED:20160901T232425Z +DTSTAMP:20160901T232425Z +DTSTART;VALUE=DATE:20161225 +DTEND;VALUE=DATE:20161226 +SUMMARY:Christmas 2016 +DESCRIPTION:A minor religious holiday. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics new file mode 100644 index 0000000000..a2fbc81a40 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics @@ -0,0 +1,15 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:office-party +CREATED:20161001T120000Z +DTSTAMP:20161001T120000Z +DTSTART:20161215T200000Z +DTEND:20161215T230000Z +SUMMARY:Office Party +ORGANIZER;CN="Big Boss":mailto:big.boss@example.com +ATTENDEE;CN=Milton;PARTSTAT=NEEDS-ACTION:mailto:milton@example.com +ATTENDEE;CN=Nancy;PARTSTAT=ACCEPTED:mailto:nancy@example.com +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics new file mode 100644 index 0000000000..2774bb5bb9 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:recurring-christmas +CREATED:20001225T000000Z +DTSTAMP:20001225T000000Z +DTSTART:20001225T000000Z +DTEND:20001226T000000Z +SUMMARY:Christmas +DESCRIPTION:Festival holiday first occurring in the year 2000. +RRULE:FREQ=YEARLY;BYMONTH=12;BYMONTHDAY=25 +EXDATE:20071225T000000Z +RDATE:20091125T000000Z +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics new file mode 100644 index 0000000000..e275fa9730 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:tea-time +CREATED:20160915T070000Z +DTSTAMP:20160915T070000Z +DTSTART:20160916T150000Z +DTEND:20160916T160000Z +SUMMARY:Tea Time +DESCRIPTION:Tea and\, perhaps\, crumpets.\nYour presence is requested!\nThi + s is a long list of types of tea to test line wrapping: earl grey tea\, En + glish breakfast tea\, black tea\, green tea\, t-rex\, oolong tea\, mint te + a\, tea with milk. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics b/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics new file mode 100644 index 0000000000..6066b57f7d --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics @@ -0,0 +1,12 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +BEGIN:VEVENT +CREATED:20161104T220244Z +UID:zimbra-timezone +SUMMARY:Zimbra Timezone +DTSTART;TZID="(GMT-05.00) Auto-Detected":20170303T090000 +DTSTAMP:20161104T220244Z +SEQUENCE:0 +END:VEVENT +END:VCALENDAR diff --git a/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php new file mode 100644 index 0000000000..02fab53b93 --- /dev/null +++ b/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php @@ -0,0 +1,107 @@ + array( + '[words].', + '[words].', + '[words].', + '[words]: [word], [word], [word] [word].', + '[words]; [lowerwords].', + '[words]!', + '[words], "[words]."', + '[words] ("[upperword] [upperword] [upperword]") [lowerwords].', + '[words]?', + ), + 'words' => array( + '[upperword] [lowerwords]', + ), + 'upperword' => array( + 'Lorem', + 'Ipsum', + 'Dolor', + 'Sit', + 'Amet', + ), + 'lowerwords' => array( + '[word]', + '[word] [word]', + '[word] [word] [word]', + '[word] [word] [word] [word]', + '[word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word] [word]', + ), + 'word' => array( + 'ad', + 'adipisicing', + 'aliqua', + 'aliquip', + 'amet', + 'anim', + 'aute', + 'cillum', + 'commodo', + 'consectetur', + 'consequat', + 'culpa', + 'cupidatat', + 'deserunt', + 'do', + 'dolor', + 'dolore', + 'duis', + 'ea', + 'eiusmod', + 'elit', + 'enim', + 'esse', + 'est', + 'et', + 'eu', + 'ex', + 'excepteur', + 'exercitation', + 'fugiat', + 'id', + 'in', + 'incididunt', + 'ipsum', + 'irure', + 'labore', + 'laboris', + 'laborum', + 'lorem', + 'magna', + 'minim', + 'mollit', + 'nisi', + 'non', + 'nostrud', + 'nulla', + 'occaecat', + 'officia', + 'pariatur', + 'proident', + 'qui', + 'quis', + 'reprehenderit', + 'sed', + 'sint', + 'sit', + 'sunt', + 'tempor', + 'ullamco', + 'ut', + 'velit', + 'veniam', + 'voluptate', + ), + ); + } + +} diff --git a/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php new file mode 100644 index 0000000000..95777e8d30 --- /dev/null +++ b/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php @@ -0,0 +1,155 @@ + array( + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]-[last]', + '[first] [middle] [last]', + '[first] "[nick]" [last]', + '[first] [particle] [particle] [particle]', + ), + 'first' => array( + 'Mohamed', + 'Youssef', + 'Ahmed', + 'Mahmoud', + 'Mustafa', + 'Fatma', + 'Aya', + 'Noam', + 'Adam', + 'Lucas', + 'Noah', + 'Jakub', + 'Victor', + 'Harry', + 'Rasmus', + 'Nathan', + 'Emil', + 'Charlie', + 'Leon', + 'Dylan', + 'Alexander', + 'Emma', + 'Marie', + 'Lea', + 'Amelia', + 'Hanna', + 'Emily', + 'Sofia', + 'Julia', + 'Santiago', + 'Sebastian', + 'Olivia', + 'Madison', + 'Isabella', + 'Esther', + 'Anya', + 'Camila', + 'Jack', + 'Oliver', + ), + 'nick' => array( + 'Buzz', + 'Juggernaut', + 'Haze', + 'Hawk', + 'Iceman', + 'Killer', + 'Apex', + 'Ocelot', + ), + 'middle' => array( + 'Rose', + 'Grace', + 'Jane', + 'Louise', + 'Jade', + 'James', + 'John', + 'William', + 'Thomas', + 'Alexander', + ), + 'last' => array( + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + 'O\'[termlast]', + 'Mc[termlast]', + ), + 'termlast' => array( + 'Smith', + 'Johnson', + 'Williams', + 'Jones', + 'Brown', + 'Davis', + 'Miller', + 'Wilson', + 'Moore', + 'Taylor', + 'Anderson', + 'Thomas', + 'Jackson', + 'White', + 'Harris', + 'Martin', + 'Thompson', + 'Garcia', + 'Marinez', + 'Robinson', + 'Clark', + 'Rodrigues', + 'Lewis', + 'Lee', + 'Walker', + 'Hall', + 'Allen', + 'Young', + 'Hernandex', + 'King', + 'Wang', + 'Li', + 'Zhang', + 'Liu', + 'Chen', + 'Yang', + 'Huang', + 'Zhao', + 'Wu', + 'Zhou', + 'Xu', + 'Sun', + 'Ma', + ), + 'particle' => array( + 'Wu', + 'Xu', + 'Ma', + 'Li', + 'Liu', + 'Shao', + 'Lin', + 'Khan', + ), + ); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..baf665ef48 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php @@ -0,0 +1,254 @@ +getStmtTerminationGrammarSet(), + $this->getVarNameGrammarSet(), + $this->getNullExprGrammarSet(), + $this->getNumberGrammarSet(), + $this->getExprGrammarSet(), + $this->getCondGrammarSet(), + $this->getLoopGrammarSet(), + $this->getStmtGrammarSet(), + $this->getAssignmentGrammarSet(), + $this->getArithExprGrammarSet(), + $this->getBoolExprGrammarSet(), + $this->getBoolValGrammarSet(), + $this->getTernaryExprGrammarSet(), + + $this->getFuncNameGrammarSet(), + $this->getFuncCallGrammarSet(), + $this->getFuncCallParamGrammarSet(), + $this->getFuncDeclGrammarSet(), + $this->getFuncParamGrammarSet(), + $this->getFuncBodyGrammarSet(), + $this->getFuncReturnGrammarSet(), + ); + } + + protected function getStartGrammarSet() { + $start_grammar = parent::getStartGrammarSet(); + + $start_grammar['start'][] = '[funcdecl]'; + + return $start_grammar; + } + + protected function getStmtTerminationGrammarSet() { + return $this->buildGrammarSet('term', array(';')); + } + + protected function getFuncCallGrammarSet() { + return $this->buildGrammarSet('funccall', + array( + '[funcname]([funccallparam])', + )); + } + + protected function getFuncCallParamGrammarSet() { + return $this->buildGrammarSet('funccallparam', + array( + '', + '[expr]', + '[expr], [expr]', + )); + } + + protected function getFuncDeclGrammarSet() { + return $this->buildGrammarSet('funcdecl', + array( + 'function [funcname]([funcparam]) '. + '{[funcbody, indent, block, trim=right]}', + )); + } + + protected function getFuncParamGrammarSet() { + return $this->buildGrammarSet('funcparam', + array( + '', + '[varname]', + '[varname], [varname]', + '[varname], [varname], [varname]', + )); + } + + protected function getFuncBodyGrammarSet() { + return $this->buildGrammarSet('funcbody', + array( + "[stmt]\n[stmt]\n[funcreturn]", + "[stmt]\n[stmt]\n[stmt]\n[funcreturn]", + "[stmt]\n[stmt]\n[stmt]\n[stmt]\n[funcreturn]", + )); + } + + protected function getFuncReturnGrammarSet() { + return $this->buildGrammarSet('funcreturn', + array( + 'return [expr][term]', + '', + )); + } + + // Not really C, but put it here because of the curly braces and mostly shared + // among Java and PHP + protected function getClassDeclGrammarSet() { + return $this->buildGrammarSet('classdecl', + array( + '[classinheritancemod] class [classname] {[classbody, indent, block]}', + 'class [classname] {[classbody, indent, block]}', + )); + } + + protected function getClassNameGrammarSet() { + return $this->buildGrammarSet('classname', + array( + 'MuffinHouse', + 'MuffinReader', + 'MuffinAwesomizer', + 'SuperException', + 'Librarian', + 'Book', + 'Ball', + 'BallOfCode', + 'AliceAndBobsSharedSecret', + 'FileInputStream', + 'FileOutputStream', + 'BufferedReader', + 'BufferedWriter', + 'Cardigan', + 'HouseOfCards', + 'UmbrellaClass', + 'GenericThing', + )); + } + + protected function getClassBodyGrammarSet() { + return $this->buildGrammarSet('classbody', + array( + '[methoddecl]', + "[methoddecl]\n\n[methoddecl]", + "[propdecl]\n[propdecl]\n\n[methoddecl]\n\n[methoddecl]", + "[propdecl]\n[propdecl]\n[propdecl]\n\n[methoddecl]\n\n[methoddecl]". + "\n\n[methoddecl]", + )); + } + + protected function getVisibilityGrammarSet() { + return $this->buildGrammarSet('visibility', + array( + 'private', + 'protected', + 'public', + )); + } + + protected function getClassInheritanceModGrammarSet() { + return $this->buildGrammarSet('classinheritancemod', + array( + 'final', + 'abstract', + )); + } + + // Keeping this separate so we won't give abstract methods a function body + protected function getMethodInheritanceModGrammarSet() { + return $this->buildGrammarSet('methodinheritancemod', + array( + 'final', + )); + } + + protected function getMethodDeclGrammarSet() { + return $this->buildGrammarSet('methoddecl', + array( + '[visibility] [methodfuncdecl]', + '[visibility] [methodfuncdecl]', + '[methodinheritancemod] [visibility] [methodfuncdecl]', + '[abstractmethoddecl]', + )); + } + + protected function getMethodFuncDeclGrammarSet() { + return $this->buildGrammarSet('methodfuncdecl', + array( + 'function [funcname]([funcparam]) '. + '{[methodbody, indent, block, trim=right]}', + )); + } + + protected function getMethodBodyGrammarSet() { + return $this->buildGrammarSet('methodbody', + array( + "[methodstmt]\n[methodbody]", + "[methodstmt]\n[funcreturn]", + )); + } + + protected function getMethodStmtGrammarSet() { + $stmts = $this->getStmtGrammarSet(); + + return $this->buildGrammarSet('methodstmt', + array_merge( + $stmts['stmt'], + array( + '[methodcall][term]', + ))); + } + + protected function getMethodCallGrammarSet() { + // Java/JavaScript + return $this->buildGrammarSet('methodcall', + array( + 'this.[funccall]', + '[varname].[funccall]', + '[classname].[funccall]', + )); + } + + protected function getAbstractMethodDeclGrammarSet() { + return $this->buildGrammarSet('abstractmethoddecl', + array( + 'abstract function [funcname]([funcparam])[term]', + )); + } + + protected function getPropDeclGrammarSet() { + return $this->buildGrammarSet('propdecl', + array( + '[visibility] [varname][term]', + )); + } + + protected function getClassRuleSets() { + return array( + $this->getClassInheritanceModGrammarSet(), + $this->getMethodInheritanceModGrammarSet(), + $this->getClassDeclGrammarSet(), + $this->getClassNameGrammarSet(), + $this->getClassBodyGrammarSet(), + $this->getMethodDeclGrammarSet(), + $this->getMethodFuncDeclGrammarSet(), + $this->getMethodBodyGrammarSet(), + $this->getMethodStmtGrammarSet(), + $this->getMethodCallGrammarSet(), + $this->getAbstractMethodDeclGrammarSet(), + $this->getPropDeclGrammarSet(), + $this->getVisibilityGrammarSet(), + ); + } + + public function generateClass() { + $rules = array_merge($this->getRules(), $this->getClassRuleSets()); + $rules['start'] = array('[classdecl]'); + $count = 0; + return $this->applyRules('[start]', $count, $rules); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..b3fe621591 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php @@ -0,0 +1,205 @@ +getStartGrammarSet(), + $this->getStmtGrammarSet(), + array_mergev($this->buildRuleSet())); + } + + abstract protected function buildRuleSet(); + + protected function buildGrammarSet($name, array $set) { + return array( + $name => $set, + ); + } + + protected function getStartGrammarSet() { + return $this->buildGrammarSet('start', + array( + "[stmt]\n[stmt]", + "[stmt]\n[stmt]\n[stmt]", + "[stmt]\n[stmt]\n[stmt]\n[stmt]", + )); + } + + protected function getStmtGrammarSet() { + return $this->buildGrammarSet('stmt', + array( + '[assignment][term]', + '[assignment][term]', + '[assignment][term]', + '[assignment][term]', + '[funccall][term]', + '[funccall][term]', + '[funccall][term]', + '[funccall][term]', + '[cond]', + '[loop]', + )); + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array( + 'do_something', + 'nonempty', + 'noOp', + 'call_user_func', + 'getenv', + 'render', + 'super', + 'derpify', + 'awesomize', + 'equals', + 'run', + 'flee', + 'fight', + 'notify', + 'listen', + 'calculate', + 'aim', + 'open', + )); + } + + protected function getVarNameGrammarSet() { + return $this->buildGrammarSet('varname', + array( + 'is_something', + 'object', + 'name', + 'token', + 'label', + 'piece_of_the_pie', + 'type', + 'state', + 'param', + 'action', + 'key', + 'timeout', + 'result', + )); + } + + protected function getNullExprGrammarSet() { + return $this->buildGrammarSet('null', array('null')); + } + + protected function getNumberGrammarSet() { + return $this->buildGrammarSet('number', + array( + mt_rand(-1, 100), + mt_rand(-100, 1000), + mt_rand(-1000, 5000), + mt_rand(0, 1).'.'.mt_rand(1, 1000), + mt_rand(0, 50).'.'.mt_rand(1, 1000), + )); + } + + protected function getExprGrammarSet() { + return $this->buildGrammarSet('expr', + array( + '[null]', + '[number]', + '[number]', + '[varname]', + '[varname]', + '[boolval]', + '[boolval]', + '[boolexpr]', + '[boolexpr]', + '[funccall]', + '[arithexpr]', + '[arithexpr]', + // Some random strings + '"'.Filesystem::readRandomCharacters(4).'"', + '"'.Filesystem::readRandomCharacters(5).'"', + )); + } + + protected function getBoolExprGrammarSet() { + return $this->buildGrammarSet('boolexpr', + array( + '[varname]', + '![varname]', + '[varname] == [boolval]', + '[varname] != [boolval]', + '[ternary]', + )); + } + + protected function getBoolValGrammarSet() { + return $this->buildGrammarSet('boolval', + array( + 'true', + 'false', + )); + } + + protected function getArithExprGrammarSet() { + return $this->buildGrammarSet('arithexpr', + array( + '[varname]++', + '++[varname]', + '[varname] + [number]', + '[varname]--', + '--[varname]', + '[varname] - [number]', + )); + } + + protected function getAssignmentGrammarSet() { + return $this->buildGrammarSet('assignment', + array( + '[varname] = [expr]', + '[varname] = [arithexpr]', + '[varname] += [expr]', + )); + } + + protected function getCondGrammarSet() { + return $this->buildGrammarSet('cond', + array( + 'if ([boolexpr]) {[stmt, indent, block]}', + 'if ([boolexpr]) {[stmt, indent, block]} else {[stmt, indent, block]}', + )); + } + + protected function getLoopGrammarSet() { + return $this->buildGrammarSet('loop', + array( + 'while ([boolexpr]) {[stmt, indent, block]}', + 'do {[stmt, indent, block]} while ([boolexpr])[term]', + 'for ([assignment]; [boolexpr]; [expr]) {[stmt, indent, block]}', + )); + } + + protected function getTernaryExprGrammarSet() { + return $this->buildGrammarSet('ternary', + array( + '[boolexpr] ? [expr] : [expr]', + )); + } + + protected function getStmtTerminationGrammarSet() { + return $this->buildGrammarSet('term', array('')); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..28a6990e77 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php @@ -0,0 +1,184 @@ +getClassRuleSets()); + + $rulesset[] = $this->getTypeNameGrammarSet(); + $rulesset[] = $this->getNamespaceDeclGrammarSet(); + $rulesset[] = $this->getNamespaceNameGrammarSet(); + $rulesset[] = $this->getImportGrammarSet(); + $rulesset[] = $this->getMethodReturnTypeGrammarSet(); + $rulesset[] = $this->getMethodNameGrammarSet(); + $rulesset[] = $this->getVarDeclGrammarSet(); + $rulesset[] = $this->getClassDerivGrammarSet(); + + return $rulesset; + } + + protected function getStartGrammarSet() { + return $this->buildGrammarSet('start', + array( + '[import, block][nmspdecl, block][classdecl, block]', + )); + } + + protected function getClassDeclGrammarSet() { + return $this->buildGrammarSet('classdecl', + array( + '[classinheritancemod] [visibility] class [classname][classderiv] '. + '{[classbody, indent, block]}', + '[visibility] class [classname][classderiv] '. + '{[classbody, indent, block]}', + )); + } + + protected function getClassDerivGrammarSet() { + return $this->buildGrammarSet('classderiv', + array( + ' extends [classname]', + '', + '', + )); + } + + protected function getTypeNameGrammarSet() { + return $this->buildGrammarSet('type', + array( + 'int', + 'boolean', + 'char', + 'short', + 'long', + 'float', + 'double', + '[classname]', + '[type][]', + )); + } + + protected function getMethodReturnTypeGrammarSet() { + return $this->buildGrammarSet('methodreturn', + array( + '[type]', + 'void', + )); + } + + protected function getNamespaceDeclGrammarSet() { + return $this->buildGrammarSet('nmspdecl', + array( + 'package [nmspname][term]', + )); + } + + protected function getNamespaceNameGrammarSet() { + return $this->buildGrammarSet('nmspname', + array( + 'java.lang', + 'java.io', + 'com.example.proj.std', + 'derp.example.www', + )); + } + + protected function getImportGrammarSet() { + return $this->buildGrammarSet('import', + array( + 'import [nmspname][term]', + 'import [nmspname].*[term]', + 'import [nmspname].[classname][term]', + )); + } + + protected function getExprGrammarSet() { + $expr = parent::getExprGrammarSet(); + + $expr['expr'][] = 'new [classname]([funccallparam])'; + + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + + // Add some 'char's + for ($ii = 0; $ii < 2; $ii++) { + $expr['expr'][] = "'".Filesystem::readRandomCharacters(1)."'"; + } + + return $expr; + } + + protected function getStmtGrammarSet() { + $stmt = parent::getStmtGrammarSet(); + + $stmt['stmt'][] = '[vardecl]'; + $stmt['stmt'][] = '[vardecl]'; + // `try` to `throw` a `Ball`! + $stmt['stmt'][] = 'throw [classname][term]'; + + return $stmt; + } + + protected function getPropDeclGrammarSet() { + return $this->buildGrammarSet('propdecl', + array( + '[visibility] [type] [varname][term]', + )); + } + + protected function getVarDeclGrammarSet() { + return $this->buildGrammarSet('vardecl', + array( + '[type] [varname][term]', + '[type] [assignment][term]', + )); + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array( + '[methodname]', + '[classname].[methodname]', + // This is just silly (too much recursion) + // '[classname].[funcname]', + // Don't do this for now, it just clutters up output (thanks to rec.) + // '[nmspname].[classname].[methodname]', + )); + } + + // Renamed from `funcname` + protected function getMethodNameGrammarSet() { + $funcnames = head(parent::getFuncNameGrammarSet()); + return $this->buildGrammarSet('methodname', $funcnames); + } + + protected function getMethodFuncDeclGrammarSet() { + return $this->buildGrammarSet('methodfuncdecl', + array( + '[methodreturn] [methodname]([funcparam]) '. + '{[methodbody, indent, block, trim=right]}', + )); + } + + protected function getFuncParamGrammarSet() { + return $this->buildGrammarSet('funcparam', + array( + '', + '[type] [varname]', + '[type] [varname], [type] [varname]', + '[type] [varname], [type] [varname], [type] [varname]', + )); + } + + protected function getAbstractMethodDeclGrammarSet() { + return $this->buildGrammarSet('abstractmethoddecl', + array( + 'abstract [methodreturn] [methodname]([funcparam])[term]', + )); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..a8f83bab2d --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php @@ -0,0 +1,57 @@ +getClassRuleSets()); + } + + protected function getStartGrammarSet() { + $start_grammar = parent::getStartGrammarSet(); + + $start_grammar['start'][] = '[classdecl]'; + $start_grammar['start'][] = '[classdecl]'; + + return $start_grammar; + } + + protected function getExprGrammarSet() { + $expr = parent::getExprGrammarSet(); + + $expr['expr'][] = 'new [classname]([funccallparam])'; + + $expr['expr'][] = '[classname]::[funccall]'; + + return $expr; + } + + protected function getVarNameGrammarSet() { + $varnames = parent::getVarNameGrammarSet(); + + foreach ($varnames as $vn_key => $vn_val) { + foreach ($vn_val as $vv_key => $vv_value) { + $varnames[$vn_key][$vv_key] = '$'.$vv_value; + } + } + + return $varnames; + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array_mergev(get_defined_functions())); + } + + protected function getMethodCallGrammarSet() { + return $this->buildGrammarSet('methodcall', + array( + '$this->[funccall]', + 'self::[funccall]', + 'static::[funccall]', + '[varname]->[funccall]', + '[classname]::[funccall]', + )); + } + +} diff --git a/src/infrastructure/markup/PhutilRemarkupBlockStorage.php b/src/infrastructure/markup/PhutilRemarkupBlockStorage.php new file mode 100644 index 0000000000..2b97c13a19 --- /dev/null +++ b/src/infrastructure/markup/PhutilRemarkupBlockStorage.php @@ -0,0 +1,176 @@ +". The first + * byte, "<0x01>" is a single byte with value 1 that marks a token. If this is + * token ID "444", the text may now look like this: + * + * //<0x01>444Z// + * + * Now the italics match and are replaced, using the next token ID: + * + * <0x01>445Z + * + * When processing completes, all the tokens are replaced with their final + * equivalents. For example, token 444 is evaluated to: + * + * ... + * + * Then token 445 is evaluated: + * + * <0x01>444Z + * + * ...and all tokens it contains are replaced: + * + * ... + * + * If we didn't do this, the italics rule could match the "//" in "http://", + * or any other number of processing mistakes could occur, some of which create + * security risks. + * + * This class generates keys, and stores the map of keys to replacement text. + */ +final class PhutilRemarkupBlockStorage extends Phobject { + + const MAGIC_BYTE = "\1"; + + private $map = array(); + private $index = 0; + + public function store($text) { + $key = self::MAGIC_BYTE.(++$this->index).'Z'; + $this->map[$key] = $text; + return $key; + } + + public function restore($corpus, $text_mode = false) { + $map = $this->map; + + if (!$text_mode) { + foreach ($map as $key => $content) { + $map[$key] = phutil_escape_html($content); + } + $corpus = phutil_escape_html($corpus); + } + + // NOTE: Tokens may contain other tokens: for example, a table may have + // links inside it. So we can't do a single simple find/replace, because + // we need to find and replace child tokens inside the content of parent + // tokens. + + // However, we know that rules which have child tokens must always store + // all their child tokens first, before they store their parent token: you + // have to pass the "store(text)" API a block of text with tokens already + // in it, so you must have created child tokens already. + + // Thus, all child tokens will appear in the list before parent tokens, so + // if we start at the beginning of the list and replace all the tokens we + // find in each piece of content, we'll end up expanding all subtokens + // correctly. + + $map[] = $corpus; + $seen = array(); + foreach ($map as $key => $content) { + $seen[$key] = true; + + // If the content contains no token magic, we don't need to replace + // anything. + if (strpos($content, self::MAGIC_BYTE) === false) { + continue; + } + + $matches = null; + preg_match_all( + '/'.self::MAGIC_BYTE.'\d+Z/', + $content, + $matches, + PREG_OFFSET_CAPTURE); + + $matches = $matches[0]; + + // See PHI1114. We're replacing all the matches in one pass because this + // is significantly faster than doing "substr_replace()" in a loop if the + // corpus is large and we have a large number of matches. + + // Build a list of string pieces in "$parts" by interleaving the + // plain strings between each token and the replacement token text, then + // implode the whole thing when we're done. + + $parts = array(); + $pos = 0; + foreach ($matches as $next) { + $subkey = $next[0]; + + // If we've matched a token pattern but don't actually have any + // corresponding token, just skip this match. This should not be + // possible, and should perhaps be an error. + if (!isset($seen[$subkey])) { + if (!isset($map[$subkey])) { + throw new Exception( + pht( + 'Matched token key "%s" while processing remarkup block, but '. + 'this token does not exist in the token map.', + $subkey)); + } else { + throw new Exception( + pht( + 'Matched token key "%s" while processing remarkup block, but '. + 'this token appears later in the list than the key being '. + 'processed ("%s").', + $subkey, + $key)); + } + } + + $subpos = $next[1]; + + // If there were any non-token bytes since the last token, add them. + if ($subpos > $pos) { + $parts[] = substr($content, $pos, $subpos - $pos); + } + + // Add the token replacement text. + $parts[] = $map[$subkey]; + + // Move the non-token cursor forward over the token. + $pos = $subpos + strlen($subkey); + } + + // Add any leftover non-token bytes after the last token. + $parts[] = substr($content, $pos); + + $content = implode('', $parts); + + $map[$key] = $content; + } + $corpus = last($map); + + if (!$text_mode) { + $corpus = phutil_safe_html($corpus); + } + + return $corpus; + } + + public function overwrite($key, $new_text) { + $this->map[$key] = $new_text; + return $this; + } + + public function getMap() { + return $this->map; + } + + public function setMap(array $map) { + $this->map = $map; + return $this; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php new file mode 100644 index 0000000000..3b02c87152 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php @@ -0,0 +1,36 @@ +engine = $engine; + return $this; + } + + final public function getEngine() { + return $this->engine; + } + + /** + * @return string + */ + abstract public function getInterpreterName(); + + abstract public function markupContent($content, array $argv); + + protected function markupError($string) { + if ($this->getEngine()->isTextMode()) { + return '('.$string.')'; + } else { + return phutil_tag( + 'div', + array( + 'class' => 'remarkup-interpreter-error', + ), + $string); + } + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php new file mode 100644 index 0000000000..feac6cffa9 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php @@ -0,0 +1,170 @@ +addInt($this->getPriority()) + ->addString(get_class($this)); + } + + abstract public function markupText($text, $children); + + /** + * This will get an array of unparsed lines and return the number of lines + * from the first array value that it can parse. + * + * @param array $lines + * @param int $cursor + * + * @return int + */ + abstract public function getMatchingLineCount(array $lines, $cursor); + + protected function didMarkupText() { + return; + } + + final public function setEngine(PhutilRemarkupEngine $engine) { + $this->engine = $engine; + $this->updateRules(); + return $this; + } + + final protected function getEngine() { + return $this->engine; + } + + public function setMarkupRules(array $rules) { + assert_instances_of($rules, 'PhutilRemarkupRule'); + $this->rules = $rules; + $this->updateRules(); + return $this; + } + + private function updateRules() { + $engine = $this->getEngine(); + if ($engine) { + $this->rules = msort($this->rules, 'getPriority'); + foreach ($this->rules as $rule) { + $rule->setEngine($engine); + } + } + return $this; + } + + final public function getMarkupRules() { + return $this->rules; + } + + final public function postprocess() { + $this->didMarkupText(); + } + + final protected function applyRules($text) { + foreach ($this->getMarkupRules() as $rule) { + $text = $rule->apply($text); + } + return $text; + } + + public function supportsChildBlocks() { + return false; + } + + public function extractChildText($text) { + throw new PhutilMethodNotImplementedException(); + } + + protected function renderRemarkupTable(array $out_rows) { + assert_instances_of($out_rows, 'array'); + + if ($this->getEngine()->isTextMode()) { + $lengths = array(); + foreach ($out_rows as $r => $row) { + foreach ($row['content'] as $c => $cell) { + $text = $this->getEngine()->restoreText($cell['content']); + $lengths[$c][$r] = phutil_utf8_strlen($text); + } + } + $max_lengths = array_map('max', $lengths); + + $out = array(); + foreach ($out_rows as $r => $row) { + $headings = false; + foreach ($row['content'] as $c => $cell) { + $length = $max_lengths[$c] - $lengths[$c][$r]; + $out[] = '| '.$cell['content'].str_repeat(' ', $length).' '; + if ($cell['type'] == 'th') { + $headings = true; + } + } + $out[] = "|\n"; + + if ($headings) { + foreach ($row['content'] as $c => $cell) { + $char = ($cell['type'] == 'th' ? '-' : ' '); + $out[] = '| '.str_repeat($char, $max_lengths[$c]).' '; + } + $out[] = "|\n"; + } + } + + return rtrim(implode('', $out), "\n"); + } + + if ($this->getEngine()->isHTMLMailMode()) { + $table_attributes = array( + 'style' => 'border-collapse: separate; + border-spacing: 1px; + background: #d3d3d3; + margin: 12px 0;', + ); + $cell_attributes = array( + 'style' => 'background: #ffffff; + padding: 3px 6px;', + ); + } else { + $table_attributes = array( + 'class' => 'remarkup-table', + ); + $cell_attributes = array(); + } + + $out = array(); + $out[] = "\n"; + foreach ($out_rows as $row) { + $cells = array(); + foreach ($row['content'] as $cell) { + $cells[] = phutil_tag( + $cell['type'], + $cell_attributes, + $cell['content']); + } + $out[] = phutil_tag($row['type'], array(), $cells); + $out[] = "\n"; + } + + $table = phutil_tag('table', $table_attributes, $out); + return phutil_tag_div('remarkup-table-wrap', $table); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php new file mode 100644 index 0000000000..b8fef85b98 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php @@ -0,0 +1,252 @@ + false, + 'lang' => null, + 'name' => null, + 'lines' => null, + ); + + $parser = new PhutilSimpleOptions(); + $custom = $parser->parse(head($lines)); + if ($custom) { + $valid = true; + foreach ($custom as $key => $value) { + if (!array_key_exists($key, $options)) { + $valid = false; + break; + } + } + if ($valid) { + array_shift($lines); + $options = $custom + $options; + } + } + + // Normalize the text back to a 0-level indent. + $min_indent = 80; + foreach ($lines as $line) { + for ($ii = 0; $ii < strlen($line); $ii++) { + if ($line[$ii] != ' ') { + $min_indent = min($ii, $min_indent); + break; + } + } + } + + $text = implode("\n", $lines); + if ($min_indent) { + $indent_string = str_repeat(' ', $min_indent); + $text = preg_replace('/^'.$indent_string.'/m', '', $text); + } + + if ($this->getEngine()->isTextMode()) { + $out = array(); + + $header = array(); + if ($options['counterexample']) { + $header[] = 'counterexample'; + } + if ($options['name'] != '') { + $header[] = 'name='.$options['name']; + } + if ($header) { + $out[] = implode(', ', $header); + } + + $text = preg_replace('/^/m', ' ', $text); + $out[] = $text; + + return implode("\n", $out); + } + + if (empty($options['lang'])) { + // If the user hasn't specified "lang=..." explicitly, try to guess the + // language. If we fail, fall back to configured defaults. + $lang = PhutilLanguageGuesser::guessLanguage($text); + if (!$lang) { + $lang = nonempty( + $this->getEngine()->getConfig('phutil.codeblock.language-default'), + 'text'); + } + $options['lang'] = $lang; + } + + $code_body = $this->highlightSource($text, $options); + + $name_header = null; + $block_style = null; + if ($this->getEngine()->isHTMLMailMode()) { + $map = $this->getEngine()->getConfig('phutil.codeblock.style-map'); + + if ($map) { + $raw_body = id(new PhutilPygmentizeParser()) + ->setMap($map) + ->parse((string)$code_body); + $code_body = phutil_safe_html($raw_body); + } + + $style_rules = array( + 'padding: 6px 12px;', + 'font-size: 13px;', + 'font-weight: bold;', + 'display: inline-block;', + 'border-top-left-radius: 3px;', + 'border-top-right-radius: 3px;', + 'color: rgba(0,0,0,.75);', + ); + + if ($options['counterexample']) { + $style_rules[] = 'background: #f7e6e6'; + } else { + $style_rules[] = 'background: rgba(71, 87, 120, 0.08);'; + } + + $header_attributes = array( + 'style' => implode(' ', $style_rules), + ); + + $block_style = 'margin: 12px 0;'; + } else { + $header_attributes = array( + 'class' => 'remarkup-code-header', + ); + } + + if ($options['name']) { + $name_header = phutil_tag( + 'div', + $header_attributes, + $options['name']); + } + + $class = 'remarkup-code-block'; + if ($options['counterexample']) { + $class = 'remarkup-code-block code-block-counterexample'; + } + + $attributes = array( + 'class' => $class, + 'style' => $block_style, + 'data-code-lang' => $options['lang'], + 'data-sigil' => 'remarkup-code-block', + ); + + return phutil_tag( + 'div', + $attributes, + array($name_header, $code_body)); + } + + private function highlightSource($text, array $options) { + if ($options['counterexample']) { + $aux_class = ' remarkup-counterexample'; + } else { + $aux_class = null; + } + + $aux_style = null; + + if ($this->getEngine()->isHTMLMailMode()) { + $aux_style = array( + 'font: 11px/15px "Menlo", "Consolas", "Monaco", monospace;', + 'padding: 12px;', + 'margin: 0;', + ); + + if ($options['counterexample']) { + $aux_style[] = 'background: #f7e6e6;'; + } else { + $aux_style[] = 'background: rgba(71, 87, 120, 0.08);'; + } + + $aux_style = implode(' ', $aux_style); + } + + if ($options['lines']) { + // Put a minimum size on this because the scrollbar is otherwise + // unusable. + $height = max(6, (int)$options['lines']); + $aux_style = $aux_style + .' ' + .'max-height: ' + .(2 * $height) + .'em; overflow: auto;'; + } + + $engine = $this->getEngine()->getConfig('syntax-highlighter.engine'); + if (!$engine) { + $engine = 'PhutilDefaultSyntaxHighlighterEngine'; + } + $engine = newv($engine, array()); + $engine->setConfig( + 'pygments.enabled', + $this->getEngine()->getConfig('pygments.enabled')); + return phutil_tag( + 'pre', + array( + 'class' => 'remarkup-code'.$aux_class, + 'style' => $aux_style, + ), + PhutilSafeHTML::applyFunction( + 'rtrim', + $engine->highlightSource($options['lang'], $text))); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php new file mode 100644 index 0000000000..38de7e16b4 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php @@ -0,0 +1,44 @@ +getEngine(); + + $text = trim($text); + $text = $this->applyRules($text); + + if ($engine->isTextMode()) { + if (!$this->getEngine()->getConfig('preserve-linebreaks')) { + $text = preg_replace('/ *\n */', ' ', $text); + } + return $text; + } + + if ($engine->getConfig('preserve-linebreaks')) { + $text = phutil_escape_html_newlines($text); + } + + if (!strlen($text)) { + return null; + } + + $default_attributes = $engine->getConfig('default.p.attributes'); + if ($default_attributes) { + $attributes = $default_attributes; + } else { + $attributes = array(); + } + + return phutil_tag('p', $attributes, $text); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php new file mode 100644 index 0000000000..cbcb143339 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php @@ -0,0 +1,162 @@ + 1) { + $level = ($lines[1][0] == '=') ? 1 : 2; + $text = trim($lines[0]); + } else { + $level = 0; + for ($ii = 0; $ii < min(5, strlen($text)); $ii++) { + if ($text[$ii] == '=' || $text[$ii] == '#') { + ++$level; + } else { + break; + } + } + $text = trim($text, ' =#'); + } + + $engine = $this->getEngine(); + + if ($engine->isTextMode()) { + $char = ($level == 1) ? '=' : '-'; + return $text."\n".str_repeat($char, phutil_utf8_strlen($text)); + } + + $use_anchors = $engine->getConfig('header.generate-toc'); + + $anchor = null; + if ($use_anchors) { + $anchor = $this->generateAnchor($level, $text); + } + + $text = phutil_tag( + 'h'.($level + 1), + array( + 'class' => 'remarkup-header', + ), + array($anchor, $this->applyRules($text))); + + return $text; + } + + private function generateAnchor($level, $text) { + $anchor = strtolower($text); + $anchor = preg_replace('/[^a-z0-9]/', '-', $anchor); + $anchor = preg_replace('/--+/', '-', $anchor); + $anchor = trim($anchor, '-'); + $anchor = substr($anchor, 0, 24); + $anchor = trim($anchor, '-'); + $base = $anchor; + + $key = self::KEY_HEADER_TOC; + $engine = $this->getEngine(); + $anchors = $engine->getTextMetadata($key, array()); + + $suffix = 1; + while (!strlen($anchor) || isset($anchors[$anchor])) { + $anchor = $base.'-'.$suffix; + $anchor = trim($anchor, '-'); + $suffix++; + } + + // When a document contains a link inside a header, like this: + // + // = [[ http://wwww.example.com/ | example ]] = + // + // ...we want to generate a TOC entry with just "example", but link the + // header itself. We push the 'toc' state so all the link rules generate + // just names. + $engine->pushState('toc'); + $text = $this->applyRules($text); + $text = $engine->restoreText($text); + + $anchors[$anchor] = array($level, $text); + $engine->popState('toc'); + + $engine->setTextMetadata($key, $anchors); + + return phutil_tag( + 'a', + array( + 'name' => $anchor, + ), + ''); + } + + public static function renderTableOfContents(PhutilRemarkupEngine $engine) { + + $key = self::KEY_HEADER_TOC; + $anchors = $engine->getTextMetadata($key, array()); + + if (count($anchors) < 2) { + // Don't generate a TOC if there are no headers, or if there's only + // one header (since such a TOC would be silly). + return null; + } + + $depth = 0; + $toc = array(); + foreach ($anchors as $anchor => $info) { + list($level, $name) = $info; + + while ($depth < $level) { + $toc[] = hsprintf('
    '); + $depth++; + } + while ($depth > $level) { + $toc[] = hsprintf('
'); + $depth--; + } + + $toc[] = phutil_tag( + 'li', + array(), + phutil_tag( + 'a', + array( + 'href' => '#'.$anchor, + ), + $name)); + } + while ($depth > 0) { + $toc[] = hsprintf(''); + $depth--; + } + + return phutil_implode_html("\n", $toc); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php new file mode 100644 index 0000000000..7b815b2846 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php @@ -0,0 +1,37 @@ +getEngine()->isTextMode()) { + return rtrim($text); + } + + return phutil_tag('hr', array('class' => 'remarkup-hr')); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php new file mode 100644 index 0000000000..3a72197341 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php @@ -0,0 +1,13 @@ +applyRules($text); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php new file mode 100644 index 0000000000..a54e6b8b13 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php @@ -0,0 +1,89 @@ +parse($matches[2]); + } + + $interpreters = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhutilRemarkupBlockInterpreter') + ->execute(); + + foreach ($interpreters as $interpreter) { + $interpreter->setEngine($this->getEngine()); + } + + $lines[$first_key] = preg_replace( + self::START_BLOCK_PATTERN, + '', + $lines[$first_key]); + $lines[$last_key] = preg_replace( + self::END_BLOCK_PATTERN, + '', + $lines[$last_key]); + + if (trim($lines[$first_key]) === '') { + unset($lines[$first_key]); + } + if (trim($lines[$last_key]) === '') { + unset($lines[$last_key]); + } + + $content = implode("\n", $lines); + + $interpreters = mpull($interpreters, null, 'getInterpreterName'); + + if (isset($interpreters[$matches[1]])) { + return $interpreters[$matches[1]]->markupContent($content, $argv); + } + + $message = pht('No interpreter found: %s', $matches[1]); + + if ($this->getEngine()->isTextMode()) { + return '('.$message.')'; + } + + return phutil_tag( + 'div', + array( + 'class' => 'remarkup-interpreter-error', + ), + $message); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php new file mode 100644 index 0000000000..5bdcab2b8f --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php @@ -0,0 +1,567 @@ + $line) { + $matches = null; + if (preg_match($regex, $line)) { + $regex = self::CONT_BLOCK_PATTERN; + if (preg_match('/^(\s+)/', $line, $matches)) { + $space = strlen($matches[1]); + } else { + $space = 0; + } + $min_space = min($min_space, $space); + } + } + + $regex = self::START_BLOCK_PATTERN; + if ($min_space) { + foreach ($lines as $key => $line) { + if (preg_match($regex, $line)) { + $regex = self::CONT_BLOCK_PATTERN; + $lines[$key] = substr($line, $min_space); + } + } + } + + + // The input text may have linewraps in it, like this: + // + // - derp derp derp derp + // derp derp derp derp + // - blarp blarp blarp blarp + // + // Group text lines together into list items, stored in $items. So the + // result in the above case will be: + // + // array( + // array( + // "- derp derp derp derp", + // " derp derp derp derp", + // ), + // array( + // "- blarp blarp blarp blarp", + // ), + // ); + + $item = array(); + $starts_at = null; + $regex = self::START_BLOCK_PATTERN; + foreach ($lines as $line) { + $match = null; + if (preg_match($regex, $line, $match)) { + if (!$starts_at && !empty($match[1])) { + $starts_at = $match[1]; + } + $regex = self::CONT_BLOCK_PATTERN; + if ($item) { + $items[] = $item; + $item = array(); + } + } + $item[] = $line; + } + if ($item) { + $items[] = $item; + } + if (!$starts_at) { + $starts_at = 1; + } + + + // Process each item to normalize the text, remove line wrapping, and + // determine its depth (indentation level) and style (ordered vs unordered). + // + // We preserve consecutive linebreaks and interpret them as paragraph + // breaks. + // + // Given the above example, the processed array will look like: + // + // array( + // array( + // 'text' => 'derp derp derp derp derp derp derp derp', + // 'depth' => 0, + // 'style' => '-', + // ), + // array( + // 'text' => 'blarp blarp blarp blarp', + // 'depth' => 0, + // 'style' => '-', + // ), + // ); + + $has_marks = false; + foreach ($items as $key => $item) { + // Trim space around newlines, to strip trailing whitespace and formatting + // indentation. + $item = preg_replace('/ *(\n+) */', '\1', implode("\n", $item)); + + // Replace single newlines with a space. Preserve multiple newlines as + // paragraph breaks. + $item = preg_replace('/(? $text, + 'depth' => $depth, + 'style' => $style, + 'mark' => $mark, + ); + } + $items = array_values($items); + + + // Users can create a sub-list by indenting any deeper amount than the + // previous list, so these are both valid: + // + // - a + // - b + // + // - a + // - b + // + // In the former case, we'll have depths (0, 2). In the latter case, depths + // (0, 4). We don't actually care about how many spaces there are, only + // how many list indentation levels (that is, we want to map both of + // those cases to (0, 1), indicating "outermost list" and "first sublist"). + // + // This is made more complicated because lists at two different indentation + // levels might be at the same list level: + // + // - a + // - b + // - c + // - d + // + // Here, 'b' and 'd' are at the same list level (2) but different indent + // levels (2, 4). + // + // Users can also create "staircases" like this: + // + // - a + // - b + // # c + // + // While this is silly, we'd like to render it as faithfully as possible. + // + // In order to do this, we convert the list of nodes into a tree, + // normalizing indentation levels and inserting dummy nodes as necessary to + // make the tree well-formed. See additional notes at buildTree(). + // + // In the case above, the result is a tree like this: + // + // - + // - + // - a + // - b + // # c + + $l = 0; + $r = count($items); + $tree = $this->buildTree($items, $l, $r, $cur_level = 0); + + + // We may need to open a list on a node, but they do not have + // list style information yet. We need to propagate list style information + // backward through the tree. In the above example, the tree now looks + // like this: + // + // - + // - + // - a + // - b + // # c + + $this->adjustTreeStyleInformation($tree); + + // Finally, we have enough information to render the tree. + + $out = $this->renderTree($tree, 0, $has_marks, $starts_at); + + if ($this->getEngine()->isTextMode()) { + $out = implode('', $out); + $out = rtrim($out, "\n"); + $out = preg_replace('/ +$/m', '', $out); + return $out; + } + + return phutil_implode_html('', $out); + } + + /** + * See additional notes in @{method:markupText}. + */ + private function buildTree(array $items, $l, $r, $cur_level) { + if ($l == $r) { + return array(); + } + + if ($cur_level > self::MAXIMUM_LIST_NESTING_DEPTH) { + // This algorithm is recursive and we don't need you blowing the stack + // with your oh-so-clever 50,000-item-deep list. Cap indentation levels + // at a reasonable number and just shove everything deeper up to this + // level. + $nodes = array(); + for ($ii = $l; $ii < $r; $ii++) { + $nodes[] = array( + 'level' => $cur_level, + 'items' => array(), + ) + $items[$ii]; + } + return $nodes; + } + + $min = $l; + for ($ii = $r - 1; $ii >= $l; $ii--) { + if ($items[$ii]['depth'] <= $items[$min]['depth']) { + $min = $ii; + } + } + + $min_depth = $items[$min]['depth']; + + $nodes = array(); + if ($min != $l) { + $nodes[] = array( + 'text' => null, + 'level' => $cur_level, + 'style' => null, + 'mark' => null, + 'items' => $this->buildTree($items, $l, $min, $cur_level + 1), + ); + } + + $last = $min; + for ($ii = $last + 1; $ii < $r; $ii++) { + if ($items[$ii]['depth'] == $min_depth) { + $nodes[] = array( + 'level' => $cur_level, + 'items' => $this->buildTree($items, $last + 1, $ii, $cur_level + 1), + ) + $items[$last]; + $last = $ii; + } + } + $nodes[] = array( + 'level' => $cur_level, + 'items' => $this->buildTree($items, $last + 1, $r, $cur_level + 1), + ) + $items[$last]; + + return $nodes; + } + + + /** + * See additional notes in @{method:markupText}. + */ + private function adjustTreeStyleInformation(array &$tree) { + // The effect here is just to walk backward through the nodes at this level + // and apply the first style in the list to any empty nodes we inserted + // before it. As we go, also recurse down the tree. + + $style = '-'; + for ($ii = count($tree) - 1; $ii >= 0; $ii--) { + if ($tree[$ii]['style'] !== null) { + // This is the earliest node we've seen with style, so set the + // style to its style. + $style = $tree[$ii]['style']; + } else { + // This node has no style, so apply the current style. + $tree[$ii]['style'] = $style; + } + if ($tree[$ii]['items']) { + $this->adjustTreeStyleInformation($tree[$ii]['items']); + } + } + } + + + /** + * See additional notes in @{method:markupText}. + */ + private function renderTree( + array $tree, + $level, + $has_marks, + $starts_at = 1) { + + $style = idx(head($tree), 'style'); + + $out = array(); + + if (!$this->getEngine()->isTextMode()) { + switch ($style) { + case '#': + $tag = 'ol'; + break; + case '-': + $tag = 'ul'; + break; + } + + $start_attr = null; + if (ctype_digit($starts_at) && $starts_at > 1) { + $start_attr = hsprintf(' start="%d"', $starts_at); + } + + if ($has_marks) { + $out[] = hsprintf( + '<%s class="remarkup-list remarkup-list-with-checkmarks"%s>', + $tag, + $start_attr); + } else { + $out[] = hsprintf( + '<%s class="remarkup-list"%s>', + $tag, + $start_attr); + } + + $out[] = "\n"; + } + + $number = $starts_at; + foreach ($tree as $item) { + if ($this->getEngine()->isTextMode()) { + if ($item['text'] === null) { + // Don't render anything. + } else { + $indent = str_repeat(' ', 2 * $level); + $out[] = $indent; + if ($item['mark'] !== null) { + if ($item['mark']) { + $out[] = '[X] '; + } else { + $out[] = '[ ] '; + } + } else { + switch ($style) { + case '#': + $out[] = $number.'. '; + $number++; + break; + case '-': + $out[] = '- '; + break; + } + } + + $parts = preg_split('/\n{2,}/', $item['text']); + foreach ($parts as $key => $part) { + if ($key != 0) { + $out[] = "\n\n ".$indent; + } + $out[] = $this->applyRules($part); + } + $out[] = "\n"; + } + } else { + if ($item['text'] === null) { + $out[] = hsprintf('
  • '); + } else { + if ($item['mark'] !== null) { + if ($item['mark'] == true) { + $out[] = hsprintf( + '
  • '); + } else { + $out[] = hsprintf( + '
  • '); + } + $out[] = phutil_tag( + 'input', + array( + 'type' => 'checkbox', + 'checked' => ($item['mark'] ? 'checked' : null), + 'disabled' => 'disabled', + )); + $out[] = ' '; + } else { + $out[] = hsprintf('
  • '); + } + + $parts = preg_split('/\n{2,}/', $item['text']); + foreach ($parts as $key => $part) { + if ($key != 0) { + $out[] = array( + "\n", + phutil_tag('br'), + phutil_tag('br'), + "\n", + ); + } + $out[] = $this->applyRules($part); + } + } + } + + if ($item['items']) { + $subitems = $this->renderTree($item['items'], $level + 1, $has_marks); + foreach ($subitems as $i) { + $out[] = $i; + } + } + if (!$this->getEngine()->isTextMode()) { + $out[] = hsprintf("
    • \n"); + } + } + + if (!$this->getEngine()->isTextMode()) { + switch ($style) { + case '#': + $out[] = hsprintf(''); + break; + case '-': + $out[] = hsprintf(''); + break; + } + } + + return $out; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php new file mode 100644 index 0000000000..527db4fb61 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php @@ -0,0 +1,93 @@ + $line) { + $line = preg_replace('/^\s*%%%/', '', $line); + $line = preg_replace('/%%%(\s*)\z/', '\1', $line); + $text[$key] = $line; + } + + if ($this->getEngine()->isTextMode()) { + return implode('', $text); + } + + return phutil_tag( + 'p', + array( + 'class' => 'remarkup-literal', + ), + phutil_implode_html(phutil_tag('br', array()), $text)); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php new file mode 100644 index 0000000000..e80f3e1953 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php @@ -0,0 +1,121 @@ +getRegEx(), $lines[$cursor])) { + $num_lines++; + $cursor++; + + while (isset($lines[$cursor])) { + if (trim($lines[$cursor])) { + $num_lines++; + $cursor++; + continue; + } + break; + } + } + + return $num_lines; + } + + public function markupText($text, $children) { + $matches = array(); + preg_match($this->getRegEx(), $text, $matches); + + if (idx($matches, 'showword')) { + $word = $matches['showword']; + $show = true; + } else { + $word = $matches['hideword']; + $show = false; + } + + $class_suffix = phutil_utf8_strtolower($word); + + // This is the "(IMPORTANT)" or "NOTE:" part. + $word_part = rtrim(substr($text, 0, strlen($matches[0]))); + + // This is the actual text. + $text_part = substr($text, strlen($matches[0])); + $text_part = $this->applyRules(rtrim($text_part)); + + $text_mode = $this->getEngine()->isTextMode(); + $html_mail_mode = $this->getEngine()->isHTMLMailMode(); + if ($text_mode) { + return $word_part.' '.$text_part; + } + + if ($show) { + $content = array( + phutil_tag( + 'span', + array( + 'class' => 'remarkup-note-word', + ), + $word_part), + ' ', + $text_part, + ); + } else { + $content = $text_part; + } + + if ($html_mail_mode) { + if ($class_suffix == 'important') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #c0392b; + background: #f4dddb;', + ); + } else if ($class_suffix == 'note') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #2980b9; + background: #daeaf3;', + ); + } else if ($class_suffix == 'warning') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #f1c40f; + background: #fdf5d4;', + ); + } + } else { + $attributes = array( + 'class' => 'remarkup-'.$class_suffix, + ); + } + + return phutil_tag( + 'div', + $attributes, + $content); + } + + private function getRegEx() { + $words = array( + 'NOTE', + 'IMPORTANT', + 'WARNING', + ); + + foreach ($words as $k => $word) { + $words[$k] = preg_quote($word, '/'); + } + $words = implode('|', $words); + + return + '/^(?:'. + '(?:\((?P'.$words.')\))'. + '|'. + '(?:(?P'.$words.'):))\s*'. + '/'; + } +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php new file mode 100644 index 0000000000..ed87f7063d --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php @@ -0,0 +1,108 @@ + $line) { + $text[$key] = substr($line, 1); + } + + // If every line in the block is empty or begins with at least one leading + // space, strip the initial space off each line. When we quote text, we + // normally add "> " (with a space) to the beginning of each line, which + // can disrupt some other rules. If the block appears to have this space + // in front of each line, remove it. + + $strip_space = true; + foreach ($text as $key => $line) { + $len = strlen($line); + + if (!$len) { + // We'll still strip spaces if there are some completely empty + // lines, they may have just had trailing whitespace trimmed. + continue; + } + + // If this line is part of a nested quote block, just ignore it when + // realigning this quote block. It's either an author attribution + // line with ">>!", or we'll deal with it in a subrule when processing + // the nested quote block. + if ($line[0] == '>') { + continue; + } + + if ($line[0] == ' ' || $line[0] == "\n") { + continue; + } + + // The first character of this line is something other than a space, so + // we can't strip spaces. + $strip_space = false; + break; + } + + if ($strip_space) { + foreach ($text as $key => $line) { + $len = strlen($line); + if (!$len) { + continue; + } + + if ($line[0] !== ' ') { + continue; + } + + $text[$key] = substr($line, 1); + } + } + + // Strip leading empty lines. + foreach ($text as $key => $line) { + if (!strlen(trim($line))) { + unset($text[$key]); + } + } + + return implode('', $text); + } + + final protected function getQuotedText($text) { + $text = rtrim($text, "\n"); + + $no_whitespace = array( + // For readability, we render nested quotes as ">> quack", + // not "> > quack". + '>' => true, + + // If the line is empty except for a newline, do not add an + // unnecessary dangling space. + "\n" => true, + ); + + $text = phutil_split_lines($text, true); + foreach ($text as $key => $line) { + $c = null; + if (isset($line[0])) { + $c = $line[0]; + } else { + $c = null; + } + + if (isset($no_whitespace[$c])) { + $text[$key] = '>'.$line; + } else { + $text[$key] = '> '.$line; + } + } + $text = implode('', $text); + + return $text; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php new file mode 100644 index 0000000000..f5ff9ef815 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php @@ -0,0 +1,47 @@ +/', $lines[$pos])) { + do { + ++$pos; + } while (isset($lines[$pos]) && preg_match('/^>/', $lines[$pos])); + } + + return ($pos - $cursor); + } + + public function extractChildText($text) { + return array('', $this->normalizeQuotedBody($text)); + } + + public function markupText($text, $children) { + if ($this->getEngine()->isTextMode()) { + return $this->getQuotedText($children); + } + + $attributes = array(); + if ($this->getEngine()->isHTMLMailMode()) { + $style = array( + 'border-left: 3px solid #a7b5bf;', + 'color: #464c5c;', + 'font-style: italic;', + 'margin: 4px 0 12px 0;', + 'padding: 4px 12px;', + 'background-color: #f8f9fc;', + ); + + $attributes['style'] = implode(' ', $style); + } + + return phutil_tag( + 'blockquote', + $attributes, + $children); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php new file mode 100644 index 0000000000..584fca1349 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php @@ -0,0 +1,91 @@ +>!/', $lines[$pos])) { + do { + ++$pos; + } while (isset($lines[$pos]) && preg_match('/^>/', $lines[$pos])); + } + + return ($pos - $cursor); + } + + public function extractChildText($text) { + $text = phutil_split_lines($text, true); + + $head = substr(reset($text), 3); + + $body = array_slice($text, 1); + $body = implode('', $body); + $body = $this->normalizeQuotedBody($body); + + return array(trim($head), $body); + } + + public function markupText($text, $children) { + $text = $this->applyRules($text); + + if ($this->getEngine()->isTextMode()) { + $children = $this->getQuotedText($children); + return $text."\n\n".$children; + } + + if ($this->getEngine()->isHTMLMailMode()) { + $block_attributes = array( + 'style' => 'border-left: 3px solid #8C98B8; + color: #6B748C; + font-style: italic; + margin: 4px 0 12px 0; + padding: 8px 12px; + background-color: #F8F9FC;', + ); + $head_attributes = array( + 'style' => 'font-style: normal; + padding-bottom: 4px;', + ); + $reply_attributes = array( + 'style' => 'margin: 0; + padding: 0; + border: 0; + color: rgb(107, 116, 140);', + ); + } else { + $block_attributes = array( + 'class' => 'remarkup-reply-block', + ); + $head_attributes = array( + 'class' => 'remarkup-reply-head', + ); + $reply_attributes = array( + 'class' => 'remarkup-reply-body', + ); + } + + return phutil_tag( + 'blockquote', + $block_attributes, + array( + "\n", + phutil_tag( + 'div', + $head_attributes, + $text), + "\n", + phutil_tag( + 'div', + $reply_attributes, + $children), + "\n", + )); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php new file mode 100644 index 0000000000..72aae08d12 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php @@ -0,0 +1,96 @@ + cells + // instead of cells. + + // If it has other types of cells, it's always a content row. + + // If it has only empty cells, it's an empty row. + + if (strlen($cell)) { + if (preg_match('/^--+\z/', $cell)) { + $any_header = true; + } else { + $any_content = true; + } + } + + $cells[] = array('type' => 'td', 'content' => $this->applyRules($cell)); + } + + $is_header = ($any_header && !$any_content); + + if (!$is_header) { + $rows[] = array('type' => 'tr', 'content' => $cells); + } else if ($rows) { + // Mark previous row with headings. + foreach ($cells as $i => $cell) { + if ($cell['content']) { + $last_key = last_key($rows); + if (!isset($rows[$last_key]['content'][$i])) { + // If this row has more cells than the previous row, there may + // not be a cell above this one to turn into a . + continue; + } + + $rows[$last_key]['content'][$i]['type'] = 'th'; + } + } + } + } + + if (!$rows) { + return $this->applyRules($text); + } + + return $this->renderRemarkupTable($rows); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php new file mode 100644 index 0000000000..57f2fe1f1d --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php @@ -0,0 +1,142 @@ +/i', $lines[$cursor])) { + $num_lines++; + $cursor++; + + while (isset($lines[$cursor])) { + $num_lines++; + if (preg_match('@\s*$@i', $lines[$cursor])) { + break; + } + $cursor++; + } + } + + return $num_lines; + } + + public function markupText($text, $children) { + $root = id(new PhutilHTMLParser()) + ->parseDocument($text); + + $nodes = $root->selectChildrenWithTags(array('table')); + + $out = array(); + $seen_table = false; + foreach ($nodes as $node) { + if ($node->isContentNode()) { + $content = $node->getContent(); + + if (!strlen(trim($content))) { + // Ignore whitespace. + continue; + } + + // If we find other content, fail the rule. This can happen if the + // input is two consecutive table tags on one line with some text + // in between them, which we currently forbid. + return $text; + } else { + // If we have multiple table tags, just return the raw text. + if ($seen_table) { + return $text; + } + $seen_table = true; + + $out[] = $this->newTable($node); + } + } + + if ($this->getEngine()->isTextMode()) { + return implode('', $out); + } else { + return phutil_implode_html('', $out); + } + } + + private function newTable(PhutilDOMNode $table) { + $nodes = $table->selectChildrenWithTags( + array( + 'colgroup', + 'tr', + )); + + $colgroup = null; + $rows = array(); + + foreach ($nodes as $node) { + if ($node->isContentNode()) { + $content = $node->getContent(); + + // If this is whitespace, ignore it. + if (!strlen(trim($content))) { + continue; + } + + // If we have nonempty content between the rows, this isn't a valid + // table. We can't really do anything reasonable with this, so just + // fail out and render the raw text. + return $table->newRawString(); + } + + if ($node->getTagName() === 'colgroup') { + // This table has multiple "" tags. Just bail out. + if ($colgroup !== null) { + return $table->newRawString(); + } + + // This table has a "" after a "". We could parse + // this, but just reject it out of an abundance of caution. + if ($rows) { + return $table->newRawString(); + } + + $colgroup = $node; + continue; + } + + $rows[] = $node; + } + + $row_specs = array(); + + foreach ($rows as $row) { + $cells = $row->selectChildrenWithTags(array('td', 'th')); + + $cell_specs = array(); + foreach ($cells as $cell) { + if ($cell->isContentNode()) { + $content = $node->getContent(); + + if (!strlen(trim($content))) { + continue; + } + + return $table->newRawString(); + } + + $content = $cell->newRawContentString(); + $content = $this->applyRules($content); + + $cell_specs[] = array( + 'type' => $cell->getTagName(), + 'content' => $content, + ); + } + + $row_specs[] = array( + 'type' => 'tr', + 'content' => $cell_specs, + ); + } + + return $this->renderRemarkupTable($row_specs); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php new file mode 100644 index 0000000000..ca2b9b5ae7 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php @@ -0,0 +1,17 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@\\*\\*(.+?)\\*\\*@s', + array($this, 'applyCallback'), + $text); + } + + protected function applyCallback(array $matches) { + return hsprintf('%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php new file mode 100644 index 0000000000..82f23d2d59 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php new file mode 100644 index 0000000000..a6effa00ac --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php @@ -0,0 +1,175 @@ +getEngine(); + + $is_anchor = false; + if (strncmp($link, '/', 1) == 0) { + $base = $engine->getConfig('uri.base'); + $base = rtrim($base, '/'); + $link = $base.$link; + } else if (strncmp($link, '#', 1) == 0) { + $here = $engine->getConfig('uri.here'); + $link = $here.$link; + + $is_anchor = true; + } + + if ($engine->isTextMode()) { + // If present, strip off "mailto:" or "tel:". + $link = preg_replace('/^(?:mailto|tel):/', '', $link); + + if (!strlen($name)) { + return $link; + } + + return $name.' <'.$link.'>'; + } + + if (!strlen($name)) { + $name = $link; + $name = preg_replace('/^(?:mailto|tel):/', '', $name); + } + + if ($engine->getState('toc')) { + return $name; + } + + $same_window = $engine->getConfig('uri.same-window', false); + if ($same_window) { + $target = null; + } else { + $target = '_blank'; + } + + // For anchors on the same page, always stay here. + if ($is_anchor) { + $target = null; + } + + return phutil_tag( + 'a', + array( + 'href' => $link, + 'class' => 'remarkup-link', + 'target' => $target, + 'rel' => 'noreferrer', + ), + $name); + } + + public function markupAlternateLink(array $matches) { + $uri = trim($matches[2]); + + if (!strlen($uri)) { + return $matches[0]; + } + + // NOTE: We apply some special rules to avoid false positives here. The + // major concern is that we do not want to convert `x[0][1](y)` in a + // discussion about C source code into a link. To this end, we: + // + // - Don't match at word boundaries; + // - require the URI to contain a "/" character or "@" character; and + // - reject URIs which being with a quote character. + + if ($uri[0] == '"' || $uri[0] == "'" || $uri[0] == '`') { + return $matches[0]; + } + + if (strpos($uri, '/') === false && + strpos($uri, '@') === false && + strncmp($uri, 'tel:', 4)) { + return $matches[0]; + } + + return $this->markupDocumentLink( + array( + $matches[0], + $matches[2], + $matches[1], + )); + } + + public function markupDocumentLink(array $matches) { + $uri = trim($matches[1]); + $name = trim(idx($matches, 2)); + + // If whatever is being linked to begins with "/" or "#", or has "://", + // or is "mailto:" or "tel:", treat it as a URI instead of a wiki page. + $is_uri = preg_match('@(^/)|(://)|(^#)|(^(?:mailto|tel):)@', $uri); + + if ($is_uri && strncmp('/', $uri, 1) && strncmp('#', $uri, 1)) { + $protocols = $this->getEngine()->getConfig( + 'uri.allowed-protocols', + array()); + + try { + $protocol = id(new PhutilURI($uri))->getProtocol(); + if (!idx($protocols, $protocol)) { + // Don't treat this as a URI if it's not an allowed protocol. + $is_uri = false; + } + } catch (Exception $ex) { + // We can end up here if we try to parse an ambiguous URI, see + // T12796. + $is_uri = false; + } + } + + // As a special case, skip "[[ / ]]" so that Phriction picks it up as a + // link to the Phriction root. It is more useful to be able to use this + // syntax to link to the root document than the home page of the install. + if ($uri == '/') { + $is_uri = false; + } + + if (!$is_uri) { + return $matches[0]; + } + + return $this->getEngine()->storeText($this->renderHyperlink($uri, $name)); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php new file mode 100644 index 0000000000..d9f56e2375 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php @@ -0,0 +1,19 @@ +getEngine()->storeText("\1"); + + return str_replace("\1", $replace, $text); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php new file mode 100644 index 0000000000..900e355cd4 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php @@ -0,0 +1,37 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@!!(.+?)(!{2,})@', + array($this, 'applyCallback'), + $text); + } + + protected function applyCallback(array $matches) { + // Remove the two exclamation points that represent syntax. + $excitement = substr($matches[2], 2); + + // If the internal content consists of ONLY exclamation points, leave it + // untouched so "!!!!!" is five exclamation points instead of one + // highlighted exclamation point. + if (preg_match('/^!+\z/', $matches[1])) { + return $matches[0]; + } + + // $excitement now has two fewer !'s than we started with. + return hsprintf('%s%s', + $matches[1], $excitement); + + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php new file mode 100644 index 0000000000..681ec46a09 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php @@ -0,0 +1,30 @@ +getPhobjectClassConstant('LINKENGINEKEY', 32); + } + + final public static function getAllLinkEngines() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getHyperlinkEngineKey') + ->execute(); + } + + final public function setEngine(PhutilRemarkupEngine $engine) { + $this->engine = $engine; + return $this; + } + + final public function getEngine() { + return $this->engine; + } + + abstract public function processHyperlinks(array $hyperlinks); + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php new file mode 100644 index 0000000000..2c81f0bb7b --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php @@ -0,0 +1,38 @@ +token = $map['token']; + $this->uri = $map['uri']; + $this->embed = ($map['mode'] === '{'); + } + + public function getToken() { + return $this->token; + } + + public function getURI() { + return $this->uri; + } + + public function isEmbed() { + return $this->embed; + } + + public function setResult($result) { + $this->result = $result; + return $this; + } + + public function getResult() { + return $this->result; + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php new file mode 100644 index 0000000000..a926ea44c1 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php @@ -0,0 +1,234 @@ +" around them get linked exactly, without + // the "<>". Angle brackets are basically special and mean "this is a URL + // with weird characters". This is assumed to be reasonable because they + // don't appear in normal text or normal URLs. + $text = preg_replace_callback( + '@<(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+?)>@', + array($this, 'markupHyperlinkAngle'), + $text); + + // We match "{uri}", but do not link it by default. + $text = preg_replace_callback( + '@{(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+?)}@', + array($this, 'markupHyperlinkCurly'), + $text); + + // Anything else we match "ungreedily", which means we'll look for + // stuff that's probably puncutation or otherwise not part of the URL and + // not link it. This lets someone write "QuicK! Go to + // http://www.example.com/!". We also apply some paren balancing rules. + + // NOTE: We're explicitly avoiding capturing stored blocks, so text like + // `http://www.example.com/[[x | y]]` doesn't get aggressively captured. + $text = preg_replace_callback( + '@(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+)@', + array($this, 'markupHyperlinkUngreedy'), + $text); + + return $text; + } + + public function markupHyperlinkAngle(array $matches) { + return $this->markupHyperlink('<', $matches); + } + + public function markupHyperlinkCurly(array $matches) { + return $this->markupHyperlink('{', $matches); + } + + protected function markupHyperlink($mode, array $matches) { + $raw_uri = $matches[1]; + + try { + $uri = new PhutilURI($raw_uri); + } catch (Exception $ex) { + return $matches[0]; + } + + $engine = $this->getEngine(); + + $token = $engine->storeText($raw_uri); + + $list_key = self::KEY_HYPERLINKS; + $link_list = $engine->getTextMetadata($list_key, array()); + + $link_list[] = array( + 'token' => $token, + 'uri' => $raw_uri, + 'mode' => $mode, + ); + + $engine->setTextMetadata($list_key, $link_list); + + return $token; + } + + protected function renderHyperlink($link, $is_embed) { + // If the URI is "{uri}" and no handler picked it up, we just render it + // as plain text. + if ($is_embed) { + return $this->renderRawLink($link, $is_embed); + } + + $engine = $this->getEngine(); + + $same_window = $engine->getConfig('uri.same-window', false); + if ($same_window) { + $target = null; + } else { + $target = '_blank'; + } + + return phutil_tag( + 'a', + array( + 'href' => $link, + 'class' => 'remarkup-link', + 'target' => $target, + 'rel' => 'noreferrer', + ), + $link); + } + + private function renderRawLink($link, $is_embed) { + if ($is_embed) { + return '{'.$link.'}'; + } else { + return $link; + } + } + + protected function markupHyperlinkUngreedy($matches) { + $match = $matches[1]; + $tail = null; + $trailing = null; + if (preg_match('/[;,.:!?]+$/', $match, $trailing)) { + $tail = $trailing[0]; + $match = substr($match, 0, -strlen($tail)); + } + + // If there's a closing paren at the end but no balancing open paren in + // the URL, don't link the close paren. This is an attempt to gracefully + // handle the two common paren cases, Wikipedia links and English language + // parentheticals, e.g.: + // + // http://en.wikipedia.org/wiki/Noun_(disambiguation) + // (see also http://www.example.com) + // + // We could apply a craftier heuristic here which tries to actually balance + // the parens, but this is probably sufficient. + if (preg_match('/\\)$/', $match) && !preg_match('/\\(/', $match)) { + $tail = ')'.$tail; + $match = substr($match, 0, -1); + } + + try { + $uri = new PhutilURI($match); + } catch (Exception $ex) { + return $matches[0]; + } + + $link = $this->markupHyperlink(null, array(null, $match)); + + return hsprintf('%s%s', $link, $tail); + } + + public function didMarkupText() { + $engine = $this->getEngine(); + + $protocols = $engine->getConfig('uri.allowed-protocols', array()); + $is_toc = $engine->getState('toc'); + $is_text = $engine->isTextMode(); + $is_mail = $engine->isHTMLMailMode(); + + $list_key = self::KEY_HYPERLINKS; + $raw_list = $engine->getTextMetadata($list_key, array()); + + $links = array(); + foreach ($raw_list as $key => $link) { + $token = $link['token']; + $raw_uri = $link['uri']; + $mode = $link['mode']; + + $is_embed = ($mode === '{'); + $is_literal = ($mode === '<'); + + // If we're rendering in a "Table of Contents" or a plain text mode, + // we're going to render the raw URI without modifications. + if ($is_toc || $is_text) { + $result = $this->renderRawLink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // If this URI doesn't use a whitelisted protocol, don't link it. This + // is primarily intended to prevent "javascript://" silliness. + $uri = new PhutilURI($raw_uri); + $protocol = $uri->getProtocol(); + $valid_protocol = idx($protocols, $protocol); + if (!$valid_protocol) { + $result = $this->renderRawLink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // If the URI is written as "", we'll render it literally even if + // some handler would otherwise deal with it. + // If we're rendering for HTML mail, we also render literally. + if ($is_literal || $is_mail) { + $result = $this->renderHyperlink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // Otherwise, this link is a valid resource which extensions are allowed + // to handle. + $links[$key] = $link; + } + + if (!$links) { + return; + } + + foreach ($links as $key => $link) { + $links[$key] = new PhutilRemarkupHyperlinkRef($link); + } + + $extensions = PhutilRemarkupHyperlinkEngineExtension::getAllLinkEngines(); + foreach ($extensions as $extension) { + $extension = id(clone $extension) + ->setEngine($engine) + ->processHyperlinks($links); + + foreach ($links as $key => $link) { + $result = $link->getResult(); + if ($result !== null) { + $engine->overwriteStoredText($link->getToken(), $result); + unset($links[$key]); + } + } + + if (!$links) { + break; + } + } + + // Render any remaining links in a normal way. + foreach ($links as $link) { + $result = $this->renderHyperlink($link->getURI(), $link->isEmbed()); + $engine->overwriteStoredText($link->getToken(), $result); + } + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php new file mode 100644 index 0000000000..9eefe2a7a2 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php new file mode 100644 index 0000000000..2db33d9516 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php @@ -0,0 +1,13 @@ +getEngine()->isTextMode()) { + return $text; + } + + return phutil_escape_html_newlines($text); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php new file mode 100644 index 0000000000..cd5ab8ad0e --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php @@ -0,0 +1,49 @@ +getEngine()->isTextMode()) { + $result = $matches[0]; + + } else + if ($this->getEngine()->isHTMLMailMode()) { + $match = isset($matches[2]) ? $matches[2] : $matches[1]; + $result = phutil_tag( + 'tt', + array( + 'style' => 'background: #ebebeb; font-size: 13px;', + ), + $match); + + } else { + $match = isset($matches[2]) ? $matches[2] : $matches[1]; + $result = phutil_tag( + 'tt', + array( + 'class' => 'remarkup-monospaced', + ), + $match); + } + + return $this->getEngine()->storeText($result); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php new file mode 100644 index 0000000000..a0ef7dc3ec --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php @@ -0,0 +1,109 @@ +engine = $engine; + return $this; + } + + public function getEngine() { + return $this->engine; + } + + public function getPriority() { + return 500.0; + } + + abstract public function apply($text); + + public function getPostprocessKey() { + return spl_object_hash($this); + } + + public function didMarkupText() { + return; + } + + protected function replaceHTML($pattern, $callback, $text) { + $this->replaceCallback = $callback; + return phutil_safe_html(preg_replace_callback( + $pattern, + array($this, 'replaceHTMLCallback'), + phutil_escape_html($text))); + } + + private function replaceHTMLCallback(array $match) { + return phutil_escape_html(call_user_func( + $this->replaceCallback, + array_map('phutil_safe_html', $match))); + } + + + /** + * Safely generate a tag. + * + * In Remarkup contexts, it's not safe to use arbitrary text in tag + * attributes: even though it will be escaped, it may contain replacement + * tokens which are then replaced with markup. + * + * This method acts as @{function:phutil_tag}, but checks attributes before + * using them. + * + * @param string Tag name. + * @param dict Tag attributes. + * @param wild Tag content. + * @return PhutilSafeHTML Tag object. + */ + protected function newTag($name, array $attrs, $content = null) { + foreach ($attrs as $key => $attr) { + if ($attr !== null) { + $attrs[$key] = $this->assertFlatText($attr); + } + } + + return phutil_tag($name, $attrs, $content); + } + + /** + * Assert that a text token is flat (it contains no replacement tokens). + * + * Because tokens can be replaced with markup, it is dangerous to use + * arbitrary input text in tag attributes. Normally, rule precedence should + * prevent this. Asserting that text is flat before using it as an attribute + * provides an extra layer of security. + * + * Normally, you can call @{method:newTag} rather than calling this method + * directly. @{method:newTag} will check attributes for you. + * + * @param wild Ostensibly flat text. + * @return string Flat text. + */ + protected function assertFlatText($text) { + $text = (string)hsprintf('%s', phutil_safe_html($text)); + $rich = (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) !== false); + if ($rich) { + throw new Exception( + pht( + 'Remarkup rule precedence is dangerous: rendering text with tokens '. + 'as flat text!')); + } + + return $text; + } + + /** + * Check whether text is flat (contains no replacement tokens) or not. + * + * @param wild Ostensibly flat text. + * @return bool True if the text is flat. + */ + protected function isFlatText($text) { + $text = (string)hsprintf('%s', phutil_safe_html($text)); + return (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) === false); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php new file mode 100644 index 0000000000..1f1572863b --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php b/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php new file mode 100644 index 0000000000..1c5ff78607 --- /dev/null +++ b/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php @@ -0,0 +1,302 @@ +config[$key] = $value; + return $this; + } + + public function getConfig($key, $default = null) { + return idx($this->config, $key, $default); + } + + public function setMode($mode) { + $this->mode = $mode; + return $this; + } + + public function isTextMode() { + return $this->mode & self::MODE_TEXT; + } + + public function isHTMLMailMode() { + return $this->mode & self::MODE_HTML_MAIL; + } + + public function setBlockRules(array $rules) { + assert_instances_of($rules, 'PhutilRemarkupBlockRule'); + + $rules = msortv($rules, 'getPriorityVector'); + + $this->blockRules = $rules; + foreach ($this->blockRules as $rule) { + $rule->setEngine($this); + } + + $post_rules = array(); + foreach ($this->blockRules as $block_rule) { + foreach ($block_rule->getMarkupRules() as $rule) { + $key = $rule->getPostprocessKey(); + if ($key !== null) { + $post_rules[$key] = $rule; + } + } + } + + $this->postprocessRules = $post_rules; + + return $this; + } + + public function getTextMetadata($key, $default = null) { + if (isset($this->metadata[$key])) { + return $this->metadata[$key]; + } + return idx($this->metadata, $key, $default); + } + + public function setTextMetadata($key, $value) { + $this->metadata[$key] = $value; + return $this; + } + + public function storeText($text) { + if ($this->isTextMode()) { + $text = phutil_safe_html($text); + } + return $this->storage->store($text); + } + + public function overwriteStoredText($token, $new_text) { + if ($this->isTextMode()) { + $new_text = phutil_safe_html($new_text); + } + $this->storage->overwrite($token, $new_text); + return $this; + } + + public function markupText($text) { + return $this->postprocessText($this->preprocessText($text)); + } + + public function pushState($state) { + if (empty($this->states[$state])) { + $this->states[$state] = 0; + } + $this->states[$state]++; + return $this; + } + + public function popState($state) { + if (empty($this->states[$state])) { + throw new Exception(pht("State '%s' pushed more than popped!", $state)); + } + $this->states[$state]--; + if (!$this->states[$state]) { + unset($this->states[$state]); + } + return $this; + } + + public function getState($state) { + return !empty($this->states[$state]); + } + + public function preprocessText($text) { + $this->metadata = array(); + $this->storage = new PhutilRemarkupBlockStorage(); + + $blocks = $this->splitTextIntoBlocks($text); + + $output = array(); + foreach ($blocks as $block) { + $output[] = $this->markupBlock($block); + } + $output = $this->flattenOutput($output); + + $map = $this->storage->getMap(); + $this->storage = null; + $metadata = $this->metadata; + + + return array( + 'output' => $output, + 'storage' => $map, + 'metadata' => $metadata, + ); + } + + private function splitTextIntoBlocks($text, $depth = 0) { + // Apply basic block and paragraph normalization to the text. NOTE: We don't + // strip trailing whitespace because it is semantic in some contexts, + // notably inlined diffs that the author intends to show as a code block. + $text = phutil_split_lines($text, true); + $block_rules = $this->blockRules; + $blocks = array(); + $cursor = 0; + $prev_block = array(); + + while (isset($text[$cursor])) { + $starting_cursor = $cursor; + foreach ($block_rules as $block_rule) { + $num_lines = $block_rule->getMatchingLineCount($text, $cursor); + + if ($num_lines) { + if ($blocks) { + $prev_block = last($blocks); + } + + $curr_block = array( + 'start' => $cursor, + 'num_lines' => $num_lines, + 'rule' => $block_rule, + 'is_empty' => self::isEmptyBlock($text, $cursor, $num_lines), + 'children' => array(), + ); + + if ($prev_block + && self::shouldMergeBlocks($text, $prev_block, $curr_block)) { + $blocks[last_key($blocks)]['num_lines'] += $curr_block['num_lines']; + $blocks[last_key($blocks)]['is_empty'] = + $blocks[last_key($blocks)]['is_empty'] && $curr_block['is_empty']; + } else { + $blocks[] = $curr_block; + } + + $cursor += $num_lines; + break; + } + } + + if ($starting_cursor === $cursor) { + throw new Exception(pht('Block in text did not match any block rule.')); + } + } + + foreach ($blocks as $key => $block) { + $lines = array_slice($text, $block['start'], $block['num_lines']); + $blocks[$key]['text'] = implode('', $lines); + } + + // Stop splitting child blocks apart if we get too deep. This arrests + // any blocks which have looping child rules, and stops the stack from + // exploding if someone writes a hilarious comment with 5,000 levels of + // quoted text. + + if ($depth < self::MAX_CHILD_DEPTH) { + foreach ($blocks as $key => $block) { + $rule = $block['rule']; + if (!$rule->supportsChildBlocks()) { + continue; + } + + list($parent_text, $child_text) = $rule->extractChildText( + $block['text']); + $blocks[$key]['text'] = $parent_text; + $blocks[$key]['children'] = $this->splitTextIntoBlocks( + $child_text, + $depth + 1); + } + } + + return $blocks; + } + + private function markupBlock(array $block) { + $children = array(); + foreach ($block['children'] as $child) { + $children[] = $this->markupBlock($child); + } + + if ($children) { + $children = $this->flattenOutput($children); + } else { + $children = null; + } + + return $block['rule']->markupText($block['text'], $children); + } + + private function flattenOutput(array $output) { + if ($this->isTextMode()) { + $output = implode("\n\n", $output)."\n"; + } else { + $output = phutil_implode_html("\n\n", $output); + } + + return $output; + } + + private static function shouldMergeBlocks($text, $prev_block, $curr_block) { + $block_rules = ipull(array($prev_block, $curr_block), 'rule'); + + $default_rule = 'PhutilRemarkupDefaultBlockRule'; + try { + assert_instances_of($block_rules, $default_rule); + + // If the last block was empty keep merging + if ($prev_block['is_empty']) { + return true; + } + + // If this line is blank keep merging + if ($curr_block['is_empty']) { + return true; + } + + // If the current line and the last line have content, keep merging + if (strlen(trim($text[$curr_block['start'] - 1]))) { + if (strlen(trim($text[$curr_block['start']]))) { + return true; + } + } + } catch (Exception $e) {} + + return false; + } + + private static function isEmptyBlock($text, $start, $num_lines) { + for ($cursor = $start; $cursor < $start + $num_lines; $cursor++) { + if (strlen(trim($text[$cursor]))) { + return false; + } + } + return true; + } + + public function postprocessText(array $dict) { + $this->metadata = idx($dict, 'metadata', array()); + + $this->storage = new PhutilRemarkupBlockStorage(); + $this->storage->setMap(idx($dict, 'storage', array())); + + foreach ($this->blockRules as $block_rule) { + $block_rule->postprocess(); + } + + foreach ($this->postprocessRules as $rule) { + $rule->didMarkupText(); + } + + return $this->restoreText(idx($dict, 'output')); + } + + public function restoreText($text) { + return $this->storage->restore($text, $this->isTextMode()); + } +} diff --git a/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php b/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php new file mode 100644 index 0000000000..c3b4960d0c --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php @@ -0,0 +1,132 @@ +markupText($root.$file); + } + } + + private function markupText($markup_file) { + $contents = Filesystem::readFile($markup_file); + $file = basename($markup_file); + + $parts = explode("\n~~~~~~~~~~\n", $contents); + $this->assertEqual(3, count($parts), $markup_file); + + list($input_remarkup, $expected_output, $expected_text) = $parts; + + $input_remarkup = $this->unescapeTrailingWhitespace($input_remarkup); + $expected_output = $this->unescapeTrailingWhitespace($expected_output); + $expected_text = $this->unescapeTrailingWhitespace($expected_text); + + $engine = $this->buildNewTestEngine(); + + switch ($file) { + case 'raw-escape.txt': + + // NOTE: Here, we want to test PhutilRemarkupEscapeRemarkupRule and + // PhutilRemarkupBlockStorage, which are triggered by "\1". In the + // test, "~" is used as a placeholder for "\1" since it's hard to type + // "\1". + + $input_remarkup = str_replace('~', "\1", $input_remarkup); + $expected_output = str_replace('~', "\1", $expected_output); + $expected_text = str_replace('~', "\1", $expected_text); + break; + case 'toc.txt': + $engine->setConfig('header.generate-toc', true); + break; + case 'link-same-window.txt': + $engine->setConfig('uri.same-window', true); + break; + case 'link-square.txt': + $engine->setConfig('uri.base', 'http://www.example.com/'); + $engine->setConfig('uri.here', 'http://www.example.com/page/'); + break; + } + + $actual_output = (string)$engine->markupText($input_remarkup); + + switch ($file) { + case 'toc.txt': + $table_of_contents = + PhutilRemarkupHeaderBlockRule::renderTableOfContents($engine); + $actual_output = $table_of_contents."\n\n".$actual_output; + break; + } + + $this->assertEqual( + $expected_output, + $actual_output, + pht("Failed to markup HTML in file '%s'.", $file)); + + $engine->setMode(PhutilRemarkupEngine::MODE_TEXT); + $actual_output = (string)$engine->markupText($input_remarkup); + + $this->assertEqual( + $expected_text, + $actual_output, + pht("Failed to markup text in file '%s'.", $file)); + } + + private function buildNewTestEngine() { + $engine = new PhutilRemarkupEngine(); + + $engine->setConfig( + 'uri.allowed-protocols', + array( + 'http' => true, + 'mailto' => true, + 'tel' => true, + )); + + $rules = array(); + $rules[] = new PhutilRemarkupEscapeRemarkupRule(); + $rules[] = new PhutilRemarkupMonospaceRule(); + $rules[] = new PhutilRemarkupDocumentLinkRule(); + $rules[] = new PhutilRemarkupHyperlinkRule(); + $rules[] = new PhutilRemarkupBoldRule(); + $rules[] = new PhutilRemarkupItalicRule(); + $rules[] = new PhutilRemarkupDelRule(); + $rules[] = new PhutilRemarkupUnderlineRule(); + $rules[] = new PhutilRemarkupHighlightRule(); + + $blocks = array(); + $blocks[] = new PhutilRemarkupQuotesBlockRule(); + $blocks[] = new PhutilRemarkupReplyBlockRule(); + $blocks[] = new PhutilRemarkupHeaderBlockRule(); + $blocks[] = new PhutilRemarkupHorizontalRuleBlockRule(); + $blocks[] = new PhutilRemarkupCodeBlockRule(); + $blocks[] = new PhutilRemarkupLiteralBlockRule(); + $blocks[] = new PhutilRemarkupNoteBlockRule(); + $blocks[] = new PhutilRemarkupTableBlockRule(); + $blocks[] = new PhutilRemarkupSimpleTableBlockRule(); + $blocks[] = new PhutilRemarkupDefaultBlockRule(); + $blocks[] = new PhutilRemarkupListBlockRule(); + $blocks[] = new PhutilRemarkupInterpreterBlockRule(); + + foreach ($blocks as $block) { + if (!($block instanceof PhutilRemarkupCodeBlockRule)) { + $block->setMarkupRules($rules); + } + } + + $engine->setBlockRules($blocks); + + return $engine; + } + + + private function unescapeTrailingWhitespace($input) { + // Remove up to one "~" at the end of each line so trailing whitespace may + // be written in tests as " ~". + return preg_replace('/~$/m', '', $input); + } + +} diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt new file mode 100644 index 0000000000..94886201cd --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt @@ -0,0 +1,7 @@ +**duck +quack** +~~~~~~~~~~ +

    duck +quack

    +~~~~~~~~~~ +**duck quack** diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt new file mode 100644 index 0000000000..8357cfbc35 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt @@ -0,0 +1,17 @@ +```x``` + +``` +y +``` +~~~~~~~~~~ +
    x
    + + + +
    y
    +~~~~~~~~~~ + x + + + + y diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt new file mode 100644 index 0000000000..c18ebfdbab --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt @@ -0,0 +1,12 @@ + lang=txt + code block + + - still a code block +~~~~~~~~~~ +
    code block
+
+- still a code block
    +~~~~~~~~~~ + code block + + - still a code block diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt new file mode 100644 index 0000000000..f743ab74d7 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt @@ -0,0 +1,9 @@ + lang=txt + x + y +~~~~~~~~~~ +
      x
+y
    +~~~~~~~~~~ + x + y diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt new file mode 100644 index 0000000000..955ad865eb --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt @@ -0,0 +1,11 @@ +omg~~ wtf~~~~~ bbq~~~ lol~~~ +~~deleted text~~~ +~~This is a great idea~~~ die forever please +~~~~~~~ +~~~~~~~~~~ +

    omg~~ wtf~~~~~ bbq~~~ lol~~~ +deleted text +This is a great idea~ die forever please +~~~~~~

    +~~~~~~~~~~ +omg~~ wtf~~~~~ bbq~~~ lol~~ ~~deleted text~~ ~~This is a great idea~~~ die forever please ~~~~~~~ diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt new file mode 100644 index 0000000000..602e8b3da5 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt @@ -0,0 +1,36 @@ +here is a diff + + lang=diff + @@ derp derp @@ + x + y + + - x + - y + + z + +derp derp +~~~~~~~~~~ +

    here is a diff

    + +
    @@ derp derp @@
+x
+y
+
+- x
+- y
++ z
    + +

    derp derp

    +~~~~~~~~~~ +here is a diff + + @@ derp derp @@ + x + y + + - x + - y + + z + +derp derp diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt new file mode 100644 index 0000000000..3d3bf125a1 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt @@ -0,0 +1,5 @@ +javascript://www.example.com/ +~~~~~~~~~~ +

    javascript://www.example.com/

    +~~~~~~~~~~ +javascript://www.example.com/ diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt new file mode 100644 index 0000000000..2fccce52d1 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt @@ -0,0 +1,5 @@ +< > & " +~~~~~~~~~~ +

    < > & "

    +~~~~~~~~~~ +< > & " diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt new file mode 100644 index 0000000000..282706f6e7 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt @@ -0,0 +1,11 @@ +#2 is my favorite. + +#project +~~~~~~~~~~ +

    #2 is my favorite.

    + +

    #project

    +~~~~~~~~~~ +#2 is my favorite. + +#project diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt new file mode 100644 index 0000000000..0c3768a3dd --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt @@ -0,0 +1,57 @@ +@nolint (UTF8) + +=a= + +blah blah blah + + += b = + +Markdown-Style Large Header +==== + +Markdown-Style Small Header +---- + +=== Remarkup-Style Smaller Header + + += ☃☃☃ UTF8 Header ☃☃☃ = +~~~~~~~~~~ +

    @nolint (UTF8)

    + +

    a

    + +

    blah blah blah

    + +

    b

    + +

    Markdown-Style Large Header

    + +

    Markdown-Style Small Header

    + +

    Remarkup-Style Smaller Header

    + +

    ☃☃☃ UTF8 Header ☃☃☃

    +~~~~~~~~~~ +@nolint (UTF8) + +a += + +blah blah blah + +b += + +Markdown-Style Large Header +=========================== + +Markdown-Style Small Header +--------------------------- + +Remarkup-Style Smaller Header +----------------------------- + +☃☃☃ UTF8 Header ☃☃☃ +=================== diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt new file mode 100644 index 0000000000..5fb8895d19 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt @@ -0,0 +1,9 @@ +how about we !!highlight!! some !!TEXT!!! +wow this must be **!!very important!!** +omg!!!!! +~~~~~~~~~~ +

    how about we highlight some TEXT! +wow this must be very important +omg!!!!!

    +~~~~~~~~~~ +how about we !!highlight!! some !!TEXT!!! wow this must be **!!very important!!** omg!!!!! diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt new file mode 100644 index 0000000000..c36a7f2650 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt @@ -0,0 +1,41 @@ +___ + +_____ + +*** + +* * * * * * * + +--- + +- - - - - - - + + --- +~~~~~~~~~~ +
    + +
    + +
    + +
    + +
    + +
    + +
    +~~~~~~~~~~ +___ + +_____ + +*** + +* * * * * * * + +--- + +- - - - - - - + + --- diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt new file mode 100644 index 0000000000..e527ee9e4d --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt @@ -0,0 +1,15 @@ +IMPORTANT: interesting **stuff** + +(IMPORTANT) interesting **stuff** +~~~~~~~~~~ +
    IMPORTANT: interesting stuff
    + + + +
    interesting stuff
    +~~~~~~~~~~ +IMPORTANT: interesting **stuff** + + + +(IMPORTANT) interesting **stuff** diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt new file mode 100644 index 0000000000..477f3eeea0 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt @@ -0,0 +1,58 @@ +phutil_test_block_interpreter (foo=bar) {{{ +content +}}} + +phutil_test_block_interpreter {{{ content +content }}} + +phutil_test_block_interpreter {{{ content }}} + +phutil_test_block_interpreter(x=y){{{content}}} + +phutil_fake_test_block_interpreter {{{ content }}} +~~~~~~~~~~ +Content: (content) +Argv: (foo=bar) + + + +Content: ( content +content ) +Argv: () + + + +Content: ( content ) +Argv: () + + + +Content: (content) +Argv: (x=y) + + + +
    No interpreter found: phutil_fake_test_block_interpreter
    +~~~~~~~~~~ +Content: (content) +Argv: (foo=bar) + + + +Content: ( content +content ) +Argv: () + + + +Content: ( content ) +Argv: () + + + +Content: (content) +Argv: (x=y) + + + +(No interpreter found: phutil_fake_test_block_interpreter) diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt new file mode 100644 index 0000000000..568e6df792 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt @@ -0,0 +1,5 @@ +``` +~~~~~~~~~~ +
    
+~~~~~~~~~~
+  
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt
new file mode 100644
index 0000000000..a6f800db95
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt
@@ -0,0 +1,6 @@
+
+a
+~~~~~~~~~~
+
    a
    
+~~~~~~~~~~
+a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt
new file mode 100644
index 0000000000..8e6129141e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt
@@ -0,0 +1,12 @@
+[Example](http://www.example.com/)
+
+x[0][1](**ptr);
+
+~~~~~~~~~~
+
    Example
    
+
+
    x[0][1](**ptr);
    
+~~~~~~~~~~
+Example 
+
+x[0][1](**ptr);
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt
new file mode 100644
index 0000000000..3808aa976c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt
@@ -0,0 +1,5 @@
+
+~~~~~~~~~~
+
    http://www.zany.com/omg/weird_url,,,
    
+~~~~~~~~~~
+http://www.zany.com/omg/weird_url,,,
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt
new file mode 100644
index 0000000000..64c93eaea6
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt
@@ -0,0 +1,35 @@
+http://www.example.com/
+
+(http://www.example.com/)
+
+
+
+http://www.example.com/wiki/example_(disambiguation)
+
+(example http://www.example.com/)
+
+Quick! http://www.example.com/!
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    (http://www.example.com/)
    
+
+
    http://www.example.com/
    
+
+
    http://www.example.com/wiki/example_(disambiguation)
    
+
+
    (example http://www.example.com/)
    
+
+
    Quick! http://www.example.com/!
    
+~~~~~~~~~~
+http://www.example.com/
+
+(http://www.example.com/)
+
+http://www.example.com/
+
+http://www.example.com/wiki/example_(disambiguation)
+
+(example http://www.example.com/)
+
+Quick! http://www.example.com/!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt
new file mode 100644
index 0000000000..e449c15013
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt
@@ -0,0 +1,18 @@
+[[ mailto:alincoln@example.com | mail me ]]
+
+[ mail me ]( mailto:alincoln@example.com )
+
+[[mailto:alincoln@example.com]]
+
+~~~~~~~~~~
+
    mail me
    
+
+
    mail me
    
+
+
    alincoln@example.com
    
+~~~~~~~~~~
+mail me 
+
+mail me 
+
+alincoln@example.com
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt
new file mode 100644
index 0000000000..bf433e9588
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt
@@ -0,0 +1,18 @@
+[[http://www.example.com/ | Example]](http://www.alternate.org/)
+
+(http://www.alternate.org/)[[http://www.example.com/ | Example]]
+
+
+
+~~~~~~~~~~
+
    Example(http://www.alternate.org/)
    
+
+
    (http://www.alternate.org/)Example
    
+
+
    <http://www.example.com/ Example>
    
+~~~~~~~~~~
+Example (http://www.alternate.org/)
+
+(http://www.alternate.org/)Example 
+
+>
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt
new file mode 100644
index 0000000000..65a86815c0
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt
@@ -0,0 +1,16 @@
+[[ /\evil.com ]]
+
+[[ /
+/evil.com ]]
+
+~~~~~~~~~~
+
    /\evil.com
    
+
+
    /
+/evil.com
    
+~~~~~~~~~~
+/\evil.com
+
+/
+/evil.com
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt
new file mode 100644
index 0000000000..937c83f67e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt
@@ -0,0 +1,11 @@
+[[http://www.example.com/]]
+
+http://www.example.com/
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    http://www.example.com/
    
+~~~~~~~~~~
+http://www.example.com/
+
+http://www.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt
new file mode 100644
index 0000000000..86f0c64d06
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt
@@ -0,0 +1,29 @@
+[[http://www.example.com/]]
+
+[[http://www.example.com/ | example.com]]
+
+[[/x/]]
+
+[[#anchor]]
+
+[[#anchor | Anchors ]]
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    example.com
    
+
+
    http://www.example.com/x/
    
+
+
    http://www.example.com/page/#anchor
    
+
+
    Anchors
    
+~~~~~~~~~~
+http://www.example.com/
+
+example.com 
+
+http://www.example.com/x/
+
+http://www.example.com/page/#anchor
+
+Anchors 
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt
new file mode 100644
index 0000000000..aac13c2cca
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt
@@ -0,0 +1,18 @@
+[[ tel:18005555555 | call me ]]
+
+[ call me ]( tel:18005555555 )
+
+[[tel:18005555555]]
+
+~~~~~~~~~~
+
    call me
    
+
+
    call me
    
+
+
    18005555555
    
+~~~~~~~~~~
+call me <18005555555>
+
+call me <18005555555>
+
+18005555555
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt
new file mode 100644
index 0000000000..847e9a1746
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt
@@ -0,0 +1,5 @@
+http://.example.com/
+~~~~~~~~~~
+
    http://<www>.example.com/
    
+~~~~~~~~~~
+http://.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt
new file mode 100644
index 0000000000..d823de6bcf
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt
@@ -0,0 +1,5 @@
+
+~~~~~~~~~~
+
    <http://x.y#http://x.y#>
    
+~~~~~~~~~~
+
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt
new file mode 100644
index 0000000000..e8f6d6536a
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt
@@ -0,0 +1,5 @@
+http://x.y#http://x.y#
+~~~~~~~~~~
+
    http://x.y#http://x.y#
    
+~~~~~~~~~~
+http://x.y#http://x.y#
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt
new file mode 100644
index 0000000000..c187b16751
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt
@@ -0,0 +1,9 @@
+http://www.example.com/,
+http://www.example.com/..
+http://www.example.com/!!!
+~~~~~~~~~~
+
    http://www.example.com/,
+http://www.example.com/..
+http://www.example.com/!!!
    
+~~~~~~~~~~
+http://www.example.com/, http://www.example.com/.. http://www.example.com/!!!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt
new file mode 100644
index 0000000000..615814d984
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt
@@ -0,0 +1,5 @@
+http://www.example.com/~~
+~~~~~~~~~~
+
    http://www.example.com/~
    
+~~~~~~~~~~
+http://www.example.com/~~
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt
new file mode 100644
index 0000000000..903ab48a5b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt
@@ -0,0 +1,5 @@
+http://www.example.com/
+~~~~~~~~~~
+
    http://www.example.com/
    
+~~~~~~~~~~
+http://www.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt
new file mode 100644
index 0000000000..420cd89cc9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt
@@ -0,0 +1,15 @@
+- a
+-- b
+--- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
          
+
        • c
          • 
+
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+    - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt
new file mode 100644
index 0000000000..c5b1631577
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt
@@ -0,0 +1,138 @@
+- a
+  - a
+    - a
+      - a
+        - a
+          - a
+            - a
+              - a
+                - a
+                  - a
+                    - a
+                      - a
+                        - a
+                          - a
+                            - a
+                              - a
+                                - a
+                                  - a
+                                    - a
+                                      - a
+                                        - a
+                                          - a
+                                            - a
+                                              - a
+                                                - a
+                                                  - a
+                                                    - a
+                                                      - a
+                                                        - a
+                                                          - a
+                                                            - a
+                                                              - a
+                                                                - a
+                                                                  - a
+                                                                    - a
+                                                                      - a
+                                                                        - a
+                                                                          - a
+
+
+derp
+~~~~~~~~~~
+
      
+
    • a
        
+
      • a
          
+
        • a
            
+
          • a
              
+
            • a
                
+
              • a
                  
+
                • a
                    
+
                  • a
                      
+
                    • a
                        
+
                      • a
                          
+
                        • a
                            
+
                          • a
                              
+
                            • a
                                
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • 
+
                            • 
+
                          • 
+
                        • 
+
                      • 
+
                    • 
+
                  • 
+
                • 
+
              • 
+
            • 
+
          • 
+
        • 
+
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- a
+  - a
+    - a
+      - a
+        - a
+          - a
+            - a
+              - a
+                - a
+                  - a
+                    - a
+                      - a
+                        - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt
new file mode 100644
index 0000000000..26c5a3f7d4
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt
@@ -0,0 +1,41 @@
+- [] a
+- [ ] b
+- [X] c
+- d
+
+[ ] A
+[X] B
+  [ ] C
+  [ ] D
+
+[1] footnote
+
+~~~~~~~~~~
+
      
+
    •  a
      • 
+
    •  b
      • 
+
    •  c
      • 
+
    • d
      • 
+
    
+
+
      
+
    •  A
      • 
+
    •  B
        
+
      •  C
        • 
+
      •  D
        • 
+
      • 
+
    
+
+
    [1] footnote
    
+~~~~~~~~~~
+[ ] a
+[ ] b
+[X] c
+- d
+
+[ ] A
+[X] B
+  [ ] C
+  [ ] D
+
+[1] footnote
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt
new file mode 100644
index 0000000000..98e55c3534
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt
@@ -0,0 +1,15 @@
+## Fruit
+- Apple
+- Banana
+~~~~~~~~~~
+
      
+
      • 
+
      1. Fruit
        2. 
+
      • 
+
    • Apple
      • 
+
    • Banana
      • 
+
    
+~~~~~~~~~~
+  1. Fruit
+- Apple
+- Banana
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt
new file mode 100644
index 0000000000..55bd4c5429
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt
@@ -0,0 +1,19 @@
+# item
+- item
+- item
+
+derp
+~~~~~~~~~~
+
      
+
    1. item
      2. 
+
    2. item
      3. 
+
    3. item
      4. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+1. item
+2. item
+3. item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt
new file mode 100644
index 0000000000..d1323090eb
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt
@@ -0,0 +1,19 @@
+# item
+# item
+# item
+
+derp
+~~~~~~~~~~
+
      
+
    1. item
      2. 
+
    2. item
      3. 
+
    3. item
      4. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+1. item
+2. item
+3. item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt
new file mode 100644
index 0000000000..acb74781af
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt
@@ -0,0 +1,7 @@
+# At the end of a block, this should be a list.
+~~~~~~~~~~
+
      
+
    1. At the end of a block, this should be a list.
      2. 
+
    
+~~~~~~~~~~
+1. At the end of a block, this should be a list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt
new file mode 100644
index 0000000000..4dd60f5413
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt
@@ -0,0 +1,12 @@
+## Small Header
+
+This should be a small header.
+~~~~~~~~~~
+
    Small Header
    
+
+
    This should be a small header.
    
+~~~~~~~~~~
+Small Header
+------------
+
+This should be a small header.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt
new file mode 100644
index 0000000000..dcd6732563
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt
@@ -0,0 +1,15 @@
+  - a
+  -- b
+  --- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
          
+
        • c
          • 
+
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+    - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt
new file mode 100644
index 0000000000..ee7b307fc2
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt
@@ -0,0 +1,14 @@
+- a
+  -- b
+  -- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
        • 
+
      • c
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+  - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt
new file mode 100644
index 0000000000..362133aaf1
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt
@@ -0,0 +1,16 @@
+- a
+  a
+- b
+b
+~~~~~~~~~~
+
      
+
    • a a
      • 
+
    • b
      • 
+
    
+
+
    b
    
+~~~~~~~~~~
+- a a
+- b
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt
new file mode 100644
index 0000000000..9ce47191d9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt
@@ -0,0 +1,30 @@
+- item
+  - sub
+- item
+  # sub
+  # sub
+- item
+
+derp
+~~~~~~~~~~
+
      
+
    • item
        
+
      • sub
        • 
+
      • 
+
    • item
        
+
      1. sub
        2. 
+
      2. sub
        3. 
+
      • 
+
    • item
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- item
+  - sub
+- item
+  1. sub
+  2. sub
+- item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt
new file mode 100644
index 0000000000..90e972c8ad
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt
@@ -0,0 +1,27 @@
+- This is a list item
+  with several paragraphs.
+
+  This is the second paragraph
+  of the first list item.
+- This is the second item
+  in the list.
+  - This is a sublist.
+- This is the third item in the list.
+
+~~~~~~~~~~
+
      
+
    • This is a list item with several paragraphs.
+

      
+This is the second paragraph of the first list item.
      • 
+
    • This is the second item in the list.
        
+
      • This is a sublist.
        • 
+
      • 
+
    • This is the third item in the list.
      • 
+
    
+~~~~~~~~~~
+- This is a list item with several paragraphs.
+
+  This is the second paragraph of the first list item.
+- This is the second item in the list.
+  - This is a sublist.
+- This is the third item in the list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt
new file mode 100644
index 0000000000..232f599662
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt
@@ -0,0 +1,23 @@
+    - top
+  - mid
+# bot
+
+derp
+~~~~~~~~~~
+
      
+
      1. 
+
        • 
+
        • top
          • 
+
        • 
+
      • mid
        • 
+
      2. 
+
    2. bot
      3. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+    - top
+  - mid
+1. bot
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt
new file mode 100644
index 0000000000..f86e489e3c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt
@@ -0,0 +1,19 @@
+* item
+* item
+* item
+
+derp
+~~~~~~~~~~
+
      
+
    • item
      • 
+
    • item
      • 
+
    • item
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- item
+- item
+- item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt
new file mode 100644
index 0000000000..365d11aafc
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt
@@ -0,0 +1,15 @@
+1) one
+
+- a
+~~~~~~~~~~
+
      
+
    1. one
      2. 
+
    
+
+
      
+
    • a
      • 
+
    
+~~~~~~~~~~
+1. one
+
+- a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt
new file mode 100644
index 0000000000..1b51134903
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt
@@ -0,0 +1,17 @@
+This should be a list:
+
+  - apple
+  - banana
+
+~~~~~~~~~~
+
    This should be a list:
    
+
+
      
+
    • apple
      • 
+
    • banana
      • 
+
    
+~~~~~~~~~~
+This should be a list:
+
+- apple
+- banana
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt
new file mode 100644
index 0000000000..eda8781b8e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt
@@ -0,0 +1,13 @@
+  - < > & "
+
+text block
+~~~~~~~~~~
+
      
+
    • < > & "
      • 
+
    
+
+
    text block
    
+~~~~~~~~~~
+- < > & "
+
+text block
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt
new file mode 100644
index 0000000000..8c94a16f47
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt
@@ -0,0 +1,18 @@
+query ##SELECT * FROM `table`##
+
+`SELECT * FROM ##table##`
+
+`**x**`
+
+~~~~~~~~~~
+
    query SELECT * FROM `table`
    
+
+
    SELECT * FROM ##table##
    
+
+
    **x**
    
+~~~~~~~~~~
+query ##SELECT * FROM `table`##
+
+`SELECT * FROM ##table##`
+
+`**x**`
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt
new file mode 100644
index 0000000000..cb78ae923b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt
@@ -0,0 +1,11 @@
+`Zebra`s
+
+I can`t and I won`t.
+~~~~~~~~~~
+
    Zebras
    
+
+
    I can`t and I won`t.
    
+~~~~~~~~~~
+`Zebra`s
+
+I can`t and I won`t.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt
new file mode 100644
index 0000000000..db9cd50198
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt
@@ -0,0 +1,5 @@
+cmd ##ls --color > /dev/null##
+~~~~~~~~~~
+
    cmd ls --color > /dev/null
    
+~~~~~~~~~~
+cmd ##ls --color > /dev/null##
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt
new file mode 100644
index 0000000000..159e7c9e54
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt
@@ -0,0 +1,30 @@
+This is a paragraph.
+
+
+  lang=txt
+  First line of code block.
+  Second line of code block.
+
+
+
+  
+    
+    
+  
+
    Cell 1Cell 2
    
+~~~~~~~~~~
+
    This is a paragraph.
    
+
+
    First line of code block.
+Second line of code block.
    
+
+
    
+
+
    Cell 1Cell 2
    
+~~~~~~~~~~
+This is a paragraph.
+
+  First line of code block.
+  Second line of code block.
+
+| Cell 1 | Cell 2 |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt
new file mode 100644
index 0000000000..389da296bc
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt
@@ -0,0 +1,14 @@
+NOTE: a
+a
+
+b
+~~~~~~~~~~
+
    NOTE: a
+a
    
+
+
    b
    
+~~~~~~~~~~
+NOTE: a
+a
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt
new file mode 100644
index 0000000000..30541832e3
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt
@@ -0,0 +1,15 @@
+NOTE: interesting **stuff**
+
+(NOTE) interesting **stuff**
+~~~~~~~~~~
+
    NOTE: interesting stuff
    
+
+
+
+
    interesting stuff
    
+~~~~~~~~~~
+NOTE: interesting **stuff**
+
+
+
+(NOTE) interesting **stuff**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt
new file mode 100644
index 0000000000..f2e5e58114
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt
@@ -0,0 +1,64 @@
+# aasdx
+# asdf
+
+1. asa
+ # asdf
+234) asdf
+
+234) asd
+
+1. asd
+234) asd
+
+10. ten
+11. eleven
+12. twelve
+
+1/ This explicitly should not be formatted as a list.
+~~~~~~~~~~
+
      
+
    1. aasdx
      2. 
+
    2. asdf
      3. 
+
    
+
+
      
+
    1. asa
        
+
      1. asdf
        2. 
+
      2. 
+
    2. asdf
      3. 
+
    
+
+
      
+
    1. asd
      2. 
+
    
+
+
      
+
    1. asd
      2. 
+
    2. asd
      3. 
+
    
+
+
      
+
    1. ten
      2. 
+
    2. eleven
      3. 
+
    3. twelve
      4. 
+
    
+
+
    1/ This explicitly should not be formatted as a list.
    
+~~~~~~~~~~
+1. aasdx
+2. asdf
+
+1. asa
+  1. asdf
+2. asdf
+
+234. asd
+
+1. asd
+2. asd
+
+10. ten
+11. eleven
+12. twelve
+
+1/ This explicitly should not be formatted as a list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt
new file mode 100644
index 0000000000..b6bc405bb9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt
@@ -0,0 +1,29 @@
+%%%a%%%
+%%%b%%%
+
+%%%a
+b%%%
+
+%%%a%%%
+
+%%%b%%%
+~~~~~~~~~~
+
    a
+
    b
    
+
+
    a
+
    b
    
+
+
    a
    
+
+
    b
    
+~~~~~~~~~~
+a
+b
+
+a
+b
+
+a
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt
new file mode 100644
index 0000000000..90cb4e0b41
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt
@@ -0,0 +1,21 @@
+**foo**
+%%%- first
+- second
+- third%%%
+[[http://hello | world]]
+~~~~~~~~~~
+
    foo
    
+
+
    - first
+
    - second
+
    - third
    
+
+
    world
    
+~~~~~~~~~~
+**foo**
+
+- first
+- second
+- third
+
+world 
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt
new file mode 100644
index 0000000000..a63e93d41c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt
@@ -0,0 +1,11 @@
+%%%[[http://hello | world]] **bold**%%%
+
+  %%%[[http://hello | world]] **bold**%%%
+~~~~~~~~~~
+
    [[http://hello | world]] **bold**
    
+
+
    [[http://hello | world]] **bold**
    
+~~~~~~~~~~
+[[http://hello | world]] **bold**
+
+[[http://hello | world]] **bold**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt
new file mode 100644
index 0000000000..34ddbd38d3
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt
@@ -0,0 +1,8 @@
+%%%
+**x**%%%
+~~~~~~~~~~
+
    
+
    **x**
    
+~~~~~~~~~~
+
+**x**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt
new file mode 100644
index 0000000000..da78b31fd6
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt
@@ -0,0 +1,5 @@
+>>> REQUESTING CHANGES BECAUSE I'M ANGRY!
+~~~~~~~~~~
+
    REQUESTING CHANGES BECAUSE I'M ANGRY!
    
+~~~~~~~~~~
+>>> REQUESTING CHANGES BECAUSE I'M ANGRY!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt
new file mode 100644
index 0000000000..9ef852d56d
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt
@@ -0,0 +1,16 @@
+> This should be a code block:
+>
+> ```lang=php
+>  $foo = 'bar';
+> ```
+~~~~~~~~~~
+
    This should be a code block:
    
+
+
    <?php
+$foo = 'bar';
    
+~~~~~~~~~~
+> This should be a code block:
+>
+>      $foo = 'bar';
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt
new file mode 100644
index 0000000000..80a7428fce
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt
@@ -0,0 +1,5 @@
+>   xyz
+~~~~~~~~~~
+
    xyz
    
+~~~~~~~~~~
+>   xyz
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt
new file mode 100644
index 0000000000..8d3aaf9c41
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt
@@ -0,0 +1,24 @@
+> # X
+> # Y
+>
+> B
+>
+> * C
+~~~~~~~~~~
+
      
+
    1. X
      2. 
+
    2. Y
      3. 
+
    
+
+
    B
    
+
+
      
+
    • C
      • 
+
    
+~~~~~~~~~~
+> 1. X
+> 2. Y
+>
+> B
+>
+> - C
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt
new file mode 100644
index 0000000000..1ca2f2f2a5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt
@@ -0,0 +1,19 @@
+>>! In U, W wrote:
+> - Y
+>
+> Z
+~~~~~~~~~~
+
    
+
    In U, W wrote:
    
+
      
+
    • Y
      • 
+
    
+
+
    Z
    
+
    
+~~~~~~~~~~
+In U, W wrote:
+
+> - Y
+>
+> Z
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt
new file mode 100644
index 0000000000..212f87222d
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt
@@ -0,0 +1,9 @@
+> Dear Sir,
+> I am utterly disgusted with the quality
+> of your inflight food service.
+~~~~~~~~~~
+
    Dear Sir,
+I am utterly disgusted with the quality
+of your inflight food service.
    
+~~~~~~~~~~
+> Dear Sir, I am utterly disgusted with the quality of your inflight food service.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt
new file mode 100644
index 0000000000..4cb635ebfb
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt
@@ -0,0 +1,17 @@
+~1~~
+
+~2Z
+
+~a
+~~~~~~~~~~
+
    ~1~
    
+
+
    ~2Z
    
+
+
    ~a
    
+~~~~~~~~~~
+~1~~
+
+~2Z
+
+~a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt
new file mode 100644
index 0000000000..4afadedd6e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt
@@ -0,0 +1,11 @@
+>>! In comment #123, alincoln wrote:
+> Four score and twenty years ago...
+~~~~~~~~~~
+
    
+
    In comment #123, alincoln wrote:
    
+
    Four score and twenty years ago...
    
+
    
+~~~~~~~~~~
+In comment #123, alincoln wrote:
+
+> Four score and twenty years ago...
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt
new file mode 100644
index 0000000000..85440ddeac
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt
@@ -0,0 +1,48 @@
+>>! Previously, fruit:
+>
+> - Apple
+> - Banana
+> - Cherry
+>
+>>>! More previously, vegetables:
+>>
+>> - Potato
+>> - Potato
+>> - Potato
+>
+> The end.
+
+~~~~~~~~~~
+
    
+
    Previously, fruit:
    
+
      
+
    • Apple
      • 
+
    • Banana
      • 
+
    • Cherry
      • 
+
    
+
+
    
+
    More previously, vegetables:
    
+
      
+
    • Potato
      • 
+
    • Potato
      • 
+
    • Potato
      • 
+
    
+
    
+
+
    The end.
    
+
    
+~~~~~~~~~~
+Previously, fruit:
+
+> - Apple
+> - Banana
+> - Cherry
+>
+> More previously, vegetables:
+>
+>> - Potato
+>> - Potato
+>> - Potato
+>
+> The end.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt
new file mode 100644
index 0000000000..40c27ca941
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt
@@ -0,0 +1,13 @@
+| Alpaca |
+| |
+| Zebra  |
+~~~~~~~~~~
+
    
+
+
+
+
    Alpaca
    Zebra
    
+~~~~~~~~~~
+| Alpaca |
+|        |
+| Zebra  |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt
new file mode 100644
index 0000000000..418baa5ba5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt
@@ -0,0 +1,7 @@
+ |a|b|
+~~~~~~~~~~
+
    
+
+
    ab
    
+~~~~~~~~~~
+| a | b |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt
new file mode 100644
index 0000000000..d4e53b3426
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt
@@ -0,0 +1,7 @@
+| [[ http://example.com | name ]] | [x] |
+~~~~~~~~~~
+
    
+
+
    name[x]
    
+~~~~~~~~~~
+| name  | [x] |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt
new file mode 100644
index 0000000000..d86a8189b9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt
@@ -0,0 +1,24 @@
+| analyze_resources | original | mobile only | www only | both |
+|                   | -------- | ----------- | -------- | ---- |
+| //real//          | 31 s     | 24 s        | 31 s     | 31 s
+| --------
+| //user//          | 49 s     | 25 s        | 31 s     | 49 s
+| --------
+| //sys//           | 24 s     | 12 s        | 13 s     | 24 s
+| -------
+~~~~~~~~~~
+
    
+
+
+
+
+
    analyze_resourcesoriginalmobile onlywww onlyboth
    real31 s24 s31 s31 s
    user49 s25 s31 s49 s
    sys24 s12 s13 s24 s
    
+~~~~~~~~~~
+| analyze_resources | original | mobile only | www only | both |
+|                   | -------- | ----------- | -------- | ---- |
+| //real//          | 31 s     | 24 s        | 31 s     | 31 s |
+| ----------------- |          |             |          |      |
+| //user//          | 49 s     | 25 s        | 31 s     | 49 s |
+| ----------------- |          |             |          |      |
+| //sys//           | 24 s     | 12 s        | 13 s     | 24 s |
+| ----------------- |          |             |          |      |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt
new file mode 100644
index 0000000000..bd9b136d72
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt
@@ -0,0 +1,5 @@
+hello
+~~~~~~~~~~
+
    hello
    
+~~~~~~~~~~
+hello
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt
new file mode 100644
index 0000000000..eab88dec77
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt
@@ -0,0 +1,5 @@
+quack
    
+~~~~~~~~~~
+<table>quack</table>
+~~~~~~~~~~
+quack
    
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt
new file mode 100644
index 0000000000..b0a45cd669
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt
@@ -0,0 +1,7 @@
+ 
    cell
    
+~~~~~~~~~~
+
    
+
+
    cell
    
+~~~~~~~~~~
+| cell |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt
new file mode 100644
index 0000000000..84b5cb84ad
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt
@@ -0,0 +1,8 @@
+|x|
+||--
+~~~~~~~~~~
+
    
+
+
    x
    
+~~~~~~~~~~
+| x |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt
new file mode 100644
index 0000000000..6a3afe5d1b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt
@@ -0,0 +1,16 @@
+
+
+
+
+
    TableStorage
    `differential_diff`InnoDB
    `edge`?
    
+~~~~~~~~~~
+
    
+
+
+
+
    TableStorage
    differential_diffInnoDB
    edge?
    
+~~~~~~~~~~
+| Table               | Storage |
+| ------------------- | ------- |
+| `differential_diff` | InnoDB  |
+| `edge`              | ?       |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt
new file mode 100644
index 0000000000..b05c52442e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt
@@ -0,0 +1,18 @@
+```code
+
+more code
+
+more code```
+
+~~~~~~~~~~
+
    code
+
+more code
+
+more code
    
+~~~~~~~~~~
+  code
+  
+  more code
+  
+  more code
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt
new file mode 100644
index 0000000000..4cc7607d32
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt
@@ -0,0 +1,5 @@
+```code```
+~~~~~~~~~~
+
    code
    
+~~~~~~~~~~
+  code
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt
new file mode 100644
index 0000000000..43448f75b5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt
@@ -0,0 +1,29 @@
+= [[ http://www.example.com/ | link_name ]] =
+
+== **bold** ==
+
+= http://www.example.com =
+
+~~~~~~~~~~
+
+
+
    link_name
    
+
+
    bold
    
+
+
    http://www.example.com
    
+~~~~~~~~~~
+[[ http://www.example.com/ | link_name ]]
+=========================================
+
+**bold**
+--------
+
+http://www.example.com
+======================
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt
new file mode 100644
index 0000000000..2187cca939
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt
@@ -0,0 +1,39 @@
+  lang=txt
+  code block
+  code block
+  
+  
+  
+  
+  code block
+  
+  
+            
+  
+  code block
+~~~~~~~~~~
+
    code block
+code block
+
+
+
+
+code block
+
+
+          
+
+code block
    
+~~~~~~~~~~
+  code block
+  code block
+  
+  
+  
+  
+  code block
+  
+  
+            
+  
+  code block
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt
new file mode 100644
index 0000000000..511c0cea84
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt
@@ -0,0 +1,13 @@
+omg__ wtf_____ bbq___ lol__
+__underlined text__
+__This is a great idea___ die forever please
+__
+/__notunderlined__/ and also /__notunderlined__.c
+~~~~~~~~~~
+
    omg__ wtf_____ bbq___ lol__
+underlined text
+This is a great idea_ die forever please
+__
+/__notunderlined__/ and also /__notunderlined__.c
    
+~~~~~~~~~~
+omg__ wtf_____ bbq___ lol__ __underlined text__ __This is a great idea___ die forever please __ /__notunderlined__/ and also /__notunderlined__.c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt
new file mode 100644
index 0000000000..6de7f0cd6c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt
@@ -0,0 +1,15 @@
+WARNING: interesting **stuff**
+
+(WARNING) interesting **stuff**
+~~~~~~~~~~
+
    WARNING: interesting stuff
    
+
+
+
+
    interesting stuff
    
+~~~~~~~~~~
+WARNING: interesting **stuff**
+
+
+
+(WARNING) interesting **stuff**
diff --git a/src/infrastructure/storage/connection/AphrontDatabaseConnection.php b/src/infrastructure/storage/connection/AphrontDatabaseConnection.php
new file mode 100644
index 0000000000..b3bd2c8299
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontDatabaseConnection.php
@@ -0,0 +1,305 @@
+close();
+  }
+
+  final public function setLastActiveEpoch($epoch) {
+    $this->lastActiveEpoch = $epoch;
+    return $this;
+  }
+
+  final public function getLastActiveEpoch() {
+    return $this->lastActiveEpoch;
+  }
+
+  final public function setPersistent($persistent) {
+    $this->persistent = $persistent;
+    return $this;
+  }
+
+  final public function getPersistent() {
+    return $this->persistent;
+  }
+
+  public function queryData($pattern/* , $arg, $arg, ... */) {
+    $args = func_get_args();
+    array_unshift($args, $this);
+    return call_user_func_array('queryfx_all', $args);
+  }
+
+  public function query($pattern/* , $arg, $arg, ... */) {
+    $args = func_get_args();
+    array_unshift($args, $this);
+    return call_user_func_array('queryfx', $args);
+  }
+
+
+  public function supportsAsyncQueries() {
+    return false;
+  }
+
+  public function supportsParallelQueries() {
+    return false;
+  }
+
+  public function setReadOnly($read_only) {
+    $this->readOnly = $read_only;
+    return $this;
+  }
+
+  public function getReadOnly() {
+    return $this->readOnly;
+  }
+
+  public function setQueryTimeout($query_timeout) {
+    $this->queryTimeout = $query_timeout;
+    return $this;
+  }
+
+  public function getQueryTimeout() {
+    return $this->queryTimeout;
+  }
+
+  public function asyncQuery($raw_query) {
+    throw new Exception(pht('Async queries are not supported.'));
+  }
+
+  public static function resolveAsyncQueries(array $conns, array $asyncs) {
+    throw new Exception(pht('Async queries are not supported.'));
+  }
+
+  /**
+   * Is this connection idle and safe to close?
+   *
+   * A connection is "idle" if it can be safely closed without loss of state.
+   * Connections inside a transaction or holding locks are not idle, even
+   * though they may not actively be executing queries.
+   *
+   * @return bool True if the connection is idle and can be safely closed.
+   */
+  public function isIdle() {
+    if ($this->isInsideTransaction()) {
+      return false;
+    }
+
+    if ($this->isHoldingAnyLock()) {
+      return false;
+    }
+
+    return true;
+  }
+
+
+/* -(  Global Locks  )------------------------------------------------------- */
+
+
+  public function rememberLock($lock) {
+    if (isset($this->locks[$lock])) {
+      throw new Exception(
+        pht(
+          'Trying to remember lock "%s", but this lock has already been '.
+          'remembered.',
+          $lock));
+    }
+
+    $this->locks[$lock] = true;
+    return $this;
+  }
+
+
+  public function forgetLock($lock) {
+    if (empty($this->locks[$lock])) {
+      throw new Exception(
+        pht(
+          'Trying to forget lock "%s", but this connection does not remember '.
+          'that lock.',
+          $lock));
+    }
+
+    unset($this->locks[$lock]);
+    return $this;
+  }
+
+
+  public function forgetAllLocks() {
+    $this->locks = array();
+    return $this;
+  }
+
+
+  public function isHoldingAnyLock() {
+    return (bool)$this->locks;
+  }
+
+
+/* -(  Transaction Management  )--------------------------------------------- */
+
+
+  /**
+   * Begin a transaction, or set a savepoint if the connection is already
+   * transactional.
+   *
+   * @return this
+   * @task xaction
+   */
+  public function openTransaction() {
+    $state = $this->getTransactionState();
+    $point = $state->getSavepointName();
+    $depth = $state->getDepth();
+
+    $new_transaction = ($depth == 0);
+    if ($new_transaction) {
+      $this->query('START TRANSACTION');
+    } else {
+      $this->query('SAVEPOINT '.$point);
+    }
+
+    $state->increaseDepth();
+
+    return $this;
+  }
+
+
+  /**
+   * Commit a transaction, or stage a savepoint for commit once the entire
+   * transaction completes if inside a transaction stack.
+   *
+   * @return this
+   * @task xaction
+   */
+  public function saveTransaction() {
+    $state = $this->getTransactionState();
+    $depth = $state->decreaseDepth();
+
+    if ($depth == 0) {
+      $this->query('COMMIT');
+    }
+
+    return $this;
+  }
+
+
+  /**
+   * Rollback a transaction, or unstage the last savepoint if inside a
+   * transaction stack.
+   *
+   * @return this
+   */
+  public function killTransaction() {
+    $state = $this->getTransactionState();
+    $depth = $state->decreaseDepth();
+
+    if ($depth == 0) {
+      $this->query('ROLLBACK');
+    } else {
+      $this->query('ROLLBACK TO SAVEPOINT '.$state->getSavepointName());
+    }
+
+    return $this;
+  }
+
+
+  /**
+   * Returns true if the connection is transactional.
+   *
+   * @return bool True if the connection is currently transactional.
+   * @task xaction
+   */
+  public function isInsideTransaction() {
+    $state = $this->getTransactionState();
+    return ($state->getDepth() > 0);
+  }
+
+
+  /**
+   * Get the current @{class:AphrontDatabaseTransactionState} object, or create
+   * one if none exists.
+   *
+   * @return AphrontDatabaseTransactionState Current transaction state.
+   * @task xaction
+   */
+  protected function getTransactionState() {
+    if (!$this->transactionState) {
+      $this->transactionState = new AphrontDatabaseTransactionState();
+    }
+    return $this->transactionState;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function beginReadLocking() {
+    $this->getTransactionState()->beginReadLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function endReadLocking() {
+    $this->getTransactionState()->endReadLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function isReadLocking() {
+    return $this->getTransactionState()->isReadLocking();
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function beginWriteLocking() {
+    $this->getTransactionState()->beginWriteLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function endWriteLocking() {
+    $this->getTransactionState()->endWriteLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function isWriteLocking() {
+    return $this->getTransactionState()->isWriteLocking();
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php b/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php
new file mode 100644
index 0000000000..67f8b5b65b
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php
@@ -0,0 +1,105 @@
+depth;
+  }
+
+  public function increaseDepth() {
+    return ++$this->depth;
+  }
+
+  public function decreaseDepth() {
+    if ($this->depth == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s or %s!',
+          'saveTransaction()',
+          'killTransaction()'));
+    }
+
+    return --$this->depth;
+  }
+
+  public function getSavepointName() {
+    return 'Aphront_Savepoint_'.$this->depth;
+  }
+
+  public function beginReadLocking() {
+    $this->readLockLevel++;
+    return $this;
+  }
+
+  public function endReadLocking() {
+    if ($this->readLockLevel == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s!',
+          __FUNCTION__.'()'));
+    }
+    $this->readLockLevel--;
+    return $this;
+  }
+
+  public function isReadLocking() {
+    return ($this->readLockLevel > 0);
+  }
+
+  public function beginWriteLocking() {
+    $this->writeLockLevel++;
+    return $this;
+  }
+
+  public function endWriteLocking() {
+    if ($this->writeLockLevel == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s!',
+          __FUNCTION__.'()'));
+    }
+    $this->writeLockLevel--;
+    return $this;
+  }
+
+  public function isWriteLocking() {
+    return ($this->writeLockLevel > 0);
+  }
+
+  public function __destruct() {
+    if ($this->depth) {
+      throw new Exception(
+        pht(
+          'Process exited with an open transaction! The transaction '.
+          'will be implicitly rolled back. Calls to %s must always be '.
+          'paired with a call to %s or %s.',
+          'openTransaction()',
+          'saveTransaction()',
+          'killTransaction()'));
+    }
+    if ($this->readLockLevel) {
+      throw new Exception(
+        pht(
+          'Process exited with an open read lock! Call to %s '.
+          'must always be paired with a call to %s.',
+          'beginReadLocking()',
+          'endReadLocking()'));
+    }
+    if ($this->writeLockLevel) {
+      throw new Exception(
+        pht(
+          'Process exited with an open write lock! Call to %s '.
+          'must always be paired with a call to %s.',
+          'beginWriteLocking()',
+          'endWriteLocking()'));
+    }
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php b/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php
new file mode 100644
index 0000000000..9638b6f00e
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php
@@ -0,0 +1,132 @@
+configuration = $configuration;
+
+    if (self::$nextInsertID === null) {
+      // Generate test IDs into a distant ID space to reduce the risk of
+      // collisions and make them distinctive.
+      self::$nextInsertID = 55555000000 + mt_rand(0, 1000);
+    }
+  }
+
+  public function openConnection() {
+    return;
+  }
+
+  public function close() {
+    return;
+  }
+
+  public function escapeUTF8String($string) {
+    return '';
+  }
+
+  public function escapeBinaryString($string) {
+    return '';
+  }
+
+  public function escapeColumnName($name) {
+    return '';
+  }
+
+  public function escapeMultilineComment($comment) {
+    return '';
+  }
+
+  public function escapeStringForLikeClause($value) {
+    return '';
+  }
+
+  private function getConfiguration($key, $default = null) {
+    return idx($this->configuration, $key, $default);
+  }
+
+  public function getInsertID() {
+    return $this->insertID;
+  }
+
+  public function getAffectedRows() {
+    return $this->affectedRows;
+  }
+
+  public function selectAllResults() {
+    return $this->allResults;
+  }
+
+  public function executeQuery(PhutilQueryString $query) {
+
+    // NOTE: "[\s<>K]*" allows any number of (properly escaped) comments to
+    // appear prior to the allowed keyword, since this connection escapes
+    // them as "" (above).
+
+    $display_query = $query->getMaskedString();
+    $raw_query = $query->getUnmaskedString();
+
+    $keywords = array(
+      'INSERT',
+      'UPDATE',
+      'DELETE',
+      'START',
+      'SAVEPOINT',
+      'COMMIT',
+      'ROLLBACK',
+    );
+    $preg_keywords = array();
+    foreach ($keywords as $key => $word) {
+      $preg_keywords[] = preg_quote($word, '/');
+    }
+    $preg_keywords = implode('|', $preg_keywords);
+
+    if (!preg_match('/^[\s<>K]*('.$preg_keywords.')\s*/i', $raw_query)) {
+      throw new AphrontNotSupportedQueryException(
+        pht(
+          "Database isolation currently only supports some queries. You are ".
+          "trying to issue a query which does not begin with an allowed ".
+          "keyword (%s): '%s'.",
+          implode(', ', $keywords),
+          $display_query));
+    }
+
+    $this->transcript[] = $display_query;
+
+    // NOTE: This method is intentionally simplified for now, since we're only
+    // using it to stub out inserts/updates. In the future it will probably need
+    // to grow more powerful.
+
+    $this->allResults = array();
+
+    // NOTE: We jitter the insert IDs to keep tests honest; a test should cover
+    // the relationship between objects, not their exact insertion order. This
+    // guarantees that IDs are unique but makes it impossible to hard-code tests
+    // against this specific implementation detail.
+    self::$nextInsertID += mt_rand(1, 10);
+    $this->insertID = self::$nextInsertID;
+    $this->affectedRows = 1;
+  }
+
+  public function executeRawQueries(array $raw_queries) {
+    $results = array();
+    foreach ($raw_queries as $id => $raw_query) {
+      $results[$id] = array();
+    }
+    return $results;
+  }
+
+  public function getQueryTranscript() {
+    return $this->transcript;
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
new file mode 100644
index 0000000000..0f9201b02d
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
@@ -0,0 +1,405 @@
+configuration = $configuration;
+  }
+
+  public function __clone() {
+    $this->establishConnection();
+  }
+
+  public function openConnection() {
+    $this->requireConnection();
+  }
+
+  public function close() {
+    if ($this->lastResult) {
+      $this->lastResult = null;
+    }
+    if ($this->connection) {
+      $this->closeConnection();
+      $this->connection = null;
+    }
+  }
+
+  public function escapeColumnName($name) {
+    return '`'.str_replace('`', '``', $name).'`';
+  }
+
+
+  public function escapeMultilineComment($comment) {
+    // These can either terminate a comment, confuse the hell out of the parser,
+    // make MySQL execute the comment as a query, or, in the case of semicolon,
+    // are quasi-dangerous because the semicolon could turn a broken query into
+    // a working query plus an ignored query.
+
+    static $map = array(
+      '--'  => '(DOUBLEDASH)',
+      '*/'  => '(STARSLASH)',
+      '//'  => '(SLASHSLASH)',
+      '#'   => '(HASH)',
+      '!'   => '(BANG)',
+      ';'   => '(SEMICOLON)',
+    );
+
+    $comment = str_replace(
+      array_keys($map),
+      array_values($map),
+      $comment);
+
+    // For good measure, kill anything else that isn't a nice printable
+    // character.
+    $comment = preg_replace('/[^\x20-\x7F]+/', ' ', $comment);
+
+    return '/* '.$comment.' */';
+  }
+
+  public function escapeStringForLikeClause($value) {
+    $value = addcslashes($value, '\%_');
+    $value = $this->escapeUTF8String($value);
+    return $value;
+  }
+
+  protected function getConfiguration($key, $default = null) {
+    return idx($this->configuration, $key, $default);
+  }
+
+  private function establishConnection() {
+    $host = $this->getConfiguration('host');
+    $database = $this->getConfiguration('database');
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'      => 'connect',
+        'host'      => $host,
+        'database'  => $database,
+      ));
+
+    // If we receive these errors, we'll retry the connection up to the
+    // retry limit. For other errors, we'll fail immediately.
+    $retry_codes = array(
+      // "Connection Timeout"
+      2002 => true,
+
+      // "Unable to Connect"
+      2003 => true,
+    );
+
+    $max_retries = max(1, $this->getConfiguration('retries', 3));
+    for ($attempt = 1; $attempt <= $max_retries; $attempt++) {
+      try {
+        $conn = $this->connect();
+        $profiler->endServiceCall($call_id, array());
+        break;
+      } catch (AphrontQueryException $ex) {
+        $code = $ex->getCode();
+        if (($attempt < $max_retries) && isset($retry_codes[$code])) {
+          $message = pht(
+            'Retrying database connection to "%s" after connection '.
+            'failure (attempt %d; "%s"; error #%d): %s',
+            $host,
+            $attempt,
+            get_class($ex),
+            $code,
+            $ex->getMessage());
+
+          phlog($message);
+        } else {
+          $profiler->endServiceCall($call_id, array());
+          throw $ex;
+        }
+      }
+    }
+
+    $this->connection = $conn;
+  }
+
+  protected function requireConnection() {
+    if (!$this->connection) {
+      if ($this->connectionPool) {
+        $this->connection = array_pop($this->connectionPool);
+      } else {
+        $this->establishConnection();
+      }
+    }
+    return $this->connection;
+  }
+
+  protected function beginAsyncConnection() {
+    $connection = $this->requireConnection();
+    $this->connection = null;
+    return $connection;
+  }
+
+  protected function endAsyncConnection($connection) {
+    if ($this->connection) {
+      $this->connectionPool[] = $this->connection;
+    }
+    $this->connection = $connection;
+  }
+
+  public function selectAllResults() {
+    $result = array();
+    $res = $this->lastResult;
+    if ($res == null) {
+      throw new Exception(pht('No query result to fetch from!'));
+    }
+    while (($row = $this->fetchAssoc($res))) {
+      $result[] = $row;
+    }
+    return $result;
+  }
+
+  public function executeQuery(PhutilQueryString $query) {
+    $display_query = $query->getMaskedString();
+    $raw_query = $query->getUnmaskedString();
+
+    $this->lastResult = null;
+    $retries = max(1, $this->getConfiguration('retries', 3));
+    while ($retries--) {
+      try {
+        $this->requireConnection();
+        $is_write = $this->checkWrite($raw_query);
+
+        $profiler = PhutilServiceProfiler::getInstance();
+        $call_id = $profiler->beginServiceCall(
+          array(
+            'type'    => 'query',
+            'config'  => $this->configuration,
+            'query'   => $display_query,
+            'write'   => $is_write,
+          ));
+
+        $result = $this->rawQuery($raw_query);
+
+        $profiler->endServiceCall($call_id, array());
+
+        if ($this->nextError) {
+          $result = null;
+        }
+
+        if ($result) {
+          $this->lastResult = $result;
+          break;
+        }
+
+        $this->throwQueryException($this->connection);
+      } catch (AphrontConnectionLostQueryException $ex) {
+        $can_retry = ($retries > 0);
+
+        if ($this->isInsideTransaction()) {
+          // Zero out the transaction state to prevent a second exception
+          // ("program exited with open transaction") from being thrown, since
+          // we're about to throw a more relevant/useful one instead.
+          $state = $this->getTransactionState();
+          while ($state->getDepth()) {
+            $state->decreaseDepth();
+          }
+
+          $can_retry = false;
+        }
+
+        if ($this->isHoldingAnyLock()) {
+          $this->forgetAllLocks();
+          $can_retry = false;
+        }
+
+        $this->close();
+
+        if (!$can_retry) {
+          throw $ex;
+        }
+      }
+    }
+  }
+
+  public function executeRawQueries(array $raw_queries) {
+    if (!$raw_queries) {
+      return array();
+    }
+
+    $is_write = false;
+    foreach ($raw_queries as $key => $raw_query) {
+      $is_write = $is_write || $this->checkWrite($raw_query);
+      $raw_queries[$key] = rtrim($raw_query, "\r\n\t ;");
+    }
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'    => 'multi-query',
+        'config'  => $this->configuration,
+        'queries' => $raw_queries,
+        'write'   => $is_write,
+      ));
+
+    $results = $this->rawQueries($raw_queries);
+
+    $profiler->endServiceCall($call_id, array());
+
+    return $results;
+  }
+
+  protected function processResult($result) {
+    if (!$result) {
+      try {
+        $this->throwQueryException($this->requireConnection());
+      } catch (Exception $ex) {
+        return $ex;
+      }
+    } else if (is_bool($result)) {
+      return $this->getAffectedRows();
+    }
+    $rows = array();
+    while (($row = $this->fetchAssoc($result))) {
+      $rows[] = $row;
+    }
+    $this->freeResult($result);
+    return $rows;
+  }
+
+  protected function checkWrite($raw_query) {
+    // NOTE: The opening "(" allows queries in the form of:
+    //
+    //   (SELECT ...) UNION (SELECT ...)
+    $is_write = !preg_match('/^[(]*(SELECT|SHOW|EXPLAIN)\s/', $raw_query);
+    if ($is_write) {
+      if ($this->getReadOnly()) {
+        throw new Exception(
+          pht(
+            'Attempting to issue a write query on a read-only '.
+            'connection (to database "%s")!',
+            $this->getConfiguration('database')));
+      }
+      AphrontWriteGuard::willWrite();
+      return true;
+    }
+
+    return false;
+  }
+
+  protected function throwQueryException($connection) {
+    if ($this->nextError) {
+      $errno = $this->nextError;
+      $error = pht('Simulated error.');
+      $this->nextError = null;
+    } else {
+      $errno = $this->getErrorCode($connection);
+      $error = $this->getErrorDescription($connection);
+    }
+    $this->throwQueryCodeException($errno, $error);
+  }
+
+  private function throwCommonException($errno, $error) {
+    $message = pht('#%d: %s', $errno, $error);
+
+    switch ($errno) {
+      case 2013: // Connection Dropped
+        throw new AphrontConnectionLostQueryException($message);
+      case 2006: // Gone Away
+        $more = pht(
+          'This error may occur if your configured MySQL "wait_timeout" or '.
+          '"max_allowed_packet" values are too small. This may also indicate '.
+          'that something used the MySQL "KILL " command to kill '.
+          'the connection running the query.');
+        throw new AphrontConnectionLostQueryException("{$message}\n\n{$more}");
+      case 1213: // Deadlock
+        throw new AphrontDeadlockQueryException($message);
+      case 1205: // Lock wait timeout exceeded
+        throw new AphrontLockTimeoutQueryException($message);
+      case 1062: // Duplicate Key
+        // NOTE: In some versions of MySQL we get a key name back here, but
+        // older versions just give us a key index ("key 2") so it's not
+        // portable to parse the key out of the error and attach it to the
+        // exception.
+        throw new AphrontDuplicateKeyQueryException($message);
+      case 1044: // Access denied to database
+      case 1142: // Access denied to table
+      case 1143: // Access denied to column
+      case 1227: // Access denied (e.g., no SUPER for SHOW SLAVE STATUS).
+        throw new AphrontAccessDeniedQueryException($message);
+      case 1045: // Access denied (auth)
+        throw new AphrontInvalidCredentialsQueryException($message);
+      case 1146: // No such table
+      case 1049: // No such database
+      case 1054: // Unknown column "..." in field list
+        throw new AphrontSchemaQueryException($message);
+    }
+
+    // TODO: 1064 is syntax error, and quite terrible in production.
+
+    return null;
+  }
+
+  protected function throwConnectionException($errno, $error, $user, $host) {
+    $this->throwCommonException($errno, $error);
+
+    $message = pht(
+      'Attempt to connect to %s@%s failed with error #%d: %s.',
+      $user,
+      $host,
+      $errno,
+      $error);
+
+    throw new AphrontConnectionQueryException($message, $errno);
+  }
+
+
+  protected function throwQueryCodeException($errno, $error) {
+    $this->throwCommonException($errno, $error);
+
+    $message = pht(
+      '#%d: %s',
+      $errno,
+      $error);
+
+    throw new AphrontQueryException($message, $errno);
+  }
+
+  /**
+   * Force the next query to fail with a simulated error. This should be used
+   * ONLY for unit tests.
+   */
+  public function simulateErrorOnNextQuery($error) {
+    $this->nextError = $error;
+    return $this;
+  }
+
+  /**
+   * Check inserts for characters outside of the BMP. Even with the strictest
+   * settings, MySQL will silently truncate data when it encounters these, which
+   * can lead to data loss and security problems.
+   */
+  protected function validateUTF8String($string) {
+    if (phutil_is_utf8($string)) {
+      return;
+    }
+
+    throw new AphrontCharacterSetQueryException(
+      pht(
+        'Attempting to construct a query using a non-utf8 string when '.
+        'utf8 is expected. Use the `%%B` conversion to escape binary '.
+        'strings data.'));
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php
new file mode 100644
index 0000000000..40a2c6c357
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php
@@ -0,0 +1,233 @@
+validateUTF8String($string);
+    return $this->escapeBinaryString($string);
+  }
+
+  public function escapeBinaryString($string) {
+    return mysql_real_escape_string($string, $this->requireConnection());
+  }
+
+  public function getInsertID() {
+    return mysql_insert_id($this->requireConnection());
+  }
+
+  public function getAffectedRows() {
+    return mysql_affected_rows($this->requireConnection());
+  }
+
+  protected function closeConnection() {
+    mysql_close($this->requireConnection());
+  }
+
+  protected function connect() {
+    if (!function_exists('mysql_connect')) {
+      // We have to '@' the actual call since it can spew all sorts of silly
+      // noise, but it will also silence fatals caused by not having MySQL
+      // installed, which has bitten me on three separate occasions. Make sure
+      // such failures are explicit and loud.
+      throw new Exception(
+        pht(
+          'About to call %s, but the PHP MySQL extension is not available!',
+          'mysql_connect()'));
+    }
+
+    $user = $this->getConfiguration('user');
+    $host = $this->getConfiguration('host');
+    $port = $this->getConfiguration('port');
+
+    if ($port) {
+      $host .= ':'.$port;
+    }
+
+    $database = $this->getConfiguration('database');
+
+    $pass = $this->getConfiguration('pass');
+    if ($pass instanceof PhutilOpaqueEnvelope) {
+      $pass = $pass->openEnvelope();
+    }
+
+    $timeout = $this->getConfiguration('timeout');
+    $timeout_ini = 'mysql.connect_timeout';
+    if ($timeout) {
+      $old_timeout = ini_get($timeout_ini);
+      ini_set($timeout_ini, $timeout);
+    }
+
+    try {
+      $conn = @mysql_connect(
+        $host,
+        $user,
+        $pass,
+        $new_link = true,
+        $flags = 0);
+    } catch (Exception $ex) {
+      if ($timeout) {
+        ini_set($timeout_ini, $old_timeout);
+      }
+      throw $ex;
+    }
+
+    if ($timeout) {
+      ini_set($timeout_ini, $old_timeout);
+    }
+
+    if (!$conn) {
+      $errno = mysql_errno();
+      $error = mysql_error();
+      $this->throwConnectionException($errno, $error, $user, $host);
+    }
+
+    if ($database !== null) {
+      $ret = @mysql_select_db($database, $conn);
+      if (!$ret) {
+        $this->throwQueryException($conn);
+      }
+    }
+
+    $ok = @mysql_set_charset('utf8mb4', $conn);
+    if (!$ok) {
+      mysql_set_charset('binary', $conn);
+    }
+
+    return $conn;
+  }
+
+  protected function rawQuery($raw_query) {
+    return @mysql_query($raw_query, $this->requireConnection());
+  }
+
+  /**
+   * @phutil-external-symbol function mysql_multi_query
+   * @phutil-external-symbol function mysql_fetch_result
+   * @phutil-external-symbol function mysql_more_results
+   * @phutil-external-symbol function mysql_next_result
+   */
+  protected function rawQueries(array $raw_queries) {
+    $conn = $this->requireConnection();
+    $results = array();
+
+    if (!function_exists('mysql_multi_query')) {
+      foreach ($raw_queries as $key => $raw_query) {
+        $results[$key] = $this->processResult($this->rawQuery($raw_query));
+      }
+      return $results;
+    }
+
+    if (!mysql_multi_query(implode("\n;\n\n", $raw_queries), $conn)) {
+      $ex = $this->processResult(false);
+      return array_fill_keys(array_keys($raw_queries), $ex);
+    }
+
+    $processed_all = false;
+    foreach ($raw_queries as $key => $raw_query) {
+      $results[$key] = $this->processResult(@mysql_fetch_result($conn));
+      if (!mysql_more_results($conn)) {
+        $processed_all = true;
+        break;
+      }
+      mysql_next_result($conn);
+    }
+
+    if (!$processed_all) {
+      throw new Exception(
+        pht('There are some results left in the result set.'));
+    }
+
+    return $results;
+  }
+
+  protected function freeResult($result) {
+    mysql_free_result($result);
+  }
+
+  public function supportsParallelQueries() {
+    // fb_parallel_query() doesn't support results with different columns.
+    return false;
+  }
+
+  /**
+   * @phutil-external-symbol function fb_parallel_query
+   */
+  public function executeParallelQueries(
+    array $queries,
+    array $conns = array()) {
+    assert_instances_of($conns, __CLASS__);
+
+    $map = array();
+    $is_write = false;
+    foreach ($queries as $id => $query) {
+      $is_write = $is_write || $this->checkWrite($query);
+      $conn = idx($conns, $id, $this);
+
+      $host = $conn->getConfiguration('host');
+      $port = 0;
+      $match = null;
+      if (preg_match('/(.+):(.+)/', $host, $match)) {
+        list(, $host, $port) = $match;
+      }
+
+      $pass = $conn->getConfiguration('pass');
+      if ($pass instanceof PhutilOpaqueEnvelope) {
+        $pass = $pass->openEnvelope();
+      }
+
+      $map[$id] = array(
+        'sql' => $query,
+        'ip' => $host,
+        'port' => $port,
+        'username' => $conn->getConfiguration('user'),
+        'password' => $pass,
+        'db' => $conn->getConfiguration('database'),
+      );
+    }
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'    => 'multi-query',
+        'queries' => $queries,
+        'write'   => $is_write,
+      ));
+
+    $map = fb_parallel_query($map);
+
+    $profiler->endServiceCall($call_id, array());
+
+    $results = array();
+    $pos = 0;
+    $err_pos = 0;
+    foreach ($queries as $id => $query) {
+      $errno = idx(idx($map, 'errno', array()), $err_pos);
+      $err_pos++;
+      if ($errno) {
+        try {
+          $this->throwQueryCodeException($errno, $map['error'][$id]);
+        } catch (Exception $ex) {
+          $results[$id] = $ex;
+        }
+        continue;
+      }
+      $results[$id] = $map['result'][$pos];
+      $pos++;
+    }
+    return $results;
+  }
+
+  protected function fetchAssoc($result) {
+    return mysql_fetch_assoc($result);
+  }
+
+  protected function getErrorCode($connection) {
+    return mysql_errno($connection);
+  }
+
+  protected function getErrorDescription($connection) {
+    return mysql_error($connection);
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
new file mode 100644
index 0000000000..7a4b5193d5
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
@@ -0,0 +1,244 @@
+validateUTF8String($string);
+    return $this->escapeBinaryString($string);
+  }
+
+  public function escapeBinaryString($string) {
+    return $this->requireConnection()->escape_string($string);
+  }
+
+  public function getInsertID() {
+    return $this->requireConnection()->insert_id;
+  }
+
+  public function getAffectedRows() {
+    return $this->requireConnection()->affected_rows;
+  }
+
+  protected function closeConnection() {
+    if ($this->connectionOpen) {
+      $this->requireConnection()->close();
+      $this->connectionOpen = false;
+    }
+  }
+
+  protected function connect() {
+    if (!class_exists('mysqli', false)) {
+      throw new Exception(pht(
+        'About to call new %s, but the PHP MySQLi extension is not available!',
+        'mysqli()'));
+    }
+
+    $user = $this->getConfiguration('user');
+    $host = $this->getConfiguration('host');
+    $port = $this->getConfiguration('port');
+    $database = $this->getConfiguration('database');
+
+    $pass = $this->getConfiguration('pass');
+    if ($pass instanceof PhutilOpaqueEnvelope) {
+      $pass = $pass->openEnvelope();
+    }
+
+    // If the host is "localhost", the port is ignored and mysqli attempts to
+    // connect over a socket.
+    if ($port) {
+      if ($host === 'localhost' || $host === null) {
+        $host = '127.0.0.1';
+      }
+    }
+
+    $conn = mysqli_init();
+
+    $timeout = $this->getConfiguration('timeout');
+    if ($timeout) {
+      $conn->options(MYSQLI_OPT_CONNECT_TIMEOUT, $timeout);
+    }
+
+    if ($this->getPersistent()) {
+      $host = 'p:'.$host;
+    }
+
+    @$conn->real_connect(
+      $host,
+      $user,
+      $pass,
+      $database,
+      $port);
+
+    $errno = $conn->connect_errno;
+    if ($errno) {
+      $error = $conn->connect_error;
+      $this->throwConnectionException($errno, $error, $user, $host);
+    }
+
+    // See T13238. Attempt to prevent "LOAD DATA LOCAL INFILE", which allows a
+    // malicious server to ask the client for any file. At time of writing,
+    // this option MUST be set after "real_connect()" on all PHP versions.
+    $conn->options(MYSQLI_OPT_LOCAL_INFILE, 0);
+
+    $this->connectionOpen = true;
+
+    $ok = @$conn->set_charset('utf8mb4');
+    if (!$ok) {
+      $ok = $conn->set_charset('binary');
+    }
+
+    return $conn;
+  }
+
+  protected function rawQuery($raw_query) {
+    $conn = $this->requireConnection();
+    $time_limit = $this->getQueryTimeout();
+
+    // If we have a query time limit, run this query synchronously but use
+    // the async API. This allows us to kill queries which take too long
+    // without requiring any configuration on the server side.
+    if ($time_limit && $this->supportsAsyncQueries()) {
+      $conn->query($raw_query, MYSQLI_ASYNC);
+
+      $read = array($conn);
+      $error = array($conn);
+      $reject = array($conn);
+
+      $result = mysqli::poll($read, $error, $reject, $time_limit);
+
+      if ($result === false) {
+        $this->closeConnection();
+        throw new Exception(
+          pht('Failed to poll mysqli connection!'));
+      } else if ($result === 0) {
+        $this->closeConnection();
+        throw new AphrontQueryTimeoutQueryException(
+          pht(
+            'Query timed out after %s second(s)!',
+            new PhutilNumber($time_limit)));
+      }
+
+      return @$conn->reap_async_query();
+    }
+
+    $trap = new PhutilErrorTrap();
+
+    $result = @$conn->query($raw_query);
+
+    $err = $trap->getErrorsAsString();
+    $trap->destroy();
+
+    // See T13238 and PHI1014. Sometimes, the call to "$conn->query()" may fail
+    // without setting an error code on the connection. One way to reproduce
+    // this is to use "LOAD DATA LOCAL INFILE" with "mysqli.allow_local_infile"
+    // disabled.
+
+    // If we have no result and no error code, raise a synthetic query error
+    // with whatever error message was raised as a local PHP warning.
+
+    if (!$result) {
+      $error_code = $this->getErrorCode($conn);
+      if (!$error_code) {
+        if (strlen($err)) {
+          $message = $err;
+         } else {
+          $message = pht(
+            'Call to "mysqli->query()" failed, but did not set an error '.
+            'code or emit an error message.');
+        }
+        $this->throwQueryCodeException(777777, $message);
+      }
+    }
+
+    return $result;
+  }
+
+  protected function rawQueries(array $raw_queries) {
+    $conn = $this->requireConnection();
+
+    $have_result = false;
+    $results = array();
+
+    foreach ($raw_queries as $key => $raw_query) {
+      if (!$have_result) {
+        // End line in front of semicolon to allow single line comments at the
+        // end of queries.
+        $have_result = $conn->multi_query(implode("\n;\n\n", $raw_queries));
+      } else {
+        $have_result = $conn->next_result();
+      }
+
+      array_shift($raw_queries);
+
+      $result = $conn->store_result();
+      if (!$result && !$this->getErrorCode($conn)) {
+        $result = true;
+      }
+      $results[$key] = $this->processResult($result);
+    }
+
+    if ($conn->more_results()) {
+      throw new Exception(
+        pht('There are some results left in the result set.'));
+    }
+
+    return $results;
+  }
+
+  protected function freeResult($result) {
+    $result->free_result();
+  }
+
+  protected function fetchAssoc($result) {
+    return $result->fetch_assoc();
+  }
+
+  protected function getErrorCode($connection) {
+    return $connection->errno;
+  }
+
+  protected function getErrorDescription($connection) {
+    return $connection->error;
+  }
+
+  public function supportsAsyncQueries() {
+    return defined('MYSQLI_ASYNC');
+  }
+
+  public function asyncQuery($raw_query) {
+    $this->checkWrite($raw_query);
+    $async = $this->beginAsyncConnection();
+    $async->query($raw_query, MYSQLI_ASYNC);
+    return $async;
+  }
+
+  public static function resolveAsyncQueries(array $conns, array $asyncs) {
+    assert_instances_of($conns, __CLASS__);
+    assert_instances_of($asyncs, 'mysqli');
+
+    $read = $error = $reject = array();
+    foreach ($asyncs as $async) {
+      $read[] = $error[] = $reject[] = $async;
+    }
+
+    if (!mysqli::poll($read, $error, $reject, 0)) {
+      return array();
+    }
+
+    $results = array();
+    foreach ($read as $async) {
+      $key = array_search($async, $asyncs, $strict = true);
+      $conn = $conns[$key];
+      $conn->endAsyncConnection($async);
+      $results[$key] = $conn->processResult($async->reap_async_query());
+    }
+    return $results;
+  }
+
+}
diff --git a/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php b/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php
new file mode 100644
index 0000000000..a7ca91cfde
--- /dev/null
+++ b/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php
@@ -0,0 +1,4 @@
+query = $query;
+  }
+
+  public function getQuery() {
+    return $this->query;
+  }
+
+}
diff --git a/src/infrastructure/storage/exception/AphrontQueryException.php b/src/infrastructure/storage/exception/AphrontQueryException.php
new file mode 100644
index 0000000000..6b0f51ad02
--- /dev/null
+++ b/src/infrastructure/storage/exception/AphrontQueryException.php
@@ -0,0 +1,6 @@
+resolve();
+ *     } catch (AphrontQueryException $ex) {
+ *     }
+ *   }
+ *
+ * `$result` contains a list of dicts for select queries or number of modified
+ * rows for modification queries.
+ */
+final class QueryFuture extends Future {
+
+  private static $futures = array();
+
+  private $conn;
+  private $query;
+  private $id;
+  private $async;
+  private $profilerCallID;
+
+  public function __construct(
+    AphrontDatabaseConnection $conn,
+    $pattern/* , ... */) {
+
+    $this->conn = $conn;
+
+    $args = func_get_args();
+    $args = array_slice($args, 2);
+    $this->query = vqsprintf($conn, $pattern, $args);
+
+    self::$futures[] = $this;
+    $this->id = last_key(self::$futures);
+  }
+
+  public function isReady() {
+    if ($this->result !== null || $this->exception) {
+      return true;
+    }
+
+    if (!$this->conn->supportsAsyncQueries()) {
+      if ($this->conn->supportsParallelQueries()) {
+        $queries = array();
+        $conns = array();
+        foreach (self::$futures as $id => $future) {
+          $queries[$id] = $future->query;
+          $conns[$id] = $future->conn;
+        }
+        $results = $this->conn->executeParallelQueries($queries, $conns);
+        $this->processResults($results);
+        return true;
+      }
+
+      $conns = array();
+      $conn_queries = array();
+      foreach (self::$futures as $id => $future) {
+        $hash = spl_object_hash($future->conn);
+        $conns[$hash] = $future->conn;
+        $conn_queries[$hash][$id] = $future->query;
+      }
+      foreach ($conn_queries as $hash => $queries) {
+        $this->processResults($conns[$hash]->executeRawQueries($queries));
+      }
+      return true;
+    }
+
+    if (!$this->async) {
+      $profiler = PhutilServiceProfiler::getInstance();
+      $this->profilerCallID = $profiler->beginServiceCall(
+        array(
+          'type'    => 'query',
+          'query'   => $this->query,
+          'async'   => true,
+        ));
+
+      $this->async = $this->conn->asyncQuery($this->query);
+      return false;
+    }
+
+    $conns = array();
+    $asyncs = array();
+    foreach (self::$futures as $id => $future) {
+      if ($future->async) {
+        $conns[$id] = $future->conn;
+        $asyncs[$id] = $future->async;
+      }
+    }
+
+    $this->processResults($this->conn->resolveAsyncQueries($conns, $asyncs));
+
+    if ($this->result !== null || $this->exception) {
+      return true;
+    }
+    return false;
+  }
+
+  private function processResults(array $results) {
+    foreach ($results as $id => $result) {
+      $future = self::$futures[$id];
+      if ($result instanceof Exception) {
+        $future->exception = $result;
+      } else {
+        $future->result = $result;
+      }
+      unset(self::$futures[$id]);
+      if ($future->profilerCallID) {
+        $profiler = PhutilServiceProfiler::getInstance();
+        $profiler->endServiceCall($future->profilerCallID, array());
+      }
+    }
+  }
+}
diff --git a/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php
new file mode 100644
index 0000000000..4a4cf986c4
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php
@@ -0,0 +1,23 @@
+database = $database;
+    $this->table = $table;
+  }
+
+  public function getAphrontRefDatabaseName() {
+    return $this->database;
+  }
+
+  public function getAphrontRefTableName() {
+    return $this->table;
+  }
+
+}
diff --git a/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php
new file mode 100644
index 0000000000..851cd46294
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php
@@ -0,0 +1,8 @@
+setTableName('X');
+    //   $query = qsprintf($conn, '%R', $object);
+    //   $object->setTableName('Y');
+    //
+    // We'd like "$query" to reference "X", reflecting the object as it
+    // existed when it was passed to "qsprintf(...)". It's surprising if the
+    // modification to the object after "qsprintf(...)" can affect "$query".
+
+    $masked_string = xsprintf(
+      'xsprintf_query',
+      array(
+        'escaper' => $escaper,
+        'unmasked' => false,
+      ),
+      $argv);
+
+    $unmasked_string = xsprintf(
+      'xsprintf_query',
+      array(
+        'escaper' => $escaper,
+        'unmasked' => true,
+      ),
+      $argv);
+
+    $this->maskedString = $masked_string;
+    $this->unmaskedString = $unmasked_string;
+  }
+
+  public function __toString() {
+    return $this->getMaskedString();
+  }
+
+  public function getUnmaskedString() {
+    return $this->unmaskedString;
+  }
+
+  public function getMaskedString() {
+    return $this->maskedString;
+  }
+
+}
diff --git a/src/infrastructure/storage/xsprintf/qsprintf.php b/src/infrastructure/storage/xsprintf/qsprintf.php
new file mode 100644
index 0000000000..7b46b34cd4
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/qsprintf.php
@@ -0,0 +1,516 @@
+ and %<.
+ *
+ *   %> ("Prefix")
+ *     Escapes a prefix query for a LIKE clause. For example:
+ *
+ *       //  Find all rows where `name` starts with $prefix.
+ *       qsprintf($escaper, 'WHERE name LIKE %>', $prefix);
+ *
+ *   %< ("Suffix")
+ *     Escapes a suffix query for a LIKE clause. For example:
+ *
+ *       //  Find all rows where `name` ends with $suffix.
+ *       qsprintf($escaper, 'WHERE name LIKE %<', $suffix);
+ *
+ *   %T ("Table")
+ *     Escapes a table name. In most cases, you should use "%R" instead.
+ */
+function qsprintf(PhutilQsprintfInterface $escaper, $pattern /* , ... */) {
+  $args = func_get_args();
+  array_shift($args);
+  return new PhutilQueryString($escaper, $args);
+}
+
+function vqsprintf(PhutilQsprintfInterface $escaper, $pattern, array $argv) {
+  array_unshift($argv, $pattern);
+  return new PhutilQueryString($escaper, $argv);
+}
+
+/**
+ * @{function:xsprintf} callback for encoding SQL queries. See
+ * @{function:qsprintf}.
+ */
+function xsprintf_query($userdata, &$pattern, &$pos, &$value, &$length) {
+  $type = $pattern[$pos];
+
+  if (is_array($userdata)) {
+    $escaper = $userdata['escaper'];
+    $unmasked = $userdata['unmasked'];
+  } else {
+    $escaper = $userdata;
+    $unmasked = false;
+  }
+
+  $next = (strlen($pattern) > $pos + 1) ? $pattern[$pos + 1] : null;
+  $nullable = false;
+  $done = false;
+
+  $prefix   = '';
+
+  if (!($escaper instanceof PhutilQsprintfInterface)) {
+    throw new InvalidArgumentException(pht('Invalid database escaper.'));
+  }
+
+  switch ($type) {
+    case '=': // Nullable test
+      switch ($next) {
+        case 'd':
+        case 'f':
+        case 's':
+          $pattern = substr_replace($pattern, '', $pos, 1);
+          $length  = strlen($pattern);
+          $type    = 's';
+          if ($value === null) {
+            $value = 'IS NULL';
+            $done = true;
+          } else {
+            $prefix = '= ';
+            $type = $next;
+          }
+          break;
+        default:
+          throw new Exception(
+            pht(
+              'Unknown conversion, try %s, %s, or %s.',
+              '%=d',
+              '%=s',
+              '%=f'));
+      }
+      break;
+
+    case 'n': // Nullable...
+      switch ($next) {
+        case 'd': //  ...integer.
+        case 'f': //  ...float.
+        case 's': //  ...string.
+        case 'B': //  ...binary string.
+          $pattern = substr_replace($pattern, '', $pos, 1);
+          $length = strlen($pattern);
+          $type = $next;
+          $nullable = true;
+          break;
+        default:
+          throw new XsprintfUnknownConversionException("%n{$next}");
+      }
+      break;
+
+    case 'L': // List of..
+      qsprintf_check_type($value, "L{$next}", $pattern);
+      $pattern = substr_replace($pattern, '', $pos, 1);
+      $length  = strlen($pattern);
+      $type = 's';
+      $done = true;
+
+      switch ($next) {
+        case 'd': //  ...integers.
+          $value = implode(', ', array_map('intval', $value));
+          break;
+        case 'f': // ...floats.
+          $value = implode(', ', array_map('floatval', $value));
+          break;
+        case 's': // ...strings.
+          foreach ($value as $k => $v) {
+            $value[$k] = "'".$escaper->escapeUTF8String((string)$v)."'";
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'B': // ...binary strings.
+          foreach ($value as $k => $v) {
+            $value[$k] = "'".$escaper->escapeBinaryString((string)$v)."'";
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'C': // ...columns.
+          foreach ($value as $k => $v) {
+            $value[$k] = $escaper->escapeColumnName($v);
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'K': // ...key columns.
+          // This is like "%LC", but for escaping column lists passed to key
+          // specifications. These should be escaped as "`column`(123)". For
+          // example:
+          //
+          //   ALTER TABLE `x` ADD KEY `y` (`u`(16), `v`(32));
+
+          foreach ($value as $k => $v) {
+            $matches = null;
+            if (preg_match('/\((\d+)\)\z/', $v, $matches)) {
+              $v = substr($v, 0, -(strlen($matches[1]) + 2));
+              $prefix_len = '('.((int)$matches[1]).')';
+            } else {
+              $prefix_len = '';
+            }
+
+            $value[$k] = $escaper->escapeColumnName($v).$prefix_len;
+          }
+
+          $value = implode(', ', $value);
+          break;
+        case 'Q':
+          // TODO: Here, and in "%LO", "%LA", and "%LJ", we should eventually
+          // stop accepting strings.
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'O':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          if (count($value) == 1) {
+            $value = '('.head($value).')';
+          } else {
+            $value = '(('.implode(') OR (', $value).'))';
+          }
+          break;
+        case 'A':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          if (count($value) == 1) {
+            $value = '('.head($value).')';
+          } else {
+            $value = '(('.implode(') AND (', $value).'))';
+          }
+          break;
+        case 'J':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          $value = implode(' ', $value);
+          break;
+        default:
+          throw new XsprintfUnknownConversionException("%L{$next}");
+      }
+      break;
+  }
+
+  if (!$done) {
+    qsprintf_check_type($value, $type, $pattern);
+    switch ($type) {
+      case 's': // String
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = "'".$escaper->escapeUTF8String((string)$value)."'";
+        }
+        $type = 's';
+        break;
+
+      case 'B': // Binary String
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = "'".$escaper->escapeBinaryString((string)$value)."'";
+        }
+        $type = 's';
+        break;
+
+      case 'Q': // Query Fragment
+        if ($value instanceof PhutilQueryString) {
+          $value = $value->getUnmaskedString();
+        }
+        $type = 's';
+        break;
+
+      case 'Z': // Raw Query Fragment
+        $type = 's';
+        break;
+
+      case '~': // Like Substring
+      case '>': // Like Prefix
+      case '<': // Like Suffix
+        $value = $escaper->escapeStringForLikeClause($value);
+        switch ($type) {
+          case '~': $value = "'%".$value."%'"; break;
+          case '>': $value = "'".$value."%'"; break;
+          case '<': $value = "'%".$value."'"; break;
+        }
+        $type  = 's';
+        break;
+
+      case 'f': // Float
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = (float)$value;
+        }
+        $type = 's';
+        break;
+
+      case 'd': // Integer
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = (int)$value;
+        }
+        $type = 's';
+        break;
+
+      case 'T': // Table
+      case 'C': // Column
+        $value = $escaper->escapeColumnName($value);
+        $type = 's';
+        break;
+
+      case 'K': // Komment
+        $value = $escaper->escapeMultilineComment($value);
+        $type = 's';
+        break;
+
+      case 'R': // Database + Table Reference
+        $database_name = $value->getAphrontRefDatabaseName();
+        $database_name = $escaper->escapeColumnName($database_name);
+
+        $table_name = $value->getAphrontRefTableName();
+        $table_name = $escaper->escapeColumnName($table_name);
+
+        $value = $database_name.'.'.$table_name;
+        $type = 's';
+        break;
+
+      case 'P': // Password or Secret
+        if ($unmasked) {
+          $value = $value->openEnvelope();
+          $value = "'".$escaper->escapeUTF8String($value)."'";
+        } else {
+          $value = '********';
+        }
+        $type = 's';
+        break;
+
+      default:
+        throw new XsprintfUnknownConversionException($type);
+    }
+  }
+
+  if ($prefix) {
+    $value = $prefix.$value;
+  }
+
+  $pattern[$pos] = $type;
+}
+
+function qsprintf_check_type($value, $type, $query) {
+  switch ($type) {
+    case 'Ld':
+    case 'Ls':
+    case 'LC':
+    case 'LK':
+    case 'LB':
+    case 'Lf':
+    case 'LQ':
+    case 'LA':
+    case 'LO':
+    case 'LJ':
+      if (!is_array($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected array argument for %%%s conversion.', $type));
+      }
+      if (empty($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Array for %%%s conversion is empty.', $type));
+      }
+
+      foreach ($value as $scalar) {
+        qsprintf_check_scalar_type($scalar, $type, $query);
+      }
+      break;
+    default:
+      qsprintf_check_scalar_type($value, $type, $query);
+      break;
+  }
+}
+
+function qsprintf_check_scalar_type($value, $type, $query) {
+  switch ($type) {
+    case 'LQ':
+    case 'LA':
+    case 'LO':
+    case 'LJ':
+      // TODO: See T13217. Remove this eventually.
+      if (is_string($value)) {
+        phlog(
+          pht(
+            'UNSAFE: Raw string ("%s") passed to query ("%s") subclause '.
+            'for "%%%s" conversion. Subclause conversions should be passed '.
+            'a list of PhutilQueryString objects.',
+            $value,
+            $query,
+            $type));
+        break;
+      }
+
+      if (!($value instanceof PhutilQueryString)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Expected a list of PhutilQueryString objects for %%%s '.
+            'conversion.',
+            $type));
+      }
+      break;
+
+    case 'Q':
+      // TODO: See T13217. Remove this eventually.
+      if (is_string($value)) {
+        phlog(
+          pht(
+            'UNSAFE: Raw string ("%s") passed to query ("%s") for "%%Q" '.
+            'conversion. %%Q should be passed a query string.',
+            $value,
+            $query));
+        break;
+      }
+
+      if (!($value instanceof PhutilQueryString)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a PhutilQueryString for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Z':
+      if (!is_string($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Value for "%%Z" conversion should be a raw string.'));
+      }
+      break;
+
+    case 'LC':
+    case 'LK':
+    case 'T':
+    case 'C':
+      if (!is_string($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a string for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Ld':
+    case 'Lf':
+    case 'd':
+    case 'f':
+      if (!is_null($value) && !is_numeric($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a numeric scalar or null for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Ls':
+    case 's':
+    case 'LB':
+    case 'B':
+    case '~':
+    case '>':
+    case '<':
+    case 'K':
+      if (!is_null($value) && !is_scalar($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a scalar or null for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'R':
+      if (!($value instanceof AphrontDatabaseTableRefInterface)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Parameter to "%s" conversion in "qsprintf(...)" is not an '.
+            'instance of AphrontDatabaseTableRefInterface.',
+            '%R'));
+      }
+      break;
+
+    case 'P':
+      if (!($value instanceof PhutilOpaqueEnvelope)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Parameter to "%s" conversion in "qsprintf(...)" is not an '.
+            'instance of PhutilOpaqueEnvelope.',
+            '%P'));
+      }
+      break;
+
+    default:
+      throw new XsprintfUnknownConversionException($type);
+  }
+}
diff --git a/src/infrastructure/storage/xsprintf/queryfx.php b/src/infrastructure/storage/xsprintf/queryfx.php
new file mode 100644
index 0000000000..d1eef612b5
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/queryfx.php
@@ -0,0 +1,27 @@
+setLastActiveEpoch(time());
+  $conn->executeQuery($query);
+}
+
+function queryfx_all(AphrontDatabaseConnection $conn, $sql /* , ... */) {
+  $argv = func_get_args();
+  call_user_func_array('queryfx', $argv);
+  return $conn->selectAllResults();
+}
+
+function queryfx_one(AphrontDatabaseConnection $conn, $sql /* , ... */) {
+  $argv = func_get_args();
+  $ret = call_user_func_array('queryfx_all', $argv);
+  if (count($ret) > 1) {
+    throw new AphrontCountQueryException(
+      pht('Query returned more than one row.'));
+  } else if (count($ret)) {
+    return reset($ret);
+  }
+  return null;
+}

From b6420e0f0ad8f0c14d018880700d0481d3d4f39c Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 08:56:24 -0700
Subject: [PATCH 03/16] Allow repository service lookups to return an ordered
 list of service refs

Summary:
Ref T13286. To support request retries, allow the service lookup method to return an ordered list of structured service references.

Existing callsites continue to immediately discard all but the first reference and pull a URI out of it.

Test Plan: Ran `git pull` in a clustered repository with an "up" node and a "down" node, saw 50% serivce failures and 50% clean pulls.

Maniphest Tasks: T13286

Differential Revision: https://secure.phabricator.com/D20775
---
 src/__phutil_library_map__.php                |  2 +
 .../diffusion/ref/DiffusionServiceRef.php     | 48 ++++++++++++++
 .../diffusion/ssh/DiffusionSSHWorkflow.php    | 22 +++++--
 .../storage/PhabricatorRepository.php         | 62 +++++++++++++------
 4 files changed, 112 insertions(+), 22 deletions(-)
 create mode 100644 src/applications/diffusion/ref/DiffusionServiceRef.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 96d3713b48..212a7b1607 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1021,6 +1021,7 @@ phutil_register_library_map(array(
     'DiffusionSSHWorkflow' => 'applications/diffusion/ssh/DiffusionSSHWorkflow.php',
     'DiffusionSearchQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionSearchQueryConduitAPIMethod.php',
     'DiffusionServeController' => 'applications/diffusion/controller/DiffusionServeController.php',
+    'DiffusionServiceRef' => 'applications/diffusion/ref/DiffusionServiceRef.php',
     'DiffusionSetPasswordSettingsPanel' => 'applications/diffusion/panel/DiffusionSetPasswordSettingsPanel.php',
     'DiffusionSetupException' => 'applications/diffusion/exception/DiffusionSetupException.php',
     'DiffusionSourceHyperlinkEngineExtension' => 'applications/diffusion/engineextension/DiffusionSourceHyperlinkEngineExtension.php',
@@ -6967,6 +6968,7 @@ phutil_register_library_map(array(
     'DiffusionSSHWorkflow' => 'PhabricatorSSHWorkflow',
     'DiffusionSearchQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
     'DiffusionServeController' => 'DiffusionController',
+    'DiffusionServiceRef' => 'Phobject',
     'DiffusionSetPasswordSettingsPanel' => 'PhabricatorSettingsPanel',
     'DiffusionSetupException' => 'Exception',
     'DiffusionSourceHyperlinkEngineExtension' => 'PhabricatorRemarkupHyperlinkEngineExtension',
diff --git a/src/applications/diffusion/ref/DiffusionServiceRef.php b/src/applications/diffusion/ref/DiffusionServiceRef.php
new file mode 100644
index 0000000000..d6e6948e5d
--- /dev/null
+++ b/src/applications/diffusion/ref/DiffusionServiceRef.php
@@ -0,0 +1,48 @@
+uri = $map['uri'];
+    $ref->isWritable = $map['writable'];
+    $ref->devicePHID = $map['devicePHID'];
+    $ref->protocol = $map['protocol'];
+    $ref->deviceName = $map['device'];
+
+    return $ref;
+  }
+
+  public function isWritable() {
+    return $this->isWritable;
+  }
+
+  public function getDevicePHID() {
+    return $this->devicePHID;
+  }
+
+  public function getURI() {
+    return $this->uri;
+  }
+
+  public function getProtocol() {
+    return $this->protocol;
+  }
+
+  public function getDeviceName() {
+    return $this->deviceName;
+  }
+
+}
diff --git a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php
index 08144eb0c9..358418f44c 100644
--- a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php
@@ -73,13 +73,13 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow {
     return $this->shouldProxy;
   }
 
-  protected function getProxyCommand($for_write) {
+  final protected function getAlmanacServiceRefs($for_write) {
     $viewer = $this->getSSHUser();
     $repository = $this->getRepository();
 
     $is_cluster_request = $this->getIsClusterRequest();
 
-    $uri = $repository->getAlmanacServiceURI(
+    $refs = $repository->getAlmanacServiceRefs(
       $viewer,
       array(
         'neverProxy' => $is_cluster_request,
@@ -89,14 +89,28 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow {
         'writable' => $for_write,
       ));
 
-    if (!$uri) {
+    if (!$refs) {
       throw new Exception(
         pht(
           'Failed to generate an intracluster proxy URI even though this '.
           'request was routed as a proxy request.'));
     }
 
-    $uri = new PhutilURI($uri);
+    return $refs;
+  }
+
+  final protected function getProxyCommand($for_write) {
+    $refs = $this->getAlmanacServiceRefs($for_write);
+
+    $ref = head($refs);
+
+    return $this->getProxyCommandForServiceRef($ref);
+  }
+
+  final protected function getProxyCommandForServiceRef(
+    DiffusionServiceRef $ref) {
+
+    $uri = new PhutilURI($ref->getURI());
 
     $username = AlmanacKeys::getClusterSSHUser();
     if ($username === null) {
diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php
index bd15e3e8de..fdc9a695c4 100644
--- a/src/applications/repository/storage/PhabricatorRepository.php
+++ b/src/applications/repository/storage/PhabricatorRepository.php
@@ -1842,6 +1842,20 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
     PhabricatorUser $viewer,
     array $options) {
 
+    $refs = $this->getAlmanacServiceRefs($viewer, $options);
+
+    if (!$refs) {
+      return null;
+    }
+
+    $ref = head($refs);
+    return $ref->getURI();
+  }
+
+  public function getAlmanacServiceRefs(
+    PhabricatorUser $viewer,
+    array $options) {
+
     PhutilTypeSpec::checkMap(
       $options,
       array(
@@ -1856,7 +1870,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
 
     $cache_key = $this->getAlmanacServiceCacheKey();
     if (!$cache_key) {
-      return null;
+      return array();
     }
 
     $cache = PhabricatorCaches::getMutableStructureCache();
@@ -1869,7 +1883,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
     }
 
     if ($uris === null) {
-      return null;
+      return array();
     }
 
     $local_device = AlmanacKeys::getDeviceID();
@@ -1893,7 +1907,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
 
       if ($local_device && $never_proxy) {
         if ($uri['device'] == $local_device) {
-          return null;
+          return array();
         }
       }
 
@@ -1954,15 +1968,20 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
       }
     }
 
+    $refs = array();
+    foreach ($results as $result) {
+      $refs[] = DiffusionServiceRef::newFromDictionary($result);
+    }
+
     // If we require a writable device, remove URIs which aren't writable.
     if ($writable) {
-      foreach ($results as $key => $uri) {
-        if (!$uri['writable']) {
+      foreach ($refs as $key => $ref) {
+        if (!$ref->isWritable()) {
           unset($results[$key]);
         }
       }
 
-      if (!$results) {
+      if (!$refs) {
         throw new Exception(
           pht(
             'This repository ("%s") is not writable with the given '.
@@ -1974,23 +1993,30 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
     }
 
     if ($writable) {
-      $results = $this->sortWritableAlmanacServiceURIs($results);
+      $refs = $this->sortWritableAlmanacServiceRefs($refs);
     } else {
-      shuffle($results);
+      $refs = $this->sortReadableAlmanacServiceRefs($refs);
     }
 
-    $result = head($results);
-    return $result['uri'];
+    return array_values($refs);
   }
 
-  private function sortWritableAlmanacServiceURIs(array $results) {
+  private function sortReadableAlmanacServiceRefs(array $refs) {
+    assert_instances_of($refs, 'DiffusionServiceRef');
+    shuffle($refs);
+    return $refs;
+  }
+
+  private function sortWritableAlmanacServiceRefs(array $refs) {
+    assert_instances_of($refs, 'DiffusionServiceRef');
+
     // See T13109 for discussion of how this method routes requests.
 
     // In the absence of other rules, we'll send traffic to devices randomly.
     // We also want to select randomly among nodes which are equally good
     // candidates to receive the write, and accomplish that by shuffling the
     // list up front.
-    shuffle($results);
+    shuffle($refs);
 
     $order = array();
 
@@ -2002,8 +2028,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
       $this->getPHID());
     if ($writer) {
       $device_phid = $writer->getWriteProperty('devicePHID');
-      foreach ($results as $key => $result) {
-        if ($result['devicePHID'] === $device_phid) {
+      foreach ($refs as $key => $ref) {
+        if ($ref->getDevicePHID() === $device_phid) {
           $order[] = $key;
         }
       }
@@ -2025,8 +2051,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
       }
       $max_devices = array_fuse($max_devices);
 
-      foreach ($results as $key => $result) {
-        if (isset($max_devices[$result['devicePHID']])) {
+      foreach ($refs as $key => $ref) {
+        if (isset($max_devices[$ref->getDevicePHID()])) {
           $order[] = $key;
         }
       }
@@ -2034,9 +2060,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
 
     // Reorder the results, putting any we've selected as preferred targets for
     // the write at the head of the list.
-    $results = array_select_keys($results, $order) + $results;
+    $refs = array_select_keys($refs, $order) + $refs;
 
-    return $results;
+    return $refs;
   }
 
   public function supportsSynchronization() {

From 95fb237ab393989b09a6354d22da622e1e6640c2 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 09:33:04 -0700
Subject: [PATCH 04/16] On Git cluster read failure, retry safe requests

Summary:
Depends on D20775. Ref T13286. When a Git read request fails against a cluster and there are other nodes we could safely try, try more nodes.

We DO NOT retry the request if:

  - the client read anything;
  - the client wrote anything;
  - or we've already retried several times.

Although //some// requests where bytes went over the wire in either direction may be safe to retry, they're rare in practice under Git, and we'd need to puzzle out what state we can safely emit.

Since most types of failure result in an outright connection failure and this catches all of them, it's likely to almost always be sufficient in practice.

Test Plan:
  - Started a cluster with one up node and one down node, pulled it.
  - Half the time, hit the up node and got a clean pull.
  - Half the time, hit the down node and got a connection failure followed by a retry and a clean pull.
  - Forced `$err = 1` so even successful attempts would retry.
  - On hitting the up node, got a "failure" and a decline to retry (bytes already written).
  - On hitting the down node, got a failure and a real retry.
  - (Note that, in both cases, "git pull" exits "0" after the valid wire transaction takes place, even though the remote exited non-zero. If the server gave Git everything it asked for, it doesn't seem to care if the server then exited with an error code.)

Maniphest Tasks: T13286

Differential Revision: https://secure.phabricator.com/D20776
---
 .../diffusion/ssh/DiffusionGitSSHWorkflow.php |  18 ++
 .../ssh/DiffusionGitUploadPackSSHWorkflow.php | 183 ++++++++++++++----
 2 files changed, 163 insertions(+), 38 deletions(-)

diff --git a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
index d9cc8063d5..d8d0116017 100644
--- a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
@@ -8,6 +8,8 @@ abstract class DiffusionGitSSHWorkflow
   private $protocolLog;
 
   private $wireProtocol;
+  private $ioBytesRead = 0;
+  private $ioBytesWritten = 0;
 
   protected function writeError($message) {
     // Git assumes we'll add our own newlines.
@@ -98,6 +100,8 @@ abstract class DiffusionGitSSHWorkflow
     PhabricatorSSHPassthruCommand $command,
     $message) {
 
+    $this->ioBytesWritten += strlen($message);
+
     $log = $this->getProtocolLog();
     if ($log) {
       $log->didWriteBytes($message);
@@ -125,7 +129,21 @@ abstract class DiffusionGitSSHWorkflow
       $message = $protocol->willReadBytes($message);
     }
 
+    // Note that bytes aren't counted until they're emittted by the protocol
+    // layer. This means the underlying command might emit bytes, but if they
+    // are buffered by the protocol layer they won't count as read bytes yet.
+
+    $this->ioBytesRead += strlen($message);
+
     return $message;
   }
 
+  final protected function getIOBytesRead() {
+    return $this->ioBytesRead;
+  }
+
+  final protected function getIOBytesWritten() {
+    return $this->ioBytesWritten;
+  }
+
 }
diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
index 7e1f4a4f33..3e8186190a 100644
--- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
@@ -1,6 +1,10 @@
 setName('git-upload-pack');
@@ -14,39 +18,33 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
   }
 
   protected function executeRepositoryOperations() {
-    $repository = $this->getRepository();
+    $is_proxy = $this->shouldProxy();
+    if ($is_proxy) {
+      return $this->executeRepositoryProxyOperations();
+    }
+
     $viewer = $this->getSSHUser();
+    $repository = $this->getRepository();
     $device = AlmanacKeys::getLiveDevice();
 
     $skip_sync = $this->shouldSkipReadSynchronization();
-    $is_proxy = $this->shouldProxy();
 
-    if ($is_proxy) {
-      $command = $this->getProxyCommand(false);
+    $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
+    if (!$skip_sync) {
+      $cluster_engine = id(new DiffusionRepositoryClusterEngine())
+        ->setViewer($viewer)
+        ->setRepository($repository)
+        ->setLog($this)
+        ->synchronizeWorkingCopyBeforeRead();
 
       if ($device) {
         $this->writeClusterEngineLogMessage(
           pht(
-            "# Fetch received by \"%s\", forwarding to cluster host.\n",
+            "# Cleared to fetch on cluster host \"%s\".\n",
             $device->getName()));
       }
-    } else {
-      $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
-      if (!$skip_sync) {
-        $cluster_engine = id(new DiffusionRepositoryClusterEngine())
-          ->setViewer($viewer)
-          ->setRepository($repository)
-          ->setLog($this)
-          ->synchronizeWorkingCopyBeforeRead();
-
-        if ($device) {
-          $this->writeClusterEngineLogMessage(
-            pht(
-              "# Cleared to fetch on cluster host \"%s\".\n",
-              $device->getName()));
-        }
-      }
     }
+
     $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
 
     $pull_event = $this->newPullEvent();
@@ -60,14 +58,12 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
       $log->didStartSession($command);
     }
 
-    if (!$is_proxy) {
-      if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) {
-        $protocol = new DiffusionGitUploadPackWireProtocol();
-        if ($log) {
-          $protocol->setProtocolLog($log);
-        }
-        $this->setWireProtocol($protocol);
+    if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) {
+      $protocol = new DiffusionGitUploadPackWireProtocol();
+      if ($log) {
+        $protocol->setProtocolLog($log);
       }
+      $this->setWireProtocol($protocol);
     }
 
     $err = $this->newPassthruCommand()
@@ -89,15 +85,7 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
         ->setResultCode(0);
     }
 
-    // TODO: Currently, when proxying, we do not write a log on the proxy.
-    // Perhaps we should write a "proxy log". This is not very useful for
-    // statistics or auditing, but could be useful for diagnostics. Marking
-    // the proxy logs as proxied (and recording devicePHID on all logs) would
-    // make differentiating between these use cases easier.
-
-    if (!$is_proxy) {
-      $pull_event->save();
-    }
+    $pull_event->save();
 
     if (!$err) {
       $this->waitForGitClient();
@@ -106,4 +94,123 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
     return $err;
   }
 
+  private function executeRepositoryProxyOperations() {
+    $device = AlmanacKeys::getLiveDevice();
+    $for_write = false;
+
+    $refs = $this->getAlmanacServiceRefs($for_write);
+    $err = 1;
+
+    while (true) {
+      $ref = head($refs);
+
+      $command = $this->getProxyCommandForServiceRef($ref);
+
+      if ($device) {
+        $this->writeClusterEngineLogMessage(
+          pht(
+            "# Fetch received by \"%s\", forwarding to cluster host \"%s\".\n",
+            $device->getName(),
+            $ref->getDeviceName()));
+      }
+
+      $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
+
+      $future = id(new ExecFuture('%C', $command))
+        ->setEnv($this->getEnvironment());
+
+      $this->didBeginRequest();
+
+      $err = $this->newPassthruCommand()
+        ->setIOChannel($this->getIOChannel())
+        ->setCommandChannelFromExecFuture($future)
+        ->execute();
+
+      $err = 1;
+
+      // TODO: Currently, when proxying, we do not write an event log on the
+      // proxy. Perhaps we should write a "proxy log". This is not very useful
+      // for statistics or auditing, but could be useful for diagnostics.
+      // Marking the proxy logs as proxied (and recording devicePHID on all
+      // logs) would make differentiating between these use cases easier.
+
+      if (!$err) {
+        $this->waitForGitClient();
+        return $err;
+      }
+
+      // Throw away this service: the request failed and we're treating the
+      // failure as persistent, so we don't want to retry another request to
+      // the same host.
+      array_shift($refs);
+
+      // Check if we have more services we can try. If we do, we'll make an
+      // effort to fall back to them below. If not, we can't do anything to
+      // recover so just bail out.
+      if (!$refs) {
+        return $err;
+      }
+
+      $should_retry = $this->shouldRetryRequest();
+      if (!$should_retry) {
+        return $err;
+      }
+
+      // If we haven't bailed out yet, we'll retry the request with the next
+      // service.
+    }
+
+    throw new Exception(pht('Reached an unreachable place.'));
+  }
+
+  private function didBeginRequest() {
+    $this->requestAttempts++;
+    return $this;
+  }
+
+  private function shouldRetryRequest() {
+    $this->requestFailures++;
+
+    if ($this->requestFailures > $this->requestAttempts) {
+      throw new Exception(
+        pht(
+          "Workflow has recorded more failures than attempts; there is a ".
+          "missing call to \"didBeginRequest()\".\n"));
+    }
+
+    $max_failures = 3;
+    if ($this->requestFailures >= $max_failures) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Reached maximum number of retry attempts, giving up.\n"));
+      return false;
+    }
+
+    $read_len = $this->getIOBytesRead();
+    if ($read_len) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Client already read from service (%s bytes), unable to retry.\n",
+          new PhutilNumber($read_len)));
+      return false;
+    }
+
+    $write_len = $this->getIOBytesWritten();
+    if ($write_len) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Client already wrote to service (%s bytes), unable to retry.\n",
+          new PhutilNumber($write_len)));
+      return false;
+    }
+
+    $this->writeClusterEngineLogMessage(
+      pht(
+        "# Service request failed, retrying (making attempt %s of %s).\n",
+        new PhutilNumber($this->requestAttempts + 1),
+        new PhutilNumber($max_failures)));
+
+    return true;
+  }
+
 }

From ff3d1769b47538630de7bf180a360f8cf835ca86 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 10:38:41 -0700
Subject: [PATCH 05/16] Instead of retrying safe reads 3 times, retry each
 eligible service once

Summary: Ref T13286. When retrying a read request, keep retrying as long as we have canididate services. Since we consume a service with each attempt, there's no real reason to abort early, and trying every service allows reads to always succeed even if (for example) 8 nodes of a 16-node cluster are dead because of a severed network link between datacenters.

Test Plan: Ran `git pull` in a clustered repository with an up node and a down node; saw retry count dynamically adjust to available node count.

Maniphest Tasks: T13286

Differential Revision: https://secure.phabricator.com/D20777
---
 .../ssh/DiffusionGitUploadPackSSHWorkflow.php | 21 ++++++-------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
index 3e8186190a..5c0e2588b7 100644
--- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
@@ -126,8 +126,6 @@ final class DiffusionGitUploadPackSSHWorkflow
         ->setCommandChannelFromExecFuture($future)
         ->execute();
 
-      $err = 1;
-
       // TODO: Currently, when proxying, we do not write an event log on the
       // proxy. Perhaps we should write a "proxy log". This is not very useful
       // for statistics or auditing, but could be useful for diagnostics.
@@ -144,14 +142,7 @@ final class DiffusionGitUploadPackSSHWorkflow
       // the same host.
       array_shift($refs);
 
-      // Check if we have more services we can try. If we do, we'll make an
-      // effort to fall back to them below. If not, we can't do anything to
-      // recover so just bail out.
-      if (!$refs) {
-        return $err;
-      }
-
-      $should_retry = $this->shouldRetryRequest();
+      $should_retry = $this->shouldRetryRequest($refs);
       if (!$should_retry) {
         return $err;
       }
@@ -168,7 +159,7 @@ final class DiffusionGitUploadPackSSHWorkflow
     return $this;
   }
 
-  private function shouldRetryRequest() {
+  private function shouldRetryRequest(array $remaining_refs) {
     $this->requestFailures++;
 
     if ($this->requestFailures > $this->requestAttempts) {
@@ -178,11 +169,11 @@ final class DiffusionGitUploadPackSSHWorkflow
           "missing call to \"didBeginRequest()\".\n"));
     }
 
-    $max_failures = 3;
-    if ($this->requestFailures >= $max_failures) {
+    if (!$remaining_refs) {
       $this->writeClusterEngineLogMessage(
         pht(
-          "# Reached maximum number of retry attempts, giving up.\n"));
+          "# All available services failed to serve the request, ".
+          "giving up.\n"));
       return false;
     }
 
@@ -208,7 +199,7 @@ final class DiffusionGitUploadPackSSHWorkflow
       pht(
         "# Service request failed, retrying (making attempt %s of %s).\n",
         new PhutilNumber($this->requestAttempts + 1),
-        new PhutilNumber($max_failures)));
+        new PhutilNumber($this->requestAttempts + count($remaining_refs))));
 
     return true;
   }

From 8ff3a133c4d7ffc02a47d6fc5fb4dae52b2cd0ab Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 11:26:20 -0700
Subject: [PATCH 06/16] Generalize repository proxy retry logic to writes

Summary:
Ref T13286. The current (very safe / conservative) rules for retrying git reads generalize to git writes, so we can use the same ruleset in both cases.

Normally, writes converge rapidly to only having good nodes at the head of the list, so this has less impact than the similar change to reads, but it generally improves consistency and allows us to assert that writes which can be served will be served.

Test Plan:
  - In a cluster with an up node and a down node, pushed changes.
  - Saw a push to the down node fail, retry, and succeed.
  - Did some pulls, saw appropriate retries and success.
  - Note that once one write goes through, the node which received the write always ends up at the head of the writable list, so nodes need to be explicitly thawed to reproduce the failure/retry behavior.

Maniphest Tasks: T13286

Differential Revision: https://secure.phabricator.com/D20778
---
 .../DiffusionGitReceivePackSSHWorkflow.php    |  61 ++++------
 .../diffusion/ssh/DiffusionGitSSHWorkflow.php | 112 +++++++++++++++++
 .../ssh/DiffusionGitUploadPackSSHWorkflow.php | 115 +-----------------
 3 files changed, 137 insertions(+), 151 deletions(-)

diff --git a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php
index abf2a4323e..f59a9b58b4 100644
--- a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php
@@ -14,42 +14,33 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow {
   }
 
   protected function executeRepositoryOperations() {
+    // This is a write, and must have write access.
+    $this->requireWriteAccess();
+
+    $is_proxy = $this->shouldProxy();
+    if ($is_proxy) {
+      return $this->executeRepositoryProxyOperations($for_write = true);
+    }
+
     $host_wait_start = microtime(true);
 
     $repository = $this->getRepository();
     $viewer = $this->getSSHUser();
     $device = AlmanacKeys::getLiveDevice();
 
-    // This is a write, and must have write access.
-    $this->requireWriteAccess();
-
     $cluster_engine = id(new DiffusionRepositoryClusterEngine())
       ->setViewer($viewer)
       ->setRepository($repository)
       ->setLog($this);
 
-    $is_proxy = $this->shouldProxy();
-    if ($is_proxy) {
-      $command = $this->getProxyCommand(true);
-      $did_write = false;
+    $command = csprintf('git-receive-pack %s', $repository->getLocalPath());
+    $cluster_engine->synchronizeWorkingCopyBeforeWrite();
 
-      if ($device) {
-        $this->writeClusterEngineLogMessage(
-          pht(
-            "# Push received by \"%s\", forwarding to cluster host.\n",
-            $device->getName()));
-      }
-    } else {
-      $command = csprintf('git-receive-pack %s', $repository->getLocalPath());
-      $did_write = true;
-      $cluster_engine->synchronizeWorkingCopyBeforeWrite();
-
-      if ($device) {
-        $this->writeClusterEngineLogMessage(
-          pht(
-            "# Ready to receive on cluster host \"%s\".\n",
-            $device->getName()));
-      }
+    if ($device) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Ready to receive on cluster host \"%s\".\n",
+          $device->getName()));
     }
 
     $log = $this->newProtocolLog($is_proxy);
@@ -71,9 +62,7 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow {
 
     // We've committed the write (or rejected it), so we can release the lock
     // without waiting for the client to receive the acknowledgement.
-    if ($did_write) {
-      $cluster_engine->synchronizeWorkingCopyAfterWrite();
-    }
+    $cluster_engine->synchronizeWorkingCopyAfterWrite();
 
     if ($caught) {
       throw $caught;
@@ -85,18 +74,16 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow {
       // When a repository is clustered, we reach this cleanup code on both
       // the proxy and the actual final endpoint node. Don't do more cleanup
       // or logging than we need to.
-      if ($did_write) {
-        $repository->writeStatusMessage(
-          PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
-          PhabricatorRepositoryStatusMessage::CODE_OKAY);
+      $repository->writeStatusMessage(
+        PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
+        PhabricatorRepositoryStatusMessage::CODE_OKAY);
 
-        $host_wait_end = microtime(true);
+      $host_wait_end = microtime(true);
 
-        $this->updatePushLogWithTimingInformation(
-          $this->getClusterEngineLogProperty('writeWait'),
-          $this->getClusterEngineLogProperty('readWait'),
-          ($host_wait_end - $host_wait_start));
-      }
+      $this->updatePushLogWithTimingInformation(
+        $this->getClusterEngineLogProperty('writeWait'),
+        $this->getClusterEngineLogProperty('readWait'),
+        ($host_wait_end - $host_wait_start));
     }
 
     return $err;
diff --git a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
index d8d0116017..292741e34d 100644
--- a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php
@@ -10,6 +10,8 @@ abstract class DiffusionGitSSHWorkflow
   private $wireProtocol;
   private $ioBytesRead = 0;
   private $ioBytesWritten = 0;
+  private $requestAttempts = 0;
+  private $requestFailures = 0;
 
   protected function writeError($message) {
     // Git assumes we'll add our own newlines.
@@ -146,4 +148,114 @@ abstract class DiffusionGitSSHWorkflow
     return $this->ioBytesWritten;
   }
 
+  final protected function executeRepositoryProxyOperations($for_write) {
+    $device = AlmanacKeys::getLiveDevice();
+
+    $refs = $this->getAlmanacServiceRefs($for_write);
+    $err = 1;
+
+    while (true) {
+      $ref = head($refs);
+
+      $command = $this->getProxyCommandForServiceRef($ref);
+
+      if ($device) {
+        $this->writeClusterEngineLogMessage(
+          pht(
+            "# Request received by \"%s\", forwarding to cluster ".
+            "host \"%s\".\n",
+            $device->getName(),
+            $ref->getDeviceName()));
+      }
+
+      $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
+
+      $future = id(new ExecFuture('%C', $command))
+        ->setEnv($this->getEnvironment());
+
+      $this->didBeginRequest();
+
+      $err = $this->newPassthruCommand()
+        ->setIOChannel($this->getIOChannel())
+        ->setCommandChannelFromExecFuture($future)
+        ->execute();
+
+      // TODO: Currently, when proxying, we do not write an event log on the
+      // proxy. Perhaps we should write a "proxy log". This is not very useful
+      // for statistics or auditing, but could be useful for diagnostics.
+      // Marking the proxy logs as proxied (and recording devicePHID on all
+      // logs) would make differentiating between these use cases easier.
+
+      if (!$err) {
+        $this->waitForGitClient();
+        return $err;
+      }
+
+      // Throw away this service: the request failed and we're treating the
+      // failure as persistent, so we don't want to retry another request to
+      // the same host.
+      array_shift($refs);
+
+      $should_retry = $this->shouldRetryRequest($refs);
+      if (!$should_retry) {
+        return $err;
+      }
+
+      // If we haven't bailed out yet, we'll retry the request with the next
+      // service.
+    }
+
+    throw new Exception(pht('Reached an unreachable place.'));
+  }
+
+  private function didBeginRequest() {
+    $this->requestAttempts++;
+    return $this;
+  }
+
+  private function shouldRetryRequest(array $remaining_refs) {
+    $this->requestFailures++;
+
+    if ($this->requestFailures > $this->requestAttempts) {
+      throw new Exception(
+        pht(
+          "Workflow has recorded more failures than attempts; there is a ".
+          "missing call to \"didBeginRequest()\".\n"));
+    }
+
+    if (!$remaining_refs) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# All available services failed to serve the request, ".
+          "giving up.\n"));
+      return false;
+    }
+
+    $read_len = $this->getIOBytesRead();
+    if ($read_len) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Client already read from service (%s bytes), unable to retry.\n",
+          new PhutilNumber($read_len)));
+      return false;
+    }
+
+    $write_len = $this->getIOBytesWritten();
+    if ($write_len) {
+      $this->writeClusterEngineLogMessage(
+        pht(
+          "# Client already wrote to service (%s bytes), unable to retry.\n",
+          new PhutilNumber($write_len)));
+      return false;
+    }
+
+    $this->writeClusterEngineLogMessage(
+      pht(
+        "# Service request failed, retrying (making attempt %s of %s).\n",
+        new PhutilNumber($this->requestAttempts + 1),
+        new PhutilNumber($this->requestAttempts + count($remaining_refs))));
+
+    return true;
+  }
+
 }
diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
index 5c0e2588b7..57c43b5a12 100644
--- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
+++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
@@ -3,9 +3,6 @@
 final class DiffusionGitUploadPackSSHWorkflow
   extends DiffusionGitSSHWorkflow {
 
-  private $requestAttempts = 0;
-  private $requestFailures = 0;
-
   protected function didConstruct() {
     $this->setName('git-upload-pack');
     $this->setArguments(
@@ -20,7 +17,7 @@ final class DiffusionGitUploadPackSSHWorkflow
   protected function executeRepositoryOperations() {
     $is_proxy = $this->shouldProxy();
     if ($is_proxy) {
-      return $this->executeRepositoryProxyOperations();
+      return $this->executeRepositoryProxyOperations($for_write = false);
     }
 
     $viewer = $this->getSSHUser();
@@ -94,114 +91,4 @@ final class DiffusionGitUploadPackSSHWorkflow
     return $err;
   }
 
-  private function executeRepositoryProxyOperations() {
-    $device = AlmanacKeys::getLiveDevice();
-    $for_write = false;
-
-    $refs = $this->getAlmanacServiceRefs($for_write);
-    $err = 1;
-
-    while (true) {
-      $ref = head($refs);
-
-      $command = $this->getProxyCommandForServiceRef($ref);
-
-      if ($device) {
-        $this->writeClusterEngineLogMessage(
-          pht(
-            "# Fetch received by \"%s\", forwarding to cluster host \"%s\".\n",
-            $device->getName(),
-            $ref->getDeviceName()));
-      }
-
-      $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
-
-      $future = id(new ExecFuture('%C', $command))
-        ->setEnv($this->getEnvironment());
-
-      $this->didBeginRequest();
-
-      $err = $this->newPassthruCommand()
-        ->setIOChannel($this->getIOChannel())
-        ->setCommandChannelFromExecFuture($future)
-        ->execute();
-
-      // TODO: Currently, when proxying, we do not write an event log on the
-      // proxy. Perhaps we should write a "proxy log". This is not very useful
-      // for statistics or auditing, but could be useful for diagnostics.
-      // Marking the proxy logs as proxied (and recording devicePHID on all
-      // logs) would make differentiating between these use cases easier.
-
-      if (!$err) {
-        $this->waitForGitClient();
-        return $err;
-      }
-
-      // Throw away this service: the request failed and we're treating the
-      // failure as persistent, so we don't want to retry another request to
-      // the same host.
-      array_shift($refs);
-
-      $should_retry = $this->shouldRetryRequest($refs);
-      if (!$should_retry) {
-        return $err;
-      }
-
-      // If we haven't bailed out yet, we'll retry the request with the next
-      // service.
-    }
-
-    throw new Exception(pht('Reached an unreachable place.'));
-  }
-
-  private function didBeginRequest() {
-    $this->requestAttempts++;
-    return $this;
-  }
-
-  private function shouldRetryRequest(array $remaining_refs) {
-    $this->requestFailures++;
-
-    if ($this->requestFailures > $this->requestAttempts) {
-      throw new Exception(
-        pht(
-          "Workflow has recorded more failures than attempts; there is a ".
-          "missing call to \"didBeginRequest()\".\n"));
-    }
-
-    if (!$remaining_refs) {
-      $this->writeClusterEngineLogMessage(
-        pht(
-          "# All available services failed to serve the request, ".
-          "giving up.\n"));
-      return false;
-    }
-
-    $read_len = $this->getIOBytesRead();
-    if ($read_len) {
-      $this->writeClusterEngineLogMessage(
-        pht(
-          "# Client already read from service (%s bytes), unable to retry.\n",
-          new PhutilNumber($read_len)));
-      return false;
-    }
-
-    $write_len = $this->getIOBytesWritten();
-    if ($write_len) {
-      $this->writeClusterEngineLogMessage(
-        pht(
-          "# Client already wrote to service (%s bytes), unable to retry.\n",
-          new PhutilNumber($write_len)));
-      return false;
-    }
-
-    $this->writeClusterEngineLogMessage(
-      pht(
-        "# Service request failed, retrying (making attempt %s of %s).\n",
-        new PhutilNumber($this->requestAttempts + 1),
-        new PhutilNumber($this->requestAttempts + count($remaining_refs))));
-
-    return true;
-  }
-
 }

From d9badba14786e786d6c76e11a5860e81151ce708 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 12:06:17 -0700
Subject: [PATCH 07/16] Give "bin/config" a friendlier error message if
 "local.json" is not writable

Summary: Ref T13403. We currently emit a useful error message, but it's not tailored and has a stack trace. Since this is a relatively routine error and on the first-time-setup path, tailor it so it's a bit nicer.

Test Plan:
  - Ran `bin/config set ...` with an unwritable "local.json".
  - Ran `bin/config set ...` normally.

Maniphest Tasks: T13403

Differential Revision: https://secure.phabricator.com/D20779
---
 .../PhabricatorConfigManagementSetWorkflow.php    | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
index 9eb83bd61e..6ad2db4471 100644
--- a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
+++ b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
@@ -140,11 +140,22 @@ final class PhabricatorConfigManagementSetWorkflow
         'Wrote configuration key "%s" to database storage.',
         $key);
     } else {
-      $config_source = id(new PhabricatorConfigLocalSource())
-        ->setKeys(array($key => $value));
+      $config_source = new PhabricatorConfigLocalSource();
 
       $local_path = $config_source->getReadablePath();
 
+      try {
+        Filesystem::assertWritable($local_path);
+      } catch (FilesystemException $ex) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Local path "%s" is not writable. This file must be writable '.
+            'so that "bin/config" can store configuration.',
+            Filesystem::readablePath($local_path)));
+      }
+
+      $config_source->setKeys(array($key => $value));
+
       $write_message = pht(
         'Wrote configuration key "%s" to local storage (in file "%s").',
         $key,

From f8eec38c941954b870940e99ffb4860e4a16eb8d Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 12:09:20 -0700
Subject: [PATCH 08/16] When "mysqli->real_connect()" fails without setting an
 error code, recover more gracefully

Summary: Depends on D20779. Ref T13403. Bad parameters may cause this call to fail without setting an error code; if it does, catch the issue and go down the normal connection error pathway.

Test Plan:
  - With "mysql.port" set to "quack", ran `bin/storage probe`.
  - Before: wild mess of warnings as the code continued below and failed when trying to interact with the connection.
  - After: clean connection failure with a useful error message.

Maniphest Tasks: T13403

Differential Revision: https://secure.phabricator.com/D20780
---
 .../AphrontBaseMySQLDatabaseConnection.php    |  3 ++
 .../mysql/AphrontMySQLiDatabaseConnection.php | 32 +++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
index 0f9201b02d..313ea5a3b0 100644
--- a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
+++ b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
@@ -10,6 +10,9 @@ abstract class AphrontBaseMySQLDatabaseConnection
 
   private $nextError;
 
+  const CALLERROR_QUERY = 777777;
+  const CALLERROR_CONNECT = 777778;
+
   abstract protected function connect();
   abstract protected function rawQuery($raw_query);
   abstract protected function rawQueries(array $raw_queries);
diff --git a/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
index 7a4b5193d5..6a0bc759a7 100644
--- a/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
+++ b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
@@ -68,19 +68,47 @@ final class AphrontMySQLiDatabaseConnection
       $host = 'p:'.$host;
     }
 
-    @$conn->real_connect(
+    $trap = new PhutilErrorTrap();
+
+    $ok = @$conn->real_connect(
       $host,
       $user,
       $pass,
       $database,
       $port);
 
+    $call_error = $trap->getErrorsAsString();
+    $trap->destroy();
+
     $errno = $conn->connect_errno;
     if ($errno) {
       $error = $conn->connect_error;
       $this->throwConnectionException($errno, $error, $user, $host);
     }
 
+    // See T13403. If the parameters to "real_connect()" are wrong, it may
+    // fail without setting an error code. In this case, raise a generic
+    // exception. (One way to reproduce this is to pass a string to the
+    // "port" parameter.)
+
+    if (!$ok) {
+      if (strlen($call_error)) {
+        $message = pht(
+          'mysqli->real_connect() failed: %s',
+          $call_error);
+      } else {
+        $message = pht(
+          'mysqli->real_connect() failed, but did not set an error code '.
+          'or emit a message.');
+      }
+
+      $this->throwConnectionException(
+        self::CALLERROR_CONNECT,
+        $message,
+        $user,
+        $host);
+    }
+
     // See T13238. Attempt to prevent "LOAD DATA LOCAL INFILE", which allows a
     // malicious server to ask the client for any file. At time of writing,
     // this option MUST be set after "real_connect()" on all PHP versions.
@@ -152,7 +180,7 @@ final class AphrontMySQLiDatabaseConnection
             'Call to "mysqli->query()" failed, but did not set an error '.
             'code or emit an error message.');
         }
-        $this->throwQueryCodeException(777777, $message);
+        $this->throwQueryCodeException(self::CALLERROR_QUERY, $message);
       }
     }
 

From e0d6994adb1ce5b961a67808d53c924741da4850 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 12:16:17 -0700
Subject: [PATCH 09/16] Use the "@" operator to silence connection retry
 messages if initializing the stack with database config optional

Summary:
Depends on D20780. Ref T13403. During initial setup, it's routine to run "bin/config" with a bad database config. We start the stack in "config optional" mode to anticipate this.

However, even in this mode, we may emit warnings if the connection fails in certain ways. These warnings aren't useful; suppress them with "@".

(Possibly this message should move from "phlog()" to "--trace" at some point, but it has a certain amount of context/history around it.)

Test Plan:
  - Configured MySQL to fail with a retryable error, e.g. good host but bad port.
  - Ran `bin/config set ...`.
  - Before: saw retry warnings on stderr.
  - After: no retry warnings on stderr.
  - (Turned off suppression code artificially and verified warnings still appear under normal startup.)

Maniphest Tasks: T13403

Differential Revision: https://secure.phabricator.com/D20781
---
 src/infrastructure/env/PhabricatorEnv.php          | 14 +++++++++++---
 .../mysql/AphrontBaseMySQLDatabaseConnection.php   |  9 ++++++++-
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/infrastructure/env/PhabricatorEnv.php b/src/infrastructure/env/PhabricatorEnv.php
index 24fb940c9a..d5f990065d 100644
--- a/src/infrastructure/env/PhabricatorEnv.php
+++ b/src/infrastructure/env/PhabricatorEnv.php
@@ -249,9 +249,17 @@ final class PhabricatorEnv extends Phobject {
     }
 
     try {
-      $stack->pushSource(
-        id(new PhabricatorConfigDatabaseSource('default'))
-          ->setName(pht('Database')));
+      // See T13403. If we're starting up in "config optional" mode, suppress
+      // messages about connection retries.
+      if ($config_optional) {
+        $database_source = @new PhabricatorConfigDatabaseSource('default');
+      } else {
+        $database_source = new PhabricatorConfigDatabaseSource('default');
+      }
+
+      $database_source->setName(pht('Database'));
+
+      $stack->pushSource($database_source);
     } catch (AphrontSchemaQueryException $exception) {
       // If the database is not available, just skip this configuration
       // source. This happens during `bin/storage upgrade`, `bin/conf` before
diff --git a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
index 313ea5a3b0..6faf10e2c6 100644
--- a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
+++ b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
@@ -126,7 +126,14 @@ abstract class AphrontBaseMySQLDatabaseConnection
             $code,
             $ex->getMessage());
 
-          phlog($message);
+          // See T13403. If we're silenced with the "@" operator, don't log
+          // this connection attempt. This keeps things quiet if we're
+          // running a setup workflow like "bin/config" and expect that the
+          // database credentials will often be incorrect.
+
+          if (error_reporting()) {
+            phlog($message);
+          }
         } else {
           $profiler->endServiceCall($call_id, array());
           throw $ex;

From 22b075df97168e0812205482346638f53c62da14 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Tue, 3 Sep 2019 16:33:07 -0700
Subject: [PATCH 10/16] Fix "ONLY_FULL_GROUP_BY" issue in SystemAction queries

Summary: Ref T13404. This query is invalid under "sql_mode=ONLY_FULL_GROUP_BY". Rewrite it to avoid interacting with `actorIdentity` at all; this is a little more robust in the presence of weird data and not really more complicated.

Test Plan:
  - Enabled "ONLY_FULL_GROUP_BY".
  - Hit system actions (e.g., login).
    - Before: error.
    - After: clean login.
  - Tried to login with a bad password many times in a row, got properly limited by the system action rate limiter.

Maniphest Tasks: T13404

Differential Revision: https://secure.phabricator.com/D20782
---
 .../engine/PhabricatorSystemActionEngine.php  | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/applications/system/engine/PhabricatorSystemActionEngine.php b/src/applications/system/engine/PhabricatorSystemActionEngine.php
index c097fa04a4..6d6f9eacfd 100644
--- a/src/applications/system/engine/PhabricatorSystemActionEngine.php
+++ b/src/applications/system/engine/PhabricatorSystemActionEngine.php
@@ -100,32 +100,34 @@ final class PhabricatorSystemActionEngine extends Phobject {
 
     $actor_hashes = array();
     foreach ($actors as $actor) {
-      $actor_hashes[] = PhabricatorHash::digestForIndex($actor);
+      $digest = PhabricatorHash::digestForIndex($actor);
+      $actor_hashes[$digest] = $actor;
     }
 
     $log = new PhabricatorSystemActionLog();
 
     $window = self::getWindow();
 
-    $conn_r = $log->establishConnection('r');
-    $scores = queryfx_all(
-      $conn_r,
-      'SELECT actorIdentity, SUM(score) totalScore FROM %T
+    $conn = $log->establishConnection('r');
+
+    $rows = queryfx_all(
+      $conn,
+      'SELECT actorHash, SUM(score) totalScore FROM %T
         WHERE action = %s AND actorHash IN (%Ls)
           AND epoch >= %d GROUP BY actorHash',
       $log->getTableName(),
       $action->getActionConstant(),
-      $actor_hashes,
-      (time() - $window));
+      array_keys($actor_hashes),
+      (PhabricatorTime::getNow() - $window));
 
-    $scores = ipull($scores, 'totalScore', 'actorIdentity');
+    $rows = ipull($rows, 'totalScore', 'actorHash');
 
-    foreach ($scores as $key => $score) {
-      $scores[$key] = $score / $window;
+    $scores = array();
+    foreach ($actor_hashes as $digest => $actor) {
+      $score = idx($rows, $digest, 0);
+      $scores[$actor] = ($score / $window);
     }
 
-    $scores = $scores + array_fill_keys($actors, 0);
-
     return $scores;
   }
 

From f7290bbbf220a362e7b81ab82bd04e93cacdbcf4 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Wed, 4 Sep 2019 07:09:39 -0700
Subject: [PATCH 11/16] Update a straggling "getAuthorities()" call in Fund

Summary: Ref T13366. The "authorities" mechanism was replaced, but I missed this callsite. Update it to use the request cache mechanism.

Test Plan: As a user without permission to view some initiatives, viewed a list of initiatives.

Maniphest Tasks: T13366

Differential Revision: https://secure.phabricator.com/D20783
---
 src/applications/fund/storage/FundInitiative.php | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/applications/fund/storage/FundInitiative.php b/src/applications/fund/storage/FundInitiative.php
index 5e4dd48026..1ebbb35ef1 100644
--- a/src/applications/fund/storage/FundInitiative.php
+++ b/src/applications/fund/storage/FundInitiative.php
@@ -136,12 +136,12 @@ final class FundInitiative extends FundDAO
     }
 
     if ($capability == PhabricatorPolicyCapability::CAN_VIEW) {
-      foreach ($viewer->getAuthorities() as $authority) {
-        if ($authority instanceof PhortuneMerchant) {
-          if ($authority->getPHID() == $this->getMerchantPHID()) {
-            return true;
-          }
-        }
+      $can_merchant = PhortuneMerchantQuery::canViewersEditMerchants(
+        array($viewer->getPHID()),
+        array($this->getMerchantPHID()));
+
+      if ($can_merchant) {
+        return true;
       }
     }
 

From 764db4869cb05b5fc7894414a7cb915a274ba476 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Wed, 4 Sep 2019 10:02:51 -0700
Subject: [PATCH 12/16] Make "bin/storage destroy" target individual hosts in
 database cluster mode

Summary:
Ref T13336. Currently, "bin/storage destroy" destroys every master. This is wonderfully destructive, but if replication fails it's useful to be able to destroy only a replica.

Operate on a single host, and require "--host" to target the operation in cluster mode, so `bin/storage destroy --host dbreplica001` is a useful operation.

Test Plan: Ran `bin/storage destroy` with various flags locally. Will destroy `secure002` and refresh replication.

Maniphest Tasks: T13336

Differential Revision: https://secure.phabricator.com/D20784
---
 .../cluster/PhabricatorDatabaseRef.php        |   3 +
 .../PhabricatorStorageManagementAPI.php       |   4 +
 ...icatorStorageManagementDestroyWorkflow.php | 134 +++++++++++-------
 3 files changed, 87 insertions(+), 54 deletions(-)

diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
index 89435b5869..478f95750b 100644
--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
@@ -221,6 +221,9 @@ final class PhabricatorDatabaseRef
     return $this->replicaRefs;
   }
 
+  public function getDisplayName() {
+    return $this->getRefKey();
+  }
 
   public function getRefKey() {
     $host = $this->getHost();
diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
index b838c8a5d9..a6a0d74593 100644
--- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
+++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
@@ -89,6 +89,10 @@ final class PhabricatorStorageManagementAPI extends Phobject {
     return $this->namespace.'_'.$fragment;
   }
 
+  public function getDisplayName() {
+    return $this->getRef()->getDisplayName();
+  }
+
   public function getDatabaseList(array $patches, $only_living = false) {
     assert_instances_of($patches, 'PhabricatorStoragePatch');
 
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
index 7d0946c8c3..9b718e231d 100644
--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
@@ -21,86 +21,112 @@ final class PhabricatorStorageManagementDestroyWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
-    $console = PhutilConsole::getConsole();
+    $api = $this->getSingleAPI();
+
+    $host_display = $api->getDisplayName();
 
     if (!$this->isDryRun() && !$this->isForce()) {
       if ($args->getArg('unittest-fixtures')) {
-        $console->writeOut(
-          phutil_console_wrap(
-            pht(
-              'Are you completely sure you really want to destroy all unit '.
-              'test fixure data? This operation can not be undone.')));
+        $warning = pht(
+          'Are you completely sure you really want to destroy all unit '.
+          'test fixure data on host "%s"? This operation can not be undone.',
+          $host_display);
+
+        echo tsprintf(
+          '%B',
+          id(new PhutilConsoleBlock())
+            ->addParagraph($warning)
+            ->drawConsoleString());
+
         if (!phutil_console_confirm(pht('Destroy all unit test data?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
       } else {
-        $console->writeOut(
-          phutil_console_wrap(
-            pht(
-              'Are you completely sure you really want to permanently destroy '.
-              'all storage for Phabricator data? This operation can not be '.
-              'undone and your data will not be recoverable if you proceed.')));
+        $warning = pht(
+          'Are you completely sure you really want to permanently destroy '.
+          'all storage for Phabricator data on host "%s"? This operation '.
+          'can not be undone and your data will not be recoverable if '.
+          'you proceed.',
+          $host_display);
+
+        echo tsprintf(
+          '%B',
+          id(new PhutilConsoleBlock())
+            ->addParagraph($warning)
+            ->drawConsoleString());
 
         if (!phutil_console_confirm(pht('Permanently destroy all data?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
 
         if (!phutil_console_confirm(pht('Really destroy all data forever?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
       }
     }
 
-    $apis = $this->getMasterAPIs();
-    foreach ($apis as $api) {
-      $patches = $this->getPatches();
+    $patches = $this->getPatches();
 
-      if ($args->getArg('unittest-fixtures')) {
-        $conn = $api->getConn(null);
-        $databases = queryfx_all(
-          $conn,
-          'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
-          'FROM INFORMATION_SCHEMA.TABLES '.
-          'WHERE TABLE_SCHEMA LIKE %>',
-          PhabricatorTestCase::NAMESPACE_PREFIX);
-        $databases = ipull($databases, 'db');
-      } else {
-        $databases   = $api->getDatabaseList($patches);
-        $databases[] = $api->getDatabaseName('meta_data');
+    if ($args->getArg('unittest-fixtures')) {
+      $conn = $api->getConn(null);
+      $databases = queryfx_all(
+        $conn,
+        'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
+        'FROM INFORMATION_SCHEMA.TABLES '.
+        'WHERE TABLE_SCHEMA LIKE %>',
+        PhabricatorTestCase::NAMESPACE_PREFIX);
+      $databases = ipull($databases, 'db');
+    } else {
+      $databases = $api->getDatabaseList($patches);
+      $databases[] = $api->getDatabaseName('meta_data');
 
-        // These are legacy databases that were dropped long ago. See T2237.
-        $databases[] = $api->getDatabaseName('phid');
-        $databases[] = $api->getDatabaseName('directory');
-      }
+      // These are legacy databases that were dropped long ago. See T2237.
+      $databases[] = $api->getDatabaseName('phid');
+      $databases[] = $api->getDatabaseName('directory');
+    }
 
-      foreach ($databases as $database) {
-        if ($this->isDryRun()) {
-          $console->writeOut(
-            "%s\n",
-            pht("DRYRUN: Would drop database '%s'.", $database));
-        } else {
-          $console->writeOut(
-            "%s\n",
-            pht("Dropping database '%s'...", $database));
-          queryfx(
-            $api->getConn(null),
-            'DROP DATABASE IF EXISTS %T',
-            $database);
-        }
-      }
+    asort($databases);
 
-      if (!$this->isDryRun()) {
-        $console->writeOut(
-          "%s\n",
+    foreach ($databases as $database) {
+      if ($this->isDryRun()) {
+        $this->logInfo(
+          pht('DRY RUN'),
           pht(
-            'Storage on "%s" was destroyed.',
-            $api->getRef()->getRefKey()));
+            'Would drop database "%s" on host "%s".',
+            $database,
+            $host_display));
+      } else {
+        $this->logWarn(
+          pht('DESTROY'),
+          pht(
+            'Dropping database "%s" on host "%s"...',
+            $database,
+            $host_display));
+
+        queryfx(
+          $api->getConn(null),
+          'DROP DATABASE IF EXISTS %T',
+          $database);
       }
     }
 
+    if (!$this->isDryRun()) {
+      $this->logOkay(
+        pht('DONE'),
+        pht(
+          'Storage on "%s" was destroyed.',
+          $host_display));
+    }
+
     return 0;
   }
 

From adc2002d2870f3ca277723a2e81fff1b6922cd81 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Thu, 5 Sep 2019 03:43:22 -0700
Subject: [PATCH 13/16] Make it easier to parse "X-Forwarded-For" with one or
 more load balancers

Summary:
Fixes T13392. If you have 17 load balancers in sequence, Phabricator will receive requests with at least 17 "X-Forwarded-For" components in the header.

We want to select the 17th-from-last element, since prior elements are not trustworthy.

This currently isn't very easy/obvious, and you have to add a kind of sketchy piece of custom code to `preamble.php` to do any "X-Forwarded-For" parsing. Make handling this correctly easier.

Test Plan:
  - Ran unit tests.
  - Configured my local `preamble.php` to call `preamble_trust_x_forwarded_for_header(4)`, then made `/debug/` dump the header and the final value of `REMOTE_ADDR`.

```
$ curl http://local.phacility.com/debug/

    
HTTP_X_FORWARDED_FOR =
   FINAL REMOTE_ADDR = 127.0.0.1

    
```

```
$ curl -H 'X-Forwarded-For: 1.1.1.1, 2.2.2.2, 3.3.3.3, 4.4.4.4, 5.5.5.5, 6.6.6.6' http://local.phacility.com/debug/

    
HTTP_X_FORWARDED_FOR = 1.1.1.1, 2.2.2.2, 3.3.3.3, 4.4.4.4, 5.5.5.5, 6.6.6.6
   FINAL REMOTE_ADDR = 3.3.3.3

    
```

```
$ curl -H 'X-Forwarded-For: 5.5.5.5, 6.6.6.6' http://local.phacility.com/debug/

    
HTTP_X_FORWARDED_FOR = 5.5.5.5, 6.6.6.6
   FINAL REMOTE_ADDR = 5.5.5.5

    
```

Maniphest Tasks: T13392

Differential Revision: https://secure.phabricator.com/D20785
---
 src/__phutil_library_map__.php                |  2 +
 .../configuring_preamble.diviner              | 51 ++++++------
 src/infrastructure/env/PhabricatorEnv.php     |  5 ++
 .../__tests__/PhabricatorPreambleTestCase.php | 74 ++++++++++++++++++
 support/startup/preamble-utils.php            | 77 +++++++++++++++++++
 webroot/index.php                             |  1 +
 6 files changed, 187 insertions(+), 23 deletions(-)
 create mode 100644 src/infrastructure/util/__tests__/PhabricatorPreambleTestCase.php
 create mode 100644 support/startup/preamble-utils.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 212a7b1607..fa9a5ce903 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -4201,6 +4201,7 @@ phutil_register_library_map(array(
     'PhabricatorPolicyTestObject' => 'applications/policy/__tests__/PhabricatorPolicyTestObject.php',
     'PhabricatorPolicyType' => 'applications/policy/constants/PhabricatorPolicyType.php',
     'PhabricatorPonderApplication' => 'applications/ponder/application/PhabricatorPonderApplication.php',
+    'PhabricatorPreambleTestCase' => 'infrastructure/util/__tests__/PhabricatorPreambleTestCase.php',
     'PhabricatorPrimaryEmailUserLogType' => 'applications/people/userlog/PhabricatorPrimaryEmailUserLogType.php',
     'PhabricatorProfileMenuEditEngine' => 'applications/search/editor/PhabricatorProfileMenuEditEngine.php',
     'PhabricatorProfileMenuEditor' => 'applications/search/editor/PhabricatorProfileMenuEditor.php',
@@ -10681,6 +10682,7 @@ phutil_register_library_map(array(
     ),
     'PhabricatorPolicyType' => 'PhabricatorPolicyConstants',
     'PhabricatorPonderApplication' => 'PhabricatorApplication',
+    'PhabricatorPreambleTestCase' => 'PhabricatorTestCase',
     'PhabricatorPrimaryEmailUserLogType' => 'PhabricatorUserLogType',
     'PhabricatorProfileMenuEditEngine' => 'PhabricatorEditEngine',
     'PhabricatorProfileMenuEditor' => 'PhabricatorApplicationTransactionEditor',
diff --git a/src/docs/user/configuration/configuring_preamble.diviner b/src/docs/user/configuration/configuring_preamble.diviner
index fc804e9072..5299afa27d 100644
--- a/src/docs/user/configuration/configuring_preamble.diviner
+++ b/src/docs/user/configuration/configuring_preamble.diviner
@@ -15,10 +15,9 @@ You can use a special preamble script to make arbitrary adjustments to the
 environment and some parts of Phabricator's configuration in order to fix these
 problems and set up the environment which Phabricator expects.
 
-NOTE: This is an advanced feature. Most installs should not need to configure
-a preamble script.
 
-= Creating a Preamble Script =
+Creating a Preamble Script
+==========================
 
 To create a preamble script, write a file to:
 
@@ -37,6 +36,7 @@ If present, this script will be executed at the very beginning of each web
 request, allowing you to adjust the environment. For common adjustments and
 examples, see the next sections.
 
+
 Adjusting Client IPs
 ====================
 
@@ -44,9 +44,15 @@ If your install is behind a load balancer, Phabricator may incorrectly detect
 all requests as originating from the load balancer, rather than from the
 correct client IPs.
 
-If this is the case and some other header (like `X-Forwarded-For`) is known to
-be trustworthy, you can read the header and overwrite the `REMOTE_ADDR` value
-so Phabricator can figure out the client IP correctly.
+In common cases where networks are configured like this, the `X-Forwarded-For`
+header will have trustworthy information about the real client IP. You
+can use the function `preamble_trust_x_forwarded_for_header()` in your
+preamble to tell Phabricator that you expect to receive requests from a
+load balancer or proxy which modifies this header:
+
+```name="Trust X-Forwarded-For Header", lang=php
+preamble_trust_x_forwarded_for_header();
+```
 
 You should do this //only// if the `X-Forwarded-For` header is known to be
 trustworthy. In particular, if users can make requests to the web server
@@ -54,30 +60,29 @@ directly, they can provide an arbitrary `X-Forwarded-For` header, and thereby
 spoof an arbitrary client IP.
 
 The `X-Forwarded-For` header may also contain a list of addresses if a request
-has been forwarded through multiple loadbalancers. Using a snippet like this
-will usually handle most situations correctly:
+has been forwarded through multiple load balancers. If you know that requests
+on your network are routed through `N` trustworthy devices, you can specify
+that `N` to tell the function how many layers of `X-Forwarded-For` to discard:
 
+```name="Trust X-Forwarded-For Header, Multiple Layers", lang=php
+preamble_trust_x_forwarded_for_header(3);
 ```
-name=Overwrite REMOTE_ADDR with X-Forwarded-For
- '1.2.3.4',
+        'layers' => 1,
+        'expect' => '1.2.3.4',
+      ),
+
+      // In this case, the LB received a request which already had an
+      // "X-Forwarded-For" header. This might be legitimate (in the case of
+      // a CDN request) or illegitimate (in the case of a client making
+      // things up). We don't want to trust it.
+      array(
+        'header' => '9.9.9.9, 1.2.3.4',
+        'layers' => 1,
+        'expect' => '1.2.3.4',
+      ),
+
+      // Multiple layers of load balancers.
+      array(
+        'header' => '9.9.9.9, 1.2.3.4',
+        'layers' => 2,
+        'expect' => '9.9.9.9',
+      ),
+
+      // Multiple layers of load balancers, plus a client-supplied value.
+      array(
+        'header' => '8.8.8.8, 9.9.9.9, 1.2.3.4',
+        'layers' => 2,
+        'expect' => '9.9.9.9',
+      ),
+
+      // Multiple layers of load balancers, but this request came from
+      // somewhere inside the network.
+      array(
+        'header' => '1.2.3.4',
+        'layers' => 2,
+        'expect' => '1.2.3.4',
+      ),
+
+      array(
+        'header' => 'A, B, C, D, E, F, G, H, I',
+        'layers' => 7,
+        'expect' => 'C',
+      ),
+    );
+
+    foreach ($tests as $test) {
+      $header = $test['header'];
+      $layers = $test['layers'];
+      $expect = $test['expect'];
+
+      $actual = preamble_get_x_forwarded_for_address($header, $layers);
+
+      $this->assertEqual(
+        $expect,
+        $actual,
+        pht(
+          'Address after stripping %d layers from: %s',
+          $layers,
+          $header));
+    }
+  }
+
+}
diff --git a/support/startup/preamble-utils.php b/support/startup/preamble-utils.php
new file mode 100644
index 0000000000..8dd3b502d6
--- /dev/null
+++ b/support/startup/preamble-utils.php
@@ -0,0 +1,77 @@
+): '.
+      '"layers" parameter must an integer larger than 0.'."\n";
+    echo "\n";
+    exit(1);
+  }
+
+  if (!isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
+    return;
+  }
+
+  $forwarded_for = $_SERVER['HTTP_X_FORWARDED_FOR'];
+  if (!strlen($forwarded_for)) {
+    return;
+  }
+
+  $address = preamble_get_x_forwarded_for_address($forwarded_for, $layers);
+
+  $_SERVER['REMOTE_ADDR'] = $address;
+}
+
+function preamble_get_x_forwarded_for_address($raw_header, $layers) {
+  // The raw header may be a list of IPs, like "1.2.3.4, 4.5.6.7", if the
+  // request the load balancer received also had this header. In particular,
+  // this happens routinely with requests received through a CDN, but can also
+  // happen illegitimately if the client just makes up an "X-Forwarded-For"
+  // header full of lies.
+
+  // We can only trust the N elements at the end of the list which correspond
+  // to network-adjacent devices we control. Usually, we're behind a single
+  // load balancer and "N" is 1, so we want to take the last element in the
+  // list.
+
+  // In some cases, "N" may be more than 1, if the network is configured so
+  // that that requests are routed through multiple layers of load balancers
+  // and proxies. In this case, we want to take the Nth-to-last element of
+  // the list.
+
+  $addresses = explode(',', $raw_header);
+
+  // If we have more than one trustworthy device on the network path, discard
+  // corresponding elements from the list. For example, if we have 7 devices,
+  // we want to discard the last 6 elements of the list.
+
+  // The final device address does not appear in the list, since devices do
+  // not append their own addresses to "X-Forwarded-For".
+
+  $discard_addresses = ($layers - 1);
+
+  // However, we don't want to throw away all of the addresses. Some requests
+  // may originate from within the network, and may thus not have as many
+  // addresses as we expect. If we have fewer addresses than trustworthy
+  // devices, discard all but one address.
+
+  $max_discard = (count($addresses) - 1);
+
+  $discard_count = min($discard_addresses, $max_discard);
+  if ($discard_count) {
+    $addresses = array_slice($addresses, 0, -$discard_count);
+  }
+
+  $original_address = end($addresses);
+  $original_address = trim($original_address);
+
+  return $original_address;
+}
diff --git a/webroot/index.php b/webroot/index.php
index 0014edfa2c..38c5c77809 100644
--- a/webroot/index.php
+++ b/webroot/index.php
@@ -85,6 +85,7 @@ function phabricator_startup() {
   require_once $root.'/support/startup/PhabricatorClientLimit.php';
   require_once $root.'/support/startup/PhabricatorClientRateLimit.php';
   require_once $root.'/support/startup/PhabricatorClientConnectionLimit.php';
+  require_once $root.'/support/startup/preamble-utils.php';
 
   // If the preamble script exists, load it.
   $t_preamble = microtime(true);

From 7e2bec92807d2a6c432fd7f690e9cb7dda0b1fc3 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Fri, 6 Sep 2019 08:18:28 -0700
Subject: [PATCH 14/16] Add a global setting for controlling the default main
 menu search scope

Summary: Fixes T13405. The default behavior of the global search bar isn't currently configurable, but can be made configurable fairly easily.

Test Plan: Changed setting as an administrator, saw setting reflected as a user with no previous preference. As a user with an existing preference, saw preference retained.

Maniphest Tasks: T13405

Differential Revision: https://secure.phabricator.com/D20787
---
 src/__phutil_library_map__.php                |  4 ++-
 .../panel/PhabricatorSearchSettingsPanel.php  | 28 +++++++++++++++++++
 .../setting/PhabricatorSearchScopeSetting.php | 27 +++++++++++++++++-
 .../menu/PhabricatorMainMenuSearchView.php    | 21 ++++++++++++--
 4 files changed, 75 insertions(+), 5 deletions(-)
 create mode 100644 src/applications/settings/panel/PhabricatorSearchSettingsPanel.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index fa9a5ce903..e6e9063453 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -4655,6 +4655,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchScopeSetting' => 'applications/settings/setting/PhabricatorSearchScopeSetting.php',
     'PhabricatorSearchSelectField' => 'applications/search/field/PhabricatorSearchSelectField.php',
     'PhabricatorSearchService' => 'infrastructure/cluster/search/PhabricatorSearchService.php',
+    'PhabricatorSearchSettingsPanel' => 'applications/settings/panel/PhabricatorSearchSettingsPanel.php',
     'PhabricatorSearchStringListField' => 'applications/search/field/PhabricatorSearchStringListField.php',
     'PhabricatorSearchSubscribersField' => 'applications/search/field/PhabricatorSearchSubscribersField.php',
     'PhabricatorSearchTextField' => 'applications/search/field/PhabricatorSearchTextField.php',
@@ -11248,9 +11249,10 @@ phutil_register_library_map(array(
     'PhabricatorSearchResultBucketGroup' => 'Phobject',
     'PhabricatorSearchResultView' => 'AphrontView',
     'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec',
-    'PhabricatorSearchScopeSetting' => 'PhabricatorInternalSetting',
+    'PhabricatorSearchScopeSetting' => 'PhabricatorSelectSetting',
     'PhabricatorSearchSelectField' => 'PhabricatorSearchField',
     'PhabricatorSearchService' => 'Phobject',
+    'PhabricatorSearchSettingsPanel' => 'PhabricatorEditEngineSettingsPanel',
     'PhabricatorSearchStringListField' => 'PhabricatorSearchField',
     'PhabricatorSearchSubscribersField' => 'PhabricatorSearchTokenizerField',
     'PhabricatorSearchTextField' => 'PhabricatorSearchField',
diff --git a/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php b/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php
new file mode 100644
index 0000000000..37b0ea919d
--- /dev/null
+++ b/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php
@@ -0,0 +1,28 @@
+getViewer(),
+      new PhabricatorSettingsApplication());
+
+    $scope_map = array();
+    foreach ($scopes as $scope) {
+      if (!isset($scope['value'])) {
+        continue;
+      }
+      $scope_map[$scope['value']] = $scope['name'];
+    }
+
+    return $scope_map;
+  }
+
 }
diff --git a/src/view/page/menu/PhabricatorMainMenuSearchView.php b/src/view/page/menu/PhabricatorMainMenuSearchView.php
index 15319a357e..aaa7c5a160 100644
--- a/src/view/page/menu/PhabricatorMainMenuSearchView.php
+++ b/src/view/page/menu/PhabricatorMainMenuSearchView.php
@@ -116,8 +116,9 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
     return $form;
   }
 
-  private function buildModeSelector($selector_id, $application_id) {
-    $viewer = $this->getViewer();
+  public static function getGlobalSearchScopeItems(
+    PhabricatorUser $viewer,
+    PhabricatorApplication $application) {
 
     $items = array();
     $items[] = array(
@@ -132,7 +133,6 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
 
     $application_value = null;
     $application_icon = self::DEFAULT_APPLICATION_ICON;
-    $application = $this->getApplication();
     if ($application) {
       $application_value = get_class($application);
       if ($application->getApplicationSearchDocumentTypes()) {
@@ -185,6 +185,14 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
       'href' => PhabricatorEnv::getDoclink('Search User Guide'),
     );
 
+    return $items;
+  }
+
+  private function buildModeSelector($selector_id, $application_id) {
+    $viewer = $this->getViewer();
+
+    $items = self::getGlobalSearchScopeItems($viewer, $this->getApplication());
+
     $scope_key = PhabricatorSearchScopeSetting::SETTINGKEY;
     $current_value = $viewer->getUserSetting($scope_key);
 
@@ -196,6 +204,13 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
       }
     }
 
+    $application = $this->getApplication();
+
+    $application_value = null;
+    if ($application) {
+      $application_value = get_class($application);
+    }
+
     $selector = id(new PHUIButtonView())
       ->setID($selector_id)
       ->addClass('phabricator-main-menu-search-dropdown')

From 318e8ebdac9511d1529bbfa22de08a0c5fd81b80 Mon Sep 17 00:00:00 2001
From: Aviv Eyal 
Date: Sun, 8 Sep 2019 00:16:19 +0000
Subject: [PATCH 15/16] Allow bin/config to create config file

Summary:
See D20779, https://discourse.phabricator-community.org/t/3089. `bin/config set` complains about
missing config file as if it's un-writable.

Test Plan: run `bin/config set` with missing, writable, unwritable conf.json and parent dir.

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley, #blessed_reviewers

Subscribers: Korvin

Differential Revision: https://secure.phabricator.com/D20788
---
 .../management/PhabricatorConfigManagementSetWorkflow.php     | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
index 6ad2db4471..d69e903bcc 100644
--- a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
+++ b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php
@@ -145,7 +145,7 @@ final class PhabricatorConfigManagementSetWorkflow
       $local_path = $config_source->getReadablePath();
 
       try {
-        Filesystem::assertWritable($local_path);
+        $config_source->setKeys(array($key => $value));
       } catch (FilesystemException $ex) {
         throw new PhutilArgumentUsageException(
           pht(
@@ -154,8 +154,6 @@ final class PhabricatorConfigManagementSetWorkflow
             Filesystem::readablePath($local_path)));
       }
 
-      $config_source->setKeys(array($key => $value));
-
       $write_message = pht(
         'Wrote configuration key "%s" to local storage (in file "%s").',
         $key,

From caccbb69d20bc61d3fc2e328bc7fe2d2789071a1 Mon Sep 17 00:00:00 2001
From: epriestley 
Date: Sun, 8 Sep 2019 09:45:53 -0700
Subject: [PATCH 16/16] When users try to log out with no providers configured,
 warn them of the consequences

Summary: Fixes T13406. On the logout screen, test for no configured providers and warn users they may be getting into more trouble than they expect.

Test Plan:
  - Logged out of a normal install and a fresh (unconfigured) install.

{F6847659}

Maniphest Tasks: T13406

Differential Revision: https://secure.phabricator.com/D20789
---
 .../PhabricatorLogoutController.php           | 36 +++++++++++++++++--
 .../config/PhabricatorAuthListController.php  |  2 +-
 src/view/AphrontDialogView.php                | 14 ++++++++
 3 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/src/applications/auth/controller/PhabricatorLogoutController.php b/src/applications/auth/controller/PhabricatorLogoutController.php
index dccf6bb45b..d71d080cbd 100644
--- a/src/applications/auth/controller/PhabricatorLogoutController.php
+++ b/src/applications/auth/controller/PhabricatorLogoutController.php
@@ -68,12 +68,42 @@ final class PhabricatorLogoutController
         ->setURI('/auth/loggedout/');
     }
 
+
     if ($viewer->getPHID()) {
-      return $this->newDialog()
+      $dialog = $this->newDialog()
         ->setTitle(pht('Log Out?'))
-        ->appendChild(pht('Are you sure you want to log out?'))
-        ->addSubmitButton(pht('Log Out'))
+        ->appendParagraph(pht('Are you sure you want to log out?'))
         ->addCancelButton('/');
+
+      $configs = id(new PhabricatorAuthProviderConfigQuery())
+        ->setViewer(PhabricatorUser::getOmnipotentUser())
+        ->execute();
+      if (!$configs) {
+        $dialog
+          ->appendRemarkup(
+            pht(
+              'WARNING: You have not configured any authentication providers '.
+              'yet, so your account has no login credentials. If you log out '.
+              'now, you will not be able to log back in normally.'))
+          ->appendParagraph(
+            pht(
+              'To enable the login flow, follow setup guidance and configure '.
+              'at least one authentication provider, then associate '.
+              'credentials with your account. After completing these steps, '.
+              'you will be able to log out and log back in normally.'))
+          ->appendParagraph(
+            pht(
+              'If you log out now, you can still regain access to your '.
+              'account later by using the account recovery workflow. The '.
+              'login screen will prompt you with recovery instructions.'));
+
+        $button = pht('Log Out Anyway');
+      } else {
+        $button = pht('Log Out');
+      }
+
+      $dialog->addSubmitButton($button);
+      return $dialog;
     }
 
     return id(new AphrontRedirectResponse())->setURI('/');
diff --git a/src/applications/auth/controller/config/PhabricatorAuthListController.php b/src/applications/auth/controller/config/PhabricatorAuthListController.php
index 5d1d85cca6..b25c791e27 100644
--- a/src/applications/auth/controller/config/PhabricatorAuthListController.php
+++ b/src/applications/auth/controller/config/PhabricatorAuthListController.php
@@ -64,7 +64,7 @@ final class PhabricatorAuthListController
           array(
             'href' => $this->getApplicationURI('config/new/'),
           ),
-          pht('Add Authentication Provider'))));
+          pht('Add Provider'))));
 
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('Login and Registration'));
diff --git a/src/view/AphrontDialogView.php b/src/view/AphrontDialogView.php
index 09fc8e7a16..b8b00a6b3e 100644
--- a/src/view/AphrontDialogView.php
+++ b/src/view/AphrontDialogView.php
@@ -160,6 +160,20 @@ final class AphrontDialogView
     return $this->appendChild($box);
   }
 
+  public function appendRemarkup($remarkup) {
+    $viewer = $this->getViewer();
+    $view = new PHUIRemarkupView($viewer, $remarkup);
+
+    $view_tag = phutil_tag(
+      'div',
+      array(
+        'class' => 'aphront-dialog-view-paragraph',
+      ),
+      $view);
+
+    return $this->appendChild($view_tag);
+  }
+
   public function appendParagraph($paragraph) {
     return $this->appendParagraphTag($paragraph);
   }