diff --git a/.arclint b/.arclint index 6047e2ccb6..d065c3d12e 100644 --- a/.arclint +++ b/.arclint @@ -1,7 +1,8 @@ { "exclude": [ "(^externals/)", - "(^webroot/rsrc/externals/(?!javelin/))" + "(^webroot/rsrc/externals/(?!javelin/))", + "(/__tests__/data/)" ], "linters": { "chmod": { diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 5d9a0aed3c..e6e9063453 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -176,15 +176,27 @@ phutil_register_library_map(array( 'Aphront400Response' => 'aphront/response/Aphront400Response.php', 'Aphront403Response' => 'aphront/response/Aphront403Response.php', 'Aphront404Response' => 'aphront/response/Aphront404Response.php', + 'AphrontAccessDeniedQueryException' => 'infrastructure/storage/exception/AphrontAccessDeniedQueryException.php', 'AphrontAjaxResponse' => 'aphront/response/AphrontAjaxResponse.php', 'AphrontApplicationConfiguration' => 'aphront/configuration/AphrontApplicationConfiguration.php', 'AphrontBarView' => 'view/widget/bars/AphrontBarView.php', + 'AphrontBaseMySQLDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php', 'AphrontBoolHTTPParameterType' => 'aphront/httpparametertype/AphrontBoolHTTPParameterType.php', 'AphrontCalendarEventView' => 'applications/calendar/view/AphrontCalendarEventView.php', + 'AphrontCharacterSetQueryException' => 'infrastructure/storage/exception/AphrontCharacterSetQueryException.php', + 'AphrontConnectionLostQueryException' => 'infrastructure/storage/exception/AphrontConnectionLostQueryException.php', + 'AphrontConnectionQueryException' => 'infrastructure/storage/exception/AphrontConnectionQueryException.php', 'AphrontController' => 'aphront/AphrontController.php', + 'AphrontCountQueryException' => 'infrastructure/storage/exception/AphrontCountQueryException.php', 'AphrontCursorPagerView' => 'view/control/AphrontCursorPagerView.php', + 'AphrontDatabaseConnection' => 'infrastructure/storage/connection/AphrontDatabaseConnection.php', + 'AphrontDatabaseTableRef' => 'infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php', + 'AphrontDatabaseTableRefInterface' => 'infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php', + 'AphrontDatabaseTransactionState' => 'infrastructure/storage/connection/AphrontDatabaseTransactionState.php', + 'AphrontDeadlockQueryException' => 'infrastructure/storage/exception/AphrontDeadlockQueryException.php', 'AphrontDialogResponse' => 'aphront/response/AphrontDialogResponse.php', 'AphrontDialogView' => 'view/AphrontDialogView.php', + 'AphrontDuplicateKeyQueryException' => 'infrastructure/storage/exception/AphrontDuplicateKeyQueryException.php', 'AphrontEpochHTTPParameterType' => 'aphront/httpparametertype/AphrontEpochHTTPParameterType.php', 'AphrontException' => 'aphront/exception/AphrontException.php', 'AphrontFileHTTPParameterType' => 'aphront/httpparametertype/AphrontFileHTTPParameterType.php', @@ -217,6 +229,8 @@ phutil_register_library_map(array( 'AphrontHTTPSink' => 'aphront/sink/AphrontHTTPSink.php', 'AphrontHTTPSinkTestCase' => 'aphront/sink/__tests__/AphrontHTTPSinkTestCase.php', 'AphrontIntHTTPParameterType' => 'aphront/httpparametertype/AphrontIntHTTPParameterType.php', + 'AphrontInvalidCredentialsQueryException' => 'infrastructure/storage/exception/AphrontInvalidCredentialsQueryException.php', + 'AphrontIsolatedDatabaseConnection' => 'infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php', 'AphrontIsolatedDatabaseConnectionTestCase' => 'infrastructure/storage/__tests__/AphrontIsolatedDatabaseConnectionTestCase.php', 'AphrontIsolatedHTTPSink' => 'aphront/sink/AphrontIsolatedHTTPSink.php', 'AphrontJSONResponse' => 'aphront/response/AphrontJSONResponse.php', @@ -224,19 +238,28 @@ phutil_register_library_map(array( 'AphrontKeyboardShortcutsAvailableView' => 'view/widget/AphrontKeyboardShortcutsAvailableView.php', 'AphrontListFilterView' => 'view/layout/AphrontListFilterView.php', 'AphrontListHTTPParameterType' => 'aphront/httpparametertype/AphrontListHTTPParameterType.php', + 'AphrontLockTimeoutQueryException' => 'infrastructure/storage/exception/AphrontLockTimeoutQueryException.php', 'AphrontMalformedRequestException' => 'aphront/exception/AphrontMalformedRequestException.php', 'AphrontMoreView' => 'view/layout/AphrontMoreView.php', 'AphrontMultiColumnView' => 'view/layout/AphrontMultiColumnView.php', + 'AphrontMySQLDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php', 'AphrontMySQLDatabaseConnectionTestCase' => 'infrastructure/storage/__tests__/AphrontMySQLDatabaseConnectionTestCase.php', + 'AphrontMySQLiDatabaseConnection' => 'infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php', + 'AphrontNotSupportedQueryException' => 'infrastructure/storage/exception/AphrontNotSupportedQueryException.php', 'AphrontNullView' => 'view/AphrontNullView.php', + 'AphrontObjectMissingQueryException' => 'infrastructure/storage/exception/AphrontObjectMissingQueryException.php', 'AphrontPHIDHTTPParameterType' => 'aphront/httpparametertype/AphrontPHIDHTTPParameterType.php', 'AphrontPHIDListHTTPParameterType' => 'aphront/httpparametertype/AphrontPHIDListHTTPParameterType.php', 'AphrontPHPHTTPSink' => 'aphront/sink/AphrontPHPHTTPSink.php', 'AphrontPageView' => 'view/page/AphrontPageView.php', + 'AphrontParameterQueryException' => 'infrastructure/storage/exception/AphrontParameterQueryException.php', 'AphrontPlainTextResponse' => 'aphront/response/AphrontPlainTextResponse.php', 'AphrontProgressBarView' => 'view/widget/bars/AphrontProgressBarView.php', 'AphrontProjectListHTTPParameterType' => 'aphront/httpparametertype/AphrontProjectListHTTPParameterType.php', 'AphrontProxyResponse' => 'aphront/response/AphrontProxyResponse.php', + 'AphrontQueryException' => 'infrastructure/storage/exception/AphrontQueryException.php', + 'AphrontQueryTimeoutQueryException' => 'infrastructure/storage/exception/AphrontQueryTimeoutQueryException.php', + 'AphrontRecoverableQueryException' => 'infrastructure/storage/exception/AphrontRecoverableQueryException.php', 'AphrontRedirectResponse' => 'aphront/response/AphrontRedirectResponse.php', 'AphrontRedirectResponseTestCase' => 'aphront/response/__tests__/AphrontRedirectResponseTestCase.php', 'AphrontReloadResponse' => 'aphront/response/AphrontReloadResponse.php', @@ -247,6 +270,7 @@ phutil_register_library_map(array( 'AphrontResponseProducerInterface' => 'aphront/interface/AphrontResponseProducerInterface.php', 'AphrontRoutingMap' => 'aphront/site/AphrontRoutingMap.php', 'AphrontRoutingResult' => 'aphront/site/AphrontRoutingResult.php', + 'AphrontSchemaQueryException' => 'infrastructure/storage/exception/AphrontSchemaQueryException.php', 'AphrontSelectHTTPParameterType' => 'aphront/httpparametertype/AphrontSelectHTTPParameterType.php', 'AphrontSideNavFilterView' => 'view/layout/AphrontSideNavFilterView.php', 'AphrontSite' => 'aphront/site/AphrontSite.php', @@ -997,6 +1021,7 @@ phutil_register_library_map(array( 'DiffusionSSHWorkflow' => 'applications/diffusion/ssh/DiffusionSSHWorkflow.php', 'DiffusionSearchQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionSearchQueryConduitAPIMethod.php', 'DiffusionServeController' => 'applications/diffusion/controller/DiffusionServeController.php', + 'DiffusionServiceRef' => 'applications/diffusion/ref/DiffusionServiceRef.php', 'DiffusionSetPasswordSettingsPanel' => 'applications/diffusion/panel/DiffusionSetPasswordSettingsPanel.php', 'DiffusionSetupException' => 'applications/diffusion/exception/DiffusionSetupException.php', 'DiffusionSourceHyperlinkEngineExtension' => 'applications/diffusion/engineextension/DiffusionSourceHyperlinkEngineExtension.php', @@ -4176,6 +4201,7 @@ phutil_register_library_map(array( 'PhabricatorPolicyTestObject' => 'applications/policy/__tests__/PhabricatorPolicyTestObject.php', 'PhabricatorPolicyType' => 'applications/policy/constants/PhabricatorPolicyType.php', 'PhabricatorPonderApplication' => 'applications/ponder/application/PhabricatorPonderApplication.php', + 'PhabricatorPreambleTestCase' => 'infrastructure/util/__tests__/PhabricatorPreambleTestCase.php', 'PhabricatorPrimaryEmailUserLogType' => 'applications/people/userlog/PhabricatorPrimaryEmailUserLogType.php', 'PhabricatorProfileMenuEditEngine' => 'applications/search/editor/PhabricatorProfileMenuEditEngine.php', 'PhabricatorProfileMenuEditor' => 'applications/search/editor/PhabricatorProfileMenuEditor.php', @@ -4629,6 +4655,7 @@ phutil_register_library_map(array( 'PhabricatorSearchScopeSetting' => 'applications/settings/setting/PhabricatorSearchScopeSetting.php', 'PhabricatorSearchSelectField' => 'applications/search/field/PhabricatorSearchSelectField.php', 'PhabricatorSearchService' => 'infrastructure/cluster/search/PhabricatorSearchService.php', + 'PhabricatorSearchSettingsPanel' => 'applications/settings/panel/PhabricatorSearchSettingsPanel.php', 'PhabricatorSearchStringListField' => 'applications/search/field/PhabricatorSearchStringListField.php', 'PhabricatorSearchSubscribersField' => 'applications/search/field/PhabricatorSearchSubscribersField.php', 'PhabricatorSearchTextField' => 'applications/search/field/PhabricatorSearchTextField.php', @@ -5512,6 +5539,93 @@ phutil_register_library_map(array( 'PhrictionTransactionComment' => 'applications/phriction/storage/PhrictionTransactionComment.php', 'PhrictionTransactionEditor' => 'applications/phriction/editor/PhrictionTransactionEditor.php', 'PhrictionTransactionQuery' => 'applications/phriction/query/PhrictionTransactionQuery.php', + 'PhutilAmazonAuthAdapter' => 'applications/auth/adapter/PhutilAmazonAuthAdapter.php', + 'PhutilAsanaAuthAdapter' => 'applications/auth/adapter/PhutilAsanaAuthAdapter.php', + 'PhutilAuthAdapter' => 'applications/auth/adapter/PhutilAuthAdapter.php', + 'PhutilAuthConfigurationException' => 'applications/auth/exception/PhutilAuthConfigurationException.php', + 'PhutilAuthCredentialException' => 'applications/auth/exception/PhutilAuthCredentialException.php', + 'PhutilAuthException' => 'applications/auth/exception/PhutilAuthException.php', + 'PhutilAuthUserAbortedException' => 'applications/auth/exception/PhutilAuthUserAbortedException.php', + 'PhutilBitbucketAuthAdapter' => 'applications/auth/adapter/PhutilBitbucketAuthAdapter.php', + 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php', + 'PhutilCalendarAbsoluteDateTime' => 'applications/calendar/parser/data/PhutilCalendarAbsoluteDateTime.php', + 'PhutilCalendarContainerNode' => 'applications/calendar/parser/data/PhutilCalendarContainerNode.php', + 'PhutilCalendarDateTime' => 'applications/calendar/parser/data/PhutilCalendarDateTime.php', + 'PhutilCalendarDateTimeTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php', + 'PhutilCalendarDocumentNode' => 'applications/calendar/parser/data/PhutilCalendarDocumentNode.php', + 'PhutilCalendarDuration' => 'applications/calendar/parser/data/PhutilCalendarDuration.php', + 'PhutilCalendarEventNode' => 'applications/calendar/parser/data/PhutilCalendarEventNode.php', + 'PhutilCalendarNode' => 'applications/calendar/parser/data/PhutilCalendarNode.php', + 'PhutilCalendarProxyDateTime' => 'applications/calendar/parser/data/PhutilCalendarProxyDateTime.php', + 'PhutilCalendarRawNode' => 'applications/calendar/parser/data/PhutilCalendarRawNode.php', + 'PhutilCalendarRecurrenceList' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceList.php', + 'PhutilCalendarRecurrenceRule' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php', + 'PhutilCalendarRecurrenceRuleTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php', + 'PhutilCalendarRecurrenceSet' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php', + 'PhutilCalendarRecurrenceSource' => 'applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php', + 'PhutilCalendarRecurrenceTestCase' => 'applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php', + 'PhutilCalendarRelativeDateTime' => 'applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php', + 'PhutilCalendarRootNode' => 'applications/calendar/parser/data/PhutilCalendarRootNode.php', + 'PhutilCalendarUserNode' => 'applications/calendar/parser/data/PhutilCalendarUserNode.php', + 'PhutilCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php', + 'PhutilDisqusAuthAdapter' => 'applications/auth/adapter/PhutilDisqusAuthAdapter.php', + 'PhutilEmptyAuthAdapter' => 'applications/auth/adapter/PhutilEmptyAuthAdapter.php', + 'PhutilFacebookAuthAdapter' => 'applications/auth/adapter/PhutilFacebookAuthAdapter.php', + 'PhutilGitHubAuthAdapter' => 'applications/auth/adapter/PhutilGitHubAuthAdapter.php', + 'PhutilGoogleAuthAdapter' => 'applications/auth/adapter/PhutilGoogleAuthAdapter.php', + 'PhutilICSParser' => 'applications/calendar/parser/ics/PhutilICSParser.php', + 'PhutilICSParserException' => 'applications/calendar/parser/ics/PhutilICSParserException.php', + 'PhutilICSParserTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php', + 'PhutilICSWriter' => 'applications/calendar/parser/ics/PhutilICSWriter.php', + 'PhutilICSWriterTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php', + 'PhutilJIRAAuthAdapter' => 'applications/auth/adapter/PhutilJIRAAuthAdapter.php', + 'PhutilJavaCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php', + 'PhutilLDAPAuthAdapter' => 'applications/auth/adapter/PhutilLDAPAuthAdapter.php', + 'PhutilLipsumContextFreeGrammar' => 'infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php', + 'PhutilOAuth1AuthAdapter' => 'applications/auth/adapter/PhutilOAuth1AuthAdapter.php', + 'PhutilOAuthAuthAdapter' => 'applications/auth/adapter/PhutilOAuthAuthAdapter.php', + 'PhutilPHPCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php', + 'PhutilPhabricatorAuthAdapter' => 'applications/auth/adapter/PhutilPhabricatorAuthAdapter.php', + 'PhutilQsprintfInterface' => 'infrastructure/storage/xsprintf/PhutilQsprintfInterface.php', + 'PhutilQueryString' => 'infrastructure/storage/xsprintf/PhutilQueryString.php', + 'PhutilRealNameContextFreeGrammar' => 'infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php', + 'PhutilRemarkupBlockInterpreter' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php', + 'PhutilRemarkupBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php', + 'PhutilRemarkupBlockStorage' => 'infrastructure/markup/PhutilRemarkupBlockStorage.php', + 'PhutilRemarkupBoldRule' => 'infrastructure/markup/markuprule/PhutilRemarkupBoldRule.php', + 'PhutilRemarkupCodeBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php', + 'PhutilRemarkupDefaultBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php', + 'PhutilRemarkupDelRule' => 'infrastructure/markup/markuprule/PhutilRemarkupDelRule.php', + 'PhutilRemarkupDocumentLinkRule' => 'infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php', + 'PhutilRemarkupEngine' => 'infrastructure/markup/remarkup/PhutilRemarkupEngine.php', + 'PhutilRemarkupEngineTestCase' => 'infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php', + 'PhutilRemarkupEscapeRemarkupRule' => 'infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php', + 'PhutilRemarkupHeaderBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php', + 'PhutilRemarkupHighlightRule' => 'infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php', + 'PhutilRemarkupHorizontalRuleBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php', + 'PhutilRemarkupHyperlinkEngineExtension' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php', + 'PhutilRemarkupHyperlinkRef' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php', + 'PhutilRemarkupHyperlinkRule' => 'infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php', + 'PhutilRemarkupInlineBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php', + 'PhutilRemarkupInterpreterBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php', + 'PhutilRemarkupItalicRule' => 'infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php', + 'PhutilRemarkupLinebreaksRule' => 'infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php', + 'PhutilRemarkupListBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php', + 'PhutilRemarkupLiteralBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php', + 'PhutilRemarkupMonospaceRule' => 'infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php', + 'PhutilRemarkupNoteBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php', + 'PhutilRemarkupQuotedBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php', + 'PhutilRemarkupQuotesBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php', + 'PhutilRemarkupReplyBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php', + 'PhutilRemarkupRule' => 'infrastructure/markup/markuprule/PhutilRemarkupRule.php', + 'PhutilRemarkupSimpleTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php', + 'PhutilRemarkupTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php', + 'PhutilRemarkupTestInterpreterRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php', + 'PhutilRemarkupUnderlineRule' => 'infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php', + 'PhutilSlackAuthAdapter' => 'applications/auth/adapter/PhutilSlackAuthAdapter.php', + 'PhutilTwitchAuthAdapter' => 'applications/auth/adapter/PhutilTwitchAuthAdapter.php', + 'PhutilTwitterAuthAdapter' => 'applications/auth/adapter/PhutilTwitterAuthAdapter.php', + 'PhutilWordPressAuthAdapter' => 'applications/auth/adapter/PhutilWordPressAuthAdapter.php', 'PolicyLockOptionType' => 'applications/policy/config/PolicyLockOptionType.php', 'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php', 'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php', @@ -5587,6 +5701,7 @@ phutil_register_library_map(array( 'ProjectReplyHandler' => 'applications/project/mail/ProjectReplyHandler.php', 'ProjectSearchConduitAPIMethod' => 'applications/project/conduit/ProjectSearchConduitAPIMethod.php', 'QueryFormattingTestCase' => 'infrastructure/storage/__tests__/QueryFormattingTestCase.php', + 'QueryFuture' => 'infrastructure/storage/future/QueryFuture.php', 'ReleephAuthorFieldSpecification' => 'applications/releeph/field/specification/ReleephAuthorFieldSpecification.php', 'ReleephBranch' => 'applications/releeph/storage/ReleephBranch.php', 'ReleephBranchAccessController' => 'applications/releeph/controller/branch/ReleephBranchAccessController.php', @@ -5713,7 +5828,15 @@ phutil_register_library_map(array( 'phid_get_subtype' => 'applications/phid/utils.php', 'phid_get_type' => 'applications/phid/utils.php', 'phid_group_by_type' => 'applications/phid/utils.php', + 'qsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'qsprintf_check_scalar_type' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'qsprintf_check_type' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'queryfx' => 'infrastructure/storage/xsprintf/queryfx.php', + 'queryfx_all' => 'infrastructure/storage/xsprintf/queryfx.php', + 'queryfx_one' => 'infrastructure/storage/xsprintf/queryfx.php', 'require_celerity_resource' => 'applications/celerity/api.php', + 'vqsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php', + 'xsprintf_query' => 'infrastructure/storage/xsprintf/qsprintf.php', ), 'xmap' => array( 'AlmanacAddress' => 'Phobject', @@ -5937,18 +6060,35 @@ phutil_register_library_map(array( 'Aphront400Response' => 'AphrontResponse', 'Aphront403Response' => 'AphrontHTMLResponse', 'Aphront404Response' => 'AphrontHTMLResponse', + 'AphrontAccessDeniedQueryException' => 'AphrontQueryException', 'AphrontAjaxResponse' => 'AphrontResponse', 'AphrontApplicationConfiguration' => 'Phobject', 'AphrontBarView' => 'AphrontView', + 'AphrontBaseMySQLDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontBoolHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontCalendarEventView' => 'AphrontView', + 'AphrontCharacterSetQueryException' => 'AphrontQueryException', + 'AphrontConnectionLostQueryException' => 'AphrontRecoverableQueryException', + 'AphrontConnectionQueryException' => 'AphrontQueryException', 'AphrontController' => 'Phobject', + 'AphrontCountQueryException' => 'AphrontQueryException', 'AphrontCursorPagerView' => 'AphrontView', + 'AphrontDatabaseConnection' => array( + 'Phobject', + 'PhutilQsprintfInterface', + ), + 'AphrontDatabaseTableRef' => array( + 'Phobject', + 'AphrontDatabaseTableRefInterface', + ), + 'AphrontDatabaseTransactionState' => 'Phobject', + 'AphrontDeadlockQueryException' => 'AphrontRecoverableQueryException', 'AphrontDialogResponse' => 'AphrontResponse', 'AphrontDialogView' => array( 'AphrontView', 'AphrontResponseProducerInterface', ), + 'AphrontDuplicateKeyQueryException' => 'AphrontQueryException', 'AphrontEpochHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontException' => 'Exception', 'AphrontFileHTTPParameterType' => 'AphrontHTTPParameterType', @@ -5981,6 +6121,8 @@ phutil_register_library_map(array( 'AphrontHTTPSink' => 'Phobject', 'AphrontHTTPSinkTestCase' => 'PhabricatorTestCase', 'AphrontIntHTTPParameterType' => 'AphrontHTTPParameterType', + 'AphrontInvalidCredentialsQueryException' => 'AphrontQueryException', + 'AphrontIsolatedDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontIsolatedDatabaseConnectionTestCase' => 'PhabricatorTestCase', 'AphrontIsolatedHTTPSink' => 'AphrontHTTPSink', 'AphrontJSONResponse' => 'AphrontResponse', @@ -5988,15 +6130,21 @@ phutil_register_library_map(array( 'AphrontKeyboardShortcutsAvailableView' => 'AphrontView', 'AphrontListFilterView' => 'AphrontView', 'AphrontListHTTPParameterType' => 'AphrontHTTPParameterType', + 'AphrontLockTimeoutQueryException' => 'AphrontRecoverableQueryException', 'AphrontMalformedRequestException' => 'AphrontException', 'AphrontMoreView' => 'AphrontView', 'AphrontMultiColumnView' => 'AphrontView', + 'AphrontMySQLDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', 'AphrontMySQLDatabaseConnectionTestCase' => 'PhabricatorTestCase', + 'AphrontMySQLiDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', + 'AphrontNotSupportedQueryException' => 'AphrontQueryException', 'AphrontNullView' => 'AphrontView', + 'AphrontObjectMissingQueryException' => 'AphrontQueryException', 'AphrontPHIDHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontPHIDListHTTPParameterType' => 'AphrontListHTTPParameterType', 'AphrontPHPHTTPSink' => 'AphrontHTTPSink', 'AphrontPageView' => 'AphrontView', + 'AphrontParameterQueryException' => 'AphrontQueryException', 'AphrontPlainTextResponse' => 'AphrontResponse', 'AphrontProgressBarView' => 'AphrontBarView', 'AphrontProjectListHTTPParameterType' => 'AphrontListHTTPParameterType', @@ -6004,6 +6152,9 @@ phutil_register_library_map(array( 'AphrontResponse', 'AphrontResponseProducerInterface', ), + 'AphrontQueryException' => 'Exception', + 'AphrontQueryTimeoutQueryException' => 'AphrontRecoverableQueryException', + 'AphrontRecoverableQueryException' => 'AphrontQueryException', 'AphrontRedirectResponse' => 'AphrontResponse', 'AphrontRedirectResponseTestCase' => 'PhabricatorTestCase', 'AphrontReloadResponse' => 'AphrontRedirectResponse', @@ -6013,6 +6164,7 @@ phutil_register_library_map(array( 'AphrontResponse' => 'Phobject', 'AphrontRoutingMap' => 'Phobject', 'AphrontRoutingResult' => 'Phobject', + 'AphrontSchemaQueryException' => 'AphrontQueryException', 'AphrontSelectHTTPParameterType' => 'AphrontHTTPParameterType', 'AphrontSideNavFilterView' => 'AphrontView', 'AphrontSite' => 'Phobject', @@ -6818,6 +6970,7 @@ phutil_register_library_map(array( 'DiffusionSSHWorkflow' => 'PhabricatorSSHWorkflow', 'DiffusionSearchQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 'DiffusionServeController' => 'DiffusionController', + 'DiffusionServiceRef' => 'Phobject', 'DiffusionSetPasswordSettingsPanel' => 'PhabricatorSettingsPanel', 'DiffusionSetupException' => 'Exception', 'DiffusionSourceHyperlinkEngineExtension' => 'PhabricatorRemarkupHyperlinkEngineExtension', @@ -10530,6 +10683,7 @@ phutil_register_library_map(array( ), 'PhabricatorPolicyType' => 'PhabricatorPolicyConstants', 'PhabricatorPonderApplication' => 'PhabricatorApplication', + 'PhabricatorPreambleTestCase' => 'PhabricatorTestCase', 'PhabricatorPrimaryEmailUserLogType' => 'PhabricatorUserLogType', 'PhabricatorProfileMenuEditEngine' => 'PhabricatorEditEngine', 'PhabricatorProfileMenuEditor' => 'PhabricatorApplicationTransactionEditor', @@ -11095,9 +11249,10 @@ phutil_register_library_map(array( 'PhabricatorSearchResultBucketGroup' => 'Phobject', 'PhabricatorSearchResultView' => 'AphrontView', 'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec', - 'PhabricatorSearchScopeSetting' => 'PhabricatorInternalSetting', + 'PhabricatorSearchScopeSetting' => 'PhabricatorSelectSetting', 'PhabricatorSearchSelectField' => 'PhabricatorSearchField', 'PhabricatorSearchService' => 'Phobject', + 'PhabricatorSearchSettingsPanel' => 'PhabricatorEditEngineSettingsPanel', 'PhabricatorSearchStringListField' => 'PhabricatorSearchField', 'PhabricatorSearchSubscribersField' => 'PhabricatorSearchTokenizerField', 'PhabricatorSearchTextField' => 'PhabricatorSearchField', @@ -12169,6 +12324,92 @@ phutil_register_library_map(array( 'PhrictionTransactionComment' => 'PhabricatorApplicationTransactionComment', 'PhrictionTransactionEditor' => 'PhabricatorApplicationTransactionEditor', 'PhrictionTransactionQuery' => 'PhabricatorApplicationTransactionQuery', + 'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilAuthAdapter' => 'Phobject', + 'PhutilAuthConfigurationException' => 'PhutilAuthException', + 'PhutilAuthCredentialException' => 'PhutilAuthException', + 'PhutilAuthException' => 'Exception', + 'PhutilAuthUserAbortedException' => 'PhutilAuthException', + 'PhutilBitbucketAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'PhutilCodeSnippetContextFreeGrammar', + 'PhutilCalendarAbsoluteDateTime' => 'PhutilCalendarDateTime', + 'PhutilCalendarContainerNode' => 'PhutilCalendarNode', + 'PhutilCalendarDateTime' => 'Phobject', + 'PhutilCalendarDateTimeTestCase' => 'PhutilTestCase', + 'PhutilCalendarDocumentNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarDuration' => 'Phobject', + 'PhutilCalendarEventNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarNode' => 'Phobject', + 'PhutilCalendarProxyDateTime' => 'PhutilCalendarDateTime', + 'PhutilCalendarRawNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarRecurrenceList' => 'PhutilCalendarRecurrenceSource', + 'PhutilCalendarRecurrenceRule' => 'PhutilCalendarRecurrenceSource', + 'PhutilCalendarRecurrenceRuleTestCase' => 'PhutilTestCase', + 'PhutilCalendarRecurrenceSet' => 'Phobject', + 'PhutilCalendarRecurrenceSource' => 'Phobject', + 'PhutilCalendarRecurrenceTestCase' => 'PhutilTestCase', + 'PhutilCalendarRelativeDateTime' => 'PhutilCalendarProxyDateTime', + 'PhutilCalendarRootNode' => 'PhutilCalendarContainerNode', + 'PhutilCalendarUserNode' => 'PhutilCalendarNode', + 'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilGoogleAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilICSParser' => 'Phobject', + 'PhutilICSParserException' => 'Exception', + 'PhutilICSParserTestCase' => 'PhutilTestCase', + 'PhutilICSWriter' => 'Phobject', + 'PhutilICSWriterTestCase' => 'PhutilTestCase', + 'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter', + 'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilQueryString' => 'Phobject', + 'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilRemarkupBlockInterpreter' => 'Phobject', + 'PhutilRemarkupBlockRule' => 'Phobject', + 'PhutilRemarkupBlockStorage' => 'Phobject', + 'PhutilRemarkupBoldRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupCodeBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupDefaultBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupDelRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupDocumentLinkRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupEngine' => 'PhutilMarkupEngine', + 'PhutilRemarkupEngineTestCase' => 'PhutilTestCase', + 'PhutilRemarkupEscapeRemarkupRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupHeaderBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupHighlightRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupHorizontalRuleBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupHyperlinkEngineExtension' => 'Phobject', + 'PhutilRemarkupHyperlinkRef' => 'Phobject', + 'PhutilRemarkupHyperlinkRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupInlineBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupInterpreterBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupItalicRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupLinebreaksRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupListBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupLiteralBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupMonospaceRule' => 'PhutilRemarkupRule', + 'PhutilRemarkupNoteBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupQuotedBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupQuotesBlockRule' => 'PhutilRemarkupQuotedBlockRule', + 'PhutilRemarkupReplyBlockRule' => 'PhutilRemarkupQuotedBlockRule', + 'PhutilRemarkupRule' => 'Phobject', + 'PhutilRemarkupSimpleTableBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule', + 'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter', + 'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule', + 'PhutilSlackAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PolicyLockOptionType' => 'PhabricatorConfigJSONOptionType', 'PonderAddAnswerView' => 'AphrontView', 'PonderAnswer' => array( @@ -12265,6 +12506,7 @@ phutil_register_library_map(array( 'ProjectReplyHandler' => 'PhabricatorApplicationTransactionReplyHandler', 'ProjectSearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod', 'QueryFormattingTestCase' => 'PhabricatorTestCase', + 'QueryFuture' => 'Future', 'ReleephAuthorFieldSpecification' => 'ReleephFieldSpecification', 'ReleephBranch' => array( 'ReleephDAO', diff --git a/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php b/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php new file mode 100644 index 0000000000..94c529ae2d --- /dev/null +++ b/src/applications/auth/adapter/PhutilAmazonAuthAdapter.php @@ -0,0 +1,80 @@ +getOAuthAccountData('user_id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return null; + } + + public function getAccountImageURI() { + return null; + } + + public function getAccountURI() { + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://www.amazon.com/ap/oa'; + } + + protected function getTokenBaseURI() { + return 'https://api.amazon.com/auth/o2/token'; + } + + public function getScope() { + return 'profile'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://api.amazon.com/user/profile'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + list($body) = $future->resolvex(); + + try { + return phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Amazon account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php b/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php new file mode 100644 index 0000000000..5d9a9ec478 --- /dev/null +++ b/src/applications/auth/adapter/PhutilAsanaAuthAdapter.php @@ -0,0 +1,86 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return null; + } + + public function getAccountImageURI() { + $photo = $this->getOAuthAccountData('photo', array()); + if (is_array($photo)) { + return idx($photo, 'image_128x128'); + } else { + return null; + } + } + + public function getAccountURI() { + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://app.asana.com/-/oauth_authorize'; + } + + protected function getTokenBaseURI() { + return 'https://app.asana.com/-/oauth_token'; + } + + public function getScope() { + return null; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + public function getExtraRefreshParameters() { + return array( + 'grant_type' => 'refresh_token', + ); + } + + public function supportsTokenRefresh() { + return true; + } + + protected function loadOAuthAccountData() { + return id(new PhutilAsanaFuture()) + ->setAccessToken($this->getAccessToken()) + ->setRawAsanaQuery('users/me') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilAuthAdapter.php b/src/applications/auth/adapter/PhutilAuthAdapter.php new file mode 100644 index 0000000000..73ef9e4834 --- /dev/null +++ b/src/applications/auth/adapter/PhutilAuthAdapter.php @@ -0,0 +1,123 @@ +` is globally unique and always identifies the + * same identity. + * + * If the adapter was unable to authenticate an identity, it should return + * `null`. + * + * @return string|null Unique account identifier, or `null` if authentication + * failed. + */ + abstract public function getAccountID(); + + + /** + * Get a string identifying this adapter, like "ldap". This string should be + * unique to the adapter class. + * + * @return string Unique adapter identifier. + */ + abstract public function getAdapterType(); + + + /** + * Get a string identifying the domain this adapter is acting on. This allows + * an adapter (like LDAP) to act against different identity domains without + * conflating credentials. For providers like Facebook or Google, the adapters + * just return the relevant domain name. + * + * @return string Domain the adapter is associated with. + */ + abstract public function getAdapterDomain(); + + + /** + * Generate a string uniquely identifying this adapter configuration. Within + * the scope of a given key, all account IDs must uniquely identify exactly + * one identity. + * + * @return string Unique identifier for this adapter configuration. + */ + public function getAdapterKey() { + return $this->getAdapterType().':'.$this->getAdapterDomain(); + } + + + /** + * Optionally, return an email address associated with this account. + * + * @return string|null An email address associated with the account, or + * `null` if data is not available. + */ + public function getAccountEmail() { + return null; + } + + + /** + * Optionally, return a human readable username associated with this account. + * + * @return string|null Account username, or `null` if data isn't available. + */ + public function getAccountName() { + return null; + } + + + /** + * Optionally, return a URI corresponding to a human-viewable profile for + * this account. + * + * @return string|null A profile URI associated with this account, or + * `null` if the data isn't available. + */ + public function getAccountURI() { + return null; + } + + + /** + * Optionally, return a profile image URI associated with this account. + * + * @return string|null URI for an account profile image, or `null` if one is + * not available. + */ + public function getAccountImageURI() { + return null; + } + + + /** + * Optionally, return a real name associated with this account. + * + * @return string|null A human real name, or `null` if this data is not + * available. + */ + public function getAccountRealName() { + return null; + } + +} diff --git a/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php b/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php new file mode 100644 index 0000000000..1384548d5e --- /dev/null +++ b/src/applications/auth/adapter/PhutilBitbucketAuthAdapter.php @@ -0,0 +1,73 @@ +getUserInfo(), 'username'); + } + + public function getAccountName() { + return idx($this->getUserInfo(), 'display_name'); + } + + public function getAccountURI() { + $name = $this->getAccountID(); + if (strlen($name)) { + return 'https://bitbucket.org/'.$name; + } + return null; + } + + public function getAccountImageURI() { + return idx($this->getUserInfo(), 'avatar'); + } + + public function getAccountRealName() { + $parts = array( + idx($this->getUserInfo(), 'first_name'), + idx($this->getUserInfo(), 'last_name'), + ); + $parts = array_filter($parts); + return implode(' ', $parts); + } + + public function getAdapterType() { + return 'bitbucket'; + } + + public function getAdapterDomain() { + return 'bitbucket.org'; + } + + protected function getRequestTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/request_token'; + } + + protected function getAuthorizeTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/authenticate'; + } + + protected function getValidateTokenURI() { + return 'https://bitbucket.org/api/1.0/oauth/access_token'; + } + + private function getUserInfo() { + if ($this->userInfo === null) { + // We don't need any of the data in the handshake, but do need to + // finish the process. This makes sure we've completed the handshake. + $this->getHandshakeData(); + + $uri = new PhutilURI('https://bitbucket.org/api/1.0/user'); + + $data = $this->newOAuth1Future($uri) + ->setMethod('GET') + ->resolveJSON(); + + $this->userInfo = idx($data, 'user', array()); + } + return $this->userInfo; + } + +} diff --git a/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php b/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php new file mode 100644 index 0000000000..b9a33b293a --- /dev/null +++ b/src/applications/auth/adapter/PhutilDisqusAuthAdapter.php @@ -0,0 +1,84 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('username'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar', 'permalink'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('profileUrl'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://disqus.com/api/oauth/2.0/authorize/'; + } + + protected function getTokenBaseURI() { + return 'https://disqus.com/api/oauth/2.0/access_token/'; + } + + public function getScope() { + return 'read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://disqus.com/api/3.0/users/details.json'); + $uri->replaceQueryParam('api_key', $this->getClientID()); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + $uri = (string)$uri; + + $future = new HTTPSFuture($uri); + $future->setMethod('GET'); + list($body) = $future->resolvex(); + + try { + $data = phutil_json_decode($body); + return $data['response']; + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Disqus account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php b/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php new file mode 100644 index 0000000000..a59dddd262 --- /dev/null +++ b/src/applications/auth/adapter/PhutilEmptyAuthAdapter.php @@ -0,0 +1,42 @@ +adapterDomain = $adapter_domain; + return $this; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterType($adapter_type) { + $this->adapterType = $adapter_type; + return $this; + } + + public function getAdapterType() { + return $this->adapterType; + } + + public function setAccountID($account_id) { + $this->accountID = $account_id; + return $this; + } + + public function getAccountID() { + return $this->accountID; + } + +} diff --git a/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php b/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php new file mode 100644 index 0000000000..ab569a14a0 --- /dev/null +++ b/src/applications/auth/adapter/PhutilFacebookAuthAdapter.php @@ -0,0 +1,114 @@ +requireSecureBrowsing = $require_secure_browsing; + return $this; + } + + public function getAdapterType() { + return 'facebook'; + } + + public function getAdapterDomain() { + return 'facebook.com'; + } + + public function getAccountID() { + return $this->getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + $link = $this->getOAuthAccountData('link'); + if (!$link) { + return null; + } + + $matches = null; + if (!preg_match('@/([^/]+)$@', $link, $matches)) { + return null; + } + + return $matches[1]; + } + + public function getAccountImageURI() { + $picture = $this->getOAuthAccountData('picture'); + if ($picture) { + $picture_data = idx($picture, 'data'); + if ($picture_data) { + return idx($picture_data, 'url'); + } + } + return null; + } + + public function getAccountURI() { + return $this->getOAuthAccountData('link'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + public function getAccountSecuritySettings() { + return $this->getOAuthAccountData('security_settings'); + } + + protected function getAuthenticateBaseURI() { + return 'https://www.facebook.com/dialog/oauth'; + } + + protected function getTokenBaseURI() { + return 'https://graph.facebook.com/oauth/access_token'; + } + + protected function loadOAuthAccountData() { + $fields = array( + 'id', + 'name', + 'email', + 'link', + 'security_settings', + 'picture', + ); + + $uri = new PhutilURI('https://graph.facebook.com/me'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + $uri->replaceQueryParam('fields', implode(',', $fields)); + list($body) = id(new HTTPSFuture($uri))->resolvex(); + + $data = null; + try { + $data = phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Facebook account data request.'), + $ex); + } + + if ($this->requireSecureBrowsing) { + if (empty($data['security_settings']['secure_browsing']['enabled'])) { + throw new Exception( + pht( + 'This Phabricator install requires you to enable Secure Browsing '. + 'on your Facebook account in order to use it to log in to '. + 'Phabricator. For more information, see %s', + 'https://www.facebook.com/help/156201551113407/')); + } + } + + return $data; + } + +} diff --git a/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php b/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php new file mode 100644 index 0000000000..7fa8a660fa --- /dev/null +++ b/src/applications/auth/adapter/PhutilGitHubAuthAdapter.php @@ -0,0 +1,72 @@ +getOAuthAccountData('id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('login'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar_url'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if (strlen($name)) { + return 'https://github.com/'.$name; + } + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://github.com/login/oauth/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://github.com/login/oauth/access_token'; + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://api.github.com/user'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + + // NOTE: GitHub requires a User-Agent string. + $future->addHeader('User-Agent', __CLASS__); + + list($body) = $future->resolvex(); + + try { + return phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from GitHub account data request.'), + $ex); + } + } + +} diff --git a/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php b/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php new file mode 100644 index 0000000000..11c10087ba --- /dev/null +++ b/src/applications/auth/adapter/PhutilGoogleAuthAdapter.php @@ -0,0 +1,105 @@ +getAccountEmail(); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + // Guess account name from email address, this is just a hint anyway. + $email = $this->getAccountEmail(); + $email = explode('@', $email); + $email = head($email); + return $email; + } + + public function getAccountImageURI() { + $uri = $this->getOAuthAccountData('picture'); + + // Change the "sz" parameter ("size") from the default to 100 to ask for + // a 100x100px image. + if ($uri !== null) { + $uri = new PhutilURI($uri); + $uri->replaceQueryParam('sz', 100); + $uri = (string)$uri; + } + + return $uri; + } + + public function getAccountURI() { + return $this->getOAuthAccountData('link'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://accounts.google.com/o/oauth2/auth'; + } + + protected function getTokenBaseURI() { + return 'https://accounts.google.com/o/oauth2/token'; + } + + public function getScope() { + $scopes = array( + 'email', + 'profile', + ); + + return implode(' ', $scopes); + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = new PhutilURI('https://www.googleapis.com/userinfo/v2/me'); + $uri->replaceQueryParam('access_token', $this->getAccessToken()); + + $future = new HTTPSFuture($uri); + list($status, $body) = $future->resolve(); + + if ($status->isError()) { + throw $status; + } + + try { + $result = phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new PhutilProxyException( + pht('Expected valid JSON response from Google account data request.'), + $ex); + } + + return $result; + } + +} diff --git a/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php b/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php new file mode 100644 index 0000000000..a045065590 --- /dev/null +++ b/src/applications/auth/adapter/PhutilJIRAAuthAdapter.php @@ -0,0 +1,164 @@ +jiraBaseURI = $jira_base_uri; + return $this; + } + + public function getJIRABaseURI() { + return $this->jiraBaseURI; + } + + public function getAccountID() { + // Make sure the handshake is finished; this method is used for its + // side effect by Auth providers. + $this->getHandshakeData(); + + return idx($this->getUserInfo(), 'key'); + } + + public function getAccountName() { + return idx($this->getUserInfo(), 'name'); + } + + public function getAccountImageURI() { + $avatars = idx($this->getUserInfo(), 'avatarUrls'); + if ($avatars) { + return idx($avatars, '48x48'); + } + return null; + } + + public function getAccountRealName() { + return idx($this->getUserInfo(), 'displayName'); + } + + public function getAccountEmail() { + return idx($this->getUserInfo(), 'emailAddress'); + } + + public function getAdapterType() { + return 'jira'; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterDomain($domain) { + $this->adapterDomain = $domain; + return $this; + } + + protected function getSignatureMethod() { + return 'RSA-SHA1'; + } + + protected function getRequestTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/request-token'); + } + + protected function getAuthorizeTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/authorize'); + } + + protected function getValidateTokenURI() { + return $this->getJIRAURI('plugins/servlet/oauth/access-token'); + } + + private function getJIRAURI($path) { + return rtrim($this->jiraBaseURI, '/').'/'.ltrim($path, '/'); + } + + private function getUserInfo() { + if ($this->userInfo === null) { + $this->currentSession = $this->newJIRAFuture('rest/auth/1/session', 'GET') + ->resolveJSON(); + + // The session call gives us the username, but not the user key or other + // information. Make a second call to get additional information. + + $params = array( + 'username' => $this->currentSession['name'], + ); + + $this->userInfo = $this->newJIRAFuture('rest/api/2/user', 'GET', $params) + ->resolveJSON(); + } + + return $this->userInfo; + } + + public static function newJIRAKeypair() { + $config = array( + 'digest_alg' => 'sha512', + 'private_key_bits' => 4096, + 'private_key_type' => OPENSSL_KEYTYPE_RSA, + ); + + $res = openssl_pkey_new($config); + if (!$res) { + throw new Exception(pht('%s failed!', 'openssl_pkey_new()')); + } + + $private_key = null; + $ok = openssl_pkey_export($res, $private_key); + if (!$ok) { + throw new Exception(pht('%s failed!', 'openssl_pkey_export()')); + } + + $public_key = openssl_pkey_get_details($res); + if (!$ok || empty($public_key['key'])) { + throw new Exception(pht('%s failed!', 'openssl_pkey_get_details()')); + } + $public_key = $public_key['key']; + + return array($public_key, $private_key); + } + + + /** + * JIRA indicates that the user has clicked the "Deny" button by passing a + * well known `oauth_verifier` value ("denied"), which we check for here. + */ + protected function willFinishOAuthHandshake() { + $jira_magic_word = 'denied'; + if ($this->getVerifier() == $jira_magic_word) { + throw new PhutilAuthUserAbortedException(); + } + } + + public function newJIRAFuture($path, $method, $params = array()) { + if ($method == 'GET') { + $uri_params = $params; + $body_params = array(); + } else { + // For other types of requests, JIRA expects the request body to be + // JSON encoded. + $uri_params = array(); + $body_params = phutil_json_encode($params); + } + + $uri = new PhutilURI($this->getJIRAURI($path), $uri_params); + + // JIRA returns a 415 error if we don't provide a Content-Type header. + + return $this->newOAuth1Future($uri, $body_params) + ->setMethod($method) + ->addHeader('Content-Type', 'application/json'); + } + +} diff --git a/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php b/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php new file mode 100644 index 0000000000..14047c1761 --- /dev/null +++ b/src/applications/auth/adapter/PhutilLDAPAuthAdapter.php @@ -0,0 +1,505 @@ +hostname = $host; + return $this; + } + + public function setPort($port) { + $this->port = $port; + return $this; + } + + public function getAdapterDomain() { + return 'self'; + } + + public function setBaseDistinguishedName($base_distinguished_name) { + $this->baseDistinguishedName = $base_distinguished_name; + return $this; + } + + public function setSearchAttributes(array $search_attributes) { + $this->searchAttributes = $search_attributes; + return $this; + } + + public function setUsernameAttribute($username_attribute) { + $this->usernameAttribute = $username_attribute; + return $this; + } + + public function setRealNameAttributes(array $attributes) { + $this->realNameAttributes = $attributes; + return $this; + } + + public function setLDAPVersion($ldap_version) { + $this->ldapVersion = $ldap_version; + return $this; + } + + public function setLDAPReferrals($ldap_referrals) { + $this->ldapReferrals = $ldap_referrals; + return $this; + } + + public function setLDAPStartTLS($ldap_start_tls) { + $this->ldapStartTLS = $ldap_start_tls; + return $this; + } + + public function setAnonymousUsername($anonymous_username) { + $this->anonymousUsername = $anonymous_username; + return $this; + } + + public function setAnonymousPassword( + PhutilOpaqueEnvelope $anonymous_password) { + $this->anonymousPassword = $anonymous_password; + return $this; + } + + public function setLoginUsername($login_username) { + $this->loginUsername = $login_username; + return $this; + } + + public function setLoginPassword(PhutilOpaqueEnvelope $login_password) { + $this->loginPassword = $login_password; + return $this; + } + + public function setActiveDirectoryDomain($domain) { + $this->activeDirectoryDomain = $domain; + return $this; + } + + public function setAlwaysSearch($always_search) { + $this->alwaysSearch = $always_search; + return $this; + } + + public function getAccountID() { + return $this->readLDAPRecordAccountID($this->getLDAPUserData()); + } + + public function getAccountName() { + return $this->readLDAPRecordAccountName($this->getLDAPUserData()); + } + + public function getAccountRealName() { + return $this->readLDAPRecordRealName($this->getLDAPUserData()); + } + + public function getAccountEmail() { + return $this->readLDAPRecordEmail($this->getLDAPUserData()); + } + + public function readLDAPRecordAccountID(array $record) { + $key = $this->usernameAttribute; + if (!strlen($key)) { + $key = head($this->searchAttributes); + } + return $this->readLDAPData($record, $key); + } + + public function readLDAPRecordAccountName(array $record) { + return $this->readLDAPRecordAccountID($record); + } + + public function readLDAPRecordRealName(array $record) { + $parts = array(); + foreach ($this->realNameAttributes as $attribute) { + $parts[] = $this->readLDAPData($record, $attribute); + } + $parts = array_filter($parts); + + if ($parts) { + return implode(' ', $parts); + } + + return null; + } + + public function readLDAPRecordEmail(array $record) { + return $this->readLDAPData($record, 'mail'); + } + + private function getLDAPUserData() { + if ($this->ldapUserData === null) { + $this->ldapUserData = $this->loadLDAPUserData(); + } + + return $this->ldapUserData; + } + + private function readLDAPData(array $data, $key, $default = null) { + $list = idx($data, $key); + if ($list === null) { + // At least in some cases (and maybe in all cases) the results from + // ldap_search() are keyed in lowercase. If we missed on the first + // try, retry with a lowercase key. + $list = idx($data, phutil_utf8_strtolower($key)); + } + + // NOTE: In most cases, the property is an array, like: + // + // array( + // 'count' => 1, + // 0 => 'actual-value-we-want', + // ) + // + // However, in at least the case of 'dn', the property is a bare string. + + if (is_scalar($list) && strlen($list)) { + return $list; + } else if (is_array($list)) { + return $list[0]; + } else { + return $default; + } + } + + private function formatLDAPAttributeSearch($attribute, $login_user) { + // If the attribute contains the literal token "${login}", treat it as a + // query and substitute the user's login name for the token. + + if (strpos($attribute, '${login}') !== false) { + $escaped_user = ldap_sprintf('%S', $login_user); + $attribute = str_replace('${login}', $escaped_user, $attribute); + return $attribute; + } + + // Otherwise, treat it as a simple attribute search. + + return ldap_sprintf( + '%Q=%S', + $attribute, + $login_user); + } + + private function loadLDAPUserData() { + $conn = $this->establishConnection(); + + $login_user = $this->loginUsername; + $login_pass = $this->loginPassword; + + if ($this->shouldBindWithoutIdentity()) { + $distinguished_name = null; + $search_query = null; + foreach ($this->searchAttributes as $attribute) { + $search_query = $this->formatLDAPAttributeSearch( + $attribute, + $login_user); + $record = $this->searchLDAPForRecord($search_query); + if ($record) { + $distinguished_name = $this->readLDAPData($record, 'dn'); + break; + } + } + if ($distinguished_name === null) { + throw new PhutilAuthCredentialException(); + } + } else { + $search_query = $this->formatLDAPAttributeSearch( + head($this->searchAttributes), + $login_user); + if ($this->activeDirectoryDomain) { + $distinguished_name = ldap_sprintf( + '%s@%Q', + $login_user, + $this->activeDirectoryDomain); + } else { + $distinguished_name = ldap_sprintf( + '%Q,%Q', + $search_query, + $this->baseDistinguishedName); + } + } + + $this->bindLDAP($conn, $distinguished_name, $login_pass); + + $result = $this->searchLDAPForRecord($search_query); + if (!$result) { + // This is unusual (since the bind succeeded) but we've seen it at least + // once in the wild, where the anonymous user is allowed to search but + // the credentialed user is not. + + // If we don't have anonymous credentials, raise an explicit exception + // here since we'll fail a typehint if we don't return an array anyway + // and this is a more useful error. + + // If we do have anonymous credentials, we'll rebind and try the search + // again below. Doing this automatically means things work correctly more + // often without requiring additional configuration. + if (!$this->shouldBindWithoutIdentity()) { + // No anonymous credentials, so we just fail here. + throw new Exception( + pht( + 'LDAP: Failed to retrieve record for user "%s" when searching. '. + 'Credentialed users may not be able to search your LDAP server. '. + 'Try configuring anonymous credentials or fully anonymous binds.', + $login_user)); + } else { + // Rebind as anonymous and try the search again. + $user = $this->anonymousUsername; + $pass = $this->anonymousPassword; + $this->bindLDAP($conn, $user, $pass); + + $result = $this->searchLDAPForRecord($search_query); + if (!$result) { + throw new Exception( + pht( + 'LDAP: Failed to retrieve record for user "%s" when searching '. + 'with both user and anonymous credentials.', + $login_user)); + } + } + } + + return $result; + } + + private function establishConnection() { + if (!$this->ldapConnection) { + $host = $this->hostname; + $port = $this->port; + + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'connect', + 'host' => $host, + 'port' => $this->port, + )); + + $conn = @ldap_connect($host, $this->port); + + $profiler->endServiceCall( + $call_id, + array( + 'ok' => (bool)$conn, + )); + + if (!$conn) { + throw new Exception( + pht('Unable to connect to LDAP server (%s:%d).', $host, $port)); + } + + $options = array( + LDAP_OPT_PROTOCOL_VERSION => (int)$this->ldapVersion, + LDAP_OPT_REFERRALS => (int)$this->ldapReferrals, + ); + + foreach ($options as $name => $value) { + $ok = @ldap_set_option($conn, $name, $value); + if (!$ok) { + $this->raiseConnectionException( + $conn, + pht( + "Unable to set LDAP option '%s' to value '%s'!", + $name, + $value)); + } + } + + if ($this->ldapStartTLS) { + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'start-tls', + )); + + // NOTE: This boils down to a function call to ldap_start_tls_s() in + // C, which is a service call. + $ok = @ldap_start_tls($conn); + + $profiler->endServiceCall( + $call_id, + array()); + + if (!$ok) { + $this->raiseConnectionException( + $conn, + pht('Unable to start TLS connection when connecting to LDAP.')); + } + } + + if ($this->shouldBindWithoutIdentity()) { + $user = $this->anonymousUsername; + $pass = $this->anonymousPassword; + $this->bindLDAP($conn, $user, $pass); + } + + $this->ldapConnection = $conn; + } + + return $this->ldapConnection; + } + + + private function searchLDAPForRecord($dn) { + $conn = $this->establishConnection(); + + $results = $this->searchLDAP('%Q', $dn); + + if (!$results) { + return null; + } + + if (count($results) > 1) { + throw new Exception( + pht( + 'LDAP record query returned more than one result. The query must '. + 'uniquely identify a record.')); + } + + return head($results); + } + + public function searchLDAP($pattern /* ... */) { + $args = func_get_args(); + $query = call_user_func_array('ldap_sprintf', $args); + + $conn = $this->establishConnection(); + + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'search', + 'dn' => $this->baseDistinguishedName, + 'query' => $query, + )); + + $result = @ldap_search($conn, $this->baseDistinguishedName, $query); + + $profiler->endServiceCall($call_id, array()); + + if (!$result) { + $this->raiseConnectionException( + $conn, + pht('LDAP search failed.')); + } + + $entries = @ldap_get_entries($conn, $result); + + if (!$entries) { + $this->raiseConnectionException( + $conn, + pht('Failed to get LDAP entries from search result.')); + } + + $results = array(); + for ($ii = 0; $ii < $entries['count']; $ii++) { + $results[] = $entries[$ii]; + } + + return $results; + } + + private function raiseConnectionException($conn, $message) { + $errno = @ldap_errno($conn); + $error = @ldap_error($conn); + + // This is `LDAP_INVALID_CREDENTIALS`. + if ($errno == 49) { + throw new PhutilAuthCredentialException(); + } + + if ($errno || $error) { + $full_message = pht( + "LDAP Exception: %s\nLDAP Error #%d: %s", + $message, + $errno, + $error); + } else { + $full_message = pht( + 'LDAP Exception: %s', + $message); + } + + throw new Exception($full_message); + } + + private function bindLDAP($conn, $user, PhutilOpaqueEnvelope $pass) { + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall( + array( + 'type' => 'ldap', + 'call' => 'bind', + 'user' => $user, + )); + + // NOTE: ldap_bind() dumps cleartext passwords into logs by default. Keep + // it quiet. + if (strlen($user)) { + $ok = @ldap_bind($conn, $user, $pass->openEnvelope()); + } else { + $ok = @ldap_bind($conn); + } + + $profiler->endServiceCall($call_id, array()); + + if (!$ok) { + if (strlen($user)) { + $this->raiseConnectionException( + $conn, + pht('Failed to bind to LDAP server (as user "%s").', $user)); + } else { + $this->raiseConnectionException( + $conn, + pht('Failed to bind to LDAP server (without username).')); + } + } + } + + + /** + * Determine if this adapter should attempt to bind to the LDAP server + * without a user identity. + * + * Generally, we can bind directly if we have a username/password, or if the + * "Always Search" flag is set, indicating that the empty username and + * password are sufficient. + * + * @return bool True if the adapter should perform binds without identity. + */ + private function shouldBindWithoutIdentity() { + return $this->alwaysSearch || strlen($this->anonymousUsername); + } + +} diff --git a/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php b/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php new file mode 100644 index 0000000000..aad5f0649d --- /dev/null +++ b/src/applications/auth/adapter/PhutilOAuth1AuthAdapter.php @@ -0,0 +1,211 @@ +privateKey = $private_key; + return $this; + } + + public function getPrivateKey() { + return $this->privateKey; + } + + public function setCallbackURI($callback_uri) { + $this->callbackURI = $callback_uri; + return $this; + } + + public function getCallbackURI() { + return $this->callbackURI; + } + + public function setVerifier($verifier) { + $this->verifier = $verifier; + return $this; + } + + public function getVerifier() { + return $this->verifier; + } + + public function setConsumerSecret(PhutilOpaqueEnvelope $consumer_secret) { + $this->consumerSecret = $consumer_secret; + return $this; + } + + public function getConsumerSecret() { + return $this->consumerSecret; + } + + public function setConsumerKey($consumer_key) { + $this->consumerKey = $consumer_key; + return $this; + } + + public function getConsumerKey() { + return $this->consumerKey; + } + + public function setTokenSecret($token_secret) { + $this->tokenSecret = $token_secret; + return $this; + } + + public function getTokenSecret() { + return $this->tokenSecret; + } + + public function setToken($token) { + $this->token = $token; + return $this; + } + + public function getToken() { + return $this->token; + } + + protected function getHandshakeData() { + if ($this->handshakeData === null) { + $this->finishOAuthHandshake(); + } + return $this->handshakeData; + } + + abstract protected function getRequestTokenURI(); + abstract protected function getAuthorizeTokenURI(); + abstract protected function getValidateTokenURI(); + + protected function getSignatureMethod() { + return 'HMAC-SHA1'; + } + + public function getContentSecurityPolicyFormActions() { + return array( + $this->getAuthorizeTokenURI(), + ); + } + + protected function newOAuth1Future($uri, $data = array()) { + $future = id(new PhutilOAuth1Future($uri, $data)) + ->setMethod('POST') + ->setSignatureMethod($this->getSignatureMethod()); + + $consumer_key = $this->getConsumerKey(); + if (strlen($consumer_key)) { + $future->setConsumerKey($consumer_key); + } else { + throw new Exception( + pht( + '%s is required!', + 'setConsumerKey()')); + } + + $consumer_secret = $this->getConsumerSecret(); + if ($consumer_secret) { + $future->setConsumerSecret($consumer_secret); + } + + if (strlen($this->getToken())) { + $future->setToken($this->getToken()); + } + + if (strlen($this->getTokenSecret())) { + $future->setTokenSecret($this->getTokenSecret()); + } + + if ($this->getPrivateKey()) { + $future->setPrivateKey($this->getPrivateKey()); + } + + return $future; + } + + public function getClientRedirectURI() { + $request_token_uri = $this->getRequestTokenURI(); + + $future = $this->newOAuth1Future($request_token_uri); + if (strlen($this->getCallbackURI())) { + $future->setCallbackURI($this->getCallbackURI()); + } + + list($body) = $future->resolvex(); + $data = id(new PhutilQueryStringParser())->parseQueryString($body); + + // NOTE: Per the spec, this value MUST be the string 'true'. + $confirmed = idx($data, 'oauth_callback_confirmed'); + if ($confirmed !== 'true') { + throw new Exception( + pht("Expected '%s' to be '%s'!", 'oauth_callback_confirmed', 'true')); + } + + $this->readTokenAndTokenSecret($data); + + $authorize_token_uri = new PhutilURI($this->getAuthorizeTokenURI()); + $authorize_token_uri->replaceQueryParam('oauth_token', $this->getToken()); + + return (string)$authorize_token_uri; + } + + protected function finishOAuthHandshake() { + $this->willFinishOAuthHandshake(); + + if (!$this->getToken()) { + throw new Exception(pht('Expected token to finish OAuth handshake!')); + } + if (!$this->getVerifier()) { + throw new Exception(pht('Expected verifier to finish OAuth handshake!')); + } + + $validate_uri = $this->getValidateTokenURI(); + $params = array( + 'oauth_verifier' => $this->getVerifier(), + ); + + list($body) = $this->newOAuth1Future($validate_uri, $params)->resolvex(); + $data = id(new PhutilQueryStringParser())->parseQueryString($body); + + $this->readTokenAndTokenSecret($data); + + $this->handshakeData = $data; + } + + private function readTokenAndTokenSecret(array $data) { + $token = idx($data, 'oauth_token'); + if (!$token) { + throw new Exception(pht("Expected '%s' in response!", 'oauth_token')); + } + + $token_secret = idx($data, 'oauth_token_secret'); + if (!$token_secret) { + throw new Exception( + pht("Expected '%s' in response!", 'oauth_token_secret')); + } + + $this->setToken($token); + $this->setTokenSecret($token_secret); + + return $this; + } + + /** + * Hook that allows subclasses to take actions before the OAuth handshake + * is completed. + */ + protected function willFinishOAuthHandshake() { + return; + } + +} diff --git a/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php b/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php new file mode 100644 index 0000000000..47d299ee36 --- /dev/null +++ b/src/applications/auth/adapter/PhutilOAuthAuthAdapter.php @@ -0,0 +1,228 @@ + $this->getClientID(), + 'scope' => $this->getScope(), + 'redirect_uri' => $this->getRedirectURI(), + 'state' => $this->getState(), + ) + $this->getExtraAuthenticateParameters(); + + $uri = new PhutilURI($this->getAuthenticateBaseURI(), $params); + + return phutil_string_cast($uri); + } + + public function getAdapterType() { + $this_class = get_class($this); + $type_name = str_replace('PhutilAuthAdapterOAuth', '', $this_class); + return strtolower($type_name); + } + + public function setState($state) { + $this->state = $state; + return $this; + } + + public function getState() { + return $this->state; + } + + public function setCode($code) { + $this->code = $code; + return $this; + } + + public function getCode() { + return $this->code; + } + + public function setRedirectURI($redirect_uri) { + $this->redirectURI = $redirect_uri; + return $this; + } + + public function getRedirectURI() { + return $this->redirectURI; + } + + public function getExtraAuthenticateParameters() { + return array(); + } + + public function getExtraTokenParameters() { + return array(); + } + + public function getExtraRefreshParameters() { + return array(); + } + + public function setScope($scope) { + $this->scope = $scope; + return $this; + } + + public function getScope() { + return $this->scope; + } + + public function setClientSecret(PhutilOpaqueEnvelope $client_secret) { + $this->clientSecret = $client_secret; + return $this; + } + + public function getClientSecret() { + return $this->clientSecret; + } + + public function setClientID($client_id) { + $this->clientID = $client_id; + return $this; + } + + public function getClientID() { + return $this->clientID; + } + + public function getAccessToken() { + return $this->getAccessTokenData('access_token'); + } + + public function getAccessTokenExpires() { + return $this->getAccessTokenData('expires_epoch'); + } + + public function getRefreshToken() { + return $this->getAccessTokenData('refresh_token'); + } + + protected function getAccessTokenData($key, $default = null) { + if ($this->accessTokenData === null) { + $this->accessTokenData = $this->loadAccessTokenData(); + } + + return idx($this->accessTokenData, $key, $default); + } + + public function supportsTokenRefresh() { + return false; + } + + public function refreshAccessToken($refresh_token) { + $this->accessTokenData = $this->loadRefreshTokenData($refresh_token); + return $this; + } + + protected function loadRefreshTokenData($refresh_token) { + $params = array( + 'refresh_token' => $refresh_token, + ) + $this->getExtraRefreshParameters(); + + // NOTE: Make sure we return the refresh_token so that subsequent + // calls to getRefreshToken() return it; providers normally do not echo + // it back for token refresh requests. + + return $this->makeTokenRequest($params) + array( + 'refresh_token' => $refresh_token, + ); + } + + protected function loadAccessTokenData() { + $code = $this->getCode(); + if (!$code) { + throw new PhutilInvalidStateException('setCode'); + } + + $params = array( + 'code' => $this->getCode(), + ) + $this->getExtraTokenParameters(); + + return $this->makeTokenRequest($params); + } + + private function makeTokenRequest(array $params) { + $uri = $this->getTokenBaseURI(); + $query_data = array( + 'client_id' => $this->getClientID(), + 'client_secret' => $this->getClientSecret()->openEnvelope(), + 'redirect_uri' => $this->getRedirectURI(), + ) + $params; + + $future = new HTTPSFuture($uri, $query_data); + $future->setMethod('POST'); + list($body) = $future->resolvex(); + + $data = $this->readAccessTokenResponse($body); + + if (isset($data['expires_in'])) { + $data['expires_epoch'] = $data['expires_in']; + } else if (isset($data['expires'])) { + $data['expires_epoch'] = $data['expires']; + } + + // If we got some "expires" value back, interpret it as an epoch timestamp + // if it's after the year 2010 and as a relative number of seconds + // otherwise. + if (isset($data['expires_epoch'])) { + if ($data['expires_epoch'] < (60 * 60 * 24 * 365 * 40)) { + $data['expires_epoch'] += time(); + } + } + + if (isset($data['error'])) { + throw new Exception(pht('Access token error: %s', $data['error'])); + } + + return $data; + } + + protected function readAccessTokenResponse($body) { + // NOTE: Most providers either return JSON or HTTP query strings, so try + // both mechanisms. If your provider does something else, override this + // method. + + $data = json_decode($body, true); + + if (!is_array($data)) { + $data = array(); + parse_str($body, $data); + } + + if (empty($data['access_token']) && + empty($data['error'])) { + throw new Exception( + pht('Failed to decode OAuth access token response: %s', $body)); + } + + return $data; + } + + protected function getOAuthAccountData($key, $default = null) { + if ($this->oauthAccountData === null) { + $this->oauthAccountData = $this->loadOAuthAccountData(); + } + + return idx($this->oauthAccountData, $key, $default); + } + +} diff --git a/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php b/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php new file mode 100644 index 0000000000..e66ba32f94 --- /dev/null +++ b/src/applications/auth/adapter/PhutilPhabricatorAuthAdapter.php @@ -0,0 +1,102 @@ +phabricatorBaseURI = $uri; + return $this; + } + + public function getPhabricatorBaseURI() { + return $this->phabricatorBaseURI; + } + + public function getAdapterDomain() { + return $this->adapterDomain; + } + + public function setAdapterDomain($domain) { + $this->adapterDomain = $domain; + return $this; + } + + public function getAdapterType() { + return 'phabricator'; + } + + public function getAccountID() { + return $this->getOAuthAccountData('phid'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('primaryEmail'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('userName'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('image'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('uri'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('realName'); + } + + protected function getAuthenticateBaseURI() { + return $this->getPhabricatorURI('oauthserver/auth/'); + } + + protected function getTokenBaseURI() { + return $this->getPhabricatorURI('oauthserver/token/'); + } + + public function getScope() { + return ''; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + $uri = id(new PhutilURI($this->getPhabricatorURI('api/user.whoami'))) + ->replaceQueryParam('access_token', $this->getAccessToken()); + list($body) = id(new HTTPSFuture($uri))->resolvex(); + + try { + $data = phutil_json_decode($body); + return $data['result']; + } catch (PhutilJSONParserException $ex) { + throw new Exception( + pht( + 'Expected valid JSON response from Phabricator %s request.', + 'user.whoami'), + $ex); + } + } + + private function getPhabricatorURI($path) { + return rtrim($this->phabricatorBaseURI, '/').'/'.ltrim($path, '/'); + } + +} diff --git a/src/applications/auth/adapter/PhutilSlackAuthAdapter.php b/src/applications/auth/adapter/PhutilSlackAuthAdapter.php new file mode 100644 index 0000000000..6578a9af7a --- /dev/null +++ b/src/applications/auth/adapter/PhutilSlackAuthAdapter.php @@ -0,0 +1,61 @@ +getOAuthAccountData('user'); + return idx($user, 'id'); + } + + public function getAccountEmail() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'email'); + } + + public function getAccountImageURI() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'image_512'); + } + + public function getAccountRealName() { + $user = $this->getOAuthAccountData('user'); + return idx($user, 'name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://slack.com/oauth/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://slack.com/api/oauth.access'; + } + + public function getScope() { + return 'identity.basic,identity.team,identity.avatar'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilSlackFuture()) + ->setAccessToken($this->getAccessToken()) + ->setRawSlackQuery('users.identity') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php b/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php new file mode 100644 index 0000000000..dce2c7e2f0 --- /dev/null +++ b/src/applications/auth/adapter/PhutilTwitchAuthAdapter.php @@ -0,0 +1,76 @@ +getOAuthAccountData('_id'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('name'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('logo'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if ($name) { + return 'http://www.twitch.tv/'.$name; + } + return null; + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('display_name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://api.twitch.tv/kraken/oauth2/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://api.twitch.tv/kraken/oauth2/token'; + } + + public function getScope() { + return 'user_read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilTwitchFuture()) + ->setClientID($this->getClientID()) + ->setAccessToken($this->getAccessToken()) + ->setRawTwitchQuery('user') + ->resolve(); + } + +} diff --git a/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php b/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php new file mode 100644 index 0000000000..6f738c75f6 --- /dev/null +++ b/src/applications/auth/adapter/PhutilTwitterAuthAdapter.php @@ -0,0 +1,75 @@ +getHandshakeData(), 'user_id'); + } + + public function getAccountName() { + return idx($this->getHandshakeData(), 'screen_name'); + } + + public function getAccountURI() { + $name = $this->getAccountName(); + if (strlen($name)) { + return 'https://twitter.com/'.$name; + } + return null; + } + + public function getAccountImageURI() { + $info = $this->getUserInfo(); + return idx($info, 'profile_image_url'); + } + + public function getAccountRealName() { + $info = $this->getUserInfo(); + return idx($info, 'name'); + } + + public function getAdapterType() { + return 'twitter'; + } + + public function getAdapterDomain() { + return 'twitter.com'; + } + + protected function getRequestTokenURI() { + return 'https://api.twitter.com/oauth/request_token'; + } + + protected function getAuthorizeTokenURI() { + return 'https://api.twitter.com/oauth/authorize'; + } + + protected function getValidateTokenURI() { + return 'https://api.twitter.com/oauth/access_token'; + } + + private function getUserInfo() { + if ($this->userInfo === null) { + $params = array( + 'user_id' => $this->getAccountID(), + ); + + $uri = new PhutilURI( + 'https://api.twitter.com/1.1/users/show.json', + $params); + + $data = $this->newOAuth1Future($uri) + ->setMethod('GET') + ->resolveJSON(); + + $this->userInfo = $data; + } + return $this->userInfo; + } + +} diff --git a/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php b/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php new file mode 100644 index 0000000000..c09c7cffab --- /dev/null +++ b/src/applications/auth/adapter/PhutilWordPressAuthAdapter.php @@ -0,0 +1,73 @@ +getOAuthAccountData('ID'); + } + + public function getAccountEmail() { + return $this->getOAuthAccountData('email'); + } + + public function getAccountName() { + return $this->getOAuthAccountData('username'); + } + + public function getAccountImageURI() { + return $this->getOAuthAccountData('avatar_URL'); + } + + public function getAccountURI() { + return $this->getOAuthAccountData('profile_URL'); + } + + public function getAccountRealName() { + return $this->getOAuthAccountData('display_name'); + } + + protected function getAuthenticateBaseURI() { + return 'https://public-api.wordpress.com/oauth2/authorize'; + } + + protected function getTokenBaseURI() { + return 'https://public-api.wordpress.com/oauth2/token'; + } + + public function getScope() { + return 'user_read'; + } + + public function getExtraAuthenticateParameters() { + return array( + 'response_type' => 'code', + 'blog_id' => 0, + ); + } + + public function getExtraTokenParameters() { + return array( + 'grant_type' => 'authorization_code', + ); + } + + protected function loadOAuthAccountData() { + return id(new PhutilWordPressFuture()) + ->setClientID($this->getClientID()) + ->setAccessToken($this->getAccessToken()) + ->setRawWordPressQuery('/me/') + ->resolve(); + } + +} diff --git a/src/applications/auth/controller/PhabricatorLogoutController.php b/src/applications/auth/controller/PhabricatorLogoutController.php index dccf6bb45b..d71d080cbd 100644 --- a/src/applications/auth/controller/PhabricatorLogoutController.php +++ b/src/applications/auth/controller/PhabricatorLogoutController.php @@ -68,12 +68,42 @@ final class PhabricatorLogoutController ->setURI('/auth/loggedout/'); } + if ($viewer->getPHID()) { - return $this->newDialog() + $dialog = $this->newDialog() ->setTitle(pht('Log Out?')) - ->appendChild(pht('Are you sure you want to log out?')) - ->addSubmitButton(pht('Log Out')) + ->appendParagraph(pht('Are you sure you want to log out?')) ->addCancelButton('/'); + + $configs = id(new PhabricatorAuthProviderConfigQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->execute(); + if (!$configs) { + $dialog + ->appendRemarkup( + pht( + 'WARNING: You have not configured any authentication providers '. + 'yet, so your account has no login credentials. If you log out '. + 'now, you will not be able to log back in normally.')) + ->appendParagraph( + pht( + 'To enable the login flow, follow setup guidance and configure '. + 'at least one authentication provider, then associate '. + 'credentials with your account. After completing these steps, '. + 'you will be able to log out and log back in normally.')) + ->appendParagraph( + pht( + 'If you log out now, you can still regain access to your '. + 'account later by using the account recovery workflow. The '. + 'login screen will prompt you with recovery instructions.')); + + $button = pht('Log Out Anyway'); + } else { + $button = pht('Log Out'); + } + + $dialog->addSubmitButton($button); + return $dialog; } return id(new AphrontRedirectResponse())->setURI('/'); diff --git a/src/applications/auth/controller/config/PhabricatorAuthListController.php b/src/applications/auth/controller/config/PhabricatorAuthListController.php index 5d1d85cca6..b25c791e27 100644 --- a/src/applications/auth/controller/config/PhabricatorAuthListController.php +++ b/src/applications/auth/controller/config/PhabricatorAuthListController.php @@ -64,7 +64,7 @@ final class PhabricatorAuthListController array( 'href' => $this->getApplicationURI('config/new/'), ), - pht('Add Authentication Provider')))); + pht('Add Provider')))); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Login and Registration')); diff --git a/src/applications/auth/exception/PhutilAuthConfigurationException.php b/src/applications/auth/exception/PhutilAuthConfigurationException.php new file mode 100644 index 0000000000..e684076d84 --- /dev/null +++ b/src/applications/auth/exception/PhutilAuthConfigurationException.php @@ -0,0 +1,6 @@ +\d{4})(?P\d{2})(?P\d{2})'. + '(?:'. + 'T(?P\d{2})(?P\d{2})(?P\d{2})(?Z)?'. + ')?'. + '\z/'; + + $matches = null; + $ok = preg_match($pattern, $value, $matches); + if (!$ok) { + throw new Exception( + pht( + 'Expected ISO8601 datetime in the format "19990105T112233Z", '. + 'found "%s".', + $value)); + } + + if (isset($matches['z'])) { + if ($timezone != 'UTC') { + throw new Exception( + pht( + 'ISO8601 date ends in "Z" indicating UTC, but a timezone other '. + 'than UTC ("%s") was specified.', + $timezone)); + } + } + + $datetime = id(new self()) + ->setYear((int)$matches['y']) + ->setMonth((int)$matches['m']) + ->setDay((int)$matches['d']) + ->setTimezone($timezone); + + if (isset($matches['h'])) { + $datetime + ->setHour((int)$matches['h']) + ->setMinute((int)$matches['i']) + ->setSecond((int)$matches['s']); + } else { + $datetime + ->setIsAllDay(true); + } + + return $datetime; + } + + public static function newFromEpoch($epoch, $timezone = 'UTC') { + $date = new DateTime('@'.$epoch); + + $zone = new DateTimeZone($timezone); + $date->setTimezone($zone); + + return id(new self()) + ->setYear((int)$date->format('Y')) + ->setMonth((int)$date->format('m')) + ->setDay((int)$date->format('d')) + ->setHour((int)$date->format('H')) + ->setMinute((int)$date->format('i')) + ->setSecond((int)$date->format('s')) + ->setTimezone($timezone); + } + + public static function newFromDictionary(array $dict) { + static $keys; + if ($keys === null) { + $keys = array_fuse( + array( + 'kind', + 'year', + 'month', + 'day', + 'hour', + 'minute', + 'second', + 'timezone', + 'isAllDay', + )); + } + + foreach ($dict as $key => $value) { + if (!isset($keys[$key])) { + throw new Exception( + pht( + 'Unexpected key "%s" in datetime dictionary, expected keys: %s.', + $key, + implode(', ', array_keys($keys)))); + } + } + + if (idx($dict, 'kind') !== 'absolute') { + throw new Exception( + pht( + 'Expected key "%s" with value "%s" in datetime dictionary.', + 'kind', + 'absolute')); + } + + if (!isset($dict['year'])) { + throw new Exception( + pht( + 'Expected key "%s" in datetime dictionary.', + 'year')); + } + + $datetime = id(new self()) + ->setYear(idx($dict, 'year')) + ->setMonth(idx($dict, 'month', 1)) + ->setDay(idx($dict, 'day', 1)) + ->setHour(idx($dict, 'hour', 0)) + ->setMinute(idx($dict, 'minute', 0)) + ->setSecond(idx($dict, 'second', 0)) + ->setTimezone(idx($dict, 'timezone')) + ->setIsAllDay((bool)idx($dict, 'isAllDay', false)); + + return $datetime; + } + + public function newRelativeDateTime($duration) { + if (is_string($duration)) { + $duration = PhutilCalendarDuration::newFromISO8601($duration); + } + + if (!($duration instanceof PhutilCalendarDuration)) { + throw new Exception( + pht( + 'Expected "PhutilCalendarDuration" object or ISO8601 duration '. + 'string.')); + } + + return id(new PhutilCalendarRelativeDateTime()) + ->setOrigin($this) + ->setDuration($duration); + } + + public function toDictionary() { + return array( + 'kind' => 'absolute', + 'year' => (int)$this->getYear(), + 'month' => (int)$this->getMonth(), + 'day' => (int)$this->getDay(), + 'hour' => (int)$this->getHour(), + 'minute' => (int)$this->getMinute(), + 'second' => (int)$this->getSecond(), + 'timezone' => $this->getTimezone(), + 'isAllDay' => (bool)$this->getIsAllDay(), + ); + } + + public function setYear($year) { + $this->year = $year; + return $this; + } + + public function getYear() { + return $this->year; + } + + public function setMonth($month) { + $this->month = $month; + return $this; + } + + public function getMonth() { + return $this->month; + } + + public function setDay($day) { + $this->day = $day; + return $this; + } + + public function getDay() { + return $this->day; + } + + public function setHour($hour) { + $this->hour = $hour; + return $this; + } + + public function getHour() { + return $this->hour; + } + + public function setMinute($minute) { + $this->minute = $minute; + return $this; + } + + public function getMinute() { + return $this->minute; + } + + public function setSecond($second) { + $this->second = $second; + return $this; + } + + public function getSecond() { + return $this->second; + } + + public function setTimezone($timezone) { + $this->timezone = $timezone; + return $this; + } + + public function getTimezone() { + return $this->timezone; + } + + private function getEffectiveTimezone() { + $date_timezone = $this->getTimezone(); + $viewer_timezone = $this->getViewerTimezone(); + + // Because all-day events are always "floating", the effective timezone + // is the viewer timezone if it is available. Otherwise, we'll return a + // DateTime object with the correct values, but it will be incorrectly + // adjusted forward or backward to the viewer's zone later. + + $zones = array(); + if ($this->getIsAllDay()) { + $zones[] = $viewer_timezone; + $zones[] = $date_timezone; + } else { + $zones[] = $date_timezone; + $zones[] = $viewer_timezone; + } + $zones = array_filter($zones); + + if (!$zones) { + throw new Exception( + pht( + 'Datetime has no timezone or viewer timezone.')); + } + + return head($zones); + } + + public function newPHPDateTimeZone() { + $zone = $this->getEffectiveTimezone(); + return new DateTimeZone($zone); + } + + public function newPHPDateTime() { + $zone = $this->newPHPDateTimeZone(); + + $y = $this->getYear(); + $m = $this->getMonth(); + $d = $this->getDay(); + + if ($this->getIsAllDay()) { + $h = 0; + $i = 0; + $s = 0; + } else { + $h = $this->getHour(); + $i = $this->getMinute(); + $s = $this->getSecond(); + } + + $format = sprintf('%04d-%02d-%02d %02d:%02d:%02d', $y, $m, $d, $h, $i, $s); + + return new DateTime($format, $zone); + } + + + public function newAbsoluteDateTime() { + return clone $this; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php b/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php new file mode 100644 index 0000000000..5beebb784e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarContainerNode.php @@ -0,0 +1,30 @@ +children; + } + + final public function getChildrenOfType($type) { + $result = array(); + + foreach ($this->getChildren() as $key => $child) { + if ($child->getNodeType() != $type) { + continue; + } + $result[$key] = $child; + } + + return $result; + } + + final public function appendChild(PhutilCalendarNode $node) { + $this->children[] = $node; + return $this; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarDateTime.php new file mode 100644 index 0000000000..f9bf5a07a4 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDateTime.php @@ -0,0 +1,54 @@ +viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function setIsAllDay($is_all_day) { + $this->isAllDay = $is_all_day; + return $this; + } + + public function getIsAllDay() { + return $this->isAllDay; + } + + public function getEpoch() { + $datetime = $this->newPHPDateTime(); + return (int)$datetime->format('U'); + } + + public function getISO8601() { + $datetime = $this->newPHPDateTime(); + + if ($this->getIsAllDay()) { + return $datetime->format('Ymd'); + } else if ($this->getTimezone()) { + // With a timezone, the event occurs at a specific second universally. + // We return the UTC representation of that point in time. + $datetime->setTimezone(new DateTimeZone('UTC')); + return $datetime->format('Ymd\\THis\\Z'); + } else { + // With no timezone, events are "floating" and occur at local time. + // We return a representation without the "Z". + return $datetime->format('Ymd\\THis'); + } + } + + abstract public function newPHPDateTimeZone(); + abstract public function newPHPDateTime(); + abstract public function newAbsoluteDateTime(); + + abstract public function getTimezone(); +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php b/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php new file mode 100644 index 0000000000..b2e92dd432 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDocumentNode.php @@ -0,0 +1,12 @@ +getChildrenOfType(PhutilCalendarEventNode::NODETYPE); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarDuration.php b/src/applications/calendar/parser/data/PhutilCalendarDuration.php new file mode 100644 index 0000000000..863150e17e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarDuration.php @@ -0,0 +1,181 @@ + $value) { + if (!isset($keys[$key])) { + throw new Exception( + pht( + 'Unexpected key "%s" in duration dictionary, expected keys: %s.', + $key, + implode(', ', array_keys($keys)))); + } + } + + $duration = id(new self()) + ->setIsNegative(idx($dict, 'isNegative', false)) + ->setWeeks(idx($dict, 'weeks', 0)) + ->setDays(idx($dict, 'days', 0)) + ->setHours(idx($dict, 'hours', 0)) + ->setMinutes(idx($dict, 'minutes', 0)) + ->setSeconds(idx($dict, 'seconds', 0)); + + return $duration; + } + + public function toDictionary() { + return array( + 'isNegative' => $this->getIsNegative(), + 'weeks' => $this->getWeeks(), + 'days' => $this->getDays(), + 'hours' => $this->getHours(), + 'minutes' => $this->getMinutes(), + 'seconds' => $this->getSeconds(), + ); + } + + public static function newFromISO8601($value) { + $pattern = + '/^'. + '(?P[+-])?'. + 'P'. + '(?:'. + '(?P\d+)W'. + '|'. + '(?:(?:(?P\d+)D)?'. + '(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?(?:(?P\d+)S)?)?'. + ')'. + ')'. + '\z/'; + + $matches = null; + $ok = preg_match($pattern, $value, $matches); + if (!$ok) { + throw new Exception( + pht( + 'Expected ISO8601 duration in the format "P12DT3H4M5S", found '. + '"%s".', + $value)); + } + + $is_negative = (idx($matches, 'sign') == '-'); + + return id(new self()) + ->setIsNegative($is_negative) + ->setWeeks((int)idx($matches, 'W', 0)) + ->setDays((int)idx($matches, 'D', 0)) + ->setHours((int)idx($matches, 'H', 0)) + ->setMinutes((int)idx($matches, 'M', 0)) + ->setSeconds((int)idx($matches, 'S', 0)); + } + + public function toISO8601() { + $parts = array(); + $parts[] = 'P'; + + $weeks = $this->getWeeks(); + if ($weeks) { + $parts[] = $weeks.'W'; + } else { + $days = $this->getDays(); + if ($days) { + $parts[] = $days.'D'; + } + + $parts[] = 'T'; + + $hours = $this->getHours(); + if ($hours) { + $parts[] = $hours.'H'; + } + + $minutes = $this->getMinutes(); + if ($minutes) { + $parts[] = $minutes.'M'; + } + + $seconds = $this->getSeconds(); + if ($seconds) { + $parts[] = $seconds.'S'; + } + } + + return implode('', $parts); + } + + public function setIsNegative($is_negative) { + $this->isNegative = $is_negative; + return $this; + } + + public function getIsNegative() { + return $this->isNegative; + } + + public function setWeeks($weeks) { + $this->weeks = $weeks; + return $this; + } + + public function getWeeks() { + return $this->weeks; + } + + public function setDays($days) { + $this->days = $days; + return $this; + } + + public function getDays() { + return $this->days; + } + + public function setHours($hours) { + $this->hours = $hours; + return $this; + } + + public function getHours() { + return $this->hours; + } + + public function setMinutes($minutes) { + $this->minutes = $minutes; + return $this; + } + + public function getMinutes() { + return $this->minutes; + } + + public function setSeconds($seconds) { + $this->seconds = $seconds; + return $this; + } + + public function getSeconds() { + return $this->seconds; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarEventNode.php b/src/applications/calendar/parser/data/PhutilCalendarEventNode.php new file mode 100644 index 0000000000..d3d33aa77e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarEventNode.php @@ -0,0 +1,172 @@ +uid = $uid; + return $this; + } + + public function getUID() { + return $this->uid; + } + + public function setName($name) { + $this->name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setDescription($description) { + $this->description = $description; + return $this; + } + + public function getDescription() { + return $this->description; + } + + public function setStartDateTime(PhutilCalendarDateTime $start) { + $this->startDateTime = $start; + return $this; + } + + public function getStartDateTime() { + return $this->startDateTime; + } + + public function setEndDateTime(PhutilCalendarDateTime $end) { + $this->endDateTime = $end; + return $this; + } + + public function getEndDateTime() { + $end = $this->endDateTime; + if ($end) { + return $end; + } + + $start = $this->getStartDateTime(); + $duration = $this->getDuration(); + if ($start && $duration) { + return id(new PhutilCalendarRelativeDateTime()) + ->setOrigin($start) + ->setDuration($duration); + } + + // If no end date or duration are specified, the event is instantaneous. + return $start; + } + + public function setDuration(PhutilCalendarDuration $duration) { + $this->duration = $duration; + return $this; + } + + public function getDuration() { + return $this->duration; + } + + public function setCreatedDateTime(PhutilCalendarDateTime $created) { + $this->createdDateTime = $created; + return $this; + } + + public function getCreatedDateTime() { + return $this->createdDateTime; + } + + public function setModifiedDateTime(PhutilCalendarDateTime $modified) { + $this->modifiedDateTime = $modified; + return $this; + } + + public function getModifiedDateTime() { + return $this->modifiedDateTime; + } + + public function setOrganizer(PhutilCalendarUserNode $organizer) { + $this->organizer = $organizer; + return $this; + } + + public function getOrganizer() { + return $this->organizer; + } + + public function setAttendees(array $attendees) { + assert_instances_of($attendees, 'PhutilCalendarUserNode'); + $this->attendees = $attendees; + return $this; + } + + public function getAttendees() { + return $this->attendees; + } + + public function addAttendee(PhutilCalendarUserNode $attendee) { + $this->attendees[] = $attendee; + return $this; + } + + public function setRecurrenceRule( + PhutilCalendarRecurrenceRule $recurrence_rule) { + $this->recurrenceRule = $recurrence_rule; + return $this; + } + + public function getRecurrenceRule() { + return $this->recurrenceRule; + } + + public function setRecurrenceExceptions(array $recurrence_exceptions) { + assert_instances_of($recurrence_exceptions, 'PhutilCalendarDateTime'); + $this->recurrenceExceptions = $recurrence_exceptions; + return $this; + } + + public function getRecurrenceExceptions() { + return $this->recurrenceExceptions; + } + + public function setRecurrenceDates(array $recurrence_dates) { + assert_instances_of($recurrence_dates, 'PhutilCalendarDateTime'); + $this->recurrenceDates = $recurrence_dates; + return $this; + } + + public function getRecurrenceDates() { + return $this->recurrenceDates; + } + + public function setRecurrenceID($recurrence_id) { + $this->recurrenceID = $recurrence_id; + return $this; + } + + public function getRecurrenceID() { + return $this->recurrenceID; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarNode.php b/src/applications/calendar/parser/data/PhutilCalendarNode.php new file mode 100644 index 0000000000..e4d7905bc3 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarNode.php @@ -0,0 +1,20 @@ +getPhobjectClassConstant('NODETYPE'); + } + + final public function setAttribute($key, $value) { + $this->attributes[$key] = $value; + return $this; + } + + final public function getAttribute($key, $default = null) { + return idx($this->attributes, $key, $default); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php new file mode 100644 index 0000000000..1293c484ab --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarProxyDateTime.php @@ -0,0 +1,51 @@ +proxy = $proxy; + return $this; + } + + final protected function getProxy() { + return $this->proxy; + } + + public function __clone() { + $this->proxy = clone $this->proxy; + } + + public function setViewerTimezone($timezone) { + $this->getProxy()->setViewerTimezone($timezone); + return $this; + } + + public function getViewerTimezone() { + return $this->getProxy()->getViewerTimezone(); + } + + public function setIsAllDay($is_all_day) { + $this->getProxy()->setIsAllDay($is_all_day); + return $this; + } + + public function getIsAllDay() { + return $this->getProxy()->getIsAllDay(); + } + + public function newPHPDateTimezone() { + return $this->getProxy()->newPHPDateTimezone(); + } + + public function newPHPDateTime() { + return $this->getProxy()->newPHPDateTime(); + } + + public function getTimezone() { + return $this->getProxy()->getTimezone(); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRawNode.php b/src/applications/calendar/parser/data/PhutilCalendarRawNode.php new file mode 100644 index 0000000000..228b3208fb --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRawNode.php @@ -0,0 +1,8 @@ +dates = $dates; + return $this; + } + + public function getDates() { + return $this->dates; + } + + public function resetSource() { + foreach ($this->getDates() as $date) { + $date->setViewerTimezone($this->getViewerTimezone()); + } + + $order = msort($this->getDates(), 'getEpoch'); + $order = array_reverse($order); + $this->order = $order; + + return $this; + } + + public function getNextEvent($cursor) { + while ($this->order) { + $next = array_pop($this->order); + if ($next->getEpoch() >= $cursor) { + return $next; + } + } + + return null; + } + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php new file mode 100644 index 0000000000..504f7d8e9e --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceRule.php @@ -0,0 +1,1820 @@ +getFrequency(); + + $interval = $this->getInterval(); + if ($interval != 1) { + $parts['INTERVAL'] = $interval; + } + + $by_second = $this->getBySecond(); + if ($by_second) { + $parts['BYSECOND'] = $by_second; + } + + $by_minute = $this->getByMinute(); + if ($by_minute) { + $parts['BYMINUTE'] = $by_minute; + } + + $by_hour = $this->getByHour(); + if ($by_hour) { + $parts['BYHOUR'] = $by_hour; + } + + $by_day = $this->getByDay(); + if ($by_day) { + $parts['BYDAY'] = $by_day; + } + + $by_month = $this->getByMonth(); + if ($by_month) { + $parts['BYMONTH'] = $by_month; + } + + $by_monthday = $this->getByMonthDay(); + if ($by_monthday) { + $parts['BYMONTHDAY'] = $by_monthday; + } + + $by_yearday = $this->getByYearDay(); + if ($by_yearday) { + $parts['BYYEARDAY'] = $by_yearday; + } + + $by_weekno = $this->getByWeekNumber(); + if ($by_weekno) { + $parts['BYWEEKNO'] = $by_weekno; + } + + $by_setpos = $this->getBySetPosition(); + if ($by_setpos) { + $parts['BYSETPOS'] = $by_setpos; + } + + $wkst = $this->getWeekStart(); + if ($wkst != self::WEEKDAY_MONDAY) { + $parts['WKST'] = $wkst; + } + + $count = $this->getCount(); + if ($count) { + $parts['COUNT'] = $count; + } + + $until = $this->getUntil(); + if ($until) { + $parts['UNTIL'] = $until->getISO8601(); + } + + return $parts; + } + + public static function newFromDictionary(array $dict) { + static $expect; + if ($expect === null) { + $expect = array_fuse( + array( + 'FREQ', + 'INTERVAL', + 'BYSECOND', + 'BYMINUTE', + 'BYHOUR', + 'BYDAY', + 'BYMONTH', + 'BYMONTHDAY', + 'BYYEARDAY', + 'BYWEEKNO', + 'BYSETPOS', + 'WKST', + 'UNTIL', + 'COUNT', + )); + } + + foreach ($dict as $key => $value) { + if (empty($expect[$key])) { + throw new Exception( + pht( + 'RRULE dictionary includes unknown key "%s". Expected keys '. + 'are: %s.', + $key, + implode(', ', array_keys($expect)))); + } + } + + $rrule = id(new self()) + ->setFrequency(idx($dict, 'FREQ')) + ->setInterval(idx($dict, 'INTERVAL', 1)) + ->setBySecond(idx($dict, 'BYSECOND', array())) + ->setByMinute(idx($dict, 'BYMINUTE', array())) + ->setByHour(idx($dict, 'BYHOUR', array())) + ->setByDay(idx($dict, 'BYDAY', array())) + ->setByMonth(idx($dict, 'BYMONTH', array())) + ->setByMonthDay(idx($dict, 'BYMONTHDAY', array())) + ->setByYearDay(idx($dict, 'BYYEARDAY', array())) + ->setByWeekNumber(idx($dict, 'BYWEEKNO', array())) + ->setBySetPosition(idx($dict, 'BYSETPOS', array())) + ->setWeekStart(idx($dict, 'WKST', self::WEEKDAY_MONDAY)); + + $count = idx($dict, 'COUNT'); + if ($count) { + $rrule->setCount($count); + } + + $until = idx($dict, 'UNTIL'); + if ($until) { + $until = PhutilCalendarAbsoluteDateTime::newFromISO8601($until); + $rrule->setUntil($until); + } + + return $rrule; + } + + public function toRRULE() { + $dict = $this->toDictionary(); + + $parts = array(); + foreach ($dict as $key => $value) { + if (is_array($value)) { + $value = implode(',', $value); + } + $parts[] = "{$key}={$value}"; + } + + return implode(';', $parts); + } + + public static function newFromRRULE($rrule) { + $parts = explode(';', $rrule); + + $dict = array(); + foreach ($parts as $part) { + list($key, $value) = explode('=', $part, 2); + switch ($key) { + case 'FREQ': + case 'INTERVAL': + case 'WKST': + case 'COUNT': + case 'UNTIL'; + break; + default: + $value = explode(',', $value); + break; + } + $dict[$key] = $value; + } + + $int_lists = array_fuse( + array( + // NOTE: "BYDAY" is absent, and takes a list like "MO, TU, WE". + 'BYSECOND', + 'BYMINUTE', + 'BYHOUR', + 'BYMONTH', + 'BYMONTHDAY', + 'BYYEARDAY', + 'BYWEEKNO', + 'BYSETPOS', + )); + + $int_values = array_fuse( + array( + 'COUNT', + 'INTERVAL', + )); + + foreach ($dict as $key => $value) { + if (isset($int_values[$key])) { + // None of these values may be negative. + if (!preg_match('/^\d+\z/', $value)) { + throw new Exception( + pht( + 'Unexpected value "%s" in "%s" RULE property: expected an '. + 'integer.', + $value, + $key)); + } + $dict[$key] = (int)$value; + } + + if (isset($int_lists[$key])) { + foreach ($value as $k => $v) { + if (!preg_match('/^-?\d+\z/', $v)) { + throw new Exception( + pht( + 'Unexpected value "%s" in "%s" RRULE property: expected '. + 'only integers.', + $v, + $key)); + } + $value[$k] = (int)$v; + } + $dict[$key] = $value; + } + } + + return self::newFromDictionary($dict); + } + + private static function getAllWeekdayConstants() { + return array_keys(self::getWeekdayIndexMap()); + } + + private static function getWeekdayIndexMap() { + static $map = array( + self::WEEKDAY_SUNDAY => self::WEEKINDEX_SUNDAY, + self::WEEKDAY_MONDAY => self::WEEKINDEX_MONDAY, + self::WEEKDAY_TUESDAY => self::WEEKINDEX_TUESDAY, + self::WEEKDAY_WEDNESDAY => self::WEEKINDEX_WEDNESDAY, + self::WEEKDAY_THURSDAY => self::WEEKINDEX_THURSDAY, + self::WEEKDAY_FRIDAY => self::WEEKINDEX_FRIDAY, + self::WEEKDAY_SATURDAY => self::WEEKINDEX_SATURDAY, + ); + + return $map; + } + + private static function getWeekdayIndex($weekday) { + $map = self::getWeekdayIndexMap(); + if (!isset($map[$weekday])) { + $constants = array_keys($map); + throw new Exception( + pht( + 'Weekday "%s" is not a valid weekday constant. Valid constants '. + 'are: %s.', + $weekday, + implode(', ', $constants))); + } + + return $map[$weekday]; + } + + public function setStartDateTime(PhutilCalendarDateTime $start) { + $this->startDateTime = $start; + return $this; + } + + public function getStartDateTime() { + return $this->startDateTime; + } + + public function setCount($count) { + if ($count < 1) { + throw new Exception( + pht( + 'RRULE COUNT value "%s" is invalid: count must be at least 1.', + $count)); + } + + $this->count = $count; + return $this; + } + + public function getCount() { + return $this->count; + } + + public function setUntil(PhutilCalendarDateTime $until) { + $this->until = $until; + return $this; + } + + public function getUntil() { + return $this->until; + } + + public function setFrequency($frequency) { + static $map = array( + self::FREQUENCY_SECONDLY => self::SCALE_SECONDLY, + self::FREQUENCY_MINUTELY => self::SCALE_MINUTELY, + self::FREQUENCY_HOURLY => self::SCALE_HOURLY, + self::FREQUENCY_DAILY => self::SCALE_DAILY, + self::FREQUENCY_WEEKLY => self::SCALE_WEEKLY, + self::FREQUENCY_MONTHLY => self::SCALE_MONTHLY, + self::FREQUENCY_YEARLY => self::SCALE_YEARLY, + ); + + if (empty($map[$frequency])) { + throw new Exception( + pht( + 'RRULE FREQ "%s" is invalid. Valid frequencies are: %s.', + $frequency, + implode(', ', array_keys($map)))); + } + + $this->frequency = $frequency; + $this->frequencyScale = $map[$frequency]; + + return $this; + } + + public function getFrequency() { + return $this->frequency; + } + + public function getFrequencyScale() { + return $this->frequencyScale; + } + + public function setInterval($interval) { + if (!is_int($interval)) { + throw new Exception( + pht( + 'RRULE INTERVAL "%s" is invalid: interval must be an integer.', + $interval)); + } + + if ($interval < 1) { + throw new Exception( + pht( + 'RRULE INTERVAL "%s" is invalid: interval must be 1 or more.', + $interval)); + } + + $this->interval = $interval; + return $this; + } + + public function getInterval() { + return $this->interval; + } + + public function setBySecond(array $by_second) { + $this->assertByRange('BYSECOND', $by_second, 0, 60); + $this->bySecond = array_fuse($by_second); + return $this; + } + + public function getBySecond() { + return $this->bySecond; + } + + public function setByMinute(array $by_minute) { + $this->assertByRange('BYMINUTE', $by_minute, 0, 59); + $this->byMinute = array_fuse($by_minute); + return $this; + } + + public function getByMinute() { + return $this->byMinute; + } + + public function setByHour(array $by_hour) { + $this->assertByRange('BYHOUR', $by_hour, 0, 23); + $this->byHour = array_fuse($by_hour); + return $this; + } + + public function getByHour() { + return $this->byHour; + } + + public function setByDay(array $by_day) { + $constants = self::getAllWeekdayConstants(); + $constants = implode('|', $constants); + + $pattern = '/^(?:[+-]?([1-9]\d?))?('.$constants.')\z/'; + foreach ($by_day as $key => $value) { + $matches = null; + if (!preg_match($pattern, $value, $matches)) { + throw new Exception( + pht( + 'RRULE BYDAY value "%s" is invalid: rule part must be in the '. + 'expected form (like "MO", "-3TH", or "+2SU").', + $value)); + } + + // The maximum allowed value is 53, which corresponds to "the 53rd + // Monday every year" or similar when evaluated against a YEARLY rule. + + $maximum = 53; + $magnitude = (int)$matches[1]; + if ($magnitude > $maximum) { + throw new Exception( + pht( + 'RRULE BYDAY value "%s" has an offset with magnitude "%s", but '. + 'the maximum permitted value is "%s".', + $value, + $magnitude, + $maximum)); + } + + // Normalize "+3FR" into "3FR". + $by_day[$key] = ltrim($value, '+'); + } + + $this->byDay = array_fuse($by_day); + return $this; + } + + public function getByDay() { + return $this->byDay; + } + + public function setByMonthDay(array $by_month_day) { + $this->assertByRange('BYMONTHDAY', $by_month_day, -31, 31, false); + $this->byMonthDay = array_fuse($by_month_day); + return $this; + } + + public function getByMonthDay() { + return $this->byMonthDay; + } + + public function setByYearDay($by_year_day) { + $this->assertByRange('BYYEARDAY', $by_year_day, -366, 366, false); + $this->byYearDay = array_fuse($by_year_day); + return $this; + } + + public function getByYearDay() { + return $this->byYearDay; + } + + public function setByMonth(array $by_month) { + $this->assertByRange('BYMONTH', $by_month, 1, 12); + $this->byMonth = array_fuse($by_month); + return $this; + } + + public function getByMonth() { + return $this->byMonth; + } + + public function setByWeekNumber(array $by_week_number) { + $this->assertByRange('BYWEEKNO', $by_week_number, -53, 53, false); + $this->byWeekNumber = array_fuse($by_week_number); + return $this; + } + + public function getByWeekNumber() { + return $this->byWeekNumber; + } + + public function setBySetPosition(array $by_set_position) { + $this->assertByRange('BYSETPOS', $by_set_position, -366, 366, false); + $this->bySetPosition = $by_set_position; + return $this; + } + + public function getBySetPosition() { + return $this->bySetPosition; + } + + public function setWeekStart($week_start) { + // Make sure this is a valid weekday constant. + self::getWeekdayIndex($week_start); + + $this->weekStart = $week_start; + return $this; + } + + public function getWeekStart() { + return $this->weekStart; + } + + public function resetSource() { + $frequency = $this->getFrequency(); + + if ($this->getByMonthDay()) { + switch ($frequency) { + case self::FREQUENCY_WEEKLY: + // RFC5545: "The BYMONTHDAY rule part MUST NOT be specified when the + // FREQ rule part is set to WEEKLY." + throw new Exception( + pht( + 'RRULE specifies BYMONTHDAY with FREQ set to WEEKLY, which '. + 'violates RFC5545.')); + break; + default: + break; + } + + } + + if ($this->getByYearDay()) { + switch ($frequency) { + case self::FREQUENCY_DAILY: + case self::FREQUENCY_WEEKLY: + case self::FREQUENCY_MONTHLY: + // RFC5545: "The BYYEARDAY rule part MUST NOT be specified when the + // FREQ rule part is set to DAILY, WEEKLY, or MONTHLY." + throw new Exception( + pht( + 'RRULE specifies BYYEARDAY with FREQ of DAILY, WEEKLY or '. + 'MONTHLY, which violates RFC5545.')); + default: + break; + } + } + + // TODO + // RFC5545: "The BYDAY rule part MUST NOT be specified with a numeric + // value when the FREQ rule part is not set to MONTHLY or YEARLY." + // RFC5545: "Furthermore, the BYDAY rule part MUST NOT be specified with a + // numeric value with the FREQ rule part set to YEARLY when the BYWEEKNO + // rule part is specified." + + + $date = $this->getStartDateTime(); + + $this->cursorSecond = $date->getSecond(); + $this->cursorMinute = $date->getMinute(); + $this->cursorHour = $date->getHour(); + + $this->cursorDay = $date->getDay(); + $this->cursorMonth = $date->getMonth(); + $this->cursorYear = $date->getYear(); + + $year_map = $this->getYearMap($this->cursorYear, $this->getWeekStart()); + $key = $this->cursorMonth.'M'.$this->cursorDay.'D'; + $this->cursorWeek = $year_map['info'][$key]['week']; + $this->cursorWeekday = $year_map['info'][$key]['weekday']; + + $this->setSeconds = array(); + $this->setMinutes = array(); + $this->setHours = array(); + $this->setDays = array(); + $this->setMonths = array(); + $this->setYears = array(); + + $this->stateSecond = null; + $this->stateMinute = null; + $this->stateHour = null; + $this->stateDay = null; + $this->stateWeek = null; + $this->stateMonth = null; + $this->stateYear = null; + + // If we have a BYSETPOS, we need to generate the entire set before we + // can filter it and return results. Normally, we start generating at + // the start date, but we need to go back one interval to generate + // BYSETPOS events so we can make sure the entire set is generated. + if ($this->getBySetPosition()) { + $interval = $this->getInterval(); + switch ($frequency) { + case self::FREQUENCY_YEARLY: + $this->cursorYear -= $interval; + break; + case self::FREQUENCY_MONTHLY: + $this->cursorMonth -= $interval; + $this->rewindMonth(); + break; + case self::FREQUENCY_WEEKLY: + $this->cursorWeek -= $interval; + $this->rewindWeek(); + break; + case self::FREQUENCY_DAILY: + $this->cursorDay -= $interval; + $this->rewindDay(); + break; + case self::FREQUENCY_HOURLY: + $this->cursorHour -= $interval; + $this->rewindHour(); + break; + case self::FREQUENCY_MINUTELY: + $this->cursorMinute -= $interval; + $this->rewindMinute(); + break; + case self::FREQUENCY_SECONDLY: + default: + throw new Exception( + pht( + 'RRULE specifies BYSETPOS with FREQ "%s", but this is invalid.', + $frequency)); + } + } + + // We can generate events from before the cursor when evaluating rules + // with BYSETPOS or FREQ=WEEKLY. + $this->minimumEpoch = $this->getStartDateTime()->getEpoch(); + + $cursor_state = array( + 'year' => $this->cursorYear, + 'month' => $this->cursorMonth, + 'week' => $this->cursorWeek, + 'day' => $this->cursorDay, + 'hour' => $this->cursorHour, + ); + + $this->cursorDayState = $cursor_state; + $this->cursorWeekState = $cursor_state; + $this->cursorHourState = $cursor_state; + + $by_hour = $this->getByHour(); + $by_minute = $this->getByMinute(); + $by_second = $this->getBySecond(); + + $scale = $this->getFrequencyScale(); + + // We return all-day events if the start date is an all-day event and we + // don't have more granular selectors or a more granular frequency. + $this->isAllDay = $date->getIsAllDay() + && !$by_hour + && !$by_minute + && !$by_second + && ($scale > self::SCALE_HOURLY); + } + + public function getNextEvent($cursor) { + while (true) { + $event = $this->generateNextEvent(); + if (!$event) { + break; + } + + $epoch = $event->getEpoch(); + if ($this->minimumEpoch) { + if ($epoch < $this->minimumEpoch) { + continue; + } + } + + if ($epoch < $cursor) { + continue; + } + + break; + } + + return $event; + } + + private function generateNextEvent() { + if ($this->activeSet) { + return array_pop($this->activeSet); + } + + $this->baseYear = $this->cursorYear; + + $by_setpos = $this->getBySetPosition(); + if ($by_setpos) { + $old_state = $this->getSetPositionState(); + } + + while (!$this->activeSet) { + $this->activeSet = $this->nextSet; + $this->nextSet = array(); + + while (true) { + if ($this->isAllDay) { + $this->nextDay(); + } else { + $this->nextSecond(); + } + + $result = id(new PhutilCalendarAbsoluteDateTime()) + ->setTimezone($this->getStartDateTime()->getTimezone()) + ->setViewerTimezone($this->getViewerTimezone()) + ->setYear($this->stateYear) + ->setMonth($this->stateMonth) + ->setDay($this->stateDay); + + if ($this->isAllDay) { + $result->setIsAllDay(true); + } else { + $result + ->setHour($this->stateHour) + ->setMinute($this->stateMinute) + ->setSecond($this->stateSecond); + } + + // If we don't have BYSETPOS, we're all done. We put this into the + // set and will immediately return it. + if (!$by_setpos) { + $this->activeSet[] = $result; + break; + } + + // Otherwise, check if we've completed a set. The set is complete if + // the state has moved past the span we were examining (for example, + // with a YEARLY event, if the state is now in the next year). + $new_state = $this->getSetPositionState(); + if ($new_state == $old_state) { + $this->activeSet[] = $result; + continue; + } + + $this->activeSet = $this->applySetPos($this->activeSet, $by_setpos); + $this->activeSet = array_reverse($this->activeSet); + $this->nextSet[] = $result; + $old_state = $new_state; + break; + } + } + + return array_pop($this->activeSet); + } + + + protected function nextSecond() { + if ($this->setSeconds) { + $this->stateSecond = array_pop($this->setSeconds); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $is_secondly = ($frequency == self::FREQUENCY_SECONDLY); + $by_second = $this->getBySecond(); + + while (!$this->setSeconds) { + $this->nextMinute(); + + if ($is_secondly || $by_second) { + $seconds = $this->newSecondsSet( + ($is_secondly ? $interval : 1), + $by_second); + } else { + $seconds = array( + $this->cursorSecond, + ); + } + + $this->setSeconds = array_reverse($seconds); + } + + $this->stateSecond = array_pop($this->setSeconds); + } + + protected function nextMinute() { + if ($this->setMinutes) { + $this->stateMinute = array_pop($this->setMinutes); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_minutely = ($frequency === self::FREQUENCY_MINUTELY); + $by_minute = $this->getByMinute(); + + while (!$this->setMinutes) { + $this->nextHour(); + + if ($is_minutely || $by_minute) { + $minutes = $this->newMinutesSet( + ($is_minutely ? $interval : 1), + $by_minute); + } else if ($scale < self::SCALE_MINUTELY) { + $minutes = $this->newMinutesSet( + 1, + array()); + } else { + $minutes = array( + $this->cursorMinute, + ); + } + + $this->setMinutes = array_reverse($minutes); + } + + $this->stateMinute = array_pop($this->setMinutes); + } + + protected function nextHour() { + if ($this->setHours) { + $this->stateHour = array_pop($this->setHours); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_hourly = ($frequency === self::FREQUENCY_HOURLY); + $by_hour = $this->getByHour(); + + while (!$this->setHours) { + $this->nextDay(); + + $is_dynamic = $is_hourly + || $by_hour + || ($scale < self::SCALE_HOURLY); + + if ($is_dynamic) { + $hours = $this->newHoursSet( + ($is_hourly ? $interval : 1), + $by_hour); + } else { + $hours = array( + $this->cursorHour, + ); + } + + $this->setHours = array_reverse($hours); + } + + $this->stateHour = array_pop($this->setHours); + } + + protected function nextDay() { + if ($this->setDays) { + $info = array_pop($this->setDays); + $this->setDayState($info); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_daily = ($frequency === self::FREQUENCY_DAILY); + $is_weekly = ($frequency === self::FREQUENCY_WEEKLY); + + $by_day = $this->getByDay(); + $by_monthday = $this->getByMonthDay(); + $by_yearday = $this->getByYearDay(); + $by_weekno = $this->getByWeekNumber(); + $by_month = $this->getByMonth(); + $week_start = $this->getWeekStart(); + + while (!$this->setDays) { + if ($is_weekly) { + $this->nextWeek(); + } else { + $this->nextMonth(); + } + + // NOTE: We normally handle BYMONTH when iterating months, but it acts + // like a filter if FREQ=WEEKLY. + + $is_dynamic = $is_daily + || $is_weekly + || $by_day + || $by_monthday + || $by_yearday + || $by_weekno + || ($by_month && $is_weekly) + || ($scale < self::SCALE_DAILY); + + if ($is_dynamic) { + $weeks = $this->newDaysSet( + ($is_daily ? $interval : 1), + $by_day, + $by_monthday, + $by_yearday, + $by_weekno, + $by_month, + $week_start); + } else { + // The cursor day may not actually exist in the current month, so + // make sure the day is valid before we generate a set which contains + // it. + $year_map = $this->getYearMap($this->stateYear, $week_start); + if ($this->cursorDay > $year_map['monthDays'][$this->stateMonth]) { + $weeks = array( + array(), + ); + } else { + $key = $this->stateMonth.'M'.$this->cursorDay.'D'; + $weeks = array( + array($year_map['info'][$key]), + ); + } + } + + // Unpack the weeks into days. + $days = array_mergev($weeks); + + $this->setDays = array_reverse($days); + } + + $info = array_pop($this->setDays); + $this->setDayState($info); + } + + private function setDayState(array $info) { + $this->stateDay = $info['monthday']; + $this->stateWeek = $info['week']; + $this->stateMonth = $info['month']; + } + + protected function nextMonth() { + if ($this->setMonths) { + $this->stateMonth = array_pop($this->setMonths); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $is_monthly = ($frequency === self::FREQUENCY_MONTHLY); + + $by_month = $this->getByMonth(); + + // If we have a BYMONTHDAY, we consider that set of days in every month. + // For example, "FREQ=YEARLY;BYMONTHDAY=3" means "the third day of every + // month", so we need to expand the month set if the constraint is present. + $by_monthday = $this->getByMonthDay(); + + // Likewise, we need to generate all months if we have BYYEARDAY or + // BYWEEKNO or BYDAY. + $by_yearday = $this->getByYearDay(); + $by_weekno = $this->getByWeekNumber(); + $by_day = $this->getByDay(); + + while (!$this->setMonths) { + $this->nextYear(); + + $is_dynamic = $is_monthly + || $by_month + || $by_monthday + || $by_yearday + || $by_weekno + || $by_day + || ($scale < self::SCALE_MONTHLY); + + if ($is_dynamic) { + $months = $this->newMonthsSet( + ($is_monthly ? $interval : 1), + $by_month); + } else { + $months = array( + $this->cursorMonth, + ); + } + + $this->setMonths = array_reverse($months); + } + + $this->stateMonth = array_pop($this->setMonths); + } + + protected function nextWeek() { + if ($this->setWeeks) { + $this->stateWeek = array_pop($this->setWeeks); + return; + } + + $frequency = $this->getFrequency(); + $interval = $this->getInterval(); + $scale = $this->getFrequencyScale(); + $by_weekno = $this->getByWeekNumber(); + + while (!$this->setWeeks) { + $this->nextYear(); + + $weeks = $this->newWeeksSet( + $interval, + $by_weekno); + + $this->setWeeks = array_reverse($weeks); + } + + $this->stateWeek = array_pop($this->setWeeks); + } + + protected function nextYear() { + $this->stateYear = $this->cursorYear; + + $frequency = $this->getFrequency(); + $is_yearly = ($frequency === self::FREQUENCY_YEARLY); + + if ($is_yearly) { + $interval = $this->getInterval(); + } else { + $interval = 1; + } + + $this->cursorYear = $this->cursorYear + $interval; + + if ($this->cursorYear > ($this->baseYear + 100)) { + throw new Exception( + pht( + 'RRULE evaluation failed to generate more events in the next 100 '. + 'years. This RRULE is likely invalid or degenerate.')); + } + + } + + private function newSecondsSet($interval, $set) { + // TODO: This doesn't account for leap seconds. In theory, it probably + // should, although this shouldn't impact any real events. + $seconds_in_minute = 60; + + if ($this->cursorSecond >= $seconds_in_minute) { + $this->cursorSecond -= $seconds_in_minute; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorSecond, + $interval, + $set, + $seconds_in_minute); + + $this->cursorSecond = ($cursor - $seconds_in_minute); + + return $result; + } + + private function newMinutesSet($interval, $set) { + // NOTE: This value is legitimately a constant! Amazing! + $minutes_in_hour = 60; + + if ($this->cursorMinute >= $minutes_in_hour) { + $this->cursorMinute -= $minutes_in_hour; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorMinute, + $interval, + $set, + $minutes_in_hour); + + $this->cursorMinute = ($cursor - $minutes_in_hour); + + return $result; + } + + private function newHoursSet($interval, $set) { + // TODO: This doesn't account for hours caused by daylight savings time. + // It probably should, although this seems unlikely to impact any real + // events. + $hours_in_day = 24; + + // If the hour cursor is behind the current time, we need to forward it in + // INTERVAL increments so we end up with the right offset. + list($skip, $this->cursorHourState) = $this->advanceCursorState( + $this->cursorHourState, + self::SCALE_HOURLY, + $interval, + $this->getWeekStart()); + + if ($skip) { + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorHour, + $interval, + $set, + $hours_in_day); + + $this->cursorHour = ($cursor - $hours_in_day); + + return $result; + } + + private function newWeeksSet($interval, $set) { + $week_start = $this->getWeekStart(); + + list($skip, $this->cursorWeekState) = $this->advanceCursorState( + $this->cursorWeekState, + self::SCALE_WEEKLY, + $interval, + $week_start); + + if ($skip) { + return array(); + } + + $year_map = $this->getYearMap($this->stateYear, $week_start); + + $result = array(); + while (true) { + if (!isset($year_map['weekMap'][$this->cursorWeek])) { + break; + } + $result[] = $this->cursorWeek; + $this->cursorWeek += $interval; + } + + $this->cursorWeek -= $year_map['weekCount']; + + return $result; + } + + private function newDaysSet( + $interval_day, + $by_day, + $by_monthday, + $by_yearday, + $by_weekno, + $by_month, + $week_start) { + + $frequency = $this->getFrequency(); + $is_yearly = ($frequency == self::FREQUENCY_YEARLY); + $is_monthly = ($frequency == self::FREQUENCY_MONTHLY); + $is_weekly = ($frequency == self::FREQUENCY_WEEKLY); + + $selection = array(); + if ($is_weekly) { + $year_map = $this->getYearMap($this->stateYear, $week_start); + + if (isset($year_map['weekMap'][$this->stateWeek])) { + foreach ($year_map['weekMap'][$this->stateWeek] as $key) { + $selection[] = $year_map['info'][$key]; + } + } + } else { + // If the day cursor is behind the current year and month, we need to + // forward it in INTERVAL increments so we end up with the right offset + // in the current month. + list($skip, $this->cursorDayState) = $this->advanceCursorState( + $this->cursorDayState, + self::SCALE_DAILY, + $interval_day, + $week_start); + + if (!$skip) { + $year_map = $this->getYearMap($this->stateYear, $week_start); + while (true) { + $month_idx = $this->stateMonth; + $month_days = $year_map['monthDays'][$month_idx]; + if ($this->cursorDay > $month_days) { + // NOTE: The year map is now out of date, but we're about to break + // out of the loop anyway so it doesn't matter. + break; + } + + $day_idx = $this->cursorDay; + + $key = "{$month_idx}M{$day_idx}D"; + $selection[] = $year_map['info'][$key]; + + $this->cursorDay += $interval_day; + } + } + } + + // As a special case, BYDAY applies to relative month offsets if BYMONTH + // is present in a YEARLY rule. + if ($is_yearly) { + if ($this->getByMonth()) { + $is_yearly = false; + $is_monthly = true; + } + } + + // As a special case, BYDAY makes us examine all week days. This doesn't + // check BYMONTHDAY or BYYEARDAY because they are not valid with WEEKLY. + $filter_weekday = true; + if ($is_weekly) { + if ($by_day) { + $filter_weekday = false; + } + } + + $weeks = array(); + foreach ($selection as $key => $info) { + if ($is_weekly) { + if ($filter_weekday) { + if ($info['weekday'] != $this->cursorWeekday) { + continue; + } + } + } else { + if ($info['month'] != $this->stateMonth) { + continue; + } + } + + if ($by_day) { + if (empty($by_day[$info['weekday']])) { + if ($is_yearly) { + if (empty($by_day[$info['weekday.yearly']]) && + empty($by_day[$info['-weekday.yearly']])) { + continue; + } + } else if ($is_monthly) { + if (empty($by_day[$info['weekday.monthly']]) && + empty($by_day[$info['-weekday.monthly']])) { + continue; + } + } else { + continue; + } + } + } + + if ($by_monthday) { + if (empty($by_monthday[$info['monthday']]) && + empty($by_monthday[$info['-monthday']])) { + continue; + } + } + + if ($by_yearday) { + if (empty($by_yearday[$info['yearday']]) && + empty($by_yearday[$info['-yearday']])) { + continue; + } + } + + if ($by_weekno) { + if (empty($by_weekno[$info['week']]) && + empty($by_weekno[$info['-week']])) { + continue; + } + } + + if ($by_month) { + if (empty($by_month[$info['month']])) { + continue; + } + } + + $weeks[$info['week']][] = $info; + } + + return array_values($weeks); + } + + private function newMonthsSet($interval, $set) { + // NOTE: This value is also a real constant! Wow! + $months_in_year = 12; + + if ($this->cursorMonth > $months_in_year) { + $this->cursorMonth -= $months_in_year; + return array(); + } + + list($cursor, $result) = $this->newIteratorSet( + $this->cursorMonth, + $interval, + $set, + $months_in_year + 1); + + $this->cursorMonth = ($cursor - $months_in_year); + + return $result; + } + + public static function getYearMap($year, $week_start) { + static $maps = array(); + + $key = "{$year}/{$week_start}"; + if (isset($maps[$key])) { + return $maps[$key]; + } + + $map = self::newYearMap($year, $week_start); + $maps[$key] = $map; + + return $maps[$key]; + } + + private static function newYearMap($year, $weekday_start) { + $weekday_index = self::getWeekdayIndex($weekday_start); + + $is_leap = (($year % 4 === 0) && ($year % 100 !== 0)) || + ($year % 400 === 0); + + // There may be some clever way to figure out which day of the week a given + // year starts on and avoid the cost of a DateTime construction, but I + // wasn't able to turn it up and we only need to do this once per year. + $datetime = new DateTime("{$year}-01-01", new DateTimeZone('UTC')); + $weekday = (int)$datetime->format('w'); + + if ($is_leap) { + $max_day = 366; + } else { + $max_day = 365; + } + + $month_days = array( + 1 => 31, + 2 => $is_leap ? 29 : 28, + 3 => 31, + 4 => 30, + 5 => 31, + 6 => 30, + 7 => 31, + 8 => 31, + 9 => 30, + 10 => 31, + 11 => 30, + 12 => 31, + ); + + // Per the spec, the first week of the year must contain at least four + // days. If the week starts on a Monday but the year starts on a Saturday, + // the first couple of days don't count as a week. In this case, the first + // week will begin on January 3. + $first_week_size = 0; + $first_weekday = $weekday; + for ($year_day = 1; $year_day <= $max_day; $year_day++) { + $first_weekday = ($first_weekday + 1) % 7; + $first_week_size++; + if ($first_weekday === $weekday_index) { + break; + } + } + + if ($first_week_size >= 4) { + $week_number = 1; + } else { + $week_number = 0; + } + + $info_map = array(); + + $weekday_map = self::getWeekdayIndexMap(); + $weekday_map = array_flip($weekday_map); + + $yearly_counts = array(); + $monthly_counts = array(); + + $month_number = 1; + $month_day = 1; + for ($year_day = 1; $year_day <= $max_day; $year_day++) { + $key = "{$month_number}M{$month_day}D"; + + $short_day = $weekday_map[$weekday]; + if (empty($yearly_counts[$short_day])) { + $yearly_counts[$short_day] = 0; + } + $yearly_counts[$short_day]++; + + if (empty($monthly_counts[$month_number][$short_day])) { + $monthly_counts[$month_number][$short_day] = 0; + } + $monthly_counts[$month_number][$short_day]++; + + $info = array( + 'year' => $year, + 'key' => $key, + 'month' => $month_number, + 'monthday' => $month_day, + '-monthday' => -$month_days[$month_number] + $month_day - 1, + 'yearday' => $year_day, + '-yearday' => -$max_day + $year_day - 1, + 'week' => $week_number, + 'weekday' => $short_day, + 'weekday.yearly' => $yearly_counts[$short_day], + 'weekday.monthly' => $monthly_counts[$month_number][$short_day], + ); + + $info_map[$key] = $info; + + $weekday = ($weekday + 1) % 7; + if ($weekday === $weekday_index) { + $week_number++; + } + + $month_day = ($month_day + 1); + if ($month_day > $month_days[$month_number]) { + $month_day = 1; + $month_number++; + } + } + + // Check how long the final week is. If it doesn't have four days, this + // is really the first week of the next year. + $final_week = array(); + foreach ($info_map as $key => $info) { + if ($info['week'] == $week_number) { + $final_week[] = $key; + } + } + + if (count($final_week) < 4) { + $week_number = $week_number - 1; + $next_year = self::getYearMap($year + 1, $weekday_start); + $next_year_weeks = $next_year['weekCount']; + } else { + $next_year_weeks = null; + } + + if ($first_week_size < 4) { + $last_year = self::getYearMap($year - 1, $weekday_start); + $last_year_weeks = $last_year['weekCount']; + } else { + $last_year_weeks = null; + } + + // Now that we know how many weeks the year has, we can compute the + // negative offsets. + foreach ($info_map as $key => $info) { + $week = $info['week']; + + if ($week === 0) { + // If this day is part of the first partial week of the year, give + // it the week number of the last week of the prior year instead. + $info['week'] = $last_year_weeks; + $info['-week'] = -1; + } else if ($week > $week_number) { + // If this day is part of the last partial week of the year, give + // it week numbers from the next year. + $info['week'] = 1; + $info['-week'] = -$next_year_weeks; + } else { + $info['-week'] = -$week_number + $week - 1; + } + + // Do all the arithmetic to figure out if this is the -19th Thursday + // in the year and such. + $month_number = $info['month']; + $short_day = $info['weekday']; + $monthly_count = $monthly_counts[$month_number][$short_day]; + $monthly_index = $info['weekday.monthly']; + $info['-weekday.monthly'] = -$monthly_count + $monthly_index - 1; + $info['-weekday.monthly'] .= $short_day; + $info['weekday.monthly'] .= $short_day; + + $yearly_count = $yearly_counts[$short_day]; + $yearly_index = $info['weekday.yearly']; + $info['-weekday.yearly'] = -$yearly_count + $yearly_index - 1; + $info['-weekday.yearly'] .= $short_day; + $info['weekday.yearly'] .= $short_day; + + $info_map[$key] = $info; + } + + $week_map = array(); + foreach ($info_map as $key => $info) { + $week_map[$info['week']][] = $key; + } + + return array( + 'info' => $info_map, + 'weekCount' => $week_number, + 'dayCount' => $max_day, + 'monthDays' => $month_days, + 'weekMap' => $week_map, + ); + } + + private function newIteratorSet($cursor, $interval, $set, $limit) { + if ($interval < 1) { + throw new Exception( + pht( + 'Invalid iteration interval ("%d"), must be at least 1.', + $interval)); + } + + $result = array(); + $seen = array(); + + $ii = $cursor; + while (true) { + if (!$set || isset($set[$ii])) { + $result[] = $ii; + } + + $ii = ($ii + $interval); + + if ($ii >= $limit) { + break; + } + } + + sort($result); + $result = array_values($result); + + return array($ii, $result); + } + + private function applySetPos(array $values, array $setpos) { + $select = array(); + + $count = count($values); + foreach ($setpos as $pos) { + if ($pos > 0 && $pos <= $count) { + $select[] = ($pos - 1); + } else if ($pos < 0 && $pos >= -$count) { + $select[] = ($count + $pos); + } + } + + sort($select); + $select = array_unique($select); + + return array_select_keys($values, $select); + } + + private function assertByRange( + $source, + array $values, + $min, + $max, + $allow_zero = true) { + + foreach ($values as $value) { + if (!is_int($value)) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: values must be '. + 'integers.', + $value, + $source)); + } + + if ($value < $min || $value > $max) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: it must be '. + 'between %s and %s.', + $value, + $source, + $min, + $max)); + } + + if (!$value && !$allow_zero) { + throw new Exception( + pht( + 'Value "%s" in RRULE "%s" parameter is invalid: it must not '. + 'be zero.', + $value, + $source)); + } + } + } + + private function getSetPositionState() { + $scale = $this->getFrequencyScale(); + + $parts = array(); + $parts[] = $this->stateYear; + + if ($scale == self::SCALE_WEEKLY) { + $parts[] = $this->stateWeek; + } else { + if ($scale < self::SCALE_YEARLY) { + $parts[] = $this->stateMonth; + } + if ($scale < self::SCALE_MONTHLY) { + $parts[] = $this->stateDay; + } + if ($scale < self::SCALE_DAILY) { + $parts[] = $this->stateHour; + } + if ($scale < self::SCALE_HOURLY) { + $parts[] = $this->stateMinute; + } + } + + return implode('/', $parts); + } + + private function rewindMonth() { + while ($this->cursorMonth < 1) { + $this->cursorYear--; + $this->cursorMonth += 12; + } + } + + private function rewindWeek() { + $week_start = $this->getWeekStart(); + while ($this->cursorWeek < 1) { + $this->cursorYear--; + $year_map = $this->getYearMap($this->cursorYear, $week_start); + $this->cursorWeek += $year_map['weekCount']; + } + } + + private function rewindDay() { + $week_start = $this->getWeekStart(); + while ($this->cursorDay < 1) { + $year_map = $this->getYearMap($this->cursorYear, $week_start); + $this->cursorDay += $year_map['monthDays'][$this->cursorMonth]; + $this->cursorMonth--; + $this->rewindMonth(); + } + } + + private function rewindHour() { + while ($this->cursorHour < 0) { + $this->cursorHour += 24; + $this->cursorDay--; + $this->rewindDay(); + } + } + + private function rewindMinute() { + while ($this->cursorMinute < 0) { + $this->cursorMinute += 60; + $this->cursorHour--; + $this->rewindHour(); + } + } + + private function advanceCursorState( + array $cursor, + $scale, + $interval, + $week_start) { + + $state = array( + 'year' => $this->stateYear, + 'month' => $this->stateMonth, + 'week' => $this->stateWeek, + 'day' => $this->stateDay, + 'hour' => $this->stateHour, + ); + + // In the common case when the interval is 1, we'll visit every possible + // value so we don't need to do any math and can just jump to the first + // hour, day, etc. + if ($interval == 1) { + if ($this->isCursorBehind($cursor, $state, $scale)) { + switch ($scale) { + case self::SCALE_DAILY: + $this->cursorDay = 1; + break; + case self::SCALE_HOURLY: + $this->cursorHour = 0; + break; + case self::SCALE_WEEKLY: + $this->cursorWeek = 1; + break; + } + } + + return array(false, $state); + } + + $year_map = $this->getYearMap($cursor['year'], $week_start); + while ($this->isCursorBehind($cursor, $state, $scale)) { + switch ($scale) { + case self::SCALE_DAILY: + $cursor['day'] += $interval; + break; + case self::SCALE_HOURLY: + $cursor['hour'] += $interval; + break; + case self::SCALE_WEEKLY: + $cursor['week'] += $interval; + break; + } + + if ($scale <= self::SCALE_HOURLY) { + while ($cursor['hour'] >= 24) { + $cursor['hour'] -= 24; + $cursor['day']++; + } + } + + if ($scale == self::SCALE_WEEKLY) { + while ($cursor['week'] > $year_map['weekCount']) { + $cursor['week'] -= $year_map['weekCount']; + $cursor['year']++; + $year_map = $this->getYearMap($cursor['year'], $week_start); + } + } + + if ($scale <= self::SCALE_DAILY) { + while ($cursor['day'] > $year_map['monthDays'][$cursor['month']]) { + $cursor['day'] -= $year_map['monthDays'][$cursor['month']]; + $cursor['month']++; + if ($cursor['month'] > 12) { + $cursor['month'] -= 12; + $cursor['year']++; + $year_map = $this->getYearMap($cursor['year'], $week_start); + } + } + } + } + + switch ($scale) { + case self::SCALE_DAILY: + $this->cursorDay = $cursor['day']; + break; + case self::SCALE_HOURLY: + $this->cursorHour = $cursor['hour']; + break; + case self::SCALE_WEEKLY: + $this->cursorWeek = $cursor['week']; + break; + } + + $skip = $this->isCursorBehind($state, $cursor, $scale); + + return array($skip, $cursor); + } + + private function isCursorBehind(array $cursor, array $state, $scale) { + if ($cursor['year'] < $state['year']) { + return true; + } else if ($cursor['year'] > $state['year']) { + return false; + } + + if ($scale == self::SCALE_WEEKLY) { + return false; + } + + if ($cursor['month'] < $state['month']) { + return true; + } else if ($cursor['month'] > $state['month']) { + return false; + } + + if ($scale >= self::SCALE_DAILY) { + return false; + } + + if ($cursor['day'] < $state['day']) { + return true; + } else if ($cursor['day'] > $state['day']) { + return false; + } + + if ($scale >= self::SCALE_HOURLY) { + return false; + } + + if ($cursor['hour'] < $state['hour']) { + return true; + } else if ($cursor['hour'] > $state['hour']) { + return false; + } + + return false; + } + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php new file mode 100644 index 0000000000..6ebcd4a58a --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSet.php @@ -0,0 +1,162 @@ +sources[] = $source; + return $this; + } + + public function setViewerTimezone($viewer_timezone) { + $this->viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function getEventsBetween( + PhutilCalendarDateTime $start = null, + PhutilCalendarDateTime $end = null, + $limit = null) { + + if ($end === null && $limit === null) { + throw new Exception( + pht( + 'Recurring event range queries must have an end date, a limit, or '. + 'both.')); + } + + $timezone = $this->getViewerTimezone(); + + $sources = array(); + foreach ($this->sources as $source) { + $source = clone $source; + $source->setViewerTimezone($timezone); + $source->resetSource(); + + $sources[] = array( + 'source' => $source, + 'state' => null, + 'epoch' => null, + ); + } + + if ($start) { + $start = clone $start; + $start->setViewerTimezone($timezone); + $min_epoch = $start->getEpoch(); + } else { + $min_epoch = 0; + } + + if ($end) { + $end = clone $end; + $end->setViewerTimezone($timezone); + $end_epoch = $end->getEpoch(); + } else { + $end_epoch = null; + } + + $results = array(); + $index = 0; + $cursor = 0; + while (true) { + // Get the next event for each source which we don't have a future + // event for. + foreach ($sources as $key => $source) { + $state = $source['state']; + $epoch = $source['epoch']; + + if ($state !== null && $epoch >= $cursor) { + // We have an event for this source, and it's a future event, so + // we don't need to do anything. + continue; + } + + $next = $source['source']->getNextEvent($cursor); + if ($next === null) { + // This source doesn't have any more events, so we're all done. + unset($sources[$key]); + continue; + } + + $next_epoch = $next->getEpoch(); + + if ($end_epoch !== null && $next_epoch > $end_epoch) { + // We have an end time and the next event from this source is + // past that end, so we know there are no more relevant events + // coming from this source. + unset($sources[$key]); + continue; + } + + $sources[$key]['state'] = $next; + $sources[$key]['epoch'] = $next_epoch; + } + + if (!$sources) { + // We've run out of sources which can produce valid events in the + // window, so we're all done. + break; + } + + // Find the minimum event time across all sources. + $next_epoch = null; + foreach ($sources as $source) { + if ($next_epoch === null) { + $next_epoch = $source['epoch']; + } else { + $next_epoch = min($next_epoch, $source['epoch']); + } + } + + $is_exception = false; + $next_source = null; + foreach ($sources as $source) { + if ($source['epoch'] == $next_epoch) { + if ($source['source']->getIsExceptionSource()) { + $is_exception = true; + } else { + $next_source = $source; + } + } + } + + // If this is an exception, it means the event does NOT occur. We + // skip it and move on. If it's not an exception, it does occur, so + // we record it. + if (!$is_exception) { + + // Only actually include this event in the results if it starts after + // any specified start time. We increment the index regardless, so we + // return results with proper offsets. + if ($next_source['epoch'] >= $min_epoch) { + $results[$index] = $next_source['state']; + } + $index++; + + if ($limit !== null && (count($results) >= $limit)) { + break; + } + } + + $cursor = $next_epoch + 1; + + // If we have an end of the window and we've reached it, we're done. + if ($end_epoch) { + if ($cursor > $end_epoch) { + break; + } + } + } + + return $results; + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php new file mode 100644 index 0000000000..214fd82933 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRecurrenceSource.php @@ -0,0 +1,34 @@ +isExceptionSource = $is_exception_source; + return $this; + } + + public function getIsExceptionSource() { + return $this->isExceptionSource; + } + + public function setViewerTimezone($viewer_timezone) { + $this->viewerTimezone = $viewer_timezone; + return $this; + } + + public function getViewerTimezone() { + return $this->viewerTimezone; + } + + public function resetSource() { + return; + } + + abstract public function getNextEvent($cursor); + + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php b/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php new file mode 100644 index 0000000000..15c5dc231c --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRelativeDateTime.php @@ -0,0 +1,74 @@ +setProxy($origin); + } + + public function getOrigin() { + return $this->getProxy(); + } + + public function setDuration(PhutilCalendarDuration $duration) { + $this->duration = $duration; + return $this; + } + + public function getDuration() { + return $this->duration; + } + + public function newPHPDateTime() { + $datetime = parent::newPHPDateTime(); + $duration = $this->getDuration(); + + if ($duration->getIsNegative()) { + $sign = '-'; + } else { + $sign = '+'; + } + + $map = array( + 'weeks' => $duration->getWeeks(), + 'days' => $duration->getDays(), + 'hours' => $duration->getHours(), + 'minutes' => $duration->getMinutes(), + 'seconds' => $duration->getSeconds(), + ); + + foreach ($map as $unit => $value) { + if (!$value) { + continue; + } + $datetime->modify("{$sign}{$value} {$unit}"); + } + + return $datetime; + } + + public function newAbsoluteDateTime() { + $clone = clone $this; + + if ($clone->getTimezone()) { + $clone->setViewerTimezone(null); + } + + $datetime = $clone->newPHPDateTime(); + + return id(new PhutilCalendarAbsoluteDateTime()) + ->setYear((int)$datetime->format('Y')) + ->setMonth((int)$datetime->format('m')) + ->setDay((int)$datetime->format('d')) + ->setHour((int)$datetime->format('H')) + ->setMinute((int)$datetime->format('i')) + ->setSecond((int)$datetime->format('s')) + ->setIsAllDay($clone->getIsAllDay()) + ->setTimezone($clone->getTimezone()) + ->setViewerTimezone($this->getViewerTimezone()); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarRootNode.php b/src/applications/calendar/parser/data/PhutilCalendarRootNode.php new file mode 100644 index 0000000000..de22587f0b --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarRootNode.php @@ -0,0 +1,12 @@ +getChildrenOfType(PhutilCalendarDocumentNode::NODETYPE); + } + +} diff --git a/src/applications/calendar/parser/data/PhutilCalendarUserNode.php b/src/applications/calendar/parser/data/PhutilCalendarUserNode.php new file mode 100644 index 0000000000..ea81e0d9d8 --- /dev/null +++ b/src/applications/calendar/parser/data/PhutilCalendarUserNode.php @@ -0,0 +1,40 @@ +name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setURI($uri) { + $this->uri = $uri; + return $this; + } + + public function getURI() { + return $this->uri; + } + + public function setStatus($status) { + $this->status = $status; + return $this; + } + + public function getStatus() { + return $this->status; + } + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php new file mode 100644 index 0000000000..d80aef92fb --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarDateTimeTestCase.php @@ -0,0 +1,49 @@ +setTimezone('America/Los_Angeles') + ->setViewerTimezone('America/Chicago') + ->setIsAllDay(true); + + $this->assertEqual( + '20161128', + $start->getISO8601()); + + $end = $start + ->newAbsoluteDateTime() + ->setHour(0) + ->setMinute(0) + ->setSecond(0) + ->newRelativeDateTime('P1D') + ->newAbsoluteDateTime(); + + $this->assertEqual( + '20161129', + $end->getISO8601()); + + // This is a date which explicitly has no specified timezone. + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601('20161128', null) + ->setViewerTimezone('UTC'); + + $this->assertEqual( + '20161128', + $start->getISO8601()); + + $end = $start + ->newAbsoluteDateTime() + ->setHour(0) + ->setMinute(0) + ->setSecond(0) + ->newRelativeDateTime('P1D') + ->newAbsoluteDateTime(); + + $this->assertEqual( + '20161129', + $end->getISO8601()); + } + + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php new file mode 100644 index 0000000000..228a921b87 --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceRuleTestCase.php @@ -0,0 +1,1750 @@ +setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_DAILY); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Simple daily event.')); + + + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_HOURLY) + ->setByHour(array(12, 13)); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 5); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T130000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T130000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Hourly event with BYHOUR.')); + + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_YEARLY); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 2); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20170101T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Yearly event.')); + + + // This is an efficiency test for bizarre rules: it defines a secondly + // event which only occurs one a year, and generates 3 instances of it. + // This implementation should be fast enough that this test doesn't take + // a significant amount of time. + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_SECONDLY) + ->setByMonth(array(1)) + ->setByMonthDay(array(1)) + ->setByHour(array(12)) + ->setByMinute(array(0)) + ->setBySecond(array(0)); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20170101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20180101T120000Z'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Secondly event with many constraints.')); + } + + public function testYearlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19980902', + '19990902', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19990902', + '20010902', + ); + + $tests[] = array( + 'DTSTART' => '20000229', + ); + $expect[] = array( + '20000229', + '20040229', + '20080229', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980102', + '19980302', + '19990102', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19970907', + '19970914', + '19970921', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19971225', + '19980106', + '19981231', + ); + + // Same test as above, just making sure the optional "+" syntax works. + $tests[] = array( + 'BYDAY' => array('+1TU', '-1TH'), + ); + $expect[] = array( + '19971225', + '19980106', + '19981231', + ); + + $tests[] = array( + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19971211', + '19980120', + '19981217', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19980106', + '19980129', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19980115', + '19980120', + '19980312', + ); + + $tests[] = array( + 'BYYEARDAY' => array(1, 100, 200, 365), + 'COUNT' => 4, + ); + $expect[] = array( + '19971231', + '19980101', + '19980410', + '19980719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(-365, -266, -166, -1), + 'COUNT' => 4, + ); + $expect[] = array( + '19971231', + '19980101', + '19980410', + '19980719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(1, 100, 200, 365), + 'BYMONTH' => array(4, 7), + 'COUNT' => 4, + ); + $expect[] = array( + '19980410', + '19980719', + '19990410', + '19990719', + ); + + $tests[] = array( + 'BYYEARDAY' => array(-365, -266, -166, -1), + 'BYMONTH' => array(4, 7), + 'COUNT' => 4, + ); + $expect[] = array( + '19980410', + '19980719', + '19990410', + '19990719', + ); + + $tests[] = array( + 'BYWEEKNO' => array(20), + ); + $expect[] = array( + '19980511', + '19980512', + '19980513', + ); + + $tests[] = array( + 'BYWEEKNO' => array(1), + 'BYDAY' => array('MO'), + ); + $expect[] = array( + '19971229', + '19990104', + '20000103', + ); + + $tests[] = array( + 'BYWEEKNO' => array(52), + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19971228', + '19981227', + '20000102', + ); + + $tests[] = array( + 'BYWEEKNO' => array(-1), + 'BYDAY' => array('SU'), + ); + $expect[] = array( + '19971228', + '19990103', + '20000102', + ); + + $tests[] = array( + 'BYWEEKNO' => array(53), + 'BYDAY' => array('MO'), + ); + $expect[] = array( + '19981228', + '20041227', + '20091228', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19980902T060000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 30), + ); + $expect[] = array( + '19970902T001500Z', + '19970902T003000Z', + '19980902T001500Z', + ); + + $tests[] = array( + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T000010Z', + '19970902T000020Z', + '19980902T000010Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 30), + ); + $expect[] = array( + '19970902T061500Z', + '19970902T063000Z', + '19970902T181500Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T060010Z', + '19970902T060020Z', + '19970902T180010Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 30), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T001510Z', + '19970902T001520Z', + '19970902T003010Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 30), + 'BYSECOND' => array(10, 20), + ); + $expect[] = array( + '19970902T061510Z', + '19970902T061520Z', + '19970902T063010Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(15), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19971115T180000Z', + '19980215T060000Z', + '19981115T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testMonthlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19971002', + '19971102', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19971102', + '19980102', + ); + + $tests[] = array( + 'INTERVAL' => 18, + ); + $expect[] = array( + '19970902', + '19990302', + '20000902', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980102', + '19980302', + '19990102', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(5, 7), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYDAY' => array('3MO'), + ); + $expect[] = array( + '19970915', + '19971020', + '19971117', + ); + + $tests[] = array( + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19970902', + '19970925', + '19971007', + ); + + $tests[] = array( + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19970911', + '19970916', + '19971016', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('1TU', '-1TH'), + ); + $expect[] = array( + '19980106', + '19980129', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('3TU', '-3TH'), + ); + $expect[] = array( + '19980115', + '19980120', + '19980312', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR'), + 'BYSETPOS' => array(-1), + ); + $expect[] = array( + '19970930', + '19971031', + '19971128', + ); + + $tests[] = array( + 'BYDAY' => array('1MO', '1TU', '1WE', '1TH', '1FR', '-1FR'), + 'BYMONTHDAY' => array(1, -1, -2), + ); + $expect[] = array( + '19971001', + '19971031', + '19971201', + ); + + $tests[] = array( + 'BYDAY' => array('1MO', '1TU', '1WE', '1TH', 'FR'), + 'BYMONTHDAY' => array(1, -1, -2), + ); + $expect[] = array( + '19971001', + '19971031', + '19971201', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19971002T060000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(6, 18), + ); + $expect[] = array( + '19970902T000600Z', + '19970902T001800Z', + '19971002T000600Z', + ); + + $tests[] = array( + 'BYSECOND' => array(6, 18), + ); + $expect[] = array( + '19970902T000006Z', + '19970902T000018Z', + '19971002T000006Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970913T180000Z', + '19970917T060000Z', + '19971013T180000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, 3, -3), + ); + $expect[] = array( + '19970913T180000Z', + '19970917T060000Z', + '19971013T180000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(13, 17), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(4, -1), + ); + $expect[] = array( + '19970917T180000Z', + '19971017T180000Z', + '19971117T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testWeeklyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19970909', + '19970916', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19970916', + '19970930', + ); + + $tests[] = array( + 'INTERVAL' => 20, + ); + $expect[] = array( + '19970902', + '19980120', + '19980609', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980106', + '19980113', + '19980120', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T060000Z', + '19970902T180000Z', + '19970909T060000Z', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + 'BYHOUR' => array(6, 18), + 'BYSETPOS' => array(3, -3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T180000Z', + '19970904T060000Z', + '19970909T180000Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testDailyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902', + '19970903', + '19970904', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902', + '19970904', + '19970906', + ); + + $tests[] = array( + 'INTERVAL' => 92, + ); + $expect[] = array( + '19970902', + '19971203', + '19980305', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101', + '19980102', + '19980103', + ); + + // This is testing that INTERVAL is respected in the presence of a BYMONTH + // filter which skips some months. + $tests[] = array( + 'BYMONTH' => array(12), + 'INTERVAL' => 17, + ); + $expect[] = array( + '19971213', + '19971230', + '19981205', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903', + '19971001', + '19971003', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105', + '19980107', + '19980305', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902', + '19970904', + '19970909', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980106', + '19980108', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980203', + '19980303', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101', + '19980303', + '20010301', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(15, 45), + 'BYSETPOS' => array(3, -3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T181500Z', + '19970903T064500Z', + '19970903T181500Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'DAILY', + 'COUNT' => 3, + 'DTSTART' => '19970902', + ), + $tests, + $expect); + } + + public function testHourlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902T090000Z', + '19970902T100000Z', + '19970902T110000Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T110000Z', + '19970902T130000Z', + ); + + $tests[] = array( + 'INTERVAL' => 769, + ); + $expect[] = array( + '19970902T090000Z', + '19971004T100000Z', + '19971105T110000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + ); + $expect[] = array( + '19970903T000000Z', + '19970903T010000Z', + '19970903T020000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYMONTHDAY' => array(5, 7), + ); + $expect[] = array( + '19980105T000000Z', + '19980105T010000Z', + '19980105T020000Z', + ); + + $tests[] = array( + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19970902T090000Z', + '19970902T100000Z', + '19970902T110000Z', + ); + + $tests[] = array( + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'BYMONTHDAY' => array(1, 3), + 'BYMONTH' => array(1, 3), + 'BYDAY' => array('TU', 'TH'), + ); + $expect[] = array( + '19980101T000000Z', + '19980101T010000Z', + '19980101T020000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYYEARDAY' => array(1, 100, 200, 365), + ); + $expect[] = array( + '19971231T000000Z', + '19971231T010000Z', + '19971231T020000Z', + '19971231T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYYEARDAY' => array(-365, -266, -166, -1), + ); + $expect[] = array( + '19971231T000000Z', + '19971231T010000Z', + '19971231T020000Z', + '19971231T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYMONTH' => array(4, 7), + 'BYYEARDAY' => array(1, 100, 200, 365), + ); + $expect[] = array( + '19980410T000000Z', + '19980410T010000Z', + '19980410T020000Z', + '19980410T030000Z', + ); + + $tests[] = array( + 'COUNT' => 4, + 'BYMONTH' => array(4, 7), + 'BYYEARDAY' => array(-365, -266, -166, -1), + ); + $expect[] = array( + '19980410T000000Z', + '19980410T010000Z', + '19980410T020000Z', + '19980410T030000Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + ); + $expect[] = array( + '19970902T180000Z', + '19970903T060000Z', + '19970903T180000Z', + ); + + $tests[] = array( + 'BYMINUTE' => array(15, 45), + 'BYSECOND' => array(15, 45), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970902T091545Z', + '19970902T094515Z', + '19970902T101545Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'HOURLY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testMinutelyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array( + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090100Z', + '19970902T090200Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090200Z', + '19970902T090400Z', + ); + + $tests[] = array( + 'BYHOUR' => array(6, 18), + 'BYMINUTE' => array(6, 18), + 'BYSECOND' => array(6, 18), + ); + $expect[] = array( + '19970902T180606Z', + '19970902T180618Z', + '19970902T181806Z', + ); + + $tests[] = array( + 'BYSECOND' => array(15, 30, 45), + 'BYSETPOS' => array(3, -3), + ); + $expect[] = array( + '19970902T090015Z', + '19970902T090045Z', + '19970902T090115Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'MINUTELY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testSecondlyRecurrenceRules() { + $tests = array(); + $expect = array(); + + $tests[] = array(); + $expect[] = array( + '19970902T090000Z', + '19970902T090001Z', + '19970902T090002Z', + ); + + $tests[] = array( + 'INTERVAL' => 2, + ); + $expect[] = array( + '19970902T090000Z', + '19970902T090002Z', + '19970902T090004Z', + ); + + $tests[] = array( + 'INTERVAL' => 90061, + ); + $expect[] = array( + '19970902T090000Z', + '19970903T100101Z', + '19970904T110202Z', + ); + + $tests[] = array( + 'BYSECOND' => array(0), + 'BYMINUTE' => array(1), + 'DTSTART' => '20100322T120100Z', + ); + $expect[] = array( + '20100322T120100Z', + '20100322T130100Z', + '20100322T140100Z', + ); + + $this->assertRules( + array( + 'FREQ' => 'SECONDLY', + 'COUNT' => 3, + 'DTSTART' => '19970902T090000Z', + ), + $tests, + $expect); + } + + public function testRFC5545RecurrenceRules() { + // These tests are derived from the examples in RFC5545. + $tests = array(); + $expect = array(); + + $tests[] = array( + 'FREQ' => 'DAILY', + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970903T090000Z', + '19970904T090000Z', + '19970905T090000Z', + '19970906T090000Z', + '19970907T090000Z', + '19970908T090000Z', + '19970909T090000Z', + '19970910T090000Z', + '19970911T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'DAILY', + 'INTERVAL' => 2, + 'DTSTART' => '19970902T090000Z', + 'COUNT' => 5, + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970906T090000Z', + '19970908T090000Z', + '19970910T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYMONTH' => array(1), + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR', 'SA', 'SU'), + 'DTSTART' => '19970902T090000Z', + 'COUNT' => 3, + ); + $expect[] = array( + '19980101T090000Z', + '19980102T090000Z', + '19980103T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 3, + 'BYDAY' => array('1FR'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970905T090000Z', + '19971003T090000Z', + '19971107T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'INTERVAL' => 2, + 'COUNT' => 5, + 'BYDAY' => array('1SU', '-1SU'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970907T090000Z', + '19970928T090000Z', + '19971102T090000Z', + '19971130T090000Z', + '19980104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'BYDAY' => array('-2MO'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970922T090000Z', + '19971020T090000Z', + '19971117T090000Z', + '19971222T090000Z', + '19980119T090000Z', + '19980216T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'BYMONTHDAY' => array(-3), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970928T090000Z', + '19971029T090000Z', + '19971128T090000Z', + '19971229T090000Z', + '19980129T090000Z', + '19980226T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 5, + 'BYMONTHDAY' => array(2, 15), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970915T090000Z', + '19971002T090000Z', + '19971015T090000Z', + '19971102T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 5, + 'BYMONTHDAY' => array(-1, 1), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970930T090000Z', + '19971001T090000Z', + '19971031T090000Z', + '19971101T090000Z', + '19971130T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 7, + 'INTERVAL' => 18, + 'BYMONTHDAY' => array(10, 11, 12, 13, 14, 15), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970910T090000Z', + '19970911T090000Z', + '19970912T090000Z', + '19970913T090000Z', + '19970914T090000Z', + '19970915T090000Z', + '19990310T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'COUNT' => 6, + 'INTERVAL' => 2, + 'BYDAY' => array('TU'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970909T090000Z', + '19970916T090000Z', + '19970923T090000Z', + '19970930T090000Z', + '19971104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 10, + 'BYMONTH' => array(6, 7), + 'DTSTART' => '19970610T090000Z', + ); + $expect[] = array( + '19970610T090000Z', + '19970710T090000Z', + '19980610T090000Z', + '19980710T090000Z', + '19990610T090000Z', + '19990710T090000Z', + '20000610T090000Z', + '20000710T090000Z', + '20010610T090000Z', + '20010710T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 4, + 'INTERVAL' => 3, + 'BYYEARDAY' => array(1, 100, 200), + 'DTSTART' => '19970101T090000Z', + ); + $expect[] = array( + '19970101T090000Z', + '19970410T090000Z', + '19970719T090000Z', + '20000101T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'BYDAY' => array('20MO'), + 'DTSTART' => '19970519T090000Z', + ); + $expect[] = array( + '19970519T090000Z', + '19980518T090000Z', + '19990517T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'COUNT' => 3, + 'BYWEEKNO' => array(20), + 'BYDAY' => array('MO'), + 'DTSTART' => '19970512T090000Z', + ); + $expect[] = array( + '19970512T090000Z', + '19980511T090000Z', + '19990517T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('TH'), + 'BYMONTH' => array(3), + 'DTSTART' => '19970313T090000Z', + 'COUNT' => 5, + ); + $expect[] = array( + '19970313T090000Z', + '19970320T090000Z', + '19970327T090000Z', + '19980305T090000Z', + '19980312T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('TH'), + 'BYMONTH' => array(6, 7, 8), + 'DTSTART' => '19970101T090000Z', + 'COUNT' => 15, + ); + $expect[] = array( + '19970605T090000Z', + '19970612T090000Z', + '19970619T090000Z', + '19970626T090000Z', + '19970703T090000Z', + '19970710T090000Z', + '19970717T090000Z', + '19970724T090000Z', + '19970731T090000Z', + '19970807T090000Z', + '19970814T090000Z', + '19970821T090000Z', + '19970828T090000Z', + '19980604T090000Z', + '19980611T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'BYDAY' => array('FR'), + 'BYMONTHDAY' => array(13), + 'COUNT' => 4, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19980213T090000Z', + '19980313T090000Z', + '19981113T090000Z', + '19990813T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('SA'), + 'BYMONTHDAY' => array(7, 8, 9, 10, 11, 12, 13), + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970913T090000Z', + '19971011T090000Z', + '19971108T090000Z', + '19971213T090000Z', + '19980110T090000Z', + '19980207T090000Z', + '19980307T090000Z', + '19980411T090000Z', + '19980509T090000Z', + '19980613T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'YEARLY', + 'INTERVAL' => 4, + 'BYMONTH' => array(11), + 'BYDAY' => array('TU'), + 'BYMONTHDAY' => array(2, 3, 4, 5, 6, 7, 8), + 'COUNT' => 6, + 'DTSTART' => '19961105T090000Z', + ); + $expect[] = array( + '19961105T090000Z', + '20001107T090000Z', + '20041102T090000Z', + '20081104T090000Z', + '20121106T090000Z', + '20161108T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('TU', 'WE', 'TH'), + 'BYSETPOS' => array(3), + 'COUNT' => 3, + 'DTSTART' => '19970904T090000Z', + ); + $expect[] = array( + '19970904T090000Z', + '19971007T090000Z', + '19971106T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'MONTHLY', + 'BYDAY' => array('MO', 'TU', 'WE', 'TH', 'FR'), + 'BYSETPOS' => array(-2), + 'COUNT' => 3, + 'DTSTART' => '19970929T090000Z', + ); + $expect[] = array( + '19970929T090000Z', + '19971030T090000Z', + '19971127T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'HOURLY', + 'INTERVAL' => 3, + 'DTSTART' => '19970929T090000Z', + 'COUNT' => 3, + ); + $expect[] = array( + '19970929T090000Z', + '19970929T120000Z', + '19970929T150000Z', + ); + + $tests[] = array( + 'FREQ' => 'MINUTELY', + 'INTERVAL' => 15, + 'COUNT' => 6, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970902T091500Z', + '19970902T093000Z', + '19970902T094500Z', + '19970902T100000Z', + '19970902T101500Z', + ); + + $tests[] = array( + 'FREQ' => 'MINUTELY', + 'INTERVAL' => 90, + 'COUNT' => 4, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970902T103000Z', + '19970902T120000Z', + '19970902T133000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 10, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970909T090000Z', + '19970916T090000Z', + '19970923T090000Z', + '19970930T090000Z', + '19971007T090000Z', + '19971014T090000Z', + '19971021T090000Z', + '19971028T090000Z', + '19971104T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 6, + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970916T090000Z', + '19970930T090000Z', + '19971014T090000Z', + '19971028T090000Z', + '19971111T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'COUNT' => 10, + 'WKST' => 'SU', + 'BYDAY' => array('TU', 'TH'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970909T090000Z', + '19970911T090000Z', + '19970916T090000Z', + '19970918T090000Z', + '19970923T090000Z', + '19970925T090000Z', + '19970930T090000Z', + '19971002T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 8, + 'WKST' => 'SU', + 'BYDAY' => array('TU', 'TH'), + 'DTSTART' => '19970902T090000Z', + ); + $expect[] = array( + '19970902T090000Z', + '19970904T090000Z', + '19970916T090000Z', + '19970918T090000Z', + '19970930T090000Z', + '19971002T090000Z', + '19971014T090000Z', + '19971016T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 4, + 'BYDAY' => array('TU', 'SU'), + 'WKST' => 'MO', + 'DTSTART' => '19970805T090000Z', + ); + $expect[] = array( + '19970805T090000Z', + '19970810T090000Z', + '19970819T090000Z', + '19970824T090000Z', + ); + + $tests[] = array( + 'FREQ' => 'WEEKLY', + 'INTERVAL' => 2, + 'COUNT' => 4, + 'BYDAY' => array('TU', 'SU'), + 'WKST' => 'SU', + 'DTSTART' => '19970805T090000Z', + ); + $expect[] = array( + '19970805T090000Z', + '19970817T090000Z', + '19970819T090000Z', + '19970831T090000Z', + ); + + + $this->assertRules(array(), $tests, $expect); + } + + + private function assertRules(array $defaults, array $tests, array $expect) { + foreach ($tests as $key => $test) { + $options = $test + $defaults; + + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601( + $options['DTSTART']); + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setStartDateTime($start) + ->setFrequency($options['FREQ']); + + $interval = idx($options, 'INTERVAL'); + if ($interval) { + $rrule->setInterval($interval); + } + + $by_day = idx($options, 'BYDAY'); + if ($by_day) { + $rrule->setByDay($by_day); + } + + $by_month = idx($options, 'BYMONTH'); + if ($by_month) { + $rrule->setByMonth($by_month); + } + + $by_monthday = idx($options, 'BYMONTHDAY'); + if ($by_monthday) { + $rrule->setByMonthDay($by_monthday); + } + + $by_yearday = idx($options, 'BYYEARDAY'); + if ($by_yearday) { + $rrule->setByYearDay($by_yearday); + } + + $by_weekno = idx($options, 'BYWEEKNO'); + if ($by_weekno) { + $rrule->setByWeekNumber($by_weekno); + } + + $by_hour = idx($options, 'BYHOUR'); + if ($by_hour) { + $rrule->setByHour($by_hour); + } + + $by_minute = idx($options, 'BYMINUTE'); + if ($by_minute) { + $rrule->setByMinute($by_minute); + } + + $by_second = idx($options, 'BYSECOND'); + if ($by_second) { + $rrule->setBySecond($by_second); + } + + $by_setpos = idx($options, 'BYSETPOS'); + if ($by_setpos) { + $rrule->setBySetPosition($by_setpos); + } + + $week_start = idx($options, 'WKST'); + if ($week_start) { + $rrule->setWeekStart($week_start); + } + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $set->getEventsBetween(null, null, $options['COUNT']); + + $this->assertEqual( + $expect[$key], + mpull($result, 'getISO8601')); + } + } + + +} diff --git a/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php new file mode 100644 index 0000000000..989975ee59 --- /dev/null +++ b/src/applications/calendar/parser/data/__tests__/PhutilCalendarRecurrenceTestCase.php @@ -0,0 +1,196 @@ +getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + array(), + $result, + pht('Set with no sources.')); + + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource(new PhutilCalendarRecurrenceList()); + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + array(), + $result, + pht('Set with empty list source.')); + + + $list = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source = id(new PhutilCalendarRecurrenceList()) + ->setDates($list); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Simple date list.')); + + $list_a = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + ); + + $list_b = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source_a = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_a); + + $source_b = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_b); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source_a) + ->addSource($source_b); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + ); + + $result = $set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple date lists.')); + + $list_a = array( + // This is Jan 1, 3, 5, 7, 8 and 10, but listed out-of-order. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160110T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160107T120000Z'), + ); + + $list_b = array( + // This is Jan 2, 4, 5, 8, but listed out of order. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + ); + + $list_c = array( + // We're going to use this as an exception list. + + // This is Jan 7 (listed in one other source), 8 (listed in two) + // and 9 (listed in none). + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160107T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160108T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160109T120000Z'), + ); + + $expect = array( + // From source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + // From source B. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + // From source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + // From source B. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + // From source A and B. Should appear only once. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160105T120000Z'), + // The 6th appears in no source. + // The 7th, 8th and 9th are excluded. + // The 10th is from source A. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160110T120000Z'), + ); + + $list_a = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_a); + + $list_b = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_b); + + $list_c = id(new PhutilCalendarRecurrenceList()) + ->setDates($list_c) + ->setIsExceptionSource(true); + + $date_set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($list_b) + ->addSource($list_c) + ->addSource($list_a); + + $date_set->setViewerTimezone('UTC'); + + $result = $date_set->getEventsBetween(null, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Set of all results in multiple lists with exclusions.')); + + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + ); + $result = $date_set->getEventsBetween(null, null, 1); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple lists, one result.')); + + $expect = array( + 2 => PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + 3 => PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z'), + ); + $result = $date_set->getEventsBetween( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160104T120000Z')); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Multiple lists, time window.')); + } + + public function testCalendarRecurrenceOffsets() { + $list = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160101T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120000Z'), + ); + + $source = id(new PhutilCalendarRecurrenceList()) + ->setDates($list); + + $set = id(new PhutilCalendarRecurrenceSet()) + ->addSource($source); + + $t1 = PhutilCalendarAbsoluteDateTime::newFromISO8601('20160102T120001Z'); + $t2 = PhutilCalendarAbsoluteDateTime::newFromISO8601('20160103T120000Z'); + + $expect = array( + 2 => $t2, + ); + + $result = $set->getEventsBetween($t1, null, 0xFFFF); + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Correct event indexes with start date.')); + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSParser.php b/src/applications/calendar/parser/ics/PhutilICSParser.php new file mode 100644 index 0000000000..015bea601c --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSParser.php @@ -0,0 +1,919 @@ +stack = array(); + $this->node = null; + $this->cursor = null; + $this->warnings = array(); + + $lines = $this->unfoldICSLines($data); + $this->lines = $lines; + + $root = $this->newICSNode(''); + $this->stack[] = $root; + $this->node = $root; + + foreach ($lines as $key => $line) { + $this->cursor = $key; + $matches = null; + if (preg_match('(^BEGIN:(.*)\z)', $line, $matches)) { + $this->beginParsingNode($matches[1]); + } else if (preg_match('(^END:(.*)\z)', $line, $matches)) { + $this->endParsingNode($matches[1]); + } else { + if (count($this->stack) < 2) { + $this->raiseParseFailure( + self::PARSE_ROOT_PROPERTY, + pht( + 'Found unexpected property at ICS document root.')); + } + $this->parseICSProperty($line); + } + } + + if (count($this->stack) > 1) { + $this->raiseParseFailure( + self::PARSE_MISSING_END, + pht( + 'Expected all "BEGIN:" sections in ICS document to have '. + 'corresponding "END:" sections.')); + } + + $this->node = null; + $this->lines = null; + $this->cursor = null; + + return $root; + } + + private function getNode() { + return $this->node; + } + + private function unfoldICSLines($data) { + $lines = phutil_split_lines($data, $retain_endings = false); + $this->lines = $lines; + + // ICS files are wrapped at 75 characters, with overlong lines continued + // on the following line with an initial space or tab. Unwrap all of the + // lines in the file. + + // This unwrapping is specifically byte-oriented, not character oriented, + // and RFC5545 anticipates that simple implementations may even split UTF8 + // characters in the middle. + + $last = null; + foreach ($lines as $idx => $line) { + $this->cursor = $idx; + if (!preg_match('/^[ \t]/', $line)) { + $last = $idx; + continue; + } + + if ($last === null) { + $this->raiseParseFailure( + self::PARSE_INITIAL_UNFOLD, + pht( + 'First line of ICS file begins with a space or tab, but this '. + 'marks a line which should be unfolded.')); + } + + $lines[$last] = $lines[$last].substr($line, 1); + unset($lines[$idx]); + } + + return $lines; + } + + private function beginParsingNode($type) { + $node = $this->getNode(); + $new_node = $this->newICSNode($type); + + if ($node instanceof PhutilCalendarContainerNode) { + $node->appendChild($new_node); + } else { + $this->raiseParseFailure( + self::PARSE_UNEXPECTED_CHILD, + pht( + 'Found unexpected node "%s" inside node "%s".', + $new_node->getAttribute('ics.type'), + $node->getAttribute('ics.type'))); + } + + $this->stack[] = $new_node; + $this->node = $new_node; + + return $this; + } + + private function newICSNode($type) { + switch ($type) { + case '': + $node = new PhutilCalendarRootNode(); + break; + case 'VCALENDAR': + $node = new PhutilCalendarDocumentNode(); + break; + case 'VEVENT': + $node = new PhutilCalendarEventNode(); + break; + default: + $node = new PhutilCalendarRawNode(); + break; + } + + $node->setAttribute('ics.type', $type); + + return $node; + } + + private function endParsingNode($type) { + $node = $this->getNode(); + if ($node instanceof PhutilCalendarRootNode) { + $this->raiseParseFailure( + self::PARSE_EXTRA_END, + pht( + 'Found unexpected "END" without a "BEGIN".')); + } + + $old_type = $node->getAttribute('ics.type'); + if ($old_type != $type) { + $this->raiseParseFailure( + self::PARSE_MISMATCHED_SECTIONS, + pht( + 'Found mismatched "BEGIN" ("%s") and "END" ("%s") sections.', + $old_type, + $type)); + } + + array_pop($this->stack); + $this->node = last($this->stack); + + return $this; + } + + private function parseICSProperty($line) { + $matches = null; + + // Properties begin with an alphanumeric name with no escaping, followed + // by either a ";" (to begin a list of parameters) or a ":" (to begin + // the actual field body). + + $ok = preg_match('(^([A-Za-z0-9-]+)([;:])(.*)\z)', $line, $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_PROPERTY, + pht( + 'Found malformed property in ICS document.')); + } + + $name = $matches[1]; + $body = $matches[3]; + $has_parameters = ($matches[2] == ';'); + + $parameters = array(); + if ($has_parameters) { + // Parameters are a sensible name, a literal "=", a pile of magic, + // and then maybe a comma and another parameter. + + while (true) { + // We're going to get the first couple of parts first. + $ok = preg_match('(^([^=]+)=)', $body, $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_PARAMETER_NAME, + pht( + 'Found malformed property in ICS document: %s', + $body)); + } + + $param_name = $matches[1]; + $body = substr($body, strlen($matches[0])); + + // Now we're going to match zero or more values. + $param_values = array(); + while (true) { + // The value can either be a double-quoted string or an unquoted + // string, with some characters forbidden. + if (strlen($body) && $body[0] == '"') { + $is_quoted = true; + $ok = preg_match( + '(^"([^\x00-\x08\x10-\x19"]*)")', + $body, + $matches); + if (!$ok) { + $this->raiseParseFailure( + self::PARSE_MALFORMED_DOUBLE_QUOTE, + pht( + 'Found malformed double-quoted string in ICS document '. + 'parameter value.')); + } + } else { + $is_quoted = false; + + // It's impossible for this not to match since it can match + // nothing, and it's valid for it to match nothing. + preg_match('(^([^\x00-\x08\x10-\x19";:,]*))', $body, $matches); + } + + // NOTE: RFC5545 says "Property parameter values that are not in + // quoted-strings are case-insensitive." -- that is, the quoted and + // unquoted representations are not equivalent. Thus, preserve the + // original formatting in case we ever need to respect this. + + $param_values[] = array( + 'value' => $this->unescapeParameterValue($matches[1]), + 'quoted' => $is_quoted, + ); + + $body = substr($body, strlen($matches[0])); + if (!strlen($body)) { + $this->raiseParseFailure( + self::PARSE_MISSING_VALUE, + pht( + 'Expected ":" after parameters in ICS document property.')); + } + + // If we have a comma now, we're going to read another value. Strip + // it off and keep going. + if ($body[0] == ',') { + $body = substr($body, 1); + continue; + } + + // If we have a semicolon, we're going to read another parameter. + if ($body[0] == ';') { + break; + } + + // If we have a colon, this is the last value and also the last + // property. Break, then handle the colon below. + if ($body[0] == ':') { + break; + } + + $short_body = id(new PhutilUTF8StringTruncator()) + ->setMaximumGlyphs(32) + ->truncateString($body); + + // We aren't expecting anything else. + $this->raiseParseFailure( + self::PARSE_UNEXPECTED_TEXT, + pht( + 'Found unexpected text ("%s") after reading parameter value.', + $short_body)); + } + + $parameters[] = array( + 'name' => $param_name, + 'values' => $param_values, + ); + + if ($body[0] == ';') { + $body = substr($body, 1); + continue; + } + + if ($body[0] == ':') { + $body = substr($body, 1); + break; + } + } + } + + $value = $this->unescapeFieldValue($name, $parameters, $body); + + $node = $this->getNode(); + + + $raw = $node->getAttribute('ics.properties', array()); + $raw[] = array( + 'name' => $name, + 'parameters' => $parameters, + 'value' => $value, + ); + $node->setAttribute('ics.properties', $raw); + + switch ($node->getAttribute('ics.type')) { + case 'VEVENT': + $this->didParseEventProperty($node, $name, $parameters, $value); + break; + } + } + + private function unescapeParameterValue($data) { + // The parameter grammar is adjusted by RFC6868 to permit escaping with + // carets. Remove that escaping. + + // This escaping is a bit weird because it's trying to be backwards + // compatible and the original spec didn't think about this and didn't + // provide much room to fix things. + + $out = ''; + $esc = false; + foreach (phutil_utf8v($data) as $c) { + if (!$esc) { + if ($c != '^') { + $out .= $c; + } else { + $esc = true; + } + } else { + switch ($c) { + case 'n': + $out .= "\n"; + break; + case '^': + $out .= '^'; + break; + case "'": + // NOTE: This is " " being decoded into a + // double quote! + $out .= '"'; + break; + default: + // NOTE: The caret is NOT an escape for any other characters. + // This is a "MUST" requirement of RFC6868. + $out .= '^'.$c; + break; + } + } + } + + // NOTE: Because caret on its own just means "caret" for backward + // compatibility, we don't warn if we're still in escaped mode once we + // reach the end of the string. + + return $out; + } + + private function unescapeFieldValue($name, array $parameters, $data) { + // NOTE: The encoding of the field value data is dependent on the field + // name (which defines a default encoding) and the parameters (which may + // include "VALUE", specifying a type of the data. + + $default_types = array( + 'CALSCALE' => 'TEXT', + 'METHOD' => 'TEXT', + 'PRODID' => 'TEXT', + 'VERSION' => 'TEXT', + + 'ATTACH' => 'URI', + 'CATEGORIES' => 'TEXT', + 'CLASS' => 'TEXT', + 'COMMENT' => 'TEXT', + 'DESCRIPTION' => 'TEXT', + + // TODO: The spec appears to contradict itself: it says that the value + // type is FLOAT, but it also says that this property value is actually + // two semicolon-separated values, which is not what FLOAT is defined as. + 'GEO' => 'TEXT', + + 'LOCATION' => 'TEXT', + 'PERCENT-COMPLETE' => 'INTEGER', + 'PRIORITY' => 'INTEGER', + 'RESOURCES' => 'TEXT', + 'STATUS' => 'TEXT', + 'SUMMARY' => 'TEXT', + + 'COMPLETED' => 'DATE-TIME', + 'DTEND' => 'DATE-TIME', + 'DUE' => 'DATE-TIME', + 'DTSTART' => 'DATE-TIME', + 'DURATION' => 'DURATION', + 'FREEBUSY' => 'PERIOD', + 'TRANSP' => 'TEXT', + + 'TZID' => 'TEXT', + 'TZNAME' => 'TEXT', + 'TZOFFSETFROM' => 'UTC-OFFSET', + 'TZOFFSETTO' => 'UTC-OFFSET', + 'TZURL' => 'URI', + + 'ATTENDEE' => 'CAL-ADDRESS', + 'CONTACT' => 'TEXT', + 'ORGANIZER' => 'CAL-ADDRESS', + 'RECURRENCE-ID' => 'DATE-TIME', + 'RELATED-TO' => 'TEXT', + 'URL' => 'URI', + 'UID' => 'TEXT', + 'EXDATE' => 'DATE-TIME', + 'RDATE' => 'DATE-TIME', + 'RRULE' => 'RECUR', + + 'ACTION' => 'TEXT', + 'REPEAT' => 'INTEGER', + 'TRIGGER' => 'DURATION', + + 'CREATED' => 'DATE-TIME', + 'DTSTAMP' => 'DATE-TIME', + 'LAST-MODIFIED' => 'DATE-TIME', + 'SEQUENCE' => 'INTEGER', + + 'REQUEST-STATUS' => 'TEXT', + ); + + $value_type = idx($default_types, $name, 'TEXT'); + + foreach ($parameters as $parameter) { + if ($parameter['name'] == 'VALUE') { + $value_type = idx(head($parameter['values']), 'value'); + } + } + + switch ($value_type) { + case 'BINARY': + $result = base64_decode($data, true); + if ($result === false) { + $this->raiseParseFailure( + self::PARSE_BAD_BASE64, + pht( + 'Unable to decode base64 data: %s', + $data)); + } + break; + case 'BOOLEAN': + $map = array( + 'true' => true, + 'false' => false, + ); + $result = phutil_utf8_strtolower($data); + if (!isset($map[$result])) { + $this->raiseParseFailure( + self::PARSE_BAD_BOOLEAN, + pht( + 'Unexpected BOOLEAN value "%s".', + $data)); + } + $result = $map[$result]; + break; + case 'CAL-ADDRESS': + $result = $data; + break; + case 'DATE': + // This is a comma-separated list of "YYYYMMDD" values. + $result = explode(',', $data); + break; + case 'DATE-TIME': + if (!strlen($data)) { + $result = array(); + } else { + $result = explode(',', $data); + } + break; + case 'DURATION': + if (!strlen($data)) { + $result = array(); + } else { + $result = explode(',', $data); + } + break; + case 'FLOAT': + $result = explode(',', $data); + foreach ($result as $k => $v) { + $result[$k] = (float)$v; + } + break; + case 'INTEGER': + $result = explode(',', $data); + foreach ($result as $k => $v) { + $result[$k] = (int)$v; + } + break; + case 'PERIOD': + $result = explode(',', $data); + break; + case 'RECUR': + $result = $data; + break; + case 'TEXT': + $result = $this->unescapeTextValue($data); + break; + case 'TIME': + $result = explode(',', $data); + break; + case 'URI': + $result = $data; + break; + case 'UTC-OFFSET': + $result = $data; + break; + default: + // RFC5545 says we MUST preserve the data for any types we don't + // recognize. + $result = $data; + break; + } + + return array( + 'type' => $value_type, + 'value' => $result, + 'raw' => $data, + ); + } + + private function unescapeTextValue($data) { + $result = array(); + + $buf = ''; + $esc = false; + foreach (phutil_utf8v($data) as $c) { + if (!$esc) { + if ($c == '\\') { + $esc = true; + } else if ($c == ',') { + $result[] = $buf; + $buf = ''; + } else { + $buf .= $c; + } + } else { + switch ($c) { + case 'n': + case 'N': + $buf .= "\n"; + break; + default: + $buf .= $c; + break; + } + $esc = false; + } + } + + if ($esc) { + $this->raiseParseFailure( + self::PARSE_UNESCAPED_BACKSLASH, + pht( + 'ICS document contains TEXT value ending with unescaped '. + 'backslash.')); + } + + $result[] = $buf; + + return $result; + } + + private function raiseParseFailure($code, $message) { + if ($this->lines && isset($this->lines[$this->cursor])) { + $message = pht( + "ICS Parse Error near line %s:\n\n>>> %s\n\n%s", + $this->cursor + 1, + $this->lines[$this->cursor], + $message); + } else { + $message = pht( + 'ICS Parse Error: %s', + $message); + } + + throw id(new PhutilICSParserException($message)) + ->setParserFailureCode($code); + } + + private function raiseWarning($code, $message) { + $this->warnings[] = array( + 'code' => $code, + 'line' => $this->cursor, + 'text' => $this->lines[$this->cursor], + 'message' => $message, + ); + + return $this; + } + + public function getWarnings() { + return $this->warnings; + } + + private function didParseEventProperty( + PhutilCalendarEventNode $node, + $name, + array $parameters, + array $value) { + + switch ($name) { + case 'UID': + $text = $this->newTextFromProperty($parameters, $value); + $node->setUID($text); + break; + case 'CREATED': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setCreatedDateTime($datetime); + break; + case 'DTSTAMP': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setModifiedDateTime($datetime); + break; + case 'SUMMARY': + $text = $this->newTextFromProperty($parameters, $value); + $node->setName($text); + break; + case 'DESCRIPTION': + $text = $this->newTextFromProperty($parameters, $value); + $node->setDescription($text); + break; + case 'DTSTART': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setStartDateTime($datetime); + break; + case 'DTEND': + $datetime = $this->newDateTimeFromProperty($parameters, $value); + $node->setEndDateTime($datetime); + break; + case 'DURATION': + $duration = $this->newDurationFromProperty($parameters, $value); + $node->setDuration($duration); + break; + case 'RRULE': + $rrule = $this->newRecurrenceRuleFromProperty($parameters, $value); + $node->setRecurrenceRule($rrule); + break; + case 'RECURRENCE-ID': + $text = $this->newTextFromProperty($parameters, $value); + $node->setRecurrenceID($text); + break; + case 'ATTENDEE': + $attendee = $this->newAttendeeFromProperty($parameters, $value); + $node->addAttendee($attendee); + break; + } + + } + + private function newTextFromProperty(array $parameters, array $value) { + $value = $value['value']; + return implode("\n\n", $value); + } + + private function newAttendeeFromProperty(array $parameters, array $value) { + $uri = $value['value']; + + switch (idx($parameters, 'PARTSTAT')) { + case 'ACCEPTED': + $status = PhutilCalendarUserNode::STATUS_ACCEPTED; + break; + case 'DECLINED': + $status = PhutilCalendarUserNode::STATUS_DECLINED; + break; + case 'NEEDS-ACTION': + default: + $status = PhutilCalendarUserNode::STATUS_INVITED; + break; + } + + $name = $this->getScalarParameterValue($parameters, 'CN'); + + return id(new PhutilCalendarUserNode()) + ->setURI($uri) + ->setName($name) + ->setStatus($status); + } + + private function newDateTimeFromProperty(array $parameters, array $value) { + $value = $value['value']; + + if (!$value) { + $this->raiseParseFailure( + self::PARSE_EMPTY_DATETIME, + pht( + 'Expected DATE-TIME to have exactly one value, found none.')); + + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MANY_DATETIME, + pht( + 'Expected DATE-TIME to have exactly one value, found more than '. + 'one.')); + } + + $value = head($value); + $tzid = $this->getScalarParameterValue($parameters, 'TZID'); + + if (preg_match('/Z\z/', $value)) { + if ($tzid) { + $this->raiseWarning( + self::WARN_TZID_UTC, + pht( + 'DATE-TIME "%s" uses "Z" to specify UTC, but also has a TZID '. + 'parameter with value "%s". This violates RFC5545. The TZID '. + 'will be ignored, and the value will be interpreted as UTC.', + $value, + $tzid)); + } + $tzid = 'UTC'; + } else if ($tzid !== null) { + $tzid = $this->guessTimezone($tzid); + } + + try { + $datetime = PhutilCalendarAbsoluteDateTime::newFromISO8601( + $value, + $tzid); + } catch (Exception $ex) { + $this->raiseParseFailure( + self::PARSE_BAD_DATETIME, + pht( + 'Error parsing DATE-TIME: %s', + $ex->getMessage())); + } + + return $datetime; + } + + private function newDurationFromProperty(array $parameters, array $value) { + $value = $value['value']; + + if (!$value) { + $this->raiseParseFailure( + self::PARSE_EMPTY_DURATION, + pht( + 'Expected DURATION to have exactly one value, found none.')); + + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MANY_DURATION, + pht( + 'Expected DURATION to have exactly one value, found more than '. + 'one.')); + } + + $value = head($value); + + try { + $duration = PhutilCalendarDuration::newFromISO8601($value); + } catch (Exception $ex) { + $this->raiseParseFailure( + self::PARSE_BAD_DURATION, + pht( + 'Invalid DURATION: %s', + $ex->getMessage())); + } + + return $duration; + } + + private function newRecurrenceRuleFromProperty(array $parameters, $value) { + return PhutilCalendarRecurrenceRule::newFromRRULE($value['value']); + } + + private function getScalarParameterValue( + array $parameters, + $name, + $default = null) { + + $match = null; + foreach ($parameters as $parameter) { + if ($parameter['name'] == $name) { + $match = $parameter; + } + } + + if ($match === null) { + return $default; + } + + $value = $match['values']; + if (!$value) { + // Parameter is specified, but with no value, like "KEY=". Just return + // the default, as though the parameter was not specified. + return $default; + } + + if (count($value) > 1) { + $this->raiseParseFailure( + self::PARSE_MULTIPLE_PARAMETERS, + pht( + 'Expected parameter "%s" to have at most one value, but found '. + 'more than one.', + $name)); + } + + return idx(head($value), 'value'); + } + + private function guessTimezone($tzid) { + $map = DateTimeZone::listIdentifiers(); + $map = array_fuse($map); + if (isset($map[$tzid])) { + // This is a real timezone we recognize, so just use it as provided. + return $tzid; + } + + // These are alternate names for timezones. + static $aliases; + + if ($aliases === null) { + $aliases = array( + 'Etc/GMT' => 'UTC', + ); + + // Load the map of Windows timezones. + $root_path = dirname(phutil_get_library_root('phutil')); + $windows_path = $root_path.'/resources/timezones/windows_timezones.json'; + $windows_data = Filesystem::readFile($windows_path); + $windows_zones = phutil_json_decode($windows_data); + + $aliases = $aliases + $windows_zones; + } + + if (isset($aliases[$tzid])) { + return $aliases[$tzid]; + } + + // Look for something that looks like "UTC+3" or "GMT -05.00". If we find + // anything, pick a timezone with that offset. + $offset_pattern = + '/'. + '(?:UTC|GMT)'. + '\s*'. + '(?P[+-])'. + '\s*'. + '(?P\d+)'. + '(?:'. + '[:.](?P\d+)'. + ')?'. + '/i'; + + $matches = null; + if (preg_match($offset_pattern, $tzid, $matches)) { + $hours = (int)$matches['h']; + $minutes = (int)idx($matches, 'm'); + $offset = ($hours * 60 * 60) + ($minutes * 60); + + if (idx($matches, 'sign') == '-') { + $offset = -$offset; + } + + // NOTE: We could possibly do better than this, by using the event start + // time to guess a timezone. However, that won't work for recurring + // events and would require us to do this work after finishing initial + // parsing. Since these unusual offset-based timezones appear to be rare, + // the benefit may not be worth the complexity. + $now = new DateTime('@'.time()); + + foreach ($map as $identifier) { + $zone = new DateTimeZone($identifier); + if ($zone->getOffset($now) == $offset) { + $this->raiseWarning( + self::WARN_TZID_GUESS, + pht( + 'TZID "%s" is unknown, guessing "%s" based on pattern "%s".', + $tzid, + $identifier, + $matches[0])); + return $identifier; + } + } + } + + $this->raiseWarning( + self::WARN_TZID_IGNORED, + pht( + 'TZID "%s" is unknown, using UTC instead.', + $tzid)); + + return 'UTC'; + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSParserException.php b/src/applications/calendar/parser/ics/PhutilICSParserException.php new file mode 100644 index 0000000000..09563ffaef --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSParserException.php @@ -0,0 +1,16 @@ +parserFailureCode = $code; + return $this; + } + + public function getParserFailureCode() { + return $this->parserFailureCode; + } + +} diff --git a/src/applications/calendar/parser/ics/PhutilICSWriter.php b/src/applications/calendar/parser/ics/PhutilICSWriter.php new file mode 100644 index 0000000000..c5baa791de --- /dev/null +++ b/src/applications/calendar/parser/ics/PhutilICSWriter.php @@ -0,0 +1,387 @@ +getChildren() as $child) { + $out[] = $this->writeNode($child); + } + + return implode('', $out); + } + + private function writeNode(PhutilCalendarNode $node) { + if (!$this->getICSNodeType($node)) { + return null; + } + + $out = array(); + + $out[] = $this->writeBeginNode($node); + $out[] = $this->writeNodeProperties($node); + + if ($node instanceof PhutilCalendarContainerNode) { + foreach ($node->getChildren() as $child) { + $out[] = $this->writeNode($child); + } + } + + $out[] = $this->writeEndNode($node); + + return implode('', $out); + } + + private function writeBeginNode(PhutilCalendarNode $node) { + $type = $this->getICSNodeType($node); + return $this->wrapICSLine("BEGIN:{$type}"); + } + + private function writeEndNode(PhutilCalendarNode $node) { + $type = $this->getICSNodeType($node); + return $this->wrapICSLine("END:{$type}"); + } + + private function writeNodeProperties(PhutilCalendarNode $node) { + $properties = $this->getNodeProperties($node); + + $out = array(); + foreach ($properties as $property) { + $propname = $property['name']; + $propvalue = $property['value']; + + $propline = array(); + $propline[] = $propname; + + foreach ($property['parameters'] as $parameter) { + $paramname = $parameter['name']; + $paramvalue = $parameter['value']; + $propline[] = ";{$paramname}={$paramvalue}"; + } + + $propline[] = ":{$propvalue}"; + $propline = implode('', $propline); + + $out[] = $this->wrapICSLine($propline); + } + + return implode('', $out); + } + + private function getICSNodeType(PhutilCalendarNode $node) { + switch ($node->getNodeType()) { + case PhutilCalendarDocumentNode::NODETYPE: + return 'VCALENDAR'; + case PhutilCalendarEventNode::NODETYPE: + return 'VEVENT'; + default: + return null; + } + } + + private function wrapICSLine($line) { + $out = array(); + $buf = ''; + + // NOTE: The line may contain sequences of combining characters which are + // more than 80 bytes in length. If it does, we'll split them in the + // middle of the sequence. This is okay and generally anticipated by + // RFC5545, which even allows implementations to split multibyte + // characters. The sequence will be stitched back together properly by + // whatever is parsing things. + + foreach (phutil_utf8v($line) as $character) { + // If adding this character would bring the line over 75 bytes, start + // a new line. + if (strlen($buf) + strlen($character) > 75) { + $out[] = $buf."\r\n"; + $buf = ' '; + } + + $buf .= $character; + } + + $out[] = $buf."\r\n"; + + return implode('', $out); + } + + private function getNodeProperties(PhutilCalendarNode $node) { + switch ($node->getNodeType()) { + case PhutilCalendarDocumentNode::NODETYPE: + return $this->getDocumentNodeProperties($node); + case PhutilCalendarEventNode::NODETYPE: + return $this->getEventNodeProperties($node); + default: + return array(); + } + } + + private function getDocumentNodeProperties( + PhutilCalendarDocumentNode $event) { + $properties = array(); + + $properties[] = $this->newTextProperty( + 'VERSION', + '2.0'); + + $properties[] = $this->newTextProperty( + 'PRODID', + '-//Phacility//Phabricator//EN'); + + return $properties; + } + + private function getEventNodeProperties(PhutilCalendarEventNode $event) { + $properties = array(); + + $uid = $event->getUID(); + if (!strlen($uid)) { + throw new Exception( + pht( + 'Unable to write ICS document: event has no UID, but each event '. + 'MUST have a UID.')); + } + $properties[] = $this->newTextProperty( + 'UID', + $uid); + + $created = $event->getCreatedDateTime(); + if ($created) { + $properties[] = $this->newDateTimeProperty( + 'CREATED', + $event->getCreatedDateTime()); + } + + $dtstamp = $event->getModifiedDateTime(); + if (!$dtstamp) { + throw new Exception( + pht( + 'Unable to write ICS document: event has no modified time, but '. + 'each event MUST have a modified time.')); + } + $properties[] = $this->newDateTimeProperty( + 'DTSTAMP', + $dtstamp); + + $dtstart = $event->getStartDateTime(); + if ($dtstart) { + $properties[] = $this->newDateTimeProperty( + 'DTSTART', + $dtstart); + } + + $dtend = $event->getEndDateTime(); + if ($dtend) { + $properties[] = $this->newDateTimeProperty( + 'DTEND', + $event->getEndDateTime()); + } + + $name = $event->getName(); + if (strlen($name)) { + $properties[] = $this->newTextProperty( + 'SUMMARY', + $name); + } + + $description = $event->getDescription(); + if (strlen($description)) { + $properties[] = $this->newTextProperty( + 'DESCRIPTION', + $description); + } + + $organizer = $event->getOrganizer(); + if ($organizer) { + $properties[] = $this->newUserProperty( + 'ORGANIZER', + $organizer); + } + + $attendees = $event->getAttendees(); + if ($attendees) { + foreach ($attendees as $attendee) { + $properties[] = $this->newUserProperty( + 'ATTENDEE', + $attendee); + } + } + + $rrule = $event->getRecurrenceRule(); + if ($rrule) { + $properties[] = $this->newRRULEProperty( + 'RRULE', + $rrule); + } + + $recurrence_id = $event->getRecurrenceID(); + if ($recurrence_id) { + $properties[] = $this->newTextProperty( + 'RECURRENCE-ID', + $recurrence_id); + } + + $exdates = $event->getRecurrenceExceptions(); + if ($exdates) { + $properties[] = $this->newDateTimesProperty( + 'EXDATE', + $exdates); + } + + $rdates = $event->getRecurrenceDates(); + if ($rdates) { + $properties[] = $this->newDateTimesProperty( + 'RDATE', + $rdates); + } + + return $properties; + } + + private function newTextProperty( + $name, + $value, + array $parameters = array()) { + + $map = array( + '\\' => '\\\\', + ',' => '\\,', + "\n" => '\\n', + ); + + $value = (array)$value; + foreach ($value as $k => $v) { + $v = str_replace(array_keys($map), array_values($map), $v); + $value[$k] = $v; + } + + $value = implode(',', $value); + + return $this->newProperty($name, $value, $parameters); + } + + private function newDateTimeProperty( + $name, + PhutilCalendarDateTime $value, + array $parameters = array()) { + + return $this->newDateTimesProperty($name, array($value), $parameters); + } + + private function newDateTimesProperty( + $name, + array $values, + array $parameters = array()) { + assert_instances_of($values, 'PhutilCalendarDateTime'); + + if (head($values)->getIsAllDay()) { + $parameters[] = array( + 'name' => 'VALUE', + 'values' => array( + 'DATE', + ), + ); + } + + $datetimes = array(); + foreach ($values as $value) { + $datetimes[] = $value->getISO8601(); + } + $datetimes = implode(';', $datetimes); + + return $this->newProperty($name, $datetimes, $parameters); + } + + private function newUserProperty( + $name, + PhutilCalendarUserNode $value, + array $parameters = array()) { + + $parameters[] = array( + 'name' => 'CN', + 'values' => array( + $value->getName(), + ), + ); + + $partstat = null; + switch ($value->getStatus()) { + case PhutilCalendarUserNode::STATUS_INVITED: + $partstat = 'NEEDS-ACTION'; + break; + case PhutilCalendarUserNode::STATUS_ACCEPTED: + $partstat = 'ACCEPTED'; + break; + case PhutilCalendarUserNode::STATUS_DECLINED: + $partstat = 'DECLINED'; + break; + } + + if ($partstat !== null) { + $parameters[] = array( + 'name' => 'PARTSTAT', + 'values' => array( + $partstat, + ), + ); + } + + // TODO: We could reasonably fill in "ROLE" and "RSVP" here too, but it + // isn't clear if these are important to external programs or not. + + return $this->newProperty($name, $value->getURI(), $parameters); + } + + private function newRRULEProperty( + $name, + PhutilCalendarRecurrenceRule $rule, + array $parameters = array()) { + + $value = $rule->toRRULE(); + return $this->newProperty($name, $value, $parameters); + } + + private function newProperty( + $name, + $value, + array $parameters = array()) { + + $map = array( + '^' => '^^', + "\n" => '^n', + '"' => "^'", + ); + + $writable_params = array(); + foreach ($parameters as $k => $parameter) { + $value_list = array(); + foreach ($parameter['values'] as $v) { + $v = str_replace(array_keys($map), array_values($map), $v); + + // If the parameter value isn't a very simple one, quote it. + + // RFC5545 says that we MUST quote it if it has a colon, a semicolon, + // or a comma, and that we MUST quote it if it's a URI. + if (!preg_match('/^[A-Za-z0-9-]*\z/', $v)) { + $v = '"'.$v.'"'; + } + + $value_list[] = $v; + } + + $writable_params[] = array( + 'name' => $parameter['name'], + 'value' => implode(',', $value_list), + ); + } + + return array( + 'name' => $name, + 'value' => $value, + 'parameters' => $writable_params, + ); + } + +} diff --git a/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php b/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php new file mode 100644 index 0000000000..e99acaf8ab --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php @@ -0,0 +1,341 @@ +parseICSSingleEvent('simple.ics'); + + $this->assertEqual( + array( + array( + 'name' => 'CREATED', + 'parameters' => array(), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160908T172702Z', + ), + 'raw' => '20160908T172702Z', + ), + ), + array( + 'name' => 'UID', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + '1CEB57AF-0C9C-402D-B3BD-D75BD4843F68', + ), + 'raw' => '1CEB57AF-0C9C-402D-B3BD-D75BD4843F68', + ), + ), + array( + 'name' => 'DTSTART', + 'parameters' => array( + array( + 'name' => 'TZID', + 'values' => array( + array( + 'value' => 'America/Los_Angeles', + 'quoted' => false, + ), + ), + ), + ), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160915T090000', + ), + 'raw' => '20160915T090000', + ), + ), + array( + 'name' => 'DTEND', + 'parameters' => array( + array( + 'name' => 'TZID', + 'values' => array( + array( + 'value' => 'America/Los_Angeles', + 'quoted' => false, + ), + ), + ), + ), + 'value' => array( + 'type' => 'DATE-TIME', + 'value' => array( + '20160915T100000', + ), + 'raw' => '20160915T100000', + ), + ), + array( + 'name' => 'SUMMARY', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + 'Simple Event', + ), + 'raw' => 'Simple Event', + ), + ), + array( + 'name' => 'DESCRIPTION', + 'parameters' => array(), + 'value' => array( + 'type' => 'TEXT', + 'value' => array( + 'This is a simple event.', + ), + 'raw' => 'This is a simple event.', + ), + ), + ), + $event->getAttribute('ics.properties')); + + $this->assertEqual( + 'Simple Event', + $event->getName()); + + $this->assertEqual( + 'This is a simple event.', + $event->getDescription()); + + $this->assertEqual( + 1473955200, + $event->getStartDateTime()->getEpoch()); + + $this->assertEqual( + 1473955200 + phutil_units('1 hour in seconds'), + $event->getEndDateTime()->getEpoch()); + } + + public function testICSOddTimezone() { + $event = $this->parseICSSingleEvent('zimbra-timezone.ics'); + + $start = $event->getStartDateTime(); + + $this->assertEqual( + '20170303T140000Z', + $start->getISO8601()); + } + + public function testICSFloatingTime() { + // This tests "floating" event times, which have no absolute time and are + // supposed to be interpreted using the viewer's timezone. It also uses + // a duration, and the duration needs to float along with the viewer + // timezone. + + $event = $this->parseICSSingleEvent('floating.ics'); + + $start = $event->getStartDateTime(); + + $caught = null; + try { + $start->getEpoch(); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue( + ($caught instanceof Exception), + pht('Expected exception for floating time with no viewer timezone.')); + + $newyears_utc = strtotime('2015-01-01 00:00:00 UTC'); + $this->assertEqual(1420070400, $newyears_utc); + + $start->setViewerTimezone('UTC'); + $this->assertEqual( + $newyears_utc, + $start->getEpoch()); + + $start->setViewerTimezone('America/Los_Angeles'); + $this->assertEqual( + $newyears_utc + phutil_units('8 hours in seconds'), + $start->getEpoch()); + + $start->setViewerTimezone('America/New_York'); + $this->assertEqual( + $newyears_utc + phutil_units('5 hours in seconds'), + $start->getEpoch()); + + $end = $event->getEndDateTime(); + $end->setViewerTimezone('UTC'); + $this->assertEqual( + $newyears_utc + phutil_units('24 hours in seconds'), + $end->getEpoch()); + + $end->setViewerTimezone('America/Los_Angeles'); + $this->assertEqual( + $newyears_utc + phutil_units('32 hours in seconds'), + $end->getEpoch()); + + $end->setViewerTimezone('America/New_York'); + $this->assertEqual( + $newyears_utc + phutil_units('29 hours in seconds'), + $end->getEpoch()); + } + + public function testICSVALARM() { + $event = $this->parseICSSingleEvent('valarm.ics'); + + // For now, we parse but ignore VALARM sections. This test just makes + // sure they survive parsing. + + $start_epoch = strtotime('2016-10-19 22:00:00 UTC'); + $this->assertEqual(1476914400, $start_epoch); + + $this->assertEqual( + $start_epoch, + $event->getStartDateTime()->getEpoch()); + } + + public function testICSDuration() { + $event = $this->parseICSSingleEvent('duration.ics'); + + // Raw value is "20160719T095722Z". + $start_epoch = strtotime('2016-07-19 09:57:22 UTC'); + $this->assertEqual(1468922242, $start_epoch); + + // Raw value is "P1DT17H4M23S". + $duration = + phutil_units('1 day in seconds') + + phutil_units('17 hours in seconds') + + phutil_units('4 minutes in seconds') + + phutil_units('23 seconds in seconds'); + + $this->assertEqual( + $start_epoch, + $event->getStartDateTime()->getEpoch()); + + $this->assertEqual( + $start_epoch + $duration, + $event->getEndDateTime()->getEpoch()); + } + + public function testICSWeeklyEvent() { + $event = $this->parseICSSingleEvent('weekly.ics'); + + $start = $event->getStartDateTime(); + $start->setViewerTimezone('UTC'); + + $rrule = $event->getRecurrenceRule() + ->setStartDateTime($start); + + $rset = id(new PhutilCalendarRecurrenceSet()) + ->addSource($rrule); + + $result = $rset->getEventsBetween(null, null, 3); + + $expect = array( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150811'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150818'), + PhutilCalendarAbsoluteDateTime::newFromISO8601('20150825'), + ); + + $this->assertEqual( + mpull($expect, 'getISO8601'), + mpull($result, 'getISO8601'), + pht('Weekly recurring event.')); + } + + public function testICSParserErrors() { + $map = array( + 'err-missing-end.ics' => PhutilICSParser::PARSE_MISSING_END, + 'err-bad-base64.ics' => PhutilICSParser::PARSE_BAD_BASE64, + 'err-bad-boolean.ics' => PhutilICSParser::PARSE_BAD_BOOLEAN, + 'err-extra-end.ics' => PhutilICSParser::PARSE_EXTRA_END, + 'err-initial-unfold.ics' => PhutilICSParser::PARSE_INITIAL_UNFOLD, + 'err-malformed-double-quote.ics' => + PhutilICSParser::PARSE_MALFORMED_DOUBLE_QUOTE, + 'err-malformed-parameter.ics' => + PhutilICSParser::PARSE_MALFORMED_PARAMETER_NAME, + 'err-malformed-property.ics' => + PhutilICSParser::PARSE_MALFORMED_PROPERTY, + 'err-missing-value.ics' => PhutilICSParser::PARSE_MISSING_VALUE, + 'err-mixmatched-sections.ics' => + PhutilICSParser::PARSE_MISMATCHED_SECTIONS, + 'err-root-property.ics' => PhutilICSParser::PARSE_ROOT_PROPERTY, + 'err-unescaped-backslash.ics' => + PhutilICSParser::PARSE_UNESCAPED_BACKSLASH, + 'err-unexpected-text.ics' => PhutilICSParser::PARSE_UNEXPECTED_TEXT, + 'err-multiple-parameters.ics' => + PhutilICSParser::PARSE_MULTIPLE_PARAMETERS, + 'err-empty-datetime.ics' => + PhutilICSParser::PARSE_EMPTY_DATETIME, + 'err-many-datetime.ics' => + PhutilICSParser::PARSE_MANY_DATETIME, + 'err-bad-datetime.ics' => + PhutilICSParser::PARSE_BAD_DATETIME, + 'err-empty-duration.ics' => + PhutilICSParser::PARSE_EMPTY_DURATION, + 'err-many-duration.ics' => + PhutilICSParser::PARSE_MANY_DURATION, + 'err-bad-duration.ics' => + PhutilICSParser::PARSE_BAD_DURATION, + + 'simple.ics' => null, + 'good-boolean.ics' => null, + 'multiple-vcalendars.ics' => null, + ); + + foreach ($map as $test_file => $expect) { + $caught = null; + try { + $this->parseICSDocument($test_file); + } catch (PhutilICSParserException $ex) { + $caught = $ex; + } + + if ($expect === null) { + $this->assertTrue( + ($caught === null), + pht( + 'Expected no exception parsing "%s", got: %s', + $test_file, + (string)$ex)); + } else { + if ($caught) { + $code = $ex->getParserFailureCode(); + $explain = pht( + 'Expected one exception parsing "%s", got a different '. + 'one: %s', + $test_file, + (string)$ex); + } else { + $code = null; + $explain = pht( + 'Expected exception parsing "%s", got none.', + $test_file); + } + + $this->assertEqual($expect, $code, $explain); + } + } + } + + private function parseICSSingleEvent($name) { + $root = $this->parseICSDocument($name); + + $documents = $root->getDocuments(); + $this->assertEqual(1, count($documents)); + $document = head($documents); + + $events = $document->getEvents(); + $this->assertEqual(1, count($events)); + + return head($events); + } + + private function parseICSDocument($name) { + $path = dirname(__FILE__).'/data/'.$name; + $data = Filesystem::readFile($path); + return id(new PhutilICSParser()) + ->parseICSData($data); + } + + +} diff --git a/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php b/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php new file mode 100644 index 0000000000..dec1bf27b0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php @@ -0,0 +1,144 @@ +setUID('tea-time') + ->setName('Tea Time') + ->setDescription( + "Tea and, perhaps, crumpets.\n". + "Your presence is requested!\n". + "This is a long list of types of tea to test line wrapping: {$teas}.") + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160915T070000Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160915T070000Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160916T150000Z')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160916T160000Z')); + + $ics_data = $this->writeICSSingleEvent($event); + + $this->assertICS('writer-tea-time.ics', $ics_data); + } + + public function testICSWriterChristmas() { + $start = PhutilCalendarAbsoluteDateTime::newFromISO8601('20001225T000000Z'); + $end = PhutilCalendarAbsoluteDateTime::newFromISO8601('20001226T000000Z'); + + $rrule = id(new PhutilCalendarRecurrenceRule()) + ->setFrequency(PhutilCalendarRecurrenceRule::FREQUENCY_YEARLY) + ->setByMonth(array(12)) + ->setByMonthDay(array(25)); + + $event = id(new PhutilCalendarEventNode()) + ->setUID('recurring-christmas') + ->setName('Christmas') + ->setDescription('Festival holiday first occurring in the year 2000.') + ->setStartDateTime($start) + ->setEndDateTime($end) + ->setCreatedDateTime($start) + ->setModifiedDateTime($start) + ->setRecurrenceRule($rrule) + ->setRecurrenceExceptions( + array( + // In 2007, Christmas was cancelled. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20071225T000000Z'), + )) + ->setRecurrenceDates( + array( + // We had an extra early Christmas in 2009. + PhutilCalendarAbsoluteDateTime::newFromISO8601('20091125T000000Z'), + )); + + $ics_data = $this->writeICSSingleEvent($event); + $this->assertICS('writer-recurring-christmas.ics', $ics_data); + } + + public function testICSWriterAllDay() { + $event = id(new PhutilCalendarEventNode()) + ->setUID('christmas-day') + ->setName('Christmas 2016') + ->setDescription('A minor religious holiday.') + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160901T232425Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20160901T232425Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161225')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161226')); + + $ics_data = $this->writeICSSingleEvent($event); + + $this->assertICS('writer-christmas.ics', $ics_data); + } + + public function testICSWriterUsers() { + $event = id(new PhutilCalendarEventNode()) + ->setUID('office-party') + ->setName('Office Party') + ->setCreatedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161001T120000Z')) + ->setModifiedDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161001T120000Z')) + ->setStartDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161215T200000Z')) + ->setEndDateTime( + PhutilCalendarAbsoluteDateTime::newFromISO8601('20161215T230000Z')) + ->setOrganizer( + id(new PhutilCalendarUserNode()) + ->setName('Big Boss') + ->setURI('mailto:big.boss@example.com')) + ->addAttendee( + id(new PhutilCalendarUserNode()) + ->setName('Milton') + ->setStatus(PhutilCalendarUserNode::STATUS_INVITED) + ->setURI('mailto:milton@example.com')) + ->addAttendee( + id(new PhutilCalendarUserNode()) + ->setName('Nancy') + ->setStatus(PhutilCalendarUserNode::STATUS_ACCEPTED) + ->setURI('mailto:nancy@example.com')); + + $ics_data = $this->writeICSSingleEvent($event); + $this->assertICS('writer-office-party.ics', $ics_data); + } + + private function writeICSSingleEvent(PhutilCalendarEventNode $event) { + $calendar = id(new PhutilCalendarDocumentNode()) + ->appendChild($event); + + $root = id(new PhutilCalendarRootNode()) + ->appendChild($calendar); + + return $this->writeICS($root); + } + + private function writeICS(PhutilCalendarRootNode $root) { + return id(new PhutilICSWriter()) + ->writeICSDocument($root); + } + + private function assertICS($name, $actual) { + $path = dirname(__FILE__).'/data/'.$name; + $data = Filesystem::readFile($path); + $this->assertEqual($data, $actual, pht('ICS: %s', $name)); + } + +} diff --git a/src/applications/calendar/parser/ics/__tests__/data/duration.ics b/src/applications/calendar/parser/ics/__tests__/data/duration.ics new file mode 100644 index 0000000000..8f2a8b5c6e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/duration.ics @@ -0,0 +1,8 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20160719T095722Z +DURATION:P1DT17H4M23S +SUMMARY:Duration Event +DESCRIPTION:This is an event with a complex duration. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics new file mode 100644 index 0000000000..ff1997e9e5 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-base64.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DATA;VALUE=BINARY;ENCODING=BASE64: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics new file mode 100644 index 0000000000..f5cd06ca83 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-boolean.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DUCK;VALUE=BOOLEAN:QUACK +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics new file mode 100644 index 0000000000..b6dada8812 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:quack +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics new file mode 100644 index 0000000000..3d0eb7b04e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-bad-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION:quack +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics new file mode 100644 index 0000000000..554fb2393f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-empty-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics new file mode 100644 index 0000000000..33b4d78796 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-empty-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION: +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics b/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics new file mode 100644 index 0000000000..b0bb4bc920 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-extra-end.ics @@ -0,0 +1 @@ +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics b/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics new file mode 100644 index 0000000000..b57778242a --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-initial-unfold.ics @@ -0,0 +1,2 @@ + BEGIN:VCALENDAR +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics new file mode 100644 index 0000000000..cd7623e51b --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-double-quote.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +A;B="C:D +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics new file mode 100644 index 0000000000..8968cfba70 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-parameter.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +A;B:C +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics new file mode 100644 index 0000000000..2121531d2d --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-malformed-property.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +PEANUTBUTTER&JELLY:sandwich +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics b/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics new file mode 100644 index 0000000000..5e617a466e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-many-datetime.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20130101,20130101 +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics b/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics new file mode 100644 index 0000000000..d43d9efd8f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-many-duration.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DURATION:P1W,P2W +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics b/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics new file mode 100644 index 0000000000..d33f05bad7 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-missing-end.ics @@ -0,0 +1,2 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics b/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics new file mode 100644 index 0000000000..39d83aa24f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-missing-value.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +TRIANGLE;color=red +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics b/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics new file mode 100644 index 0000000000..e3a1ab0009 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-mixmatched-sections.ics @@ -0,0 +1,4 @@ +BEGIN:A +BEGIN:B +END:A +END:B diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics b/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics new file mode 100644 index 0000000000..38da44f3ff --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-multiple-parameters.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART;TZID=A,B:20160915T090000 +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics b/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics new file mode 100644 index 0000000000..a36bd4c0b3 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-root-property.ics @@ -0,0 +1 @@ +NAME:value diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics b/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics new file mode 100644 index 0000000000..a1ba94b67a --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-unescaped-backslash.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +STORY:The duck coughed up an unescaped backslash: \ +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics b/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics new file mode 100644 index 0000000000..30873a2f60 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/err-unexpected-text.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +SQUARE;color=red" +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/floating.ics b/src/applications/calendar/parser/ics/__tests__/data/floating.ics new file mode 100644 index 0000000000..eecfe23f3e --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/floating.ics @@ -0,0 +1,8 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART:20150101T000000 +DURATION:P1D +SUMMARY:New Year's 2015 +DESCRIPTION:This is an event with a floating start time. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics b/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics new file mode 100644 index 0000000000..d281b17fb2 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/good-boolean.ics @@ -0,0 +1,5 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DUCK;VALUE=BOOLEAN:TRUE +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics b/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics new file mode 100644 index 0000000000..68e99a6e18 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/multiple-vcalendars.ics @@ -0,0 +1,4 @@ +BEGIN:VCALENDAR +END:VCALENDAR +BEGIN:VCALENDAR +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/simple.ics b/src/applications/calendar/parser/ics/__tests__/data/simple.ics new file mode 100644 index 0000000000..8181a24ff0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/simple.ics @@ -0,0 +1,12 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +BEGIN:VEVENT +CREATED:20160908T172702Z +UID:1CEB57AF-0C9C-402D-B3BD-D75BD4843F68 +DTSTART;TZID=America/Los_Angeles:20160915T090000 +DTEND;TZID=America/Los_Angeles:20160915T100000 +SUMMARY:Simple Event +DESCRIPTION:This is a simple event. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/valarm.ics b/src/applications/calendar/parser/ics/__tests__/data/valarm.ics new file mode 100644 index 0000000000..060f5ef85f --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/valarm.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +CREATED:20161027T173727 +DTSTAMP:20161027T173727 +LAST-MODIFIED:20161027T173727 +UID:aic4zm86mg +SUMMARY:alarm event +DTSTART;TZID=Europe/Berlin:20161020T000000 +DTEND;TZID=Europe/Berlin:20161020T010000 +BEGIN:VALARM +ACTION:AUDIO +TRIGGER:-PT15M +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/weekly.ics b/src/applications/calendar/parser/ics/__tests__/data/weekly.ics new file mode 100644 index 0000000000..8e067bf8ea --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/weekly.ics @@ -0,0 +1,14 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND;VALUE=DATE:20150812 +LAST-MODIFIED:20160822T130015Z +UID:4AE69E91-4A51-4B77-8849-85981E037A83 +DTSTAMP:20161129T152151Z +SUMMARY:Weekly Event +DTSTART;VALUE=DATE:20150811 +CREATED:20141109T163445Z +RRULE:FREQ=WEEKLY +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics new file mode 100644 index 0000000000..4624d151d0 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-christmas.ics @@ -0,0 +1,13 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:christmas-day +CREATED:20160901T232425Z +DTSTAMP:20160901T232425Z +DTSTART;VALUE=DATE:20161225 +DTEND;VALUE=DATE:20161226 +SUMMARY:Christmas 2016 +DESCRIPTION:A minor religious holiday. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics new file mode 100644 index 0000000000..a2fbc81a40 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-office-party.ics @@ -0,0 +1,15 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:office-party +CREATED:20161001T120000Z +DTSTAMP:20161001T120000Z +DTSTART:20161215T200000Z +DTEND:20161215T230000Z +SUMMARY:Office Party +ORGANIZER;CN="Big Boss":mailto:big.boss@example.com +ATTENDEE;CN=Milton;PARTSTAT=NEEDS-ACTION:mailto:milton@example.com +ATTENDEE;CN=Nancy;PARTSTAT=ACCEPTED:mailto:nancy@example.com +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics new file mode 100644 index 0000000000..2774bb5bb9 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-recurring-christmas.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:recurring-christmas +CREATED:20001225T000000Z +DTSTAMP:20001225T000000Z +DTSTART:20001225T000000Z +DTEND:20001226T000000Z +SUMMARY:Christmas +DESCRIPTION:Festival holiday first occurring in the year 2000. +RRULE:FREQ=YEARLY;BYMONTH=12;BYMONTHDAY=25 +EXDATE:20071225T000000Z +RDATE:20091125T000000Z +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics b/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics new file mode 100644 index 0000000000..e275fa9730 --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/writer-tea-time.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Phacility//Phabricator//EN +BEGIN:VEVENT +UID:tea-time +CREATED:20160915T070000Z +DTSTAMP:20160915T070000Z +DTSTART:20160916T150000Z +DTEND:20160916T160000Z +SUMMARY:Tea Time +DESCRIPTION:Tea and\, perhaps\, crumpets.\nYour presence is requested!\nThi + s is a long list of types of tea to test line wrapping: earl grey tea\, En + glish breakfast tea\, black tea\, green tea\, t-rex\, oolong tea\, mint te + a\, tea with milk. +END:VEVENT +END:VCALENDAR diff --git a/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics b/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics new file mode 100644 index 0000000000..6066b57f7d --- /dev/null +++ b/src/applications/calendar/parser/ics/__tests__/data/zimbra-timezone.ics @@ -0,0 +1,12 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +BEGIN:VEVENT +CREATED:20161104T220244Z +UID:zimbra-timezone +SUMMARY:Zimbra Timezone +DTSTART;TZID="(GMT-05.00) Auto-Detected":20170303T090000 +DTSTAMP:20161104T220244Z +SEQUENCE:0 +END:VEVENT +END:VCALENDAR diff --git a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php index 9eb83bd61e..d69e903bcc 100644 --- a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php +++ b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php @@ -140,11 +140,20 @@ final class PhabricatorConfigManagementSetWorkflow 'Wrote configuration key "%s" to database storage.', $key); } else { - $config_source = id(new PhabricatorConfigLocalSource()) - ->setKeys(array($key => $value)); + $config_source = new PhabricatorConfigLocalSource(); $local_path = $config_source->getReadablePath(); + try { + $config_source->setKeys(array($key => $value)); + } catch (FilesystemException $ex) { + throw new PhutilArgumentUsageException( + pht( + 'Local path "%s" is not writable. This file must be writable '. + 'so that "bin/config" can store configuration.', + Filesystem::readablePath($local_path))); + } + $write_message = pht( 'Wrote configuration key "%s" to local storage (in file "%s").', $key, diff --git a/src/applications/diffusion/ref/DiffusionServiceRef.php b/src/applications/diffusion/ref/DiffusionServiceRef.php new file mode 100644 index 0000000000..d6e6948e5d --- /dev/null +++ b/src/applications/diffusion/ref/DiffusionServiceRef.php @@ -0,0 +1,48 @@ +uri = $map['uri']; + $ref->isWritable = $map['writable']; + $ref->devicePHID = $map['devicePHID']; + $ref->protocol = $map['protocol']; + $ref->deviceName = $map['device']; + + return $ref; + } + + public function isWritable() { + return $this->isWritable; + } + + public function getDevicePHID() { + return $this->devicePHID; + } + + public function getURI() { + return $this->uri; + } + + public function getProtocol() { + return $this->protocol; + } + + public function getDeviceName() { + return $this->deviceName; + } + +} diff --git a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php index abf2a4323e..f59a9b58b4 100644 --- a/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitReceivePackSSHWorkflow.php @@ -14,42 +14,33 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { } protected function executeRepositoryOperations() { + // This is a write, and must have write access. + $this->requireWriteAccess(); + + $is_proxy = $this->shouldProxy(); + if ($is_proxy) { + return $this->executeRepositoryProxyOperations($for_write = true); + } + $host_wait_start = microtime(true); $repository = $this->getRepository(); $viewer = $this->getSSHUser(); $device = AlmanacKeys::getLiveDevice(); - // This is a write, and must have write access. - $this->requireWriteAccess(); - $cluster_engine = id(new DiffusionRepositoryClusterEngine()) ->setViewer($viewer) ->setRepository($repository) ->setLog($this); - $is_proxy = $this->shouldProxy(); - if ($is_proxy) { - $command = $this->getProxyCommand(true); - $did_write = false; + $command = csprintf('git-receive-pack %s', $repository->getLocalPath()); + $cluster_engine->synchronizeWorkingCopyBeforeWrite(); - if ($device) { - $this->writeClusterEngineLogMessage( - pht( - "# Push received by \"%s\", forwarding to cluster host.\n", - $device->getName())); - } - } else { - $command = csprintf('git-receive-pack %s', $repository->getLocalPath()); - $did_write = true; - $cluster_engine->synchronizeWorkingCopyBeforeWrite(); - - if ($device) { - $this->writeClusterEngineLogMessage( - pht( - "# Ready to receive on cluster host \"%s\".\n", - $device->getName())); - } + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Ready to receive on cluster host \"%s\".\n", + $device->getName())); } $log = $this->newProtocolLog($is_proxy); @@ -71,9 +62,7 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { // We've committed the write (or rejected it), so we can release the lock // without waiting for the client to receive the acknowledgement. - if ($did_write) { - $cluster_engine->synchronizeWorkingCopyAfterWrite(); - } + $cluster_engine->synchronizeWorkingCopyAfterWrite(); if ($caught) { throw $caught; @@ -85,18 +74,16 @@ final class DiffusionGitReceivePackSSHWorkflow extends DiffusionGitSSHWorkflow { // When a repository is clustered, we reach this cleanup code on both // the proxy and the actual final endpoint node. Don't do more cleanup // or logging than we need to. - if ($did_write) { - $repository->writeStatusMessage( - PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, - PhabricatorRepositoryStatusMessage::CODE_OKAY); + $repository->writeStatusMessage( + PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, + PhabricatorRepositoryStatusMessage::CODE_OKAY); - $host_wait_end = microtime(true); + $host_wait_end = microtime(true); - $this->updatePushLogWithTimingInformation( - $this->getClusterEngineLogProperty('writeWait'), - $this->getClusterEngineLogProperty('readWait'), - ($host_wait_end - $host_wait_start)); - } + $this->updatePushLogWithTimingInformation( + $this->getClusterEngineLogProperty('writeWait'), + $this->getClusterEngineLogProperty('readWait'), + ($host_wait_end - $host_wait_start)); } return $err; diff --git a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php index d9cc8063d5..292741e34d 100644 --- a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php @@ -8,6 +8,10 @@ abstract class DiffusionGitSSHWorkflow private $protocolLog; private $wireProtocol; + private $ioBytesRead = 0; + private $ioBytesWritten = 0; + private $requestAttempts = 0; + private $requestFailures = 0; protected function writeError($message) { // Git assumes we'll add our own newlines. @@ -98,6 +102,8 @@ abstract class DiffusionGitSSHWorkflow PhabricatorSSHPassthruCommand $command, $message) { + $this->ioBytesWritten += strlen($message); + $log = $this->getProtocolLog(); if ($log) { $log->didWriteBytes($message); @@ -125,7 +131,131 @@ abstract class DiffusionGitSSHWorkflow $message = $protocol->willReadBytes($message); } + // Note that bytes aren't counted until they're emittted by the protocol + // layer. This means the underlying command might emit bytes, but if they + // are buffered by the protocol layer they won't count as read bytes yet. + + $this->ioBytesRead += strlen($message); + return $message; } + final protected function getIOBytesRead() { + return $this->ioBytesRead; + } + + final protected function getIOBytesWritten() { + return $this->ioBytesWritten; + } + + final protected function executeRepositoryProxyOperations($for_write) { + $device = AlmanacKeys::getLiveDevice(); + + $refs = $this->getAlmanacServiceRefs($for_write); + $err = 1; + + while (true) { + $ref = head($refs); + + $command = $this->getProxyCommandForServiceRef($ref); + + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Request received by \"%s\", forwarding to cluster ". + "host \"%s\".\n", + $device->getName(), + $ref->getDeviceName())); + } + + $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); + + $future = id(new ExecFuture('%C', $command)) + ->setEnv($this->getEnvironment()); + + $this->didBeginRequest(); + + $err = $this->newPassthruCommand() + ->setIOChannel($this->getIOChannel()) + ->setCommandChannelFromExecFuture($future) + ->execute(); + + // TODO: Currently, when proxying, we do not write an event log on the + // proxy. Perhaps we should write a "proxy log". This is not very useful + // for statistics or auditing, but could be useful for diagnostics. + // Marking the proxy logs as proxied (and recording devicePHID on all + // logs) would make differentiating between these use cases easier. + + if (!$err) { + $this->waitForGitClient(); + return $err; + } + + // Throw away this service: the request failed and we're treating the + // failure as persistent, so we don't want to retry another request to + // the same host. + array_shift($refs); + + $should_retry = $this->shouldRetryRequest($refs); + if (!$should_retry) { + return $err; + } + + // If we haven't bailed out yet, we'll retry the request with the next + // service. + } + + throw new Exception(pht('Reached an unreachable place.')); + } + + private function didBeginRequest() { + $this->requestAttempts++; + return $this; + } + + private function shouldRetryRequest(array $remaining_refs) { + $this->requestFailures++; + + if ($this->requestFailures > $this->requestAttempts) { + throw new Exception( + pht( + "Workflow has recorded more failures than attempts; there is a ". + "missing call to \"didBeginRequest()\".\n")); + } + + if (!$remaining_refs) { + $this->writeClusterEngineLogMessage( + pht( + "# All available services failed to serve the request, ". + "giving up.\n")); + return false; + } + + $read_len = $this->getIOBytesRead(); + if ($read_len) { + $this->writeClusterEngineLogMessage( + pht( + "# Client already read from service (%s bytes), unable to retry.\n", + new PhutilNumber($read_len))); + return false; + } + + $write_len = $this->getIOBytesWritten(); + if ($write_len) { + $this->writeClusterEngineLogMessage( + pht( + "# Client already wrote to service (%s bytes), unable to retry.\n", + new PhutilNumber($write_len))); + return false; + } + + $this->writeClusterEngineLogMessage( + pht( + "# Service request failed, retrying (making attempt %s of %s).\n", + new PhutilNumber($this->requestAttempts + 1), + new PhutilNumber($this->requestAttempts + count($remaining_refs)))); + + return true; + } + } diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php index 7e1f4a4f33..57c43b5a12 100644 --- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php @@ -1,6 +1,7 @@ setName('git-upload-pack'); @@ -14,39 +15,33 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow { } protected function executeRepositoryOperations() { - $repository = $this->getRepository(); + $is_proxy = $this->shouldProxy(); + if ($is_proxy) { + return $this->executeRepositoryProxyOperations($for_write = false); + } + $viewer = $this->getSSHUser(); + $repository = $this->getRepository(); $device = AlmanacKeys::getLiveDevice(); $skip_sync = $this->shouldSkipReadSynchronization(); - $is_proxy = $this->shouldProxy(); - if ($is_proxy) { - $command = $this->getProxyCommand(false); + $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath()); + if (!$skip_sync) { + $cluster_engine = id(new DiffusionRepositoryClusterEngine()) + ->setViewer($viewer) + ->setRepository($repository) + ->setLog($this) + ->synchronizeWorkingCopyBeforeRead(); if ($device) { $this->writeClusterEngineLogMessage( pht( - "# Fetch received by \"%s\", forwarding to cluster host.\n", + "# Cleared to fetch on cluster host \"%s\".\n", $device->getName())); } - } else { - $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath()); - if (!$skip_sync) { - $cluster_engine = id(new DiffusionRepositoryClusterEngine()) - ->setViewer($viewer) - ->setRepository($repository) - ->setLog($this) - ->synchronizeWorkingCopyBeforeRead(); - - if ($device) { - $this->writeClusterEngineLogMessage( - pht( - "# Cleared to fetch on cluster host \"%s\".\n", - $device->getName())); - } - } } + $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); $pull_event = $this->newPullEvent(); @@ -60,14 +55,12 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow { $log->didStartSession($command); } - if (!$is_proxy) { - if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) { - $protocol = new DiffusionGitUploadPackWireProtocol(); - if ($log) { - $protocol->setProtocolLog($log); - } - $this->setWireProtocol($protocol); + if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) { + $protocol = new DiffusionGitUploadPackWireProtocol(); + if ($log) { + $protocol->setProtocolLog($log); } + $this->setWireProtocol($protocol); } $err = $this->newPassthruCommand() @@ -89,15 +82,7 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow { ->setResultCode(0); } - // TODO: Currently, when proxying, we do not write a log on the proxy. - // Perhaps we should write a "proxy log". This is not very useful for - // statistics or auditing, but could be useful for diagnostics. Marking - // the proxy logs as proxied (and recording devicePHID on all logs) would - // make differentiating between these use cases easier. - - if (!$is_proxy) { - $pull_event->save(); - } + $pull_event->save(); if (!$err) { $this->waitForGitClient(); diff --git a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php index 08144eb0c9..358418f44c 100644 --- a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php @@ -73,13 +73,13 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow { return $this->shouldProxy; } - protected function getProxyCommand($for_write) { + final protected function getAlmanacServiceRefs($for_write) { $viewer = $this->getSSHUser(); $repository = $this->getRepository(); $is_cluster_request = $this->getIsClusterRequest(); - $uri = $repository->getAlmanacServiceURI( + $refs = $repository->getAlmanacServiceRefs( $viewer, array( 'neverProxy' => $is_cluster_request, @@ -89,14 +89,28 @@ abstract class DiffusionSSHWorkflow extends PhabricatorSSHWorkflow { 'writable' => $for_write, )); - if (!$uri) { + if (!$refs) { throw new Exception( pht( 'Failed to generate an intracluster proxy URI even though this '. 'request was routed as a proxy request.')); } - $uri = new PhutilURI($uri); + return $refs; + } + + final protected function getProxyCommand($for_write) { + $refs = $this->getAlmanacServiceRefs($for_write); + + $ref = head($refs); + + return $this->getProxyCommandForServiceRef($ref); + } + + final protected function getProxyCommandForServiceRef( + DiffusionServiceRef $ref) { + + $uri = new PhutilURI($ref->getURI()); $username = AlmanacKeys::getClusterSSHUser(); if ($username === null) { diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index bd15e3e8de..fdc9a695c4 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1842,6 +1842,20 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO PhabricatorUser $viewer, array $options) { + $refs = $this->getAlmanacServiceRefs($viewer, $options); + + if (!$refs) { + return null; + } + + $ref = head($refs); + return $ref->getURI(); + } + + public function getAlmanacServiceRefs( + PhabricatorUser $viewer, + array $options) { + PhutilTypeSpec::checkMap( $options, array( @@ -1856,7 +1870,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $cache_key = $this->getAlmanacServiceCacheKey(); if (!$cache_key) { - return null; + return array(); } $cache = PhabricatorCaches::getMutableStructureCache(); @@ -1869,7 +1883,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } if ($uris === null) { - return null; + return array(); } $local_device = AlmanacKeys::getDeviceID(); @@ -1893,7 +1907,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO if ($local_device && $never_proxy) { if ($uri['device'] == $local_device) { - return null; + return array(); } } @@ -1954,15 +1968,20 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } } + $refs = array(); + foreach ($results as $result) { + $refs[] = DiffusionServiceRef::newFromDictionary($result); + } + // If we require a writable device, remove URIs which aren't writable. if ($writable) { - foreach ($results as $key => $uri) { - if (!$uri['writable']) { + foreach ($refs as $key => $ref) { + if (!$ref->isWritable()) { unset($results[$key]); } } - if (!$results) { + if (!$refs) { throw new Exception( pht( 'This repository ("%s") is not writable with the given '. @@ -1974,23 +1993,30 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } if ($writable) { - $results = $this->sortWritableAlmanacServiceURIs($results); + $refs = $this->sortWritableAlmanacServiceRefs($refs); } else { - shuffle($results); + $refs = $this->sortReadableAlmanacServiceRefs($refs); } - $result = head($results); - return $result['uri']; + return array_values($refs); } - private function sortWritableAlmanacServiceURIs(array $results) { + private function sortReadableAlmanacServiceRefs(array $refs) { + assert_instances_of($refs, 'DiffusionServiceRef'); + shuffle($refs); + return $refs; + } + + private function sortWritableAlmanacServiceRefs(array $refs) { + assert_instances_of($refs, 'DiffusionServiceRef'); + // See T13109 for discussion of how this method routes requests. // In the absence of other rules, we'll send traffic to devices randomly. // We also want to select randomly among nodes which are equally good // candidates to receive the write, and accomplish that by shuffling the // list up front. - shuffle($results); + shuffle($refs); $order = array(); @@ -2002,8 +2028,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO $this->getPHID()); if ($writer) { $device_phid = $writer->getWriteProperty('devicePHID'); - foreach ($results as $key => $result) { - if ($result['devicePHID'] === $device_phid) { + foreach ($refs as $key => $ref) { + if ($ref->getDevicePHID() === $device_phid) { $order[] = $key; } } @@ -2025,8 +2051,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } $max_devices = array_fuse($max_devices); - foreach ($results as $key => $result) { - if (isset($max_devices[$result['devicePHID']])) { + foreach ($refs as $key => $ref) { + if (isset($max_devices[$ref->getDevicePHID()])) { $order[] = $key; } } @@ -2034,9 +2060,9 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO // Reorder the results, putting any we've selected as preferred targets for // the write at the head of the list. - $results = array_select_keys($results, $order) + $results; + $refs = array_select_keys($refs, $order) + $refs; - return $results; + return $refs; } public function supportsSynchronization() { diff --git a/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php b/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php new file mode 100644 index 0000000000..37b0ea919d --- /dev/null +++ b/src/applications/settings/panel/PhabricatorSearchSettingsPanel.php @@ -0,0 +1,28 @@ +getViewer(), + new PhabricatorSettingsApplication()); + + $scope_map = array(); + foreach ($scopes as $scope) { + if (!isset($scope['value'])) { + continue; + } + $scope_map[$scope['value']] = $scope['name']; + } + + return $scope_map; + } + } diff --git a/src/applications/system/engine/PhabricatorSystemActionEngine.php b/src/applications/system/engine/PhabricatorSystemActionEngine.php index c097fa04a4..6d6f9eacfd 100644 --- a/src/applications/system/engine/PhabricatorSystemActionEngine.php +++ b/src/applications/system/engine/PhabricatorSystemActionEngine.php @@ -100,32 +100,34 @@ final class PhabricatorSystemActionEngine extends Phobject { $actor_hashes = array(); foreach ($actors as $actor) { - $actor_hashes[] = PhabricatorHash::digestForIndex($actor); + $digest = PhabricatorHash::digestForIndex($actor); + $actor_hashes[$digest] = $actor; } $log = new PhabricatorSystemActionLog(); $window = self::getWindow(); - $conn_r = $log->establishConnection('r'); - $scores = queryfx_all( - $conn_r, - 'SELECT actorIdentity, SUM(score) totalScore FROM %T + $conn = $log->establishConnection('r'); + + $rows = queryfx_all( + $conn, + 'SELECT actorHash, SUM(score) totalScore FROM %T WHERE action = %s AND actorHash IN (%Ls) AND epoch >= %d GROUP BY actorHash', $log->getTableName(), $action->getActionConstant(), - $actor_hashes, - (time() - $window)); + array_keys($actor_hashes), + (PhabricatorTime::getNow() - $window)); - $scores = ipull($scores, 'totalScore', 'actorIdentity'); + $rows = ipull($rows, 'totalScore', 'actorHash'); - foreach ($scores as $key => $score) { - $scores[$key] = $score / $window; + $scores = array(); + foreach ($actor_hashes as $digest => $actor) { + $score = idx($rows, $digest, 0); + $scores[$actor] = ($score / $window); } - $scores = $scores + array_fill_keys($actors, 0); - return $scores; } diff --git a/src/docs/user/configuration/configuring_preamble.diviner b/src/docs/user/configuration/configuring_preamble.diviner index fc804e9072..5299afa27d 100644 --- a/src/docs/user/configuration/configuring_preamble.diviner +++ b/src/docs/user/configuration/configuring_preamble.diviner @@ -15,10 +15,9 @@ You can use a special preamble script to make arbitrary adjustments to the environment and some parts of Phabricator's configuration in order to fix these problems and set up the environment which Phabricator expects. -NOTE: This is an advanced feature. Most installs should not need to configure -a preamble script. -= Creating a Preamble Script = +Creating a Preamble Script +========================== To create a preamble script, write a file to: @@ -37,6 +36,7 @@ If present, this script will be executed at the very beginning of each web request, allowing you to adjust the environment. For common adjustments and examples, see the next sections. + Adjusting Client IPs ==================== @@ -44,9 +44,15 @@ If your install is behind a load balancer, Phabricator may incorrectly detect all requests as originating from the load balancer, rather than from the correct client IPs. -If this is the case and some other header (like `X-Forwarded-For`) is known to -be trustworthy, you can read the header and overwrite the `REMOTE_ADDR` value -so Phabricator can figure out the client IP correctly. +In common cases where networks are configured like this, the `X-Forwarded-For` +header will have trustworthy information about the real client IP. You +can use the function `preamble_trust_x_forwarded_for_header()` in your +preamble to tell Phabricator that you expect to receive requests from a +load balancer or proxy which modifies this header: + +```name="Trust X-Forwarded-For Header", lang=php +preamble_trust_x_forwarded_for_header(); +``` You should do this //only// if the `X-Forwarded-For` header is known to be trustworthy. In particular, if users can make requests to the web server @@ -54,30 +60,29 @@ directly, they can provide an arbitrary `X-Forwarded-For` header, and thereby spoof an arbitrary client IP. The `X-Forwarded-For` header may also contain a list of addresses if a request -has been forwarded through multiple loadbalancers. Using a snippet like this -will usually handle most situations correctly: +has been forwarded through multiple load balancers. If you know that requests +on your network are routed through `N` trustworthy devices, you can specify +that `N` to tell the function how many layers of `X-Forwarded-For` to discard: +```name="Trust X-Forwarded-For Header, Multiple Layers", lang=php +preamble_trust_x_forwarded_for_header(3); ``` -name=Overwrite REMOTE_ADDR with X-Forwarded-For -replicaRefs; } + public function getDisplayName() { + return $this->getRefKey(); + } public function getRefKey() { $host = $this->getHost(); diff --git a/src/infrastructure/env/PhabricatorEnv.php b/src/infrastructure/env/PhabricatorEnv.php index ff6a3db04f..5e265b6dce 100644 --- a/src/infrastructure/env/PhabricatorEnv.php +++ b/src/infrastructure/env/PhabricatorEnv.php @@ -135,6 +135,11 @@ final class PhabricatorEnv extends Phobject { // TODO: Add a "locale.default" config option once we have some reasonable // defaults which aren't silly nonsense. self::setLocaleCode('en_US'); + + // Load the preamble utility library if we haven't already. On web + // requests this loaded earlier, but we want to load it for non-web + // requests so that unit tests can call these functions. + require_once $phabricator_path.'/support/startup/preamble-utils.php'; } public static function beginScopedLocale($locale_code) { @@ -249,9 +254,17 @@ final class PhabricatorEnv extends Phobject { } try { - $stack->pushSource( - id(new PhabricatorConfigDatabaseSource('default')) - ->setName(pht('Database'))); + // See T13403. If we're starting up in "config optional" mode, suppress + // messages about connection retries. + if ($config_optional) { + $database_source = @new PhabricatorConfigDatabaseSource('default'); + } else { + $database_source = new PhabricatorConfigDatabaseSource('default'); + } + + $database_source->setName(pht('Database')); + + $stack->pushSource($database_source); } catch (AphrontSchemaQueryException $exception) { // If the database is not available, just skip this configuration // source. This happens during `bin/storage upgrade`, `bin/conf` before diff --git a/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php new file mode 100644 index 0000000000..02fab53b93 --- /dev/null +++ b/src/infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php @@ -0,0 +1,107 @@ + array( + '[words].', + '[words].', + '[words].', + '[words]: [word], [word], [word] [word].', + '[words]; [lowerwords].', + '[words]!', + '[words], "[words]."', + '[words] ("[upperword] [upperword] [upperword]") [lowerwords].', + '[words]?', + ), + 'words' => array( + '[upperword] [lowerwords]', + ), + 'upperword' => array( + 'Lorem', + 'Ipsum', + 'Dolor', + 'Sit', + 'Amet', + ), + 'lowerwords' => array( + '[word]', + '[word] [word]', + '[word] [word] [word]', + '[word] [word] [word] [word]', + '[word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word] [word]', + '[word] [word] [word] [word] [word] [word]', + ), + 'word' => array( + 'ad', + 'adipisicing', + 'aliqua', + 'aliquip', + 'amet', + 'anim', + 'aute', + 'cillum', + 'commodo', + 'consectetur', + 'consequat', + 'culpa', + 'cupidatat', + 'deserunt', + 'do', + 'dolor', + 'dolore', + 'duis', + 'ea', + 'eiusmod', + 'elit', + 'enim', + 'esse', + 'est', + 'et', + 'eu', + 'ex', + 'excepteur', + 'exercitation', + 'fugiat', + 'id', + 'in', + 'incididunt', + 'ipsum', + 'irure', + 'labore', + 'laboris', + 'laborum', + 'lorem', + 'magna', + 'minim', + 'mollit', + 'nisi', + 'non', + 'nostrud', + 'nulla', + 'occaecat', + 'officia', + 'pariatur', + 'proident', + 'qui', + 'quis', + 'reprehenderit', + 'sed', + 'sint', + 'sit', + 'sunt', + 'tempor', + 'ullamco', + 'ut', + 'velit', + 'veniam', + 'voluptate', + ), + ); + } + +} diff --git a/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php new file mode 100644 index 0000000000..95777e8d30 --- /dev/null +++ b/src/infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php @@ -0,0 +1,155 @@ + array( + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]', + '[first] [last]-[last]', + '[first] [middle] [last]', + '[first] "[nick]" [last]', + '[first] [particle] [particle] [particle]', + ), + 'first' => array( + 'Mohamed', + 'Youssef', + 'Ahmed', + 'Mahmoud', + 'Mustafa', + 'Fatma', + 'Aya', + 'Noam', + 'Adam', + 'Lucas', + 'Noah', + 'Jakub', + 'Victor', + 'Harry', + 'Rasmus', + 'Nathan', + 'Emil', + 'Charlie', + 'Leon', + 'Dylan', + 'Alexander', + 'Emma', + 'Marie', + 'Lea', + 'Amelia', + 'Hanna', + 'Emily', + 'Sofia', + 'Julia', + 'Santiago', + 'Sebastian', + 'Olivia', + 'Madison', + 'Isabella', + 'Esther', + 'Anya', + 'Camila', + 'Jack', + 'Oliver', + ), + 'nick' => array( + 'Buzz', + 'Juggernaut', + 'Haze', + 'Hawk', + 'Iceman', + 'Killer', + 'Apex', + 'Ocelot', + ), + 'middle' => array( + 'Rose', + 'Grace', + 'Jane', + 'Louise', + 'Jade', + 'James', + 'John', + 'William', + 'Thomas', + 'Alexander', + ), + 'last' => array( + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + '[termlast]', + 'O\'[termlast]', + 'Mc[termlast]', + ), + 'termlast' => array( + 'Smith', + 'Johnson', + 'Williams', + 'Jones', + 'Brown', + 'Davis', + 'Miller', + 'Wilson', + 'Moore', + 'Taylor', + 'Anderson', + 'Thomas', + 'Jackson', + 'White', + 'Harris', + 'Martin', + 'Thompson', + 'Garcia', + 'Marinez', + 'Robinson', + 'Clark', + 'Rodrigues', + 'Lewis', + 'Lee', + 'Walker', + 'Hall', + 'Allen', + 'Young', + 'Hernandex', + 'King', + 'Wang', + 'Li', + 'Zhang', + 'Liu', + 'Chen', + 'Yang', + 'Huang', + 'Zhao', + 'Wu', + 'Zhou', + 'Xu', + 'Sun', + 'Ma', + ), + 'particle' => array( + 'Wu', + 'Xu', + 'Ma', + 'Li', + 'Liu', + 'Shao', + 'Lin', + 'Khan', + ), + ); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..baf665ef48 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilCLikeCodeSnippetContextFreeGrammar.php @@ -0,0 +1,254 @@ +getStmtTerminationGrammarSet(), + $this->getVarNameGrammarSet(), + $this->getNullExprGrammarSet(), + $this->getNumberGrammarSet(), + $this->getExprGrammarSet(), + $this->getCondGrammarSet(), + $this->getLoopGrammarSet(), + $this->getStmtGrammarSet(), + $this->getAssignmentGrammarSet(), + $this->getArithExprGrammarSet(), + $this->getBoolExprGrammarSet(), + $this->getBoolValGrammarSet(), + $this->getTernaryExprGrammarSet(), + + $this->getFuncNameGrammarSet(), + $this->getFuncCallGrammarSet(), + $this->getFuncCallParamGrammarSet(), + $this->getFuncDeclGrammarSet(), + $this->getFuncParamGrammarSet(), + $this->getFuncBodyGrammarSet(), + $this->getFuncReturnGrammarSet(), + ); + } + + protected function getStartGrammarSet() { + $start_grammar = parent::getStartGrammarSet(); + + $start_grammar['start'][] = '[funcdecl]'; + + return $start_grammar; + } + + protected function getStmtTerminationGrammarSet() { + return $this->buildGrammarSet('term', array(';')); + } + + protected function getFuncCallGrammarSet() { + return $this->buildGrammarSet('funccall', + array( + '[funcname]([funccallparam])', + )); + } + + protected function getFuncCallParamGrammarSet() { + return $this->buildGrammarSet('funccallparam', + array( + '', + '[expr]', + '[expr], [expr]', + )); + } + + protected function getFuncDeclGrammarSet() { + return $this->buildGrammarSet('funcdecl', + array( + 'function [funcname]([funcparam]) '. + '{[funcbody, indent, block, trim=right]}', + )); + } + + protected function getFuncParamGrammarSet() { + return $this->buildGrammarSet('funcparam', + array( + '', + '[varname]', + '[varname], [varname]', + '[varname], [varname], [varname]', + )); + } + + protected function getFuncBodyGrammarSet() { + return $this->buildGrammarSet('funcbody', + array( + "[stmt]\n[stmt]\n[funcreturn]", + "[stmt]\n[stmt]\n[stmt]\n[funcreturn]", + "[stmt]\n[stmt]\n[stmt]\n[stmt]\n[funcreturn]", + )); + } + + protected function getFuncReturnGrammarSet() { + return $this->buildGrammarSet('funcreturn', + array( + 'return [expr][term]', + '', + )); + } + + // Not really C, but put it here because of the curly braces and mostly shared + // among Java and PHP + protected function getClassDeclGrammarSet() { + return $this->buildGrammarSet('classdecl', + array( + '[classinheritancemod] class [classname] {[classbody, indent, block]}', + 'class [classname] {[classbody, indent, block]}', + )); + } + + protected function getClassNameGrammarSet() { + return $this->buildGrammarSet('classname', + array( + 'MuffinHouse', + 'MuffinReader', + 'MuffinAwesomizer', + 'SuperException', + 'Librarian', + 'Book', + 'Ball', + 'BallOfCode', + 'AliceAndBobsSharedSecret', + 'FileInputStream', + 'FileOutputStream', + 'BufferedReader', + 'BufferedWriter', + 'Cardigan', + 'HouseOfCards', + 'UmbrellaClass', + 'GenericThing', + )); + } + + protected function getClassBodyGrammarSet() { + return $this->buildGrammarSet('classbody', + array( + '[methoddecl]', + "[methoddecl]\n\n[methoddecl]", + "[propdecl]\n[propdecl]\n\n[methoddecl]\n\n[methoddecl]", + "[propdecl]\n[propdecl]\n[propdecl]\n\n[methoddecl]\n\n[methoddecl]". + "\n\n[methoddecl]", + )); + } + + protected function getVisibilityGrammarSet() { + return $this->buildGrammarSet('visibility', + array( + 'private', + 'protected', + 'public', + )); + } + + protected function getClassInheritanceModGrammarSet() { + return $this->buildGrammarSet('classinheritancemod', + array( + 'final', + 'abstract', + )); + } + + // Keeping this separate so we won't give abstract methods a function body + protected function getMethodInheritanceModGrammarSet() { + return $this->buildGrammarSet('methodinheritancemod', + array( + 'final', + )); + } + + protected function getMethodDeclGrammarSet() { + return $this->buildGrammarSet('methoddecl', + array( + '[visibility] [methodfuncdecl]', + '[visibility] [methodfuncdecl]', + '[methodinheritancemod] [visibility] [methodfuncdecl]', + '[abstractmethoddecl]', + )); + } + + protected function getMethodFuncDeclGrammarSet() { + return $this->buildGrammarSet('methodfuncdecl', + array( + 'function [funcname]([funcparam]) '. + '{[methodbody, indent, block, trim=right]}', + )); + } + + protected function getMethodBodyGrammarSet() { + return $this->buildGrammarSet('methodbody', + array( + "[methodstmt]\n[methodbody]", + "[methodstmt]\n[funcreturn]", + )); + } + + protected function getMethodStmtGrammarSet() { + $stmts = $this->getStmtGrammarSet(); + + return $this->buildGrammarSet('methodstmt', + array_merge( + $stmts['stmt'], + array( + '[methodcall][term]', + ))); + } + + protected function getMethodCallGrammarSet() { + // Java/JavaScript + return $this->buildGrammarSet('methodcall', + array( + 'this.[funccall]', + '[varname].[funccall]', + '[classname].[funccall]', + )); + } + + protected function getAbstractMethodDeclGrammarSet() { + return $this->buildGrammarSet('abstractmethoddecl', + array( + 'abstract function [funcname]([funcparam])[term]', + )); + } + + protected function getPropDeclGrammarSet() { + return $this->buildGrammarSet('propdecl', + array( + '[visibility] [varname][term]', + )); + } + + protected function getClassRuleSets() { + return array( + $this->getClassInheritanceModGrammarSet(), + $this->getMethodInheritanceModGrammarSet(), + $this->getClassDeclGrammarSet(), + $this->getClassNameGrammarSet(), + $this->getClassBodyGrammarSet(), + $this->getMethodDeclGrammarSet(), + $this->getMethodFuncDeclGrammarSet(), + $this->getMethodBodyGrammarSet(), + $this->getMethodStmtGrammarSet(), + $this->getMethodCallGrammarSet(), + $this->getAbstractMethodDeclGrammarSet(), + $this->getPropDeclGrammarSet(), + $this->getVisibilityGrammarSet(), + ); + } + + public function generateClass() { + $rules = array_merge($this->getRules(), $this->getClassRuleSets()); + $rules['start'] = array('[classdecl]'); + $count = 0; + return $this->applyRules('[start]', $count, $rules); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..b3fe621591 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php @@ -0,0 +1,205 @@ +getStartGrammarSet(), + $this->getStmtGrammarSet(), + array_mergev($this->buildRuleSet())); + } + + abstract protected function buildRuleSet(); + + protected function buildGrammarSet($name, array $set) { + return array( + $name => $set, + ); + } + + protected function getStartGrammarSet() { + return $this->buildGrammarSet('start', + array( + "[stmt]\n[stmt]", + "[stmt]\n[stmt]\n[stmt]", + "[stmt]\n[stmt]\n[stmt]\n[stmt]", + )); + } + + protected function getStmtGrammarSet() { + return $this->buildGrammarSet('stmt', + array( + '[assignment][term]', + '[assignment][term]', + '[assignment][term]', + '[assignment][term]', + '[funccall][term]', + '[funccall][term]', + '[funccall][term]', + '[funccall][term]', + '[cond]', + '[loop]', + )); + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array( + 'do_something', + 'nonempty', + 'noOp', + 'call_user_func', + 'getenv', + 'render', + 'super', + 'derpify', + 'awesomize', + 'equals', + 'run', + 'flee', + 'fight', + 'notify', + 'listen', + 'calculate', + 'aim', + 'open', + )); + } + + protected function getVarNameGrammarSet() { + return $this->buildGrammarSet('varname', + array( + 'is_something', + 'object', + 'name', + 'token', + 'label', + 'piece_of_the_pie', + 'type', + 'state', + 'param', + 'action', + 'key', + 'timeout', + 'result', + )); + } + + protected function getNullExprGrammarSet() { + return $this->buildGrammarSet('null', array('null')); + } + + protected function getNumberGrammarSet() { + return $this->buildGrammarSet('number', + array( + mt_rand(-1, 100), + mt_rand(-100, 1000), + mt_rand(-1000, 5000), + mt_rand(0, 1).'.'.mt_rand(1, 1000), + mt_rand(0, 50).'.'.mt_rand(1, 1000), + )); + } + + protected function getExprGrammarSet() { + return $this->buildGrammarSet('expr', + array( + '[null]', + '[number]', + '[number]', + '[varname]', + '[varname]', + '[boolval]', + '[boolval]', + '[boolexpr]', + '[boolexpr]', + '[funccall]', + '[arithexpr]', + '[arithexpr]', + // Some random strings + '"'.Filesystem::readRandomCharacters(4).'"', + '"'.Filesystem::readRandomCharacters(5).'"', + )); + } + + protected function getBoolExprGrammarSet() { + return $this->buildGrammarSet('boolexpr', + array( + '[varname]', + '![varname]', + '[varname] == [boolval]', + '[varname] != [boolval]', + '[ternary]', + )); + } + + protected function getBoolValGrammarSet() { + return $this->buildGrammarSet('boolval', + array( + 'true', + 'false', + )); + } + + protected function getArithExprGrammarSet() { + return $this->buildGrammarSet('arithexpr', + array( + '[varname]++', + '++[varname]', + '[varname] + [number]', + '[varname]--', + '--[varname]', + '[varname] - [number]', + )); + } + + protected function getAssignmentGrammarSet() { + return $this->buildGrammarSet('assignment', + array( + '[varname] = [expr]', + '[varname] = [arithexpr]', + '[varname] += [expr]', + )); + } + + protected function getCondGrammarSet() { + return $this->buildGrammarSet('cond', + array( + 'if ([boolexpr]) {[stmt, indent, block]}', + 'if ([boolexpr]) {[stmt, indent, block]} else {[stmt, indent, block]}', + )); + } + + protected function getLoopGrammarSet() { + return $this->buildGrammarSet('loop', + array( + 'while ([boolexpr]) {[stmt, indent, block]}', + 'do {[stmt, indent, block]} while ([boolexpr])[term]', + 'for ([assignment]; [boolexpr]; [expr]) {[stmt, indent, block]}', + )); + } + + protected function getTernaryExprGrammarSet() { + return $this->buildGrammarSet('ternary', + array( + '[boolexpr] ? [expr] : [expr]', + )); + } + + protected function getStmtTerminationGrammarSet() { + return $this->buildGrammarSet('term', array('')); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..28a6990e77 --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php @@ -0,0 +1,184 @@ +getClassRuleSets()); + + $rulesset[] = $this->getTypeNameGrammarSet(); + $rulesset[] = $this->getNamespaceDeclGrammarSet(); + $rulesset[] = $this->getNamespaceNameGrammarSet(); + $rulesset[] = $this->getImportGrammarSet(); + $rulesset[] = $this->getMethodReturnTypeGrammarSet(); + $rulesset[] = $this->getMethodNameGrammarSet(); + $rulesset[] = $this->getVarDeclGrammarSet(); + $rulesset[] = $this->getClassDerivGrammarSet(); + + return $rulesset; + } + + protected function getStartGrammarSet() { + return $this->buildGrammarSet('start', + array( + '[import, block][nmspdecl, block][classdecl, block]', + )); + } + + protected function getClassDeclGrammarSet() { + return $this->buildGrammarSet('classdecl', + array( + '[classinheritancemod] [visibility] class [classname][classderiv] '. + '{[classbody, indent, block]}', + '[visibility] class [classname][classderiv] '. + '{[classbody, indent, block]}', + )); + } + + protected function getClassDerivGrammarSet() { + return $this->buildGrammarSet('classderiv', + array( + ' extends [classname]', + '', + '', + )); + } + + protected function getTypeNameGrammarSet() { + return $this->buildGrammarSet('type', + array( + 'int', + 'boolean', + 'char', + 'short', + 'long', + 'float', + 'double', + '[classname]', + '[type][]', + )); + } + + protected function getMethodReturnTypeGrammarSet() { + return $this->buildGrammarSet('methodreturn', + array( + '[type]', + 'void', + )); + } + + protected function getNamespaceDeclGrammarSet() { + return $this->buildGrammarSet('nmspdecl', + array( + 'package [nmspname][term]', + )); + } + + protected function getNamespaceNameGrammarSet() { + return $this->buildGrammarSet('nmspname', + array( + 'java.lang', + 'java.io', + 'com.example.proj.std', + 'derp.example.www', + )); + } + + protected function getImportGrammarSet() { + return $this->buildGrammarSet('import', + array( + 'import [nmspname][term]', + 'import [nmspname].*[term]', + 'import [nmspname].[classname][term]', + )); + } + + protected function getExprGrammarSet() { + $expr = parent::getExprGrammarSet(); + + $expr['expr'][] = 'new [classname]([funccallparam])'; + + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + $expr['expr'][] = '[methodcall]'; + + // Add some 'char's + for ($ii = 0; $ii < 2; $ii++) { + $expr['expr'][] = "'".Filesystem::readRandomCharacters(1)."'"; + } + + return $expr; + } + + protected function getStmtGrammarSet() { + $stmt = parent::getStmtGrammarSet(); + + $stmt['stmt'][] = '[vardecl]'; + $stmt['stmt'][] = '[vardecl]'; + // `try` to `throw` a `Ball`! + $stmt['stmt'][] = 'throw [classname][term]'; + + return $stmt; + } + + protected function getPropDeclGrammarSet() { + return $this->buildGrammarSet('propdecl', + array( + '[visibility] [type] [varname][term]', + )); + } + + protected function getVarDeclGrammarSet() { + return $this->buildGrammarSet('vardecl', + array( + '[type] [varname][term]', + '[type] [assignment][term]', + )); + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array( + '[methodname]', + '[classname].[methodname]', + // This is just silly (too much recursion) + // '[classname].[funcname]', + // Don't do this for now, it just clutters up output (thanks to rec.) + // '[nmspname].[classname].[methodname]', + )); + } + + // Renamed from `funcname` + protected function getMethodNameGrammarSet() { + $funcnames = head(parent::getFuncNameGrammarSet()); + return $this->buildGrammarSet('methodname', $funcnames); + } + + protected function getMethodFuncDeclGrammarSet() { + return $this->buildGrammarSet('methodfuncdecl', + array( + '[methodreturn] [methodname]([funcparam]) '. + '{[methodbody, indent, block, trim=right]}', + )); + } + + protected function getFuncParamGrammarSet() { + return $this->buildGrammarSet('funcparam', + array( + '', + '[type] [varname]', + '[type] [varname], [type] [varname]', + '[type] [varname], [type] [varname], [type] [varname]', + )); + } + + protected function getAbstractMethodDeclGrammarSet() { + return $this->buildGrammarSet('abstractmethoddecl', + array( + 'abstract [methodreturn] [methodname]([funcparam])[term]', + )); + } + +} diff --git a/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php b/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php new file mode 100644 index 0000000000..a8f83bab2d --- /dev/null +++ b/src/infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php @@ -0,0 +1,57 @@ +getClassRuleSets()); + } + + protected function getStartGrammarSet() { + $start_grammar = parent::getStartGrammarSet(); + + $start_grammar['start'][] = '[classdecl]'; + $start_grammar['start'][] = '[classdecl]'; + + return $start_grammar; + } + + protected function getExprGrammarSet() { + $expr = parent::getExprGrammarSet(); + + $expr['expr'][] = 'new [classname]([funccallparam])'; + + $expr['expr'][] = '[classname]::[funccall]'; + + return $expr; + } + + protected function getVarNameGrammarSet() { + $varnames = parent::getVarNameGrammarSet(); + + foreach ($varnames as $vn_key => $vn_val) { + foreach ($vn_val as $vv_key => $vv_value) { + $varnames[$vn_key][$vv_key] = '$'.$vv_value; + } + } + + return $varnames; + } + + protected function getFuncNameGrammarSet() { + return $this->buildGrammarSet('funcname', + array_mergev(get_defined_functions())); + } + + protected function getMethodCallGrammarSet() { + return $this->buildGrammarSet('methodcall', + array( + '$this->[funccall]', + 'self::[funccall]', + 'static::[funccall]', + '[varname]->[funccall]', + '[classname]::[funccall]', + )); + } + +} diff --git a/src/infrastructure/markup/PhutilRemarkupBlockStorage.php b/src/infrastructure/markup/PhutilRemarkupBlockStorage.php new file mode 100644 index 0000000000..2b97c13a19 --- /dev/null +++ b/src/infrastructure/markup/PhutilRemarkupBlockStorage.php @@ -0,0 +1,176 @@ +". The first + * byte, "<0x01>" is a single byte with value 1 that marks a token. If this is + * token ID "444", the text may now look like this: + * + * //<0x01>444Z// + * + * Now the italics match and are replaced, using the next token ID: + * + * <0x01>445Z + * + * When processing completes, all the tokens are replaced with their final + * equivalents. For example, token 444 is evaluated to: + * + * ... + * + * Then token 445 is evaluated: + * + * <0x01>444Z + * + * ...and all tokens it contains are replaced: + * + * ... + * + * If we didn't do this, the italics rule could match the "//" in "http://", + * or any other number of processing mistakes could occur, some of which create + * security risks. + * + * This class generates keys, and stores the map of keys to replacement text. + */ +final class PhutilRemarkupBlockStorage extends Phobject { + + const MAGIC_BYTE = "\1"; + + private $map = array(); + private $index = 0; + + public function store($text) { + $key = self::MAGIC_BYTE.(++$this->index).'Z'; + $this->map[$key] = $text; + return $key; + } + + public function restore($corpus, $text_mode = false) { + $map = $this->map; + + if (!$text_mode) { + foreach ($map as $key => $content) { + $map[$key] = phutil_escape_html($content); + } + $corpus = phutil_escape_html($corpus); + } + + // NOTE: Tokens may contain other tokens: for example, a table may have + // links inside it. So we can't do a single simple find/replace, because + // we need to find and replace child tokens inside the content of parent + // tokens. + + // However, we know that rules which have child tokens must always store + // all their child tokens first, before they store their parent token: you + // have to pass the "store(text)" API a block of text with tokens already + // in it, so you must have created child tokens already. + + // Thus, all child tokens will appear in the list before parent tokens, so + // if we start at the beginning of the list and replace all the tokens we + // find in each piece of content, we'll end up expanding all subtokens + // correctly. + + $map[] = $corpus; + $seen = array(); + foreach ($map as $key => $content) { + $seen[$key] = true; + + // If the content contains no token magic, we don't need to replace + // anything. + if (strpos($content, self::MAGIC_BYTE) === false) { + continue; + } + + $matches = null; + preg_match_all( + '/'.self::MAGIC_BYTE.'\d+Z/', + $content, + $matches, + PREG_OFFSET_CAPTURE); + + $matches = $matches[0]; + + // See PHI1114. We're replacing all the matches in one pass because this + // is significantly faster than doing "substr_replace()" in a loop if the + // corpus is large and we have a large number of matches. + + // Build a list of string pieces in "$parts" by interleaving the + // plain strings between each token and the replacement token text, then + // implode the whole thing when we're done. + + $parts = array(); + $pos = 0; + foreach ($matches as $next) { + $subkey = $next[0]; + + // If we've matched a token pattern but don't actually have any + // corresponding token, just skip this match. This should not be + // possible, and should perhaps be an error. + if (!isset($seen[$subkey])) { + if (!isset($map[$subkey])) { + throw new Exception( + pht( + 'Matched token key "%s" while processing remarkup block, but '. + 'this token does not exist in the token map.', + $subkey)); + } else { + throw new Exception( + pht( + 'Matched token key "%s" while processing remarkup block, but '. + 'this token appears later in the list than the key being '. + 'processed ("%s").', + $subkey, + $key)); + } + } + + $subpos = $next[1]; + + // If there were any non-token bytes since the last token, add them. + if ($subpos > $pos) { + $parts[] = substr($content, $pos, $subpos - $pos); + } + + // Add the token replacement text. + $parts[] = $map[$subkey]; + + // Move the non-token cursor forward over the token. + $pos = $subpos + strlen($subkey); + } + + // Add any leftover non-token bytes after the last token. + $parts[] = substr($content, $pos); + + $content = implode('', $parts); + + $map[$key] = $content; + } + $corpus = last($map); + + if (!$text_mode) { + $corpus = phutil_safe_html($corpus); + } + + return $corpus; + } + + public function overwrite($key, $new_text) { + $this->map[$key] = $new_text; + return $this; + } + + public function getMap() { + return $this->map; + } + + public function setMap(array $map) { + $this->map = $map; + return $this; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php new file mode 100644 index 0000000000..3b02c87152 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php @@ -0,0 +1,36 @@ +engine = $engine; + return $this; + } + + final public function getEngine() { + return $this->engine; + } + + /** + * @return string + */ + abstract public function getInterpreterName(); + + abstract public function markupContent($content, array $argv); + + protected function markupError($string) { + if ($this->getEngine()->isTextMode()) { + return '('.$string.')'; + } else { + return phutil_tag( + 'div', + array( + 'class' => 'remarkup-interpreter-error', + ), + $string); + } + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php new file mode 100644 index 0000000000..feac6cffa9 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupBlockRule.php @@ -0,0 +1,170 @@ +addInt($this->getPriority()) + ->addString(get_class($this)); + } + + abstract public function markupText($text, $children); + + /** + * This will get an array of unparsed lines and return the number of lines + * from the first array value that it can parse. + * + * @param array $lines + * @param int $cursor + * + * @return int + */ + abstract public function getMatchingLineCount(array $lines, $cursor); + + protected function didMarkupText() { + return; + } + + final public function setEngine(PhutilRemarkupEngine $engine) { + $this->engine = $engine; + $this->updateRules(); + return $this; + } + + final protected function getEngine() { + return $this->engine; + } + + public function setMarkupRules(array $rules) { + assert_instances_of($rules, 'PhutilRemarkupRule'); + $this->rules = $rules; + $this->updateRules(); + return $this; + } + + private function updateRules() { + $engine = $this->getEngine(); + if ($engine) { + $this->rules = msort($this->rules, 'getPriority'); + foreach ($this->rules as $rule) { + $rule->setEngine($engine); + } + } + return $this; + } + + final public function getMarkupRules() { + return $this->rules; + } + + final public function postprocess() { + $this->didMarkupText(); + } + + final protected function applyRules($text) { + foreach ($this->getMarkupRules() as $rule) { + $text = $rule->apply($text); + } + return $text; + } + + public function supportsChildBlocks() { + return false; + } + + public function extractChildText($text) { + throw new PhutilMethodNotImplementedException(); + } + + protected function renderRemarkupTable(array $out_rows) { + assert_instances_of($out_rows, 'array'); + + if ($this->getEngine()->isTextMode()) { + $lengths = array(); + foreach ($out_rows as $r => $row) { + foreach ($row['content'] as $c => $cell) { + $text = $this->getEngine()->restoreText($cell['content']); + $lengths[$c][$r] = phutil_utf8_strlen($text); + } + } + $max_lengths = array_map('max', $lengths); + + $out = array(); + foreach ($out_rows as $r => $row) { + $headings = false; + foreach ($row['content'] as $c => $cell) { + $length = $max_lengths[$c] - $lengths[$c][$r]; + $out[] = '| '.$cell['content'].str_repeat(' ', $length).' '; + if ($cell['type'] == 'th') { + $headings = true; + } + } + $out[] = "|\n"; + + if ($headings) { + foreach ($row['content'] as $c => $cell) { + $char = ($cell['type'] == 'th' ? '-' : ' '); + $out[] = '| '.str_repeat($char, $max_lengths[$c]).' '; + } + $out[] = "|\n"; + } + } + + return rtrim(implode('', $out), "\n"); + } + + if ($this->getEngine()->isHTMLMailMode()) { + $table_attributes = array( + 'style' => 'border-collapse: separate; + border-spacing: 1px; + background: #d3d3d3; + margin: 12px 0;', + ); + $cell_attributes = array( + 'style' => 'background: #ffffff; + padding: 3px 6px;', + ); + } else { + $table_attributes = array( + 'class' => 'remarkup-table', + ); + $cell_attributes = array(); + } + + $out = array(); + $out[] = "\n"; + foreach ($out_rows as $row) { + $cells = array(); + foreach ($row['content'] as $cell) { + $cells[] = phutil_tag( + $cell['type'], + $cell_attributes, + $cell['content']); + } + $out[] = phutil_tag($row['type'], array(), $cells); + $out[] = "\n"; + } + + $table = phutil_tag('table', $table_attributes, $out); + return phutil_tag_div('remarkup-table-wrap', $table); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php new file mode 100644 index 0000000000..b8fef85b98 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupCodeBlockRule.php @@ -0,0 +1,252 @@ + false, + 'lang' => null, + 'name' => null, + 'lines' => null, + ); + + $parser = new PhutilSimpleOptions(); + $custom = $parser->parse(head($lines)); + if ($custom) { + $valid = true; + foreach ($custom as $key => $value) { + if (!array_key_exists($key, $options)) { + $valid = false; + break; + } + } + if ($valid) { + array_shift($lines); + $options = $custom + $options; + } + } + + // Normalize the text back to a 0-level indent. + $min_indent = 80; + foreach ($lines as $line) { + for ($ii = 0; $ii < strlen($line); $ii++) { + if ($line[$ii] != ' ') { + $min_indent = min($ii, $min_indent); + break; + } + } + } + + $text = implode("\n", $lines); + if ($min_indent) { + $indent_string = str_repeat(' ', $min_indent); + $text = preg_replace('/^'.$indent_string.'/m', '', $text); + } + + if ($this->getEngine()->isTextMode()) { + $out = array(); + + $header = array(); + if ($options['counterexample']) { + $header[] = 'counterexample'; + } + if ($options['name'] != '') { + $header[] = 'name='.$options['name']; + } + if ($header) { + $out[] = implode(', ', $header); + } + + $text = preg_replace('/^/m', ' ', $text); + $out[] = $text; + + return implode("\n", $out); + } + + if (empty($options['lang'])) { + // If the user hasn't specified "lang=..." explicitly, try to guess the + // language. If we fail, fall back to configured defaults. + $lang = PhutilLanguageGuesser::guessLanguage($text); + if (!$lang) { + $lang = nonempty( + $this->getEngine()->getConfig('phutil.codeblock.language-default'), + 'text'); + } + $options['lang'] = $lang; + } + + $code_body = $this->highlightSource($text, $options); + + $name_header = null; + $block_style = null; + if ($this->getEngine()->isHTMLMailMode()) { + $map = $this->getEngine()->getConfig('phutil.codeblock.style-map'); + + if ($map) { + $raw_body = id(new PhutilPygmentizeParser()) + ->setMap($map) + ->parse((string)$code_body); + $code_body = phutil_safe_html($raw_body); + } + + $style_rules = array( + 'padding: 6px 12px;', + 'font-size: 13px;', + 'font-weight: bold;', + 'display: inline-block;', + 'border-top-left-radius: 3px;', + 'border-top-right-radius: 3px;', + 'color: rgba(0,0,0,.75);', + ); + + if ($options['counterexample']) { + $style_rules[] = 'background: #f7e6e6'; + } else { + $style_rules[] = 'background: rgba(71, 87, 120, 0.08);'; + } + + $header_attributes = array( + 'style' => implode(' ', $style_rules), + ); + + $block_style = 'margin: 12px 0;'; + } else { + $header_attributes = array( + 'class' => 'remarkup-code-header', + ); + } + + if ($options['name']) { + $name_header = phutil_tag( + 'div', + $header_attributes, + $options['name']); + } + + $class = 'remarkup-code-block'; + if ($options['counterexample']) { + $class = 'remarkup-code-block code-block-counterexample'; + } + + $attributes = array( + 'class' => $class, + 'style' => $block_style, + 'data-code-lang' => $options['lang'], + 'data-sigil' => 'remarkup-code-block', + ); + + return phutil_tag( + 'div', + $attributes, + array($name_header, $code_body)); + } + + private function highlightSource($text, array $options) { + if ($options['counterexample']) { + $aux_class = ' remarkup-counterexample'; + } else { + $aux_class = null; + } + + $aux_style = null; + + if ($this->getEngine()->isHTMLMailMode()) { + $aux_style = array( + 'font: 11px/15px "Menlo", "Consolas", "Monaco", monospace;', + 'padding: 12px;', + 'margin: 0;', + ); + + if ($options['counterexample']) { + $aux_style[] = 'background: #f7e6e6;'; + } else { + $aux_style[] = 'background: rgba(71, 87, 120, 0.08);'; + } + + $aux_style = implode(' ', $aux_style); + } + + if ($options['lines']) { + // Put a minimum size on this because the scrollbar is otherwise + // unusable. + $height = max(6, (int)$options['lines']); + $aux_style = $aux_style + .' ' + .'max-height: ' + .(2 * $height) + .'em; overflow: auto;'; + } + + $engine = $this->getEngine()->getConfig('syntax-highlighter.engine'); + if (!$engine) { + $engine = 'PhutilDefaultSyntaxHighlighterEngine'; + } + $engine = newv($engine, array()); + $engine->setConfig( + 'pygments.enabled', + $this->getEngine()->getConfig('pygments.enabled')); + return phutil_tag( + 'pre', + array( + 'class' => 'remarkup-code'.$aux_class, + 'style' => $aux_style, + ), + PhutilSafeHTML::applyFunction( + 'rtrim', + $engine->highlightSource($options['lang'], $text))); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php new file mode 100644 index 0000000000..38de7e16b4 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupDefaultBlockRule.php @@ -0,0 +1,44 @@ +getEngine(); + + $text = trim($text); + $text = $this->applyRules($text); + + if ($engine->isTextMode()) { + if (!$this->getEngine()->getConfig('preserve-linebreaks')) { + $text = preg_replace('/ *\n */', ' ', $text); + } + return $text; + } + + if ($engine->getConfig('preserve-linebreaks')) { + $text = phutil_escape_html_newlines($text); + } + + if (!strlen($text)) { + return null; + } + + $default_attributes = $engine->getConfig('default.p.attributes'); + if ($default_attributes) { + $attributes = $default_attributes; + } else { + $attributes = array(); + } + + return phutil_tag('p', $attributes, $text); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php new file mode 100644 index 0000000000..cbcb143339 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupHeaderBlockRule.php @@ -0,0 +1,162 @@ + 1) { + $level = ($lines[1][0] == '=') ? 1 : 2; + $text = trim($lines[0]); + } else { + $level = 0; + for ($ii = 0; $ii < min(5, strlen($text)); $ii++) { + if ($text[$ii] == '=' || $text[$ii] == '#') { + ++$level; + } else { + break; + } + } + $text = trim($text, ' =#'); + } + + $engine = $this->getEngine(); + + if ($engine->isTextMode()) { + $char = ($level == 1) ? '=' : '-'; + return $text."\n".str_repeat($char, phutil_utf8_strlen($text)); + } + + $use_anchors = $engine->getConfig('header.generate-toc'); + + $anchor = null; + if ($use_anchors) { + $anchor = $this->generateAnchor($level, $text); + } + + $text = phutil_tag( + 'h'.($level + 1), + array( + 'class' => 'remarkup-header', + ), + array($anchor, $this->applyRules($text))); + + return $text; + } + + private function generateAnchor($level, $text) { + $anchor = strtolower($text); + $anchor = preg_replace('/[^a-z0-9]/', '-', $anchor); + $anchor = preg_replace('/--+/', '-', $anchor); + $anchor = trim($anchor, '-'); + $anchor = substr($anchor, 0, 24); + $anchor = trim($anchor, '-'); + $base = $anchor; + + $key = self::KEY_HEADER_TOC; + $engine = $this->getEngine(); + $anchors = $engine->getTextMetadata($key, array()); + + $suffix = 1; + while (!strlen($anchor) || isset($anchors[$anchor])) { + $anchor = $base.'-'.$suffix; + $anchor = trim($anchor, '-'); + $suffix++; + } + + // When a document contains a link inside a header, like this: + // + // = [[ http://wwww.example.com/ | example ]] = + // + // ...we want to generate a TOC entry with just "example", but link the + // header itself. We push the 'toc' state so all the link rules generate + // just names. + $engine->pushState('toc'); + $text = $this->applyRules($text); + $text = $engine->restoreText($text); + + $anchors[$anchor] = array($level, $text); + $engine->popState('toc'); + + $engine->setTextMetadata($key, $anchors); + + return phutil_tag( + 'a', + array( + 'name' => $anchor, + ), + ''); + } + + public static function renderTableOfContents(PhutilRemarkupEngine $engine) { + + $key = self::KEY_HEADER_TOC; + $anchors = $engine->getTextMetadata($key, array()); + + if (count($anchors) < 2) { + // Don't generate a TOC if there are no headers, or if there's only + // one header (since such a TOC would be silly). + return null; + } + + $depth = 0; + $toc = array(); + foreach ($anchors as $anchor => $info) { + list($level, $name) = $info; + + while ($depth < $level) { + $toc[] = hsprintf('
    '); + $depth++; + } + while ($depth > $level) { + $toc[] = hsprintf('
'); + $depth--; + } + + $toc[] = phutil_tag( + 'li', + array(), + phutil_tag( + 'a', + array( + 'href' => '#'.$anchor, + ), + $name)); + } + while ($depth > 0) { + $toc[] = hsprintf(''); + $depth--; + } + + return phutil_implode_html("\n", $toc); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php new file mode 100644 index 0000000000..7b815b2846 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php @@ -0,0 +1,37 @@ +getEngine()->isTextMode()) { + return rtrim($text); + } + + return phutil_tag('hr', array('class' => 'remarkup-hr')); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php new file mode 100644 index 0000000000..3a72197341 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupInlineBlockRule.php @@ -0,0 +1,13 @@ +applyRules($text); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php new file mode 100644 index 0000000000..a54e6b8b13 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupInterpreterBlockRule.php @@ -0,0 +1,89 @@ +parse($matches[2]); + } + + $interpreters = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhutilRemarkupBlockInterpreter') + ->execute(); + + foreach ($interpreters as $interpreter) { + $interpreter->setEngine($this->getEngine()); + } + + $lines[$first_key] = preg_replace( + self::START_BLOCK_PATTERN, + '', + $lines[$first_key]); + $lines[$last_key] = preg_replace( + self::END_BLOCK_PATTERN, + '', + $lines[$last_key]); + + if (trim($lines[$first_key]) === '') { + unset($lines[$first_key]); + } + if (trim($lines[$last_key]) === '') { + unset($lines[$last_key]); + } + + $content = implode("\n", $lines); + + $interpreters = mpull($interpreters, null, 'getInterpreterName'); + + if (isset($interpreters[$matches[1]])) { + return $interpreters[$matches[1]]->markupContent($content, $argv); + } + + $message = pht('No interpreter found: %s', $matches[1]); + + if ($this->getEngine()->isTextMode()) { + return '('.$message.')'; + } + + return phutil_tag( + 'div', + array( + 'class' => 'remarkup-interpreter-error', + ), + $message); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php new file mode 100644 index 0000000000..5bdcab2b8f --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupListBlockRule.php @@ -0,0 +1,567 @@ + $line) { + $matches = null; + if (preg_match($regex, $line)) { + $regex = self::CONT_BLOCK_PATTERN; + if (preg_match('/^(\s+)/', $line, $matches)) { + $space = strlen($matches[1]); + } else { + $space = 0; + } + $min_space = min($min_space, $space); + } + } + + $regex = self::START_BLOCK_PATTERN; + if ($min_space) { + foreach ($lines as $key => $line) { + if (preg_match($regex, $line)) { + $regex = self::CONT_BLOCK_PATTERN; + $lines[$key] = substr($line, $min_space); + } + } + } + + + // The input text may have linewraps in it, like this: + // + // - derp derp derp derp + // derp derp derp derp + // - blarp blarp blarp blarp + // + // Group text lines together into list items, stored in $items. So the + // result in the above case will be: + // + // array( + // array( + // "- derp derp derp derp", + // " derp derp derp derp", + // ), + // array( + // "- blarp blarp blarp blarp", + // ), + // ); + + $item = array(); + $starts_at = null; + $regex = self::START_BLOCK_PATTERN; + foreach ($lines as $line) { + $match = null; + if (preg_match($regex, $line, $match)) { + if (!$starts_at && !empty($match[1])) { + $starts_at = $match[1]; + } + $regex = self::CONT_BLOCK_PATTERN; + if ($item) { + $items[] = $item; + $item = array(); + } + } + $item[] = $line; + } + if ($item) { + $items[] = $item; + } + if (!$starts_at) { + $starts_at = 1; + } + + + // Process each item to normalize the text, remove line wrapping, and + // determine its depth (indentation level) and style (ordered vs unordered). + // + // We preserve consecutive linebreaks and interpret them as paragraph + // breaks. + // + // Given the above example, the processed array will look like: + // + // array( + // array( + // 'text' => 'derp derp derp derp derp derp derp derp', + // 'depth' => 0, + // 'style' => '-', + // ), + // array( + // 'text' => 'blarp blarp blarp blarp', + // 'depth' => 0, + // 'style' => '-', + // ), + // ); + + $has_marks = false; + foreach ($items as $key => $item) { + // Trim space around newlines, to strip trailing whitespace and formatting + // indentation. + $item = preg_replace('/ *(\n+) */', '\1', implode("\n", $item)); + + // Replace single newlines with a space. Preserve multiple newlines as + // paragraph breaks. + $item = preg_replace('/(? $text, + 'depth' => $depth, + 'style' => $style, + 'mark' => $mark, + ); + } + $items = array_values($items); + + + // Users can create a sub-list by indenting any deeper amount than the + // previous list, so these are both valid: + // + // - a + // - b + // + // - a + // - b + // + // In the former case, we'll have depths (0, 2). In the latter case, depths + // (0, 4). We don't actually care about how many spaces there are, only + // how many list indentation levels (that is, we want to map both of + // those cases to (0, 1), indicating "outermost list" and "first sublist"). + // + // This is made more complicated because lists at two different indentation + // levels might be at the same list level: + // + // - a + // - b + // - c + // - d + // + // Here, 'b' and 'd' are at the same list level (2) but different indent + // levels (2, 4). + // + // Users can also create "staircases" like this: + // + // - a + // - b + // # c + // + // While this is silly, we'd like to render it as faithfully as possible. + // + // In order to do this, we convert the list of nodes into a tree, + // normalizing indentation levels and inserting dummy nodes as necessary to + // make the tree well-formed. See additional notes at buildTree(). + // + // In the case above, the result is a tree like this: + // + // - + // - + // - a + // - b + // # c + + $l = 0; + $r = count($items); + $tree = $this->buildTree($items, $l, $r, $cur_level = 0); + + + // We may need to open a list on a node, but they do not have + // list style information yet. We need to propagate list style information + // backward through the tree. In the above example, the tree now looks + // like this: + // + // - + // - + // - a + // - b + // # c + + $this->adjustTreeStyleInformation($tree); + + // Finally, we have enough information to render the tree. + + $out = $this->renderTree($tree, 0, $has_marks, $starts_at); + + if ($this->getEngine()->isTextMode()) { + $out = implode('', $out); + $out = rtrim($out, "\n"); + $out = preg_replace('/ +$/m', '', $out); + return $out; + } + + return phutil_implode_html('', $out); + } + + /** + * See additional notes in @{method:markupText}. + */ + private function buildTree(array $items, $l, $r, $cur_level) { + if ($l == $r) { + return array(); + } + + if ($cur_level > self::MAXIMUM_LIST_NESTING_DEPTH) { + // This algorithm is recursive and we don't need you blowing the stack + // with your oh-so-clever 50,000-item-deep list. Cap indentation levels + // at a reasonable number and just shove everything deeper up to this + // level. + $nodes = array(); + for ($ii = $l; $ii < $r; $ii++) { + $nodes[] = array( + 'level' => $cur_level, + 'items' => array(), + ) + $items[$ii]; + } + return $nodes; + } + + $min = $l; + for ($ii = $r - 1; $ii >= $l; $ii--) { + if ($items[$ii]['depth'] <= $items[$min]['depth']) { + $min = $ii; + } + } + + $min_depth = $items[$min]['depth']; + + $nodes = array(); + if ($min != $l) { + $nodes[] = array( + 'text' => null, + 'level' => $cur_level, + 'style' => null, + 'mark' => null, + 'items' => $this->buildTree($items, $l, $min, $cur_level + 1), + ); + } + + $last = $min; + for ($ii = $last + 1; $ii < $r; $ii++) { + if ($items[$ii]['depth'] == $min_depth) { + $nodes[] = array( + 'level' => $cur_level, + 'items' => $this->buildTree($items, $last + 1, $ii, $cur_level + 1), + ) + $items[$last]; + $last = $ii; + } + } + $nodes[] = array( + 'level' => $cur_level, + 'items' => $this->buildTree($items, $last + 1, $r, $cur_level + 1), + ) + $items[$last]; + + return $nodes; + } + + + /** + * See additional notes in @{method:markupText}. + */ + private function adjustTreeStyleInformation(array &$tree) { + // The effect here is just to walk backward through the nodes at this level + // and apply the first style in the list to any empty nodes we inserted + // before it. As we go, also recurse down the tree. + + $style = '-'; + for ($ii = count($tree) - 1; $ii >= 0; $ii--) { + if ($tree[$ii]['style'] !== null) { + // This is the earliest node we've seen with style, so set the + // style to its style. + $style = $tree[$ii]['style']; + } else { + // This node has no style, so apply the current style. + $tree[$ii]['style'] = $style; + } + if ($tree[$ii]['items']) { + $this->adjustTreeStyleInformation($tree[$ii]['items']); + } + } + } + + + /** + * See additional notes in @{method:markupText}. + */ + private function renderTree( + array $tree, + $level, + $has_marks, + $starts_at = 1) { + + $style = idx(head($tree), 'style'); + + $out = array(); + + if (!$this->getEngine()->isTextMode()) { + switch ($style) { + case '#': + $tag = 'ol'; + break; + case '-': + $tag = 'ul'; + break; + } + + $start_attr = null; + if (ctype_digit($starts_at) && $starts_at > 1) { + $start_attr = hsprintf(' start="%d"', $starts_at); + } + + if ($has_marks) { + $out[] = hsprintf( + '<%s class="remarkup-list remarkup-list-with-checkmarks"%s>', + $tag, + $start_attr); + } else { + $out[] = hsprintf( + '<%s class="remarkup-list"%s>', + $tag, + $start_attr); + } + + $out[] = "\n"; + } + + $number = $starts_at; + foreach ($tree as $item) { + if ($this->getEngine()->isTextMode()) { + if ($item['text'] === null) { + // Don't render anything. + } else { + $indent = str_repeat(' ', 2 * $level); + $out[] = $indent; + if ($item['mark'] !== null) { + if ($item['mark']) { + $out[] = '[X] '; + } else { + $out[] = '[ ] '; + } + } else { + switch ($style) { + case '#': + $out[] = $number.'. '; + $number++; + break; + case '-': + $out[] = '- '; + break; + } + } + + $parts = preg_split('/\n{2,}/', $item['text']); + foreach ($parts as $key => $part) { + if ($key != 0) { + $out[] = "\n\n ".$indent; + } + $out[] = $this->applyRules($part); + } + $out[] = "\n"; + } + } else { + if ($item['text'] === null) { + $out[] = hsprintf('
  • '); + } else { + if ($item['mark'] !== null) { + if ($item['mark'] == true) { + $out[] = hsprintf( + '
  • '); + } else { + $out[] = hsprintf( + '
  • '); + } + $out[] = phutil_tag( + 'input', + array( + 'type' => 'checkbox', + 'checked' => ($item['mark'] ? 'checked' : null), + 'disabled' => 'disabled', + )); + $out[] = ' '; + } else { + $out[] = hsprintf('
  • '); + } + + $parts = preg_split('/\n{2,}/', $item['text']); + foreach ($parts as $key => $part) { + if ($key != 0) { + $out[] = array( + "\n", + phutil_tag('br'), + phutil_tag('br'), + "\n", + ); + } + $out[] = $this->applyRules($part); + } + } + } + + if ($item['items']) { + $subitems = $this->renderTree($item['items'], $level + 1, $has_marks); + foreach ($subitems as $i) { + $out[] = $i; + } + } + if (!$this->getEngine()->isTextMode()) { + $out[] = hsprintf("
    • \n"); + } + } + + if (!$this->getEngine()->isTextMode()) { + switch ($style) { + case '#': + $out[] = hsprintf(''); + break; + case '-': + $out[] = hsprintf(''); + break; + } + } + + return $out; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php new file mode 100644 index 0000000000..527db4fb61 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupLiteralBlockRule.php @@ -0,0 +1,93 @@ + $line) { + $line = preg_replace('/^\s*%%%/', '', $line); + $line = preg_replace('/%%%(\s*)\z/', '\1', $line); + $text[$key] = $line; + } + + if ($this->getEngine()->isTextMode()) { + return implode('', $text); + } + + return phutil_tag( + 'p', + array( + 'class' => 'remarkup-literal', + ), + phutil_implode_html(phutil_tag('br', array()), $text)); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php new file mode 100644 index 0000000000..e80f3e1953 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupNoteBlockRule.php @@ -0,0 +1,121 @@ +getRegEx(), $lines[$cursor])) { + $num_lines++; + $cursor++; + + while (isset($lines[$cursor])) { + if (trim($lines[$cursor])) { + $num_lines++; + $cursor++; + continue; + } + break; + } + } + + return $num_lines; + } + + public function markupText($text, $children) { + $matches = array(); + preg_match($this->getRegEx(), $text, $matches); + + if (idx($matches, 'showword')) { + $word = $matches['showword']; + $show = true; + } else { + $word = $matches['hideword']; + $show = false; + } + + $class_suffix = phutil_utf8_strtolower($word); + + // This is the "(IMPORTANT)" or "NOTE:" part. + $word_part = rtrim(substr($text, 0, strlen($matches[0]))); + + // This is the actual text. + $text_part = substr($text, strlen($matches[0])); + $text_part = $this->applyRules(rtrim($text_part)); + + $text_mode = $this->getEngine()->isTextMode(); + $html_mail_mode = $this->getEngine()->isHTMLMailMode(); + if ($text_mode) { + return $word_part.' '.$text_part; + } + + if ($show) { + $content = array( + phutil_tag( + 'span', + array( + 'class' => 'remarkup-note-word', + ), + $word_part), + ' ', + $text_part, + ); + } else { + $content = $text_part; + } + + if ($html_mail_mode) { + if ($class_suffix == 'important') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #c0392b; + background: #f4dddb;', + ); + } else if ($class_suffix == 'note') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #2980b9; + background: #daeaf3;', + ); + } else if ($class_suffix == 'warning') { + $attributes = array( + 'style' => 'margin: 16px 0; + padding: 12px; + border-left: 3px solid #f1c40f; + background: #fdf5d4;', + ); + } + } else { + $attributes = array( + 'class' => 'remarkup-'.$class_suffix, + ); + } + + return phutil_tag( + 'div', + $attributes, + $content); + } + + private function getRegEx() { + $words = array( + 'NOTE', + 'IMPORTANT', + 'WARNING', + ); + + foreach ($words as $k => $word) { + $words[$k] = preg_quote($word, '/'); + } + $words = implode('|', $words); + + return + '/^(?:'. + '(?:\((?P'.$words.')\))'. + '|'. + '(?:(?P'.$words.'):))\s*'. + '/'; + } +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php new file mode 100644 index 0000000000..ed87f7063d --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotedBlockRule.php @@ -0,0 +1,108 @@ + $line) { + $text[$key] = substr($line, 1); + } + + // If every line in the block is empty or begins with at least one leading + // space, strip the initial space off each line. When we quote text, we + // normally add "> " (with a space) to the beginning of each line, which + // can disrupt some other rules. If the block appears to have this space + // in front of each line, remove it. + + $strip_space = true; + foreach ($text as $key => $line) { + $len = strlen($line); + + if (!$len) { + // We'll still strip spaces if there are some completely empty + // lines, they may have just had trailing whitespace trimmed. + continue; + } + + // If this line is part of a nested quote block, just ignore it when + // realigning this quote block. It's either an author attribution + // line with ">>!", or we'll deal with it in a subrule when processing + // the nested quote block. + if ($line[0] == '>') { + continue; + } + + if ($line[0] == ' ' || $line[0] == "\n") { + continue; + } + + // The first character of this line is something other than a space, so + // we can't strip spaces. + $strip_space = false; + break; + } + + if ($strip_space) { + foreach ($text as $key => $line) { + $len = strlen($line); + if (!$len) { + continue; + } + + if ($line[0] !== ' ') { + continue; + } + + $text[$key] = substr($line, 1); + } + } + + // Strip leading empty lines. + foreach ($text as $key => $line) { + if (!strlen(trim($line))) { + unset($text[$key]); + } + } + + return implode('', $text); + } + + final protected function getQuotedText($text) { + $text = rtrim($text, "\n"); + + $no_whitespace = array( + // For readability, we render nested quotes as ">> quack", + // not "> > quack". + '>' => true, + + // If the line is empty except for a newline, do not add an + // unnecessary dangling space. + "\n" => true, + ); + + $text = phutil_split_lines($text, true); + foreach ($text as $key => $line) { + $c = null; + if (isset($line[0])) { + $c = $line[0]; + } else { + $c = null; + } + + if (isset($no_whitespace[$c])) { + $text[$key] = '>'.$line; + } else { + $text[$key] = '> '.$line; + } + } + $text = implode('', $text); + + return $text; + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php new file mode 100644 index 0000000000..f5ff9ef815 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupQuotesBlockRule.php @@ -0,0 +1,47 @@ +/', $lines[$pos])) { + do { + ++$pos; + } while (isset($lines[$pos]) && preg_match('/^>/', $lines[$pos])); + } + + return ($pos - $cursor); + } + + public function extractChildText($text) { + return array('', $this->normalizeQuotedBody($text)); + } + + public function markupText($text, $children) { + if ($this->getEngine()->isTextMode()) { + return $this->getQuotedText($children); + } + + $attributes = array(); + if ($this->getEngine()->isHTMLMailMode()) { + $style = array( + 'border-left: 3px solid #a7b5bf;', + 'color: #464c5c;', + 'font-style: italic;', + 'margin: 4px 0 12px 0;', + 'padding: 4px 12px;', + 'background-color: #f8f9fc;', + ); + + $attributes['style'] = implode(' ', $style); + } + + return phutil_tag( + 'blockquote', + $attributes, + $children); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php new file mode 100644 index 0000000000..584fca1349 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupReplyBlockRule.php @@ -0,0 +1,91 @@ +>!/', $lines[$pos])) { + do { + ++$pos; + } while (isset($lines[$pos]) && preg_match('/^>/', $lines[$pos])); + } + + return ($pos - $cursor); + } + + public function extractChildText($text) { + $text = phutil_split_lines($text, true); + + $head = substr(reset($text), 3); + + $body = array_slice($text, 1); + $body = implode('', $body); + $body = $this->normalizeQuotedBody($body); + + return array(trim($head), $body); + } + + public function markupText($text, $children) { + $text = $this->applyRules($text); + + if ($this->getEngine()->isTextMode()) { + $children = $this->getQuotedText($children); + return $text."\n\n".$children; + } + + if ($this->getEngine()->isHTMLMailMode()) { + $block_attributes = array( + 'style' => 'border-left: 3px solid #8C98B8; + color: #6B748C; + font-style: italic; + margin: 4px 0 12px 0; + padding: 8px 12px; + background-color: #F8F9FC;', + ); + $head_attributes = array( + 'style' => 'font-style: normal; + padding-bottom: 4px;', + ); + $reply_attributes = array( + 'style' => 'margin: 0; + padding: 0; + border: 0; + color: rgb(107, 116, 140);', + ); + } else { + $block_attributes = array( + 'class' => 'remarkup-reply-block', + ); + $head_attributes = array( + 'class' => 'remarkup-reply-head', + ); + $reply_attributes = array( + 'class' => 'remarkup-reply-body', + ); + } + + return phutil_tag( + 'blockquote', + $block_attributes, + array( + "\n", + phutil_tag( + 'div', + $head_attributes, + $text), + "\n", + phutil_tag( + 'div', + $reply_attributes, + $children), + "\n", + )); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php new file mode 100644 index 0000000000..72aae08d12 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupSimpleTableBlockRule.php @@ -0,0 +1,96 @@ + cells + // instead of cells. + + // If it has other types of cells, it's always a content row. + + // If it has only empty cells, it's an empty row. + + if (strlen($cell)) { + if (preg_match('/^--+\z/', $cell)) { + $any_header = true; + } else { + $any_content = true; + } + } + + $cells[] = array('type' => 'td', 'content' => $this->applyRules($cell)); + } + + $is_header = ($any_header && !$any_content); + + if (!$is_header) { + $rows[] = array('type' => 'tr', 'content' => $cells); + } else if ($rows) { + // Mark previous row with headings. + foreach ($cells as $i => $cell) { + if ($cell['content']) { + $last_key = last_key($rows); + if (!isset($rows[$last_key]['content'][$i])) { + // If this row has more cells than the previous row, there may + // not be a cell above this one to turn into a . + continue; + } + + $rows[$last_key]['content'][$i]['type'] = 'th'; + } + } + } + } + + if (!$rows) { + return $this->applyRules($text); + } + + return $this->renderRemarkupTable($rows); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php new file mode 100644 index 0000000000..57f2fe1f1d --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php @@ -0,0 +1,142 @@ +/i', $lines[$cursor])) { + $num_lines++; + $cursor++; + + while (isset($lines[$cursor])) { + $num_lines++; + if (preg_match('@\s*$@i', $lines[$cursor])) { + break; + } + $cursor++; + } + } + + return $num_lines; + } + + public function markupText($text, $children) { + $root = id(new PhutilHTMLParser()) + ->parseDocument($text); + + $nodes = $root->selectChildrenWithTags(array('table')); + + $out = array(); + $seen_table = false; + foreach ($nodes as $node) { + if ($node->isContentNode()) { + $content = $node->getContent(); + + if (!strlen(trim($content))) { + // Ignore whitespace. + continue; + } + + // If we find other content, fail the rule. This can happen if the + // input is two consecutive table tags on one line with some text + // in between them, which we currently forbid. + return $text; + } else { + // If we have multiple table tags, just return the raw text. + if ($seen_table) { + return $text; + } + $seen_table = true; + + $out[] = $this->newTable($node); + } + } + + if ($this->getEngine()->isTextMode()) { + return implode('', $out); + } else { + return phutil_implode_html('', $out); + } + } + + private function newTable(PhutilDOMNode $table) { + $nodes = $table->selectChildrenWithTags( + array( + 'colgroup', + 'tr', + )); + + $colgroup = null; + $rows = array(); + + foreach ($nodes as $node) { + if ($node->isContentNode()) { + $content = $node->getContent(); + + // If this is whitespace, ignore it. + if (!strlen(trim($content))) { + continue; + } + + // If we have nonempty content between the rows, this isn't a valid + // table. We can't really do anything reasonable with this, so just + // fail out and render the raw text. + return $table->newRawString(); + } + + if ($node->getTagName() === 'colgroup') { + // This table has multiple "" tags. Just bail out. + if ($colgroup !== null) { + return $table->newRawString(); + } + + // This table has a "" after a "". We could parse + // this, but just reject it out of an abundance of caution. + if ($rows) { + return $table->newRawString(); + } + + $colgroup = $node; + continue; + } + + $rows[] = $node; + } + + $row_specs = array(); + + foreach ($rows as $row) { + $cells = $row->selectChildrenWithTags(array('td', 'th')); + + $cell_specs = array(); + foreach ($cells as $cell) { + if ($cell->isContentNode()) { + $content = $node->getContent(); + + if (!strlen(trim($content))) { + continue; + } + + return $table->newRawString(); + } + + $content = $cell->newRawContentString(); + $content = $this->applyRules($content); + + $cell_specs[] = array( + 'type' => $cell->getTagName(), + 'content' => $content, + ); + } + + $row_specs[] = array( + 'type' => 'tr', + 'content' => $cell_specs, + ); + } + + return $this->renderRemarkupTable($row_specs); + } + +} diff --git a/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php b/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php new file mode 100644 index 0000000000..ca2b9b5ae7 --- /dev/null +++ b/src/infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php @@ -0,0 +1,17 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@\\*\\*(.+?)\\*\\*@s', + array($this, 'applyCallback'), + $text); + } + + protected function applyCallback(array $matches) { + return hsprintf('%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php new file mode 100644 index 0000000000..82f23d2d59 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupDelRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php new file mode 100644 index 0000000000..a6effa00ac --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupDocumentLinkRule.php @@ -0,0 +1,175 @@ +getEngine(); + + $is_anchor = false; + if (strncmp($link, '/', 1) == 0) { + $base = $engine->getConfig('uri.base'); + $base = rtrim($base, '/'); + $link = $base.$link; + } else if (strncmp($link, '#', 1) == 0) { + $here = $engine->getConfig('uri.here'); + $link = $here.$link; + + $is_anchor = true; + } + + if ($engine->isTextMode()) { + // If present, strip off "mailto:" or "tel:". + $link = preg_replace('/^(?:mailto|tel):/', '', $link); + + if (!strlen($name)) { + return $link; + } + + return $name.' <'.$link.'>'; + } + + if (!strlen($name)) { + $name = $link; + $name = preg_replace('/^(?:mailto|tel):/', '', $name); + } + + if ($engine->getState('toc')) { + return $name; + } + + $same_window = $engine->getConfig('uri.same-window', false); + if ($same_window) { + $target = null; + } else { + $target = '_blank'; + } + + // For anchors on the same page, always stay here. + if ($is_anchor) { + $target = null; + } + + return phutil_tag( + 'a', + array( + 'href' => $link, + 'class' => 'remarkup-link', + 'target' => $target, + 'rel' => 'noreferrer', + ), + $name); + } + + public function markupAlternateLink(array $matches) { + $uri = trim($matches[2]); + + if (!strlen($uri)) { + return $matches[0]; + } + + // NOTE: We apply some special rules to avoid false positives here. The + // major concern is that we do not want to convert `x[0][1](y)` in a + // discussion about C source code into a link. To this end, we: + // + // - Don't match at word boundaries; + // - require the URI to contain a "/" character or "@" character; and + // - reject URIs which being with a quote character. + + if ($uri[0] == '"' || $uri[0] == "'" || $uri[0] == '`') { + return $matches[0]; + } + + if (strpos($uri, '/') === false && + strpos($uri, '@') === false && + strncmp($uri, 'tel:', 4)) { + return $matches[0]; + } + + return $this->markupDocumentLink( + array( + $matches[0], + $matches[2], + $matches[1], + )); + } + + public function markupDocumentLink(array $matches) { + $uri = trim($matches[1]); + $name = trim(idx($matches, 2)); + + // If whatever is being linked to begins with "/" or "#", or has "://", + // or is "mailto:" or "tel:", treat it as a URI instead of a wiki page. + $is_uri = preg_match('@(^/)|(://)|(^#)|(^(?:mailto|tel):)@', $uri); + + if ($is_uri && strncmp('/', $uri, 1) && strncmp('#', $uri, 1)) { + $protocols = $this->getEngine()->getConfig( + 'uri.allowed-protocols', + array()); + + try { + $protocol = id(new PhutilURI($uri))->getProtocol(); + if (!idx($protocols, $protocol)) { + // Don't treat this as a URI if it's not an allowed protocol. + $is_uri = false; + } + } catch (Exception $ex) { + // We can end up here if we try to parse an ambiguous URI, see + // T12796. + $is_uri = false; + } + } + + // As a special case, skip "[[ / ]]" so that Phriction picks it up as a + // link to the Phriction root. It is more useful to be able to use this + // syntax to link to the root document than the home page of the install. + if ($uri == '/') { + $is_uri = false; + } + + if (!$is_uri) { + return $matches[0]; + } + + return $this->getEngine()->storeText($this->renderHyperlink($uri, $name)); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php new file mode 100644 index 0000000000..d9f56e2375 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupEscapeRemarkupRule.php @@ -0,0 +1,19 @@ +getEngine()->storeText("\1"); + + return str_replace("\1", $replace, $text); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php new file mode 100644 index 0000000000..900e355cd4 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHighlightRule.php @@ -0,0 +1,37 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@!!(.+?)(!{2,})@', + array($this, 'applyCallback'), + $text); + } + + protected function applyCallback(array $matches) { + // Remove the two exclamation points that represent syntax. + $excitement = substr($matches[2], 2); + + // If the internal content consists of ONLY exclamation points, leave it + // untouched so "!!!!!" is five exclamation points instead of one + // highlighted exclamation point. + if (preg_match('/^!+\z/', $matches[1])) { + return $matches[0]; + } + + // $excitement now has two fewer !'s than we started with. + return hsprintf('%s%s', + $matches[1], $excitement); + + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php new file mode 100644 index 0000000000..681ec46a09 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkEngineExtension.php @@ -0,0 +1,30 @@ +getPhobjectClassConstant('LINKENGINEKEY', 32); + } + + final public static function getAllLinkEngines() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getHyperlinkEngineKey') + ->execute(); + } + + final public function setEngine(PhutilRemarkupEngine $engine) { + $this->engine = $engine; + return $this; + } + + final public function getEngine() { + return $this->engine; + } + + abstract public function processHyperlinks(array $hyperlinks); + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php new file mode 100644 index 0000000000..2c81f0bb7b --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRef.php @@ -0,0 +1,38 @@ +token = $map['token']; + $this->uri = $map['uri']; + $this->embed = ($map['mode'] === '{'); + } + + public function getToken() { + return $this->token; + } + + public function getURI() { + return $this->uri; + } + + public function isEmbed() { + return $this->embed; + } + + public function setResult($result) { + $this->result = $result; + return $this; + } + + public function getResult() { + return $this->result; + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php new file mode 100644 index 0000000000..a926ea44c1 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupHyperlinkRule.php @@ -0,0 +1,234 @@ +" around them get linked exactly, without + // the "<>". Angle brackets are basically special and mean "this is a URL + // with weird characters". This is assumed to be reasonable because they + // don't appear in normal text or normal URLs. + $text = preg_replace_callback( + '@<(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+?)>@', + array($this, 'markupHyperlinkAngle'), + $text); + + // We match "{uri}", but do not link it by default. + $text = preg_replace_callback( + '@{(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+?)}@', + array($this, 'markupHyperlinkCurly'), + $text); + + // Anything else we match "ungreedily", which means we'll look for + // stuff that's probably puncutation or otherwise not part of the URL and + // not link it. This lets someone write "QuicK! Go to + // http://www.example.com/!". We also apply some paren balancing rules. + + // NOTE: We're explicitly avoiding capturing stored blocks, so text like + // `http://www.example.com/[[x | y]]` doesn't get aggressively captured. + $text = preg_replace_callback( + '@(\w{3,}://[^\s'.PhutilRemarkupBlockStorage::MAGIC_BYTE.']+)@', + array($this, 'markupHyperlinkUngreedy'), + $text); + + return $text; + } + + public function markupHyperlinkAngle(array $matches) { + return $this->markupHyperlink('<', $matches); + } + + public function markupHyperlinkCurly(array $matches) { + return $this->markupHyperlink('{', $matches); + } + + protected function markupHyperlink($mode, array $matches) { + $raw_uri = $matches[1]; + + try { + $uri = new PhutilURI($raw_uri); + } catch (Exception $ex) { + return $matches[0]; + } + + $engine = $this->getEngine(); + + $token = $engine->storeText($raw_uri); + + $list_key = self::KEY_HYPERLINKS; + $link_list = $engine->getTextMetadata($list_key, array()); + + $link_list[] = array( + 'token' => $token, + 'uri' => $raw_uri, + 'mode' => $mode, + ); + + $engine->setTextMetadata($list_key, $link_list); + + return $token; + } + + protected function renderHyperlink($link, $is_embed) { + // If the URI is "{uri}" and no handler picked it up, we just render it + // as plain text. + if ($is_embed) { + return $this->renderRawLink($link, $is_embed); + } + + $engine = $this->getEngine(); + + $same_window = $engine->getConfig('uri.same-window', false); + if ($same_window) { + $target = null; + } else { + $target = '_blank'; + } + + return phutil_tag( + 'a', + array( + 'href' => $link, + 'class' => 'remarkup-link', + 'target' => $target, + 'rel' => 'noreferrer', + ), + $link); + } + + private function renderRawLink($link, $is_embed) { + if ($is_embed) { + return '{'.$link.'}'; + } else { + return $link; + } + } + + protected function markupHyperlinkUngreedy($matches) { + $match = $matches[1]; + $tail = null; + $trailing = null; + if (preg_match('/[;,.:!?]+$/', $match, $trailing)) { + $tail = $trailing[0]; + $match = substr($match, 0, -strlen($tail)); + } + + // If there's a closing paren at the end but no balancing open paren in + // the URL, don't link the close paren. This is an attempt to gracefully + // handle the two common paren cases, Wikipedia links and English language + // parentheticals, e.g.: + // + // http://en.wikipedia.org/wiki/Noun_(disambiguation) + // (see also http://www.example.com) + // + // We could apply a craftier heuristic here which tries to actually balance + // the parens, but this is probably sufficient. + if (preg_match('/\\)$/', $match) && !preg_match('/\\(/', $match)) { + $tail = ')'.$tail; + $match = substr($match, 0, -1); + } + + try { + $uri = new PhutilURI($match); + } catch (Exception $ex) { + return $matches[0]; + } + + $link = $this->markupHyperlink(null, array(null, $match)); + + return hsprintf('%s%s', $link, $tail); + } + + public function didMarkupText() { + $engine = $this->getEngine(); + + $protocols = $engine->getConfig('uri.allowed-protocols', array()); + $is_toc = $engine->getState('toc'); + $is_text = $engine->isTextMode(); + $is_mail = $engine->isHTMLMailMode(); + + $list_key = self::KEY_HYPERLINKS; + $raw_list = $engine->getTextMetadata($list_key, array()); + + $links = array(); + foreach ($raw_list as $key => $link) { + $token = $link['token']; + $raw_uri = $link['uri']; + $mode = $link['mode']; + + $is_embed = ($mode === '{'); + $is_literal = ($mode === '<'); + + // If we're rendering in a "Table of Contents" or a plain text mode, + // we're going to render the raw URI without modifications. + if ($is_toc || $is_text) { + $result = $this->renderRawLink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // If this URI doesn't use a whitelisted protocol, don't link it. This + // is primarily intended to prevent "javascript://" silliness. + $uri = new PhutilURI($raw_uri); + $protocol = $uri->getProtocol(); + $valid_protocol = idx($protocols, $protocol); + if (!$valid_protocol) { + $result = $this->renderRawLink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // If the URI is written as "", we'll render it literally even if + // some handler would otherwise deal with it. + // If we're rendering for HTML mail, we also render literally. + if ($is_literal || $is_mail) { + $result = $this->renderHyperlink($raw_uri, $is_embed); + $engine->overwriteStoredText($token, $result); + continue; + } + + // Otherwise, this link is a valid resource which extensions are allowed + // to handle. + $links[$key] = $link; + } + + if (!$links) { + return; + } + + foreach ($links as $key => $link) { + $links[$key] = new PhutilRemarkupHyperlinkRef($link); + } + + $extensions = PhutilRemarkupHyperlinkEngineExtension::getAllLinkEngines(); + foreach ($extensions as $extension) { + $extension = id(clone $extension) + ->setEngine($engine) + ->processHyperlinks($links); + + foreach ($links as $key => $link) { + $result = $link->getResult(); + if ($result !== null) { + $engine->overwriteStoredText($link->getToken(), $result); + unset($links[$key]); + } + } + + if (!$links) { + break; + } + } + + // Render any remaining links in a normal way. + foreach ($links as $link) { + $result = $this->renderHyperlink($link->getURI(), $link->isEmbed()); + $engine->overwriteStoredText($link->getToken(), $result); + } + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php new file mode 100644 index 0000000000..9eefe2a7a2 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupItalicRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php new file mode 100644 index 0000000000..2db33d9516 --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupLinebreaksRule.php @@ -0,0 +1,13 @@ +getEngine()->isTextMode()) { + return $text; + } + + return phutil_escape_html_newlines($text); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php new file mode 100644 index 0000000000..cd5ab8ad0e --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupMonospaceRule.php @@ -0,0 +1,49 @@ +getEngine()->isTextMode()) { + $result = $matches[0]; + + } else + if ($this->getEngine()->isHTMLMailMode()) { + $match = isset($matches[2]) ? $matches[2] : $matches[1]; + $result = phutil_tag( + 'tt', + array( + 'style' => 'background: #ebebeb; font-size: 13px;', + ), + $match); + + } else { + $match = isset($matches[2]) ? $matches[2] : $matches[1]; + $result = phutil_tag( + 'tt', + array( + 'class' => 'remarkup-monospaced', + ), + $match); + } + + return $this->getEngine()->storeText($result); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php new file mode 100644 index 0000000000..a0ef7dc3ec --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupRule.php @@ -0,0 +1,109 @@ +engine = $engine; + return $this; + } + + public function getEngine() { + return $this->engine; + } + + public function getPriority() { + return 500.0; + } + + abstract public function apply($text); + + public function getPostprocessKey() { + return spl_object_hash($this); + } + + public function didMarkupText() { + return; + } + + protected function replaceHTML($pattern, $callback, $text) { + $this->replaceCallback = $callback; + return phutil_safe_html(preg_replace_callback( + $pattern, + array($this, 'replaceHTMLCallback'), + phutil_escape_html($text))); + } + + private function replaceHTMLCallback(array $match) { + return phutil_escape_html(call_user_func( + $this->replaceCallback, + array_map('phutil_safe_html', $match))); + } + + + /** + * Safely generate a tag. + * + * In Remarkup contexts, it's not safe to use arbitrary text in tag + * attributes: even though it will be escaped, it may contain replacement + * tokens which are then replaced with markup. + * + * This method acts as @{function:phutil_tag}, but checks attributes before + * using them. + * + * @param string Tag name. + * @param dict Tag attributes. + * @param wild Tag content. + * @return PhutilSafeHTML Tag object. + */ + protected function newTag($name, array $attrs, $content = null) { + foreach ($attrs as $key => $attr) { + if ($attr !== null) { + $attrs[$key] = $this->assertFlatText($attr); + } + } + + return phutil_tag($name, $attrs, $content); + } + + /** + * Assert that a text token is flat (it contains no replacement tokens). + * + * Because tokens can be replaced with markup, it is dangerous to use + * arbitrary input text in tag attributes. Normally, rule precedence should + * prevent this. Asserting that text is flat before using it as an attribute + * provides an extra layer of security. + * + * Normally, you can call @{method:newTag} rather than calling this method + * directly. @{method:newTag} will check attributes for you. + * + * @param wild Ostensibly flat text. + * @return string Flat text. + */ + protected function assertFlatText($text) { + $text = (string)hsprintf('%s', phutil_safe_html($text)); + $rich = (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) !== false); + if ($rich) { + throw new Exception( + pht( + 'Remarkup rule precedence is dangerous: rendering text with tokens '. + 'as flat text!')); + } + + return $text; + } + + /** + * Check whether text is flat (contains no replacement tokens) or not. + * + * @param wild Ostensibly flat text. + * @return bool True if the text is flat. + */ + protected function isFlatText($text) { + $text = (string)hsprintf('%s', phutil_safe_html($text)); + return (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) === false); + } + +} diff --git a/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php b/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php new file mode 100644 index 0000000000..1f1572863b --- /dev/null +++ b/src/infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php @@ -0,0 +1,24 @@ +getEngine()->isTextMode()) { + return $text; + } + + return $this->replaceHTML( + '@(?%s', $matches[1]); + } + +} diff --git a/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php b/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php new file mode 100644 index 0000000000..1c5ff78607 --- /dev/null +++ b/src/infrastructure/markup/remarkup/PhutilRemarkupEngine.php @@ -0,0 +1,302 @@ +config[$key] = $value; + return $this; + } + + public function getConfig($key, $default = null) { + return idx($this->config, $key, $default); + } + + public function setMode($mode) { + $this->mode = $mode; + return $this; + } + + public function isTextMode() { + return $this->mode & self::MODE_TEXT; + } + + public function isHTMLMailMode() { + return $this->mode & self::MODE_HTML_MAIL; + } + + public function setBlockRules(array $rules) { + assert_instances_of($rules, 'PhutilRemarkupBlockRule'); + + $rules = msortv($rules, 'getPriorityVector'); + + $this->blockRules = $rules; + foreach ($this->blockRules as $rule) { + $rule->setEngine($this); + } + + $post_rules = array(); + foreach ($this->blockRules as $block_rule) { + foreach ($block_rule->getMarkupRules() as $rule) { + $key = $rule->getPostprocessKey(); + if ($key !== null) { + $post_rules[$key] = $rule; + } + } + } + + $this->postprocessRules = $post_rules; + + return $this; + } + + public function getTextMetadata($key, $default = null) { + if (isset($this->metadata[$key])) { + return $this->metadata[$key]; + } + return idx($this->metadata, $key, $default); + } + + public function setTextMetadata($key, $value) { + $this->metadata[$key] = $value; + return $this; + } + + public function storeText($text) { + if ($this->isTextMode()) { + $text = phutil_safe_html($text); + } + return $this->storage->store($text); + } + + public function overwriteStoredText($token, $new_text) { + if ($this->isTextMode()) { + $new_text = phutil_safe_html($new_text); + } + $this->storage->overwrite($token, $new_text); + return $this; + } + + public function markupText($text) { + return $this->postprocessText($this->preprocessText($text)); + } + + public function pushState($state) { + if (empty($this->states[$state])) { + $this->states[$state] = 0; + } + $this->states[$state]++; + return $this; + } + + public function popState($state) { + if (empty($this->states[$state])) { + throw new Exception(pht("State '%s' pushed more than popped!", $state)); + } + $this->states[$state]--; + if (!$this->states[$state]) { + unset($this->states[$state]); + } + return $this; + } + + public function getState($state) { + return !empty($this->states[$state]); + } + + public function preprocessText($text) { + $this->metadata = array(); + $this->storage = new PhutilRemarkupBlockStorage(); + + $blocks = $this->splitTextIntoBlocks($text); + + $output = array(); + foreach ($blocks as $block) { + $output[] = $this->markupBlock($block); + } + $output = $this->flattenOutput($output); + + $map = $this->storage->getMap(); + $this->storage = null; + $metadata = $this->metadata; + + + return array( + 'output' => $output, + 'storage' => $map, + 'metadata' => $metadata, + ); + } + + private function splitTextIntoBlocks($text, $depth = 0) { + // Apply basic block and paragraph normalization to the text. NOTE: We don't + // strip trailing whitespace because it is semantic in some contexts, + // notably inlined diffs that the author intends to show as a code block. + $text = phutil_split_lines($text, true); + $block_rules = $this->blockRules; + $blocks = array(); + $cursor = 0; + $prev_block = array(); + + while (isset($text[$cursor])) { + $starting_cursor = $cursor; + foreach ($block_rules as $block_rule) { + $num_lines = $block_rule->getMatchingLineCount($text, $cursor); + + if ($num_lines) { + if ($blocks) { + $prev_block = last($blocks); + } + + $curr_block = array( + 'start' => $cursor, + 'num_lines' => $num_lines, + 'rule' => $block_rule, + 'is_empty' => self::isEmptyBlock($text, $cursor, $num_lines), + 'children' => array(), + ); + + if ($prev_block + && self::shouldMergeBlocks($text, $prev_block, $curr_block)) { + $blocks[last_key($blocks)]['num_lines'] += $curr_block['num_lines']; + $blocks[last_key($blocks)]['is_empty'] = + $blocks[last_key($blocks)]['is_empty'] && $curr_block['is_empty']; + } else { + $blocks[] = $curr_block; + } + + $cursor += $num_lines; + break; + } + } + + if ($starting_cursor === $cursor) { + throw new Exception(pht('Block in text did not match any block rule.')); + } + } + + foreach ($blocks as $key => $block) { + $lines = array_slice($text, $block['start'], $block['num_lines']); + $blocks[$key]['text'] = implode('', $lines); + } + + // Stop splitting child blocks apart if we get too deep. This arrests + // any blocks which have looping child rules, and stops the stack from + // exploding if someone writes a hilarious comment with 5,000 levels of + // quoted text. + + if ($depth < self::MAX_CHILD_DEPTH) { + foreach ($blocks as $key => $block) { + $rule = $block['rule']; + if (!$rule->supportsChildBlocks()) { + continue; + } + + list($parent_text, $child_text) = $rule->extractChildText( + $block['text']); + $blocks[$key]['text'] = $parent_text; + $blocks[$key]['children'] = $this->splitTextIntoBlocks( + $child_text, + $depth + 1); + } + } + + return $blocks; + } + + private function markupBlock(array $block) { + $children = array(); + foreach ($block['children'] as $child) { + $children[] = $this->markupBlock($child); + } + + if ($children) { + $children = $this->flattenOutput($children); + } else { + $children = null; + } + + return $block['rule']->markupText($block['text'], $children); + } + + private function flattenOutput(array $output) { + if ($this->isTextMode()) { + $output = implode("\n\n", $output)."\n"; + } else { + $output = phutil_implode_html("\n\n", $output); + } + + return $output; + } + + private static function shouldMergeBlocks($text, $prev_block, $curr_block) { + $block_rules = ipull(array($prev_block, $curr_block), 'rule'); + + $default_rule = 'PhutilRemarkupDefaultBlockRule'; + try { + assert_instances_of($block_rules, $default_rule); + + // If the last block was empty keep merging + if ($prev_block['is_empty']) { + return true; + } + + // If this line is blank keep merging + if ($curr_block['is_empty']) { + return true; + } + + // If the current line and the last line have content, keep merging + if (strlen(trim($text[$curr_block['start'] - 1]))) { + if (strlen(trim($text[$curr_block['start']]))) { + return true; + } + } + } catch (Exception $e) {} + + return false; + } + + private static function isEmptyBlock($text, $start, $num_lines) { + for ($cursor = $start; $cursor < $start + $num_lines; $cursor++) { + if (strlen(trim($text[$cursor]))) { + return false; + } + } + return true; + } + + public function postprocessText(array $dict) { + $this->metadata = idx($dict, 'metadata', array()); + + $this->storage = new PhutilRemarkupBlockStorage(); + $this->storage->setMap(idx($dict, 'storage', array())); + + foreach ($this->blockRules as $block_rule) { + $block_rule->postprocess(); + } + + foreach ($this->postprocessRules as $rule) { + $rule->didMarkupText(); + } + + return $this->restoreText(idx($dict, 'output')); + } + + public function restoreText($text) { + return $this->storage->restore($text, $this->isTextMode()); + } +} diff --git a/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php b/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php new file mode 100644 index 0000000000..c3b4960d0c --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/PhutilRemarkupEngineTestCase.php @@ -0,0 +1,132 @@ +markupText($root.$file); + } + } + + private function markupText($markup_file) { + $contents = Filesystem::readFile($markup_file); + $file = basename($markup_file); + + $parts = explode("\n~~~~~~~~~~\n", $contents); + $this->assertEqual(3, count($parts), $markup_file); + + list($input_remarkup, $expected_output, $expected_text) = $parts; + + $input_remarkup = $this->unescapeTrailingWhitespace($input_remarkup); + $expected_output = $this->unescapeTrailingWhitespace($expected_output); + $expected_text = $this->unescapeTrailingWhitespace($expected_text); + + $engine = $this->buildNewTestEngine(); + + switch ($file) { + case 'raw-escape.txt': + + // NOTE: Here, we want to test PhutilRemarkupEscapeRemarkupRule and + // PhutilRemarkupBlockStorage, which are triggered by "\1". In the + // test, "~" is used as a placeholder for "\1" since it's hard to type + // "\1". + + $input_remarkup = str_replace('~', "\1", $input_remarkup); + $expected_output = str_replace('~', "\1", $expected_output); + $expected_text = str_replace('~', "\1", $expected_text); + break; + case 'toc.txt': + $engine->setConfig('header.generate-toc', true); + break; + case 'link-same-window.txt': + $engine->setConfig('uri.same-window', true); + break; + case 'link-square.txt': + $engine->setConfig('uri.base', 'http://www.example.com/'); + $engine->setConfig('uri.here', 'http://www.example.com/page/'); + break; + } + + $actual_output = (string)$engine->markupText($input_remarkup); + + switch ($file) { + case 'toc.txt': + $table_of_contents = + PhutilRemarkupHeaderBlockRule::renderTableOfContents($engine); + $actual_output = $table_of_contents."\n\n".$actual_output; + break; + } + + $this->assertEqual( + $expected_output, + $actual_output, + pht("Failed to markup HTML in file '%s'.", $file)); + + $engine->setMode(PhutilRemarkupEngine::MODE_TEXT); + $actual_output = (string)$engine->markupText($input_remarkup); + + $this->assertEqual( + $expected_text, + $actual_output, + pht("Failed to markup text in file '%s'.", $file)); + } + + private function buildNewTestEngine() { + $engine = new PhutilRemarkupEngine(); + + $engine->setConfig( + 'uri.allowed-protocols', + array( + 'http' => true, + 'mailto' => true, + 'tel' => true, + )); + + $rules = array(); + $rules[] = new PhutilRemarkupEscapeRemarkupRule(); + $rules[] = new PhutilRemarkupMonospaceRule(); + $rules[] = new PhutilRemarkupDocumentLinkRule(); + $rules[] = new PhutilRemarkupHyperlinkRule(); + $rules[] = new PhutilRemarkupBoldRule(); + $rules[] = new PhutilRemarkupItalicRule(); + $rules[] = new PhutilRemarkupDelRule(); + $rules[] = new PhutilRemarkupUnderlineRule(); + $rules[] = new PhutilRemarkupHighlightRule(); + + $blocks = array(); + $blocks[] = new PhutilRemarkupQuotesBlockRule(); + $blocks[] = new PhutilRemarkupReplyBlockRule(); + $blocks[] = new PhutilRemarkupHeaderBlockRule(); + $blocks[] = new PhutilRemarkupHorizontalRuleBlockRule(); + $blocks[] = new PhutilRemarkupCodeBlockRule(); + $blocks[] = new PhutilRemarkupLiteralBlockRule(); + $blocks[] = new PhutilRemarkupNoteBlockRule(); + $blocks[] = new PhutilRemarkupTableBlockRule(); + $blocks[] = new PhutilRemarkupSimpleTableBlockRule(); + $blocks[] = new PhutilRemarkupDefaultBlockRule(); + $blocks[] = new PhutilRemarkupListBlockRule(); + $blocks[] = new PhutilRemarkupInterpreterBlockRule(); + + foreach ($blocks as $block) { + if (!($block instanceof PhutilRemarkupCodeBlockRule)) { + $block->setMarkupRules($rules); + } + } + + $engine->setBlockRules($blocks); + + return $engine; + } + + + private function unescapeTrailingWhitespace($input) { + // Remove up to one "~" at the end of each line so trailing whitespace may + // be written in tests as " ~". + return preg_replace('/~$/m', '', $input); + } + +} diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt new file mode 100644 index 0000000000..94886201cd --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/across-newlines.txt @@ -0,0 +1,7 @@ +**duck +quack** +~~~~~~~~~~ +

    duck +quack

    +~~~~~~~~~~ +**duck quack** diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt new file mode 100644 index 0000000000..8357cfbc35 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/backticks-whitespace.txt @@ -0,0 +1,17 @@ +```x``` + +``` +y +``` +~~~~~~~~~~ +
    x
    + + + +
    y
    +~~~~~~~~~~ + x + + + + y diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt new file mode 100644 index 0000000000..c18ebfdbab --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/block-then-list.txt @@ -0,0 +1,12 @@ + lang=txt + code block + + - still a code block +~~~~~~~~~~ +
    code block
+
+- still a code block
    +~~~~~~~~~~ + code block + + - still a code block diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt new file mode 100644 index 0000000000..f743ab74d7 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/code-block-whitespace.txt @@ -0,0 +1,9 @@ + lang=txt + x + y +~~~~~~~~~~ +
      x
+y
    +~~~~~~~~~~ + x + y diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt new file mode 100644 index 0000000000..955ad865eb --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/del.txt @@ -0,0 +1,11 @@ +omg~~ wtf~~~~~ bbq~~~ lol~~~ +~~deleted text~~~ +~~This is a great idea~~~ die forever please +~~~~~~~ +~~~~~~~~~~ +

    omg~~ wtf~~~~~ bbq~~~ lol~~~ +deleted text +This is a great idea~ die forever please +~~~~~~

    +~~~~~~~~~~ +omg~~ wtf~~~~~ bbq~~~ lol~~ ~~deleted text~~ ~~This is a great idea~~~ die forever please ~~~~~~~ diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt new file mode 100644 index 0000000000..602e8b3da5 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/diff.txt @@ -0,0 +1,36 @@ +here is a diff + + lang=diff + @@ derp derp @@ + x + y + + - x + - y + + z + +derp derp +~~~~~~~~~~ +

    here is a diff

    + +
    @@ derp derp @@
+x
+y
+
+- x
+- y
++ z
    + +

    derp derp

    +~~~~~~~~~~ +here is a diff + + @@ derp derp @@ + x + y + + - x + - y + + z + +derp derp diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt new file mode 100644 index 0000000000..3d3bf125a1 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/disallowed-link.txt @@ -0,0 +1,5 @@ +javascript://www.example.com/ +~~~~~~~~~~ +

    javascript://www.example.com/

    +~~~~~~~~~~ +javascript://www.example.com/ diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt new file mode 100644 index 0000000000..2fccce52d1 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/entities.txt @@ -0,0 +1,5 @@ +< > & " +~~~~~~~~~~ +

    < > & "

    +~~~~~~~~~~ +< > & " diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt new file mode 100644 index 0000000000..282706f6e7 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/header-skip.txt @@ -0,0 +1,11 @@ +#2 is my favorite. + +#project +~~~~~~~~~~ +

    #2 is my favorite.

    + +

    #project

    +~~~~~~~~~~ +#2 is my favorite. + +#project diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt new file mode 100644 index 0000000000..0c3768a3dd --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/headers.txt @@ -0,0 +1,57 @@ +@nolint (UTF8) + +=a= + +blah blah blah + + += b = + +Markdown-Style Large Header +==== + +Markdown-Style Small Header +---- + +=== Remarkup-Style Smaller Header + + += ☃☃☃ UTF8 Header ☃☃☃ = +~~~~~~~~~~ +

    @nolint (UTF8)

    + +

    a

    + +

    blah blah blah

    + +

    b

    + +

    Markdown-Style Large Header

    + +

    Markdown-Style Small Header

    + +

    Remarkup-Style Smaller Header

    + +

    ☃☃☃ UTF8 Header ☃☃☃

    +~~~~~~~~~~ +@nolint (UTF8) + +a += + +blah blah blah + +b += + +Markdown-Style Large Header +=========================== + +Markdown-Style Small Header +--------------------------- + +Remarkup-Style Smaller Header +----------------------------- + +☃☃☃ UTF8 Header ☃☃☃ +=================== diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt new file mode 100644 index 0000000000..5fb8895d19 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/highlight.txt @@ -0,0 +1,9 @@ +how about we !!highlight!! some !!TEXT!!! +wow this must be **!!very important!!** +omg!!!!! +~~~~~~~~~~ +

    how about we highlight some TEXT! +wow this must be very important +omg!!!!!

    +~~~~~~~~~~ +how about we !!highlight!! some !!TEXT!!! wow this must be **!!very important!!** omg!!!!! diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt new file mode 100644 index 0000000000..c36a7f2650 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/horizonal-rule.txt @@ -0,0 +1,41 @@ +___ + +_____ + +*** + +* * * * * * * + +--- + +- - - - - - - + + --- +~~~~~~~~~~ +
    + +
    + +
    + +
    + +
    + +
    + +
    +~~~~~~~~~~ +___ + +_____ + +*** + +* * * * * * * + +--- + +- - - - - - - + + --- diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt new file mode 100644 index 0000000000..e527ee9e4d --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/important.txt @@ -0,0 +1,15 @@ +IMPORTANT: interesting **stuff** + +(IMPORTANT) interesting **stuff** +~~~~~~~~~~ +
    IMPORTANT: interesting stuff
    + + + +
    interesting stuff
    +~~~~~~~~~~ +IMPORTANT: interesting **stuff** + + + +(IMPORTANT) interesting **stuff** diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt new file mode 100644 index 0000000000..477f3eeea0 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/interpreter-test.txt @@ -0,0 +1,58 @@ +phutil_test_block_interpreter (foo=bar) {{{ +content +}}} + +phutil_test_block_interpreter {{{ content +content }}} + +phutil_test_block_interpreter {{{ content }}} + +phutil_test_block_interpreter(x=y){{{content}}} + +phutil_fake_test_block_interpreter {{{ content }}} +~~~~~~~~~~ +Content: (content) +Argv: (foo=bar) + + + +Content: ( content +content ) +Argv: () + + + +Content: ( content ) +Argv: () + + + +Content: (content) +Argv: (x=y) + + + +
    No interpreter found: phutil_fake_test_block_interpreter
    +~~~~~~~~~~ +Content: (content) +Argv: (foo=bar) + + + +Content: ( content +content ) +Argv: () + + + +Content: ( content ) +Argv: () + + + +Content: (content) +Argv: (x=y) + + + +(No interpreter found: phutil_fake_test_block_interpreter) diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt new file mode 100644 index 0000000000..568e6df792 --- /dev/null +++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/just-backticks.txt @@ -0,0 +1,5 @@ +``` +~~~~~~~~~~ +
    
+~~~~~~~~~~
+  
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt
new file mode 100644
index 0000000000..a6f800db95
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/leading-newline.txt
@@ -0,0 +1,6 @@
+
+a
+~~~~~~~~~~
+
    a
    
+~~~~~~~~~~
+a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt
new file mode 100644
index 0000000000..8e6129141e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-alternate.txt
@@ -0,0 +1,12 @@
+[Example](http://www.example.com/)
+
+x[0][1](**ptr);
+
+~~~~~~~~~~
+
    Example
    
+
+
    x[0][1](**ptr);
    
+~~~~~~~~~~
+Example 
+
+x[0][1](**ptr);
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt
new file mode 100644
index 0000000000..3808aa976c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-brackets.txt
@@ -0,0 +1,5 @@
+
+~~~~~~~~~~
+
    http://www.zany.com/omg/weird_url,,,
    
+~~~~~~~~~~
+http://www.zany.com/omg/weird_url,,,
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt
new file mode 100644
index 0000000000..64c93eaea6
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-edge-cases.txt
@@ -0,0 +1,35 @@
+http://www.example.com/
+
+(http://www.example.com/)
+
+
+
+http://www.example.com/wiki/example_(disambiguation)
+
+(example http://www.example.com/)
+
+Quick! http://www.example.com/!
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    (http://www.example.com/)
    
+
+
    http://www.example.com/
    
+
+
    http://www.example.com/wiki/example_(disambiguation)
    
+
+
    (example http://www.example.com/)
    
+
+
    Quick! http://www.example.com/!
    
+~~~~~~~~~~
+http://www.example.com/
+
+(http://www.example.com/)
+
+http://www.example.com/
+
+http://www.example.com/wiki/example_(disambiguation)
+
+(example http://www.example.com/)
+
+Quick! http://www.example.com/!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt
new file mode 100644
index 0000000000..e449c15013
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mailto.txt
@@ -0,0 +1,18 @@
+[[ mailto:alincoln@example.com | mail me ]]
+
+[ mail me ]( mailto:alincoln@example.com )
+
+[[mailto:alincoln@example.com]]
+
+~~~~~~~~~~
+
    mail me
    
+
+
    mail me
    
+
+
    alincoln@example.com
    
+~~~~~~~~~~
+mail me 
+
+mail me 
+
+alincoln@example.com
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt
new file mode 100644
index 0000000000..bf433e9588
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-mixed.txt
@@ -0,0 +1,18 @@
+[[http://www.example.com/ | Example]](http://www.alternate.org/)
+
+(http://www.alternate.org/)[[http://www.example.com/ | Example]]
+
+
+
+~~~~~~~~~~
+
    Example(http://www.alternate.org/)
    
+
+
    (http://www.alternate.org/)Example
    
+
+
    <http://www.example.com/ Example>
    
+~~~~~~~~~~
+Example (http://www.alternate.org/)
+
+(http://www.alternate.org/)Example 
+
+>
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt
new file mode 100644
index 0000000000..65a86815c0
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-noreferrer.txt
@@ -0,0 +1,16 @@
+[[ /\evil.com ]]
+
+[[ /
+/evil.com ]]
+
+~~~~~~~~~~
+
    /\evil.com
    
+
+
    /
+/evil.com
    
+~~~~~~~~~~
+/\evil.com
+
+/
+/evil.com
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt
new file mode 100644
index 0000000000..937c83f67e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-same-window.txt
@@ -0,0 +1,11 @@
+[[http://www.example.com/]]
+
+http://www.example.com/
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    http://www.example.com/
    
+~~~~~~~~~~
+http://www.example.com/
+
+http://www.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt
new file mode 100644
index 0000000000..86f0c64d06
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-square.txt
@@ -0,0 +1,29 @@
+[[http://www.example.com/]]
+
+[[http://www.example.com/ | example.com]]
+
+[[/x/]]
+
+[[#anchor]]
+
+[[#anchor | Anchors ]]
+~~~~~~~~~~
+
    http://www.example.com/
    
+
+
    example.com
    
+
+
    http://www.example.com/x/
    
+
+
    http://www.example.com/page/#anchor
    
+
+
    Anchors
    
+~~~~~~~~~~
+http://www.example.com/
+
+example.com 
+
+http://www.example.com/x/
+
+http://www.example.com/page/#anchor
+
+Anchors 
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt
new file mode 100644
index 0000000000..aac13c2cca
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-tel.txt
@@ -0,0 +1,18 @@
+[[ tel:18005555555 | call me ]]
+
+[ call me ]( tel:18005555555 )
+
+[[tel:18005555555]]
+
+~~~~~~~~~~
+
    call me
    
+
+
    call me
    
+
+
    18005555555
    
+~~~~~~~~~~
+call me <18005555555>
+
+call me <18005555555>
+
+18005555555
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt
new file mode 100644
index 0000000000..847e9a1746
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-brackets.txt
@@ -0,0 +1,5 @@
+http://.example.com/
+~~~~~~~~~~
+
    http://<www>.example.com/
    
+~~~~~~~~~~
+http://.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt
new file mode 100644
index 0000000000..d823de6bcf
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-angle-link-anchor.txt
@@ -0,0 +1,5 @@
+
+~~~~~~~~~~
+
    <http://x.y#http://x.y#>
    
+~~~~~~~~~~
+
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt
new file mode 100644
index 0000000000..e8f6d6536a
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-link-anchor.txt
@@ -0,0 +1,5 @@
+http://x.y#http://x.y#
+~~~~~~~~~~
+
    http://x.y#http://x.y#
    
+~~~~~~~~~~
+http://x.y#http://x.y#
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt
new file mode 100644
index 0000000000..c187b16751
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-punctuation.txt
@@ -0,0 +1,9 @@
+http://www.example.com/,
+http://www.example.com/..
+http://www.example.com/!!!
+~~~~~~~~~~
+
    http://www.example.com/,
+http://www.example.com/..
+http://www.example.com/!!!
    
+~~~~~~~~~~
+http://www.example.com/, http://www.example.com/.. http://www.example.com/!!!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt
new file mode 100644
index 0000000000..615814d984
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link-with-tilde.txt
@@ -0,0 +1,5 @@
+http://www.example.com/~~
+~~~~~~~~~~
+
    http://www.example.com/~
    
+~~~~~~~~~~
+http://www.example.com/~~
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt
new file mode 100644
index 0000000000..903ab48a5b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/link.txt
@@ -0,0 +1,5 @@
+http://www.example.com/
+~~~~~~~~~~
+
    http://www.example.com/
    
+~~~~~~~~~~
+http://www.example.com/
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt
new file mode 100644
index 0000000000..420cd89cc9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-alternate-style.txt
@@ -0,0 +1,15 @@
+- a
+-- b
+--- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
          
+
        • c
          • 
+
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+    - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt
new file mode 100644
index 0000000000..c5b1631577
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-blow-stack.txt
@@ -0,0 +1,138 @@
+- a
+  - a
+    - a
+      - a
+        - a
+          - a
+            - a
+              - a
+                - a
+                  - a
+                    - a
+                      - a
+                        - a
+                          - a
+                            - a
+                              - a
+                                - a
+                                  - a
+                                    - a
+                                      - a
+                                        - a
+                                          - a
+                                            - a
+                                              - a
+                                                - a
+                                                  - a
+                                                    - a
+                                                      - a
+                                                        - a
+                                                          - a
+                                                            - a
+                                                              - a
+                                                                - a
+                                                                  - a
+                                                                    - a
+                                                                      - a
+                                                                        - a
+                                                                          - a
+
+
+derp
+~~~~~~~~~~
+
      
+
    • a
        
+
      • a
          
+
        • a
            
+
          • a
              
+
            • a
                
+
              • a
                  
+
                • a
                    
+
                  • a
                      
+
                    • a
                        
+
                      • a
                          
+
                        • a
                            
+
                          • a
                              
+
                            • a
                                
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • a
                                • 
+
                              • 
+
                            • 
+
                          • 
+
                        • 
+
                      • 
+
                    • 
+
                  • 
+
                • 
+
              • 
+
            • 
+
          • 
+
        • 
+
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- a
+  - a
+    - a
+      - a
+        - a
+          - a
+            - a
+              - a
+                - a
+                  - a
+                    - a
+                      - a
+                        - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+                          - a
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt
new file mode 100644
index 0000000000..26c5a3f7d4
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-checkboxes.txt
@@ -0,0 +1,41 @@
+- [] a
+- [ ] b
+- [X] c
+- d
+
+[ ] A
+[X] B
+  [ ] C
+  [ ] D
+
+[1] footnote
+
+~~~~~~~~~~
+
      
+
    •  a
      • 
+
    •  b
      • 
+
    •  c
      • 
+
    • d
      • 
+
    
+
+
      
+
    •  A
      • 
+
    •  B
        
+
      •  C
        • 
+
      •  D
        • 
+
      • 
+
    
+
+
    [1] footnote
    
+~~~~~~~~~~
+[ ] a
+[ ] b
+[X] c
+- d
+
+[ ] A
+[X] B
+  [ ] C
+  [ ] D
+
+[1] footnote
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt
new file mode 100644
index 0000000000..98e55c3534
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-crazystairs.txt
@@ -0,0 +1,15 @@
+## Fruit
+- Apple
+- Banana
+~~~~~~~~~~
+
      
+
      • 
+
      1. Fruit
        2. 
+
      • 
+
    • Apple
      • 
+
    • Banana
      • 
+
    
+~~~~~~~~~~
+  1. Fruit
+- Apple
+- Banana
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt
new file mode 100644
index 0000000000..55bd4c5429
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-first-style-wins.txt
@@ -0,0 +1,19 @@
+# item
+- item
+- item
+
+derp
+~~~~~~~~~~
+
      
+
    1. item
      2. 
+
    2. item
      3. 
+
    3. item
      4. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+1. item
+2. item
+3. item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt
new file mode 100644
index 0000000000..d1323090eb
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-hash.txt
@@ -0,0 +1,19 @@
+# item
+# item
+# item
+
+derp
+~~~~~~~~~~
+
      
+
    1. item
      2. 
+
    2. item
      3. 
+
    3. item
      4. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+1. item
+2. item
+3. item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt
new file mode 100644
index 0000000000..acb74781af
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header-last.txt
@@ -0,0 +1,7 @@
+# At the end of a block, this should be a list.
+~~~~~~~~~~
+
      
+
    1. At the end of a block, this should be a list.
      2. 
+
    
+~~~~~~~~~~
+1. At the end of a block, this should be a list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt
new file mode 100644
index 0000000000..4dd60f5413
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-header.txt
@@ -0,0 +1,12 @@
+## Small Header
+
+This should be a small header.
+~~~~~~~~~~
+
    Small Header
    
+
+
    This should be a small header.
    
+~~~~~~~~~~
+Small Header
+------------
+
+This should be a small header.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt
new file mode 100644
index 0000000000..dcd6732563
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-mixed-styles.txt
@@ -0,0 +1,15 @@
+  - a
+  -- b
+  --- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
          
+
        • c
          • 
+
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+    - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt
new file mode 100644
index 0000000000..ee7b307fc2
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multi.txt
@@ -0,0 +1,14 @@
+- a
+  -- b
+  -- c
+~~~~~~~~~~
+
      
+
    • a
        
+
      • b
        • 
+
      • c
        • 
+
      • 
+
    
+~~~~~~~~~~
+- a
+  - b
+  - c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt
new file mode 100644
index 0000000000..362133aaf1
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-multiline.txt
@@ -0,0 +1,16 @@
+- a
+  a
+- b
+b
+~~~~~~~~~~
+
      
+
    • a a
      • 
+
    • b
      • 
+
    
+
+
    b
    
+~~~~~~~~~~
+- a a
+- b
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt
new file mode 100644
index 0000000000..9ce47191d9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-nest.txt
@@ -0,0 +1,30 @@
+- item
+  - sub
+- item
+  # sub
+  # sub
+- item
+
+derp
+~~~~~~~~~~
+
      
+
    • item
        
+
      • sub
        • 
+
      • 
+
    • item
        
+
      1. sub
        2. 
+
      2. sub
        3. 
+
      • 
+
    • item
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- item
+  - sub
+- item
+  1. sub
+  2. sub
+- item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt
new file mode 100644
index 0000000000..90e972c8ad
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-paragraphs.txt
@@ -0,0 +1,27 @@
+- This is a list item
+  with several paragraphs.
+
+  This is the second paragraph
+  of the first list item.
+- This is the second item
+  in the list.
+  - This is a sublist.
+- This is the third item in the list.
+
+~~~~~~~~~~
+
      
+
    • This is a list item with several paragraphs.
+

      
+This is the second paragraph of the first list item.
      • 
+
    • This is the second item in the list.
        
+
      • This is a sublist.
        • 
+
      • 
+
    • This is the third item in the list.
      • 
+
    
+~~~~~~~~~~
+- This is a list item with several paragraphs.
+
+  This is the second paragraph of the first list item.
+- This is the second item in the list.
+  - This is a sublist.
+- This is the third item in the list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt
new file mode 100644
index 0000000000..232f599662
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-staircase.txt
@@ -0,0 +1,23 @@
+    - top
+  - mid
+# bot
+
+derp
+~~~~~~~~~~
+
      
+
      1. 
+
        • 
+
        • top
          • 
+
        • 
+
      • mid
        • 
+
      2. 
+
    2. bot
      3. 
+
    
+
+
    derp
    
+~~~~~~~~~~
+    - top
+  - mid
+1. bot
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt
new file mode 100644
index 0000000000..f86e489e3c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-star.txt
@@ -0,0 +1,19 @@
+* item
+* item
+* item
+
+derp
+~~~~~~~~~~
+
      
+
    • item
      • 
+
    • item
      • 
+
    • item
      • 
+
    
+
+
    derp
    
+~~~~~~~~~~
+- item
+- item
+- item
+
+derp
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt
new file mode 100644
index 0000000000..365d11aafc
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-then-a-list.txt
@@ -0,0 +1,15 @@
+1) one
+
+- a
+~~~~~~~~~~
+
      
+
    1. one
      2. 
+
    
+
+
      
+
    • a
      • 
+
    
+~~~~~~~~~~
+1. one
+
+- a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt
new file mode 100644
index 0000000000..1b51134903
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list-vs-codeblock.txt
@@ -0,0 +1,17 @@
+This should be a list:
+
+  - apple
+  - banana
+
+~~~~~~~~~~
+
    This should be a list:
    
+
+
      
+
    • apple
      • 
+
    • banana
      • 
+
    
+~~~~~~~~~~
+This should be a list:
+
+- apple
+- banana
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt
new file mode 100644
index 0000000000..eda8781b8e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/list.txt
@@ -0,0 +1,13 @@
+  - < > & "
+
+text block
+~~~~~~~~~~
+
      
+
    • < > & "
      • 
+
    
+
+
    text block
    
+~~~~~~~~~~
+- < > & "
+
+text block
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt
new file mode 100644
index 0000000000..8c94a16f47
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-in-monospaced.txt
@@ -0,0 +1,18 @@
+query ##SELECT * FROM `table`##
+
+`SELECT * FROM ##table##`
+
+`**x**`
+
+~~~~~~~~~~
+
    query SELECT * FROM `table`
    
+
+
    SELECT * FROM ##table##
    
+
+
    **x**
    
+~~~~~~~~~~
+query ##SELECT * FROM `table`##
+
+`SELECT * FROM ##table##`
+
+`**x**`
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt
new file mode 100644
index 0000000000..cb78ae923b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced-plural.txt
@@ -0,0 +1,11 @@
+`Zebra`s
+
+I can`t and I won`t.
+~~~~~~~~~~
+
    Zebras
    
+
+
    I can`t and I won`t.
    
+~~~~~~~~~~
+`Zebra`s
+
+I can`t and I won`t.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt
new file mode 100644
index 0000000000..db9cd50198
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/monospaced.txt
@@ -0,0 +1,5 @@
+cmd ##ls --color > /dev/null##
+~~~~~~~~~~
+
    cmd ls --color > /dev/null
    
+~~~~~~~~~~
+cmd ##ls --color > /dev/null##
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt
new file mode 100644
index 0000000000..159e7c9e54
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/newline-then-block.txt
@@ -0,0 +1,30 @@
+This is a paragraph.
+
+
+  lang=txt
+  First line of code block.
+  Second line of code block.
+
+
+
+  
+    
+    
+  
+
    Cell 1Cell 2
    
+~~~~~~~~~~
+
    This is a paragraph.
    
+
+
    First line of code block.
+Second line of code block.
    
+
+
    
+
+
    Cell 1Cell 2
    
+~~~~~~~~~~
+This is a paragraph.
+
+  First line of code block.
+  Second line of code block.
+
+| Cell 1 | Cell 2 |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt
new file mode 100644
index 0000000000..389da296bc
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/note-multiline.txt
@@ -0,0 +1,14 @@
+NOTE: a
+a
+
+b
+~~~~~~~~~~
+
    NOTE: a
+a
    
+
+
    b
    
+~~~~~~~~~~
+NOTE: a
+a
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt
new file mode 100644
index 0000000000..30541832e3
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/note.txt
@@ -0,0 +1,15 @@
+NOTE: interesting **stuff**
+
+(NOTE) interesting **stuff**
+~~~~~~~~~~
+
    NOTE: interesting stuff
    
+
+
+
+
    interesting stuff
    
+~~~~~~~~~~
+NOTE: interesting **stuff**
+
+
+
+(NOTE) interesting **stuff**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt
new file mode 100644
index 0000000000..f2e5e58114
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/ordered-list-with-numbers.txt
@@ -0,0 +1,64 @@
+# aasdx
+# asdf
+
+1. asa
+ # asdf
+234) asdf
+
+234) asd
+
+1. asd
+234) asd
+
+10. ten
+11. eleven
+12. twelve
+
+1/ This explicitly should not be formatted as a list.
+~~~~~~~~~~
+
      
+
    1. aasdx
      2. 
+
    2. asdf
      3. 
+
    
+
+
      
+
    1. asa
        
+
      1. asdf
        2. 
+
      2. 
+
    2. asdf
      3. 
+
    
+
+
      
+
    1. asd
      2. 
+
    
+
+
      
+
    1. asd
      2. 
+
    2. asd
      3. 
+
    
+
+
      
+
    1. ten
      2. 
+
    2. eleven
      3. 
+
    3. twelve
      4. 
+
    
+
+
    1/ This explicitly should not be formatted as a list.
    
+~~~~~~~~~~
+1. aasdx
+2. asdf
+
+1. asa
+  1. asdf
+2. asdf
+
+234. asd
+
+1. asd
+2. asd
+
+10. ten
+11. eleven
+12. twelve
+
+1/ This explicitly should not be formatted as a list.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt
new file mode 100644
index 0000000000..b6bc405bb9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-adjacent.txt
@@ -0,0 +1,29 @@
+%%%a%%%
+%%%b%%%
+
+%%%a
+b%%%
+
+%%%a%%%
+
+%%%b%%%
+~~~~~~~~~~
+
    a
+
    b
    
+
+
    a
+
    b
    
+
+
    a
    
+
+
    b
    
+~~~~~~~~~~
+a
+b
+
+a
+b
+
+a
+
+b
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt
new file mode 100644
index 0000000000..90cb4e0b41
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-multiline.txt
@@ -0,0 +1,21 @@
+**foo**
+%%%- first
+- second
+- third%%%
+[[http://hello | world]]
+~~~~~~~~~~
+
    foo
    
+
+
    - first
+
    - second
+
    - third
    
+
+
    world
    
+~~~~~~~~~~
+**foo**
+
+- first
+- second
+- third
+
+world 
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt
new file mode 100644
index 0000000000..a63e93d41c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-oneline.txt
@@ -0,0 +1,11 @@
+%%%[[http://hello | world]] **bold**%%%
+
+  %%%[[http://hello | world]] **bold**%%%
+~~~~~~~~~~
+
    [[http://hello | world]] **bold**
    
+
+
    [[http://hello | world]] **bold**
    
+~~~~~~~~~~
+[[http://hello | world]] **bold**
+
+[[http://hello | world]] **bold**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt
new file mode 100644
index 0000000000..34ddbd38d3
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/percent-block-solo.txt
@@ -0,0 +1,8 @@
+%%%
+**x**%%%
+~~~~~~~~~~
+
    
+
    **x**
    
+~~~~~~~~~~
+
+**x**
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt
new file mode 100644
index 0000000000..da78b31fd6
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-angry.txt
@@ -0,0 +1,5 @@
+>>> REQUESTING CHANGES BECAUSE I'M ANGRY!
+~~~~~~~~~~
+
    REQUESTING CHANGES BECAUSE I'M ANGRY!
    
+~~~~~~~~~~
+>>> REQUESTING CHANGES BECAUSE I'M ANGRY!
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt
new file mode 100644
index 0000000000..9ef852d56d
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-code-block.txt
@@ -0,0 +1,16 @@
+> This should be a code block:
+>
+> ```lang=php
+>  $foo = 'bar';
+> ```
+~~~~~~~~~~
+
    This should be a code block:
    
+
+
    <?php
+$foo = 'bar';
    
+~~~~~~~~~~
+> This should be a code block:
+>
+>      $foo = 'bar';
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt
new file mode 100644
index 0000000000..80a7428fce
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-indent-block.txt
@@ -0,0 +1,5 @@
+>   xyz
+~~~~~~~~~~
+
    xyz
    
+~~~~~~~~~~
+>   xyz
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt
new file mode 100644
index 0000000000..8d3aaf9c41
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-lists.txt
@@ -0,0 +1,24 @@
+> # X
+> # Y
+>
+> B
+>
+> * C
+~~~~~~~~~~
+
      
+
    1. X
      2. 
+
    2. Y
      3. 
+
    
+
+
    B
    
+
+
      
+
    • C
      • 
+
    
+~~~~~~~~~~
+> 1. X
+> 2. Y
+>
+> B
+>
+> - C
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt
new file mode 100644
index 0000000000..1ca2f2f2a5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quoted-quote.txt
@@ -0,0 +1,19 @@
+>>! In U, W wrote:
+> - Y
+>
+> Z
+~~~~~~~~~~
+
    
+
    In U, W wrote:
    
+
      
+
    • Y
      • 
+
    
+
+
    Z
    
+
    
+~~~~~~~~~~
+In U, W wrote:
+
+> - Y
+>
+> Z
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt
new file mode 100644
index 0000000000..212f87222d
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/quotes.txt
@@ -0,0 +1,9 @@
+> Dear Sir,
+> I am utterly disgusted with the quality
+> of your inflight food service.
+~~~~~~~~~~
+
    Dear Sir,
+I am utterly disgusted with the quality
+of your inflight food service.
    
+~~~~~~~~~~
+> Dear Sir, I am utterly disgusted with the quality of your inflight food service.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt
new file mode 100644
index 0000000000..4cb635ebfb
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/raw-escape.txt
@@ -0,0 +1,17 @@
+~1~~
+
+~2Z
+
+~a
+~~~~~~~~~~
+
    ~1~
    
+
+
    ~2Z
    
+
+
    ~a
    
+~~~~~~~~~~
+~1~~
+
+~2Z
+
+~a
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt
new file mode 100644
index 0000000000..4afadedd6e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-basic.txt
@@ -0,0 +1,11 @@
+>>! In comment #123, alincoln wrote:
+> Four score and twenty years ago...
+~~~~~~~~~~
+
    
+
    In comment #123, alincoln wrote:
    
+
    Four score and twenty years ago...
    
+
    
+~~~~~~~~~~
+In comment #123, alincoln wrote:
+
+> Four score and twenty years ago...
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt
new file mode 100644
index 0000000000..85440ddeac
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/reply-nested.txt
@@ -0,0 +1,48 @@
+>>! Previously, fruit:
+>
+> - Apple
+> - Banana
+> - Cherry
+>
+>>>! More previously, vegetables:
+>>
+>> - Potato
+>> - Potato
+>> - Potato
+>
+> The end.
+
+~~~~~~~~~~
+
    
+
    Previously, fruit:
    
+
      
+
    • Apple
      • 
+
    • Banana
      • 
+
    • Cherry
      • 
+
    
+
+
    
+
    More previously, vegetables:
    
+
      
+
    • Potato
      • 
+
    • Potato
      • 
+
    • Potato
      • 
+
    
+
    
+
+
    The end.
    
+
    
+~~~~~~~~~~
+Previously, fruit:
+
+> - Apple
+> - Banana
+> - Cherry
+>
+> More previously, vegetables:
+>
+>> - Potato
+>> - Potato
+>> - Potato
+>
+> The end.
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt
new file mode 100644
index 0000000000..40c27ca941
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-empty-row.txt
@@ -0,0 +1,13 @@
+| Alpaca |
+| |
+| Zebra  |
+~~~~~~~~~~
+
    
+
+
+
+
    Alpaca
    Zebra
    
+~~~~~~~~~~
+| Alpaca |
+|        |
+| Zebra  |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt
new file mode 100644
index 0000000000..418baa5ba5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-leading-space.txt
@@ -0,0 +1,7 @@
+ |a|b|
+~~~~~~~~~~
+
    
+
+
    ab
    
+~~~~~~~~~~
+| a | b |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt
new file mode 100644
index 0000000000..d4e53b3426
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table-with-link.txt
@@ -0,0 +1,7 @@
+| [[ http://example.com | name ]] | [x] |
+~~~~~~~~~~
+
    
+
+
    name[x]
    
+~~~~~~~~~~
+| name  | [x] |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt
new file mode 100644
index 0000000000..d86a8189b9
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple-table.txt
@@ -0,0 +1,24 @@
+| analyze_resources | original | mobile only | www only | both |
+|                   | -------- | ----------- | -------- | ---- |
+| //real//          | 31 s     | 24 s        | 31 s     | 31 s
+| --------
+| //user//          | 49 s     | 25 s        | 31 s     | 49 s
+| --------
+| //sys//           | 24 s     | 12 s        | 13 s     | 24 s
+| -------
+~~~~~~~~~~
+
    
+
+
+
+
+
    analyze_resourcesoriginalmobile onlywww onlyboth
    real31 s24 s31 s31 s
    user49 s25 s31 s49 s
    sys24 s12 s13 s24 s
    
+~~~~~~~~~~
+| analyze_resources | original | mobile only | www only | both |
+|                   | -------- | ----------- | -------- | ---- |
+| //real//          | 31 s     | 24 s        | 31 s     | 31 s |
+| ----------------- |          |             |          |      |
+| //user//          | 49 s     | 25 s        | 31 s     | 49 s |
+| ----------------- |          |             |          |      |
+| //sys//           | 24 s     | 12 s        | 13 s     | 24 s |
+| ----------------- |          |             |          |      |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt
new file mode 100644
index 0000000000..bd9b136d72
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/simple.txt
@@ -0,0 +1,5 @@
+hello
+~~~~~~~~~~
+
    hello
    
+~~~~~~~~~~
+hello
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt
new file mode 100644
index 0000000000..eab88dec77
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-direct-content.txt
@@ -0,0 +1,5 @@
+quack
    
+~~~~~~~~~~
+<table>quack</table>
+~~~~~~~~~~
+quack
    
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt
new file mode 100644
index 0000000000..b0a45cd669
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-leading-space.txt
@@ -0,0 +1,7 @@
+ 
    cell
    
+~~~~~~~~~~
+
    
+
+
    cell
    
+~~~~~~~~~~
+| cell |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt
new file mode 100644
index 0000000000..84b5cb84ad
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table-with-long-header.txt
@@ -0,0 +1,8 @@
+|x|
+||--
+~~~~~~~~~~
+
    
+
+
    x
    
+~~~~~~~~~~
+| x |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt
new file mode 100644
index 0000000000..6a3afe5d1b
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/table.txt
@@ -0,0 +1,16 @@
+
+
+
+
+
    TableStorage
    `differential_diff`InnoDB
    `edge`?
    
+~~~~~~~~~~
+
    
+
+
+
+
    TableStorage
    differential_diffInnoDB
    edge?
    
+~~~~~~~~~~
+| Table               | Storage |
+| ------------------- | ------- |
+| `differential_diff` | InnoDB  |
+| `edge`              | ?       |
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt
new file mode 100644
index 0000000000..b05c52442e
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block-multi.txt
@@ -0,0 +1,18 @@
+```code
+
+more code
+
+more code```
+
+~~~~~~~~~~
+
    code
+
+more code
+
+more code
    
+~~~~~~~~~~
+  code
+  
+  more code
+  
+  more code
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt
new file mode 100644
index 0000000000..4cc7607d32
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/tick-block.txt
@@ -0,0 +1,5 @@
+```code```
+~~~~~~~~~~
+
    code
    
+~~~~~~~~~~
+  code
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt
new file mode 100644
index 0000000000..43448f75b5
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/toc.txt
@@ -0,0 +1,29 @@
+= [[ http://www.example.com/ | link_name ]] =
+
+== **bold** ==
+
+= http://www.example.com =
+
+~~~~~~~~~~
+
+
+
    link_name
    
+
+
    bold
    
+
+
    http://www.example.com
    
+~~~~~~~~~~
+[[ http://www.example.com/ | link_name ]]
+=========================================
+
+**bold**
+--------
+
+http://www.example.com
+======================
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt
new file mode 100644
index 0000000000..2187cca939
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/trailing-whitespace-codeblock.txt
@@ -0,0 +1,39 @@
+  lang=txt
+  code block
+  code block
+  
+  
+  
+  
+  code block
+  
+  
+            
+  
+  code block
+~~~~~~~~~~
+
    code block
+code block
+
+
+
+
+code block
+
+
+          
+
+code block
    
+~~~~~~~~~~
+  code block
+  code block
+  
+  
+  
+  
+  code block
+  
+  
+            
+  
+  code block
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt
new file mode 100644
index 0000000000..511c0cea84
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/underline.txt
@@ -0,0 +1,13 @@
+omg__ wtf_____ bbq___ lol__
+__underlined text__
+__This is a great idea___ die forever please
+__
+/__notunderlined__/ and also /__notunderlined__.c
+~~~~~~~~~~
+
    omg__ wtf_____ bbq___ lol__
+underlined text
+This is a great idea_ die forever please
+__
+/__notunderlined__/ and also /__notunderlined__.c
    
+~~~~~~~~~~
+omg__ wtf_____ bbq___ lol__ __underlined text__ __This is a great idea___ die forever please __ /__notunderlined__/ and also /__notunderlined__.c
diff --git a/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt b/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt
new file mode 100644
index 0000000000..6de7f0cd6c
--- /dev/null
+++ b/src/infrastructure/markup/remarkup/__tests__/remarkup/warning.txt
@@ -0,0 +1,15 @@
+WARNING: interesting **stuff**
+
+(WARNING) interesting **stuff**
+~~~~~~~~~~
+
    WARNING: interesting stuff
    
+
+
+
+
    interesting stuff
    
+~~~~~~~~~~
+WARNING: interesting **stuff**
+
+
+
+(WARNING) interesting **stuff**
diff --git a/src/infrastructure/storage/connection/AphrontDatabaseConnection.php b/src/infrastructure/storage/connection/AphrontDatabaseConnection.php
new file mode 100644
index 0000000000..b3bd2c8299
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontDatabaseConnection.php
@@ -0,0 +1,305 @@
+close();
+  }
+
+  final public function setLastActiveEpoch($epoch) {
+    $this->lastActiveEpoch = $epoch;
+    return $this;
+  }
+
+  final public function getLastActiveEpoch() {
+    return $this->lastActiveEpoch;
+  }
+
+  final public function setPersistent($persistent) {
+    $this->persistent = $persistent;
+    return $this;
+  }
+
+  final public function getPersistent() {
+    return $this->persistent;
+  }
+
+  public function queryData($pattern/* , $arg, $arg, ... */) {
+    $args = func_get_args();
+    array_unshift($args, $this);
+    return call_user_func_array('queryfx_all', $args);
+  }
+
+  public function query($pattern/* , $arg, $arg, ... */) {
+    $args = func_get_args();
+    array_unshift($args, $this);
+    return call_user_func_array('queryfx', $args);
+  }
+
+
+  public function supportsAsyncQueries() {
+    return false;
+  }
+
+  public function supportsParallelQueries() {
+    return false;
+  }
+
+  public function setReadOnly($read_only) {
+    $this->readOnly = $read_only;
+    return $this;
+  }
+
+  public function getReadOnly() {
+    return $this->readOnly;
+  }
+
+  public function setQueryTimeout($query_timeout) {
+    $this->queryTimeout = $query_timeout;
+    return $this;
+  }
+
+  public function getQueryTimeout() {
+    return $this->queryTimeout;
+  }
+
+  public function asyncQuery($raw_query) {
+    throw new Exception(pht('Async queries are not supported.'));
+  }
+
+  public static function resolveAsyncQueries(array $conns, array $asyncs) {
+    throw new Exception(pht('Async queries are not supported.'));
+  }
+
+  /**
+   * Is this connection idle and safe to close?
+   *
+   * A connection is "idle" if it can be safely closed without loss of state.
+   * Connections inside a transaction or holding locks are not idle, even
+   * though they may not actively be executing queries.
+   *
+   * @return bool True if the connection is idle and can be safely closed.
+   */
+  public function isIdle() {
+    if ($this->isInsideTransaction()) {
+      return false;
+    }
+
+    if ($this->isHoldingAnyLock()) {
+      return false;
+    }
+
+    return true;
+  }
+
+
+/* -(  Global Locks  )------------------------------------------------------- */
+
+
+  public function rememberLock($lock) {
+    if (isset($this->locks[$lock])) {
+      throw new Exception(
+        pht(
+          'Trying to remember lock "%s", but this lock has already been '.
+          'remembered.',
+          $lock));
+    }
+
+    $this->locks[$lock] = true;
+    return $this;
+  }
+
+
+  public function forgetLock($lock) {
+    if (empty($this->locks[$lock])) {
+      throw new Exception(
+        pht(
+          'Trying to forget lock "%s", but this connection does not remember '.
+          'that lock.',
+          $lock));
+    }
+
+    unset($this->locks[$lock]);
+    return $this;
+  }
+
+
+  public function forgetAllLocks() {
+    $this->locks = array();
+    return $this;
+  }
+
+
+  public function isHoldingAnyLock() {
+    return (bool)$this->locks;
+  }
+
+
+/* -(  Transaction Management  )--------------------------------------------- */
+
+
+  /**
+   * Begin a transaction, or set a savepoint if the connection is already
+   * transactional.
+   *
+   * @return this
+   * @task xaction
+   */
+  public function openTransaction() {
+    $state = $this->getTransactionState();
+    $point = $state->getSavepointName();
+    $depth = $state->getDepth();
+
+    $new_transaction = ($depth == 0);
+    if ($new_transaction) {
+      $this->query('START TRANSACTION');
+    } else {
+      $this->query('SAVEPOINT '.$point);
+    }
+
+    $state->increaseDepth();
+
+    return $this;
+  }
+
+
+  /**
+   * Commit a transaction, or stage a savepoint for commit once the entire
+   * transaction completes if inside a transaction stack.
+   *
+   * @return this
+   * @task xaction
+   */
+  public function saveTransaction() {
+    $state = $this->getTransactionState();
+    $depth = $state->decreaseDepth();
+
+    if ($depth == 0) {
+      $this->query('COMMIT');
+    }
+
+    return $this;
+  }
+
+
+  /**
+   * Rollback a transaction, or unstage the last savepoint if inside a
+   * transaction stack.
+   *
+   * @return this
+   */
+  public function killTransaction() {
+    $state = $this->getTransactionState();
+    $depth = $state->decreaseDepth();
+
+    if ($depth == 0) {
+      $this->query('ROLLBACK');
+    } else {
+      $this->query('ROLLBACK TO SAVEPOINT '.$state->getSavepointName());
+    }
+
+    return $this;
+  }
+
+
+  /**
+   * Returns true if the connection is transactional.
+   *
+   * @return bool True if the connection is currently transactional.
+   * @task xaction
+   */
+  public function isInsideTransaction() {
+    $state = $this->getTransactionState();
+    return ($state->getDepth() > 0);
+  }
+
+
+  /**
+   * Get the current @{class:AphrontDatabaseTransactionState} object, or create
+   * one if none exists.
+   *
+   * @return AphrontDatabaseTransactionState Current transaction state.
+   * @task xaction
+   */
+  protected function getTransactionState() {
+    if (!$this->transactionState) {
+      $this->transactionState = new AphrontDatabaseTransactionState();
+    }
+    return $this->transactionState;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function beginReadLocking() {
+    $this->getTransactionState()->beginReadLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function endReadLocking() {
+    $this->getTransactionState()->endReadLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function isReadLocking() {
+    return $this->getTransactionState()->isReadLocking();
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function beginWriteLocking() {
+    $this->getTransactionState()->beginWriteLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function endWriteLocking() {
+    $this->getTransactionState()->endWriteLocking();
+    return $this;
+  }
+
+
+  /**
+   * @task xaction
+   */
+  public function isWriteLocking() {
+    return $this->getTransactionState()->isWriteLocking();
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php b/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php
new file mode 100644
index 0000000000..67f8b5b65b
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontDatabaseTransactionState.php
@@ -0,0 +1,105 @@
+depth;
+  }
+
+  public function increaseDepth() {
+    return ++$this->depth;
+  }
+
+  public function decreaseDepth() {
+    if ($this->depth == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s or %s!',
+          'saveTransaction()',
+          'killTransaction()'));
+    }
+
+    return --$this->depth;
+  }
+
+  public function getSavepointName() {
+    return 'Aphront_Savepoint_'.$this->depth;
+  }
+
+  public function beginReadLocking() {
+    $this->readLockLevel++;
+    return $this;
+  }
+
+  public function endReadLocking() {
+    if ($this->readLockLevel == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s!',
+          __FUNCTION__.'()'));
+    }
+    $this->readLockLevel--;
+    return $this;
+  }
+
+  public function isReadLocking() {
+    return ($this->readLockLevel > 0);
+  }
+
+  public function beginWriteLocking() {
+    $this->writeLockLevel++;
+    return $this;
+  }
+
+  public function endWriteLocking() {
+    if ($this->writeLockLevel == 0) {
+      throw new Exception(
+        pht(
+          'Too many calls to %s!',
+          __FUNCTION__.'()'));
+    }
+    $this->writeLockLevel--;
+    return $this;
+  }
+
+  public function isWriteLocking() {
+    return ($this->writeLockLevel > 0);
+  }
+
+  public function __destruct() {
+    if ($this->depth) {
+      throw new Exception(
+        pht(
+          'Process exited with an open transaction! The transaction '.
+          'will be implicitly rolled back. Calls to %s must always be '.
+          'paired with a call to %s or %s.',
+          'openTransaction()',
+          'saveTransaction()',
+          'killTransaction()'));
+    }
+    if ($this->readLockLevel) {
+      throw new Exception(
+        pht(
+          'Process exited with an open read lock! Call to %s '.
+          'must always be paired with a call to %s.',
+          'beginReadLocking()',
+          'endReadLocking()'));
+    }
+    if ($this->writeLockLevel) {
+      throw new Exception(
+        pht(
+          'Process exited with an open write lock! Call to %s '.
+          'must always be paired with a call to %s.',
+          'beginWriteLocking()',
+          'endWriteLocking()'));
+    }
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php b/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php
new file mode 100644
index 0000000000..9638b6f00e
--- /dev/null
+++ b/src/infrastructure/storage/connection/AphrontIsolatedDatabaseConnection.php
@@ -0,0 +1,132 @@
+configuration = $configuration;
+
+    if (self::$nextInsertID === null) {
+      // Generate test IDs into a distant ID space to reduce the risk of
+      // collisions and make them distinctive.
+      self::$nextInsertID = 55555000000 + mt_rand(0, 1000);
+    }
+  }
+
+  public function openConnection() {
+    return;
+  }
+
+  public function close() {
+    return;
+  }
+
+  public function escapeUTF8String($string) {
+    return '';
+  }
+
+  public function escapeBinaryString($string) {
+    return '';
+  }
+
+  public function escapeColumnName($name) {
+    return '';
+  }
+
+  public function escapeMultilineComment($comment) {
+    return '';
+  }
+
+  public function escapeStringForLikeClause($value) {
+    return '';
+  }
+
+  private function getConfiguration($key, $default = null) {
+    return idx($this->configuration, $key, $default);
+  }
+
+  public function getInsertID() {
+    return $this->insertID;
+  }
+
+  public function getAffectedRows() {
+    return $this->affectedRows;
+  }
+
+  public function selectAllResults() {
+    return $this->allResults;
+  }
+
+  public function executeQuery(PhutilQueryString $query) {
+
+    // NOTE: "[\s<>K]*" allows any number of (properly escaped) comments to
+    // appear prior to the allowed keyword, since this connection escapes
+    // them as "" (above).
+
+    $display_query = $query->getMaskedString();
+    $raw_query = $query->getUnmaskedString();
+
+    $keywords = array(
+      'INSERT',
+      'UPDATE',
+      'DELETE',
+      'START',
+      'SAVEPOINT',
+      'COMMIT',
+      'ROLLBACK',
+    );
+    $preg_keywords = array();
+    foreach ($keywords as $key => $word) {
+      $preg_keywords[] = preg_quote($word, '/');
+    }
+    $preg_keywords = implode('|', $preg_keywords);
+
+    if (!preg_match('/^[\s<>K]*('.$preg_keywords.')\s*/i', $raw_query)) {
+      throw new AphrontNotSupportedQueryException(
+        pht(
+          "Database isolation currently only supports some queries. You are ".
+          "trying to issue a query which does not begin with an allowed ".
+          "keyword (%s): '%s'.",
+          implode(', ', $keywords),
+          $display_query));
+    }
+
+    $this->transcript[] = $display_query;
+
+    // NOTE: This method is intentionally simplified for now, since we're only
+    // using it to stub out inserts/updates. In the future it will probably need
+    // to grow more powerful.
+
+    $this->allResults = array();
+
+    // NOTE: We jitter the insert IDs to keep tests honest; a test should cover
+    // the relationship between objects, not their exact insertion order. This
+    // guarantees that IDs are unique but makes it impossible to hard-code tests
+    // against this specific implementation detail.
+    self::$nextInsertID += mt_rand(1, 10);
+    $this->insertID = self::$nextInsertID;
+    $this->affectedRows = 1;
+  }
+
+  public function executeRawQueries(array $raw_queries) {
+    $results = array();
+    foreach ($raw_queries as $id => $raw_query) {
+      $results[$id] = array();
+    }
+    return $results;
+  }
+
+  public function getQueryTranscript() {
+    return $this->transcript;
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
new file mode 100644
index 0000000000..6faf10e2c6
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php
@@ -0,0 +1,415 @@
+configuration = $configuration;
+  }
+
+  public function __clone() {
+    $this->establishConnection();
+  }
+
+  public function openConnection() {
+    $this->requireConnection();
+  }
+
+  public function close() {
+    if ($this->lastResult) {
+      $this->lastResult = null;
+    }
+    if ($this->connection) {
+      $this->closeConnection();
+      $this->connection = null;
+    }
+  }
+
+  public function escapeColumnName($name) {
+    return '`'.str_replace('`', '``', $name).'`';
+  }
+
+
+  public function escapeMultilineComment($comment) {
+    // These can either terminate a comment, confuse the hell out of the parser,
+    // make MySQL execute the comment as a query, or, in the case of semicolon,
+    // are quasi-dangerous because the semicolon could turn a broken query into
+    // a working query plus an ignored query.
+
+    static $map = array(
+      '--'  => '(DOUBLEDASH)',
+      '*/'  => '(STARSLASH)',
+      '//'  => '(SLASHSLASH)',
+      '#'   => '(HASH)',
+      '!'   => '(BANG)',
+      ';'   => '(SEMICOLON)',
+    );
+
+    $comment = str_replace(
+      array_keys($map),
+      array_values($map),
+      $comment);
+
+    // For good measure, kill anything else that isn't a nice printable
+    // character.
+    $comment = preg_replace('/[^\x20-\x7F]+/', ' ', $comment);
+
+    return '/* '.$comment.' */';
+  }
+
+  public function escapeStringForLikeClause($value) {
+    $value = addcslashes($value, '\%_');
+    $value = $this->escapeUTF8String($value);
+    return $value;
+  }
+
+  protected function getConfiguration($key, $default = null) {
+    return idx($this->configuration, $key, $default);
+  }
+
+  private function establishConnection() {
+    $host = $this->getConfiguration('host');
+    $database = $this->getConfiguration('database');
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'      => 'connect',
+        'host'      => $host,
+        'database'  => $database,
+      ));
+
+    // If we receive these errors, we'll retry the connection up to the
+    // retry limit. For other errors, we'll fail immediately.
+    $retry_codes = array(
+      // "Connection Timeout"
+      2002 => true,
+
+      // "Unable to Connect"
+      2003 => true,
+    );
+
+    $max_retries = max(1, $this->getConfiguration('retries', 3));
+    for ($attempt = 1; $attempt <= $max_retries; $attempt++) {
+      try {
+        $conn = $this->connect();
+        $profiler->endServiceCall($call_id, array());
+        break;
+      } catch (AphrontQueryException $ex) {
+        $code = $ex->getCode();
+        if (($attempt < $max_retries) && isset($retry_codes[$code])) {
+          $message = pht(
+            'Retrying database connection to "%s" after connection '.
+            'failure (attempt %d; "%s"; error #%d): %s',
+            $host,
+            $attempt,
+            get_class($ex),
+            $code,
+            $ex->getMessage());
+
+          // See T13403. If we're silenced with the "@" operator, don't log
+          // this connection attempt. This keeps things quiet if we're
+          // running a setup workflow like "bin/config" and expect that the
+          // database credentials will often be incorrect.
+
+          if (error_reporting()) {
+            phlog($message);
+          }
+        } else {
+          $profiler->endServiceCall($call_id, array());
+          throw $ex;
+        }
+      }
+    }
+
+    $this->connection = $conn;
+  }
+
+  protected function requireConnection() {
+    if (!$this->connection) {
+      if ($this->connectionPool) {
+        $this->connection = array_pop($this->connectionPool);
+      } else {
+        $this->establishConnection();
+      }
+    }
+    return $this->connection;
+  }
+
+  protected function beginAsyncConnection() {
+    $connection = $this->requireConnection();
+    $this->connection = null;
+    return $connection;
+  }
+
+  protected function endAsyncConnection($connection) {
+    if ($this->connection) {
+      $this->connectionPool[] = $this->connection;
+    }
+    $this->connection = $connection;
+  }
+
+  public function selectAllResults() {
+    $result = array();
+    $res = $this->lastResult;
+    if ($res == null) {
+      throw new Exception(pht('No query result to fetch from!'));
+    }
+    while (($row = $this->fetchAssoc($res))) {
+      $result[] = $row;
+    }
+    return $result;
+  }
+
+  public function executeQuery(PhutilQueryString $query) {
+    $display_query = $query->getMaskedString();
+    $raw_query = $query->getUnmaskedString();
+
+    $this->lastResult = null;
+    $retries = max(1, $this->getConfiguration('retries', 3));
+    while ($retries--) {
+      try {
+        $this->requireConnection();
+        $is_write = $this->checkWrite($raw_query);
+
+        $profiler = PhutilServiceProfiler::getInstance();
+        $call_id = $profiler->beginServiceCall(
+          array(
+            'type'    => 'query',
+            'config'  => $this->configuration,
+            'query'   => $display_query,
+            'write'   => $is_write,
+          ));
+
+        $result = $this->rawQuery($raw_query);
+
+        $profiler->endServiceCall($call_id, array());
+
+        if ($this->nextError) {
+          $result = null;
+        }
+
+        if ($result) {
+          $this->lastResult = $result;
+          break;
+        }
+
+        $this->throwQueryException($this->connection);
+      } catch (AphrontConnectionLostQueryException $ex) {
+        $can_retry = ($retries > 0);
+
+        if ($this->isInsideTransaction()) {
+          // Zero out the transaction state to prevent a second exception
+          // ("program exited with open transaction") from being thrown, since
+          // we're about to throw a more relevant/useful one instead.
+          $state = $this->getTransactionState();
+          while ($state->getDepth()) {
+            $state->decreaseDepth();
+          }
+
+          $can_retry = false;
+        }
+
+        if ($this->isHoldingAnyLock()) {
+          $this->forgetAllLocks();
+          $can_retry = false;
+        }
+
+        $this->close();
+
+        if (!$can_retry) {
+          throw $ex;
+        }
+      }
+    }
+  }
+
+  public function executeRawQueries(array $raw_queries) {
+    if (!$raw_queries) {
+      return array();
+    }
+
+    $is_write = false;
+    foreach ($raw_queries as $key => $raw_query) {
+      $is_write = $is_write || $this->checkWrite($raw_query);
+      $raw_queries[$key] = rtrim($raw_query, "\r\n\t ;");
+    }
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'    => 'multi-query',
+        'config'  => $this->configuration,
+        'queries' => $raw_queries,
+        'write'   => $is_write,
+      ));
+
+    $results = $this->rawQueries($raw_queries);
+
+    $profiler->endServiceCall($call_id, array());
+
+    return $results;
+  }
+
+  protected function processResult($result) {
+    if (!$result) {
+      try {
+        $this->throwQueryException($this->requireConnection());
+      } catch (Exception $ex) {
+        return $ex;
+      }
+    } else if (is_bool($result)) {
+      return $this->getAffectedRows();
+    }
+    $rows = array();
+    while (($row = $this->fetchAssoc($result))) {
+      $rows[] = $row;
+    }
+    $this->freeResult($result);
+    return $rows;
+  }
+
+  protected function checkWrite($raw_query) {
+    // NOTE: The opening "(" allows queries in the form of:
+    //
+    //   (SELECT ...) UNION (SELECT ...)
+    $is_write = !preg_match('/^[(]*(SELECT|SHOW|EXPLAIN)\s/', $raw_query);
+    if ($is_write) {
+      if ($this->getReadOnly()) {
+        throw new Exception(
+          pht(
+            'Attempting to issue a write query on a read-only '.
+            'connection (to database "%s")!',
+            $this->getConfiguration('database')));
+      }
+      AphrontWriteGuard::willWrite();
+      return true;
+    }
+
+    return false;
+  }
+
+  protected function throwQueryException($connection) {
+    if ($this->nextError) {
+      $errno = $this->nextError;
+      $error = pht('Simulated error.');
+      $this->nextError = null;
+    } else {
+      $errno = $this->getErrorCode($connection);
+      $error = $this->getErrorDescription($connection);
+    }
+    $this->throwQueryCodeException($errno, $error);
+  }
+
+  private function throwCommonException($errno, $error) {
+    $message = pht('#%d: %s', $errno, $error);
+
+    switch ($errno) {
+      case 2013: // Connection Dropped
+        throw new AphrontConnectionLostQueryException($message);
+      case 2006: // Gone Away
+        $more = pht(
+          'This error may occur if your configured MySQL "wait_timeout" or '.
+          '"max_allowed_packet" values are too small. This may also indicate '.
+          'that something used the MySQL "KILL " command to kill '.
+          'the connection running the query.');
+        throw new AphrontConnectionLostQueryException("{$message}\n\n{$more}");
+      case 1213: // Deadlock
+        throw new AphrontDeadlockQueryException($message);
+      case 1205: // Lock wait timeout exceeded
+        throw new AphrontLockTimeoutQueryException($message);
+      case 1062: // Duplicate Key
+        // NOTE: In some versions of MySQL we get a key name back here, but
+        // older versions just give us a key index ("key 2") so it's not
+        // portable to parse the key out of the error and attach it to the
+        // exception.
+        throw new AphrontDuplicateKeyQueryException($message);
+      case 1044: // Access denied to database
+      case 1142: // Access denied to table
+      case 1143: // Access denied to column
+      case 1227: // Access denied (e.g., no SUPER for SHOW SLAVE STATUS).
+        throw new AphrontAccessDeniedQueryException($message);
+      case 1045: // Access denied (auth)
+        throw new AphrontInvalidCredentialsQueryException($message);
+      case 1146: // No such table
+      case 1049: // No such database
+      case 1054: // Unknown column "..." in field list
+        throw new AphrontSchemaQueryException($message);
+    }
+
+    // TODO: 1064 is syntax error, and quite terrible in production.
+
+    return null;
+  }
+
+  protected function throwConnectionException($errno, $error, $user, $host) {
+    $this->throwCommonException($errno, $error);
+
+    $message = pht(
+      'Attempt to connect to %s@%s failed with error #%d: %s.',
+      $user,
+      $host,
+      $errno,
+      $error);
+
+    throw new AphrontConnectionQueryException($message, $errno);
+  }
+
+
+  protected function throwQueryCodeException($errno, $error) {
+    $this->throwCommonException($errno, $error);
+
+    $message = pht(
+      '#%d: %s',
+      $errno,
+      $error);
+
+    throw new AphrontQueryException($message, $errno);
+  }
+
+  /**
+   * Force the next query to fail with a simulated error. This should be used
+   * ONLY for unit tests.
+   */
+  public function simulateErrorOnNextQuery($error) {
+    $this->nextError = $error;
+    return $this;
+  }
+
+  /**
+   * Check inserts for characters outside of the BMP. Even with the strictest
+   * settings, MySQL will silently truncate data when it encounters these, which
+   * can lead to data loss and security problems.
+   */
+  protected function validateUTF8String($string) {
+    if (phutil_is_utf8($string)) {
+      return;
+    }
+
+    throw new AphrontCharacterSetQueryException(
+      pht(
+        'Attempting to construct a query using a non-utf8 string when '.
+        'utf8 is expected. Use the `%%B` conversion to escape binary '.
+        'strings data.'));
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php
new file mode 100644
index 0000000000..40a2c6c357
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontMySQLDatabaseConnection.php
@@ -0,0 +1,233 @@
+validateUTF8String($string);
+    return $this->escapeBinaryString($string);
+  }
+
+  public function escapeBinaryString($string) {
+    return mysql_real_escape_string($string, $this->requireConnection());
+  }
+
+  public function getInsertID() {
+    return mysql_insert_id($this->requireConnection());
+  }
+
+  public function getAffectedRows() {
+    return mysql_affected_rows($this->requireConnection());
+  }
+
+  protected function closeConnection() {
+    mysql_close($this->requireConnection());
+  }
+
+  protected function connect() {
+    if (!function_exists('mysql_connect')) {
+      // We have to '@' the actual call since it can spew all sorts of silly
+      // noise, but it will also silence fatals caused by not having MySQL
+      // installed, which has bitten me on three separate occasions. Make sure
+      // such failures are explicit and loud.
+      throw new Exception(
+        pht(
+          'About to call %s, but the PHP MySQL extension is not available!',
+          'mysql_connect()'));
+    }
+
+    $user = $this->getConfiguration('user');
+    $host = $this->getConfiguration('host');
+    $port = $this->getConfiguration('port');
+
+    if ($port) {
+      $host .= ':'.$port;
+    }
+
+    $database = $this->getConfiguration('database');
+
+    $pass = $this->getConfiguration('pass');
+    if ($pass instanceof PhutilOpaqueEnvelope) {
+      $pass = $pass->openEnvelope();
+    }
+
+    $timeout = $this->getConfiguration('timeout');
+    $timeout_ini = 'mysql.connect_timeout';
+    if ($timeout) {
+      $old_timeout = ini_get($timeout_ini);
+      ini_set($timeout_ini, $timeout);
+    }
+
+    try {
+      $conn = @mysql_connect(
+        $host,
+        $user,
+        $pass,
+        $new_link = true,
+        $flags = 0);
+    } catch (Exception $ex) {
+      if ($timeout) {
+        ini_set($timeout_ini, $old_timeout);
+      }
+      throw $ex;
+    }
+
+    if ($timeout) {
+      ini_set($timeout_ini, $old_timeout);
+    }
+
+    if (!$conn) {
+      $errno = mysql_errno();
+      $error = mysql_error();
+      $this->throwConnectionException($errno, $error, $user, $host);
+    }
+
+    if ($database !== null) {
+      $ret = @mysql_select_db($database, $conn);
+      if (!$ret) {
+        $this->throwQueryException($conn);
+      }
+    }
+
+    $ok = @mysql_set_charset('utf8mb4', $conn);
+    if (!$ok) {
+      mysql_set_charset('binary', $conn);
+    }
+
+    return $conn;
+  }
+
+  protected function rawQuery($raw_query) {
+    return @mysql_query($raw_query, $this->requireConnection());
+  }
+
+  /**
+   * @phutil-external-symbol function mysql_multi_query
+   * @phutil-external-symbol function mysql_fetch_result
+   * @phutil-external-symbol function mysql_more_results
+   * @phutil-external-symbol function mysql_next_result
+   */
+  protected function rawQueries(array $raw_queries) {
+    $conn = $this->requireConnection();
+    $results = array();
+
+    if (!function_exists('mysql_multi_query')) {
+      foreach ($raw_queries as $key => $raw_query) {
+        $results[$key] = $this->processResult($this->rawQuery($raw_query));
+      }
+      return $results;
+    }
+
+    if (!mysql_multi_query(implode("\n;\n\n", $raw_queries), $conn)) {
+      $ex = $this->processResult(false);
+      return array_fill_keys(array_keys($raw_queries), $ex);
+    }
+
+    $processed_all = false;
+    foreach ($raw_queries as $key => $raw_query) {
+      $results[$key] = $this->processResult(@mysql_fetch_result($conn));
+      if (!mysql_more_results($conn)) {
+        $processed_all = true;
+        break;
+      }
+      mysql_next_result($conn);
+    }
+
+    if (!$processed_all) {
+      throw new Exception(
+        pht('There are some results left in the result set.'));
+    }
+
+    return $results;
+  }
+
+  protected function freeResult($result) {
+    mysql_free_result($result);
+  }
+
+  public function supportsParallelQueries() {
+    // fb_parallel_query() doesn't support results with different columns.
+    return false;
+  }
+
+  /**
+   * @phutil-external-symbol function fb_parallel_query
+   */
+  public function executeParallelQueries(
+    array $queries,
+    array $conns = array()) {
+    assert_instances_of($conns, __CLASS__);
+
+    $map = array();
+    $is_write = false;
+    foreach ($queries as $id => $query) {
+      $is_write = $is_write || $this->checkWrite($query);
+      $conn = idx($conns, $id, $this);
+
+      $host = $conn->getConfiguration('host');
+      $port = 0;
+      $match = null;
+      if (preg_match('/(.+):(.+)/', $host, $match)) {
+        list(, $host, $port) = $match;
+      }
+
+      $pass = $conn->getConfiguration('pass');
+      if ($pass instanceof PhutilOpaqueEnvelope) {
+        $pass = $pass->openEnvelope();
+      }
+
+      $map[$id] = array(
+        'sql' => $query,
+        'ip' => $host,
+        'port' => $port,
+        'username' => $conn->getConfiguration('user'),
+        'password' => $pass,
+        'db' => $conn->getConfiguration('database'),
+      );
+    }
+
+    $profiler = PhutilServiceProfiler::getInstance();
+    $call_id = $profiler->beginServiceCall(
+      array(
+        'type'    => 'multi-query',
+        'queries' => $queries,
+        'write'   => $is_write,
+      ));
+
+    $map = fb_parallel_query($map);
+
+    $profiler->endServiceCall($call_id, array());
+
+    $results = array();
+    $pos = 0;
+    $err_pos = 0;
+    foreach ($queries as $id => $query) {
+      $errno = idx(idx($map, 'errno', array()), $err_pos);
+      $err_pos++;
+      if ($errno) {
+        try {
+          $this->throwQueryCodeException($errno, $map['error'][$id]);
+        } catch (Exception $ex) {
+          $results[$id] = $ex;
+        }
+        continue;
+      }
+      $results[$id] = $map['result'][$pos];
+      $pos++;
+    }
+    return $results;
+  }
+
+  protected function fetchAssoc($result) {
+    return mysql_fetch_assoc($result);
+  }
+
+  protected function getErrorCode($connection) {
+    return mysql_errno($connection);
+  }
+
+  protected function getErrorDescription($connection) {
+    return mysql_error($connection);
+  }
+
+}
diff --git a/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
new file mode 100644
index 0000000000..6a0bc759a7
--- /dev/null
+++ b/src/infrastructure/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php
@@ -0,0 +1,272 @@
+validateUTF8String($string);
+    return $this->escapeBinaryString($string);
+  }
+
+  public function escapeBinaryString($string) {
+    return $this->requireConnection()->escape_string($string);
+  }
+
+  public function getInsertID() {
+    return $this->requireConnection()->insert_id;
+  }
+
+  public function getAffectedRows() {
+    return $this->requireConnection()->affected_rows;
+  }
+
+  protected function closeConnection() {
+    if ($this->connectionOpen) {
+      $this->requireConnection()->close();
+      $this->connectionOpen = false;
+    }
+  }
+
+  protected function connect() {
+    if (!class_exists('mysqli', false)) {
+      throw new Exception(pht(
+        'About to call new %s, but the PHP MySQLi extension is not available!',
+        'mysqli()'));
+    }
+
+    $user = $this->getConfiguration('user');
+    $host = $this->getConfiguration('host');
+    $port = $this->getConfiguration('port');
+    $database = $this->getConfiguration('database');
+
+    $pass = $this->getConfiguration('pass');
+    if ($pass instanceof PhutilOpaqueEnvelope) {
+      $pass = $pass->openEnvelope();
+    }
+
+    // If the host is "localhost", the port is ignored and mysqli attempts to
+    // connect over a socket.
+    if ($port) {
+      if ($host === 'localhost' || $host === null) {
+        $host = '127.0.0.1';
+      }
+    }
+
+    $conn = mysqli_init();
+
+    $timeout = $this->getConfiguration('timeout');
+    if ($timeout) {
+      $conn->options(MYSQLI_OPT_CONNECT_TIMEOUT, $timeout);
+    }
+
+    if ($this->getPersistent()) {
+      $host = 'p:'.$host;
+    }
+
+    $trap = new PhutilErrorTrap();
+
+    $ok = @$conn->real_connect(
+      $host,
+      $user,
+      $pass,
+      $database,
+      $port);
+
+    $call_error = $trap->getErrorsAsString();
+    $trap->destroy();
+
+    $errno = $conn->connect_errno;
+    if ($errno) {
+      $error = $conn->connect_error;
+      $this->throwConnectionException($errno, $error, $user, $host);
+    }
+
+    // See T13403. If the parameters to "real_connect()" are wrong, it may
+    // fail without setting an error code. In this case, raise a generic
+    // exception. (One way to reproduce this is to pass a string to the
+    // "port" parameter.)
+
+    if (!$ok) {
+      if (strlen($call_error)) {
+        $message = pht(
+          'mysqli->real_connect() failed: %s',
+          $call_error);
+      } else {
+        $message = pht(
+          'mysqli->real_connect() failed, but did not set an error code '.
+          'or emit a message.');
+      }
+
+      $this->throwConnectionException(
+        self::CALLERROR_CONNECT,
+        $message,
+        $user,
+        $host);
+    }
+
+    // See T13238. Attempt to prevent "LOAD DATA LOCAL INFILE", which allows a
+    // malicious server to ask the client for any file. At time of writing,
+    // this option MUST be set after "real_connect()" on all PHP versions.
+    $conn->options(MYSQLI_OPT_LOCAL_INFILE, 0);
+
+    $this->connectionOpen = true;
+
+    $ok = @$conn->set_charset('utf8mb4');
+    if (!$ok) {
+      $ok = $conn->set_charset('binary');
+    }
+
+    return $conn;
+  }
+
+  protected function rawQuery($raw_query) {
+    $conn = $this->requireConnection();
+    $time_limit = $this->getQueryTimeout();
+
+    // If we have a query time limit, run this query synchronously but use
+    // the async API. This allows us to kill queries which take too long
+    // without requiring any configuration on the server side.
+    if ($time_limit && $this->supportsAsyncQueries()) {
+      $conn->query($raw_query, MYSQLI_ASYNC);
+
+      $read = array($conn);
+      $error = array($conn);
+      $reject = array($conn);
+
+      $result = mysqli::poll($read, $error, $reject, $time_limit);
+
+      if ($result === false) {
+        $this->closeConnection();
+        throw new Exception(
+          pht('Failed to poll mysqli connection!'));
+      } else if ($result === 0) {
+        $this->closeConnection();
+        throw new AphrontQueryTimeoutQueryException(
+          pht(
+            'Query timed out after %s second(s)!',
+            new PhutilNumber($time_limit)));
+      }
+
+      return @$conn->reap_async_query();
+    }
+
+    $trap = new PhutilErrorTrap();
+
+    $result = @$conn->query($raw_query);
+
+    $err = $trap->getErrorsAsString();
+    $trap->destroy();
+
+    // See T13238 and PHI1014. Sometimes, the call to "$conn->query()" may fail
+    // without setting an error code on the connection. One way to reproduce
+    // this is to use "LOAD DATA LOCAL INFILE" with "mysqli.allow_local_infile"
+    // disabled.
+
+    // If we have no result and no error code, raise a synthetic query error
+    // with whatever error message was raised as a local PHP warning.
+
+    if (!$result) {
+      $error_code = $this->getErrorCode($conn);
+      if (!$error_code) {
+        if (strlen($err)) {
+          $message = $err;
+         } else {
+          $message = pht(
+            'Call to "mysqli->query()" failed, but did not set an error '.
+            'code or emit an error message.');
+        }
+        $this->throwQueryCodeException(self::CALLERROR_QUERY, $message);
+      }
+    }
+
+    return $result;
+  }
+
+  protected function rawQueries(array $raw_queries) {
+    $conn = $this->requireConnection();
+
+    $have_result = false;
+    $results = array();
+
+    foreach ($raw_queries as $key => $raw_query) {
+      if (!$have_result) {
+        // End line in front of semicolon to allow single line comments at the
+        // end of queries.
+        $have_result = $conn->multi_query(implode("\n;\n\n", $raw_queries));
+      } else {
+        $have_result = $conn->next_result();
+      }
+
+      array_shift($raw_queries);
+
+      $result = $conn->store_result();
+      if (!$result && !$this->getErrorCode($conn)) {
+        $result = true;
+      }
+      $results[$key] = $this->processResult($result);
+    }
+
+    if ($conn->more_results()) {
+      throw new Exception(
+        pht('There are some results left in the result set.'));
+    }
+
+    return $results;
+  }
+
+  protected function freeResult($result) {
+    $result->free_result();
+  }
+
+  protected function fetchAssoc($result) {
+    return $result->fetch_assoc();
+  }
+
+  protected function getErrorCode($connection) {
+    return $connection->errno;
+  }
+
+  protected function getErrorDescription($connection) {
+    return $connection->error;
+  }
+
+  public function supportsAsyncQueries() {
+    return defined('MYSQLI_ASYNC');
+  }
+
+  public function asyncQuery($raw_query) {
+    $this->checkWrite($raw_query);
+    $async = $this->beginAsyncConnection();
+    $async->query($raw_query, MYSQLI_ASYNC);
+    return $async;
+  }
+
+  public static function resolveAsyncQueries(array $conns, array $asyncs) {
+    assert_instances_of($conns, __CLASS__);
+    assert_instances_of($asyncs, 'mysqli');
+
+    $read = $error = $reject = array();
+    foreach ($asyncs as $async) {
+      $read[] = $error[] = $reject[] = $async;
+    }
+
+    if (!mysqli::poll($read, $error, $reject, 0)) {
+      return array();
+    }
+
+    $results = array();
+    foreach ($read as $async) {
+      $key = array_search($async, $asyncs, $strict = true);
+      $conn = $conns[$key];
+      $conn->endAsyncConnection($async);
+      $results[$key] = $conn->processResult($async->reap_async_query());
+    }
+    return $results;
+  }
+
+}
diff --git a/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php b/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php
new file mode 100644
index 0000000000..a7ca91cfde
--- /dev/null
+++ b/src/infrastructure/storage/exception/AphrontAccessDeniedQueryException.php
@@ -0,0 +1,4 @@
+query = $query;
+  }
+
+  public function getQuery() {
+    return $this->query;
+  }
+
+}
diff --git a/src/infrastructure/storage/exception/AphrontQueryException.php b/src/infrastructure/storage/exception/AphrontQueryException.php
new file mode 100644
index 0000000000..6b0f51ad02
--- /dev/null
+++ b/src/infrastructure/storage/exception/AphrontQueryException.php
@@ -0,0 +1,6 @@
+resolve();
+ *     } catch (AphrontQueryException $ex) {
+ *     }
+ *   }
+ *
+ * `$result` contains a list of dicts for select queries or number of modified
+ * rows for modification queries.
+ */
+final class QueryFuture extends Future {
+
+  private static $futures = array();
+
+  private $conn;
+  private $query;
+  private $id;
+  private $async;
+  private $profilerCallID;
+
+  public function __construct(
+    AphrontDatabaseConnection $conn,
+    $pattern/* , ... */) {
+
+    $this->conn = $conn;
+
+    $args = func_get_args();
+    $args = array_slice($args, 2);
+    $this->query = vqsprintf($conn, $pattern, $args);
+
+    self::$futures[] = $this;
+    $this->id = last_key(self::$futures);
+  }
+
+  public function isReady() {
+    if ($this->result !== null || $this->exception) {
+      return true;
+    }
+
+    if (!$this->conn->supportsAsyncQueries()) {
+      if ($this->conn->supportsParallelQueries()) {
+        $queries = array();
+        $conns = array();
+        foreach (self::$futures as $id => $future) {
+          $queries[$id] = $future->query;
+          $conns[$id] = $future->conn;
+        }
+        $results = $this->conn->executeParallelQueries($queries, $conns);
+        $this->processResults($results);
+        return true;
+      }
+
+      $conns = array();
+      $conn_queries = array();
+      foreach (self::$futures as $id => $future) {
+        $hash = spl_object_hash($future->conn);
+        $conns[$hash] = $future->conn;
+        $conn_queries[$hash][$id] = $future->query;
+      }
+      foreach ($conn_queries as $hash => $queries) {
+        $this->processResults($conns[$hash]->executeRawQueries($queries));
+      }
+      return true;
+    }
+
+    if (!$this->async) {
+      $profiler = PhutilServiceProfiler::getInstance();
+      $this->profilerCallID = $profiler->beginServiceCall(
+        array(
+          'type'    => 'query',
+          'query'   => $this->query,
+          'async'   => true,
+        ));
+
+      $this->async = $this->conn->asyncQuery($this->query);
+      return false;
+    }
+
+    $conns = array();
+    $asyncs = array();
+    foreach (self::$futures as $id => $future) {
+      if ($future->async) {
+        $conns[$id] = $future->conn;
+        $asyncs[$id] = $future->async;
+      }
+    }
+
+    $this->processResults($this->conn->resolveAsyncQueries($conns, $asyncs));
+
+    if ($this->result !== null || $this->exception) {
+      return true;
+    }
+    return false;
+  }
+
+  private function processResults(array $results) {
+    foreach ($results as $id => $result) {
+      $future = self::$futures[$id];
+      if ($result instanceof Exception) {
+        $future->exception = $result;
+      } else {
+        $future->result = $result;
+      }
+      unset(self::$futures[$id]);
+      if ($future->profilerCallID) {
+        $profiler = PhutilServiceProfiler::getInstance();
+        $profiler->endServiceCall($future->profilerCallID, array());
+      }
+    }
+  }
+}
diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
index b838c8a5d9..a6a0d74593 100644
--- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
+++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
@@ -89,6 +89,10 @@ final class PhabricatorStorageManagementAPI extends Phobject {
     return $this->namespace.'_'.$fragment;
   }
 
+  public function getDisplayName() {
+    return $this->getRef()->getDisplayName();
+  }
+
   public function getDatabaseList(array $patches, $only_living = false) {
     assert_instances_of($patches, 'PhabricatorStoragePatch');
 
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
index 7d0946c8c3..9b718e231d 100644
--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
@@ -21,86 +21,112 @@ final class PhabricatorStorageManagementDestroyWorkflow
   }
 
   public function didExecute(PhutilArgumentParser $args) {
-    $console = PhutilConsole::getConsole();
+    $api = $this->getSingleAPI();
+
+    $host_display = $api->getDisplayName();
 
     if (!$this->isDryRun() && !$this->isForce()) {
       if ($args->getArg('unittest-fixtures')) {
-        $console->writeOut(
-          phutil_console_wrap(
-            pht(
-              'Are you completely sure you really want to destroy all unit '.
-              'test fixure data? This operation can not be undone.')));
+        $warning = pht(
+          'Are you completely sure you really want to destroy all unit '.
+          'test fixure data on host "%s"? This operation can not be undone.',
+          $host_display);
+
+        echo tsprintf(
+          '%B',
+          id(new PhutilConsoleBlock())
+            ->addParagraph($warning)
+            ->drawConsoleString());
+
         if (!phutil_console_confirm(pht('Destroy all unit test data?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
       } else {
-        $console->writeOut(
-          phutil_console_wrap(
-            pht(
-              'Are you completely sure you really want to permanently destroy '.
-              'all storage for Phabricator data? This operation can not be '.
-              'undone and your data will not be recoverable if you proceed.')));
+        $warning = pht(
+          'Are you completely sure you really want to permanently destroy '.
+          'all storage for Phabricator data on host "%s"? This operation '.
+          'can not be undone and your data will not be recoverable if '.
+          'you proceed.',
+          $host_display);
+
+        echo tsprintf(
+          '%B',
+          id(new PhutilConsoleBlock())
+            ->addParagraph($warning)
+            ->drawConsoleString());
 
         if (!phutil_console_confirm(pht('Permanently destroy all data?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
 
         if (!phutil_console_confirm(pht('Really destroy all data forever?'))) {
-          $console->writeOut("%s\n", pht('Cancelled.'));
+          $this->logFail(
+            pht('CANCELLED'),
+            pht('User cancelled operation.'));
           exit(1);
         }
       }
     }
 
-    $apis = $this->getMasterAPIs();
-    foreach ($apis as $api) {
-      $patches = $this->getPatches();
+    $patches = $this->getPatches();
 
-      if ($args->getArg('unittest-fixtures')) {
-        $conn = $api->getConn(null);
-        $databases = queryfx_all(
-          $conn,
-          'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
-          'FROM INFORMATION_SCHEMA.TABLES '.
-          'WHERE TABLE_SCHEMA LIKE %>',
-          PhabricatorTestCase::NAMESPACE_PREFIX);
-        $databases = ipull($databases, 'db');
-      } else {
-        $databases   = $api->getDatabaseList($patches);
-        $databases[] = $api->getDatabaseName('meta_data');
+    if ($args->getArg('unittest-fixtures')) {
+      $conn = $api->getConn(null);
+      $databases = queryfx_all(
+        $conn,
+        'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
+        'FROM INFORMATION_SCHEMA.TABLES '.
+        'WHERE TABLE_SCHEMA LIKE %>',
+        PhabricatorTestCase::NAMESPACE_PREFIX);
+      $databases = ipull($databases, 'db');
+    } else {
+      $databases = $api->getDatabaseList($patches);
+      $databases[] = $api->getDatabaseName('meta_data');
 
-        // These are legacy databases that were dropped long ago. See T2237.
-        $databases[] = $api->getDatabaseName('phid');
-        $databases[] = $api->getDatabaseName('directory');
-      }
+      // These are legacy databases that were dropped long ago. See T2237.
+      $databases[] = $api->getDatabaseName('phid');
+      $databases[] = $api->getDatabaseName('directory');
+    }
 
-      foreach ($databases as $database) {
-        if ($this->isDryRun()) {
-          $console->writeOut(
-            "%s\n",
-            pht("DRYRUN: Would drop database '%s'.", $database));
-        } else {
-          $console->writeOut(
-            "%s\n",
-            pht("Dropping database '%s'...", $database));
-          queryfx(
-            $api->getConn(null),
-            'DROP DATABASE IF EXISTS %T',
-            $database);
-        }
-      }
+    asort($databases);
 
-      if (!$this->isDryRun()) {
-        $console->writeOut(
-          "%s\n",
+    foreach ($databases as $database) {
+      if ($this->isDryRun()) {
+        $this->logInfo(
+          pht('DRY RUN'),
           pht(
-            'Storage on "%s" was destroyed.',
-            $api->getRef()->getRefKey()));
+            'Would drop database "%s" on host "%s".',
+            $database,
+            $host_display));
+      } else {
+        $this->logWarn(
+          pht('DESTROY'),
+          pht(
+            'Dropping database "%s" on host "%s"...',
+            $database,
+            $host_display));
+
+        queryfx(
+          $api->getConn(null),
+          'DROP DATABASE IF EXISTS %T',
+          $database);
       }
     }
 
+    if (!$this->isDryRun()) {
+      $this->logOkay(
+        pht('DONE'),
+        pht(
+          'Storage on "%s" was destroyed.',
+          $host_display));
+    }
+
     return 0;
   }
 
diff --git a/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php
new file mode 100644
index 0000000000..4a4cf986c4
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRef.php
@@ -0,0 +1,23 @@
+database = $database;
+    $this->table = $table;
+  }
+
+  public function getAphrontRefDatabaseName() {
+    return $this->database;
+  }
+
+  public function getAphrontRefTableName() {
+    return $this->table;
+  }
+
+}
diff --git a/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php
new file mode 100644
index 0000000000..851cd46294
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/AphrontDatabaseTableRefInterface.php
@@ -0,0 +1,8 @@
+setTableName('X');
+    //   $query = qsprintf($conn, '%R', $object);
+    //   $object->setTableName('Y');
+    //
+    // We'd like "$query" to reference "X", reflecting the object as it
+    // existed when it was passed to "qsprintf(...)". It's surprising if the
+    // modification to the object after "qsprintf(...)" can affect "$query".
+
+    $masked_string = xsprintf(
+      'xsprintf_query',
+      array(
+        'escaper' => $escaper,
+        'unmasked' => false,
+      ),
+      $argv);
+
+    $unmasked_string = xsprintf(
+      'xsprintf_query',
+      array(
+        'escaper' => $escaper,
+        'unmasked' => true,
+      ),
+      $argv);
+
+    $this->maskedString = $masked_string;
+    $this->unmaskedString = $unmasked_string;
+  }
+
+  public function __toString() {
+    return $this->getMaskedString();
+  }
+
+  public function getUnmaskedString() {
+    return $this->unmaskedString;
+  }
+
+  public function getMaskedString() {
+    return $this->maskedString;
+  }
+
+}
diff --git a/src/infrastructure/storage/xsprintf/qsprintf.php b/src/infrastructure/storage/xsprintf/qsprintf.php
new file mode 100644
index 0000000000..7b46b34cd4
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/qsprintf.php
@@ -0,0 +1,516 @@
+ and %<.
+ *
+ *   %> ("Prefix")
+ *     Escapes a prefix query for a LIKE clause. For example:
+ *
+ *       //  Find all rows where `name` starts with $prefix.
+ *       qsprintf($escaper, 'WHERE name LIKE %>', $prefix);
+ *
+ *   %< ("Suffix")
+ *     Escapes a suffix query for a LIKE clause. For example:
+ *
+ *       //  Find all rows where `name` ends with $suffix.
+ *       qsprintf($escaper, 'WHERE name LIKE %<', $suffix);
+ *
+ *   %T ("Table")
+ *     Escapes a table name. In most cases, you should use "%R" instead.
+ */
+function qsprintf(PhutilQsprintfInterface $escaper, $pattern /* , ... */) {
+  $args = func_get_args();
+  array_shift($args);
+  return new PhutilQueryString($escaper, $args);
+}
+
+function vqsprintf(PhutilQsprintfInterface $escaper, $pattern, array $argv) {
+  array_unshift($argv, $pattern);
+  return new PhutilQueryString($escaper, $argv);
+}
+
+/**
+ * @{function:xsprintf} callback for encoding SQL queries. See
+ * @{function:qsprintf}.
+ */
+function xsprintf_query($userdata, &$pattern, &$pos, &$value, &$length) {
+  $type = $pattern[$pos];
+
+  if (is_array($userdata)) {
+    $escaper = $userdata['escaper'];
+    $unmasked = $userdata['unmasked'];
+  } else {
+    $escaper = $userdata;
+    $unmasked = false;
+  }
+
+  $next = (strlen($pattern) > $pos + 1) ? $pattern[$pos + 1] : null;
+  $nullable = false;
+  $done = false;
+
+  $prefix   = '';
+
+  if (!($escaper instanceof PhutilQsprintfInterface)) {
+    throw new InvalidArgumentException(pht('Invalid database escaper.'));
+  }
+
+  switch ($type) {
+    case '=': // Nullable test
+      switch ($next) {
+        case 'd':
+        case 'f':
+        case 's':
+          $pattern = substr_replace($pattern, '', $pos, 1);
+          $length  = strlen($pattern);
+          $type    = 's';
+          if ($value === null) {
+            $value = 'IS NULL';
+            $done = true;
+          } else {
+            $prefix = '= ';
+            $type = $next;
+          }
+          break;
+        default:
+          throw new Exception(
+            pht(
+              'Unknown conversion, try %s, %s, or %s.',
+              '%=d',
+              '%=s',
+              '%=f'));
+      }
+      break;
+
+    case 'n': // Nullable...
+      switch ($next) {
+        case 'd': //  ...integer.
+        case 'f': //  ...float.
+        case 's': //  ...string.
+        case 'B': //  ...binary string.
+          $pattern = substr_replace($pattern, '', $pos, 1);
+          $length = strlen($pattern);
+          $type = $next;
+          $nullable = true;
+          break;
+        default:
+          throw new XsprintfUnknownConversionException("%n{$next}");
+      }
+      break;
+
+    case 'L': // List of..
+      qsprintf_check_type($value, "L{$next}", $pattern);
+      $pattern = substr_replace($pattern, '', $pos, 1);
+      $length  = strlen($pattern);
+      $type = 's';
+      $done = true;
+
+      switch ($next) {
+        case 'd': //  ...integers.
+          $value = implode(', ', array_map('intval', $value));
+          break;
+        case 'f': // ...floats.
+          $value = implode(', ', array_map('floatval', $value));
+          break;
+        case 's': // ...strings.
+          foreach ($value as $k => $v) {
+            $value[$k] = "'".$escaper->escapeUTF8String((string)$v)."'";
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'B': // ...binary strings.
+          foreach ($value as $k => $v) {
+            $value[$k] = "'".$escaper->escapeBinaryString((string)$v)."'";
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'C': // ...columns.
+          foreach ($value as $k => $v) {
+            $value[$k] = $escaper->escapeColumnName($v);
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'K': // ...key columns.
+          // This is like "%LC", but for escaping column lists passed to key
+          // specifications. These should be escaped as "`column`(123)". For
+          // example:
+          //
+          //   ALTER TABLE `x` ADD KEY `y` (`u`(16), `v`(32));
+
+          foreach ($value as $k => $v) {
+            $matches = null;
+            if (preg_match('/\((\d+)\)\z/', $v, $matches)) {
+              $v = substr($v, 0, -(strlen($matches[1]) + 2));
+              $prefix_len = '('.((int)$matches[1]).')';
+            } else {
+              $prefix_len = '';
+            }
+
+            $value[$k] = $escaper->escapeColumnName($v).$prefix_len;
+          }
+
+          $value = implode(', ', $value);
+          break;
+        case 'Q':
+          // TODO: Here, and in "%LO", "%LA", and "%LJ", we should eventually
+          // stop accepting strings.
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          $value = implode(', ', $value);
+          break;
+        case 'O':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          if (count($value) == 1) {
+            $value = '('.head($value).')';
+          } else {
+            $value = '(('.implode(') OR (', $value).'))';
+          }
+          break;
+        case 'A':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          if (count($value) == 1) {
+            $value = '('.head($value).')';
+          } else {
+            $value = '(('.implode(') AND (', $value).'))';
+          }
+          break;
+        case 'J':
+          foreach ($value as $k => $v) {
+            if (is_string($v)) {
+              continue;
+            }
+            $value[$k] = $v->getUnmaskedString();
+          }
+          $value = implode(' ', $value);
+          break;
+        default:
+          throw new XsprintfUnknownConversionException("%L{$next}");
+      }
+      break;
+  }
+
+  if (!$done) {
+    qsprintf_check_type($value, $type, $pattern);
+    switch ($type) {
+      case 's': // String
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = "'".$escaper->escapeUTF8String((string)$value)."'";
+        }
+        $type = 's';
+        break;
+
+      case 'B': // Binary String
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = "'".$escaper->escapeBinaryString((string)$value)."'";
+        }
+        $type = 's';
+        break;
+
+      case 'Q': // Query Fragment
+        if ($value instanceof PhutilQueryString) {
+          $value = $value->getUnmaskedString();
+        }
+        $type = 's';
+        break;
+
+      case 'Z': // Raw Query Fragment
+        $type = 's';
+        break;
+
+      case '~': // Like Substring
+      case '>': // Like Prefix
+      case '<': // Like Suffix
+        $value = $escaper->escapeStringForLikeClause($value);
+        switch ($type) {
+          case '~': $value = "'%".$value."%'"; break;
+          case '>': $value = "'".$value."%'"; break;
+          case '<': $value = "'%".$value."'"; break;
+        }
+        $type  = 's';
+        break;
+
+      case 'f': // Float
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = (float)$value;
+        }
+        $type = 's';
+        break;
+
+      case 'd': // Integer
+        if ($nullable && $value === null) {
+          $value = 'NULL';
+        } else {
+          $value = (int)$value;
+        }
+        $type = 's';
+        break;
+
+      case 'T': // Table
+      case 'C': // Column
+        $value = $escaper->escapeColumnName($value);
+        $type = 's';
+        break;
+
+      case 'K': // Komment
+        $value = $escaper->escapeMultilineComment($value);
+        $type = 's';
+        break;
+
+      case 'R': // Database + Table Reference
+        $database_name = $value->getAphrontRefDatabaseName();
+        $database_name = $escaper->escapeColumnName($database_name);
+
+        $table_name = $value->getAphrontRefTableName();
+        $table_name = $escaper->escapeColumnName($table_name);
+
+        $value = $database_name.'.'.$table_name;
+        $type = 's';
+        break;
+
+      case 'P': // Password or Secret
+        if ($unmasked) {
+          $value = $value->openEnvelope();
+          $value = "'".$escaper->escapeUTF8String($value)."'";
+        } else {
+          $value = '********';
+        }
+        $type = 's';
+        break;
+
+      default:
+        throw new XsprintfUnknownConversionException($type);
+    }
+  }
+
+  if ($prefix) {
+    $value = $prefix.$value;
+  }
+
+  $pattern[$pos] = $type;
+}
+
+function qsprintf_check_type($value, $type, $query) {
+  switch ($type) {
+    case 'Ld':
+    case 'Ls':
+    case 'LC':
+    case 'LK':
+    case 'LB':
+    case 'Lf':
+    case 'LQ':
+    case 'LA':
+    case 'LO':
+    case 'LJ':
+      if (!is_array($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected array argument for %%%s conversion.', $type));
+      }
+      if (empty($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Array for %%%s conversion is empty.', $type));
+      }
+
+      foreach ($value as $scalar) {
+        qsprintf_check_scalar_type($scalar, $type, $query);
+      }
+      break;
+    default:
+      qsprintf_check_scalar_type($value, $type, $query);
+      break;
+  }
+}
+
+function qsprintf_check_scalar_type($value, $type, $query) {
+  switch ($type) {
+    case 'LQ':
+    case 'LA':
+    case 'LO':
+    case 'LJ':
+      // TODO: See T13217. Remove this eventually.
+      if (is_string($value)) {
+        phlog(
+          pht(
+            'UNSAFE: Raw string ("%s") passed to query ("%s") subclause '.
+            'for "%%%s" conversion. Subclause conversions should be passed '.
+            'a list of PhutilQueryString objects.',
+            $value,
+            $query,
+            $type));
+        break;
+      }
+
+      if (!($value instanceof PhutilQueryString)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Expected a list of PhutilQueryString objects for %%%s '.
+            'conversion.',
+            $type));
+      }
+      break;
+
+    case 'Q':
+      // TODO: See T13217. Remove this eventually.
+      if (is_string($value)) {
+        phlog(
+          pht(
+            'UNSAFE: Raw string ("%s") passed to query ("%s") for "%%Q" '.
+            'conversion. %%Q should be passed a query string.',
+            $value,
+            $query));
+        break;
+      }
+
+      if (!($value instanceof PhutilQueryString)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a PhutilQueryString for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Z':
+      if (!is_string($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Value for "%%Z" conversion should be a raw string.'));
+      }
+      break;
+
+    case 'LC':
+    case 'LK':
+    case 'T':
+    case 'C':
+      if (!is_string($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a string for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Ld':
+    case 'Lf':
+    case 'd':
+    case 'f':
+      if (!is_null($value) && !is_numeric($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a numeric scalar or null for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'Ls':
+    case 's':
+    case 'LB':
+    case 'B':
+    case '~':
+    case '>':
+    case '<':
+    case 'K':
+      if (!is_null($value) && !is_scalar($value)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht('Expected a scalar or null for %%%s conversion.', $type));
+      }
+      break;
+
+    case 'R':
+      if (!($value instanceof AphrontDatabaseTableRefInterface)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Parameter to "%s" conversion in "qsprintf(...)" is not an '.
+            'instance of AphrontDatabaseTableRefInterface.',
+            '%R'));
+      }
+      break;
+
+    case 'P':
+      if (!($value instanceof PhutilOpaqueEnvelope)) {
+        throw new AphrontParameterQueryException(
+          $query,
+          pht(
+            'Parameter to "%s" conversion in "qsprintf(...)" is not an '.
+            'instance of PhutilOpaqueEnvelope.',
+            '%P'));
+      }
+      break;
+
+    default:
+      throw new XsprintfUnknownConversionException($type);
+  }
+}
diff --git a/src/infrastructure/storage/xsprintf/queryfx.php b/src/infrastructure/storage/xsprintf/queryfx.php
new file mode 100644
index 0000000000..d1eef612b5
--- /dev/null
+++ b/src/infrastructure/storage/xsprintf/queryfx.php
@@ -0,0 +1,27 @@
+setLastActiveEpoch(time());
+  $conn->executeQuery($query);
+}
+
+function queryfx_all(AphrontDatabaseConnection $conn, $sql /* , ... */) {
+  $argv = func_get_args();
+  call_user_func_array('queryfx', $argv);
+  return $conn->selectAllResults();
+}
+
+function queryfx_one(AphrontDatabaseConnection $conn, $sql /* , ... */) {
+  $argv = func_get_args();
+  $ret = call_user_func_array('queryfx_all', $argv);
+  if (count($ret) > 1) {
+    throw new AphrontCountQueryException(
+      pht('Query returned more than one row.'));
+  } else if (count($ret)) {
+    return reset($ret);
+  }
+  return null;
+}
diff --git a/src/infrastructure/util/__tests__/PhabricatorPreambleTestCase.php b/src/infrastructure/util/__tests__/PhabricatorPreambleTestCase.php
new file mode 100644
index 0000000000..c47e8c0c9a
--- /dev/null
+++ b/src/infrastructure/util/__tests__/PhabricatorPreambleTestCase.php
@@ -0,0 +1,74 @@
+ '1.2.3.4',
+        'layers' => 1,
+        'expect' => '1.2.3.4',
+      ),
+
+      // In this case, the LB received a request which already had an
+      // "X-Forwarded-For" header. This might be legitimate (in the case of
+      // a CDN request) or illegitimate (in the case of a client making
+      // things up). We don't want to trust it.
+      array(
+        'header' => '9.9.9.9, 1.2.3.4',
+        'layers' => 1,
+        'expect' => '1.2.3.4',
+      ),
+
+      // Multiple layers of load balancers.
+      array(
+        'header' => '9.9.9.9, 1.2.3.4',
+        'layers' => 2,
+        'expect' => '9.9.9.9',
+      ),
+
+      // Multiple layers of load balancers, plus a client-supplied value.
+      array(
+        'header' => '8.8.8.8, 9.9.9.9, 1.2.3.4',
+        'layers' => 2,
+        'expect' => '9.9.9.9',
+      ),
+
+      // Multiple layers of load balancers, but this request came from
+      // somewhere inside the network.
+      array(
+        'header' => '1.2.3.4',
+        'layers' => 2,
+        'expect' => '1.2.3.4',
+      ),
+
+      array(
+        'header' => 'A, B, C, D, E, F, G, H, I',
+        'layers' => 7,
+        'expect' => 'C',
+      ),
+    );
+
+    foreach ($tests as $test) {
+      $header = $test['header'];
+      $layers = $test['layers'];
+      $expect = $test['expect'];
+
+      $actual = preamble_get_x_forwarded_for_address($header, $layers);
+
+      $this->assertEqual(
+        $expect,
+        $actual,
+        pht(
+          'Address after stripping %d layers from: %s',
+          $layers,
+          $header));
+    }
+  }
+
+}
diff --git a/src/view/AphrontDialogView.php b/src/view/AphrontDialogView.php
index 09fc8e7a16..b8b00a6b3e 100644
--- a/src/view/AphrontDialogView.php
+++ b/src/view/AphrontDialogView.php
@@ -160,6 +160,20 @@ final class AphrontDialogView
     return $this->appendChild($box);
   }
 
+  public function appendRemarkup($remarkup) {
+    $viewer = $this->getViewer();
+    $view = new PHUIRemarkupView($viewer, $remarkup);
+
+    $view_tag = phutil_tag(
+      'div',
+      array(
+        'class' => 'aphront-dialog-view-paragraph',
+      ),
+      $view);
+
+    return $this->appendChild($view_tag);
+  }
+
   public function appendParagraph($paragraph) {
     return $this->appendParagraphTag($paragraph);
   }
diff --git a/src/view/page/menu/PhabricatorMainMenuSearchView.php b/src/view/page/menu/PhabricatorMainMenuSearchView.php
index 15319a357e..aaa7c5a160 100644
--- a/src/view/page/menu/PhabricatorMainMenuSearchView.php
+++ b/src/view/page/menu/PhabricatorMainMenuSearchView.php
@@ -116,8 +116,9 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
     return $form;
   }
 
-  private function buildModeSelector($selector_id, $application_id) {
-    $viewer = $this->getViewer();
+  public static function getGlobalSearchScopeItems(
+    PhabricatorUser $viewer,
+    PhabricatorApplication $application) {
 
     $items = array();
     $items[] = array(
@@ -132,7 +133,6 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
 
     $application_value = null;
     $application_icon = self::DEFAULT_APPLICATION_ICON;
-    $application = $this->getApplication();
     if ($application) {
       $application_value = get_class($application);
       if ($application->getApplicationSearchDocumentTypes()) {
@@ -185,6 +185,14 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
       'href' => PhabricatorEnv::getDoclink('Search User Guide'),
     );
 
+    return $items;
+  }
+
+  private function buildModeSelector($selector_id, $application_id) {
+    $viewer = $this->getViewer();
+
+    $items = self::getGlobalSearchScopeItems($viewer, $this->getApplication());
+
     $scope_key = PhabricatorSearchScopeSetting::SETTINGKEY;
     $current_value = $viewer->getUserSetting($scope_key);
 
@@ -196,6 +204,13 @@ final class PhabricatorMainMenuSearchView extends AphrontView {
       }
     }
 
+    $application = $this->getApplication();
+
+    $application_value = null;
+    if ($application) {
+      $application_value = get_class($application);
+    }
+
     $selector = id(new PHUIButtonView())
       ->setID($selector_id)
       ->addClass('phabricator-main-menu-search-dropdown')
diff --git a/support/startup/preamble-utils.php b/support/startup/preamble-utils.php
new file mode 100644
index 0000000000..8dd3b502d6
--- /dev/null
+++ b/support/startup/preamble-utils.php
@@ -0,0 +1,77 @@
+): '.
+      '"layers" parameter must an integer larger than 0.'."\n";
+    echo "\n";
+    exit(1);
+  }
+
+  if (!isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
+    return;
+  }
+
+  $forwarded_for = $_SERVER['HTTP_X_FORWARDED_FOR'];
+  if (!strlen($forwarded_for)) {
+    return;
+  }
+
+  $address = preamble_get_x_forwarded_for_address($forwarded_for, $layers);
+
+  $_SERVER['REMOTE_ADDR'] = $address;
+}
+
+function preamble_get_x_forwarded_for_address($raw_header, $layers) {
+  // The raw header may be a list of IPs, like "1.2.3.4, 4.5.6.7", if the
+  // request the load balancer received also had this header. In particular,
+  // this happens routinely with requests received through a CDN, but can also
+  // happen illegitimately if the client just makes up an "X-Forwarded-For"
+  // header full of lies.
+
+  // We can only trust the N elements at the end of the list which correspond
+  // to network-adjacent devices we control. Usually, we're behind a single
+  // load balancer and "N" is 1, so we want to take the last element in the
+  // list.
+
+  // In some cases, "N" may be more than 1, if the network is configured so
+  // that that requests are routed through multiple layers of load balancers
+  // and proxies. In this case, we want to take the Nth-to-last element of
+  // the list.
+
+  $addresses = explode(',', $raw_header);
+
+  // If we have more than one trustworthy device on the network path, discard
+  // corresponding elements from the list. For example, if we have 7 devices,
+  // we want to discard the last 6 elements of the list.
+
+  // The final device address does not appear in the list, since devices do
+  // not append their own addresses to "X-Forwarded-For".
+
+  $discard_addresses = ($layers - 1);
+
+  // However, we don't want to throw away all of the addresses. Some requests
+  // may originate from within the network, and may thus not have as many
+  // addresses as we expect. If we have fewer addresses than trustworthy
+  // devices, discard all but one address.
+
+  $max_discard = (count($addresses) - 1);
+
+  $discard_count = min($discard_addresses, $max_discard);
+  if ($discard_count) {
+    $addresses = array_slice($addresses, 0, -$discard_count);
+  }
+
+  $original_address = end($addresses);
+  $original_address = trim($original_address);
+
+  return $original_address;
+}
diff --git a/webroot/index.php b/webroot/index.php
index 0014edfa2c..38c5c77809 100644
--- a/webroot/index.php
+++ b/webroot/index.php
@@ -85,6 +85,7 @@ function phabricator_startup() {
   require_once $root.'/support/startup/PhabricatorClientLimit.php';
   require_once $root.'/support/startup/PhabricatorClientRateLimit.php';
   require_once $root.'/support/startup/PhabricatorClientConnectionLimit.php';
+  require_once $root.'/support/startup/preamble-utils.php';
 
   // If the preamble script exists, load it.
   $t_preamble = microtime(true);