From c868ee9c07d0e8b7a6622f1de263027efa0773ea Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 05:23:47 -0800 Subject: [PATCH] Introduce and document a new `cluster.mailers` option for configuring multiple mailers Summary: Depends on D19002. Ref T13053. Ref T12677. Adds a new option to allow configuration of multiple mailers. Nothing actually uses this yet. Test Plan: Tried to set it to various bad values, got reasonable error messages. Read documentation. Reviewers: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19003 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMetaMTAConfigOptions.php | 20 +- .../PhabricatorMailImplementationAdapter.php | 12 + ...atorMailImplementationAmazonSESAdapter.php | 2 + ...icatorMailImplementationMailgunAdapter.php | 2 + ...atorMailImplementationPHPMailerAdapter.php | 2 + ...MailImplementationPHPMailerLiteAdapter.php | 2 + ...catorMailImplementationSendGridAdapter.php | 2 + ...abricatorMailImplementationTestAdapter.php | 2 + .../configuring_outbound_email.diviner | 303 +++++++++++------- .../PhabricatorClusterMailersConfigType.php | 100 ++++++ 11 files changed, 323 insertions(+), 126 deletions(-) create mode 100644 src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c642f13f90..fb2b0ccf0c 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2411,6 +2411,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterMailersConfigType' => 'infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php', 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', 'PhabricatorClusterSearchConfigType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigType.php', 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', @@ -7824,6 +7825,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterMailersConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterNoHostForRoleException' => 'Exception', 'PhabricatorClusterSearchConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterServiceHealthRecord' => 'Phobject', diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 8c2d2265bc..43734abea0 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -138,19 +138,14 @@ EODOC , 'metamta.public-replies')); - $adapter_doc_href = PhabricatorEnv::getDoclink( - 'Configuring Outbound Email'); - $adapter_doc_name = pht('Configuring Outbound Email'); $adapter_description = $this->deformat(pht(<<deformat(pht(<<deformat(pht(<<newOption('cluster.mailers', 'cluster.mailers', null) + ->setLocked(true) + ->setDescription($mailers_description), $this->newOption( 'metamta.default-address', 'string', diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 514306758d..923574d16d 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -5,6 +5,18 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { private $key; private $options = array(); + final public function getAdapterType() { + return $this->getPhobjectClassConstant('ADAPTERTYPE'); + } + + final public static function getAllAdapters() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getAdapterType') + ->execute(); + } + + abstract public function setFrom($email, $name = ''); abstract public function addReplyTo($email, $name = ''); abstract public function addTos(array $emails); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php index 850b83f1dd..22cc102262 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationAmazonSESAdapter extends PhabricatorMailImplementationPHPMailerLiteAdapter { + const ADAPTERTYPE = 'ses'; + private $message; private $isHTML; diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index a7be6731eb..bed0dada63 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationMailgunAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'mailgun'; + private $params = array(); private $attachments = array(); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php index 0eb59629a6..3ca6366730 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationPHPMailerAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'smtp'; + private $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index 4fd8387252..1f21a993c9 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -6,6 +6,8 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendmail'; + protected $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php index 9cd8dd19b4..be2a837053 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationSendGridAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendgrid'; + private $params = array(); protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php index bd64076a59..8a8d0de0c2 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php @@ -7,6 +7,8 @@ final class PhabricatorMailImplementationTestAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'test'; + private $guts = array(); private $config = array(); diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 2a95f49bc3..0c7ef529fa 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -3,43 +3,40 @@ Instructions for configuring Phabricator to send mail. -= Overview = +Overview +======== -Phabricator can send outbound email via several different providers, called -"Adapters". +Phabricator can send outbound email through several different mail services, +including a local mailer or various third-party services. Options include: | Send Mail With | Setup | Cost | Inbound | Notes | |---------|-------|------|---------|-------| | Mailgun | Easy | Cheap | Yes | Recommended | | Amazon SES | Easy | Cheap | No | Recommended | -| SendGrid | Medium | Cheap | Yes | Discouraged (See Note) | +| SendGrid | Medium | Cheap | Yes | Discouraged | | External SMTP | Medium | Varies | No | Gmail, etc. | -| Local SMTP | Hard | Free | No | (Default) sendmail, postfix, etc | -| Custom | Hard | Free | No | Write an adapter for some other service. | +| Local SMTP | Hard | Free | No | sendmail, postfix, etc | +| Custom | Hard | Free | No | Write a custom mailer for some other service. | | Drop in a Hole | Easy | Free | No | Drops mail in a deep, dark hole. | -Of these options, sending mail via local SMTP is the default, but usually -requires some configuration to get working. See below for details on how to -select and configure a delivery method. +See below for details on how to select and configure mail delivery for each +mailer. Overall, Mailgun and SES are much easier to set up, and using one of them is recommended. In particular, Mailgun will also let you set up inbound email easily. If you have some internal mail service you'd like to use you can also -write a custom adapter, but this requires digging into the code. +write a custom mailer, but this requires digging into the code. Phabricator sends mail in the background, so the daemons need to be running for it to be able to deliver mail. You should receive setup warnings if they are not. For more information on using daemons, see @{article:Managing Daemons with phd}. -**Note on SendGrid**: Users have experienced a number of odd issues with -SendGrid, compared to fewer issues with other mailers. We discourage SendGrid -unless you're already using it. If you send to SendGrid via SMTP, you may need -to adjust `phpmailer.smtp-encoding`. -= Basics = +Basics +====== Regardless of how outbound email is delivered, you should configure these keys in your configuration: @@ -51,33 +48,113 @@ in your configuration: - **metamta.can-send-as-user** should be left as `false` in most cases, but see the documentation for details. -= Configuring Mail Adapters = -To choose how mail will be sent, change the `metamta.mail-adapter` key in -your configuration. Possible values are listed in the UI: +Configuring Mailers +=================== - - `PhabricatorMailImplementationAmazonMailgunAdapter`: use Mailgun, see - "Adapter: Mailgun". - - `PhabricatorMailImplementationAmazonSESAdapter`: use Amazon SES, see - "Adapter: Amazon SES". - - `PhabricatorMailImplementationPHPMailerLiteAdapter`: default, uses - "sendmail", see "Adapter: Sendmail". - - `PhabricatorMailImplementationPHPMailerAdapter`: uses SMTP, see - "Adapter: SMTP". - - `PhabricatorMailImplementationSendGridAdapter`: use SendGrid, see - "Adapter: SendGrid". - - `Some Custom Class You Write`: use a custom adapter you write, see - "Adapter: Custom". - - `PhabricatorMailImplementationTestAdapter`: this will - **completely disable** outbound mail. You can use this if you don't want to - send outbound mail, or want to skip this step for now and configure it - later. +Configure one or more mailers by listing them in the the `cluster.mailers` +configuration option. Most installs only need to configure one mailer, but you +can configure multiple mailers to provide greater availability in the event of +a service disruption. -= Adapter: Sendmail = +A valid `cluster.mailers` configuration looks something like this: -This is the default, and selected by choosing -`PhabricatorMailImplementationPHPMailerLiteAdapter` as the value for -**metamta.mail-adapter**. This requires a `sendmail` binary to be installed on +```lang=json +[ + { + "key": "mycompany-mailgun", + "type": "mailgun", + "options": { + "domain": "mycompany.com", + "api-key": "..." + } + }, + ... +] +``` + +The supported keys for each mailer are: + + - `key`: Required string. A unique name for this mailer. + - `type`: Required string. Identifies the type of mailer. See below for + options. + - `priority`: Optional string. Advanced option which controls load balancing + and failover behavior. See below for details. + - `options`: Optional map. Additional options for the mailer type. + +The `type` field can be used to select these third-party mailers: + + - `mailgun`: Use Mailgun. + - `ses`: Use Amazon SES. + - `sendgrid`: Use Sendgrid. + +It also supports these local mailers: + + - `sendmail`: Use the local `sendmail` binary. + - `smtp`: Connect directly to an SMTP server. + - `test`: Internal mailer for testing. Does not send mail. + +You can also write your own mailer by extending +`PhabricatorMailImplementationAdapter`. + +Once you've selected a mailer, find the corresponding section below for +instructions on configuring it. + + +Mailer: Mailgun +=============== + +Mailgun is a third-party email delivery service. You can learn more at +. Mailgun is easy to configure and works well. + +To use this mailer, set `type` to `mailgun`, then configure these `options`: + + - `api-key`: Required string. Your Mailgun API key. + - `domain`: Required string. Your Mailgun domain. + + +Mailer: Amazon SES +================== + +Amazon SES is Amazon's cloud email service. You can learn more at +. + +To use this mailer, set `type` to `ses`, then configure these `options`: + + - `access-key`: Required string. Your Amazon SES access key. + - `secret-key`: Required string. Your Amazon SES secret key. + - `endpoint`: Required string. Your Amazon SES endpoint. + +NOTE: Amazon SES **requires you to verify your "From" address**. Configure +which "From" address to use by setting "`metamta.default-address`" in your +config, then follow the Amazon SES verification process to verify it. You +won't be able to send email until you do this! + + +Mailer: SendGrid +================ + +SendGrid is a third-party email delivery service. You can learn more at +. + +You can configure SendGrid in two ways: you can send via SMTP or via the REST +API. To use SMTP, configure Phabricator to use an `smtp` mailer. + +To use the REST API mailer, set `type` to `sendgrid`, then configure +these `options`: + + - `api-user`: Required string. Your SendGrid login name. + - `api-key`: Required string. Your SendGrid API key. + +NOTE: Users have experienced a number of odd issues with SendGrid, compared to +fewer issues with other mailers. We discourage SendGrid unless you're already +using it. + + +Mailer: Sendmail +================ + +This requires a `sendmail` binary to be installed on the system. Most MTAs (e.g., sendmail, qmail, postfix) should do this, but your machine may not have one installed by default. For install instructions, consult the documentation for your favorite MTA. @@ -88,96 +165,32 @@ document. If you can already send outbound email from the command line or know how to configure it, this option is straightforward. If you have no idea how to do any of this, strongly consider using Mailgun or Amazon SES instead. -If you experience issues with mail getting mangled (for example, arriving with -too many or too few newlines) you may try adjusting `phpmailer.smtp-encoding`. +To use this mailer, set `type` to `sendmail`. There are no `options` to +configure. -= Adapter: SMTP = + +Mailer: STMP +============ You can use this adapter to send mail via an external SMTP server, like Gmail. -To do this, set these configuration keys: - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationPHPMailerAdapter`. - - **phpmailer.mailer**: set to `smtp`. - - **phpmailer.smtp-host**: set to hostname of your SMTP server. - - **phpmailer.smtp-port**: set to port of your SMTP server. - - **phpmailer.smtp-user**: set to your username used for authentication. - - **phpmailer.smtp-password**: set to your password used for authentication. - - **phpmailer.smtp-protocol**: set to `tls` or `ssl` if necessary. Use +To use this mailer, set `type` to `smtp`, then configure these `options`: + + - `host`: Required string. The hostname of your SMTP server. + - `user`: Optional string. Username used for authentication. + - `password`: Optional string. Password for authentication. + - `protocol`: Optional string. Set to `tls` or `ssl` if necessary. Use `ssl` for Gmail. - - **phpmailer.smtp-encoding**: Normally safe to leave as the default, but - adjusting it may help resolve mail mangling issues (for example, mail - arriving with too many or too few newlines). -= Adapter: Mailgun = -Mailgun is an email delivery service. You can learn more at -. Mailgun isn't free, but is very easy to configure -and works well. +Disable Mail +============ -To use Mailgun, sign up for an account, then set these configuration keys: +To disable mail, just don't configure any mailers. - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationMailgunAdapter`. - - **mailgun.api-key**: set to your Mailgun API key. - - **mailgun.domain**: set to your Mailgun domain. -= Adapter: Amazon SES = - -Amazon SES is Amazon's cloud email service. It is not free, but is easier to -configure than sendmail and can simplify outbound email configuration. To use -Amazon SES, you need to sign up for an account with Amazon at -. - -To configure Phabricator to use Amazon SES, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationAmazonSESAdapter". - - **amazon-ses.access-key**: set to your Amazon SES access key. - - **amazon-ses.secret-key**: set to your Amazon SES secret key. - - **amazon-ses.endpoint**: Set to your Amazon SES endpoint. - -NOTE: Amazon SES **requires you to verify your "From" address**. Configure which -"From" address to use by setting "`metamta.default-address`" in your config, -then follow the Amazon SES verification process to verify it. You won't be able -to send email until you do this! - -= Adapter: SendGrid = - -SendGrid is an email delivery service like Amazon SES. You can learn more at -. It is easy to configure, but not free. - -You can configure SendGrid in two ways: you can send via SMTP or via the REST -API. To use SMTP, just configure `sendmail` and leave Phabricator's setup -with defaults. To use the REST API, follow the instructions in this section. - -To configure Phabricator to use SendGrid, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationSendGridAdapter". - - **sendgrid.api-user**: set to your SendGrid login name. - - **sendgrid.api-key**: set to your SendGrid password. - -If you're logged into your SendGrid account, you may be able to find this -information easily by visiting . - -= Adapter: Custom = - -You can provide a custom adapter by writing a concrete subclass of -@{class:PhabricatorMailImplementationAdapter} and setting it as the -`metamta.mail-adapter`. - -TODO: This should be better documented once extending Phabricator is better -documented. - -= Adapter: Disable Outbound Mail = - -You can use the @{class:PhabricatorMailImplementationTestAdapter} to completely -disable outbound mail, if you don't want to send mail or don't want to configure -it yet. Just set **metamta.mail-adapter** to -`PhabricatorMailImplementationTestAdapter`. - -= Testing and Debugging Outbound Email = +Testing and Debugging Outbound Email +==================================== You can use the `bin/mail` utility to test, debug, and examine outbound mail. In particular: @@ -191,7 +204,59 @@ Run `bin/mail help ` for more help on using these commands. You can monitor daemons using the Daemon Console (`/daemon/`, or click **Daemon Console** from the homepage). -= Next Steps = + +Priorities +========== + +By default, Phabricator will try each mailer in order: it will try the first +mailer first. If that fails (for example, because the service is not available +at the moment) it will try the second mailer, and so on. + +If you want to load balance between multiple mailers instead of using one as +a primary, you can set `priority`. Phabricator will start with mailers in the +highest priority group and go through them randomly, then fall back to the +next group. + +For example, if you have two SMTP servers and you want to balance requests +between them and then fall back to Mailgun if both fail, configure priorities +like this: + +```lang=json +[ + { + "key": "smtp-uswest", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "smtp-useast", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "mailgun-fallback", + "type": "mailgun", + "options": "..." + } +} +``` + +Phabricator will start with servers in the highest priority group (the group +with the **largest** `priority` number). In this example, the highest group is +`300`, which has the two SMTP servers. They'll be tried in random order first. + +If both fail, Phabricator will move on to the next priority group. In this +example, there are no other priority groups. + +If it still hasn't sent the mail, Phabricator will try servers which are not +in any priority group, in the configured order. In this example there is +only one such server, so it will try to send via Mailgun. + + +Next Steps +========== Continue by: diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php new file mode 100644 index 0000000000..2a7550c419 --- /dev/null +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -0,0 +1,100 @@ +newException( + pht( + 'Mailer cluster configuration is not valid: it should be a list '. + 'of mailer configurations.')); + } + + foreach ($value as $index => $spec) { + if (!is_array($spec)) { + throw $this->newException( + pht( + 'Mailer cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a mailer, but the value '. + 'with index "%s" is not a dictionary.', + $index)); + } + } + + $adapters = PhabricatorMailImplementationAdapter::getAllAdapters(); + + $map = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'key' => 'string', + 'type' => 'string', + 'priority' => 'optional int', + 'options' => 'optional wild', + )); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration has an invalid mailer specification '. + '(at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + $key = $spec['key']; + if (isset($map[$key])) { + throw $this->newException( + pht( + 'Mailer configuration is invalid: multiple mailers have the same '. + 'key ("%s"). Each mailer must have a unique key.', + $key)); + } + $map[$key] = true; + + $priority = idx($spec, 'priority', 0); + if ($priority <= 0) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: priority must be '. + 'greater than 0.', + $key)); + } + + $type = $spec['type']; + if (!isset($adapters[$type])) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: mailer type ("%s") is '. + 'unknown. Supported mailer types are: %s.', + $key, + $type, + implode(', ', array_keys($adapters)))); + } + + $options = idx($spec, 'options', array()); + try { + id(clone $adapters[$type])->validateOptions($options); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") specifies invalid options for '. + 'mailer: %s', + $key, + $ex->getMessage())); + } + } + } + +}