revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-26 08:42:41 +01:00
Code Issues Releases Wiki Activity

Refactor mail to produce an intermediate "bag of strings" object in preparation for SMS

Browse source 
Summary:
Depends on D19954. Ref T920. This is a step toward a world where "Mailers" are generic and may send messages over a broader array of channels (email, SMS, postal mail).

There are a few major parts here:

  - Instead of calling `$mailer->setSubject()`, `$mailer->setFrom()`, etc., build in intermediate `$message` object first, then pass that to the mailer.
    - This breaks every mailer! This change on its own does not fix them. I plan to fix them in a series of "update mailer X", "update mailer Y" followups.
    - This generally makes the API easier to change in the far future, and in the near future supports mailers accepting different types of `$message` objects with the same API.
  - Pull the "build an email" stuff out into a `PhabricatorMailEmailEngine`. `MetaMTAMail` is already a huge object without also doing this translation step. This is just a separation/simplification change, but also tries to fight against `MetaMTAMail` getting 5K lines of email/sms/whatsapp/postal-mail code.
  - Try to rewrite the "build an email" stuff to be a bit more straightforward while making it generate objects. Prior to this change, it had this weird flow:

```lang=php
foreach ($properties as $key => $prop) {
  switch ($key) {
    case 'xyz':
      // ...
  }
}
```

This is just inherently somewhat hard to puzzle out, and it means that processing order depends on internal property order, which is quite surprising.

Test Plan: This breaks everything on its own; adapters must be updated to use the new API. See followups.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T920

Differential Revision: https://secure.phabricator.com/D19955
This commit is contained in:
epriestley 2019-01-04 05:45:38 -08:00
parent a37b28ef79
commit b5797ce60a
6 changed files with 813 additions and 544 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/__phutil_library_map__.php
View file

@ -3389,6 +3389,7 @@ phutil_register_library_map(array(
'PhabricatorMacroViewController' => 'applications/macro/controller/PhabricatorMacroViewController.php',
'PhabricatorMailAttachment' => 'applications/metamta/message/PhabricatorMailAttachment.php',
'PhabricatorMailConfigTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php',
'PhabricatorMailEmailEngine' => 'applications/metamta/engine/PhabricatorMailEmailEngine.php',
'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php',
'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php',
'PhabricatorMailEmailMessage' => 'applications/metamta/message/PhabricatorMailEmailMessage.php',
@ -3414,6 +3415,7 @@ phutil_register_library_map(array(
'PhabricatorMailManagementUnverifyWorkflow' => 'applications/metamta/management/PhabricatorMailManagementUnverifyWorkflow.php',
'PhabricatorMailManagementVolumeWorkflow' => 'applications/metamta/management/PhabricatorMailManagementVolumeWorkflow.php',
'PhabricatorMailManagementWorkflow' => 'applications/metamta/management/PhabricatorMailManagementWorkflow.php',
'PhabricatorMailMessageEngine' => 'applications/metamta/engine/PhabricatorMailMessageEngine.php',
'PhabricatorMailMustEncryptHeraldAction' => 'applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php',
'PhabricatorMailOutboundMailHeraldAdapter' => 'applications/metamta/herald/PhabricatorMailOutboundMailHeraldAdapter.php',
'PhabricatorMailOutboundRoutingHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingHeraldAction.php',
@ -9221,6 +9223,7 @@ phutil_register_library_map(array(
'PhabricatorMacroViewController' => 'PhabricatorMacroController',
'PhabricatorMailAttachment' => 'Phobject',
'PhabricatorMailConfigTestCase' => 'PhabricatorTestCase',
'PhabricatorMailEmailEngine' => 'PhabricatorMailMessageEngine',
'PhabricatorMailEmailHeraldField' => 'HeraldField',
'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup',
'PhabricatorMailEmailMessage' => 'PhabricatorMailExternalMessage',
@ -9246,6 +9249,7 @@ phutil_register_library_map(array(
'PhabricatorMailManagementUnverifyWorkflow' => 'PhabricatorMailManagementWorkflow',
'PhabricatorMailManagementVolumeWorkflow' => 'PhabricatorMailManagementWorkflow',
'PhabricatorMailManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorMailMessageEngine' => 'Phobject',
'PhabricatorMailMustEncryptHeraldAction' => 'HeraldAction',
'PhabricatorMailOutboundMailHeraldAdapter' => 'HeraldAdapter',
'PhabricatorMailOutboundRoutingHeraldAction' => 'HeraldAction',

3
src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php
View file

@ -187,9 +187,6 @@ final class PhabricatorMetaMTAMailViewController
->setStacked(true);
$headers = $mail->getDeliveredHeaders();
if ($headers === null) {
$headers = $mail->generateHeaders();
}
// Sort headers by name.
$headers = isort($headers, 0);

649
src/applications/metamta/engine/PhabricatorMailEmailEngine.php Normal file
View file

@ -0,0 +1,649 @@
<?php
final class PhabricatorMailEmailEngine
extends PhabricatorMailMessageEngine {
public function newMessage() {
$mailer = $this->getMailer();
$mail = $this->getMail();
$message = new PhabricatorMailEmailMessage();
$from_address = $this->newFromEmailAddress();
$message->setFromAddress($from_address);
$reply_address = $this->newReplyToEmailAddress();
if ($reply_address) {
$message->setReplyToAddress($reply_address);
}
$to_addresses = $this->newToEmailAddresses();
$cc_addresses = $this->newCCEmailAddresses();
if (!$to_addresses && !$cc_addresses) {
$mail->setMessage(
pht(
'Message has no valid recipients: all To/CC are disabled, '.
'invalid, or configured not to receive this mail.'));
return null;
}
// If this email describes a mail processing error, we rate limit outbound
// messages to each individual address. This prevents messes where
// something is stuck in a loop or dumps a ton of messages on us suddenly.
if ($mail->getIsErrorEmail()) {
$all_recipients = array();
foreach ($to_addresses as $to_address) {
$all_recipients[] = $to_address->getAddress();
}
foreach ($cc_addresses as $cc_address) {
$all_recipients[] = $cc_address->getAddress();
}
if ($this->shouldRateLimitMail($all_recipients)) {
$mail->setMessage(
pht(
'This is an error email, but one or more recipients have '.
'exceeded the error email rate limit. Declining to deliver '.
'message.'));
return null;
}
}
// Some mailers require a valid "To:" in order to deliver mail. If we
// don't have any "To:", try to fill it in with a placeholder "To:".
// If that also fails, move the "Cc:" line to "To:".
if (!$to_addresses) {
$void_address = $this->newVoidEmailAddress();
$cc_addresses = $to_addresses;
$to_addresses = array($void_address);
}
$to_addresses = $this->getUniqueEmailAddresses($to_addresses);
$cc_addresses = $this->getUniqueEmailAddresses(
$cc_addresses,
$to_addresses);
$message->setToAddresses($to_addresses);
$message->setCCAddresses($cc_addresses);
$attachments = $this->newEmailAttachments();
$message->setAttachments($attachments);
$subject = $this->newEmailSubject();
$message->setSubject($subject);
$headers = $this->newEmailHeaders();
foreach ($this->newEmailThreadingHeaders($mailer) as $threading_header) {
$headers[] = $threading_header;
}
$stamps = $mail->getMailStamps();
if ($stamps) {
$headers[] = $this->newEmailHeader(
'X-Phabricator-Stamps',
implode(' ', $stamps));
}
$must_encrypt = $mail->getMustEncrypt();
$raw_body = $mail->getBody();
$body = $raw_body;
if ($must_encrypt) {
$parts = array();
$encrypt_uri = $this->getMustEncryptURI();
if (!strlen($encrypt_uri)) {
$encrypt_phid = $this->getRelatedPHID();
if ($encrypt_phid) {
$encrypt_uri = urisprintf(
'/object/%s/',
$encrypt_phid);
}
}
if (strlen($encrypt_uri)) {
$parts[] = pht(
'This secure message is notifying you of a change to this object:');
$parts[] = PhabricatorEnv::getProductionURI($encrypt_uri);
}
$parts[] = pht(
'The content for this message can only be transmitted over a '.
'secure channel. To view the message content, follow this '.
'link:');
$parts[] = PhabricatorEnv::getProductionURI($this->getURI());
$body = implode("\n\n", $parts);
} else {
$body = $raw_body;
}
$body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit');
if (strlen($body) > $body_limit) {
$body = id(new PhutilUTF8StringTruncator())
->setMaximumBytes($body_limit)
->truncateString($body);
$body .= "\n";
$body .= pht('(This email was truncated at %d bytes.)', $body_limit);
}
$message->setTextBody($body);
$body_limit -= strlen($body);
// If we sent a different message body than we were asked to, record
// what we actually sent to make debugging and diagnostics easier.
if ($body !== $raw_body) {
$mail->setDeliveredBody($body);
}
if ($must_encrypt) {
$send_html = false;
} else {
$send_html = $this->shouldSendHTML();
}
if ($send_html) {
$html_body = $mail->getHTMLBody();
if (strlen($html_body)) {
// NOTE: We just drop the entire HTML body if it won't fit. Safely
// truncating HTML is hard, and we already have the text body to fall
// back to.
if (strlen($html_body) <= $body_limit) {
$message->setHTMLBody($html_body);
$body_limit -= strlen($html_body);
}
}
}
// Pass the headers to the mailer, then save the state so we can show
// them in the web UI. If the mail must be encrypted, we remove headers
// which are not on a strict whitelist to avoid disclosing information.
$filtered_headers = $this->filterHeaders($headers, $must_encrypt);
$message->setHeaders($filtered_headers);
$mail->setUnfilteredHeaders($headers);
$mail->setDeliveredHeaders($headers);
if (PhabricatorEnv::getEnvConfig('phabricator.silent')) {
$mail->setMessage(
pht(
'Phabricator is running in silent mode. See `%s` '.
'in the configuration to change this setting.',
'phabricator.silent'));
return null;
}
return $message;
}
/* -( Message Components )------------------------------------------------- */
private function newFromEmailAddress() {
$from_address = $this->newDefaultEmailAddress();
$mail = $this->getMail();
// If the mail content must be encrypted, always disguise the sender.
$must_encrypt = $mail->getMustEncrypt();
if ($must_encrypt) {
return $from_address;
}
// If we have a raw "From" address, use that.
$raw_from = $mail->getRawFrom();
if ($raw_from) {
list($from_email, $from_name) = $raw_from;
return $this->newEmailAddress($from_email, $from_name);
}
// Otherwise, use as much of the information for any sending entity as
// we can.
$from_phid = $mail->getFrom();
$actor = $this->getActor($from_phid);
if ($actor) {
$actor_email = $actor->getEmailAddress();
$actor_name = $actor->getName();
} else {
$actor_email = null;
$actor_name = null;
}
$send_as_user = PhabricatorEnv::getEnvConfig('metamta.can-send-as-user');
if ($send_as_user) {
if ($actor_email !== null) {
$from_address->setAddress($actor_email);
}
}
if ($actor_name !== null) {
$from_address->setDisplayName($actor_name);
}
return $from_address;
}
private function newReplyToEmailAddress() {
$mail = $this->getMail();
$reply_raw = $mail->getReplyTo();
if (!strlen($reply_raw)) {
return null;
}
$reply_address = new PhutilEmailAddress($reply_raw);
// If we have a sending object, change the display name.
$from_phid = $mail->getFrom();
$actor = $this->getActor($from_phid);
if ($actor) {
$reply_address->setDisplayName($actor->getName());
}
// If we don't have a display name, fill in a default.
if (!strlen($reply_address->getDisplayName())) {
$reply_address->setDisplayName(pht('Phabricator'));
}
return $reply_address;
}
private function newToEmailAddresses() {
$mail = $this->getMail();
$phids = $mail->getToPHIDs();
$addresses = $this->newEmailAddressesFromActorPHIDs($phids);
foreach ($mail->getRawToAddresses() as $raw_address) {
$addresses[] = new PhutilEmailAddress($raw_address);
}
return $addresses;
}
private function newCCEmailAddresses() {
$mail = $this->getMail();
$phids = $mail->getCcPHIDs();
return $this->newEmailAddressesFromActorPHIDs($phids);
}
private function newEmailAddressesFromActorPHIDs(array $phids) {
$mail = $this->getMail();
$phids = $mail->expandRecipients($phids);
$addresses = array();
foreach ($phids as $phid) {
$actor = $this->getActor($phid);
if (!$actor) {
continue;
}
if (!$actor->isDeliverable()) {
continue;
}
$addresses[] = new PhutilEmailAddress($actor->getEmailAddress());
}
return $addresses;
}
private function newEmailSubject() {
$mail = $this->getMail();
$is_threaded = (bool)$mail->getThreadID();
$must_encrypt = $mail->getMustEncrypt();
$subject = array();
if ($is_threaded) {
if ($this->shouldAddRePrefix()) {
$subject[] = 'Re:';
}
}
$subject[] = trim($mail->getSubjectPrefix());
// If mail content must be encrypted, we replace the subject with
// a generic one.
if ($must_encrypt) {
$encrypt_subject = $mail->getMustEncryptSubject();
if (!strlen($encrypt_subject)) {
$encrypt_subject = pht('Object Updated');
}
$subject[] = $encrypt_subject;
} else {
$vary_prefix = $mail->getVarySubjectPrefix();
if (strlen($vary_prefix)) {
if ($this->shouldVarySubject()) {
$subject[] = $vary_prefix;
}
}
$subject[] = $mail->getSubject();
}
foreach ($subject as $key => $part) {
if (!strlen($part)) {
unset($subject[$key]);
}
}
$subject = implode(' ', $subject);
return $subject;
}
private function newEmailHeaders() {
$mail = $this->getMail();
$headers = array();
$headers[] = $this->newEmailHeader(
'X-Phabricator-Sent-This-Message',
'Yes');
$headers[] = $this->newEmailHeader(
'X-Mail-Transport-Agent',
'MetaMTA');
// Some clients respect this to suppress OOF and other auto-responses.
$headers[] = $this->newEmailHeader(
'X-Auto-Response-Suppress',
'All');
$mailtags = $mail->getMailTags();
if ($mailtags) {
$tag_header = array();
foreach ($mailtags as $mailtag) {
$tag_header[] = '<'.$mailtag.'>';
}
$tag_header = implode(', ', $tag_header);
$headers[] = $this->newEmailHeader(
'X-Phabricator-Mail-Tags',
$tag_header);
}
$value = $mail->getHeaders();
foreach ($value as $pair) {
list($header_key, $header_value) = $pair;
// NOTE: If we have \n in a header, SES rejects the email.
$header_value = str_replace("\n", ' ', $header_value);
$headers[] = $this->newEmailHeader($header_key, $header_value);
}
$is_bulk = $mail->getIsBulk();
if ($is_bulk) {
$headers[] = $this->newEmailHeader('Precedence', 'bulk');
}
if ($mail->getMustEncrypt()) {
$headers[] = $this->newEmailHeader('X-Phabricator-Must-Encrypt', 'Yes');
}
$related_phid = $mail->getRelatedPHID();
if ($related_phid) {
$headers[] = $this->newEmailHeader('Thread-Topic', $related_phid);
}
$headers[] = $this->newEmailHeader(
'X-Phabricator-Mail-ID',
$mail->getID());
$unique = Filesystem::readRandomCharacters(16);
$headers[] = $this->newEmailHeader(
'X-Phabricator-Send-Attempt',
$unique);
return $headers;
}
private function newEmailThreadingHeaders() {
$mailer = $this->getMailer();
$mail = $this->getMail();
$headers = array();
$thread_id = $mail->getThreadID();
if (!strlen($thread_id)) {
return $headers;
}
$is_first = $mail->getIsFirstMessage();
// NOTE: Gmail freaks out about In-Reply-To and References which aren't in
// the form "<string@domain.tld>"; this is also required by RFC 2822,
// although some clients are more liberal in what they accept.
$domain = $this->newMailDomain();
$thread_id = '<'.$thread_id.'@'.$domain.'>';
if ($is_first && $mailer->supportsMessageIDHeader()) {
$headers[] = $this->newEmailHeader('Message-ID', $thread_id);
} else {
$in_reply_to = $thread_id;
$references = array($thread_id);
$parent_id = $mail->getParentMessageID();
if ($parent_id) {
$in_reply_to = $parent_id;
// By RFC 2822, the most immediate parent should appear last
// in the "References" header, so this order is intentional.
$references[] = $parent_id;
}
$references = implode(' ', $references);
$headers[] = $this->newEmailHeader('In-Reply-To', $in_reply_to);
$headers[] = $this->newEmailHeader('References', $references);
}
$thread_index = $this->generateThreadIndex($thread_id, $is_first);
$headers[] = $this->newEmailHeader('Thread-Index', $thread_index);
return $headers;
}
private function newEmailAttachments() {
$mail = $this->getMail();
// If the mail content must be encrypted, don't add attachments.
$must_encrypt = $mail->getMustEncrypt();
if ($must_encrypt) {
return array();
}
return $mail->getAttachments();
}
/* -( Preferences )-------------------------------------------------------- */
private function shouldAddRePrefix() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailRePrefixSetting::SETTINGKEY);
return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX);
}
private function shouldVarySubject() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailVarySubjectsSetting::SETTINGKEY);
return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS);
}
private function shouldSendHTML() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailFormatSetting::SETTINGKEY);
return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL);
}
/* -( Utilities )---------------------------------------------------------- */
private function newEmailHeader($name, $value) {
return id(new PhabricatorMailHeader())
->setName($name)
->setValue($value);
}
private function newEmailAddress($address, $name = null) {
$object = id(new PhutilEmailAddress())
->setAddress($address);
if (strlen($name)) {
$object->setDisplayName($name);
}
return $object;
}
public function newDefaultEmailAddress() {
$raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address');
if (!strlen($raw_address)) {
$domain = $this->newMailDomain();
$raw_address = "noreply@{$domain}";
}
$address = new PhutilEmailAddress($raw_address);
if (!strlen($address->getDisplayName())) {
$address->setDisplayName(pht('Phabricator'));
}
return $address;
}
public function newVoidEmailAddress() {
return $this->newDefaultEmailAddress();
}
private function newMailDomain() {
$domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain');
if (strlen($domain)) {
return $domain;
}
$install_uri = PhabricatorEnv::getURI('/');
$install_uri = new PhutilURI($install_uri);
return $install_uri->getDomain();
}
private function filterHeaders(array $headers, $must_encrypt) {
assert_instances_of($headers, 'PhabricatorMailHeader');
if (!$must_encrypt) {
return $headers;
}
$whitelist = array(
'In-Reply-To',
'Message-ID',
'Precedence',
'References',
'Thread-Index',
'Thread-Topic',
'X-Mail-Transport-Agent',
'X-Auto-Response-Suppress',
'X-Phabricator-Sent-This-Message',
'X-Phabricator-Must-Encrypt',
'X-Phabricator-Mail-ID',
'X-Phabricator-Send-Attempt',
);
// NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags".
// This header contains a significant amount of meaningful information
// about the object.
$whitelist_map = array();
foreach ($whitelist as $term) {
$whitelist_map[phutil_utf8_strtolower($term)] = true;
}
foreach ($headers as $key => $header) {
$name = $header->getName();
$name = phutil_utf8_strtolower($name);
if (!isset($whitelist_map[$name])) {
unset($headers[$key]);
}
}
return $headers;
}
private function getUniqueEmailAddresses(
array $addresses,
array $exclude = array()) {
assert_instances_of($addresses, 'PhutilEmailAddress');
assert_instances_of($exclude, 'PhutilEmailAddress');
$seen = array();
foreach ($exclude as $address) {
$seen[$address->getAddress()] = true;
}
foreach ($addresses as $key => $address) {
$raw_address = $address->getAddress();
if (isset($seen[$raw_address])) {
unset($addresses[$key]);
continue;
}
$seen[$raw_address] = true;
}
return array_values($addresses);
}
private function generateThreadIndex($seed, $is_first_mail) {
// When threading, Outlook ignores the 'References' and 'In-Reply-To'
// headers that most clients use. Instead, it uses a custom 'Thread-Index'
// header. The format of this header is something like this (from
// camel-exchange-folder.c in Evolution Exchange):
/* A new post to a folder gets a 27-byte-long thread index. (The value
* is apparently unique but meaningless.) Each reply to a post gets a
* 32-byte-long thread index whose first 27 bytes are the same as the
* parent's thread index. Each reply to any of those gets a
* 37-byte-long thread index, etc. The Thread-Index header contains a
* base64 representation of this value.
*/
// The specific implementation uses a 27-byte header for the first email
// a recipient receives, and a random 5-byte suffix (32 bytes total)
// thereafter. This means that all the replies are (incorrectly) siblings,
// but it would be very difficult to keep track of the entire tree and this
// gets us reasonable client behavior.
$base = substr(md5($seed), 0, 27);
if (!$is_first_mail) {
// Not totally sure, but it seems like outlook orders replies by
// thread-index rather than timestamp, so to get these to show up in the
// right order we use the time as the last 4 bytes.
$base .= ' '.pack('N', time());
}
return base64_encode($base);
}
private function shouldRateLimitMail(array $all_recipients) {
try {
PhabricatorSystemActionEngine::willTakeAction(
$all_recipients,
new PhabricatorMetaMTAErrorMailAction(),
1);
return false;
} catch (PhabricatorSystemActionRateLimitException $ex) {
return true;
}
}
}

55
src/applications/metamta/engine/PhabricatorMailMessageEngine.php Normal file
View file

@ -0,0 +1,55 @@
<?php
abstract class PhabricatorMailMessageEngine
extends Phobject {
private $mailer;
private $mail;
private $actors = array();
private $preferences;
final public function setMailer(
PhabricatorMailImplementationAdapter $mailer) {
$this->mailer = $mailer;
return $this;
}
final public function getMailer() {
return $this->mailer;
}
final public function setMail(PhabricatorMetaMTAMail $mail) {
$this->mail = $mail;
return $this;
}
final public function getMail() {
return $this->mail;
}
final public function setActors(array $actors) {
assert_instances_of($actors, 'PhabricatorMetaMTAActor');
$this->actors = $actors;
return $this;
}
final public function getActors() {
return $this->actors;
}
final public function getActor($phid) {
return idx($this->actors, $phid);
}
final public function setPreferences(
PhabricatorUserPreferences $preferences) {
$this->preferences = $preferences;
return $this;
}
final public function getPreferences() {
return $this->preferences;
}
}

4
src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php
View file

@ -116,10 +116,6 @@ final class PhabricatorMailManagementShowOutboundWorkflow
$headers = $message->getDeliveredHeaders();
$unfiltered = $message->getUnfilteredHeaders();
if (!$unfiltered) {
$headers = $message->generateHeaders();
$unfiltered = $headers;
}
$header_map = array();
foreach ($headers as $header) {

642
src/applications/metamta/storage/PhabricatorMetaMTAMail.php
View file

@ -191,13 +191,17 @@ final class PhabricatorMetaMTAMail
return $this;
}
public function getHeaders() {
return $this->getParam('headers', array());
}
public function addAttachment(PhabricatorMailAttachment $attachment) {
$this->parameters['attachments'][] = $attachment->toDictionary();
return $this;
}
public function getAttachments() {
$dicts = $this->getParam('attachments');
$dicts = $this->getParam('attachments', array());
$result = array();
foreach ($dicts as $dict) {
@ -256,11 +260,19 @@ final class PhabricatorMetaMTAMail
return $this;
}
public function getRawFrom() {
return $this->getParam('raw-from');
}
public function setReplyTo($reply_to) {
$this->setParam('reply-to', $reply_to);
return $this;
}
public function getReplyTo() {
return $this->getParam('reply-to');
}
public function setSubject($subject) {
$this->setParam('subject', $subject);
return $this;
@ -271,11 +283,19 @@ final class PhabricatorMetaMTAMail
return $this;
}
public function getSubjectPrefix() {
return $this->getParam('subject-prefix');
}
public function setVarySubjectPrefix($prefix) {
$this->setParam('vary-subject-prefix', $prefix);
return $this;
}
public function getVarySubjectPrefix() {
return $this->getParam('vary-subject-prefix');
}
public function setBody($body) {
$this->setParam('body', $body);
return $this;
@ -413,6 +433,10 @@ final class PhabricatorMetaMTAMail
return $this;
}
public function getIsBulk() {
return $this->getParam('is-bulk');
}
/**
* Use this method to set an ID used for message threading. MetaMTA will
* set appropriate headers (Message-ID, In-Reply-To, References and
@ -429,6 +453,14 @@ final class PhabricatorMetaMTAMail
return $this;
}
public function getThreadID() {
return $this->getParam('thread-id');
}
public function getIsFirstMessage() {
return (bool)$this->getParam('is-first-message');
}
/**
* Save a newly created mail to the database. The mail will eventually be
* delivered by the MetaMTA daemon.
@ -597,10 +629,6 @@ final class PhabricatorMetaMTAMail
}
}
foreach ($sorted as $mailer) {
$mailer->prepareForSend();
}
return $sorted;
}
@ -627,36 +655,51 @@ final class PhabricatorMetaMTAMail
->save();
}
$exceptions = array();
foreach ($mailers as $template_mailer) {
$mailer = null;
$actors = $this->loadAllActors();
// If we're sending one mail to everyone, some recipients will be in
// "Cc" rather than "To". We'll move them to "To" later (or supply a
// dummy "To") but need to look for the recipient in either the
// "To" or "Cc" fields here.
$target_phid = head($this->getToPHIDs());
if (!$target_phid) {
$target_phid = head($this->getCcPHIDs());
}
$preferences = $this->loadPreferences($target_phid);
// Attach any files we're about to send to this message, so the recipients
// can view them.
$viewer = PhabricatorUser::getOmnipotentUser();
$files = $this->loadAttachedFiles($viewer);
foreach ($files as $file) {
$file->attachToObject($this->getPHID());
}
$exceptions = array();
foreach ($mailers as $mailer) {
try {
$mailer = $this->buildMailer($template_mailer);
$message = id(new PhabricatorMailEmailEngine())
->setMailer($mailer)
->setMail($this)
->setActors($actors)
->setPreferences($preferences)
->newMessage($mailer);
} catch (Exception $ex) {
$exceptions[] = $ex;
continue;
}
if (!$mailer) {
// If we don't get a mailer back, that means the mail doesn't
// actually need to be sent (for example, because recipients have
// declined to receive the mail). Void it and return.
if (!$message) {
// If we don't get a message back, that means the mail doesn't actually
// need to be sent (for example, because recipients have declined to
// receive the mail). Void it and return.
return $this
->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID)
->save();
}
try {
$ok = $mailer->send();
if (!$ok) {
// TODO: At some point, we should clean this up and make all mailers
// throw.
throw new Exception(
pht(
'Mail adapter encountered an unexpected, unspecified '.
'failure.'));
}
$mailer->sendMessage($message);
} catch (PhabricatorMetaMTAPermanentFailureException $ex) {
// If any mailer raises a permanent failure, stop trying to send the
// mail with other mailers.
@ -677,6 +720,19 @@ final class PhabricatorMetaMTAMail
$this->setParam('mailer.key', $mailer_key);
}
// Now that we sent the message, store the final deliverability outcomes
// and reasoning so we can explain why things happened the way they did.
$actor_list = array();
foreach ($actors as $actor) {
$actor_list[$actor->getPHID()] = array(
'deliverable' => $actor->isDeliverable(),
'reasons' => $actor->getDeliverabilityReasons(),
);
}
$this->setParam('actors.sent', $actor_list);
$this->setParam('routing.sent', $this->getParam('routing'));
$this->setParam('routingmap.sent', $this->getRoutingRuleMap());
return $this
->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT)
->save();
@ -705,368 +761,6 @@ final class PhabricatorMetaMTAMail
$exceptions);
}
private function buildMailer(PhabricatorMailImplementationAdapter $mailer) {
$headers = $this->generateHeaders();
$params = $this->parameters;
$actors = $this->loadAllActors();
$deliverable_actors = $this->filterDeliverableActors($actors);
$default_from = (string)$this->newDefaultEmailAddress();
if (empty($params['from'])) {
$mailer->setFrom($default_from);
}
$is_first = idx($params, 'is-first-message');
unset($params['is-first-message']);
$is_threaded = (bool)idx($params, 'thread-id');
$must_encrypt = $this->getMustEncrypt();
$reply_to_name = idx($params, 'reply-to-name', '');
unset($params['reply-to-name']);
$add_cc = array();
$add_to = array();
// If we're sending one mail to everyone, some recipients will be in
// "Cc" rather than "To". We'll move them to "To" later (or supply a
// dummy "To") but need to look for the recipient in either the
// "To" or "Cc" fields here.
$target_phid = head(idx($params, 'to', array()));
if (!$target_phid) {
$target_phid = head(idx($params, 'cc', array()));
}
$preferences = $this->loadPreferences($target_phid);
foreach ($params as $key => $value) {
switch ($key) {
case 'raw-from':
list($from_email, $from_name) = $value;
$mailer->setFrom($from_email, $from_name);
break;
case 'from':
// If the mail content must be encrypted, disguise the sender.
if ($must_encrypt) {
$mailer->setFrom($default_from, pht('Phabricator'));
break;
}
$from = $value;
$actor_email = null;
$actor_name = null;
$actor = idx($actors, $from);
if ($actor) {
$actor_email = $actor->getEmailAddress();
$actor_name = $actor->getName();
}
$can_send_as_user = $actor_email &&
PhabricatorEnv::getEnvConfig('metamta.can-send-as-user');
if ($can_send_as_user) {
$mailer->setFrom($actor_email, $actor_name);
} else {
$from_email = coalesce($actor_email, $default_from);
$from_name = coalesce($actor_name, pht('Phabricator'));
if (empty($params['reply-to'])) {
$params['reply-to'] = $from_email;
$params['reply-to-name'] = $from_name;
}
$mailer->setFrom($default_from, $from_name);
}
break;
case 'reply-to':
$mailer->addReplyTo($value, $reply_to_name);
break;
case 'to':
$to_phids = $this->expandRecipients($value);
$to_actors = array_select_keys($deliverable_actors, $to_phids);
$add_to = array_merge(
$add_to,
mpull($to_actors, 'getEmailAddress'));
break;
case 'raw-to':
$add_to = array_merge($add_to, $value);
break;
case 'cc':
$cc_phids = $this->expandRecipients($value);
$cc_actors = array_select_keys($deliverable_actors, $cc_phids);
$add_cc = array_merge(
$add_cc,
mpull($cc_actors, 'getEmailAddress'));
break;
case 'attachments':
$attached_viewer = PhabricatorUser::getOmnipotentUser();
$files = $this->loadAttachedFiles($attached_viewer);
foreach ($files as $file) {
$file->attachToObject($this->getPHID());
}
// If the mail content must be encrypted, don't add attachments.
if ($must_encrypt) {
break;
}
$value = $this->getAttachments();
foreach ($value as $attachment) {
$mailer->addAttachment(
$attachment->getData(),
$attachment->getFilename(),
$attachment->getMimeType());
}
break;
case 'subject':
$subject = array();
if ($is_threaded) {
if ($this->shouldAddRePrefix($preferences)) {
$subject[] = 'Re:';
}
}
$subject[] = trim(idx($params, 'subject-prefix'));
// If mail content must be encrypted, we replace the subject with
// a generic one.
if ($must_encrypt) {
$encrypt_subject = $this->getMustEncryptSubject();
if (!strlen($encrypt_subject)) {
$encrypt_subject = pht('Object Updated');
}
$subject[] = $encrypt_subject;
} else {
$vary_prefix = idx($params, 'vary-subject-prefix');
if ($vary_prefix != '') {
if ($this->shouldVarySubject($preferences)) {
$subject[] = $vary_prefix;
}
}
$subject[] = $value;
}
$mailer->setSubject(implode(' ', array_filter($subject)));
break;
case 'thread-id':
// NOTE: Gmail freaks out about In-Reply-To and References which
// aren't in the form "<string@domain.tld>"; this is also required
// by RFC 2822, although some clients are more liberal in what they
// accept.
$domain = $this->newMailDomain();
$value = '<'.$value.'@'.$domain.'>';
if ($is_first && $mailer->supportsMessageIDHeader()) {
$headers[] = array('Message-ID', $value);
} else {
$in_reply_to = $value;
$references = array($value);
$parent_id = $this->getParentMessageID();
if ($parent_id) {
$in_reply_to = $parent_id;
// By RFC 2822, the most immediate parent should appear last
// in the "References" header, so this order is intentional.
$references[] = $parent_id;
}
$references = implode(' ', $references);
$headers[] = array('In-Reply-To', $in_reply_to);
$headers[] = array('References', $references);
}
$thread_index = $this->generateThreadIndex($value, $is_first);
$headers[] = array('Thread-Index', $thread_index);
break;
default:
// Other parameters are handled elsewhere or are not relevant to
// constructing the message.
break;
}
}
$stamps = $this->getMailStamps();
if ($stamps) {
$headers[] = array('X-Phabricator-Stamps', implode(' ', $stamps));
}
$raw_body = idx($params, 'body', '');
$body = $raw_body;
if ($must_encrypt) {
$parts = array();
$encrypt_uri = $this->getMustEncryptURI();
if (!strlen($encrypt_uri)) {
$encrypt_phid = $this->getRelatedPHID();
if ($encrypt_phid) {
$encrypt_uri = urisprintf(
'/object/%s/',
$encrypt_phid);
}
}
if (strlen($encrypt_uri)) {
$parts[] = pht(
'This secure message is notifying you of a change to this object:');
$parts[] = PhabricatorEnv::getProductionURI($encrypt_uri);
}
$parts[] = pht(
'The content for this message can only be transmitted over a '.
'secure channel. To view the message content, follow this '.
'link:');
$parts[] = PhabricatorEnv::getProductionURI($this->getURI());
$body = implode("\n\n", $parts);
} else {
$body = $raw_body;
}
$body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit');
if (strlen($body) > $body_limit) {
$body = id(new PhutilUTF8StringTruncator())
->setMaximumBytes($body_limit)
->truncateString($body);
$body .= "\n";
$body .= pht('(This email was truncated at %d bytes.)', $body_limit);
}
$mailer->setBody($body);
$body_limit -= strlen($body);
// If we sent a different message body than we were asked to, record
// what we actually sent to make debugging and diagnostics easier.
if ($body !== $raw_body) {
$this->setParam('body.sent', $body);
}
if ($must_encrypt) {
$send_html = false;
} else {
$send_html = $this->shouldSendHTML($preferences);
}
if ($send_html) {
$html_body = idx($params, 'html-body');
if (strlen($html_body)) {
// NOTE: We just drop the entire HTML body if it won't fit. Safely
// truncating HTML is hard, and we already have the text body to fall
// back to.
if (strlen($html_body) <= $body_limit) {
$mailer->setHTMLBody($html_body);
$body_limit -= strlen($html_body);
}
}
}
// Pass the headers to the mailer, then save the state so we can show
// them in the web UI. If the mail must be encrypted, we remove headers
// which are not on a strict whitelist to avoid disclosing information.
$filtered_headers = $this->filterHeaders($headers, $must_encrypt);
foreach ($filtered_headers as $header) {
list($header_key, $header_value) = $header;
$mailer->addHeader($header_key, $header_value);
}
$this->setParam('headers.unfiltered', $headers);
$this->setParam('headers.sent', $filtered_headers);
// Save the final deliverability outcomes and reasoning so we can
// explain why things happened the way they did.
$actor_list = array();
foreach ($actors as $actor) {
$actor_list[$actor->getPHID()] = array(
'deliverable' => $actor->isDeliverable(),
'reasons' => $actor->getDeliverabilityReasons(),
);
}
$this->setParam('actors.sent', $actor_list);
$this->setParam('routing.sent', $this->getParam('routing'));
$this->setParam('routingmap.sent', $this->getRoutingRuleMap());
if (!$add_to && !$add_cc) {
$this->setMessage(
pht(
'Message has no valid recipients: all To/Cc are disabled, '.
'invalid, or configured not to receive this mail.'));
return null;
}
if ($this->getIsErrorEmail()) {
$all_recipients = array_merge($add_to, $add_cc);
if ($this->shouldRateLimitMail($all_recipients)) {
$this->setMessage(
pht(
'This is an error email, but one or more recipients have '.
'exceeded the error email rate limit. Declining to deliver '.
'message.'));
return null;
}
}
if (PhabricatorEnv::getEnvConfig('phabricator.silent')) {
$this->setMessage(
pht(
'Phabricator is running in silent mode. See `%s` '.
'in the configuration to change this setting.',
'phabricator.silent'));
return null;
}
// Some mailers require a valid "To:" in order to deliver mail. If we don't
// have any "To:", fill it in with a placeholder "To:". This allows client
// rules based on whether the recipient is in "To:" or "CC:" to continue
// behaving in the same way.
if (!$add_to) {
$void_recipient = $this->newVoidEmailAddress();
$add_to = array($void_recipient->getAddress());
}
$add_to = array_unique($add_to);
$add_cc = array_diff(array_unique($add_cc), $add_to);
$mailer->addTos($add_to);
if ($add_cc) {
$mailer->addCCs($add_cc);
}
return $mailer;
}
private function generateThreadIndex($seed, $is_first_mail) {
// When threading, Outlook ignores the 'References' and 'In-Reply-To'
// headers that most clients use. Instead, it uses a custom 'Thread-Index'
// header. The format of this header is something like this (from
// camel-exchange-folder.c in Evolution Exchange):
/* A new post to a folder gets a 27-byte-long thread index. (The value
* is apparently unique but meaningless.) Each reply to a post gets a
* 32-byte-long thread index whose first 27 bytes are the same as the
* parent's thread index. Each reply to any of those gets a
* 37-byte-long thread index, etc. The Thread-Index header contains a
* base64 representation of this value.
*/
// The specific implementation uses a 27-byte header for the first email
// a recipient receives, and a random 5-byte suffix (32 bytes total)
// thereafter. This means that all the replies are (incorrectly) siblings,
// but it would be very difficult to keep track of the entire tree and this
// gets us reasonable client behavior.
$base = substr(md5($seed), 0, 27);
if (!$is_first_mail) {
// Not totally sure, but it seems like outlook orders replies by
// thread-index rather than timestamp, so to get these to show up in the
// right order we use the time as the last 4 bytes.
$base .= ' '.pack('N', time());
}
return base64_encode($base);
}
public static function shouldMailEachRecipient() {
return PhabricatorEnv::getEnvConfig('metamta.one-mail-per-recipient');
@ -1120,7 +814,7 @@ final class PhabricatorMetaMTAMail
* recipients.
* @return list<phid> Deaggregated list of mailable recipients.
*/
private function expandRecipients(array $phids) {
public function expandRecipients(array $phids) {
if ($this->recipientExpansionMap === null) {
$all_phids = $this->getAllActorPHIDs();
$this->recipientExpansionMap = id(new PhabricatorMetaMTAMemberQuery())
@ -1320,72 +1014,15 @@ final class PhabricatorMetaMTAMail
return $actors;
}
private function shouldRateLimitMail(array $all_recipients) {
try {
PhabricatorSystemActionEngine::willTakeAction(
$all_recipients,
new PhabricatorMetaMTAErrorMailAction(),
1);
return false;
} catch (PhabricatorSystemActionRateLimitException $ex) {
return true;
}
}
public function generateHeaders() {
$headers = array();
$headers[] = array('X-Phabricator-Sent-This-Message', 'Yes');
$headers[] = array('X-Mail-Transport-Agent', 'MetaMTA');
// Some clients respect this to suppress OOF and other auto-responses.
$headers[] = array('X-Auto-Response-Suppress', 'All');
$mailtags = $this->getParam('mailtags');
if ($mailtags) {
$tag_header = array();
foreach ($mailtags as $mailtag) {
$tag_header[] = '<'.$mailtag.'>';
}
$tag_header = implode(', ', $tag_header);
$headers[] = array('X-Phabricator-Mail-Tags', $tag_header);
}
$value = $this->getParam('headers', array());
foreach ($value as $pair) {
list($header_key, $header_value) = $pair;
// NOTE: If we have \n in a header, SES rejects the email.
$header_value = str_replace("\n", ' ', $header_value);
$headers[] = array($header_key, $header_value);
}
$is_bulk = $this->getParam('is-bulk');
if ($is_bulk) {
$headers[] = array('Precedence', 'bulk');
}
if ($this->getMustEncrypt()) {
$headers[] = array('X-Phabricator-Must-Encrypt', 'Yes');
}
$related_phid = $this->getRelatedPHID();
if ($related_phid) {
$headers[] = array('Thread-Topic', $related_phid);
}
$headers[] = array('X-Phabricator-Mail-ID', $this->getID());
$unique = Filesystem::readRandomCharacters(16);
$headers[] = array('X-Phabricator-Send-Attempt', $unique);
return $headers;
}
public function getDeliveredHeaders() {
return $this->getParam('headers.sent');
}
public function setDeliveredHeaders(array $headers) {
$headers = $this->flattenHeaders($headers);
return $this->setParam('headers.sent', $headers);
}
public function getUnfilteredHeaders() {
$unfiltered = $this->getParam('headers.unfiltered');
@ -1399,6 +1036,25 @@ final class PhabricatorMetaMTAMail
return $unfiltered;
}
public function setUnfilteredHeaders(array $headers) {
$headers = $this->flattenHeaders($headers);
return $this->setParam('headers.unfiltered', $headers);
}
private function flattenHeaders(array $headers) {
assert_instances_of($headers, 'PhabricatorMailHeader');
$list = array();
foreach ($list as $header) {
$list[] = array(
$header->getName(),
$header->getValue(),
);
}
return $list;
}
public function getDeliveredActors() {
return $this->getParam('actors.sent');
}
@ -1415,81 +1071,14 @@ final class PhabricatorMetaMTAMail
return $this->getParam('body.sent');
}
private function filterHeaders(array $headers, $must_encrypt) {
if (!$must_encrypt) {
return $headers;
}
$whitelist = array(
'In-Reply-To',
'Message-ID',
'Precedence',
'References',
'Thread-Index',
'Thread-Topic',
'X-Mail-Transport-Agent',
'X-Auto-Response-Suppress',
'X-Phabricator-Sent-This-Message',
'X-Phabricator-Must-Encrypt',
'X-Phabricator-Mail-ID',
'X-Phabricator-Send-Attempt',
);
// NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags".
// This header contains a significant amount of meaningful information
// about the object.
$whitelist_map = array();
foreach ($whitelist as $term) {
$whitelist_map[phutil_utf8_strtolower($term)] = true;
}
foreach ($headers as $key => $header) {
list($name, $value) = $header;
$name = phutil_utf8_strtolower($name);
if (!isset($whitelist_map[$name])) {
unset($headers[$key]);
}
}
return $headers;
public function setDeliveredBody($body) {
return $this->setParam('body.sent', $body);
}
public function getURI() {
return '/mail/detail/'.$this->getID().'/';
}
private function newMailDomain() {
$domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain');
if (strlen($domain)) {
return $domain;
}
$install_uri = PhabricatorEnv::getURI('/');
$install_uri = new PhutilURI($install_uri);
return $install_uri->getDomain();
}
public function newDefaultEmailAddress() {
$raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address');
if (strlen($raw_address)) {
return new PhutilEmailAddress($raw_address);
}
$domain = $this->newMailDomain();
$address = "noreply@{$domain}";
return new PhutilEmailAddress($address);
}
public function newVoidEmailAddress() {
return $this->newDefaultEmailAddress();
}
/* -( Routing )------------------------------------------------------------ */
@ -1578,27 +1167,6 @@ final class PhabricatorMetaMTAMail
return PhabricatorUserPreferences::loadGlobalPreferences($viewer);
}
private function shouldAddRePrefix(PhabricatorUserPreferences $preferences) {
$value = $preferences->getSettingValue(
PhabricatorEmailRePrefixSetting::SETTINGKEY);
return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX);
}
private function shouldVarySubject(PhabricatorUserPreferences $preferences) {
$value = $preferences->getSettingValue(
PhabricatorEmailVarySubjectsSetting::SETTINGKEY);
return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS);
}
private function shouldSendHTML(PhabricatorUserPreferences $preferences) {
$value = $preferences->getSettingValue(
PhabricatorEmailFormatSetting::SETTINGKEY);
return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL);
}
public function shouldRenderMailStampsInBody($viewer) {
$preferences = $this->loadPreferences($viewer->getPHID());
$value = $preferences->getSettingValue(