From 7b2b5cd91e018f3479a20056da79a82dd1008d74 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 06:16:32 -0800 Subject: [PATCH 01/67] Add basic support for a "Must Encrypt" mail flag which prevents unsecured content transmission Summary: Ref T13053. See PHI291. For particularly sensitive objects (like security issues), installs may reasonably wish to prevent details from being sent in plaintext over email. This adds a "Must Encrypt" mail behavior, which discards mail content and all identifying details, replacing it with a link to the `/mail/` application. Users can follow the link to view the message over HTTPS. The flag discards body content, attachments, and headers which imply things about the content of the object. It retains threading headers and headers which may uniquely identify the object as long as they don't disclose anyting about the content. The `bin/mail list-outbound` command now flags these messages with a `#` mark. The `bin/mail show-outbound` command now shows sent/suppressed headers and the body content as delivered (if it differs from the original body content). The `/mail/` web UI now shows a tag for messages marked with this flag. For now, there is no way to actually set this flag on mail. Test Plan: - Forced this flag on, made comments and took actions to send mail. - Reviewed mail with `bin/mail` and `/mail/` in the web UI, saw all content information omitted. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18983 --- .../PhabricatorMetaMTAMailViewController.php | 17 ++ ...atorMailManagementListOutboundWorkflow.php | 2 + ...atorMailManagementShowOutboundWorkflow.php | 57 +++++-- .../storage/PhabricatorMetaMTAMail.php | 146 ++++++++++++++++-- 4 files changed, 197 insertions(+), 25 deletions(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 80da535a9e..20bbc425b5 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -32,6 +32,23 @@ final class PhabricatorMetaMTAMailViewController $color = PhabricatorMailOutboundStatus::getStatusColor($status); $header->setStatus($icon, $color, $name); + if ($mail->getMustEncrypt()) { + Javelin::initBehavior('phabricator-tooltips'); + $header->addTag( + id(new PHUITagView()) + ->setType(PHUITagView::TYPE_SHADE) + ->setColor('blue') + ->setName(pht('Must Encrypt')) + ->setIcon('fa-shield blue') + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => pht( + 'Message content can only be transmitted over secure '. + 'channels.'), + ))); + } + $crumbs = $this->buildApplicationCrumbs() ->addTextCrumb(pht('Mail %d', $mail->getID())) ->setBorder(true); diff --git a/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php index b4aa76379e..a83dafb0a8 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php @@ -37,6 +37,7 @@ final class PhabricatorMailManagementListOutboundWorkflow $table = id(new PhutilConsoleTable()) ->setShowHeader(false) ->addColumn('id', array('title' => pht('ID'))) + ->addColumn('encrypt', array('title' => pht('#'))) ->addColumn('status', array('title' => pht('Status'))) ->addColumn('subject', array('title' => pht('Subject'))); @@ -45,6 +46,7 @@ final class PhabricatorMailManagementListOutboundWorkflow $table->addRow(array( 'id' => $mail->getID(), + 'encrypt' => ($mail->getMustEncrypt() ? '#' : ' '), 'status' => PhabricatorMailOutboundStatus::getStatusName($status), 'subject' => $mail->getSubject(), )); diff --git a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php index 54a91861ae..0fc7dd14b9 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php @@ -79,7 +79,7 @@ final class PhabricatorMailManagementShowOutboundWorkflow $info = array(); - $info[] = pht('PROPERTIES'); + $info[] = $this->newSectionHeader(pht('PROPERTIES')); $info[] = pht('ID: %d', $message->getID()); $info[] = pht('Status: %s', $message->getStatus()); $info[] = pht('Related PHID: %s', $message->getRelatedPHID()); @@ -87,15 +87,17 @@ final class PhabricatorMailManagementShowOutboundWorkflow $ignore = array( 'body' => true, + 'body.sent' => true, 'html-body' => true, 'headers' => true, 'attachments' => true, 'headers.sent' => true, + 'headers.unfiltered' => true, 'authors.sent' => true, ); $info[] = null; - $info[] = pht('PARAMETERS'); + $info[] = $this->newSectionHeader(pht('PARAMETERS')); $parameters = $message->getParameters(); foreach ($parameters as $key => $value) { if (isset($ignore[$key])) { @@ -110,22 +112,40 @@ final class PhabricatorMailManagementShowOutboundWorkflow } $info[] = null; - $info[] = pht('HEADERS'); + $info[] = $this->newSectionHeader(pht('HEADERS')); $headers = $message->getDeliveredHeaders(); - if (!$headers) { + $unfiltered = $message->getUnfilteredHeaders(); + if (!$unfiltered) { $headers = $message->generateHeaders(); + $unfiltered = $headers; } + $header_map = array(); foreach ($headers as $header) { list($name, $value) = $header; - $info[] = "{$name}: {$value}"; + $header_map[$name.':'.$value] = true; + } + + foreach ($unfiltered as $header) { + list($name, $value) = $header; + $was_sent = isset($header_map[$name.':'.$value]); + + if ($was_sent) { + $marker = ' '; + } else { + $marker = '#'; + } + + $info[] = "{$marker} {$name}: {$value}"; } $attachments = idx($parameters, 'attachments'); if ($attachments) { $info[] = null; - $info[] = pht('ATTACHMENTS'); + + $info[] = $this->newSectionHeader(pht('ATTACHMENTS')); + foreach ($attachments as $attachment) { $info[] = idx($attachment, 'filename', pht('Unnamed File')); } @@ -136,7 +156,9 @@ final class PhabricatorMailManagementShowOutboundWorkflow $actors = $message->getDeliveredActors(); if ($actors) { $info[] = null; - $info[] = pht('RECIPIENTS'); + + $info[] = $this->newSectionHeader(pht('RECIPIENTS')); + foreach ($actors as $actor_phid => $actor_info) { $actor = idx($all_actors, $actor_phid); if ($actor) { @@ -162,15 +184,22 @@ final class PhabricatorMailManagementShowOutboundWorkflow } $info[] = null; - $info[] = pht('TEXT BODY'); + $info[] = $this->newSectionHeader(pht('TEXT BODY')); if (strlen($message->getBody())) { - $info[] = $message->getBody(); + $info[] = tsprintf('%B', $message->getBody()); } else { $info[] = pht('(This message has no text body.)'); } + $delivered_body = $message->getDeliveredBody(); + if ($delivered_body !== null) { + $info[] = null; + $info[] = $this->newSectionHeader(pht('BODY AS DELIVERED'), true); + $info[] = tsprintf('%B', $delivered_body); + } + $info[] = null; - $info[] = pht('HTML BODY'); + $info[] = $this->newSectionHeader(pht('HTML BODY')); if (strlen($message->getHTMLBody())) { $info[] = $message->getHTMLBody(); $info[] = null; @@ -186,4 +215,12 @@ final class PhabricatorMailManagementShowOutboundWorkflow } } + private function newSectionHeader($label, $emphasize = false) { + if ($emphasize) { + return tsprintf('** %s **', $label); + } else { + return tsprintf('** %s **', $label); + } + } + } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 20b1482036..c203e86530 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -21,7 +21,10 @@ final class PhabricatorMetaMTAMail public function __construct() { $this->status = PhabricatorMailOutboundStatus::STATUS_QUEUE; - $this->parameters = array('sensitive' => true); + $this->parameters = array( + 'sensitive' => true, + 'mustEncrypt' => false, + ); parent::__construct(); } @@ -247,6 +250,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('sensitive', true); } + public function setMustEncrypt($bool) { + $this->setParam('mustEncrypt', $bool); + return $this; + } + + public function getMustEncrypt() { + return $this->getParam('mustEncrypt', false); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -431,6 +443,7 @@ final class PhabricatorMetaMTAMail unset($params['is-first-message']); $is_threaded = (bool)idx($params, 'thread-id'); + $must_encrypt = $this->getMustEncrypt(); $reply_to_name = idx($params, 'reply-to-name', ''); unset($params['reply-to-name']); @@ -502,6 +515,11 @@ final class PhabricatorMetaMTAMail mpull($cc_actors, 'getEmailAddress')); break; case 'attachments': + // If the mail content must be encrypted, don't add attachments. + if ($must_encrypt) { + break; + } + $value = $this->getAttachments(); foreach ($value as $attachment) { $mailer->addAttachment( @@ -521,14 +539,20 @@ final class PhabricatorMetaMTAMail $subject[] = trim(idx($params, 'subject-prefix')); - $vary_prefix = idx($params, 'vary-subject-prefix'); - if ($vary_prefix != '') { - if ($this->shouldVarySubject($preferences)) { - $subject[] = $vary_prefix; + // If mail content must be encrypted, we replace the subject with + // a generic one. + if ($must_encrypt) { + $subject[] = pht('Object Updated'); + } else { + $vary_prefix = idx($params, 'vary-subject-prefix'); + if ($vary_prefix != '') { + if ($this->shouldVarySubject($preferences)) { + $subject[] = $vary_prefix; + } } - } - $subject[] = $value; + $subject[] = $value; + } $mailer->setSubject(implode(' ', array_filter($subject))); break; @@ -567,7 +591,22 @@ final class PhabricatorMetaMTAMail } } - $body = idx($params, 'body', ''); + $raw_body = idx($params, 'body', ''); + $body = $raw_body; + if ($must_encrypt) { + $parts = array(); + $parts[] = pht( + 'The content for this message can only be transmitted over a '. + 'secure channel. To view the message content, follow this '. + 'link:'); + + $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); + + $body = implode("\n\n", $parts); + } else { + $body = $raw_body; + } + $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); if (strlen($body) > $max) { $body = id(new PhutilUTF8StringTruncator()) @@ -578,18 +617,32 @@ final class PhabricatorMetaMTAMail } $mailer->setBody($body); - $html_emails = $this->shouldSendHTML($preferences); - if ($html_emails && isset($params['html-body'])) { + // If we sent a different message body than we were asked to, record + // what we actually sent to make debugging and diagnostics easier. + if ($body !== $raw_body) { + $this->setParam('body.sent', $body); + } + + if ($must_encrypt) { + $send_html = false; + } else { + $send_html = $this->shouldSendHTML($preferences); + } + + if ($send_html && isset($params['html-body'])) { $mailer->setHTMLBody($params['html-body']); } // Pass the headers to the mailer, then save the state so we can show - // them in the web UI. - foreach ($headers as $header) { + // them in the web UI. If the mail must be encrypted, we remove headers + // which are not on a strict whitelist to avoid disclosing information. + $filtered_headers = $this->filterHeaders($headers, $must_encrypt); + foreach ($filtered_headers as $header) { list($header_key, $header_value) = $header; $mailer->addHeader($header_key, $header_value); } - $this->setParam('headers.sent', $headers); + $this->setParam('headers.unfiltered', $headers); + $this->setParam('headers.sent', $filtered_headers); // Save the final deliverability outcomes and reasoning so we can // explain why things happened the way they did. @@ -1002,8 +1055,6 @@ final class PhabricatorMetaMTAMail // Some clients respect this to suppress OOF and other auto-responses. $headers[] = array('X-Auto-Response-Suppress', 'All'); - // If the message has mailtags, filter out any recipients who don't want - // to receive this type of mail. $mailtags = $this->getParam('mailtags'); if ($mailtags) { $tag_header = array(); @@ -1028,6 +1079,10 @@ final class PhabricatorMetaMTAMail $headers[] = array('Precedence', 'bulk'); } + if ($this->getMustEncrypt()) { + $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes'); + } + return $headers; } @@ -1035,6 +1090,19 @@ final class PhabricatorMetaMTAMail return $this->getParam('headers.sent'); } + public function getUnfilteredHeaders() { + $unfiltered = $this->getParam('headers.unfiltered'); + + if ($unfiltered === null) { + // Older versions of Phabricator did not filter headers, and thus did + // not record unfiltered headers. If we don't have unfiltered header + // data just return the delivered headers for compatibility. + return $this->getDeliveredHeaders(); + } + + return $unfiltered; + } + public function getDeliveredActors() { return $this->getParam('actors.sent'); } @@ -1047,6 +1115,54 @@ final class PhabricatorMetaMTAMail return $this->getParam('routingmap.sent'); } + public function getDeliveredBody() { + return $this->getParam('body.sent'); + } + + private function filterHeaders(array $headers, $must_encrypt) { + if (!$must_encrypt) { + return $headers; + } + + $whitelist = array( + 'In-Reply-To', + 'Message-ID', + 'Precedence', + 'References', + 'Thread-Index', + + 'X-Mail-Transport-Agent', + 'X-Auto-Response-Suppress', + + 'X-Phabricator-Sent-This-Message', + 'X-Phabricator-Must-Encrypt', + ); + + // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags". + // This header contains a significant amount of meaningful information + // about the object. + + $whitelist_map = array(); + foreach ($whitelist as $term) { + $whitelist_map[phutil_utf8_strtolower($term)] = true; + } + + foreach ($headers as $key => $header) { + list($name, $value) = $header; + $name = phutil_utf8_strtolower($name); + + if (!isset($whitelist_map[$name])) { + unset($headers[$key]); + } + } + + return $headers; + } + + public function getURI() { + return '/mail/detail/'.$this->getID().'/'; + } + /* -( Routing )------------------------------------------------------------ */ From cbe4e68c072239608a5b59d50dec399cb36699b0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 09:19:14 -0800 Subject: [PATCH 02/67] Add a Herald action to trigger "Must Encrypt" for mail Summary: Depends on D18983. Ref T13053. Adds a new Herald action to activate the "must encrypt" flag and drop mail content. Test Plan: - Created a new Herald rule: {F5407075} - Created a "dog task" (woof woof, unsecure) and a "duck task" (quack quack, secure). - Viewed mail for both in `bin/mail` and web UI, saw appropriate security/encryption behavior. - Viewed "Must Encrypt" in "Headers" tab for the duck mail, saw why the mail was encrypted (link to Herald rule). Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18984 --- src/__phutil_library_map__.php | 2 + .../herald/adapter/HeraldAdapter.php | 14 +++++ .../PhabricatorMetaMTAMailViewController.php | 9 +++ ...PhabricatorMailMustEncryptHeraldAction.php | 62 +++++++++++++++++++ .../PhabricatorMetaMTAEmailHeraldAction.php | 4 ++ .../storage/PhabricatorMetaMTAMail.php | 9 +++ ...habricatorApplicationTransactionEditor.php | 11 ++++ 7 files changed, 111 insertions(+) create mode 100644 src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 32985c76c0..f0e2d29cfc 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3189,6 +3189,7 @@ phutil_register_library_map(array( 'PhabricatorMailManagementUnverifyWorkflow' => 'applications/metamta/management/PhabricatorMailManagementUnverifyWorkflow.php', 'PhabricatorMailManagementVolumeWorkflow' => 'applications/metamta/management/PhabricatorMailManagementVolumeWorkflow.php', 'PhabricatorMailManagementWorkflow' => 'applications/metamta/management/PhabricatorMailManagementWorkflow.php', + 'PhabricatorMailMustEncryptHeraldAction' => 'applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php', 'PhabricatorMailOutboundMailHeraldAdapter' => 'applications/metamta/herald/PhabricatorMailOutboundMailHeraldAdapter.php', 'PhabricatorMailOutboundRoutingHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingHeraldAction.php', 'PhabricatorMailOutboundRoutingSelfEmailHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingSelfEmailHeraldAction.php', @@ -8674,6 +8675,7 @@ phutil_register_library_map(array( 'PhabricatorMailManagementUnverifyWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementVolumeWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorMailMustEncryptHeraldAction' => 'HeraldAction', 'PhabricatorMailOutboundMailHeraldAdapter' => 'HeraldAdapter', 'PhabricatorMailOutboundRoutingHeraldAction' => 'HeraldAction', 'PhabricatorMailOutboundRoutingSelfEmailHeraldAction' => 'PhabricatorMailOutboundRoutingHeraldAction', diff --git a/src/applications/herald/adapter/HeraldAdapter.php b/src/applications/herald/adapter/HeraldAdapter.php index 9d56f474ff..cc0fdbd3b5 100644 --- a/src/applications/herald/adapter/HeraldAdapter.php +++ b/src/applications/herald/adapter/HeraldAdapter.php @@ -39,6 +39,7 @@ abstract class HeraldAdapter extends Phobject { private $edgeCache = array(); private $forbiddenActions = array(); private $viewer; + private $mustEncryptReasons = array(); public function getEmailPHIDs() { return array_values($this->emailPHIDs); @@ -1182,4 +1183,17 @@ abstract class HeraldAdapter extends Phobject { return $this->forbiddenActions[$action]; } + +/* -( Must Encrypt )------------------------------------------------------- */ + + + final public function addMustEncryptReason($reason) { + $this->mustEncryptReasons[] = $reason; + return $this; + } + + final public function getMustEncryptReasons() { + return $this->mustEncryptReasons; + } + } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 20bbc425b5..1aca34c2ea 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -175,6 +175,15 @@ final class PhabricatorMetaMTAMailViewController $properties->addProperty($key, $value); } + $encrypt_phids = $mail->getMustEncryptReasons(); + if ($encrypt_phids) { + $properties->addProperty( + pht('Must Encrypt'), + $viewer->loadHandles($encrypt_phids) + ->renderList()); + } + + return $properties; } diff --git a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php new file mode 100644 index 0000000000..f8cf7ee204 --- /dev/null +++ b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php @@ -0,0 +1,62 @@ +getRule()->getPHID(); + + $adapter = $this->getAdapter(); + $adapter->addMustEncryptReason($rule_phid); + + $this->logEffect(self::DO_MUST_ENCRYPT, array($rule_phid)); + } + + protected function getActionEffectMap() { + return array( + self::DO_MUST_ENCRYPT => array( + 'icon' => 'fa-shield', + 'color' => 'blue', + 'name' => pht('Must Encrypt'), + ), + ); + } + + protected function renderActionEffectDescription($type, $data) { + switch ($type) { + case self::DO_MUST_ENCRYPT: + return pht( + 'Made it a requirement that mail content be transmitted only '. + 'over secure channels.'); + } + } + +} diff --git a/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php b/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php index 74fb879fe7..383b8ebd36 100644 --- a/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php +++ b/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php @@ -13,6 +13,10 @@ abstract class PhabricatorMetaMTAEmailHeraldAction } public function supportsObject($object) { + return self::isMailGeneratingObject($object); + } + + public static function isMailGeneratingObject($object) { // NOTE: This implementation lacks generality, but there's no great way to // figure out if something generates email right now. diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index c203e86530..a9736c1766 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -259,6 +259,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('mustEncrypt', false); } + public function setMustEncryptReasons(array $reasons) { + $this->setParam('mustEncryptReasons', $reasons); + return $this; + } + + public function getMustEncryptReasons() { + return $this->getParam('mustEncryptReasons', array()); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 155592fc4e..0dc6a06fbf 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -71,6 +71,7 @@ abstract class PhabricatorApplicationTransactionEditor private $mailShouldSend = false; private $modularTypes; private $silent; + private $mustEncrypt; private $transactionQueue = array(); @@ -2549,6 +2550,13 @@ abstract class PhabricatorApplicationTransactionEditor $this->loadHandles($xactions); $mail = $this->buildMailForTarget($object, $xactions, $target); + + if ($this->mustEncrypt) { + $mail + ->setMustEncrypt(true) + ->setMustEncryptReasons($this->mustEncrypt); + } + } catch (Exception $ex) { $caught = $ex; } @@ -3214,6 +3222,8 @@ abstract class PhabricatorApplicationTransactionEditor $adapter->getQueuedHarbormasterBuildRequests()); } + $this->mustEncrypt = $adapter->getMustEncryptReasons(); + return array_merge( $this->didApplyHeraldRules($object, $adapter, $xscript), $adapter->getQueuedTransactions()); @@ -3558,6 +3568,7 @@ abstract class PhabricatorApplicationTransactionEditor 'feedRelatedPHIDs', 'feedShouldPublish', 'mailShouldSend', + 'mustEncrypt', ); } From eb06aca951cee6b01ceecf57286a892c77348d81 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 11:28:06 -0800 Subject: [PATCH 03/67] Support DestructionEngine in MetaMTAMail Summary: Depends on D18984. Ref T13053. See D13408 for the original change and why this doesn't use DestructionEngine right now. The quick version is: - It causes us to write a destruction log, which is slightly silly (we're deleting one thing and creating another). - It's a little bit slower than not using DestructionEngine. However, it gets us some stuff for free that's likely relevant now (e.g., Herald Transcript cleanup) and I'm planning to move attachments to Files, but want to be able to delete them when mail is destroyed. The destruction log is a touch silly, but those records are very small and that log gets GC'd later without generating new logs. We could silence the log from the GC if it's ever an issue. Test Plan: Used `bin/remove destroy` and `bin/garbage collect --collector mail.sent` to destroy mail and collect garbage. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18985 --- src/__phutil_library_map__.php | 1 + .../MetaMTAMailSentGarbageCollector.php | 3 ++- .../storage/PhabricatorMetaMTAMail.php | 26 ++++++++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index f0e2d29cfc..aff0d5e36f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -8739,6 +8739,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMail' => array( 'PhabricatorMetaMTADAO', 'PhabricatorPolicyInterface', + 'PhabricatorDestructibleInterface', ), 'PhabricatorMetaMTAMailBody' => 'Phobject', 'PhabricatorMetaMTAMailBodyTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php index c9ca274436..dacd46d187 100644 --- a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php +++ b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php @@ -18,8 +18,9 @@ final class MetaMTAMailSentGarbageCollector 'dateCreated < %d LIMIT 100', $this->getGarbageEpoch()); + $engine = new PhabricatorDestructionEngine(); foreach ($mails as $mail) { - $mail->delete(); + $engine->destroyObject($mail); } return (count($mails) == 100); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a9736c1766..d5111529f3 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -5,7 +5,9 @@ */ final class PhabricatorMetaMTAMail extends PhabricatorMetaMTADAO - implements PhabricatorPolicyInterface { + implements + PhabricatorPolicyInterface, + PhabricatorDestructibleInterface { const RETRY_DELAY = 5; @@ -1041,20 +1043,6 @@ final class PhabricatorMetaMTAMail } } - public function delete() { - $this->openTransaction(); - queryfx( - $this->establishConnection('w'), - 'DELETE FROM %T WHERE src = %s AND type = %d', - PhabricatorEdgeConfig::TABLE_NAME_EDGE, - $this->getPHID(), - PhabricatorMetaMTAMailHasRecipientEdgeType::EDGECONST); - $ret = parent::delete(); - $this->saveTransaction(); - - return $ret; - } - public function generateHeaders() { $headers = array(); @@ -1306,4 +1294,12 @@ final class PhabricatorMetaMTAMail } +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + $this->delete(); + } + } From 6d90c7ad92b35e95ceda1b94772b7120cea84acd Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 11:17:47 -0800 Subject: [PATCH 04/67] Save mail attachments in Files, not on the actual objects Summary: Depends on D18985. Ref T13053. See PHI125. Currently, mail attachments are just encoded onto the actual objects in the `MetaMTAMail` table. This fails if attachments can't be encoded in JSON -- e.g., they aren't UTF8. This happens most often when revisions or commits attach patches to mail and those patches contain source code changes for files that are not encoded in UTF8. Instead, save attachments in (and load attachments from) Files. Test Plan: Enabled patches for mail, created a revision, saw it attach a patch. Viewed mail in web UI, saw link to download patch. Followed link, saw sensible file. Checked database, saw a `filePHID`. Destroyed mail with `bin/remove destroy`, saw attached files also destroyed. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18986 --- .../PhabricatorMetaMTAMailViewController.php | 6 +++ .../storage/PhabricatorMetaMTAAttachment.php | 44 ++++++++++++++++--- .../storage/PhabricatorMetaMTAMail.php | 41 +++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 1aca34c2ea..03d340bac9 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -151,6 +151,12 @@ final class PhabricatorMetaMTAMailViewController $properties->addTextContent($body); + $file_phids = $mail->getAttachmentFilePHIDs(); + if ($file_phids) { + $properties->addProperty( + pht('Attached Files'), + $viewer->loadHandles($file_phids)->renderList()); + } return $properties; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php b/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php index b26bb0b8a7..256bee46e8 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php @@ -1,9 +1,12 @@ setData($data); @@ -39,18 +42,49 @@ final class PhabricatorMetaMTAAttachment extends Phobject { } public function toDictionary() { + if (!$this->file) { + $iterator = new ArrayIterator(array($this->getData())); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($this->getFilename()) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($this->getMimeType()) + ->setIterator($iterator); + + $this->file = $source->uploadFile(); + } + return array( 'filename' => $this->getFilename(), 'mimetype' => $this->getMimeType(), - 'data' => $this->getData(), + 'filePHID' => $this->file->getPHID(), ); } public static function newFromDictionary(array $dict) { - return new PhabricatorMetaMTAAttachment( + $file = null; + + $file_phid = idx($dict, 'filePHID'); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + $dict['data'] = $file->loadFileData(); + } + } + + $attachment = new self( idx($dict, 'data'), idx($dict, 'filename'), idx($dict, 'mimetype')); + + if ($file) { + $attachment->file = $file; + } + + return $attachment; } } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index d5111529f3..a61951650c 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -197,6 +197,35 @@ final class PhabricatorMetaMTAMail return $result; } + public function getAttachmentFilePHIDs() { + $file_phids = array(); + + $dictionaries = $this->getParam('attachments'); + if ($dictionaries) { + foreach ($dictionaries as $dictionary) { + $file_phid = idx($dictionary, 'filePHID'); + if ($file_phid) { + $file_phids[] = $file_phid; + } + } + } + + return $file_phids; + } + + public function loadAttachedFiles(PhabricatorUser $viewer) { + $file_phids = $this->getAttachmentFilePHIDs(); + + if (!$file_phids) { + return array(); + } + + return id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs($file_phids) + ->execute(); + } + public function setAttachments(array $attachments) { assert_instances_of($attachments, 'PhabricatorMetaMTAAttachment'); $this->setParam('attachments', mpull($attachments, 'toDictionary')); @@ -526,6 +555,12 @@ final class PhabricatorMetaMTAMail mpull($cc_actors, 'getEmailAddress')); break; case 'attachments': + $attached_viewer = PhabricatorUser::getOmnipotentUser(); + $files = $this->loadAttachedFiles($attached_viewer); + foreach ($files as $file) { + $file->attachToObject($this->getPHID()); + } + // If the mail content must be encrypted, don't add attachments. if ($must_encrypt) { break; @@ -1299,6 +1334,12 @@ final class PhabricatorMetaMTAMail public function destroyObjectPermanently( PhabricatorDestructionEngine $engine) { + + $files = $this->loadAttachedFiles($engine->getViewer()); + foreach ($files as $file) { + $engine->destroyObject($file); + } + $this->delete(); } From 55f7cdb99b3f40b94aa07b1d29c8eaf344db34da Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 2 Feb 2018 14:57:25 -0800 Subject: [PATCH 05/67] Fix a bad classname reference in the "Must Encrypt" action --- .../metamta/herald/PhabricatorMailMustEncryptHeraldAction.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php index f8cf7ee204..027e1bb733 100644 --- a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php +++ b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php @@ -16,7 +16,7 @@ final class PhabricatorMailMustEncryptHeraldAction 'Require mail content be transmitted only over secure channels.'); } public function supportsObject($object) { - return self::isMailGeneratingObject($object); + return PhabricatorMetaMTAEmailHeraldAction::isMailGeneratingObject($object); } public function getActionGroupKey() { From 956c4058e64cf3ec338e08e5218980e7f93f72fa Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 4 Feb 2018 05:55:01 -0800 Subject: [PATCH 06/67] Add a `bin/conduit call` support binary Summary: Ref T13060. See PHI343. Triaging this bug required figuring out where in the pipeline UTF8 was being dropped, and bisecting the pipeline required making calls to Conduit. Currently, there's no easy way to debug/inspect arbitrary Conduit calls, especially when they are `diffusion.*` calls which route to a different host (even if you have a real session and use the web console for these, you just see an HTTP service call to the target host in DarkConsole). Add a `bin/conduit` utility to make this kind of debugging easier, with an eye toward the Phacility production cluster (or other similar clusters) specifically. Test Plan: - Ran `echo '{}' | bin/conduit call --method conduit.ping --input -` and similar. - Used a similar approach to successfully diagnose the UTF8 issue in T13060. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13060 Differential Revision: https://secure.phabricator.com/D18987 --- bin/conduit | 1 + scripts/setup/manage_conduit.php | 21 ++++++ src/__phutil_library_map__.php | 4 ++ ...abricatorConduitCallManagementWorkflow.php | 66 +++++++++++++++++++ .../PhabricatorConduitManagementWorkflow.php | 4 ++ 5 files changed, 96 insertions(+) create mode 120000 bin/conduit create mode 100755 scripts/setup/manage_conduit.php create mode 100644 src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php create mode 100644 src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php diff --git a/bin/conduit b/bin/conduit new file mode 120000 index 0000000000..9221340a93 --- /dev/null +++ b/bin/conduit @@ -0,0 +1 @@ +../scripts/setup/manage_conduit.php \ No newline at end of file diff --git a/scripts/setup/manage_conduit.php b/scripts/setup/manage_conduit.php new file mode 100755 index 0000000000..07384e7ed8 --- /dev/null +++ b/scripts/setup/manage_conduit.php @@ -0,0 +1,21 @@ +#!/usr/bin/env php +setTagline(pht('manage Conduit')); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorConduitManagementWorkflow') + ->execute(); +$workflows[] = new PhutilHelpArgumentWorkflow(); +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index aff0d5e36f..7076cde010 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2425,6 +2425,7 @@ phutil_register_library_map(array( 'PhabricatorCommonPasswords' => 'applications/auth/constants/PhabricatorCommonPasswords.php', 'PhabricatorConduitAPIController' => 'applications/conduit/controller/PhabricatorConduitAPIController.php', 'PhabricatorConduitApplication' => 'applications/conduit/application/PhabricatorConduitApplication.php', + 'PhabricatorConduitCallManagementWorkflow' => 'applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php', 'PhabricatorConduitCertificateToken' => 'applications/conduit/storage/PhabricatorConduitCertificateToken.php', 'PhabricatorConduitConsoleController' => 'applications/conduit/controller/PhabricatorConduitConsoleController.php', 'PhabricatorConduitContentSource' => 'infrastructure/contentsource/PhabricatorConduitContentSource.php', @@ -2435,6 +2436,7 @@ phutil_register_library_map(array( 'PhabricatorConduitLogController' => 'applications/conduit/controller/PhabricatorConduitLogController.php', 'PhabricatorConduitLogQuery' => 'applications/conduit/query/PhabricatorConduitLogQuery.php', 'PhabricatorConduitLogSearchEngine' => 'applications/conduit/query/PhabricatorConduitLogSearchEngine.php', + 'PhabricatorConduitManagementWorkflow' => 'applications/conduit/management/PhabricatorConduitManagementWorkflow.php', 'PhabricatorConduitMethodCallLog' => 'applications/conduit/storage/PhabricatorConduitMethodCallLog.php', 'PhabricatorConduitMethodQuery' => 'applications/conduit/query/PhabricatorConduitMethodQuery.php', 'PhabricatorConduitRequestExceptionHandler' => 'aphront/handler/PhabricatorConduitRequestExceptionHandler.php', @@ -7822,6 +7824,7 @@ phutil_register_library_map(array( 'PhabricatorCommonPasswords' => 'Phobject', 'PhabricatorConduitAPIController' => 'PhabricatorConduitController', 'PhabricatorConduitApplication' => 'PhabricatorApplication', + 'PhabricatorConduitCallManagementWorkflow' => 'PhabricatorConduitManagementWorkflow', 'PhabricatorConduitCertificateToken' => 'PhabricatorConduitDAO', 'PhabricatorConduitConsoleController' => 'PhabricatorConduitController', 'PhabricatorConduitContentSource' => 'PhabricatorContentSource', @@ -7832,6 +7835,7 @@ phutil_register_library_map(array( 'PhabricatorConduitLogController' => 'PhabricatorConduitController', 'PhabricatorConduitLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorConduitLogSearchEngine' => 'PhabricatorApplicationSearchEngine', + 'PhabricatorConduitManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorConduitMethodCallLog' => array( 'PhabricatorConduitDAO', 'PhabricatorPolicyInterface', diff --git a/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php b/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php new file mode 100644 index 0000000000..6cb3bd2409 --- /dev/null +++ b/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php @@ -0,0 +1,66 @@ +setName('call') + ->setSynopsis(pht('Call a Conduit method..')) + ->setArguments( + array( + array( + 'name' => 'method', + 'param' => 'method', + 'help' => pht('Method to call.'), + ), + array( + 'name' => 'input', + 'param' => 'input', + 'help' => pht( + 'File to read parameters from, or "-" to read from '. + 'stdin.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $method = $args->getArg('method'); + if (!strlen($method)) { + throw new PhutilArgumentUsageException( + pht('Specify a method to call with "--method".')); + } + + $input = $args->getArg('input'); + if (!strlen($input)) { + throw new PhutilArgumentUsageException( + pht('Specify a file to read parameters from with "--input".')); + } + + if ($input === '-') { + fprintf(STDERR, tsprintf("%s\n", pht('Reading input from stdin...'))); + $input_json = file_get_contents('php://stdin'); + } else { + $input_json = Filesystem::readFile($input); + } + + $params = phutil_json_decode($input_json); + + $result = id(new ConduitCall($method, $params)) + ->setUser($viewer) + ->execute(); + + $output = array( + 'result' => $result, + ); + + echo tsprintf( + "%B\n", + id(new PhutilJSON())->encodeFormatted($output)); + + return 0; + } + +} diff --git a/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php b/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php new file mode 100644 index 0000000000..4abb250fc4 --- /dev/null +++ b/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php @@ -0,0 +1,4 @@ + Date: Sun, 4 Feb 2018 06:01:49 -0800 Subject: [PATCH 07/67] Always setlocale() to en_US.UTF-8 for the main process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Depends on D18987. See PHI343. Fixes T13060. See also T7339. When the main process starts up with `LANG=POSIX` (this is the default on Ubuntu) and we later try to run a subprocess with a UTF8 character in the argument list (like `git cat-file blob πŸ‘.txt`), the argument is not passed to the subprocess correctly. We already set `LANG=en_US.UTF-8` in the //subprocess// environment, but this only controls behavior for the subprocess itself. It appears that the argument list encoding before the actual subprocess starts depends on the parent process's locale setting, which makes some degree of sense. Setting `putenv('LANG=en_US.UTF-8')` has no effect on this, but my guess is that the parent process's locale setting is read at startup (rather than read anew from `LANG` every time) and not changed by further modifications of `LANG`. Using `setlocale(...)` does appear to fix this. Ideally, installs would probably set some UTF-8-compatible LANG setting as the default. However, this makes setup harder and I couldn't figure out how to do it on our production Ubuntu AMI after spending a reasonable amount of time at it (see T13060). Since it's very rare that this setting matters, try to just do the right thing. This may fail if "en_US.UTF-8" isn't available, but I think warnings/remedies to this are in the scope of T7339, since we want this locale to exist for other legitimate reasons anyway. Test Plan: - Applied this fix in production, processed the failing worker task from PHI343 after kicking Apache hard enough. - Ran locally with `setlocale(LC_ALL, 'duck.quack')` to make sure a bad/invalid/unavailable setting didn't break anything, didn't hit any issues. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13060 Differential Revision: https://secure.phabricator.com/D18988 --- support/startup/PhabricatorStartup.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/support/startup/PhabricatorStartup.php b/support/startup/PhabricatorStartup.php index 1911a46b8a..212b057376 100644 --- a/support/startup/PhabricatorStartup.php +++ b/support/startup/PhabricatorStartup.php @@ -395,6 +395,11 @@ final class PhabricatorStartup { if (function_exists('libxml_disable_entity_loader')) { libxml_disable_entity_loader(true); } + + // See T13060. If the locale for this process (the parent process) is not + // a UTF-8 locale we can encounter problems when launching subprocesses + // which receive UTF-8 parameters in their command line argument list. + @setlocale(LC_ALL, 'en_US.UTF-8'); } From b3880975e5a17e0202055e235556316b34146cab Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:24:17 -0800 Subject: [PATCH 08/67] =?UTF-8?q?Add=20aliases=20for=20"party"=20emoji=20(?= =?UTF-8?q?=F0=9F=8E=89)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: This is currently `:tada:`, which I'd never have guessed. (This isn't a super scalable approach, but this emoji is in particularly common use. See also T12644.) Test Plan: Typed `:party`, `:confet`, etc. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18993 --- resources/emoji/manifest.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/resources/emoji/manifest.json b/resources/emoji/manifest.json index 2d8388d277..47568fb43d 100644 --- a/resources/emoji/manifest.json +++ b/resources/emoji/manifest.json @@ -1622,5 +1622,9 @@ "zipper_mouth": "\ud83e\udd10", "zzz": "\ud83d\udca4", "100": "\ud83d\udcaf", - "1234": "\ud83d\udd22" + "1234": "\ud83d\udd22", + + "party": "\ud83c\udf89", + "celebration": "\ud83c\udf89", + "confetti": "\ud83c\udf89" } From ef121b3e17cfa9649820938e2092a5c98183c8f1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 05:58:21 -0800 Subject: [PATCH 09/67] Fix a Herald repetition policy selection error for rule types which support only one policy Summary: Ref T13048. See . When a rule supports only one repetition policy (always "every time") like "Commit Hook" rules, we don't render a control for `repetition_policy` and fail to update it when saving. Before the changes to support the new "if the rule did not match the last time" policy, this workflow just defaulted to "every time" if the input was invalid, but this was changed by accident in D18926 when I removed some of the toInt/toString juggling code. (This patch also prevents users from fiddling with the form to create a rule which evaluates with an invalid policy; this wasn't validated before.) Test Plan: - Created new "Commit Hook" (only one policy available) rule. - Saved existing "Commit Hook" rule. - Created new "Task" (multiple policies) rule. - Saved existing Task rule. - Set task rule to each repetition policy, saved, verified the save worked. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13048 Differential Revision: https://secure.phabricator.com/D18992 --- .../controller/HeraldRuleController.php | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/applications/herald/controller/HeraldRuleController.php b/src/applications/herald/controller/HeraldRuleController.php index c61c29e90e..d400f8ae90 100644 --- a/src/applications/herald/controller/HeraldRuleController.php +++ b/src/applications/herald/controller/HeraldRuleController.php @@ -265,7 +265,15 @@ final class HeraldRuleController extends HeraldController { $new_name = $request->getStr('name'); $match_all = ($request->getStr('must_match') == 'all'); - $repetition_policy_param = $request->getStr('repetition_policy'); + $repetition_policy = $request->getStr('repetition_policy'); + + // If the user selected an invalid policy, or there's only one possible + // value so we didn't render a control, adjust the value to the first + // valid policy value. + $repetition_options = $this->getRepetitionOptionMap($adapter); + if (!isset($repetition_options[$repetition_policy])) { + $repetition_policy = head_key($repetition_options); + } $e_name = true; $errors = array(); @@ -348,7 +356,7 @@ final class HeraldRuleController extends HeraldController { $match_all, $conditions, $actions, - $repetition_policy_param); + $repetition_policy); $xactions = array(); $xactions[] = id(new HeraldRuleTransaction()) @@ -373,7 +381,7 @@ final class HeraldRuleController extends HeraldController { // mutate current rule, so it would be sent to the client in the right state $rule->setMustMatchAll((int)$match_all); $rule->setName($new_name); - $rule->setRepetitionPolicyStringConstant($repetition_policy_param); + $rule->setRepetitionPolicyStringConstant($repetition_policy); $rule->attachConditions($conditions); $rule->attachActions($actions); @@ -594,13 +602,9 @@ final class HeraldRuleController extends HeraldController { */ private function renderRepetitionSelector($rule, HeraldAdapter $adapter) { $repetition_policy = $rule->getRepetitionPolicyStringConstant(); - - $repetition_options = $adapter->getRepetitionOptions(); - $repetition_names = HeraldRule::getRepetitionPolicySelectOptionMap(); - $repetition_map = array_select_keys($repetition_names, $repetition_options); - + $repetition_map = $this->getRepetitionOptionMap($adapter); if (count($repetition_map) < 2) { - return head($repetition_names); + return head($repetition_map); } else { return AphrontFormSelectControl::renderSelectTag( $repetition_policy, @@ -611,6 +615,11 @@ final class HeraldRuleController extends HeraldController { } } + private function getRepetitionOptionMap(HeraldAdapter $adapter) { + $repetition_options = $adapter->getRepetitionOptions(); + $repetition_names = HeraldRule::getRepetitionPolicySelectOptionMap(); + return array_select_keys($repetition_names, $repetition_options); + } protected function buildTokenizerTemplates() { $template = new AphrontTokenizerTemplateView(); From c3f95bc410a3173fc1d2eccc392e0d4b6b6c65a4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 4 Feb 2018 09:04:12 -0800 Subject: [PATCH 10/67] Add basic support for mail "stamps" to improve client mail routing Summary: Ref T10448. Currently, we use "mail tags" (in {nav Settings > Email Preferences}) to give users some ability to route mail. There are a number of major issues with this: - It isn't modular and can't be extended by third-party applications. - The UI is a giant mess of 5,000 individual settings. - Settings don't map clearly to actual edits. - A lot of stuff isn't covered by any setting. This adds a new system, called "mail stamps", which is similar to "mail tags" but tries to fix all these problems. I called these "stamps" because: stamps make sense with mail; we can't throw away the old system just yet and need to keep it around for a bit; we don't use this term for anything else; it avoids confusion with project tags. (Conceptually, imagine these as ink stamps like "RETURN TO SENDER" or "FRAGILE", not actual postage stamps.) The only real "trick" here is that later versions of this will need to enumerate possible stamps for an object and maybe all possible stamps for all objects in the system. This is why stamp generation is separated into a "template" phase and a "value" phase. In future changes, the "template" phase can be used on its own to generate documentation and typeaheads and let users build rules. This may need some more refinement before it really works since I haven't built any of that yet. Also adds a preference for getting stamps in the header only (default) or header and body (better for Gmail, which can't route based on headers). Test Plan: Fiddled with preference, sent some mail and saw a "STAMPS" setting in the body and an "X-Phabricator-Stamps" header. {F5411694} Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T10448 Differential Revision: https://secure.phabricator.com/D18991 --- src/__phutil_library_map__.php | 10 ++ .../engine/PhabricatorMailEngineExtension.php | 47 ++++++ .../replyhandler/PhabricatorMailTarget.php | 33 +++- .../metamta/stamp/PhabricatorMailStamp.php | 88 +++++++++++ .../stamp/PhabricatorStringMailStamp.php | 16 ++ .../storage/PhabricatorMetaMTAMail.php | 29 ++++ .../setting/PhabricatorEmailStampsSetting.php | 47 ++++++ ...habricatorApplicationTransactionEditor.php | 149 ++++++++++++++++++ ...orApplicationObjectMailEngineExtension.php | 92 +++++++++++ 9 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 src/applications/metamta/engine/PhabricatorMailEngineExtension.php create mode 100644 src/applications/metamta/stamp/PhabricatorMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorStringMailStamp.php create mode 100644 src/applications/settings/setting/PhabricatorEmailStampsSetting.php create mode 100644 src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 7076cde010..6fa14fbacb 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1958,6 +1958,7 @@ phutil_register_library_map(array( 'PhabricatorApplicationEditHTTPParameterHelpView' => 'applications/transactions/view/PhabricatorApplicationEditHTTPParameterHelpView.php', 'PhabricatorApplicationEditor' => 'applications/meta/editor/PhabricatorApplicationEditor.php', 'PhabricatorApplicationEmailCommandsController' => 'applications/meta/controller/PhabricatorApplicationEmailCommandsController.php', + 'PhabricatorApplicationObjectMailEngineExtension' => 'applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php', 'PhabricatorApplicationPanelController' => 'applications/meta/controller/PhabricatorApplicationPanelController.php', 'PhabricatorApplicationPolicyChangeTransaction' => 'applications/meta/xactions/PhabricatorApplicationPolicyChangeTransaction.php', 'PhabricatorApplicationProfileMenuItem' => 'applications/search/menuitem/PhabricatorApplicationProfileMenuItem.php', @@ -2828,6 +2829,7 @@ phutil_register_library_map(array( 'PhabricatorEmailPreferencesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailPreferencesSettingsPanel.php', 'PhabricatorEmailRePrefixSetting' => 'applications/settings/setting/PhabricatorEmailRePrefixSetting.php', 'PhabricatorEmailSelfActionsSetting' => 'applications/settings/setting/PhabricatorEmailSelfActionsSetting.php', + 'PhabricatorEmailStampsSetting' => 'applications/settings/setting/PhabricatorEmailStampsSetting.php', 'PhabricatorEmailTagsSetting' => 'applications/settings/setting/PhabricatorEmailTagsSetting.php', 'PhabricatorEmailVarySubjectsSetting' => 'applications/settings/setting/PhabricatorEmailVarySubjectsSetting.php', 'PhabricatorEmailVerificationController' => 'applications/auth/controller/PhabricatorEmailVerificationController.php', @@ -3174,6 +3176,7 @@ phutil_register_library_map(array( 'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php', 'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php', 'PhabricatorMailEmailSubjectHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailSubjectHeraldField.php', + 'PhabricatorMailEngineExtension' => 'applications/metamta/engine/PhabricatorMailEngineExtension.php', 'PhabricatorMailImplementationAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationAdapter.php', 'PhabricatorMailImplementationAmazonSESAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php', 'PhabricatorMailImplementationMailgunAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php', @@ -3202,6 +3205,7 @@ phutil_register_library_map(array( 'PhabricatorMailReplyHandler' => 'applications/metamta/replyhandler/PhabricatorMailReplyHandler.php', 'PhabricatorMailRoutingRule' => 'applications/metamta/constants/PhabricatorMailRoutingRule.php', 'PhabricatorMailSetupCheck' => 'applications/config/check/PhabricatorMailSetupCheck.php', + 'PhabricatorMailStamp' => 'applications/metamta/stamp/PhabricatorMailStamp.php', 'PhabricatorMailTarget' => 'applications/metamta/replyhandler/PhabricatorMailTarget.php', 'PhabricatorMailgunConfigOptions' => 'applications/config/option/PhabricatorMailgunConfigOptions.php', 'PhabricatorMainMenuBarExtension' => 'view/page/menu/PhabricatorMainMenuBarExtension.php', @@ -4204,6 +4208,7 @@ phutil_register_library_map(array( 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', 'PhabricatorStringListExportField' => 'infrastructure/export/field/PhabricatorStringListExportField.php', + 'PhabricatorStringMailStamp' => 'applications/metamta/stamp/PhabricatorStringMailStamp.php', 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', 'PhabricatorSubmitEditField' => 'applications/transactions/editfield/PhabricatorSubmitEditField.php', 'PhabricatorSubscribableInterface' => 'applications/subscriptions/interface/PhabricatorSubscribableInterface.php', @@ -7269,6 +7274,7 @@ phutil_register_library_map(array( 'PhabricatorApplicationEditHTTPParameterHelpView' => 'AphrontView', 'PhabricatorApplicationEditor' => 'PhabricatorApplicationTransactionEditor', 'PhabricatorApplicationEmailCommandsController' => 'PhabricatorApplicationsController', + 'PhabricatorApplicationObjectMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorApplicationPanelController' => 'PhabricatorApplicationsController', 'PhabricatorApplicationPolicyChangeTransaction' => 'PhabricatorApplicationTransactionType', 'PhabricatorApplicationProfileMenuItem' => 'PhabricatorProfileMenuItem', @@ -8276,6 +8282,7 @@ phutil_register_library_map(array( 'PhabricatorEmailPreferencesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailRePrefixSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailSelfActionsSetting' => 'PhabricatorSelectSetting', + 'PhabricatorEmailStampsSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailTagsSetting' => 'PhabricatorInternalSetting', 'PhabricatorEmailVarySubjectsSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailVerificationController' => 'PhabricatorAuthController', @@ -8662,6 +8669,7 @@ phutil_register_library_map(array( 'PhabricatorMailEmailHeraldField' => 'HeraldField', 'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup', 'PhabricatorMailEmailSubjectHeraldField' => 'PhabricatorMailEmailHeraldField', + 'PhabricatorMailEngineExtension' => 'Phobject', 'PhabricatorMailImplementationAdapter' => 'Phobject', 'PhabricatorMailImplementationAmazonSESAdapter' => 'PhabricatorMailImplementationPHPMailerLiteAdapter', 'PhabricatorMailImplementationMailgunAdapter' => 'PhabricatorMailImplementationAdapter', @@ -8690,6 +8698,7 @@ phutil_register_library_map(array( 'PhabricatorMailReplyHandler' => 'Phobject', 'PhabricatorMailRoutingRule' => 'Phobject', 'PhabricatorMailSetupCheck' => 'PhabricatorSetupCheck', + 'PhabricatorMailStamp' => 'Phobject', 'PhabricatorMailTarget' => 'Phobject', 'PhabricatorMailgunConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMainMenuBarExtension' => 'Phobject', @@ -9907,6 +9916,7 @@ phutil_register_library_map(array( 'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType', 'PhabricatorStringListEditField' => 'PhabricatorEditField', 'PhabricatorStringListExportField' => 'PhabricatorListExportField', + 'PhabricatorStringMailStamp' => 'PhabricatorMailStamp', 'PhabricatorStringSetting' => 'PhabricatorSetting', 'PhabricatorSubmitEditField' => 'PhabricatorEditField', 'PhabricatorSubscribedToObjectEdgeType' => 'PhabricatorEdgeType', diff --git a/src/applications/metamta/engine/PhabricatorMailEngineExtension.php b/src/applications/metamta/engine/PhabricatorMailEngineExtension.php new file mode 100644 index 0000000000..36675dda4a --- /dev/null +++ b/src/applications/metamta/engine/PhabricatorMailEngineExtension.php @@ -0,0 +1,47 @@ +getPhobjectClassConstant('EXTENSIONKEY'); + } + + final public function setViewer($viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + final public function setEditor( + PhabricatorApplicationTransactionEditor $editor) { + $this->editor = $editor; + return $this; + } + + final public function getEditor() { + return $this->editor; + } + + abstract public function supportsObject($object); + abstract public function newMailStampTemplates($object); + abstract public function newMailStamps($object, array $xactions); + + final public static function getAllExtensions() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExtensionKey') + ->execute(); + } + + final protected function getMailStamp($key) { + return $this->getEditor()->getMailStamp($key); + } + +} diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index c607087b22..0463b6cdd7 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -58,23 +58,48 @@ final class PhabricatorMailTarget extends Phobject { public function willSendMail(PhabricatorMetaMTAMail $mail) { $viewer = $this->getViewer(); + $show_stamps = $mail->shouldRenderMailStampsInBody($viewer); + + $body = $mail->getBody(); + $html_body = $mail->getHTMLBody(); + $has_html = (strlen($html_body) > 0); + + if ($show_stamps) { + $stamps = $mail->getMailStamps(); + + $body .= "\n"; + $body .= pht('STAMPS'); + $body .= "\n"; + $body .= implode(', ', $stamps); + $body .= "\n"; + + if ($has_html) { + $html = array(); + $html[] = phutil_tag('strong', array(), pht('STAMPS')); + $html[] = phutil_tag('br'); + $html[] = phutil_implode_html(', ', $stamps); + $html[] = phutil_tag('br'); + $html = phutil_tag('div', array(), $html); + $html_body .= hsprintf('%s', $html); + } + } + $mail->addPHIDHeaders('X-Phabricator-To', $this->rawToPHIDs); $mail->addPHIDHeaders('X-Phabricator-Cc', $this->rawCCPHIDs); $to_handles = $viewer->loadHandles($this->rawToPHIDs); $cc_handles = $viewer->loadHandles($this->rawCCPHIDs); - $body = $mail->getBody(); $body .= "\n"; $body .= $this->getRecipientsSummary($to_handles, $cc_handles); - $mail->setBody($body); - $html_body = $mail->getHTMLBody(); - if (strlen($html_body)) { + if ($has_html) { $html_body .= hsprintf( '%s', $this->getRecipientsSummaryHTML($to_handles, $cc_handles)); } + + $mail->setBody($body); $mail->setHTMLBody($html_body); $reply_to = $this->getReplyTo(); diff --git a/src/applications/metamta/stamp/PhabricatorMailStamp.php b/src/applications/metamta/stamp/PhabricatorMailStamp.php new file mode 100644 index 0000000000..9b425a4bdf --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorMailStamp.php @@ -0,0 +1,88 @@ +getPhobjectClassConstant('STAMPTYPE'); + } + + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final protected function setRawValue($value) { + $this->value = $value; + return $this; + } + + final protected function getRawValue() { + return $this->value; + } + + final public function setViewer(PhabricatorUser $viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + final public function setLabel($label) { + $this->label = $label; + return $this; + } + + final public function getLabel() { + return $this->label; + } + + public function setValue($value) { + return $this->setRawValue($value); + } + + final public function toDictionary() { + return array( + 'type' => $this->getStampType(), + 'key' => $this->getKey(), + 'value' => $this->getValueForDictionary(), + ); + } + + final public static function getAllStamps() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getStampType') + ->execute(); + } + + protected function getValueForDictionary() { + return $this->getRawValue(); + } + + public function setValueFromDictionary($value) { + return $this->setRawValue($value); + } + + public function getValueForRendering() { + return $this->getRawValue(); + } + + abstract public function renderStamps($value); + + final protected function renderStamp($key, $value = null) { + return $key.'('.$value.')'; + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php new file mode 100644 index 0000000000..98d472ad48 --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php @@ -0,0 +1,16 @@ +renderStamp($this->getKey(), $value); + } + +} diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a61951650c..a7734b7ae8 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -299,6 +299,22 @@ final class PhabricatorMetaMTAMail return $this->getParam('mustEncryptReasons', array()); } + public function setMailStamps(array $stamps) { + return $this->setParam('stamps', $stamps); + } + + public function getMailStamps() { + return $this->getParam('stamps', array()); + } + + public function setMailStampMetadata($metadata) { + return $this->setParam('stampMetadata', $metadata); + } + + public function getMailStampMetadata() { + return $this->getParam('stampMetadata', array()); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -637,6 +653,11 @@ final class PhabricatorMetaMTAMail } } + $stamps = $this->getMailStamps(); + if ($stamps) { + $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + } + $raw_body = idx($params, 'body', ''); $body = $raw_body; if ($must_encrypt) { @@ -1304,6 +1325,14 @@ final class PhabricatorMetaMTAMail return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL); } + public function shouldRenderMailStampsInBody($viewer) { + $preferences = $this->loadPreferences($viewer->getPHID()); + $value = $preferences->getSettingValue( + PhabricatorEmailStampsSetting::SETTINGKEY); + + return ($value == PhabricatorEmailStampsSetting::VALUE_BODY_STAMPS); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/settings/setting/PhabricatorEmailStampsSetting.php b/src/applications/settings/setting/PhabricatorEmailStampsSetting.php new file mode 100644 index 0000000000..39403f40a0 --- /dev/null +++ b/src/applications/settings/setting/PhabricatorEmailStampsSetting.php @@ -0,0 +1,47 @@ + pht('Mail Headers'), + self::VALUE_BODY_STAMPS => pht('Mail Headers and Body'), + ); + } + +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 0dc6a06fbf..47b975c093 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -72,6 +72,8 @@ abstract class PhabricatorApplicationTransactionEditor private $modularTypes; private $silent; private $mustEncrypt; + private $stampTemplates = array(); + private $mailStamps = array(); private $transactionQueue = array(); @@ -1181,6 +1183,12 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailShouldSend = true; $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + + $mail_xactions = $this->getTransactionsForMail($object, $xactions); + $stamps = $this->newMailStamps($object, $xactions); + foreach ($stamps as $stamp) { + $this->mailStamps[] = $stamp->toDictionary(); + } } if ($this->shouldPublishFeedStory($object, $xactions)) { @@ -2611,6 +2619,7 @@ abstract class PhabricatorApplicationTransactionEditor $mail_tags = $this->getMailTags($object, $mail_xactions); $action = $this->getMailAction($object, $mail_xactions); + $stamps = $this->generateMailStamps($object, $this->mailStamps); if (PhabricatorEnv::getEnvConfig('metamta.email-preferences')) { $this->addEmailPreferenceSectionToMailBody( @@ -2649,6 +2658,18 @@ abstract class PhabricatorApplicationTransactionEditor $mail->setParentMessageID($this->getParentMessageID()); } + // If we have stamps, attach the raw dictionary version (not the actual + // objects) to the mail so that debugging tools can see what we used to + // render the final list. + if ($this->mailStamps) { + $mail->setMailStampMetadata($this->mailStamps); + } + + // If we have rendered stamps, attach them to the mail. + if ($stamps) { + $mail->setMailStamps($stamps); + } + return $target->willSendMail($mail); } @@ -3569,6 +3590,7 @@ abstract class PhabricatorApplicationTransactionEditor 'feedShouldPublish', 'mailShouldSend', 'mustEncrypt', + 'mailStamps', ); } @@ -3961,4 +3983,131 @@ abstract class PhabricatorApplicationTransactionEditor return $editor; } + +/* -( Stamps )------------------------------------------------------------- */ + + + public function newMailStampTemplates($object) { + $actor = $this->getActor(); + + $templates = array(); + + $extensions = $this->newMailExtensions($object); + foreach ($extensions as $extension) { + $stamps = $extension->newMailStampTemplates($object); + foreach ($stamps as $stamp) { + $key = $stamp->getKey(); + if (isset($templates[$key])) { + throw new Exception( + pht( + 'Mail extension ("%s") defines a stamp template with the '. + 'same key ("%s") as another template. Each stamp template '. + 'must have a unique key.', + get_class($extension), + $key)); + } + + $stamp->setViewer($actor); + + $templates[$key] = $stamp; + } + } + + return $templates; + } + + final public function getMailStamp($key) { + if (!isset($this->stampTemplates)) { + throw new PhutilInvalidStateException('newMailStampTemplates'); + } + + if (!isset($this->stampTemplates[$key])) { + throw new Exception( + pht( + 'Editor ("%s") has no mail stamp template with provided key ("%s").', + get_class($this), + $key)); + } + + return $this->stampTemplates[$key]; + } + + private function newMailStamps($object, array $xactions) { + $actor = $this->getActor(); + + $this->stampTemplates = $this->newMailStampTemplates($object); + + $extensions = $this->newMailExtensions($object); + $stamps = array(); + foreach ($extensions as $extension) { + $extension->newMailStamps($object, $xactions); + } + + return $this->stampTemplates; + } + + private function newMailExtensions($object) { + $actor = $this->getActor(); + + $all_extensions = PhabricatorMailEngineExtension::getAllExtensions(); + + $extensions = array(); + foreach ($all_extensions as $key => $template) { + $extension = id(clone $template) + ->setViewer($actor) + ->setEditor($this); + + if ($extension->supportsObject($object)) { + $extensions[$key] = $extension; + } + } + + return $extensions; + } + + private function generateMailStamps($object, $data) { + if (!$data || !is_array($data)) { + return null; + } + + $templates = $this->newMailStampTemplates($object); + foreach ($data as $spec) { + if (!is_array($spec)) { + continue; + } + + $key = idx($spec, 'key'); + if (!isset($templates[$key])) { + continue; + } + + $type = idx($spec, 'type'); + if ($templates[$key]->getStampType() !== $type) { + continue; + } + + $value = idx($spec, 'value'); + $templates[$key]->setValueFromDictionary($value); + } + + $results = array(); + foreach ($templates as $template) { + $value = $template->getValueForRendering(); + + $rendered = $template->renderStamps($value); + if ($rendered === null) { + continue; + } + + $rendered = (array)$rendered; + foreach ($rendered as $stamp) { + $results[] = $stamp; + } + } + + sort($results); + + return $results; + } + } diff --git a/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php new file mode 100644 index 0000000000..bf441df71c --- /dev/null +++ b/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php @@ -0,0 +1,92 @@ +setKey('application') + ->setLabel(pht('Application')), + ); + + if ($this->hasMonogram($object)) { + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('monogram') + ->setLabel(pht('Object Monogram')); + } + + if ($this->hasPHID($object)) { + // This is a PHID, but we always want to render it as a raw string, so + // use a string mail stamp. + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('phid') + ->setLabel(pht('Object PHID')); + + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('object-type') + ->setLabel(pht('Object Type')); + } + + return $templates; + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $application = null; + $class = $editor->getEditorApplicationClass(); + if (PhabricatorApplication::isClassInstalledForViewer($class, $viewer)) { + $application = newv($class, array()); + } + + if ($application) { + $application_name = $application->getName(); + $this->getMailStamp('application') + ->setValue($application_name); + } + + if ($this->hasMonogram($object)) { + $monogram = $object->getMonogram(); + $this->getMailStamp('monogram') + ->setValue($monogram); + } + + if ($this->hasPHID($object)) { + $object_phid = $object->getPHID(); + + $this->getMailStamp('phid') + ->setValue($object_phid); + + $phid_type = phid_get_type($object_phid); + if ($phid_type != PhabricatorPHIDConstants::PHID_TYPE_UNKNOWN) { + $this->getMailStamp('object-type') + ->setValue($phid_type); + } + } + } + + private function hasPHID($object) { + if (!($object instanceof LiskDAO)) { + return false; + } + + if (!$object->getConfigOption(LiskDAO::CONFIG_AUX_PHID)) { + return false; + } + + return true; + } + + private function hasMonogram($object) { + return method_exists($object, 'getMonogram'); + } + +} From 9de54aedb57801fcee9f9da839cf730063f98ca7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:37:39 -0800 Subject: [PATCH 11/67] Remove inconsistent and confusing use of the term "multiplex" in mail Summary: Ref T13053. Because I previously misunderstood what "multiplex" means, I used it in various contradictory and inconsistent ways. We can send mail in two ways: either one mail to everyone with a big "To" and a big "Cc" (not default; better for mailing lists) or one mail to each recipient with just them in "To" (default; better for almost everything else). "Multiplexing" is combining multiple signals over a single channel, so it more accurately describes the big to/cc. However, it is sometimes used to descibe the other approach. Since it's ambiguous and I've tainted it through misuse, get rid of it and use more clear language. (There's still some likely misuse in the SMS stuff, and a couple of legitimate uses in other contexts.) Test Plan: Grepped for `multiplex`, saw less of it. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18994 --- .../config/option/PhabricatorMetaMTAConfigOptions.php | 6 +++--- .../PhabricatorMailImplementationPHPMailerAdapter.php | 4 ++-- .../PhabricatorMailImplementationPHPMailerLiteAdapter.php | 4 ++-- .../metamta/storage/PhabricatorMetaMTAMail.php | 8 ++++---- .../panel/PhabricatorEmailFormatSettingsPanel.php | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 6c846daa57..8c2d2265bc 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -153,9 +153,9 @@ EODOC $adapter_doc_name)); $placeholder_description = $this->deformat(pht(<<mailer->Encoding = $encoding; // By default, PHPMailer sends one mail per recipient. We handle - // multiplexing higher in the stack, so tell it to send mail exactly - // like we ask. + // combining or separating To and Cc higher in the stack, so tell it to + // send mail exactly like we ask. $this->mailer->SingleTo = false; $mailer = PhabricatorEnv::getEnvConfig('phpmailer.mailer'); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index f072e769c3..668f9353ec 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -22,8 +22,8 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter $this->mailer->Encoding = $encoding; // By default, PHPMailerLite sends one mail per recipient. We handle - // multiplexing higher in the stack, so tell it to send mail exactly - // like we ask. + // combining or separating To and Cc higher in the stack, so tell it to + // send mail exactly like we ask. $this->mailer->SingleTo = false; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a7734b7ae8..f19e5e951a 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -507,8 +507,8 @@ final class PhabricatorMetaMTAMail $add_cc = array(); $add_to = array(); - // If multiplexing is enabled, some recipients will be in "Cc" - // rather than "To". We'll move them to "To" later (or supply a + // If we're sending one mail to everyone, some recipients will be in + // "Cc" rather than "To". We'll move them to "To" later (or supply a // dummy "To") but need to look for the recipient in either the // "To" or "Cc" fields here. $target_phid = head(idx($params, 'to', array())); @@ -847,7 +847,7 @@ final class PhabricatorMetaMTAMail return base64_encode($base); } - public static function shouldMultiplexAllMail() { + public static function shouldMailEachRecipient() { return PhabricatorEnv::getEnvConfig('metamta.one-mail-per-recipient'); } @@ -1290,7 +1290,7 @@ final class PhabricatorMetaMTAMail private function loadPreferences($target_phid) { $viewer = PhabricatorUser::getOmnipotentUser(); - if (self::shouldMultiplexAllMail()) { + if (self::shouldMailEachRecipient()) { $preferences = id(new PhabricatorUserPreferencesQuery()) ->setViewer($viewer) ->withUserPHIDs(array($target_phid)) diff --git a/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php b/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php index 107816f2eb..bdc3c994b3 100644 --- a/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php +++ b/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php @@ -14,7 +14,7 @@ final class PhabricatorEmailFormatSettingsPanel } public function isUserPanel() { - return PhabricatorMetaMTAMail::shouldMultiplexAllMail(); + return PhabricatorMetaMTAMail::shouldMailEachRecipient(); } public function isManagementPanel() { From 3131e733a85cb8be5f6119ca8d4d7a0b23f29009 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:31:56 -0800 Subject: [PATCH 12/67] Add Editor-based mail stamps: actor, via, silent, encrypted, new, mention, self-actor, self-mention Summary: Ref T13053. Adds more mail tags with information available on the Editor object. Test Plan: Banged around in Maniphest, viewed the resulting mail, all the stamps seemed to align with reality. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18995 --- src/__phutil_library_map__.php | 8 ++ .../stamp/PhabricatorBoolMailStamp.php | 16 ++++ .../stamp/PhabricatorPHIDMailStamp.php | 36 +++++++++ .../stamp/PhabricatorViewerMailStamp.php | 35 ++++++++ .../phid/PhabricatorPeopleUserPHIDType.php | 11 ++- .../phid/PhabricatorObjectHandle.php | 10 +++ .../PhabricatorProjectProjectPHIDType.php | 3 +- ...habricatorApplicationTransactionEditor.php | 6 +- .../PhabricatorEditorMailEngineExtension.php | 81 +++++++++++++++++++ 9 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 src/applications/metamta/stamp/PhabricatorBoolMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorViewerMailStamp.php create mode 100644 src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 6fa14fbacb..2a84849171 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2220,6 +2220,7 @@ phutil_register_library_map(array( 'PhabricatorBoardResponseEngine' => 'applications/project/engine/PhabricatorBoardResponseEngine.php', 'PhabricatorBoolConfigType' => 'applications/config/type/PhabricatorBoolConfigType.php', 'PhabricatorBoolEditField' => 'applications/transactions/editfield/PhabricatorBoolEditField.php', + 'PhabricatorBoolMailStamp' => 'applications/metamta/stamp/PhabricatorBoolMailStamp.php', 'PhabricatorBritishEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorBritishEnglishTranslation.php', 'PhabricatorBuiltinDraftEngine' => 'applications/transactions/draft/PhabricatorBuiltinDraftEngine.php', 'PhabricatorBuiltinFileCachePurger' => 'applications/cache/purger/PhabricatorBuiltinFileCachePurger.php', @@ -2813,6 +2814,7 @@ phutil_register_library_map(array( 'PhabricatorEditPage' => 'applications/transactions/editengine/PhabricatorEditPage.php', 'PhabricatorEditType' => 'applications/transactions/edittype/PhabricatorEditType.php', 'PhabricatorEditor' => 'infrastructure/PhabricatorEditor.php', + 'PhabricatorEditorMailEngineExtension' => 'applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php', 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', @@ -3440,6 +3442,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', 'PhabricatorPHIDListExportField' => 'infrastructure/export/field/PhabricatorPHIDListExportField.php', + 'PhabricatorPHIDMailStamp' => 'applications/metamta/stamp/PhabricatorPHIDMailStamp.php', 'PhabricatorPHIDResolver' => 'applications/phid/resolver/PhabricatorPHIDResolver.php', 'PhabricatorPHIDType' => 'applications/phid/type/PhabricatorPHIDType.php', 'PhabricatorPHIDTypeTestCase' => 'applications/phid/type/__tests__/PhabricatorPHIDTypeTestCase.php', @@ -4395,6 +4398,7 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'applications/draft/storage/PhabricatorVersionedDraft.php', 'PhabricatorVeryWowEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorVeryWowEnglishTranslation.php', 'PhabricatorViewerDatasource' => 'applications/people/typeahead/PhabricatorViewerDatasource.php', + 'PhabricatorViewerMailStamp' => 'applications/metamta/stamp/PhabricatorViewerMailStamp.php', 'PhabricatorWatcherHasObjectEdgeType' => 'applications/transactions/edges/PhabricatorWatcherHasObjectEdgeType.php', 'PhabricatorWebContentSource' => 'infrastructure/contentsource/PhabricatorWebContentSource.php', 'PhabricatorWebServerSetupCheck' => 'applications/config/check/PhabricatorWebServerSetupCheck.php', @@ -7582,6 +7586,7 @@ phutil_register_library_map(array( 'PhabricatorBoardResponseEngine' => 'Phobject', 'PhabricatorBoolConfigType' => 'PhabricatorTextConfigType', 'PhabricatorBoolEditField' => 'PhabricatorEditField', + 'PhabricatorBoolMailStamp' => 'PhabricatorMailStamp', 'PhabricatorBritishEnglishTranslation' => 'PhutilTranslation', 'PhabricatorBuiltinDraftEngine' => 'PhabricatorDraftEngine', 'PhabricatorBuiltinFileCachePurger' => 'PhabricatorCachePurger', @@ -8267,6 +8272,7 @@ phutil_register_library_map(array( 'PhabricatorEditPage' => 'Phobject', 'PhabricatorEditType' => 'Phobject', 'PhabricatorEditor' => 'Phobject', + 'PhabricatorEditorMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', @@ -8973,6 +8979,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDListEditField' => 'PhabricatorEditField', 'PhabricatorPHIDListEditType' => 'PhabricatorEditType', 'PhabricatorPHIDListExportField' => 'PhabricatorListExportField', + 'PhabricatorPHIDMailStamp' => 'PhabricatorMailStamp', 'PhabricatorPHIDResolver' => 'Phobject', 'PhabricatorPHIDType' => 'Phobject', 'PhabricatorPHIDTypeTestCase' => 'PhutilTestCase', @@ -10137,6 +10144,7 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'PhabricatorDraftDAO', 'PhabricatorVeryWowEnglishTranslation' => 'PhutilTranslation', 'PhabricatorViewerDatasource' => 'PhabricatorTypeaheadDatasource', + 'PhabricatorViewerMailStamp' => 'PhabricatorMailStamp', 'PhabricatorWatcherHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorWebContentSource' => 'PhabricatorContentSource', 'PhabricatorWebServerSetupCheck' => 'PhabricatorSetupCheck', diff --git a/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php b/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php new file mode 100644 index 0000000000..d274df67fe --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php @@ -0,0 +1,16 @@ +renderStamp($this->getKey()); + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php b/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php new file mode 100644 index 0000000000..575ad16f6a --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php @@ -0,0 +1,36 @@ +getViewer(); + $handles = $viewer->loadHandles($value); + + $results = array(); + foreach ($value as $phid) { + $handle = $handles[$phid]; + + $mail_name = $handle->getMailStampName(); + if ($mail_name === null) { + $mail_name = $handle->getPHID(); + } + + $results[] = $this->renderStamp($this->getKey(), $mail_name); + } + + return $results; + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php new file mode 100644 index 0000000000..4eb2028ed8 --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php @@ -0,0 +1,35 @@ +getViewer()->getPHID(); + if (!$viewer_phid) { + return null; + } + + if (!$value) { + return null; + } + + $value = (array)$value; + $value = array_fuse($value); + + if (!isset($value[$viewer_phid])) { + return null; + } + + return $this->renderStamp($this->getKey()); + } + +} diff --git a/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php b/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php index f0512e91f1..7867f098f1 100644 --- a/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php +++ b/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php @@ -39,11 +39,14 @@ final class PhabricatorPeopleUserPHIDType extends PhabricatorPHIDType { foreach ($handles as $phid => $handle) { $user = $objects[$phid]; $realname = $user->getRealName(); + $username = $user->getUsername(); - $handle->setName($user->getUsername()); - $handle->setURI('/p/'.$user->getUsername().'/'); - $handle->setFullName($user->getFullName()); - $handle->setImageURI($user->getProfileImageURI()); + $handle + ->setName($username) + ->setURI('/p/'.$username.'/') + ->setFullName($user->getFullName()) + ->setImageURI($user->getProfileImageURI()) + ->setMailStampName('@'.$username); if ($user->getIsMailingList()) { $handle->setIcon('fa-envelope-o'); diff --git a/src/applications/phid/PhabricatorObjectHandle.php b/src/applications/phid/PhabricatorObjectHandle.php index 1e6812b53b..ba93dbcead 100644 --- a/src/applications/phid/PhabricatorObjectHandle.php +++ b/src/applications/phid/PhabricatorObjectHandle.php @@ -31,6 +31,7 @@ final class PhabricatorObjectHandle private $subtitle; private $tokenIcon; private $commandLineObjectName; + private $mailStampName; private $stateIcon; private $stateColor; @@ -134,6 +135,15 @@ final class PhabricatorObjectHandle return $this->objectName; } + public function setMailStampName($mail_stamp_name) { + $this->mailStampName = $mail_stamp_name; + return $this; + } + + public function getMailStampName() { + return $this->mailStampName; + } + public function setURI($uri) { $this->uri = $uri; return $this; diff --git a/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php b/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php index 3aa6088780..9247966d75 100644 --- a/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php +++ b/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php @@ -45,11 +45,12 @@ final class PhabricatorProjectProjectPHIDType extends PhabricatorPHIDType { if (strlen($slug)) { $handle->setObjectName('#'.$slug); + $handle->setMailStampName('#'.$slug); $handle->setURI("/tag/{$slug}/"); } else { // We set the name to the project's PHID to avoid a parse error when a // project has no hashtag (as is the case with milestones by default). - // See T12659 for more details + // See T12659 for more details. $handle->setCommandLineObjectName($project->getPHID()); $handle->setURI("/project/view/{$id}/"); } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 47b975c093..f82cd91701 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -179,7 +179,7 @@ abstract class PhabricatorApplicationTransactionEditor return $this->isNewObject; } - protected function getMentionedPHIDs() { + public function getMentionedPHIDs() { return $this->mentionedPHIDs; } @@ -201,6 +201,10 @@ abstract class PhabricatorApplicationTransactionEditor return $this->silent; } + public function getMustEncrypt() { + return $this->mustEncrypt; + } + public function setIsInverseEdgeEditor($is_inverse_edge_editor) { $this->isInverseEdgeEditor = $is_inverse_edge_editor; return $this; diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php new file mode 100644 index 0000000000..acfc7ec833 --- /dev/null +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -0,0 +1,81 @@ +setKey('actor') + ->setLabel(pht('Acting User')); + + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('via') + ->setLabel(pht('Via Content Source')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('silent') + ->setLabel(pht('Silent Edit')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('encrypted') + ->setLabel(pht('Encryption Required')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('new') + ->setLabel(pht('New Object')); + + $templates[] = id(new PhabricatorPHIDMailStamp()) + ->setKey('mention') + ->setLabel(pht('Mentioned User')); + + $templates[] = id(new PhabricatorViewerMailStamp()) + ->setKey('self-actor') + ->setLabel(pht('You Acted')); + + $templates[] = id(new PhabricatorViewerMailStamp()) + ->setKey('self-mention') + ->setLabel(pht('You Were Mentioned')); + + return $templates; + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $this->getMailStamp('actor') + ->setValue($editor->getActingAsPHID()); + + $content_source = $editor->getContentSource(); + $this->getMailStamp('via') + ->setValue($content_source->getSourceTypeConstant()); + + $this->getMailStamp('silent') + ->setValue($editor->getIsSilent()); + + $this->getMailStamp('encrypted') + ->setValue($editor->getMustEncrypt()); + + $this->getMailStamp('new') + ->setValue($editor->getIsNewObject()); + + $mentioned_phids = $editor->getMentionedPHIDs(); + $this->getMailStamp('mention') + ->setValue($mentioned_phids); + + $this->getMailStamp('self-actor') + ->setValue($editor->getActingAsPHID()); + + $this->getMailStamp('self-mention') + ->setValue($mentioned_phids); + } + +} From 7d475eb09af74b42da52ca258d2fbf130fca6a3e Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 11:09:36 -0800 Subject: [PATCH 13/67] Add more mail stamps: tasks, subscribers, projects, spaces Summary: Ref T13053. Adds task stamps plus the major infrastructure applications. Test Plan: {F5413058} Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18996 --- src/__phutil_library_map__.php | 8 +++ .../ManiphestMailEngineExtension.php | 58 +++++++++++++++++++ .../phid/PhabricatorOwnersPackagePHIDType.php | 1 + ...PhabricatorProjectsMailEngineExtension.php | 32 ++++++++++ .../PhabricatorSpacesMailEngineExtension.php | 35 +++++++++++ .../PhabricatorSpacesNamespacePHIDType.php | 8 ++- ...icatorSubscriptionsMailEngineExtension.php | 32 ++++++++++ 7 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php create mode 100644 src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php create mode 100644 src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php create mode 100644 src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 2a84849171..4742ce410e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1528,6 +1528,7 @@ phutil_register_library_map(array( 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestGetTaskTransactionsConduitAPIMethod.php', 'ManiphestHovercardEngineExtension' => 'applications/maniphest/engineextension/ManiphestHovercardEngineExtension.php', 'ManiphestInfoConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestInfoConduitAPIMethod.php', + 'ManiphestMailEngineExtension' => 'applications/maniphest/engineextension/ManiphestMailEngineExtension.php', 'ManiphestNameIndex' => 'applications/maniphest/storage/ManiphestNameIndex.php', 'ManiphestPointsConfigType' => 'applications/maniphest/config/ManiphestPointsConfigType.php', 'ManiphestPrioritiesConfigType' => 'applications/maniphest/config/ManiphestPrioritiesConfigType.php', @@ -3853,6 +3854,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsEditField' => 'applications/transactions/editfield/PhabricatorProjectsEditField.php', 'PhabricatorProjectsExportEngineExtension' => 'infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php', 'PhabricatorProjectsFulltextEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsFulltextEngineExtension.php', + 'PhabricatorProjectsMailEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'applications/project/engineextension/PhabricatorProjectsMembersSearchEngineAttachment.php', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsMembershipIndexEngineExtension.php', 'PhabricatorProjectsPolicyRule' => 'applications/project/policyrule/PhabricatorProjectsPolicyRule.php', @@ -4146,6 +4148,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesExportEngineExtension' => 'infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php', 'PhabricatorSpacesInterface' => 'applications/spaces/interface/PhabricatorSpacesInterface.php', 'PhabricatorSpacesListController' => 'applications/spaces/controller/PhabricatorSpacesListController.php', + 'PhabricatorSpacesMailEngineExtension' => 'applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php', 'PhabricatorSpacesNamespace' => 'applications/spaces/storage/PhabricatorSpacesNamespace.php', 'PhabricatorSpacesNamespaceArchiveTransaction' => 'applications/spaces/xaction/PhabricatorSpacesNamespaceArchiveTransaction.php', 'PhabricatorSpacesNamespaceDatasource' => 'applications/spaces/typeahead/PhabricatorSpacesNamespaceDatasource.php', @@ -4230,6 +4233,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsFulltextEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsFulltextEngineExtension.php', 'PhabricatorSubscriptionsHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsHeraldAction.php', 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', + 'PhabricatorSubscriptionsMailEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSelfHeraldAction.php', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSubscribersHeraldAction.php', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php', @@ -6795,6 +6799,7 @@ phutil_register_library_map(array( 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'ManiphestConduitAPIMethod', 'ManiphestHovercardEngineExtension' => 'PhabricatorHovercardEngineExtension', 'ManiphestInfoConduitAPIMethod' => 'ManiphestConduitAPIMethod', + 'ManiphestMailEngineExtension' => 'PhabricatorMailEngineExtension', 'ManiphestNameIndex' => 'ManiphestDAO', 'ManiphestPointsConfigType' => 'PhabricatorJSONConfigType', 'ManiphestPrioritiesConfigType' => 'PhabricatorJSONConfigType', @@ -9482,6 +9487,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsEditField' => 'PhabricatorTokenizerEditField', 'PhabricatorProjectsExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorProjectsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', + 'PhabricatorProjectsMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'PhabricatorIndexEngineExtension', 'PhabricatorProjectsPolicyRule' => 'PhabricatorPolicyRule', @@ -9851,6 +9857,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorSpacesInterface' => 'PhabricatorPHIDInterface', 'PhabricatorSpacesListController' => 'PhabricatorSpacesController', + 'PhabricatorSpacesMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorSpacesNamespace' => array( 'PhabricatorSpacesDAO', 'PhabricatorPolicyInterface', @@ -9941,6 +9948,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorSubscriptionsHeraldAction' => 'HeraldAction', 'PhabricatorSubscriptionsListController' => 'PhabricatorController', + 'PhabricatorSubscriptionsMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', diff --git a/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php b/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php new file mode 100644 index 0000000000..ee38bdf604 --- /dev/null +++ b/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php @@ -0,0 +1,58 @@ +setKey('author') + ->setLabel(pht('Author')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('task-owner') + ->setLabel(pht('Task Owner')), + id(new PhabricatorBoolMailStamp()) + ->setKey('task-unassigned') + ->setLabel(pht('Task Unassigned')), + id(new PhabricatorStringMailStamp()) + ->setKey('task-priority') + ->setLabel(pht('Task Priority')), + id(new PhabricatorStringMailStamp()) + ->setKey('task-status') + ->setLabel(pht('Task Status')), + id(new PhabricatorStringMailStamp()) + ->setKey('subtype') + ->setLabel(pht('Subtype')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $this->getMailStamp('author') + ->setValue($object->getAuthorPHID()); + + $this->getMailStamp('task-owner') + ->setValue($object->getOwnerPHID()); + + $this->getMailStamp('task-unassigned') + ->setValue(!$object->getOwnerPHID()); + + $this->getMailStamp('task-priority') + ->setValue($object->getPriority()); + + $this->getMailStamp('task-status') + ->setValue($object->getStatus()); + + $this->getMailStamp('subtype') + ->setValue($object->getSubtype()); + } + +} diff --git a/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php b/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php index cfbaf6eeb2..fbff6a2103 100644 --- a/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php +++ b/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php @@ -45,6 +45,7 @@ final class PhabricatorOwnersPackagePHIDType extends PhabricatorPHIDType { ->setName($monogram) ->setFullName("{$monogram}: {$name}") ->setCommandLineObjectName("{$monogram} {$name}") + ->setMailStampName($monogram) ->setURI($uri); if ($package->isArchived()) { diff --git a/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php b/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php new file mode 100644 index 0000000000..6f92f87b11 --- /dev/null +++ b/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php @@ -0,0 +1,32 @@ +setKey('tag') + ->setLabel(pht('Tagged with Project')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $project_phids = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorProjectObjectHasProjectEdgeType::EDGECONST); + + $this->getMailStamp('tag') + ->setValue($project_phids); + } + +} diff --git a/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php b/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php new file mode 100644 index 0000000000..7ddbda05fe --- /dev/null +++ b/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php @@ -0,0 +1,35 @@ +setKey('space') + ->setLabel(pht('Space')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + if (!PhabricatorSpacesNamespaceQuery::getSpacesExist()) { + return; + } + + $space_phid = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $object); + + $this->getMailStamp('space') + ->setValue($space_phid); + } + +} diff --git a/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php b/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php index 86371d6420..1399e71c8e 100644 --- a/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php +++ b/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php @@ -36,9 +36,11 @@ final class PhabricatorSpacesNamespacePHIDType $monogram = $namespace->getMonogram(); $name = $namespace->getNamespaceName(); - $handle->setName($name); - $handle->setFullName(pht('%s %s', $monogram, $name)); - $handle->setURI('/'.$monogram); + $handle + ->setName($name) + ->setFullName(pht('%s %s', $monogram, $name)) + ->setURI('/'.$monogram) + ->setMailStampName($monogram); if ($namespace->getIsArchived()) { $handle->setStatus(PhabricatorObjectHandle::STATUS_CLOSED); diff --git a/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php new file mode 100644 index 0000000000..122fad4b0d --- /dev/null +++ b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php @@ -0,0 +1,32 @@ +setKey('subscriber') + ->setLabel(pht('Subscriber')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $subscriber_phids = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorObjectHasSubscriberEdgeType::EDGECONST); + + $this->getMailStamp('subscriber') + ->setValue($subscriber_phids); + } + +} From 1bf64e5cbcf48adf05ef60cb9eb948d4056092b2 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 11:44:31 -0800 Subject: [PATCH 14/67] Add Differential and Herald mail stamps and some refinements Summary: Ref T13053. Adds revision stamps (status, reviewers, etc). Adds Herald rule stamps, like the existing X-Herald-Rules header. Removes the "self" stamps, since you can just write a rule against `whatever(@epriestley)` equivalently. If there's routing logic around this, it can live in the routing layer. This avoids tons of self-actor, self-mention, self-reviewer, self-blocking-reviewer, self-resigned-reviewer, etc., stamps. Use `natcasesort()` instead of `sort()` so that numeric values (like monograms) sort `9, 80, 700` instead of `700, 80, 9`. Remove the commas from rendering since they don't really add anything. Test Plan: Edited tasks and revisions, looked at mail stamps, saw stamps that looked pretty reasonable (with no more self stuff, no more commas, sorting numbers, and Herald stamps). Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18997 --- src/__phutil_library_map__.php | 4 +- .../DifferentialMailEngineExtension.php | 80 +++++++++++++++++++ .../storage/DifferentialReviewer.php | 5 ++ .../replyhandler/PhabricatorMailTarget.php | 4 +- .../stamp/PhabricatorStringMailStamp.php | 14 +++- .../stamp/PhabricatorViewerMailStamp.php | 35 -------- ...habricatorRepositoryRepositoryPHIDType.php | 8 +- ...habricatorApplicationTransactionEditor.php | 21 ++++- .../PhabricatorEditorMailEngineExtension.php | 17 ++-- 9 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 src/applications/differential/engineextension/DifferentialMailEngineExtension.php delete mode 100644 src/applications/metamta/stamp/PhabricatorViewerMailStamp.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 4742ce410e..c642f13f90 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -487,6 +487,7 @@ phutil_register_library_map(array( 'DifferentialLintField' => 'applications/differential/customfield/DifferentialLintField.php', 'DifferentialLintStatus' => 'applications/differential/constants/DifferentialLintStatus.php', 'DifferentialLocalCommitsView' => 'applications/differential/view/DifferentialLocalCommitsView.php', + 'DifferentialMailEngineExtension' => 'applications/differential/engineextension/DifferentialMailEngineExtension.php', 'DifferentialMailView' => 'applications/differential/mail/DifferentialMailView.php', 'DifferentialManiphestTasksField' => 'applications/differential/customfield/DifferentialManiphestTasksField.php', 'DifferentialModernHunk' => 'applications/differential/storage/DifferentialModernHunk.php', @@ -4402,7 +4403,6 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'applications/draft/storage/PhabricatorVersionedDraft.php', 'PhabricatorVeryWowEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorVeryWowEnglishTranslation.php', 'PhabricatorViewerDatasource' => 'applications/people/typeahead/PhabricatorViewerDatasource.php', - 'PhabricatorViewerMailStamp' => 'applications/metamta/stamp/PhabricatorViewerMailStamp.php', 'PhabricatorWatcherHasObjectEdgeType' => 'applications/transactions/edges/PhabricatorWatcherHasObjectEdgeType.php', 'PhabricatorWebContentSource' => 'infrastructure/contentsource/PhabricatorWebContentSource.php', 'PhabricatorWebServerSetupCheck' => 'applications/config/check/PhabricatorWebServerSetupCheck.php', @@ -5612,6 +5612,7 @@ phutil_register_library_map(array( 'DifferentialLintField' => 'DifferentialHarbormasterField', 'DifferentialLintStatus' => 'Phobject', 'DifferentialLocalCommitsView' => 'AphrontView', + 'DifferentialMailEngineExtension' => 'PhabricatorMailEngineExtension', 'DifferentialMailView' => 'Phobject', 'DifferentialManiphestTasksField' => 'DifferentialCoreCustomField', 'DifferentialModernHunk' => 'DifferentialHunk', @@ -10152,7 +10153,6 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'PhabricatorDraftDAO', 'PhabricatorVeryWowEnglishTranslation' => 'PhutilTranslation', 'PhabricatorViewerDatasource' => 'PhabricatorTypeaheadDatasource', - 'PhabricatorViewerMailStamp' => 'PhabricatorMailStamp', 'PhabricatorWatcherHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorWebContentSource' => 'PhabricatorContentSource', 'PhabricatorWebServerSetupCheck' => 'PhabricatorSetupCheck', diff --git a/src/applications/differential/engineextension/DifferentialMailEngineExtension.php b/src/applications/differential/engineextension/DifferentialMailEngineExtension.php new file mode 100644 index 0000000000..24aa9fb329 --- /dev/null +++ b/src/applications/differential/engineextension/DifferentialMailEngineExtension.php @@ -0,0 +1,80 @@ +setKey('author') + ->setLabel(pht('Author')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('blocking-reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('resigned-reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('revision-repository') + ->setLabel(pht('Revision Repository')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('revision-status') + ->setLabel(pht('Revision Status')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $revision = id(new DifferentialRevisionQuery()) + ->setViewer($viewer) + ->needReviewers(true) + ->withPHIDs(array($object->getPHID())) + ->executeOne(); + + $reviewers = array(); + $blocking = array(); + $resigned = array(); + foreach ($revision->getReviewers() as $reviewer) { + $reviewer_phid = $reviewer->getReviewerPHID(); + + if ($reviewer->isResigned()) { + $resigned[] = $reviewer_phid; + } else { + $reviewers[] = $reviewer_phid; + if ($reviewer->isBlocking()) { + $reviewers[] = $blocking; + } + } + } + + $this->getMailStamp('author') + ->setValue($revision->getAuthorPHID()); + + $this->getMailStamp('reviewer') + ->setValue($reviewers); + + $this->getMailStamp('blocking-reviewer') + ->setValue($blocking); + + $this->getMailStamp('resigned-reviewer') + ->setValue($resigned); + + $this->getMailStamp('revision-repository') + ->setValue($revision->getRepositoryPHID()); + + $this->getMailStamp('revision-status') + ->setValue($revision->getModernRevisionStatus()); + } + +} diff --git a/src/applications/differential/storage/DifferentialReviewer.php b/src/applications/differential/storage/DifferentialReviewer.php index 9df149e788..e3f9bdaf8d 100644 --- a/src/applications/differential/storage/DifferentialReviewer.php +++ b/src/applications/differential/storage/DifferentialReviewer.php @@ -69,6 +69,11 @@ final class DifferentialReviewer return ($this->getReviewerStatus() == $status_resigned); } + public function isBlocking() { + $status_blocking = DifferentialReviewerStatus::STATUS_BLOCKING; + return ($this->getReviewerStatus() == $status_blocking); + } + public function isRejected($diff_phid) { $status_rejected = DifferentialReviewerStatus::STATUS_REJECTED; diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index 0463b6cdd7..bbf17be3fd 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -70,14 +70,14 @@ final class PhabricatorMailTarget extends Phobject { $body .= "\n"; $body .= pht('STAMPS'); $body .= "\n"; - $body .= implode(', ', $stamps); + $body .= implode(' ', $stamps); $body .= "\n"; if ($has_html) { $html = array(); $html[] = phutil_tag('strong', array(), pht('STAMPS')); $html[] = phutil_tag('br'); - $html[] = phutil_implode_html(', ', $stamps); + $html[] = phutil_implode_html(' ', $stamps); $html[] = phutil_tag('br'); $html = phutil_tag('div', array(), $html); $html_body .= hsprintf('%s', $html); diff --git a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php index 98d472ad48..b6210afb4e 100644 --- a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php +++ b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php @@ -6,11 +6,21 @@ final class PhabricatorStringMailStamp const STAMPTYPE = 'string'; public function renderStamps($value) { - if (!strlen($value)) { + if ($value === null || $value === '') { return null; } - return $this->renderStamp($this->getKey(), $value); + $value = (array)$value; + if (!$value) { + return null; + } + + $results = array(); + foreach ($value as $v) { + $results[] = $this->renderStamp($this->getKey(), $v); + } + + return $results; } } diff --git a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php deleted file mode 100644 index 4eb2028ed8..0000000000 --- a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php +++ /dev/null @@ -1,35 +0,0 @@ -getViewer()->getPHID(); - if (!$viewer_phid) { - return null; - } - - if (!$value) { - return null; - } - - $value = (array)$value; - $value = array_fuse($value); - - if (!isset($value[$viewer_phid])) { - return null; - } - - return $this->renderStamp($this->getKey()); - } - -} diff --git a/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php b/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php index c938b31a65..ba78b0fe7a 100644 --- a/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php +++ b/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php @@ -41,9 +41,11 @@ final class PhabricatorRepositoryRepositoryPHIDType $name = $repository->getName(); $uri = $repository->getURI(); - $handle->setName($monogram); - $handle->setFullName("{$monogram} {$name}"); - $handle->setURI($uri); + $handle + ->setName($monogram) + ->setFullName("{$monogram} {$name}") + ->setURI($uri) + ->setMailStampName($monogram); } } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index f82cd91701..ac4d56ba16 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -205,6 +205,25 @@ abstract class PhabricatorApplicationTransactionEditor return $this->mustEncrypt; } + public function getHeraldRuleMonograms() { + // Convert the stored "<123>, <456>" string into a list: "H123", "H456". + $list = $this->heraldHeader; + $list = preg_split('/[, ]+/', $list); + + foreach ($list as $key => $item) { + $item = trim($item, '<>'); + + if (!is_numeric($item)) { + unset($list[$key]); + continue; + } + + $list[$key] = 'H'.$item; + } + + return $list; + } + public function setIsInverseEdgeEditor($is_inverse_edge_editor) { $this->isInverseEdgeEditor = $is_inverse_edge_editor; return $this; @@ -4109,7 +4128,7 @@ abstract class PhabricatorApplicationTransactionEditor } } - sort($results); + natcasesort($results); return $results; } diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php index acfc7ec833..29d10d641f 100644 --- a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -36,13 +36,9 @@ final class PhabricatorEditorMailEngineExtension ->setKey('mention') ->setLabel(pht('Mentioned User')); - $templates[] = id(new PhabricatorViewerMailStamp()) - ->setKey('self-actor') - ->setLabel(pht('You Acted')); - - $templates[] = id(new PhabricatorViewerMailStamp()) - ->setKey('self-mention') - ->setLabel(pht('You Were Mentioned')); + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('herald') + ->setLabel(pht('Herald Rule')); return $templates; } @@ -71,11 +67,8 @@ final class PhabricatorEditorMailEngineExtension $this->getMailStamp('mention') ->setValue($mentioned_phids); - $this->getMailStamp('self-actor') - ->setValue($editor->getActingAsPHID()); - - $this->getMailStamp('self-mention') - ->setValue($mentioned_phids); + $this->getMailStamp('herald') + ->setValue($editor->getHeraldRuleMonograms()); } } From 56bf0690804fe4c77085b1320df9a5e994d3163c Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 04:07:22 -0800 Subject: [PATCH 15/67] Try running Herald when performing inverse edge edits Summary: Ref T13053. When you mention one object on another (or link two objects together with an action like "Edit Parent Revisions"), we write a transaction on each side to add the "alice added subtask X" and "alice added parent task Y" items to the timeline. This behavior now causes problems in T13053 with the "Must Encrypt" flag because it prevents the flag from being applied to the corresponding "inverse edge" mail. This was added in rP5050389f as a quick workaround for a fatal related to Editors not having enough data to apply Herald on mentions. However, that was in 2014, and since then: - Herald got a significant rewrite to modularize all the rules and adapters. - Editing got a significant upgrade in EditEngine and most edit workflows now operate through EditEngine. - We generally do more editing on more pathways, everything is more modular, and we have standardized how data is loaded to a greater degree. I suspect there's no longer a problem with just running Herald here, and can't reproduce one. If anything does crop up, it's probably easy (and desirable) to fix it. This makes Herald fire a little more often: if someone writes a rule, mentioning or creating a relationship to old tasks will now make the rule act. Offhand, that seems fine. If it turns out to be weird, we can probably tailor Herald's behavior. Test Plan: I wasn't able to break anything: - Mentioned a task on another task (original issue). - Linked tasks with commits, mocks, revisions. - Linked revisions with commits, tasks. - Mentioned users, revisions, and commits. - Verified that mail generated by creating links (e.g., Revision > Edit Tasks) now gets the "Must Encrypt" flag properly. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18999 --- .../editor/PhabricatorApplicationTransactionEditor.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index ac4d56ba16..c5390de362 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -1095,13 +1095,6 @@ abstract class PhabricatorApplicationTransactionEditor // We are the Herald editor, so stop work here and return the updated // transactions. return $xactions; - } else if ($this->getIsInverseEdgeEditor()) { - // If we're applying inverse edge transactions, don't trigger Herald. - // From a product perspective, the current set of inverse edges (most - // often, mentions) aren't things users would expect to trigger Herald. - // From a technical perspective, objects loaded by the inverse editor may - // not have enough data to execute rules. At least for now, just stop - // Herald from executing when applying inverse edges. } else if ($this->shouldApplyHeraldRules($object, $xactions)) { // We are not the Herald editor, so try to apply Herald rules. $herald_xactions = $this->applyHeraldRules($object, $xactions); From a5bbadbaba0259350ccc6f6a86786373e935aee5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 16:00:45 -0800 Subject: [PATCH 16/67] Fix another Git 2.16.0 CLI compatibility issue Summary: This command also needs a "." instead of an empty string now. (This powers the file browser typeahead in Diffusion.) Test Plan: Will test in production since there's still no easy 2.16 installer for macOS. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D19010 --- .../conduit/DiffusionQueryPathsConduitAPIMethod.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php index be2f07f2c6..09c07ec28f 100644 --- a/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php @@ -37,7 +37,11 @@ final class DiffusionQueryPathsConduitAPIMethod $commit = $request->getValue('commit'); $repository = $drequest->getRepository(); - // http://comments.gmane.org/gmane.comp.version-control.git/197735 + // Recent versions of Git don't work if you pass the empty string, and + // require "." to list everything. + if (!strlen($path)) { + $path = '.'; + } $future = $repository->getLocalCommandFuture( 'ls-tree --name-only -r -z %s -- %s', From 150a04791c06c91ab587fe11ea2a50ba8b40cd45 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 05:43:37 -0800 Subject: [PATCH 17/67] Fix bad NUX link in Legalpad search view Summary: See . This URI isn't correct. Test Plan: Visited {nav Use Results > New User State} in developer mode, clicked green button. Before: 404. After: taken to the edit screen. Differential Revision: https://secure.phabricator.com/D19024 --- .../legalpad/query/LegalpadDocumentSearchEngine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php b/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php index 1b608b02e3..a245417483 100644 --- a/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php +++ b/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php @@ -176,7 +176,7 @@ final class LegalpadDocumentSearchEngine $create_button = id(new PHUIButtonView()) ->setTag('a') ->setText(pht('Create a Document')) - ->setHref('/legalpad/create/') + ->setHref('/legalpad/edit/') ->setColor(PHUIButtonView::GREEN); $icon = $this->getApplication()->getIcon(); From 1485debcbda2bae25884dbd54055ccb3fecd9580 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 14:22:54 -0800 Subject: [PATCH 18/67] Prepare mail transmission to support failover across multiple mailers Summary: Ref T13053. Ref T12677. This restructures the calls and error handling logic so that we can pass in a list of multiple mailers and get retry logic. This doesn't actually ever use multiple mailers yet, and shouldn't change any behavior. I'll add multiple-mailer coverage a little further in, since there's currently no way to effectively test which of several mailers ended up transmitting a message. Test Plan: - This has test coverage; tests still pass. - Poked around locally doing things that send mail, saw mail appear to send. I'm not attached to a real mailer though so my confidence in local testing is only so-so. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D18998 --- .../storage/PhabricatorMetaMTAMail.php | 693 +++++++++--------- .../PhabricatorMetaMTAMailTestCase.php | 8 +- 2 files changed, 367 insertions(+), 334 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f19e5e951a..179eb18088 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -461,359 +461,392 @@ final class PhabricatorMetaMTAMail /** * Attempt to deliver an email immediately, in this process. * - * @param bool Try to deliver this email even if it has already been - * delivered or is in backoff after a failed delivery attempt. - * @param PhabricatorMailImplementationAdapter Use a specific mail adapter, - * instead of the default. - * * @return void */ - public function sendNow( - $force_send = false, - PhabricatorMailImplementationAdapter $mailer = null) { - - if ($mailer === null) { - $mailer = $this->buildDefaultMailer(); + public function sendNow() { + if ($this->getStatus() != PhabricatorMailOutboundStatus::STATUS_QUEUE) { + throw new Exception(pht('Trying to send an already-sent mail!')); } - if (!$force_send) { - if ($this->getStatus() != PhabricatorMailOutboundStatus::STATUS_QUEUE) { - throw new Exception(pht('Trying to send an already-sent mail!')); - } - } + $mailers = array( + $this->buildDefaultMailer(), + ); - try { - $headers = $this->generateHeaders(); + return $this->sendWithMailers($mailers); + } - $params = $this->parameters; - $actors = $this->loadAllActors(); - $deliverable_actors = $this->filterDeliverableActors($actors); + public function sendWithMailers(array $mailers) { + $exceptions = array(); + foreach ($mailers as $template_mailer) { + $mailer = null; - $default_from = PhabricatorEnv::getEnvConfig('metamta.default-address'); - if (empty($params['from'])) { - $mailer->setFrom($default_from); + try { + $mailer = $this->buildMailer($template_mailer); + } catch (Exception $ex) { + $exceptions[] = $ex; + continue; } - $is_first = idx($params, 'is-first-message'); - unset($params['is-first-message']); - - $is_threaded = (bool)idx($params, 'thread-id'); - $must_encrypt = $this->getMustEncrypt(); - - $reply_to_name = idx($params, 'reply-to-name', ''); - unset($params['reply-to-name']); - - $add_cc = array(); - $add_to = array(); - - // If we're sending one mail to everyone, some recipients will be in - // "Cc" rather than "To". We'll move them to "To" later (or supply a - // dummy "To") but need to look for the recipient in either the - // "To" or "Cc" fields here. - $target_phid = head(idx($params, 'to', array())); - if (!$target_phid) { - $target_phid = head(idx($params, 'cc', array())); + if (!$mailer) { + // If we don't get a mailer back, that means the mail doesn't + // actually need to be sent (for example, because recipients have + // declined to receive the mail). Void it and return. + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID) + ->save(); } - $preferences = $this->loadPreferences($target_phid); - - foreach ($params as $key => $value) { - switch ($key) { - case 'raw-from': - list($from_email, $from_name) = $value; - $mailer->setFrom($from_email, $from_name); - break; - case 'from': - $from = $value; - $actor_email = null; - $actor_name = null; - $actor = idx($actors, $from); - if ($actor) { - $actor_email = $actor->getEmailAddress(); - $actor_name = $actor->getName(); - } - $can_send_as_user = $actor_email && - PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); - - if ($can_send_as_user) { - $mailer->setFrom($actor_email, $actor_name); - } else { - $from_email = coalesce($actor_email, $default_from); - $from_name = coalesce($actor_name, pht('Phabricator')); - - if (empty($params['reply-to'])) { - $params['reply-to'] = $from_email; - $params['reply-to-name'] = $from_name; - } - - $mailer->setFrom($default_from, $from_name); - } - break; - case 'reply-to': - $mailer->addReplyTo($value, $reply_to_name); - break; - case 'to': - $to_phids = $this->expandRecipients($value); - $to_actors = array_select_keys($deliverable_actors, $to_phids); - $add_to = array_merge( - $add_to, - mpull($to_actors, 'getEmailAddress')); - break; - case 'raw-to': - $add_to = array_merge($add_to, $value); - break; - case 'cc': - $cc_phids = $this->expandRecipients($value); - $cc_actors = array_select_keys($deliverable_actors, $cc_phids); - $add_cc = array_merge( - $add_cc, - mpull($cc_actors, 'getEmailAddress')); - break; - case 'attachments': - $attached_viewer = PhabricatorUser::getOmnipotentUser(); - $files = $this->loadAttachedFiles($attached_viewer); - foreach ($files as $file) { - $file->attachToObject($this->getPHID()); - } - - // If the mail content must be encrypted, don't add attachments. - if ($must_encrypt) { - break; - } - - $value = $this->getAttachments(); - foreach ($value as $attachment) { - $mailer->addAttachment( - $attachment->getData(), - $attachment->getFilename(), - $attachment->getMimeType()); - } - break; - case 'subject': - $subject = array(); - - if ($is_threaded) { - if ($this->shouldAddRePrefix($preferences)) { - $subject[] = 'Re:'; - } - } - - $subject[] = trim(idx($params, 'subject-prefix')); - - // If mail content must be encrypted, we replace the subject with - // a generic one. - if ($must_encrypt) { - $subject[] = pht('Object Updated'); - } else { - $vary_prefix = idx($params, 'vary-subject-prefix'); - if ($vary_prefix != '') { - if ($this->shouldVarySubject($preferences)) { - $subject[] = $vary_prefix; - } - } - - $subject[] = $value; - } - - $mailer->setSubject(implode(' ', array_filter($subject))); - break; - case 'thread-id': - - // NOTE: Gmail freaks out about In-Reply-To and References which - // aren't in the form ""; this is also required - // by RFC 2822, although some clients are more liberal in what they - // accept. - $domain = PhabricatorEnv::getEnvConfig('metamta.domain'); - $value = '<'.$value.'@'.$domain.'>'; - - if ($is_first && $mailer->supportsMessageIDHeader()) { - $headers[] = array('Message-ID', $value); - } else { - $in_reply_to = $value; - $references = array($value); - $parent_id = $this->getParentMessageID(); - if ($parent_id) { - $in_reply_to = $parent_id; - // By RFC 2822, the most immediate parent should appear last - // in the "References" header, so this order is intentional. - $references[] = $parent_id; - } - $references = implode(' ', $references); - $headers[] = array('In-Reply-To', $in_reply_to); - $headers[] = array('References', $references); - } - $thread_index = $this->generateThreadIndex($value, $is_first); - $headers[] = array('Thread-Index', $thread_index); - break; - default: - // Other parameters are handled elsewhere or are not relevant to - // constructing the message. - break; - } - } - - $stamps = $this->getMailStamps(); - if ($stamps) { - $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); - } - - $raw_body = idx($params, 'body', ''); - $body = $raw_body; - if ($must_encrypt) { - $parts = array(); - $parts[] = pht( - 'The content for this message can only be transmitted over a '. - 'secure channel. To view the message content, follow this '. - 'link:'); - - $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); - - $body = implode("\n\n", $parts); - } else { - $body = $raw_body; - } - - $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); - if (strlen($body) > $max) { - $body = id(new PhutilUTF8StringTruncator()) - ->setMaximumBytes($max) - ->truncateString($body); - $body .= "\n"; - $body .= pht('(This email was truncated at %d bytes.)', $max); - } - $mailer->setBody($body); - - // If we sent a different message body than we were asked to, record - // what we actually sent to make debugging and diagnostics easier. - if ($body !== $raw_body) { - $this->setParam('body.sent', $body); - } - - if ($must_encrypt) { - $send_html = false; - } else { - $send_html = $this->shouldSendHTML($preferences); - } - - if ($send_html && isset($params['html-body'])) { - $mailer->setHTMLBody($params['html-body']); - } - - // Pass the headers to the mailer, then save the state so we can show - // them in the web UI. If the mail must be encrypted, we remove headers - // which are not on a strict whitelist to avoid disclosing information. - $filtered_headers = $this->filterHeaders($headers, $must_encrypt); - foreach ($filtered_headers as $header) { - list($header_key, $header_value) = $header; - $mailer->addHeader($header_key, $header_value); - } - $this->setParam('headers.unfiltered', $headers); - $this->setParam('headers.sent', $filtered_headers); - - // Save the final deliverability outcomes and reasoning so we can - // explain why things happened the way they did. - $actor_list = array(); - foreach ($actors as $actor) { - $actor_list[$actor->getPHID()] = array( - 'deliverable' => $actor->isDeliverable(), - 'reasons' => $actor->getDeliverabilityReasons(), - ); - } - $this->setParam('actors.sent', $actor_list); - - $this->setParam('routing.sent', $this->getParam('routing')); - $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); - - if (!$add_to && !$add_cc) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); - $this->setMessage( - pht( - 'Message has no valid recipients: all To/Cc are disabled, '. - 'invalid, or configured not to receive this mail.')); - return $this->save(); - } - - if ($this->getIsErrorEmail()) { - $all_recipients = array_merge($add_to, $add_cc); - if ($this->shouldRateLimitMail($all_recipients)) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); - $this->setMessage( + try { + $ok = $mailer->send(); + if (!$ok) { + // TODO: At some point, we should clean this up and make all mailers + // throw. + throw new Exception( pht( - 'This is an error email, but one or more recipients have '. - 'exceeded the error email rate limit. Declining to deliver '. - 'message.')); - return $this->save(); + 'Mail adapter encountered an unexpected, unspecified '. + 'failure.')); } + } catch (PhabricatorMetaMTAPermanentFailureException $ex) { + // If any mailer raises a permanent failure, stop trying to send the + // mail with other mailers. + $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) + ->setMessage($ex->getMessage()) + ->save(); + + throw $ex; + } catch (Exception $ex) { + $exceptions[] = $ex; + continue; } - if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT) + ->save(); + } + + // If we make it here, no mailer could send the mail but no mailer failed + // permanently either. We update the error message for the mail, but leave + // it in the current status (usually, STATUS_QUEUE) and try again later. + + $messages = array(); + foreach ($exceptions as $ex) { + $messages[] = $ex->getMessage(); + } + $messages = implode("\n\n", $messages); + + $this + ->setMessage($messages) + ->save(); + + if (count($exceptions) === 1) { + throw head($exceptions); + } + + throw new PhutilAggregateException( + pht('Encountered multiple exceptions while transmitting mail.'), + $exceptions); + } + + private function buildMailer(PhabricatorMailImplementationAdapter $mailer) { + $headers = $this->generateHeaders(); + + $params = $this->parameters; + + $actors = $this->loadAllActors(); + $deliverable_actors = $this->filterDeliverableActors($actors); + + $default_from = PhabricatorEnv::getEnvConfig('metamta.default-address'); + if (empty($params['from'])) { + $mailer->setFrom($default_from); + } + + $is_first = idx($params, 'is-first-message'); + unset($params['is-first-message']); + + $is_threaded = (bool)idx($params, 'thread-id'); + $must_encrypt = $this->getMustEncrypt(); + + $reply_to_name = idx($params, 'reply-to-name', ''); + unset($params['reply-to-name']); + + $add_cc = array(); + $add_to = array(); + + // If we're sending one mail to everyone, some recipients will be in + // "Cc" rather than "To". We'll move them to "To" later (or supply a + // dummy "To") but need to look for the recipient in either the + // "To" or "Cc" fields here. + $target_phid = head(idx($params, 'to', array())); + if (!$target_phid) { + $target_phid = head(idx($params, 'cc', array())); + } + + $preferences = $this->loadPreferences($target_phid); + + foreach ($params as $key => $value) { + switch ($key) { + case 'raw-from': + list($from_email, $from_name) = $value; + $mailer->setFrom($from_email, $from_name); + break; + case 'from': + $from = $value; + $actor_email = null; + $actor_name = null; + $actor = idx($actors, $from); + if ($actor) { + $actor_email = $actor->getEmailAddress(); + $actor_name = $actor->getName(); + } + $can_send_as_user = $actor_email && + PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); + + if ($can_send_as_user) { + $mailer->setFrom($actor_email, $actor_name); + } else { + $from_email = coalesce($actor_email, $default_from); + $from_name = coalesce($actor_name, pht('Phabricator')); + + if (empty($params['reply-to'])) { + $params['reply-to'] = $from_email; + $params['reply-to-name'] = $from_name; + } + + $mailer->setFrom($default_from, $from_name); + } + break; + case 'reply-to': + $mailer->addReplyTo($value, $reply_to_name); + break; + case 'to': + $to_phids = $this->expandRecipients($value); + $to_actors = array_select_keys($deliverable_actors, $to_phids); + $add_to = array_merge( + $add_to, + mpull($to_actors, 'getEmailAddress')); + break; + case 'raw-to': + $add_to = array_merge($add_to, $value); + break; + case 'cc': + $cc_phids = $this->expandRecipients($value); + $cc_actors = array_select_keys($deliverable_actors, $cc_phids); + $add_cc = array_merge( + $add_cc, + mpull($cc_actors, 'getEmailAddress')); + break; + case 'attachments': + $attached_viewer = PhabricatorUser::getOmnipotentUser(); + $files = $this->loadAttachedFiles($attached_viewer); + foreach ($files as $file) { + $file->attachToObject($this->getPHID()); + } + + // If the mail content must be encrypted, don't add attachments. + if ($must_encrypt) { + break; + } + + $value = $this->getAttachments(); + foreach ($value as $attachment) { + $mailer->addAttachment( + $attachment->getData(), + $attachment->getFilename(), + $attachment->getMimeType()); + } + break; + case 'subject': + $subject = array(); + + if ($is_threaded) { + if ($this->shouldAddRePrefix($preferences)) { + $subject[] = 'Re:'; + } + } + + $subject[] = trim(idx($params, 'subject-prefix')); + + // If mail content must be encrypted, we replace the subject with + // a generic one. + if ($must_encrypt) { + $subject[] = pht('Object Updated'); + } else { + $vary_prefix = idx($params, 'vary-subject-prefix'); + if ($vary_prefix != '') { + if ($this->shouldVarySubject($preferences)) { + $subject[] = $vary_prefix; + } + } + + $subject[] = $value; + } + + $mailer->setSubject(implode(' ', array_filter($subject))); + break; + case 'thread-id': + + // NOTE: Gmail freaks out about In-Reply-To and References which + // aren't in the form ""; this is also required + // by RFC 2822, although some clients are more liberal in what they + // accept. + $domain = PhabricatorEnv::getEnvConfig('metamta.domain'); + $value = '<'.$value.'@'.$domain.'>'; + + if ($is_first && $mailer->supportsMessageIDHeader()) { + $headers[] = array('Message-ID', $value); + } else { + $in_reply_to = $value; + $references = array($value); + $parent_id = $this->getParentMessageID(); + if ($parent_id) { + $in_reply_to = $parent_id; + // By RFC 2822, the most immediate parent should appear last + // in the "References" header, so this order is intentional. + $references[] = $parent_id; + } + $references = implode(' ', $references); + $headers[] = array('In-Reply-To', $in_reply_to); + $headers[] = array('References', $references); + } + $thread_index = $this->generateThreadIndex($value, $is_first); + $headers[] = array('Thread-Index', $thread_index); + break; + default: + // Other parameters are handled elsewhere or are not relevant to + // constructing the message. + break; + } + } + + $stamps = $this->getMailStamps(); + if ($stamps) { + $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + } + + $raw_body = idx($params, 'body', ''); + $body = $raw_body; + if ($must_encrypt) { + $parts = array(); + $parts[] = pht( + 'The content for this message can only be transmitted over a '. + 'secure channel. To view the message content, follow this '. + 'link:'); + + $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); + + $body = implode("\n\n", $parts); + } else { + $body = $raw_body; + } + + $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); + if (strlen($body) > $max) { + $body = id(new PhutilUTF8StringTruncator()) + ->setMaximumBytes($max) + ->truncateString($body); + $body .= "\n"; + $body .= pht('(This email was truncated at %d bytes.)', $max); + } + $mailer->setBody($body); + + // If we sent a different message body than we were asked to, record + // what we actually sent to make debugging and diagnostics easier. + if ($body !== $raw_body) { + $this->setParam('body.sent', $body); + } + + if ($must_encrypt) { + $send_html = false; + } else { + $send_html = $this->shouldSendHTML($preferences); + } + + if ($send_html && isset($params['html-body'])) { + $mailer->setHTMLBody($params['html-body']); + } + + // Pass the headers to the mailer, then save the state so we can show + // them in the web UI. If the mail must be encrypted, we remove headers + // which are not on a strict whitelist to avoid disclosing information. + $filtered_headers = $this->filterHeaders($headers, $must_encrypt); + foreach ($filtered_headers as $header) { + list($header_key, $header_value) = $header; + $mailer->addHeader($header_key, $header_value); + } + $this->setParam('headers.unfiltered', $headers); + $this->setParam('headers.sent', $filtered_headers); + + // Save the final deliverability outcomes and reasoning so we can + // explain why things happened the way they did. + $actor_list = array(); + foreach ($actors as $actor) { + $actor_list[$actor->getPHID()] = array( + 'deliverable' => $actor->isDeliverable(), + 'reasons' => $actor->getDeliverabilityReasons(), + ); + } + $this->setParam('actors.sent', $actor_list); + + $this->setParam('routing.sent', $this->getParam('routing')); + $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); + + if (!$add_to && !$add_cc) { + $this->setMessage( + pht( + 'Message has no valid recipients: all To/Cc are disabled, '. + 'invalid, or configured not to receive this mail.')); + + return null; + } + + if ($this->getIsErrorEmail()) { + $all_recipients = array_merge($add_to, $add_cc); + if ($this->shouldRateLimitMail($all_recipients)) { $this->setMessage( pht( - 'Phabricator is running in silent mode. See `%s` '. - 'in the configuration to change this setting.', - 'phabricator.silent')); - return $this->save(); + 'This is an error email, but one or more recipients have '. + 'exceeded the error email rate limit. Declining to deliver '. + 'message.')); + + return null; } - - // Some mailers require a valid "To:" in order to deliver mail. If we - // don't have any "To:", try to fill it in with a placeholder "To:". - // If that also fails, move the "Cc:" line to "To:". - if (!$add_to) { - $placeholder_key = 'metamta.placeholder-to-recipient'; - $placeholder = PhabricatorEnv::getEnvConfig($placeholder_key); - if ($placeholder !== null) { - $add_to = array($placeholder); - } else { - $add_to = $add_cc; - $add_cc = array(); - } - } - - $add_to = array_unique($add_to); - $add_cc = array_diff(array_unique($add_cc), $add_to); - - $mailer->addTos($add_to); - if ($add_cc) { - $mailer->addCCs($add_cc); - } - } catch (Exception $ex) { - $this - ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) - ->setMessage($ex->getMessage()) - ->save(); - - throw $ex; } - try { - $ok = $mailer->send(); - if (!$ok) { - // TODO: At some point, we should clean this up and make all mailers - // throw. - throw new Exception( - pht('Mail adapter encountered an unexpected, unspecified failure.')); - } + if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { + $this->setMessage( + pht( + 'Phabricator is running in silent mode. See `%s` '. + 'in the configuration to change this setting.', + 'phabricator.silent')); - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT); - $this->save(); - - return $this; - } catch (PhabricatorMetaMTAPermanentFailureException $ex) { - $this - ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) - ->setMessage($ex->getMessage()) - ->save(); - - throw $ex; - } catch (Exception $ex) { - $this - ->setMessage($ex->getMessage()."\n".$ex->getTraceAsString()) - ->save(); - - throw $ex; + return null; } + + // Some mailers require a valid "To:" in order to deliver mail. If we + // don't have any "To:", try to fill it in with a placeholder "To:". + // If that also fails, move the "Cc:" line to "To:". + if (!$add_to) { + $placeholder_key = 'metamta.placeholder-to-recipient'; + $placeholder = PhabricatorEnv::getEnvConfig($placeholder_key); + if ($placeholder !== null) { + $add_to = array($placeholder); + } else { + $add_to = $add_cc; + $add_cc = array(); + } + } + + $add_to = array_unique($add_to); + $add_cc = array_diff(array_unique($add_cc), $add_to); + + $mailer->addTos($add_to); + if ($add_cc) { + $mailer->addCCs($add_cc); + } + + return $mailer; } private function generateThreadIndex($seed, $is_first_mail) { diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 635913439d..9f14e0c4e1 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -18,7 +18,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mail->addTos(array($phid)); $mailer = new PhabricatorMailImplementationTestAdapter(); - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); $this->assertEqual( PhabricatorMailOutboundStatus::STATUS_SENT, $mail->getStatus()); @@ -31,7 +31,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mailer = new PhabricatorMailImplementationTestAdapter(); $mailer->setFailTemporarily(true); try { - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); } catch (Exception $ex) { // Ignore. } @@ -47,7 +47,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mailer = new PhabricatorMailImplementationTestAdapter(); $mailer->setFailPermanently(true); try { - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); } catch (Exception $ex) { // Ignore. } @@ -191,7 +191,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mail = new PhabricatorMetaMTAMail(); $mail->setThreadID($thread_id, $is_first_mail); - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); $guts = $mailer->getGuts(); $dict = ipull($guts['headers'], 1, 0); From 7765299f8399f58ed96b91ee1ef0b2cabf562617 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 04:28:03 -0800 Subject: [PATCH 19/67] Mask the sender for "Must Encrypt" mail Summary: Depends on D18998. Ref T13053. When we send "Must Encrypt" mail, we currently send it with a normal "From" address. This discloses a little information about the object (for example, if the Director of Silly Walks is interacting with a "must encrypt" object, the vulnerability is probably related to Silly Walks), so anonymize who is interacting with the object. Test Plan: Processed some mail. (The actual final "From" is ephemeral and a little tricky to examine and I didn't actually transmit mail over the network, but it should be obvious if this works or not on `secure`.) Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19000 --- src/applications/metamta/storage/PhabricatorMetaMTAMail.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 179eb18088..317f9be8df 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -592,6 +592,12 @@ final class PhabricatorMetaMTAMail $mailer->setFrom($from_email, $from_name); break; case 'from': + // If the mail content must be encrypted, disguise the sender. + if ($must_encrypt) { + $mailer->setFrom($default_from, pht('Phabricator')); + break; + } + $from = $value; $actor_email = null; $actor_name = null; From 7f2c90fbd12b10e10abc3098c080baae96d4040f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 15:58:07 -0800 Subject: [PATCH 20/67] Prepare for multiple mailers of the same type Summary: Depends on D19000. Ref T13053. Ref T12677. Currently, most mailers are configured with a bunch of `.setting-name` global config options. This means that you can't configure two different SMTP servers, which is a reasonable thing to want to do in the brave new world of mail failover. It also means you can't configure two Mailgun accounts or two SES accounts. Although this might seem a little silly, we've had more service disruptions because of policy issues / administrative error (where a particular account was disabled) than actual downtime, so maybe it's not completely ridiculous. Realign mailers so they can take configuration directly in an explicit way. A later change will add new configuration to take advantage of this and let us move away from having ~10 global options for this stuff eventually. (This also makes writing third-party mailers easier.) Test Plan: Processed some mail, ran existing unit tests. But I wasn't especially thorough. I expect later changes to provide some tools to make this more testable, so I'll vet each provider more thoroughly and add coverage for multiple mailers after that stuff is ready. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19002 --- .../PhabricatorMailImplementationAdapter.php | 40 ++++++++++++++ ...atorMailImplementationAmazonSESAdapter.php | 36 ++++++++++-- ...icatorMailImplementationMailgunAdapter.php | 27 ++++++++- ...atorMailImplementationPHPMailerAdapter.php | 55 ++++++++++++++++--- ...MailImplementationPHPMailerLiteAdapter.php | 24 +++++++- ...catorMailImplementationSendGridAdapter.php | 27 ++++++++- ...abricatorMailImplementationTestAdapter.php | 18 +++++- .../storage/PhabricatorMetaMTAMail.php | 29 +++++++--- .../PhabricatorMetaMTAMailTestCase.php | 4 +- 9 files changed, 229 insertions(+), 31 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 3363301909..514306758d 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -2,6 +2,9 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { + private $key; + private $options = array(); + abstract public function setFrom($email, $name = ''); abstract public function addReplyTo($email, $name = ''); abstract public function addTos(array $emails); @@ -12,6 +15,7 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { abstract public function setHTMLBody($html_body); abstract public function setSubject($subject); + /** * Some mailers, notably Amazon SES, do not support us setting a specific * Message-ID header. @@ -32,4 +36,40 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { */ abstract public function send(); + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final public function getOption($key) { + if (!array_key_exists($key, $this->options)) { + throw new Exception( + pht( + 'Mailer ("%s") is attempting to access unknown option ("%s").', + get_class($this), + $key)); + } + + return $this->options[$key]; + } + + final public function setOptions(array $options) { + $this->validateOptions($options); + $this->options = $options; + return $this; + } + + abstract protected function validateOptions(array $options); + + abstract public function newDefaultOptions(); + abstract public function newLegacyOptions(); + + public function prepareForSend() { + return; + } + } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php index 5b03cd86ac..850b83f1dd 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php @@ -6,8 +6,8 @@ final class PhabricatorMailImplementationAmazonSESAdapter private $message; private $isHTML; - public function __construct() { - parent::__construct(); + public function prepareForSend() { + parent::prepareForSend(); $this->mailer->Mailer = 'amazon-ses'; $this->mailer->customMailer = $this; } @@ -17,13 +17,39 @@ final class PhabricatorMailImplementationAmazonSESAdapter return false; } + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'access-key' => 'string', + 'secret-key' => 'string', + 'endpoint' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'access-key' => null, + 'secret-key' => null, + 'endpoint' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'access-key' => PhabricatorEnv::getEnvConfig('amazon-ses.access-key'), + 'secret-key' => PhabricatorEnv::getEnvConfig('amazon-ses.secret-key'), + 'endpoint' => PhabricatorEnv::getEnvConfig('amazon-ses.endpoint'), + ); + } + /** * @phutil-external-symbol class SimpleEmailService */ public function executeSend($body) { - $key = PhabricatorEnv::getEnvConfig('amazon-ses.access-key'); - $secret = PhabricatorEnv::getEnvConfig('amazon-ses.secret-key'); - $endpoint = PhabricatorEnv::getEnvConfig('amazon-ses.endpoint'); + $key = $this->getOption('access-key'); + $secret = $this->getOption('secret-key'); + $endpoint = $this->getOption('endpoint'); $root = phutil_get_library_root('phabricator'); $root = dirname($root); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index cfe6491fe0..a7be6731eb 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -71,9 +71,32 @@ final class PhabricatorMailImplementationMailgunAdapter return true; } + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'api-key' => 'string', + 'domain' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'api-key' => null, + 'domain' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'api-key' => PhabricatorEnv::getEnvConfig('mailgun.api-key'), + 'domain' => PhabricatorEnv::getEnvConfig('mailgun.domain'), + ); + } + public function send() { - $key = PhabricatorEnv::getEnvConfig('mailgun.api-key'); - $domain = PhabricatorEnv::getEnvConfig('mailgun.domain'); + $key = $this->getOption('api-key'); + $domain = $this->getOption('domain'); $params = array(); $params['to'] = implode(', ', idx($this->params, 'tos', array())); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php index f4d7e8e156..0eb59629a6 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php @@ -5,17 +5,55 @@ final class PhabricatorMailImplementationPHPMailerAdapter private $mailer; + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'host' => 'string|null', + 'port' => 'int', + 'user' => 'string|null', + 'password' => 'string|null', + 'protocol' => 'string|null', + 'encoding' => 'string', + 'mailer' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'host' => null, + 'port' => 25, + 'user' => null, + 'password' => null, + 'protocol' => null, + 'encoding' => 'base64', + 'mailer' => 'smtp', + ); + } + + public function newLegacyOptions() { + return array( + 'host' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-host'), + 'port' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-port'), + 'user' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-user'), + 'password' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-passsword'), + 'protocol' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-protocol'), + 'encoding' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'), + 'mailer' => PhabricatorEnv::getEnvConfig('phpmailer.mailer'), + ); + } + /** * @phutil-external-symbol class PHPMailer */ - public function __construct() { + public function prepareForSend() { $root = phutil_get_library_root('phabricator'); $root = dirname($root); require_once $root.'/externals/phpmailer/class.phpmailer.php'; $this->mailer = new PHPMailer($use_exceptions = true); $this->mailer->CharSet = 'utf-8'; - $encoding = PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'); + $encoding = $this->getOption('encoding'); $this->mailer->Encoding = $encoding; // By default, PHPMailer sends one mail per recipient. We handle @@ -23,20 +61,19 @@ final class PhabricatorMailImplementationPHPMailerAdapter // send mail exactly like we ask. $this->mailer->SingleTo = false; - $mailer = PhabricatorEnv::getEnvConfig('phpmailer.mailer'); + $mailer = $this->getOption('mailer'); if ($mailer == 'smtp') { $this->mailer->IsSMTP(); - $this->mailer->Host = PhabricatorEnv::getEnvConfig('phpmailer.smtp-host'); - $this->mailer->Port = PhabricatorEnv::getEnvConfig('phpmailer.smtp-port'); - $user = PhabricatorEnv::getEnvConfig('phpmailer.smtp-user'); + $this->mailer->Host = $this->getOption('host'); + $this->mailer->Port = $this->getOption('port'); + $user = $this->getOption('user'); if ($user) { $this->mailer->SMTPAuth = true; $this->mailer->Username = $user; - $this->mailer->Password = - PhabricatorEnv::getEnvConfig('phpmailer.smtp-password'); + $this->mailer->Password = $this->getOption('password'); } - $protocol = PhabricatorEnv::getEnvConfig('phpmailer.smtp-protocol'); + $protocol = $this->getOption('protocol'); if ($protocol) { $protocol = phutil_utf8_strtolower($protocol); $this->mailer->SMTPSecure = $protocol; diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index 668f9353ec..4fd8387252 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -8,17 +8,37 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter protected $mailer; + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'encoding' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'encoding' => 'base64', + ); + } + + public function newLegacyOptions() { + return array( + 'encoding' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'), + ); + } + /** * @phutil-external-symbol class PHPMailerLite */ - public function __construct() { + public function prepareForSend() { $root = phutil_get_library_root('phabricator'); $root = dirname($root); require_once $root.'/externals/phpmailer/class.phpmailer-lite.php'; $this->mailer = new PHPMailerLite($use_exceptions = true); $this->mailer->CharSet = 'utf-8'; - $encoding = PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'); + $encoding = $this->getOption('encoding'); $this->mailer->Encoding = $encoding; // By default, PHPMailerLite sends one mail per recipient. We handle diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php index 566d33fd14..9cd8dd19b4 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php @@ -8,6 +8,29 @@ final class PhabricatorMailImplementationSendGridAdapter private $params = array(); + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'api-user' => 'string', + 'api-key' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'api-user' => null, + 'api-key' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'api-user' => PhabricatorEnv::getEnvConfig('sendgrid.api-user'), + 'api-key' => PhabricatorEnv::getEnvConfig('sendgrid.api-key'), + ); + } + public function setFrom($email, $name = '') { $this->params['from'] = $email; $this->params['from-name'] = $name; @@ -73,8 +96,8 @@ final class PhabricatorMailImplementationSendGridAdapter public function send() { - $user = PhabricatorEnv::getEnvConfig('sendgrid.api-user'); - $key = PhabricatorEnv::getEnvConfig('sendgrid.api-key'); + $user = $this->getOption('api-user'); + $key = $this->getOption('api-key'); if (!$user || !$key) { throw new Exception( diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php index 0ea2af916f..bd64076a59 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php @@ -8,9 +8,23 @@ final class PhabricatorMailImplementationTestAdapter extends PhabricatorMailImplementationAdapter { private $guts = array(); - private $config; + private $config = array(); - public function __construct(array $config = array()) { + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array()); + } + + public function newDefaultOptions() { + return array(); + } + + public function newLegacyOptions() { + return array(); + } + + public function prepareForSend(array $config = array()) { $this->config = $config; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 317f9be8df..eb1f1fbea2 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -453,11 +453,6 @@ final class PhabricatorMetaMTAMail return $result; } - public function buildDefaultMailer() { - return PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); - } - - /** * Attempt to deliver an email immediately, in this process. * @@ -468,13 +463,31 @@ final class PhabricatorMetaMTAMail throw new Exception(pht('Trying to send an already-sent mail!')); } - $mailers = array( - $this->buildDefaultMailer(), - ); + $mailers = $this->newMailers(); return $this->sendWithMailers($mailers); } + private function newMailers() { + $mailers = array(); + + $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); + + $defaults = $mailer->newDefaultOptions(); + $options = $mailer->newLegacyOptions(); + + $options = $options + $defaults; + + $mailer + ->setKey('default') + ->setOptions($options); + + $mailer->prepareForSend(); + + $mailers[] = $mailer; + + return $mailers; + } public function sendWithMailers(array $mailers) { $exceptions = array(); diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 9f14e0c4e1..6e72b129b1 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -182,7 +182,9 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $supports_message_id, $is_first_mail) { - $mailer = new PhabricatorMailImplementationTestAdapter( + $mailer = new PhabricatorMailImplementationTestAdapter(); + + $mailer->prepareForSend( array( 'supportsMessageIDHeader' => $supports_message_id, )); From c868ee9c07d0e8b7a6622f1de263027efa0773ea Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 05:23:47 -0800 Subject: [PATCH 21/67] Introduce and document a new `cluster.mailers` option for configuring multiple mailers Summary: Depends on D19002. Ref T13053. Ref T12677. Adds a new option to allow configuration of multiple mailers. Nothing actually uses this yet. Test Plan: Tried to set it to various bad values, got reasonable error messages. Read documentation. Reviewers: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19003 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMetaMTAConfigOptions.php | 20 +- .../PhabricatorMailImplementationAdapter.php | 12 + ...atorMailImplementationAmazonSESAdapter.php | 2 + ...icatorMailImplementationMailgunAdapter.php | 2 + ...atorMailImplementationPHPMailerAdapter.php | 2 + ...MailImplementationPHPMailerLiteAdapter.php | 2 + ...catorMailImplementationSendGridAdapter.php | 2 + ...abricatorMailImplementationTestAdapter.php | 2 + .../configuring_outbound_email.diviner | 303 +++++++++++------- .../PhabricatorClusterMailersConfigType.php | 100 ++++++ 11 files changed, 323 insertions(+), 126 deletions(-) create mode 100644 src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c642f13f90..fb2b0ccf0c 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2411,6 +2411,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterMailersConfigType' => 'infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php', 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', 'PhabricatorClusterSearchConfigType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigType.php', 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', @@ -7824,6 +7825,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterMailersConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterNoHostForRoleException' => 'Exception', 'PhabricatorClusterSearchConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterServiceHealthRecord' => 'Phobject', diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 8c2d2265bc..43734abea0 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -138,19 +138,14 @@ EODOC , 'metamta.public-replies')); - $adapter_doc_href = PhabricatorEnv::getDoclink( - 'Configuring Outbound Email'); - $adapter_doc_name = pht('Configuring Outbound Email'); $adapter_description = $this->deformat(pht(<<deformat(pht(<<deformat(pht(<<newOption('cluster.mailers', 'cluster.mailers', null) + ->setLocked(true) + ->setDescription($mailers_description), $this->newOption( 'metamta.default-address', 'string', diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 514306758d..923574d16d 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -5,6 +5,18 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { private $key; private $options = array(); + final public function getAdapterType() { + return $this->getPhobjectClassConstant('ADAPTERTYPE'); + } + + final public static function getAllAdapters() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getAdapterType') + ->execute(); + } + + abstract public function setFrom($email, $name = ''); abstract public function addReplyTo($email, $name = ''); abstract public function addTos(array $emails); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php index 850b83f1dd..22cc102262 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationAmazonSESAdapter extends PhabricatorMailImplementationPHPMailerLiteAdapter { + const ADAPTERTYPE = 'ses'; + private $message; private $isHTML; diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index a7be6731eb..bed0dada63 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationMailgunAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'mailgun'; + private $params = array(); private $attachments = array(); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php index 0eb59629a6..3ca6366730 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationPHPMailerAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'smtp'; + private $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index 4fd8387252..1f21a993c9 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -6,6 +6,8 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendmail'; + protected $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php index 9cd8dd19b4..be2a837053 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationSendGridAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendgrid'; + private $params = array(); protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php index bd64076a59..8a8d0de0c2 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php @@ -7,6 +7,8 @@ final class PhabricatorMailImplementationTestAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'test'; + private $guts = array(); private $config = array(); diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 2a95f49bc3..0c7ef529fa 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -3,43 +3,40 @@ Instructions for configuring Phabricator to send mail. -= Overview = +Overview +======== -Phabricator can send outbound email via several different providers, called -"Adapters". +Phabricator can send outbound email through several different mail services, +including a local mailer or various third-party services. Options include: | Send Mail With | Setup | Cost | Inbound | Notes | |---------|-------|------|---------|-------| | Mailgun | Easy | Cheap | Yes | Recommended | | Amazon SES | Easy | Cheap | No | Recommended | -| SendGrid | Medium | Cheap | Yes | Discouraged (See Note) | +| SendGrid | Medium | Cheap | Yes | Discouraged | | External SMTP | Medium | Varies | No | Gmail, etc. | -| Local SMTP | Hard | Free | No | (Default) sendmail, postfix, etc | -| Custom | Hard | Free | No | Write an adapter for some other service. | +| Local SMTP | Hard | Free | No | sendmail, postfix, etc | +| Custom | Hard | Free | No | Write a custom mailer for some other service. | | Drop in a Hole | Easy | Free | No | Drops mail in a deep, dark hole. | -Of these options, sending mail via local SMTP is the default, but usually -requires some configuration to get working. See below for details on how to -select and configure a delivery method. +See below for details on how to select and configure mail delivery for each +mailer. Overall, Mailgun and SES are much easier to set up, and using one of them is recommended. In particular, Mailgun will also let you set up inbound email easily. If you have some internal mail service you'd like to use you can also -write a custom adapter, but this requires digging into the code. +write a custom mailer, but this requires digging into the code. Phabricator sends mail in the background, so the daemons need to be running for it to be able to deliver mail. You should receive setup warnings if they are not. For more information on using daemons, see @{article:Managing Daemons with phd}. -**Note on SendGrid**: Users have experienced a number of odd issues with -SendGrid, compared to fewer issues with other mailers. We discourage SendGrid -unless you're already using it. If you send to SendGrid via SMTP, you may need -to adjust `phpmailer.smtp-encoding`. -= Basics = +Basics +====== Regardless of how outbound email is delivered, you should configure these keys in your configuration: @@ -51,33 +48,113 @@ in your configuration: - **metamta.can-send-as-user** should be left as `false` in most cases, but see the documentation for details. -= Configuring Mail Adapters = -To choose how mail will be sent, change the `metamta.mail-adapter` key in -your configuration. Possible values are listed in the UI: +Configuring Mailers +=================== - - `PhabricatorMailImplementationAmazonMailgunAdapter`: use Mailgun, see - "Adapter: Mailgun". - - `PhabricatorMailImplementationAmazonSESAdapter`: use Amazon SES, see - "Adapter: Amazon SES". - - `PhabricatorMailImplementationPHPMailerLiteAdapter`: default, uses - "sendmail", see "Adapter: Sendmail". - - `PhabricatorMailImplementationPHPMailerAdapter`: uses SMTP, see - "Adapter: SMTP". - - `PhabricatorMailImplementationSendGridAdapter`: use SendGrid, see - "Adapter: SendGrid". - - `Some Custom Class You Write`: use a custom adapter you write, see - "Adapter: Custom". - - `PhabricatorMailImplementationTestAdapter`: this will - **completely disable** outbound mail. You can use this if you don't want to - send outbound mail, or want to skip this step for now and configure it - later. +Configure one or more mailers by listing them in the the `cluster.mailers` +configuration option. Most installs only need to configure one mailer, but you +can configure multiple mailers to provide greater availability in the event of +a service disruption. -= Adapter: Sendmail = +A valid `cluster.mailers` configuration looks something like this: -This is the default, and selected by choosing -`PhabricatorMailImplementationPHPMailerLiteAdapter` as the value for -**metamta.mail-adapter**. This requires a `sendmail` binary to be installed on +```lang=json +[ + { + "key": "mycompany-mailgun", + "type": "mailgun", + "options": { + "domain": "mycompany.com", + "api-key": "..." + } + }, + ... +] +``` + +The supported keys for each mailer are: + + - `key`: Required string. A unique name for this mailer. + - `type`: Required string. Identifies the type of mailer. See below for + options. + - `priority`: Optional string. Advanced option which controls load balancing + and failover behavior. See below for details. + - `options`: Optional map. Additional options for the mailer type. + +The `type` field can be used to select these third-party mailers: + + - `mailgun`: Use Mailgun. + - `ses`: Use Amazon SES. + - `sendgrid`: Use Sendgrid. + +It also supports these local mailers: + + - `sendmail`: Use the local `sendmail` binary. + - `smtp`: Connect directly to an SMTP server. + - `test`: Internal mailer for testing. Does not send mail. + +You can also write your own mailer by extending +`PhabricatorMailImplementationAdapter`. + +Once you've selected a mailer, find the corresponding section below for +instructions on configuring it. + + +Mailer: Mailgun +=============== + +Mailgun is a third-party email delivery service. You can learn more at +. Mailgun is easy to configure and works well. + +To use this mailer, set `type` to `mailgun`, then configure these `options`: + + - `api-key`: Required string. Your Mailgun API key. + - `domain`: Required string. Your Mailgun domain. + + +Mailer: Amazon SES +================== + +Amazon SES is Amazon's cloud email service. You can learn more at +. + +To use this mailer, set `type` to `ses`, then configure these `options`: + + - `access-key`: Required string. Your Amazon SES access key. + - `secret-key`: Required string. Your Amazon SES secret key. + - `endpoint`: Required string. Your Amazon SES endpoint. + +NOTE: Amazon SES **requires you to verify your "From" address**. Configure +which "From" address to use by setting "`metamta.default-address`" in your +config, then follow the Amazon SES verification process to verify it. You +won't be able to send email until you do this! + + +Mailer: SendGrid +================ + +SendGrid is a third-party email delivery service. You can learn more at +. + +You can configure SendGrid in two ways: you can send via SMTP or via the REST +API. To use SMTP, configure Phabricator to use an `smtp` mailer. + +To use the REST API mailer, set `type` to `sendgrid`, then configure +these `options`: + + - `api-user`: Required string. Your SendGrid login name. + - `api-key`: Required string. Your SendGrid API key. + +NOTE: Users have experienced a number of odd issues with SendGrid, compared to +fewer issues with other mailers. We discourage SendGrid unless you're already +using it. + + +Mailer: Sendmail +================ + +This requires a `sendmail` binary to be installed on the system. Most MTAs (e.g., sendmail, qmail, postfix) should do this, but your machine may not have one installed by default. For install instructions, consult the documentation for your favorite MTA. @@ -88,96 +165,32 @@ document. If you can already send outbound email from the command line or know how to configure it, this option is straightforward. If you have no idea how to do any of this, strongly consider using Mailgun or Amazon SES instead. -If you experience issues with mail getting mangled (for example, arriving with -too many or too few newlines) you may try adjusting `phpmailer.smtp-encoding`. +To use this mailer, set `type` to `sendmail`. There are no `options` to +configure. -= Adapter: SMTP = + +Mailer: STMP +============ You can use this adapter to send mail via an external SMTP server, like Gmail. -To do this, set these configuration keys: - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationPHPMailerAdapter`. - - **phpmailer.mailer**: set to `smtp`. - - **phpmailer.smtp-host**: set to hostname of your SMTP server. - - **phpmailer.smtp-port**: set to port of your SMTP server. - - **phpmailer.smtp-user**: set to your username used for authentication. - - **phpmailer.smtp-password**: set to your password used for authentication. - - **phpmailer.smtp-protocol**: set to `tls` or `ssl` if necessary. Use +To use this mailer, set `type` to `smtp`, then configure these `options`: + + - `host`: Required string. The hostname of your SMTP server. + - `user`: Optional string. Username used for authentication. + - `password`: Optional string. Password for authentication. + - `protocol`: Optional string. Set to `tls` or `ssl` if necessary. Use `ssl` for Gmail. - - **phpmailer.smtp-encoding**: Normally safe to leave as the default, but - adjusting it may help resolve mail mangling issues (for example, mail - arriving with too many or too few newlines). -= Adapter: Mailgun = -Mailgun is an email delivery service. You can learn more at -. Mailgun isn't free, but is very easy to configure -and works well. +Disable Mail +============ -To use Mailgun, sign up for an account, then set these configuration keys: +To disable mail, just don't configure any mailers. - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationMailgunAdapter`. - - **mailgun.api-key**: set to your Mailgun API key. - - **mailgun.domain**: set to your Mailgun domain. -= Adapter: Amazon SES = - -Amazon SES is Amazon's cloud email service. It is not free, but is easier to -configure than sendmail and can simplify outbound email configuration. To use -Amazon SES, you need to sign up for an account with Amazon at -. - -To configure Phabricator to use Amazon SES, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationAmazonSESAdapter". - - **amazon-ses.access-key**: set to your Amazon SES access key. - - **amazon-ses.secret-key**: set to your Amazon SES secret key. - - **amazon-ses.endpoint**: Set to your Amazon SES endpoint. - -NOTE: Amazon SES **requires you to verify your "From" address**. Configure which -"From" address to use by setting "`metamta.default-address`" in your config, -then follow the Amazon SES verification process to verify it. You won't be able -to send email until you do this! - -= Adapter: SendGrid = - -SendGrid is an email delivery service like Amazon SES. You can learn more at -. It is easy to configure, but not free. - -You can configure SendGrid in two ways: you can send via SMTP or via the REST -API. To use SMTP, just configure `sendmail` and leave Phabricator's setup -with defaults. To use the REST API, follow the instructions in this section. - -To configure Phabricator to use SendGrid, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationSendGridAdapter". - - **sendgrid.api-user**: set to your SendGrid login name. - - **sendgrid.api-key**: set to your SendGrid password. - -If you're logged into your SendGrid account, you may be able to find this -information easily by visiting . - -= Adapter: Custom = - -You can provide a custom adapter by writing a concrete subclass of -@{class:PhabricatorMailImplementationAdapter} and setting it as the -`metamta.mail-adapter`. - -TODO: This should be better documented once extending Phabricator is better -documented. - -= Adapter: Disable Outbound Mail = - -You can use the @{class:PhabricatorMailImplementationTestAdapter} to completely -disable outbound mail, if you don't want to send mail or don't want to configure -it yet. Just set **metamta.mail-adapter** to -`PhabricatorMailImplementationTestAdapter`. - -= Testing and Debugging Outbound Email = +Testing and Debugging Outbound Email +==================================== You can use the `bin/mail` utility to test, debug, and examine outbound mail. In particular: @@ -191,7 +204,59 @@ Run `bin/mail help ` for more help on using these commands. You can monitor daemons using the Daemon Console (`/daemon/`, or click **Daemon Console** from the homepage). -= Next Steps = + +Priorities +========== + +By default, Phabricator will try each mailer in order: it will try the first +mailer first. If that fails (for example, because the service is not available +at the moment) it will try the second mailer, and so on. + +If you want to load balance between multiple mailers instead of using one as +a primary, you can set `priority`. Phabricator will start with mailers in the +highest priority group and go through them randomly, then fall back to the +next group. + +For example, if you have two SMTP servers and you want to balance requests +between them and then fall back to Mailgun if both fail, configure priorities +like this: + +```lang=json +[ + { + "key": "smtp-uswest", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "smtp-useast", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "mailgun-fallback", + "type": "mailgun", + "options": "..." + } +} +``` + +Phabricator will start with servers in the highest priority group (the group +with the **largest** `priority` number). In this example, the highest group is +`300`, which has the two SMTP servers. They'll be tried in random order first. + +If both fail, Phabricator will move on to the next priority group. In this +example, there are no other priority groups. + +If it still hasn't sent the mail, Phabricator will try servers which are not +in any priority group, in the configured order. In this example there is +only one such server, so it will try to send via Mailgun. + + +Next Steps +========== Continue by: diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php new file mode 100644 index 0000000000..2a7550c419 --- /dev/null +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -0,0 +1,100 @@ +newException( + pht( + 'Mailer cluster configuration is not valid: it should be a list '. + 'of mailer configurations.')); + } + + foreach ($value as $index => $spec) { + if (!is_array($spec)) { + throw $this->newException( + pht( + 'Mailer cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a mailer, but the value '. + 'with index "%s" is not a dictionary.', + $index)); + } + } + + $adapters = PhabricatorMailImplementationAdapter::getAllAdapters(); + + $map = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'key' => 'string', + 'type' => 'string', + 'priority' => 'optional int', + 'options' => 'optional wild', + )); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration has an invalid mailer specification '. + '(at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + $key = $spec['key']; + if (isset($map[$key])) { + throw $this->newException( + pht( + 'Mailer configuration is invalid: multiple mailers have the same '. + 'key ("%s"). Each mailer must have a unique key.', + $key)); + } + $map[$key] = true; + + $priority = idx($spec, 'priority', 0); + if ($priority <= 0) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: priority must be '. + 'greater than 0.', + $key)); + } + + $type = $spec['type']; + if (!isset($adapters[$type])) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: mailer type ("%s") is '. + 'unknown. Supported mailer types are: %s.', + $key, + $type, + implode(', ', array_keys($adapters)))); + } + + $options = idx($spec, 'options', array()); + try { + id(clone $adapters[$type])->validateOptions($options); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") specifies invalid options for '. + 'mailer: %s', + $key, + $ex->getMessage())); + } + } + } + +} From 4236952cdbc0e1f8af11b126dc28172997827627 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:20:24 -0800 Subject: [PATCH 22/67] Add a `bin/config set --stdin < value.json` flag to make CLI configuration of complex values easier Summary: Depends on D19003. Ref T12677. Ref T13053. For the first time, we're requiring CLI configuration of a complex value (not just a string, integer, bool, etc) to do something fairly standard (send mail). Users sometimes have very reasonable difficulty figuring out how to `./bin/config set key `. Provide an easy way to handle this and make sure it gets appropriate callouts in the documentation. (Also, hide the `cluster.mailers` value rather than just locking it, since it may have API keys or SMTP passwords.) Test Plan: Read documentation, used old and new flags to set configuration. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19004 --- ...PhabricatorConfigManagementSetWorkflow.php | 46 +++++++++++++------ .../PhabricatorMetaMTAConfigOptions.php | 2 +- .../configuration_locked.diviner | 20 ++++++++ .../configuring_outbound_email.diviner | 34 ++++++++++++++ 4 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php index 9f50d2eeed..22b760872e 100644 --- a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php +++ b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php @@ -6,7 +6,9 @@ final class PhabricatorConfigManagementSetWorkflow protected function didConstruct() { $this ->setName('set') - ->setExamples('**set** __key__ __value__') + ->setExamples( + "**set** __key__ __value__\n". + "**set** __key__ --stdin < value.json") ->setSynopsis(pht('Set a local configuration value.')) ->setArguments( array( @@ -16,6 +18,10 @@ final class PhabricatorConfigManagementSetWorkflow 'Update configuration in the database instead of '. 'in local configuration.'), ), + array( + 'name' => 'stdin', + 'help' => pht('Read option value from stdin.'), + ), array( 'name' => 'args', 'wildcard' => true, @@ -31,22 +37,36 @@ final class PhabricatorConfigManagementSetWorkflow pht('Specify a configuration key and a value to set it to.')); } + $is_stdin = $args->getArg('stdin'); + $key = $argv[0]; - if (count($argv) == 1) { - throw new PhutilArgumentUsageException( - pht( - "Specify a value to set the key '%s' to.", - $key)); + if ($is_stdin) { + if (count($argv) > 1) { + throw new PhutilArgumentUsageException( + pht( + 'Too many arguments: expected only a key when using "--stdin".')); + } + + fprintf(STDERR, tsprintf("%s\n", pht('Reading value from stdin...'))); + $value = file_get_contents('php://stdin'); + } else { + if (count($argv) == 1) { + throw new PhutilArgumentUsageException( + pht( + "Specify a value to set the key '%s' to.", + $key)); + } + + if (count($argv) > 2) { + throw new PhutilArgumentUsageException( + pht( + 'Too many arguments: expected one key and one value.')); + } + + $value = $argv[1]; } - $value = $argv[1]; - - if (count($argv) > 2) { - throw new PhutilArgumentUsageException( - pht( - 'Too many arguments: expected one key and one value.')); - } $options = PhabricatorApplicationConfigOptions::loadAllOptions(); if (empty($options[$key])) { diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 43734abea0..0b916150bc 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -202,7 +202,7 @@ EODOC return array( $this->newOption('cluster.mailers', 'cluster.mailers', null) - ->setLocked(true) + ->setHidden(true) ->setDescription($mailers_description), $this->newOption( 'metamta.default-address', diff --git a/src/docs/user/configuration/configuration_locked.diviner b/src/docs/user/configuration/configuration_locked.diviner index 040b838177..fff0da9bdc 100644 --- a/src/docs/user/configuration/configuration_locked.diviner +++ b/src/docs/user/configuration/configuration_locked.diviner @@ -27,6 +27,24 @@ can edit it from the CLI instead, with `bin/config`: phabricator/ $ ./bin/config set ``` +Some configuration options take complicated values which can be difficult +to escape properly for the shell. The easiest way to set these options is +to use the `--stdin` flag. First, put your desired value in a `config.json` +file: + +```name=config.json, lang=json +{ + "duck": "quack", + "cow": "moo" +} +``` + +Then, set it with `--stdin` like this: + +``` +phabricator/ $ ./bin/config set --stdin < config.json +``` + A few settings have alternate CLI tools. Refer to the setting page for details. @@ -98,4 +116,6 @@ Next Steps Continue by: + - learning more about advanced options with + @{Configuration User Guide: Advanced Configuration}; or - returning to the @{article: Configuration Guide}. diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 0c7ef529fa..21abf92736 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -101,6 +101,40 @@ Once you've selected a mailer, find the corresponding section below for instructions on configuring it. +Setting Complex Configuration +============================= + +Mailers can not be edited from the web UI. If mailers could be edited from +the web UI, it would give an attacker who compromised an administrator account +a lot of power: they could redirect mail to a server they control and then +intercept mail for any other account, including password reset mail. + +For more information about locked configuration options, see +@{article:Configuration Guide: Locked and Hidden Configuration}. + +Setting `cluster.mailers` from the command line using `bin/config set` can be +tricky because of shell escaping. The easiest way to do it is to use the +`--stdin` flag. First, put your desired configuration in a file like this: + +```lang=json, name=mailers.json +[ + { + "key": "test-mailer", + "type": "test" + } +] +``` + +Then set the value like this: + +``` +phabricator/ $ ./bin/config set --stdin < mailers.json +``` + +For alternatives and more information on configuration, see +@{article:Configuration User Guide: Advanced Configuration} + + Mailer: Mailgun =============== From 994d2e8e156323e0684dac897e65bc9ba6db42e8 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:31:37 -0800 Subject: [PATCH 23/67] Use "cluster.mailers" if it is configured Summary: Depends on D19004. Ref T13053. Ref T12677. If the new `cluster.mailers` is configured, make use of it. Also use it in the Sengrid/Mailgun inbound stuff. Also fix a bug where "Must Encrypt" mail to no recipients could fatal because no `$mail` was returned. Test Plan: Processed some mail locally. The testing on this is still pretty flimsy, but I plan to solidify it in an upcoming change. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19005 --- .../PhabricatorMailImplementationAdapter.php | 10 +++ ...ricatorMetaMTAMailgunReceiveController.php | 23 +++++- ...icatorMetaMTASendGridReceiveController.php | 20 +++++ .../storage/PhabricatorMetaMTAMail.php | 82 ++++++++++++++++--- ...habricatorApplicationTransactionEditor.php | 11 +-- 5 files changed, 126 insertions(+), 20 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 923574d16d..14c7a63663 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -3,6 +3,7 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { private $key; + private $priority; private $options = array(); final public function getAdapterType() { @@ -57,6 +58,15 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { return $this->key; } + final public function setPriority($priority) { + $this->priority = $priority; + return $this; + } + + final public function getPriority() { + return $this->priority; + } + final public function getOption($key) { if (!array_key_exists($key, $this->options)) { throw new Exception( diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php index 467995a186..4eb53b7120 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php @@ -8,14 +8,31 @@ final class PhabricatorMetaMTAMailgunReceiveController } private function verifyMessage() { - $api_key = PhabricatorEnv::getEnvConfig('mailgun.api-key'); $request = $this->getRequest(); $timestamp = $request->getStr('timestamp'); $token = $request->getStr('token'); $sig = $request->getStr('signature'); - $hash = hash_hmac('sha256', $timestamp.$token, $api_key); - return phutil_hashes_are_identical($sig, $hash); + // An install may configure multiple Mailgun mailers, and we might receive + // inbound mail from any of them. Test the signature to see if it matches + // any configured Mailgun mailer. + + $mailers = PhabricatorMetaMTAMail::newMailers(); + $mailgun_type = PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE; + foreach ($mailers as $mailer) { + if ($mailer->getAdapterType() != $mailgun_type) { + continue; + } + + $api_key = $mailer->getOption('api-key'); + + $hash = hash_hmac('sha256', $timestamp.$token, $api_key); + if (phutil_hashes_are_identical($sig, $hash)) { + return true; + } + } + + return false; } public function handleRequest(AphrontRequest $request) { diff --git a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php index 0a5e28fcee..99e60caa05 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php @@ -8,6 +8,26 @@ final class PhabricatorMetaMTASendGridReceiveController } public function handleRequest(AphrontRequest $request) { + $mailers = PhabricatorMetaMTAMail::newMailers(); + $sendgrid_type = PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE; + + // SendGrid doesn't sign payloads so we can't be sure that SendGrid + // actually sent this request, but require a configured SendGrid mailer + // before we activate this endpoint. + + $has_sendgrid = false; + foreach ($mailers as $mailer) { + if ($mailer->getAdapterType() != $sendgrid_type) { + continue; + } + + $has_sendgrid = true; + break; + } + + if (!$has_sendgrid) { + return new Aphront404Response(); + } // No CSRF for SendGrid. $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index eb1f1fbea2..2ab3299c34 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -463,33 +463,85 @@ final class PhabricatorMetaMTAMail throw new Exception(pht('Trying to send an already-sent mail!')); } - $mailers = $this->newMailers(); + $mailers = self::newMailers(); return $this->sendWithMailers($mailers); } - private function newMailers() { + public static function newMailers() { $mailers = array(); - $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); + $config = PhabricatorEnv::getEnvConfig('cluster.mailers'); + if ($config === null) { + $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); - $defaults = $mailer->newDefaultOptions(); - $options = $mailer->newLegacyOptions(); + $defaults = $mailer->newDefaultOptions(); + $options = $mailer->newLegacyOptions(); - $options = $options + $defaults; + $options = $options + $defaults; - $mailer - ->setKey('default') - ->setOptions($options); + $mailer + ->setKey('default') + ->setPriority(-1) + ->setOptions($options); - $mailer->prepareForSend(); + $mailers[] = $mailer; + } else { + $adapters = PhabricatorMailImplementationAdapter::getAllAdapters(); + $next_priority = -1; - $mailers[] = $mailer; + foreach ($config as $spec) { + $type = $spec['type']; + if (!isset($adapters[$type])) { + throw new Exception( + pht( + 'Unknown mailer ("%s")!', + $type)); + } - return $mailers; + $key = $spec['key']; + $mailer = id(clone $adapters[$type]) + ->setKey($key); + + $priority = idx($spec, 'priority'); + if (!$priority) { + $priority = $next_priority; + $next_priority--; + } + $mailer->setPriority($priority); + + $defaults = $mailer->newDefaultOptions(); + $options = idx($spec, 'options', array()) + $defaults; + $mailer->setOptions($options); + } + } + + $sorted = array(); + $groups = mgroup($mailers, 'getPriority'); + ksort($groups); + foreach ($groups as $group) { + // Reorder services within the same priority group randomly. + shuffle($group); + foreach ($group as $mailer) { + $sorted[] = $mailer; + } + } + + foreach ($sorted as $mailer) { + $mailer->prepareForSend(); + } + + return $sorted; } public function sendWithMailers(array $mailers) { + if (!$mailers) { + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID) + ->setMessage(pht('No mailers are configured.')) + ->save(); + } + $exceptions = array(); foreach ($mailers as $template_mailer) { $mailer = null; @@ -865,6 +917,12 @@ final class PhabricatorMetaMTAMail $mailer->addCCs($add_cc); } + // Keep track of which mailer actually ended up accepting the message. + $mailer_key = $mailer->getKey(); + if ($mailer_key !== null) { + $this->setParam('mailer.key', $mailer_key); + } + return $mailer; } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index c5390de362..e4f9607801 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -2575,12 +2575,13 @@ abstract class PhabricatorApplicationTransactionEditor $mail = $this->buildMailForTarget($object, $xactions, $target); - if ($this->mustEncrypt) { - $mail - ->setMustEncrypt(true) - ->setMustEncryptReasons($this->mustEncrypt); + if ($mail) { + if ($this->mustEncrypt) { + $mail + ->setMustEncrypt(true) + ->setMustEncryptReasons($this->mustEncrypt); + } } - } catch (Exception $ex) { $caught = $ex; } From 9947eee182aa9fe04e926098eb00dfd1a758e185 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:56:44 -0800 Subject: [PATCH 24/67] Add some test coverage for mailers configuration Summary: Depends on D19005. Ref T12677. Ref T13053. Tests that turning `cluster.mailers` into an actual list of mailers more or less works as expected. Test Plan: Ran unit tests. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19006 --- src/__phutil_library_map__.php | 2 + .../storage/PhabricatorMetaMTAMail.php | 4 +- .../PhabricatorMailConfigTestCase.php | 131 ++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index fb2b0ccf0c..29a97c43d2 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3178,6 +3178,7 @@ phutil_register_library_map(array( 'PhabricatorMacroTransactionQuery' => 'applications/macro/query/PhabricatorMacroTransactionQuery.php', 'PhabricatorMacroTransactionType' => 'applications/macro/xaction/PhabricatorMacroTransactionType.php', 'PhabricatorMacroViewController' => 'applications/macro/controller/PhabricatorMacroViewController.php', + 'PhabricatorMailConfigTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php', 'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php', 'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php', 'PhabricatorMailEmailSubjectHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailSubjectHeraldField.php', @@ -8680,6 +8681,7 @@ phutil_register_library_map(array( 'PhabricatorMacroTransactionQuery' => 'PhabricatorApplicationTransactionQuery', 'PhabricatorMacroTransactionType' => 'PhabricatorModularTransactionType', 'PhabricatorMacroViewController' => 'PhabricatorMacroController', + 'PhabricatorMailConfigTestCase' => 'PhabricatorTestCase', 'PhabricatorMailEmailHeraldField' => 'HeraldField', 'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup', 'PhabricatorMailEmailSubjectHeraldField' => 'PhabricatorMailEmailHeraldField', diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 2ab3299c34..141d3b5e1c 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -513,12 +513,14 @@ final class PhabricatorMetaMTAMail $defaults = $mailer->newDefaultOptions(); $options = idx($spec, 'options', array()) + $defaults; $mailer->setOptions($options); + + $mailers[] = $mailer; } } $sorted = array(); $groups = mgroup($mailers, 'getPriority'); - ksort($groups); + krsort($groups); foreach ($groups as $group) { // Reorder services within the same priority group randomly. shuffle($group); diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php new file mode 100644 index 0000000000..25984a2c1d --- /dev/null +++ b/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php @@ -0,0 +1,131 @@ +newMailersWithConfig( + array( + array( + 'key' => 'A', + 'type' => 'test', + ), + array( + 'key' => 'B', + 'type' => 'test', + ), + )); + + $this->assertEqual( + array('A', 'B'), + mpull($mailers, 'getKey')); + + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A', + 'priority' => 1, + 'type' => 'test', + ), + array( + 'key' => 'B', + 'priority' => 2, + 'type' => 'test', + ), + )); + + $this->assertEqual( + array('B', 'A'), + mpull($mailers, 'getKey')); + + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A1', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'A2', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'B', + 'type' => 'test', + ), + array( + 'key' => 'C', + 'priority' => 400, + 'type' => 'test', + ), + array( + 'key' => 'D', + 'type' => 'test', + ), + )); + + // The "A" servers should be shuffled randomly, so either outcome is + // acceptable. + $option_1 = array('C', 'A1', 'A2', 'B', 'D'); + $option_2 = array('C', 'A2', 'A1', 'B', 'D'); + $actual = mpull($mailers, 'getKey'); + + $this->assertTrue(($actual === $option_1) || ($actual === $option_2)); + + // Make sure that when we're load balancing we actually send traffic to + // both servers reasonably often. + $saw_a1 = false; + $saw_a2 = false; + $attempts = 0; + while (true) { + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A1', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'A2', + 'priority' => 300, + 'type' => 'test', + ), + )); + + $first_key = head($mailers)->getKey(); + + if ($first_key == 'A1') { + $saw_a1 = true; + } + + if ($first_key == 'A2') { + $saw_a2 = true; + } + + if ($saw_a1 && $saw_a2) { + break; + } + + if ($attempts++ > 1024) { + throw new Exception( + pht( + 'Load balancing between two mail servers did not select both '. + 'servers after an absurd number of attempts.')); + } + } + + $this->assertTrue($saw_a1 && $saw_a2); + } + + private function newMailersWithConfig(array $config) { + $env = PhabricatorEnv::beginScopedEnv(); + $env->overrideEnvConfig('cluster.mailers', $config); + + $mailers = PhabricatorMetaMTAMail::newMailers(); + + unset($env); + return $mailers; + } + +} From 1f53aa27e4596754cd347c3a74fab3876c66f4c8 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 08:24:00 -0800 Subject: [PATCH 25/67] Add unit tests for mail failover behaviors when multiple mailers are configured Summary: Depends on D19006. Ref T13053. Ref T12677. When multiple mailers are configured but one or more fail, test that we recover (or don't) appropriately. Test Plan: Ran unit tests. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19007 --- .../storage/PhabricatorMetaMTAMail.php | 16 ++-- .../PhabricatorMetaMTAMailTestCase.php | 78 +++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 141d3b5e1c..5f859fd1d9 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -315,6 +315,10 @@ final class PhabricatorMetaMTAMail return $this->getParam('stampMetadata', array()); } + public function getMailerKey() { + return $this->getParam('mailer.key'); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -588,6 +592,12 @@ final class PhabricatorMetaMTAMail continue; } + // Keep track of which mailer actually ended up accepting the message. + $mailer_key = $mailer->getKey(); + if ($mailer_key !== null) { + $this->setParam('mailer.key', $mailer_key); + } + return $this ->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT) ->save(); @@ -919,12 +929,6 @@ final class PhabricatorMetaMTAMail $mailer->addCCs($add_cc); } - // Keep track of which mailer actually ended up accepting the message. - $mailer_key = $mailer->getKey(); - if ($mailer_key !== null) { - $this->setParam('mailer.key', $mailer_key); - } - return $mailer; } diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 6e72b129b1..c0045301fd 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -253,4 +253,82 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { ->executeOne(); } + public function testMailerFailover() { + $user = $this->generateNewTestUser(); + $phid = $user->getPHID(); + + $status_sent = PhabricatorMailOutboundStatus::STATUS_SENT; + $status_queue = PhabricatorMailOutboundStatus::STATUS_QUEUE; + $status_fail = PhabricatorMailOutboundStatus::STATUS_FAIL; + + $mailer1 = id(new PhabricatorMailImplementationTestAdapter()) + ->setKey('mailer1'); + + $mailer2 = id(new PhabricatorMailImplementationTestAdapter()) + ->setKey('mailer2'); + + $mailers = array( + $mailer1, + $mailer2, + ); + + // Send mail with both mailers active. The first mailer should be used. + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)) + ->sendWithMailers($mailers); + $this->assertEqual($status_sent, $mail->getStatus()); + $this->assertEqual('mailer1', $mail->getMailerKey()); + + + // If the first mailer fails, the mail should be sent with the second + // mailer. Since we transmitted the mail, this doesn't raise an exception. + $mailer1->setFailTemporarily(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)) + ->sendWithMailers($mailers); + $this->assertEqual($status_sent, $mail->getStatus()); + $this->assertEqual('mailer2', $mail->getMailerKey()); + + + // If both mailers fail, the mail should remain in queue. + $mailer2->setFailTemporarily(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)); + + $caught = null; + try { + $mail->sendWithMailers($mailers); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue($caught instanceof Exception); + $this->assertEqual($status_queue, $mail->getStatus()); + $this->assertEqual(null, $mail->getMailerKey()); + + $mailer1->setFailTemporarily(false); + $mailer2->setFailTemporarily(false); + + + // If the first mailer fails permanently, the mail should fail even though + // the second mailer isn't configured to fail. + $mailer1->setFailPermanently(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)); + + $caught = null; + try { + $mail->sendWithMailers($mailers); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue($caught instanceof Exception); + $this->assertEqual($status_fail, $mail->getStatus()); + $this->assertEqual(null, $mail->getMailerKey()); + } + } From 19b3fb8863d6b72dbb08d904069dc9a914f95c69 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 09:29:40 -0800 Subject: [PATCH 26/67] Add a Postmark mail adapter so it can be configured as an outbound mailer Summary: Depends on D19007. Ref T12677. Test Plan: Used `bin/mail send-test ... --mailer postmark` to deliver some mail via Postmark. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19009 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMailImplementationAdapter.php | 9 ++ ...catorMailImplementationPostmarkAdapter.php | 112 ++++++++++++++++++ ...bricatorMailManagementSendTestWorkflow.php | 20 ++++ .../storage/PhabricatorMetaMTAMail.php | 10 ++ .../configuring_outbound_email.diviner | 12 ++ 6 files changed, 165 insertions(+) create mode 100644 src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 29a97c43d2..16b3e1257a 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3188,6 +3188,7 @@ phutil_register_library_map(array( 'PhabricatorMailImplementationMailgunAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php', 'PhabricatorMailImplementationPHPMailerAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php', 'PhabricatorMailImplementationPHPMailerLiteAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php', + 'PhabricatorMailImplementationPostmarkAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php', 'PhabricatorMailImplementationSendGridAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php', 'PhabricatorMailImplementationTestAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php', 'PhabricatorMailManagementListInboundWorkflow' => 'applications/metamta/management/PhabricatorMailManagementListInboundWorkflow.php', @@ -8691,6 +8692,7 @@ phutil_register_library_map(array( 'PhabricatorMailImplementationMailgunAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationPHPMailerAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationPHPMailerLiteAdapter' => 'PhabricatorMailImplementationAdapter', + 'PhabricatorMailImplementationPostmarkAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationSendGridAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationTestAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailManagementListInboundWorkflow' => 'PhabricatorMailManagementWorkflow', diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 14c7a63663..ce56345194 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -94,4 +94,13 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { return; } + protected function renderAddress($email, $name = null) { + if (strlen($name)) { + // TODO: This needs to be escaped correctly. + return "{$name} <{$email}>"; + } else { + return $email; + } + } + } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php new file mode 100644 index 0000000000..bd5ee820af --- /dev/null +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php @@ -0,0 +1,112 @@ +parameters['From'] = $this->renderAddress($email, $name); + return $this; + } + + public function addReplyTo($email, $name = '') { + $this->parameters['ReplyTo'] = $this->renderAddress($email, $name); + return $this; + } + + public function addTos(array $emails) { + foreach ($emails as $email) { + $this->parameters['To'][] = $email; + } + return $this; + } + + public function addCCs(array $emails) { + foreach ($emails as $email) { + $this->parameters['Cc'][] = $email; + } + return $this; + } + + public function addAttachment($data, $filename, $mimetype) { + $this->parameters['Attachments'][] = array( + 'Name' => $filename, + 'ContentType' => $mimetype, + 'Content' => base64_encode($data), + ); + + return $this; + } + + public function addHeader($header_name, $header_value) { + $this->parameters['Headers'][] = array( + 'Name' => $header_name, + 'Value' => $header_value, + ); + return $this; + } + + public function setBody($body) { + $this->parameters['TextBody'] = $body; + return $this; + } + + public function setHTMLBody($html_body) { + $this->parameters['HtmlBody'] = $html_body; + return $this; + } + + public function setSubject($subject) { + $this->parameters['Subject'] = $subject; + return $this; + } + + public function supportsMessageIDHeader() { + return true; + } + + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'access-token' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'access-token' => null, + ); + } + + public function newLegacyOptions() { + return array(); + } + + public function send() { + $access_token = $this->getOption('access-token'); + + $parameters = $this->parameters; + $flatten = array( + 'To', + 'Cc', + ); + + foreach ($flatten as $key) { + if (isset($parameters[$key])) { + $parameters[$key] = implode(', ', $parameters[$key]); + } + } + + id(new PhutilPostmarkFuture()) + ->setAccessToken($access_token) + ->setMethod('email', $parameters) + ->resolve(); + + return true; + } + +} diff --git a/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php index 9f4e91ca22..152b62fd3f 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php @@ -47,6 +47,11 @@ final class PhabricatorMailManagementSendTestWorkflow 'help' => pht('Attach a file.'), 'repeat' => true, ), + array( + 'name' => 'mailer', + 'param' => 'key', + 'help' => pht('Send with a specific configured mailer.'), + ), array( 'name' => 'html', 'help' => pht('Send as HTML mail.'), @@ -161,6 +166,21 @@ final class PhabricatorMailManagementSendTestWorkflow $mail->setFrom($from->getPHID()); } + $mailer_key = $args->getArg('mailer'); + if ($mailer_key !== null) { + $mailers = PhabricatorMetaMTAMail::newMailers(); + $mailers = mpull($mailers, null, 'getKey'); + if (!isset($mailers[$mailer_key])) { + throw new PhutilArgumentUsageException( + pht( + 'Mailer key ("%s") is not configured. Available keys are: %s.', + $mailer_key, + implode(', ', array_keys($mailers)))); + } + + $mail->setTryMailers(array($mailer_key)); + } + foreach ($attach as $attachment) { $data = Filesystem::readFile($attachment); $name = basename($attachment); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 5f859fd1d9..83cdc7c40f 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -319,6 +319,10 @@ final class PhabricatorMetaMTAMail return $this->getParam('mailer.key'); } + public function setTryMailers(array $mailers) { + return $this->setParam('mailers.try', $mailers); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -469,6 +473,12 @@ final class PhabricatorMetaMTAMail $mailers = self::newMailers(); + $try_mailers = $this->getParam('mailers.try'); + if ($try_mailers) { + $mailers = mpull($mailers, null, 'getKey'); + $mailers = array_select_keys($mailers, $try_mailers); + } + return $this->sendWithMailers($mailers); } diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 21abf92736..d2daf7a40a 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -12,6 +12,7 @@ including a local mailer or various third-party services. Options include: | Send Mail With | Setup | Cost | Inbound | Notes | |---------|-------|------|---------|-------| | Mailgun | Easy | Cheap | Yes | Recommended | +| Postmark | Easy | Cheap | Yes | Recommended | | Amazon SES | Easy | Cheap | No | Recommended | | SendGrid | Medium | Cheap | Yes | Discouraged | | External SMTP | Medium | Varies | No | Gmail, etc. | @@ -147,6 +148,17 @@ To use this mailer, set `type` to `mailgun`, then configure these `options`: - `domain`: Required string. Your Mailgun domain. +Mailer: Postmark +================ + +Postmark is a third-party email delivery serivice. You can learn more at +. + +To use this mailer, set `type` to `postmark`, then configure these `options`: + + - `access-token`: Required string. Your Postmark access token. + + Mailer: Amazon SES ================== From f090fa7426e42bc085e6be25c2eead0af73eb1b4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 02:57:28 -0800 Subject: [PATCH 27/67] Use object PHIDs for "Thread-Topic" headers in mail Summary: Depends on D19009. Ref T13053. For "Must Encrypt" mail, we must currently strip the "Thread-Topic" header because it sometimes contains sensitive information about the object. I don't actually know if this header is useful or anyting uses it. My understanding is that it's an Outlook/Exchange thing, but we also implement "Thread-Index" which I think is what Outlook/Exchange actually look at. This header may have done something before we implemented "Thread-Index", or maybe never done anything. Or maybe older versions of Excel/Outlook did something with it and newer versions don't, or do less. So it's possible that an even better fix here would be to simply remove this, but I wasn't able to convince myself of that after Googling for 10 minutes and I don't think it's worth hours of installing Exchange/Outlook to figure out. Instead, I'm just trying to simplify our handling of this header for now, and maybe some day we'll learn more about Exchange/Outlook and can remove it. In a number of cases we already use the object monogram or PHID as a "Thread-Topic" without users ever complaining, so I think that if this header is useful it probably isn't shown to users, or isn't shown very often (e.g., only in a specific "conversation" sub-view?). Just use the object PHID (which should be unique and stable) as a thread-topic, everywhere, automatically. Then allow this header through for "Must Encrypt" mail. Test Plan: Processed some local mail, saw object PHIDs for "Thread-Topic" headers. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19012 --- src/applications/audit/editor/PhabricatorAuditEditor.php | 5 +---- .../auth/editor/PhabricatorAuthSSHKeyEditor.php | 4 +--- src/applications/badges/editor/PhabricatorBadgesEditor.php | 4 +--- .../calendar/editor/PhabricatorCalendarEventEditor.php | 4 +--- src/applications/conpherence/editor/ConpherenceEditor.php | 4 +--- .../countdown/editor/PhabricatorCountdownEditor.php | 3 +-- .../differential/editor/DifferentialTransactionEditor.php | 7 +------ src/applications/files/editor/PhabricatorFileEditor.php | 3 +-- src/applications/fund/editor/FundInitiativeEditor.php | 3 +-- .../legalpad/editor/LegalpadDocumentEditor.php | 4 +--- src/applications/macro/editor/PhabricatorMacroEditor.php | 3 +-- .../maniphest/editor/ManiphestTransactionEditor.php | 3 +-- .../metamta/storage/PhabricatorMetaMTAMail.php | 6 ++++++ .../editor/PhabricatorOwnersPackageTransactionEditor.php | 4 +--- src/applications/paste/editor/PhabricatorPasteEditor.php | 3 +-- src/applications/phame/editor/PhameBlogEditor.php | 4 +--- src/applications/phame/editor/PhamePostEditor.php | 4 +--- src/applications/pholio/editor/PholioMockEditor.php | 4 +--- src/applications/phortune/editor/PhortuneCartEditor.php | 3 +-- .../phriction/editor/PhrictionTransactionEditor.php | 4 +--- .../phurl/editor/PhabricatorPhurlURLEditor.php | 3 +-- src/applications/ponder/editor/PonderAnswerEditor.php | 3 +-- src/applications/ponder/editor/PonderQuestionEditor.php | 4 +--- .../project/editor/PhabricatorProjectTransactionEditor.php | 4 +--- .../releeph/editor/ReleephRequestTransactionalEditor.php | 4 +--- .../worker/PhabricatorRepositoryPushMailWorker.php | 1 - .../slowvote/editor/PhabricatorSlowvoteEditor.php | 3 +-- 27 files changed, 31 insertions(+), 70 deletions(-) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index 049733f777..c39be75366 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -473,17 +473,14 @@ final class PhabricatorAuditEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $identifier = $object->getCommitIdentifier(); $repository = $object->getRepository(); - $monogram = $repository->getMonogram(); $summary = $object->getSummary(); $name = $repository->formatCommitName($identifier); $subject = "{$name}: {$summary}"; - $thread_topic = "Commit {$monogram}{$identifier}"; $template = id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $thread_topic); + ->setSubject($subject); $this->attachPatch( $template, diff --git a/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php b/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php index 569c37403b..3f178c9855 100644 --- a/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php +++ b/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php @@ -255,11 +255,9 @@ final class PhabricatorAuthSSHKeyEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $name = $object->getName(); - $phid = $object->getPHID(); $mail = id(new PhabricatorMetaMTAMail()) - ->setSubject(pht('SSH Key %d: %s', $id, $name)) - ->addHeader('Thread-Topic', $phid); + ->setSubject(pht('SSH Key %d: %s', $id, $name)); // The primary value of this mail is alerting users to account compromises, // so force delivery. In particular, this mail should still be delivered diff --git a/src/applications/badges/editor/PhabricatorBadgesEditor.php b/src/applications/badges/editor/PhabricatorBadgesEditor.php index fddc55747c..785d8c989b 100644 --- a/src/applications/badges/editor/PhabricatorBadgesEditor.php +++ b/src/applications/badges/editor/PhabricatorBadgesEditor.php @@ -87,12 +87,10 @@ final class PhabricatorBadgesEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $name = $object->getName(); $id = $object->getID(); - $topic = pht('Badge %d', $id); $subject = pht('Badge %d: %s', $id, $name); return id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $topic); + ->setSubject($subject); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php b/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php index 4ab13fd360..f1b72dc0ea 100644 --- a/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php +++ b/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php @@ -309,13 +309,11 @@ final class PhabricatorCalendarEventEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); $monogram = $object->getMonogram(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/conpherence/editor/ConpherenceEditor.php b/src/applications/conpherence/editor/ConpherenceEditor.php index 29ffc22251..7896055f64 100644 --- a/src/applications/conpherence/editor/ConpherenceEditor.php +++ b/src/applications/conpherence/editor/ConpherenceEditor.php @@ -227,11 +227,9 @@ final class ConpherenceEditor extends PhabricatorApplicationTransactionEditor { '%s sent you a message.', $this->getActor()->getUserName()); } - $phid = $object->getPHID(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("Z{$id}: {$title}") - ->addHeader('Thread-Topic', "Z{$id}: {$phid}"); + ->setSubject("Z{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/countdown/editor/PhabricatorCountdownEditor.php b/src/applications/countdown/editor/PhabricatorCountdownEditor.php index 322b2ee2c3..2102b3785b 100644 --- a/src/applications/countdown/editor/PhabricatorCountdownEditor.php +++ b/src/applications/countdown/editor/PhabricatorCountdownEditor.php @@ -45,8 +45,7 @@ final class PhabricatorCountdownEditor $name = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index 3a1537b01d..f3583438c8 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -689,15 +689,10 @@ final class DifferentialTransactionEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $title = $object->getTitle(); - - $original_title = $object->getOriginalTitle(); - $subject = "D{$id}: {$title}"; - $thread_topic = "D{$id}: {$original_title}"; return id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $thread_topic); + ->setSubject($subject); } protected function getTransactionsForMail( diff --git a/src/applications/files/editor/PhabricatorFileEditor.php b/src/applications/files/editor/PhabricatorFileEditor.php index 6a2b797b40..db974cec65 100644 --- a/src/applications/files/editor/PhabricatorFileEditor.php +++ b/src/applications/files/editor/PhabricatorFileEditor.php @@ -47,8 +47,7 @@ final class PhabricatorFileEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("F{$id}: {$name}") - ->addHeader('Thread-Topic', "F{$id}"); + ->setSubject("F{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/fund/editor/FundInitiativeEditor.php b/src/applications/fund/editor/FundInitiativeEditor.php index e5c372fd12..9175156ffd 100644 --- a/src/applications/fund/editor/FundInitiativeEditor.php +++ b/src/applications/fund/editor/FundInitiativeEditor.php @@ -50,8 +50,7 @@ final class FundInitiativeEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/legalpad/editor/LegalpadDocumentEditor.php b/src/applications/legalpad/editor/LegalpadDocumentEditor.php index 35f2487a81..e4b43186ee 100644 --- a/src/applications/legalpad/editor/LegalpadDocumentEditor.php +++ b/src/applications/legalpad/editor/LegalpadDocumentEditor.php @@ -124,12 +124,10 @@ final class LegalpadDocumentEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); - $phid = $object->getPHID(); $title = $object->getDocumentBody()->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("L{$id}: {$title}") - ->addHeader('Thread-Topic', "L{$id}: {$phid}"); + ->setSubject("L{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/macro/editor/PhabricatorMacroEditor.php b/src/applications/macro/editor/PhabricatorMacroEditor.php index 5d28b78f5f..f59c29b426 100644 --- a/src/applications/macro/editor/PhabricatorMacroEditor.php +++ b/src/applications/macro/editor/PhabricatorMacroEditor.php @@ -35,8 +35,7 @@ final class PhabricatorMacroEditor $name = 'Image Macro "'.$name.'"'; return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $name); + ->setSubject($name); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/maniphest/editor/ManiphestTransactionEditor.php b/src/applications/maniphest/editor/ManiphestTransactionEditor.php index caf70b8f3c..9c8e3869dc 100644 --- a/src/applications/maniphest/editor/ManiphestTransactionEditor.php +++ b/src/applications/maniphest/editor/ManiphestTransactionEditor.php @@ -206,8 +206,7 @@ final class ManiphestTransactionEditor $title = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("T{$id}: {$title}") - ->addHeader('Thread-Topic', "T{$id}: ".$object->getOriginalTitle()); + ->setSubject("T{$id}: {$title}"); } protected function buildMailBody( diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 83cdc7c40f..f2c8939132 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -1262,6 +1262,11 @@ final class PhabricatorMetaMTAMail $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes'); } + $related_phid = $this->getRelatedPHID(); + if ($related_phid) { + $headers[] = array('Thread-Topic', $related_phid); + } + return $headers; } @@ -1309,6 +1314,7 @@ final class PhabricatorMetaMTAMail 'Precedence', 'References', 'Thread-Index', + 'Thread-Topic', 'X-Mail-Transport-Agent', 'X-Auto-Response-Suppress', diff --git a/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php b/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php index 40657abd57..c6aad6e2bd 100644 --- a/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php +++ b/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php @@ -46,12 +46,10 @@ final class PhabricatorOwnersPackageTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $object->getPHID()); + ->setSubject($name); } protected function buildMailBody( diff --git a/src/applications/paste/editor/PhabricatorPasteEditor.php b/src/applications/paste/editor/PhabricatorPasteEditor.php index 063b72cfc0..c312915727 100644 --- a/src/applications/paste/editor/PhabricatorPasteEditor.php +++ b/src/applications/paste/editor/PhabricatorPasteEditor.php @@ -72,8 +72,7 @@ final class PhabricatorPasteEditor $name = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("P{$id}: {$name}") - ->addHeader('Thread-Topic', "P{$id}"); + ->setSubject("P{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/phame/editor/PhameBlogEditor.php b/src/applications/phame/editor/PhameBlogEditor.php index f30a74065e..c122d8fa3b 100644 --- a/src/applications/phame/editor/PhameBlogEditor.php +++ b/src/applications/phame/editor/PhameBlogEditor.php @@ -48,12 +48,10 @@ final class PhameBlogEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $phid = $object->getPHID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $phid); + ->setSubject($name); } protected function buildReplyHandler(PhabricatorLiskDAO $object) { diff --git a/src/applications/phame/editor/PhamePostEditor.php b/src/applications/phame/editor/PhamePostEditor.php index 488d7a4938..d95389e549 100644 --- a/src/applications/phame/editor/PhamePostEditor.php +++ b/src/applications/phame/editor/PhamePostEditor.php @@ -61,12 +61,10 @@ final class PhamePostEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $phid = $object->getPHID(); $title = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($title) - ->addHeader('Thread-Topic', $phid); + ->setSubject($title); } protected function buildReplyHandler(PhabricatorLiskDAO $object) { diff --git a/src/applications/pholio/editor/PholioMockEditor.php b/src/applications/pholio/editor/PholioMockEditor.php index c0fcf31f83..6bd49d2e7b 100644 --- a/src/applications/pholio/editor/PholioMockEditor.php +++ b/src/applications/pholio/editor/PholioMockEditor.php @@ -112,11 +112,9 @@ final class PholioMockEditor extends PhabricatorApplicationTransactionEditor { protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $name = $object->getName(); - $original_name = $object->getOriginalName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("M{$id}: {$name}") - ->addHeader('Thread-Topic', "M{$id}: {$original_name}"); + ->setSubject("M{$id}: {$name}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/phortune/editor/PhortuneCartEditor.php b/src/applications/phortune/editor/PhortuneCartEditor.php index 5196e12429..dcf1f2d0e0 100644 --- a/src/applications/phortune/editor/PhortuneCartEditor.php +++ b/src/applications/phortune/editor/PhortuneCartEditor.php @@ -123,8 +123,7 @@ final class PhortuneCartEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject(pht('Order %d: %s', $id, $name)) - ->addHeader('Thread-Topic', pht('Order %s', $id)); + ->setSubject(pht('Order %d: %s', $id, $name)); } protected function buildMailBody( diff --git a/src/applications/phriction/editor/PhrictionTransactionEditor.php b/src/applications/phriction/editor/PhrictionTransactionEditor.php index fcc9fe0474..73aee3fd4c 100644 --- a/src/applications/phriction/editor/PhrictionTransactionEditor.php +++ b/src/applications/phriction/editor/PhrictionTransactionEditor.php @@ -299,12 +299,10 @@ final class PhrictionTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $title = $object->getContent()->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($title) - ->addHeader('Thread-Topic', $object->getPHID()); + ->setSubject($title); } protected function buildMailBody( diff --git a/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php b/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php index 439d62f84a..49f290c343 100644 --- a/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php +++ b/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php @@ -68,8 +68,7 @@ final class PhabricatorPhurlURLEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("U{$id}: {$name}") - ->addHeader('Thread-Topic', "U{$id}: ".$object->getName()); + ->setSubject("U{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/ponder/editor/PonderAnswerEditor.php b/src/applications/ponder/editor/PonderAnswerEditor.php index bab0e1f72d..37b2fe2cd0 100644 --- a/src/applications/ponder/editor/PonderAnswerEditor.php +++ b/src/applications/ponder/editor/PonderAnswerEditor.php @@ -57,8 +57,7 @@ final class PonderAnswerEditor extends PonderEditor { $id = $object->getID(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("ANSR{$id}") - ->addHeader('Thread-Topic', "ANSR{$id}"); + ->setSubject("ANSR{$id}"); } protected function buildMailBody( diff --git a/src/applications/ponder/editor/PonderQuestionEditor.php b/src/applications/ponder/editor/PonderQuestionEditor.php index 0720f436b9..ba9687bd0d 100644 --- a/src/applications/ponder/editor/PonderQuestionEditor.php +++ b/src/applications/ponder/editor/PonderQuestionEditor.php @@ -146,11 +146,9 @@ final class PonderQuestionEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $title = $object->getTitle(); - $original_title = $object->getOriginalTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("Q{$id}: {$title}") - ->addHeader('Thread-Topic', "Q{$id}: {$original_title}"); + ->setSubject("Q{$id}: {$title}"); } protected function buildMailBody( diff --git a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php index 2764ce6322..de61c2a09b 100644 --- a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php +++ b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php @@ -219,12 +219,10 @@ final class PhabricatorProjectTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$name}") - ->addHeader('Thread-Topic', "Project {$id}"); + ->setSubject("{$name}"); } protected function buildMailBody( diff --git a/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php b/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php index 4710557043..da488d9c72 100644 --- a/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php +++ b/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php @@ -196,11 +196,9 @@ final class ReleephRequestTransactionalEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); - $phid = $object->getPHID(); $title = $object->getSummaryForDisplay(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("RQ{$id}: {$title}") - ->addHeader('Thread-Topic', "RQ{$id}: {$phid}"); + ->setSubject("RQ{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php index 17226a1377..5ffaf0a5c2 100644 --- a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php +++ b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php @@ -124,7 +124,6 @@ final class PhabricatorRepositoryPushMailWorker ->setFrom($event->getPusherPHID()) ->setBody($body->render()) ->setThreadID($event->getPHID(), $is_new = true) - ->addHeader('Thread-Topic', $subject) ->setIsBulk(true); return $target->willSendMail($mail); diff --git a/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php b/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php index 38dbfb12d6..cf088f37d4 100644 --- a/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php +++ b/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php @@ -48,8 +48,7 @@ final class PhabricatorSlowvoteEditor $name = $object->getQuestion(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( From aa74af19834ba96ccaa7bcd54a5f7fa14bcc97bc Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:29:33 -0800 Subject: [PATCH 28/67] Remove all "originalTitle"/"originalName" fields from objects Summary: Depends on D19012. Ref T13053. In D19012, I've changed "Thread-Topic" to always use PHIDs. This change drops the selective on-object storage we have to track the original, human-readable title for objects. Even if we end up backing out the "Thread-Topic" change, we'd be better off storing this in a table in the Mail app which just has ``, since then we get the right behavior without needing every object to have this separate field. Test Plan: Grepped for `original`, `originalName`, `originalTitle`, etc. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19013 --- resources/sql/autopatches/20180207.mail.01.task.sql | 2 ++ .../sql/autopatches/20180207.mail.02.revision.sql | 2 ++ resources/sql/autopatches/20180207.mail.03.mock.sql | 2 ++ .../differential/storage/DifferentialRevision.php | 10 ---------- .../maniphest/editor/ManiphestTransactionEditor.php | 1 - src/applications/maniphest/storage/ManiphestTask.php | 10 ---------- src/applications/pholio/storage/PholioMock.php | 2 -- .../pholio/xaction/PholioMockNameTransaction.php | 3 --- src/applications/ponder/storage/PonderQuestion.php | 5 ----- 9 files changed, 6 insertions(+), 31 deletions(-) create mode 100644 resources/sql/autopatches/20180207.mail.01.task.sql create mode 100644 resources/sql/autopatches/20180207.mail.02.revision.sql create mode 100644 resources/sql/autopatches/20180207.mail.03.mock.sql diff --git a/resources/sql/autopatches/20180207.mail.01.task.sql b/resources/sql/autopatches/20180207.mail.01.task.sql new file mode 100644 index 0000000000..f04b90c809 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.01.task.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + DROP originalTitle; diff --git a/resources/sql/autopatches/20180207.mail.02.revision.sql b/resources/sql/autopatches/20180207.mail.02.revision.sql new file mode 100644 index 0000000000..881efbcc94 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.02.revision.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_differential.differential_revision + DROP originalTitle; diff --git a/resources/sql/autopatches/20180207.mail.03.mock.sql b/resources/sql/autopatches/20180207.mail.03.mock.sql new file mode 100644 index 0000000000..360d7cf9a7 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.03.mock.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_pholio.pholio_mock + DROP originalName; diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 2c82de164a..938c588857 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -20,7 +20,6 @@ final class DifferentialRevision extends DifferentialDAO PhabricatorDraftInterface { protected $title = ''; - protected $originalTitle; protected $status; protected $summary = ''; @@ -98,7 +97,6 @@ final class DifferentialRevision extends DifferentialDAO ), self::CONFIG_COLUMN_SCHEMA => array( 'title' => 'text255', - 'originalTitle' => 'text255', 'status' => 'text32', 'summary' => 'text', 'testPlan' => 'text', @@ -155,14 +153,6 @@ final class DifferentialRevision extends DifferentialDAO return '/'.$this->getMonogram(); } - public function setTitle($title) { - $this->title = $title; - if (!$this->getID()) { - $this->originalTitle = $title; - } - return $this; - } - public function loadIDsByCommitPHIDs($phids) { if (!$phids) { return array(); diff --git a/src/applications/maniphest/editor/ManiphestTransactionEditor.php b/src/applications/maniphest/editor/ManiphestTransactionEditor.php index 9c8e3869dc..66247ca6d0 100644 --- a/src/applications/maniphest/editor/ManiphestTransactionEditor.php +++ b/src/applications/maniphest/editor/ManiphestTransactionEditor.php @@ -522,7 +522,6 @@ final class ManiphestTransactionEditor 'status' => '""', 'priority' => 0, 'title' => '""', - 'originalTitle' => '""', 'description' => '""', 'dateCreated' => 0, 'dateModified' => 0, diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index f72977c5b2..e19886d3ff 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -31,7 +31,6 @@ final class ManiphestTask extends ManiphestDAO protected $subpriority = 0; protected $title = ''; - protected $originalTitle = ''; protected $description = ''; protected $originalEmailSource; protected $mailKey; @@ -83,7 +82,6 @@ final class ManiphestTask extends ManiphestDAO 'status' => 'text64', 'priority' => 'uint32', 'title' => 'sort', - 'originalTitle' => 'text', 'description' => 'text', 'mailKey' => 'bytes20', 'ownerOrdering' => 'text64?', @@ -176,14 +174,6 @@ final class ManiphestTask extends ManiphestDAO return $this; } - public function setTitle($title) { - $this->title = $title; - if (!$this->getID()) { - $this->originalTitle = $title; - } - return $this; - } - public function getMonogram() { return 'T'.$this->getID(); } diff --git a/src/applications/pholio/storage/PholioMock.php b/src/applications/pholio/storage/PholioMock.php index 4aa9ef4055..523733b3df 100644 --- a/src/applications/pholio/storage/PholioMock.php +++ b/src/applications/pholio/storage/PholioMock.php @@ -25,7 +25,6 @@ final class PholioMock extends PholioDAO protected $editPolicy; protected $name; - protected $originalName; protected $description; protected $coverPHID; protected $mailKey; @@ -65,7 +64,6 @@ final class PholioMock extends PholioDAO self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'text128', 'description' => 'text', - 'originalName' => 'text128', 'mailKey' => 'bytes20', 'status' => 'text12', ), diff --git a/src/applications/pholio/xaction/PholioMockNameTransaction.php b/src/applications/pholio/xaction/PholioMockNameTransaction.php index d1231636af..82fb92fe40 100644 --- a/src/applications/pholio/xaction/PholioMockNameTransaction.php +++ b/src/applications/pholio/xaction/PholioMockNameTransaction.php @@ -15,9 +15,6 @@ final class PholioMockNameTransaction public function applyInternalEffects($object, $value) { $object->setName($value); - if ($object->getOriginalName() === null) { - $object->setOriginalName($this->getNewValue()); - } } public function getTitle() { diff --git a/src/applications/ponder/storage/PonderQuestion.php b/src/applications/ponder/storage/PonderQuestion.php index eefcdba9be..17f7ee3fdc 100644 --- a/src/applications/ponder/storage/PonderQuestion.php +++ b/src/applications/ponder/storage/PonderQuestion.php @@ -194,11 +194,6 @@ final class PonderQuestion extends PonderDAO return parent::save(); } - public function getOriginalTitle() { - // TODO: Make this actually save/return the original title. - return $this->getTitle(); - } - public function getFullTitle() { $id = $this->getID(); $title = $this->getTitle(); From 6e5df2dd714e8360251026eb112af9d2930ba789 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:49:16 -0800 Subject: [PATCH 29/67] Document that disabling "metamta.one-mail-per-recipient" leaks recipients for "Must Encrypt" Summary: Depends on D19013. Ref T13053. When mail is marked "Must Encrypt", we normally do not include recipient information. However, when `metamta.one-mail-per-recipient` is disabled, the recipient list will leak in the "To" and "Cc" headers. This interaction is probably not very surprising, but document it explicitly for completeness. (Also use "Mail messages" instead of "Mails".) Test Plan: Read documentation in the "Config" application. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19014 --- .../config/option/PhabricatorMetaMTAConfigOptions.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 0b916150bc..8a236a883b 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -66,7 +66,9 @@ of each approach are: received a similar message, but can not prevent all stray email arising from "Reply All". - Not supported with a private reply-to address. - - Mails are sent in the server default translation. + - Mail messages are sent in the server default translation. + - Mail that must be delivered over secure channels will leak the recipient + list in the "To" and "Cc" headers. - One mail to each user: - Policy controls work correctly and are enforced per-user. - Recipients need to look in the mail body to see To/Cc. @@ -77,7 +79,7 @@ of each approach are: - "Reply All" will never send extra mail to other users involved in the thread. - Required if private reply-to addresses are configured. - - Mails are sent in the language of user preference. + - Mail messages are sent in the language of user preference. EODOC )); From 085221b0d6f6150047cd38ad5174822fa8f279af Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:56:29 -0800 Subject: [PATCH 30/67] In HTML mail, make the text for mail stamps in mail bodies smaller and lighter Summary: Depends on D19014. Ref T13053. Test Plan: Used `./bin/mail show-outbound --id --dump-html > out.html && open out.html` to look at HTML mail, saw smaller, lighter stamp text with better spacing. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19015 --- .../metamta/replyhandler/PhabricatorMailTarget.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index bbf17be3fd..5d8378e8af 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -77,7 +77,13 @@ final class PhabricatorMailTarget extends Phobject { $html = array(); $html[] = phutil_tag('strong', array(), pht('STAMPS')); $html[] = phutil_tag('br'); - $html[] = phutil_implode_html(' ', $stamps); + $html[] = phutil_tag( + 'span', + array( + 'style' => 'font-size: smaller; color: #92969D', + ), + phutil_implode_html(' ', $stamps)); + $html[] = phutil_tag('br'); $html[] = phutil_tag('br'); $html = phutil_tag('div', array(), $html); $html_body .= hsprintf('%s', $html); From 0986c7f6732e02d14f8c8f6a192595d90b9416c9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 04:05:44 -0800 Subject: [PATCH 31/67] Add a "View Object" button on the web mail view page Summary: Depends on D19015. Ref T13053. Currently, we don't link up hyperlinks in the body of mail viewed in the web UI. We should, but this is a little tricky (see T13053#235074). As a general improvement to make working with "Must Encrypt" mail less painful, add a big button to jump to the related object. Test Plan: {F5415990} Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19016 --- .../PhabricatorMetaMTAMailViewController.php | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 03d340bac9..9b33397831 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -75,8 +75,26 @@ final class PhabricatorMetaMTAMailViewController ->setKey('metadata') ->appendChild($this->buildMetadataProperties($mail))); + $header_view = id(new PHUIHeaderView()) + ->setHeader(pht('Mail')); + + $object_phid = $mail->getRelatedPHID(); + if ($object_phid) { + $handles = $viewer->loadHandles(array($object_phid)); + $handle = $handles[$object_phid]; + if ($handle->isComplete() && $handle->getURI()) { + $view_button = id(new PHUIButtonView()) + ->setTag('a') + ->setText(pht('View Object')) + ->setIcon('fa-chevron-right') + ->setHref($handle->getURI()); + + $header_view->addActionLink($view_button); + } + } + $object_box = id(new PHUIObjectBoxView()) - ->setHeaderText(pht('Mail')) + ->setHeader($header_view) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->addTabGroup($tab_group); From 5792032dc9c939112542ec32d0021e6cff9e0aea Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 04:51:09 -0800 Subject: [PATCH 32/67] Support Postmark inbound mail via webhook Summary: Depends on D19016. Ref T13053. Adds a listener for the Postmark webhook. Test Plan: Processed some test mail locally, at least: {F5416053} Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19017 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMetaMTAApplication.php | 1 + ...ricatorMetaMTAMailgunReceiveController.php | 11 +-- ...icatorMetaMTAPostmarkReceiveController.php | 87 +++++++++++++++++++ ...icatorMetaMTASendGridReceiveController.php | 20 ++--- .../storage/PhabricatorMetaMTAMail.php | 14 +++ .../configuring_inbound_email.diviner | 12 +++ 7 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 16b3e1257a..301459869f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3272,6 +3272,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMailgunReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php', 'PhabricatorMetaMTAMemberQuery' => 'applications/metamta/query/PhabricatorMetaMTAMemberQuery.php', 'PhabricatorMetaMTAPermanentFailureException' => 'applications/metamta/exception/PhabricatorMetaMTAPermanentFailureException.php', + 'PhabricatorMetaMTAPostmarkReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php', 'PhabricatorMetaMTAReceivedMail' => 'applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php', 'PhabricatorMetaMTAReceivedMailProcessingException' => 'applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php', 'PhabricatorMetaMTAReceivedMailTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php', @@ -8787,6 +8788,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMailgunReceiveController' => 'PhabricatorMetaMTAController', 'PhabricatorMetaMTAMemberQuery' => 'PhabricatorQuery', 'PhabricatorMetaMTAPermanentFailureException' => 'Exception', + 'PhabricatorMetaMTAPostmarkReceiveController' => 'PhabricatorMetaMTAController', 'PhabricatorMetaMTAReceivedMail' => 'PhabricatorMetaMTADAO', 'PhabricatorMetaMTAReceivedMailProcessingException' => 'Exception', 'PhabricatorMetaMTAReceivedMailTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/metamta/application/PhabricatorMetaMTAApplication.php b/src/applications/metamta/application/PhabricatorMetaMTAApplication.php index adb08aaa24..f53af55035 100644 --- a/src/applications/metamta/application/PhabricatorMetaMTAApplication.php +++ b/src/applications/metamta/application/PhabricatorMetaMTAApplication.php @@ -42,6 +42,7 @@ final class PhabricatorMetaMTAApplication extends PhabricatorApplication { 'detail/(?P[1-9]\d*)/' => 'PhabricatorMetaMTAMailViewController', 'sendgrid/' => 'PhabricatorMetaMTASendGridReceiveController', 'mailgun/' => 'PhabricatorMetaMTAMailgunReceiveController', + 'postmark/' => 'PhabricatorMetaMTAPostmarkReceiveController', ), ); } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php index 4eb53b7120..3ca2711dcf 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php @@ -17,15 +17,12 @@ final class PhabricatorMetaMTAMailgunReceiveController // inbound mail from any of them. Test the signature to see if it matches // any configured Mailgun mailer. - $mailers = PhabricatorMetaMTAMail::newMailers(); - $mailgun_type = PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE; + $mailers = PhabricatorMetaMTAMail::newMailersWithTypes( + array( + PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE, + )); foreach ($mailers as $mailer) { - if ($mailer->getAdapterType() != $mailgun_type) { - continue; - } - $api_key = $mailer->getOption('api-key'); - $hash = hash_hmac('sha256', $timestamp.$token, $api_key); if (phutil_hashes_are_identical($sig, $hash)) { return true; diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php new file mode 100644 index 0000000000..a54da6fb40 --- /dev/null +++ b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php @@ -0,0 +1,87 @@ + idx($data, 'To'), + 'from' => idx($data, 'From'), + 'cc' => idx($data, 'Cc'), + 'subject' => idx($data, 'Subject'), + ) + $raw_headers; + + + $received = id(new PhabricatorMetaMTAReceivedMail()) + ->setHeaders($headers) + ->setBodies( + array( + 'text' => idx($data, 'TextBody'), + 'html' => idx($data, 'HtmlBody'), + )); + + $file_phids = array(); + $attachments = idx($data, 'Attachments', array()); + foreach ($attachments as $attachment) { + $file_data = idx($attachment, 'Content'); + $file_data = base64_decode($file_data); + + try { + $file = PhabricatorFile::newFromFileData( + $file_data, + array( + 'name' => idx($attachment, 'Name'), + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + )); + $file_phids[] = $file->getPHID(); + } catch (Exception $ex) { + phlog($ex); + } + } + $received->setAttachments($file_phids); + + try { + $received->save(); + $received->processReceivedMail(); + } catch (Exception $ex) { + phlog($ex); + } + + return id(new AphrontWebpageResponse()) + ->setContent(pht("Got it! Thanks, Postmark!\n")); + } + +} diff --git a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php index 99e60caa05..6651f85d6c 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php @@ -8,24 +8,14 @@ final class PhabricatorMetaMTASendGridReceiveController } public function handleRequest(AphrontRequest $request) { - $mailers = PhabricatorMetaMTAMail::newMailers(); - $sendgrid_type = PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE; - // SendGrid doesn't sign payloads so we can't be sure that SendGrid // actually sent this request, but require a configured SendGrid mailer // before we activate this endpoint. - - $has_sendgrid = false; - foreach ($mailers as $mailer) { - if ($mailer->getAdapterType() != $sendgrid_type) { - continue; - } - - $has_sendgrid = true; - break; - } - - if (!$has_sendgrid) { + $mailers = PhabricatorMetaMTAMail::newMailersWithTypes( + array( + PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE, + )); + if (!$mailers) { return new Aphront404Response(); } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f2c8939132..4a9bc68322 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -482,6 +482,20 @@ final class PhabricatorMetaMTAMail return $this->sendWithMailers($mailers); } + public static function newMailersWithTypes(array $types) { + $mailers = self::newMailers(); + $types = array_fuse($types); + + foreach ($mailers as $key => $mailer) { + $mailer_type = $mailer->getAdapterType(); + if (!isset($types[$mailer_type])) { + unset($mailers[$key]); + } + } + + return array_values($mailers); + } + public static function newMailers() { $mailers = array(); diff --git a/src/docs/user/configuration/configuring_inbound_email.diviner b/src/docs/user/configuration/configuring_inbound_email.diviner index 5b47a17831..ada4ddb828 100644 --- a/src/docs/user/configuration/configuring_inbound_email.diviner +++ b/src/docs/user/configuration/configuring_inbound_email.diviner @@ -14,6 +14,7 @@ There are a few approaches available: | Receive Mail With | Setup | Cost | Notes | |--------|-------|------|-------| | Mailgun | Easy | Cheap | Recommended | +| Postmark | Easy | Cheap | Recommended | | SendGrid | Easy | Cheap | | | Local MTA | Extremely Difficult | Free | Strongly discouraged! | @@ -130,6 +131,17 @@ like this: example domain with your actual domain. - Set the `mailgun.api-key` config key to your Mailgun API key. +Postmark Setup +============== + +To process inbound mail from Postmark, configure this URI as your inbound +webhook URI in the Postmark control panel: + +``` +https:///mail/postmark/ +``` + + = SendGrid Setup = To use SendGrid, you need a SendGrid account with access to the "Parse API" for From dbe479f0d9dee38aee22808dc6321cd32e766a1f Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 05:09:21 -0800 Subject: [PATCH 33/67] Don't send error/exception mail to unverified addresses Summary: Depends on D19017. Fixes T12491. Ref T13053. After SES threw us in the dungeon for sending mail to a spamtrap we changed outbound mail rules to stop sending to unverified addresses, except a small amount of registration mail which we can't avoid. However, we'll still reply to random inbound messages with a helpful error, even if the sender is unverified. Instead, only send exception mail back if we know who the sender is. Test Plan: Processed inbound mail with `scripts/mail/mail_handler.php`. No more outbound mail for "bad address", etc. Still got outbound mail for "unknown command !quack". Reviewers: amckinley Maniphest Tasks: T13053, T12491 Differential Revision: https://secure.phabricator.com/D19018 --- .../PhabricatorMetaMTAReceivedMail.php | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php index 18fa7dd2ba..fc98d17010 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php @@ -105,6 +105,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { public function processReceivedMail() { + $sender = null; try { $this->dropMailFromPhabricator(); $this->dropMailAlreadyReceived(); @@ -140,7 +141,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { // This error is explicitly ignored. break; default: - $this->sendExceptionMail($ex); + $this->sendExceptionMail($ex, $sender); break; } @@ -150,7 +151,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { ->save(); return $this; } catch (Exception $ex) { - $this->sendExceptionMail($ex); + $this->sendExceptionMail($ex, $sender); $this ->setStatus(MetaMTAReceivedMailStatus::STATUS_UNHANDLED_EXCEPTION) @@ -305,9 +306,14 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { return head($accept); } - private function sendExceptionMail(Exception $ex) { - $from = $this->getHeader('from'); - if (!strlen($from)) { + private function sendExceptionMail( + Exception $ex, + PhabricatorUser $viewer = null) { + + // If we've failed to identify a legitimate sender, we don't send them + // an error message back. We want to avoid sending mail to unverified + // addresses. See T12491. + if (!$viewer) { return; } @@ -364,9 +370,8 @@ EOBODY $mail = id(new PhabricatorMetaMTAMail()) ->setIsErrorEmail(true) - ->setForceDelivery(true) ->setSubject($title) - ->addRawTos(array($from)) + ->addTos(array($viewer->getPHID())) ->setBody($body) ->saveAndSend(); } From f214abb63f9df8eb17a0438c7256d9f46957f48b Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 05:52:52 -0800 Subject: [PATCH 34/67] When a change removes recipients from an object, send them one last email Summary: Depends on D19018. Fixes T4776. Ref T13053. When you remove someone from an object's recipient list (for example, by removing them a reviewer, auditor, subscriber, owner or author) we currently do not send them mail about it because they're no longer connected to the object. In many of these cases (Commandeer, Reassign) the actual action in the UI adds them back to the object somehow (as a reviewer or subscriber, respectively) so this doesn't actually matter. However, there's no recovery mechanism for reviewer or subscriber removal. This is slightly bad from a policy/threat viewpoint since it means an attacker can remove all the recipients of an object "somewhat" silently. This isn't really silent, but it's less un-silent than it should be. It's also just not very good from a human interaction perspective: if Alice removes Bob as a reviewer, possibly "against his will", he should be notified about that. In the good case, Alice wrote a nice goodbye note that he should get to read. In the bad case, he should get a chance to correct the mistake. Also add a `removed(@user)` mail stamp so you can route these locally if you want. Test Plan: - Created and edited some different objects without catching anything broken. - Removed subscribers from tasks, saw the final email include the removed recipients with a `removed()` stamp. I'm not totally sure this doesn't have any surprising behavior or break any weird objects, but I think anything that crops up should be easy to fix. Reviewers: amckinley Subscribers: sophiebits Maniphest Tasks: T13053, T4776 Differential Revision: https://secure.phabricator.com/D19019 --- ...habricatorApplicationTransactionEditor.php | 70 +++++++++++++++++++ .../PhabricatorEditorMailEngineExtension.php | 7 ++ 2 files changed, 77 insertions(+) diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index e4f9607801..6bb7679fdc 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -74,6 +74,9 @@ abstract class PhabricatorApplicationTransactionEditor private $mustEncrypt; private $stampTemplates = array(); private $mailStamps = array(); + private $oldTo = array(); + private $oldCC = array(); + private $mailRemovedPHIDs = array(); private $transactionQueue = array(); @@ -916,6 +919,8 @@ abstract class PhabricatorApplicationTransactionEditor $this->willApplyTransactions($object, $xactions); if ($object->getID()) { + $this->buildOldRecipientLists($object, $xactions); + foreach ($xactions as $xaction) { // If any of the transactions require a read lock, hold one and @@ -1200,6 +1205,10 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + // Add any recipients who were previously on the notification list + // but were removed by this change. + $this->applyOldRecipientLists(); + $mail_xactions = $this->getTransactionsForMail($object, $xactions); $stamps = $this->newMailStamps($object, $xactions); foreach ($stamps as $stamp) { @@ -4127,4 +4136,65 @@ abstract class PhabricatorApplicationTransactionEditor return $results; } + public function getRemovedRecipientPHIDs() { + return $this->mailRemovedPHIDs; + } + + private function buildOldRecipientLists($object, $xactions) { + // See T4776. Before we start making any changes, build a list of the old + // recipients. If a change removes a user from the recipient list for an + // object we still want to notify the user about that change. This allows + // them to respond if they didn't want to be removed. + + if (!$this->shouldSendMail($object, $xactions)) { + return; + } + + $this->oldTo = $this->getMailTo($object); + $this->oldCC = $this->getMailCC($object); + + return $this; + } + + private function applyOldRecipientLists() { + $actor_phid = $this->getActingAsPHID(); + + // If you took yourself off the recipient list (for example, by + // unsubscribing or resigning) assume that you know what you did and + // don't need to be notified. + + // If you just moved from "To" to "Cc" (or vice versa), you're still a + // recipient so we don't need to add you back in. + + $map = array_fuse($this->mailToPHIDs) + array_fuse($this->mailCCPHIDs); + + foreach ($this->oldTo as $phid) { + if ($phid === $actor_phid) { + continue; + } + + if (isset($map[$phid])) { + continue; + } + + $this->mailToPHIDs[] = $phid; + $this->mailRemovedPHIDs[] = $phid; + } + + foreach ($this->oldCC as $phid) { + if ($phid === $actor_phid) { + continue; + } + + if (isset($map[$phid])) { + continue; + } + + $this->mailCCPHIDs[] = $phid; + $this->mailRemovedPHIDs[] = $phid; + } + + return $this; + } + } diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php index 29d10d641f..5365894429 100644 --- a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -40,6 +40,10 @@ final class PhabricatorEditorMailEngineExtension ->setKey('herald') ->setLabel(pht('Herald Rule')); + $templates[] = id(new PhabricatorPHIDMailStamp()) + ->setKey('removed') + ->setLabel(pht('Recipient Removed')); + return $templates; } @@ -69,6 +73,9 @@ final class PhabricatorEditorMailEngineExtension $this->getMailStamp('herald') ->setValue($editor->getHeraldRuleMonograms()); + + $this->getMailStamp('removed') + ->setValue($editor->getRemovedRecipientPHIDs()); } } From 1cd3a593784a95c9925bce443d790aaa6c90e996 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 07:08:36 -0800 Subject: [PATCH 35/67] When users resign from revisions, stop expanding projects/packages to include them Summary: Depends on D19019. Ref T13053. Fixes T12689. See PHI178. Currently, if `@alice` resigns from a revision but `#alice-fan-club` is still a subscriber or reviewer, she'll continue to get mail. This is undesirable. When users are associated with an object but have explicitly disengaged in an individal role (currently, only resign in audit/differential) mark them "unexpandable", so that they can no longer be included through implicit membership in a group (a project or package). `@alice` can still get mail if she's a explicit recipient: as an author, owner, or if she adds herself back as a subscriber. Test Plan: - Added `@ducker` and `#users-named-ducker` as reviewers. Ducker got mail. - Resigned as ducker, stopped getting future mail. - Subscribed explicitly, got mail again. - (Plus some `var_dump()` sanity checking in the internals.) Reviewers: amckinley Maniphest Tasks: T13053, T12689 Differential Revision: https://secure.phabricator.com/D19021 --- .../audit/editor/PhabricatorAuditEditor.php | 15 ++++++++-- .../editor/DifferentialTransactionEditor.php | 12 ++++++++ .../storage/DifferentialRevision.php | 10 +++++-- .../PhabricatorMailReplyHandler.php | 30 +++++++++++++++++++ .../PhabricatorRepositoryAuditRequest.php | 9 ++++++ .../storage/PhabricatorRepositoryCommit.php | 3 +- ...habricatorApplicationTransactionEditor.php | 14 +++++++++ 7 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index c39be75366..d142bd60cd 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -496,7 +496,6 @@ final class PhabricatorAuditEditor $phids[] = $object->getAuthorPHID(); } - $status_resigned = PhabricatorAuditStatusConstants::RESIGNED; foreach ($object->getAudits() as $audit) { if (!$audit->isInteresting()) { // Don't send mail to uninteresting auditors, like packages which @@ -504,7 +503,7 @@ final class PhabricatorAuditEditor continue; } - if ($audit->getAuditStatus() != $status_resigned) { + if (!$audit->isResigned()) { $phids[] = $audit->getAuditorPHID(); } } @@ -514,6 +513,18 @@ final class PhabricatorAuditEditor return $phids; } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $phids = array(); + + foreach ($object->getAudits() as $auditor) { + if ($auditor->isResigned()) { + $phids[] = $auditor->getAuditorPHID(); + } + } + + return $phids; + } + protected function buildMailBody( PhabricatorLiskDAO $object, array $xactions) { diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index f3583438c8..cc29d69fe0 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -644,6 +644,18 @@ final class DifferentialTransactionEditor return $phids; } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $phids = array(); + + foreach ($object->getReviewers() as $reviewer) { + if ($reviewer->isResigned()) { + $phids[] = $reviewer->getReviewerPHID(); + } + } + + return $phids; + } + protected function getMailAction( PhabricatorLiskDAO $object, array $xactions) { diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 938c588857..4591315207 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -820,9 +820,15 @@ final class DifferentialRevision extends DifferentialDAO } foreach ($reviewers as $reviewer) { - if ($reviewer->getReviewerPHID() == $phid) { - return true; + if ($reviewer->getReviewerPHID() !== $phid) { + continue; } + + if ($reviewer->isResigned()) { + continue; + } + + return true; } return false; diff --git a/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php b/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php index b0ae2de494..f8dd784e3b 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php @@ -6,6 +6,7 @@ abstract class PhabricatorMailReplyHandler extends Phobject { private $applicationEmail; private $actor; private $excludePHIDs = array(); + private $unexpandablePHIDs = array(); final public function setMailReceiver($mail_receiver) { $this->validateMailReceiver($mail_receiver); @@ -45,6 +46,15 @@ abstract class PhabricatorMailReplyHandler extends Phobject { return $this->excludePHIDs; } + public function setUnexpandablePHIDs(array $phids) { + $this->unexpandablePHIDs = $phids; + return $this; + } + + public function getUnexpandablePHIDs() { + return $this->unexpandablePHIDs; + } + abstract public function validateMailReceiver($mail_receiver); abstract public function getPrivateReplyHandlerEmailAddress( PhabricatorUser $user); @@ -297,6 +307,16 @@ abstract class PhabricatorMailReplyHandler extends Phobject { $to_result = array(); $cc_result = array(); + // "Unexpandable" users have disengaged from an object (for example, + // by resigning from a revision). + + // If such a user is still a direct recipient (for example, they're still + // on the Subscribers list) they're fair game, but group targets (like + // projects) will no longer include them when expanded. + + $unexpandable = $this->getUnexpandablePHIDs(); + $unexpandable = array_fuse($unexpandable); + $all_phids = array_merge($to, $cc); if ($all_phids) { $map = id(new PhabricatorMetaMTAMemberQuery()) @@ -305,11 +325,21 @@ abstract class PhabricatorMailReplyHandler extends Phobject { ->execute(); foreach ($to as $phid) { foreach ($map[$phid] as $expanded) { + if ($expanded !== $phid) { + if (isset($unexpandable[$expanded])) { + continue; + } + } $to_result[$expanded] = $expanded; } } foreach ($cc as $phid) { foreach ($map[$phid] as $expanded) { + if ($expanded !== $phid) { + if (isset($unexpandable[$expanded])) { + continue; + } + } $cc_result[$expanded] = $expanded; } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php b/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php index ea86594d9e..e05820c825 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php +++ b/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php @@ -72,6 +72,15 @@ final class PhabricatorRepositoryAuditRequest return true; } + public function isResigned() { + switch ($this->getAuditStatus()) { + case PhabricatorAuditStatusConstants::RESIGNED: + return true; + } + + return false; + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/repository/storage/PhabricatorRepositoryCommit.php b/src/applications/repository/storage/PhabricatorRepositoryCommit.php index 31a06dcbd4..1c5998d583 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryCommit.php +++ b/src/applications/repository/storage/PhabricatorRepositoryCommit.php @@ -657,7 +657,8 @@ final class PhabricatorRepositoryCommit public function isAutomaticallySubscribed($phid) { // TODO: This should also list auditors, but handling that is a bit messy - // right now because we are not guaranteed to have the data. + // right now because we are not guaranteed to have the data. (It should not + // include resigned auditors.) return ($phid == $this->getAuthorPHID()); } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 6bb7679fdc..167aa05fed 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -77,6 +77,7 @@ abstract class PhabricatorApplicationTransactionEditor private $oldTo = array(); private $oldCC = array(); private $mailRemovedPHIDs = array(); + private $mailUnexpandablePHIDs = array(); private $transactionQueue = array(); @@ -1204,6 +1205,7 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailShouldSend = true; $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + $this->mailUnexpandablePHIDs = $this->newMailUnexpandablePHIDs($object); // Add any recipients who were previously on the notification list // but were removed by this change. @@ -2562,7 +2564,13 @@ abstract class PhabricatorApplicationTransactionEditor $email_cc = $this->mailCCPHIDs; $email_cc = array_merge($email_cc, $this->heraldEmailPHIDs); + $unexpandable = $this->mailUnexpandablePHIDs; + if (!is_array($unexpandable)) { + $unexpandable = array(); + } + $targets = $this->buildReplyHandler($object) + ->setUnexpandablePHIDs($unexpandable) ->getMailTargets($email_to, $email_cc); // Set this explicitly before we start swapping out the effective actor. @@ -2817,6 +2825,11 @@ abstract class PhabricatorApplicationTransactionEditor } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + return array(); + } + + /** * @task mail */ @@ -3617,6 +3630,7 @@ abstract class PhabricatorApplicationTransactionEditor 'mailShouldSend', 'mustEncrypt', 'mailStamps', + 'mailUnexpandablePHIDs', ); } From d0a2e3c54f9ae254c7158965480c892d1c03d70c Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 09:39:27 -0800 Subject: [PATCH 36/67] Fix an issue where some Differential edit pathways may not have reviewers attached Summary: Depends on D19021. Ref T13053. When you "Subscribe", or make some other types of edits, we don't necessarily have reviewer data, but may now need it to do the new recipient list logic. I don't have a totally clean way to deal with this in the general case in mind, but just load it for now so that things don't fatal. Test Plan: Subscribed to a revision with the "Subscribe" action. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19022 --- .../editor/DifferentialTransactionEditor.php | 25 +++++++++++++++++++ .../storage/DifferentialRevision.php | 4 +++ 2 files changed, 29 insertions(+) diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index cc29d69fe0..063df6c602 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -632,6 +632,8 @@ final class DifferentialTransactionEditor } protected function getMailTo(PhabricatorLiskDAO $object) { + $this->requireReviewers($object); + $phids = array(); $phids[] = $object->getAuthorPHID(); foreach ($object->getReviewers() as $reviewer) { @@ -645,6 +647,8 @@ final class DifferentialTransactionEditor } protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $this->requireReviewers($object); + $phids = array(); foreach ($object->getReviewers() as $reviewer) { @@ -1737,4 +1741,25 @@ final class DifferentialTransactionEditor } } + private function requireReviewers(DifferentialRevision $revision) { + if ($revision->hasAttachedReviewers()) { + return; + } + + $with_reviewers = id(new DifferentialRevisionQuery()) + ->setViewer($this->getActor()) + ->needReviewers(true) + ->withPHIDs(array($revision->getPHID())) + ->executeOne(); + if (!$with_reviewers) { + throw new Exception( + pht( + 'Failed to reload revision ("%s").', + $revision->getPHID())); + } + + $revision->attachReviewers($with_reviewers->getReviewers()); + } + + } diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 4591315207..e8fdf7e514 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -583,6 +583,10 @@ final class DifferentialRevision extends DifferentialDAO return $this; } + public function hasAttachedReviewers() { + return ($this->reviewerStatus !== self::ATTACHABLE); + } + public function getReviewerPHIDs() { $reviewers = $this->getReviewers(); return mpull($reviewers, 'getReviewerPHID'); From 2bb4fc9ecea567b8df06a4965b6d1008a615c078 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 07:13:57 -0800 Subject: [PATCH 37/67] Fix a Phortune billing issue where subscription autopay could charge disabled cards Summary: See support email. There's nothing tricky here, we were just missing a check. The different parts of this got built at different times so I think this was simply overlooked. Also add a redundant check just to future-proof this and be on the safe side. Test Plan: Used `bin/phortune invoice` to charge a pact subscription. After deleting the card, the charge failed with an appropriate error. Reviewers: amckinley Differential Revision: https://secure.phabricator.com/D19020 --- .../query/PhortunePaymentMethodQuery.php | 25 ++++++------------- .../phortune/storage/PhortuneCart.php | 7 ++++++ .../storage/PhortunePaymentMethod.php | 4 +++ .../worker/PhortuneSubscriptionWorker.php | 4 +++ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/applications/phortune/query/PhortunePaymentMethodQuery.php b/src/applications/phortune/query/PhortunePaymentMethodQuery.php index 07d49a5909..42d54805e6 100644 --- a/src/applications/phortune/query/PhortunePaymentMethodQuery.php +++ b/src/applications/phortune/query/PhortunePaymentMethodQuery.php @@ -34,19 +34,12 @@ final class PhortunePaymentMethodQuery return $this; } + public function newResultObject() { + return new PhortunePaymentMethod(); + } + protected function loadPage() { - $table = new PhortunePaymentMethod(); - $conn = $table->establishConnection('r'); - - $rows = queryfx_all( - $conn, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn), - $this->buildOrderClause($conn), - $this->buildLimitClause($conn)); - - return $table->loadAllFromArray($rows); + return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $methods) { @@ -106,8 +99,8 @@ final class PhortunePaymentMethodQuery return $methods; } - protected function buildWhereClause(AphrontDatabaseConnection $conn) { - $where = array(); + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( @@ -144,9 +137,7 @@ final class PhortunePaymentMethodQuery $this->statuses); } - $where[] = $this->buildPagingClause($conn); - - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { diff --git a/src/applications/phortune/storage/PhortuneCart.php b/src/applications/phortune/storage/PhortuneCart.php index af2b386dfe..07554ccb87 100644 --- a/src/applications/phortune/storage/PhortuneCart.php +++ b/src/applications/phortune/storage/PhortuneCart.php @@ -118,6 +118,13 @@ final class PhortuneCart extends PhortuneDAO ->setAmountAsCurrency($this->getTotalPriceAsCurrency()); if ($method) { + if (!$method->isActive()) { + throw new Exception( + pht( + 'Attempting to apply a charge using an inactive '. + 'payment method ("%s")!', + $method->getPHID())); + } $charge->setPaymentMethodPHID($method->getPHID()); } diff --git a/src/applications/phortune/storage/PhortunePaymentMethod.php b/src/applications/phortune/storage/PhortunePaymentMethod.php index 8044168ba4..1712d3f973 100644 --- a/src/applications/phortune/storage/PhortunePaymentMethod.php +++ b/src/applications/phortune/storage/PhortunePaymentMethod.php @@ -128,6 +128,10 @@ final class PhortunePaymentMethod extends PhortuneDAO return $month.'/'.$year; } + public function isActive() { + return ($this->getStatus() === self::STATUS_ACTIVE); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/phortune/worker/PhortuneSubscriptionWorker.php b/src/applications/phortune/worker/PhortuneSubscriptionWorker.php index 097fb54dd9..d05aacbb7c 100644 --- a/src/applications/phortune/worker/PhortuneSubscriptionWorker.php +++ b/src/applications/phortune/worker/PhortuneSubscriptionWorker.php @@ -141,6 +141,10 @@ final class PhortuneSubscriptionWorker extends PhabricatorWorker { $method = id(new PhortunePaymentMethodQuery()) ->setViewer($viewer) ->withPHIDs(array($subscription->getDefaultPaymentMethodPHID())) + ->withStatuses( + array( + PhortunePaymentMethod::STATUS_ACTIVE, + )) ->executeOne(); if (!$method) { $issues[] = pht( From 948b0ceca466457b97798eb920554aa353aa112b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 06:47:24 -0800 Subject: [PATCH 38/67] Configure a whitelist of remote addresses for Postmark inbound webhooks Summary: Ref T13053. Postmark support recommends testing requests against a whitelist of known remote addresses to determine request authenticity. Today, the list can be found here: This is potentially less robust than, e.g., HMAC verification, since they may need to add new datacenters or support IPv6 or something. Users might also have weird network topologies where everything is proxied, and this makes testing/simulating more difficult. Allow users to configure the list so that they don't need to hack things apart if Postmark adds a new datacenter or remote addresses are unreliable for some other reason, but ship with safe defaults for today. Test Plan: Tried to make local requests, got kicked out. Added `0.0.0.0/0` to the list, stopped getting kicked out. I don't have a convenient way to route real Postmark traffic to my development laptop with an authentic remote address so I haven't verified that the published remote address is legitimate, but I'll vet that in production when I go through all the other mailers. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19025 --- ...ricatorMailImplementationPostmarkAdapter.php | 12 ++++++++++++ ...bricatorMetaMTAPostmarkReceiveController.php | 15 +++++++++++++++ .../configuring_inbound_email.diviner | 4 ++++ .../configuring_outbound_email.diviner | 17 +++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php index bd5ee820af..5792ba08f8 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php @@ -73,12 +73,24 @@ final class PhabricatorMailImplementationPostmarkAdapter $options, array( 'access-token' => 'string', + 'inbound-addresses' => 'list', )); + + // Make sure this is properly formatted. + PhutilCIDRList::newList($options['inbound-addresses']); } public function newDefaultOptions() { return array( 'access-token' => null, + 'inbound-addresses' => array( + // Via Postmark support circa February 2018, see: + // + // https://postmarkapp.com/support/article/800-ips-for-firewalls + // + // "Configuring Outbound Email" should be updated if this changes. + '50.31.156.6/32', + ), ); } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php index a54da6fb40..345cd93fe1 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php @@ -20,6 +20,21 @@ final class PhabricatorMetaMTAPostmarkReceiveController return new Aphront404Response(); } + $remote_address = $request->getRemoteAddress(); + $any_remote_match = false; + foreach ($mailers as $mailer) { + $inbound_addresses = $mailer->getOption('inbound-addresses'); + $cidr_list = PhutilCIDRList::newList($inbound_addresses); + if ($cidr_list->containsAddress($remote_address)) { + $any_remote_match = true; + break; + } + } + + if (!$any_remote_match) { + return new Aphront400Response(); + } + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $raw_input = PhabricatorStartup::getRawInput(); diff --git a/src/docs/user/configuration/configuring_inbound_email.diviner b/src/docs/user/configuration/configuring_inbound_email.diviner index ada4ddb828..f4f367d57e 100644 --- a/src/docs/user/configuration/configuring_inbound_email.diviner +++ b/src/docs/user/configuration/configuring_inbound_email.diviner @@ -141,6 +141,10 @@ webhook URI in the Postmark control panel: https:///mail/postmark/ ``` +See also the Postmark section in @{article:Configuring Outbound Email} for +discussion of the remote address whitelist used to verify that requests this +endpoint receives are authentic requests originating from Postmark. + = SendGrid Setup = diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index d2daf7a40a..37a344c275 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -157,6 +157,23 @@ Postmark is a third-party email delivery serivice. You can learn more at To use this mailer, set `type` to `postmark`, then configure these `options`: - `access-token`: Required string. Your Postmark access token. + - `inbound-addresses`: Optional list. Address ranges which you + will accept inbound Postmark HTTP webook requests from. + +The default address list is preconfigured with Postmark's address range, so +you generally will not need to set or adjust it. + +The option accepts a list of CIDR ranges, like `1.2.3.4/16` (IPv4) or +`::ffff:0:0/96` (IPv6). The default ranges are: + +```lang=json +[ + "50.31.156.6/32" +] +``` + +The default address ranges were last updated in February 2018, and were +documented at: Mailer: Amazon SES From 942b17a980888b09980217735b12c9dd583a3b8c Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 08:57:07 -0800 Subject: [PATCH 39/67] Improve correctness of email address escaping in Mailgun/Postmark Summary: Ref T13053. Uses the changes in D19026 to escape mail addresses. Those rules may not be right yet, but they're at least all in one place, have test coverage, and aren't obviously incorrect. Test Plan: Will vet this more extensively when re-testing all mailers. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19027 --- .../adapter/PhabricatorMailImplementationAdapter.php | 5 +++-- .../PhabricatorMailImplementationMailgunAdapter.php | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index ce56345194..dfbe891651 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -96,8 +96,9 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { protected function renderAddress($email, $name = null) { if (strlen($name)) { - // TODO: This needs to be escaped correctly. - return "{$name} <{$email}>"; + return (string)id(new PhutilEmailAddress()) + ->setDisplayName($name) + ->setAddress($email); } else { return $email; } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index bed0dada63..12c54e0d6a 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -21,7 +21,7 @@ final class PhabricatorMailImplementationMailgunAdapter if (empty($this->params['reply-to'])) { $this->params['reply-to'] = array(); } - $this->params['reply-to'][] = "{$name} <{$email}>"; + $this->params['reply-to'][] = $this->renderAddress($name, $email); return $this; } @@ -110,11 +110,8 @@ final class PhabricatorMailImplementationMailgunAdapter } $from = idx($this->params, 'from'); - if (idx($this->params, 'from-name')) { - $params['from'] = "\"{$this->params['from-name']}\" <{$from}>"; - } else { - $params['from'] = $from; - } + $from_name = idx($this->params, 'from-name'); + $params['from'] = $this->renderAddress($from, $from_name); if (idx($this->params, 'reply-to')) { $replyto = $this->params['reply-to']; From a8f937d3138a4b02203964e995945c58be49293c Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:03:26 -0800 Subject: [PATCH 40/67] Only add the Mail "STAMPS" body section if there are stamps Summary: Ref T13053. Some mail (like push notification mail) doesn't currently generate any stamps. Drop this section if there aren't any stamps on the mail. Test Plan: Will check push mail in production. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19028 --- .../replyhandler/PhabricatorMailTarget.php | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index 5d8378e8af..4bd5105135 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -66,27 +66,28 @@ final class PhabricatorMailTarget extends Phobject { if ($show_stamps) { $stamps = $mail->getMailStamps(); + if ($stamps) { + $body .= "\n"; + $body .= pht('STAMPS'); + $body .= "\n"; + $body .= implode(' ', $stamps); + $body .= "\n"; - $body .= "\n"; - $body .= pht('STAMPS'); - $body .= "\n"; - $body .= implode(' ', $stamps); - $body .= "\n"; - - if ($has_html) { - $html = array(); - $html[] = phutil_tag('strong', array(), pht('STAMPS')); - $html[] = phutil_tag('br'); - $html[] = phutil_tag( - 'span', - array( - 'style' => 'font-size: smaller; color: #92969D', - ), - phutil_implode_html(' ', $stamps)); - $html[] = phutil_tag('br'); - $html[] = phutil_tag('br'); - $html = phutil_tag('div', array(), $html); - $html_body .= hsprintf('%s', $html); + if ($has_html) { + $html = array(); + $html[] = phutil_tag('strong', array(), pht('STAMPS')); + $html[] = phutil_tag('br'); + $html[] = phutil_tag( + 'span', + array( + 'style' => 'font-size: smaller; color: #92969D', + ), + phutil_implode_html(' ', $stamps)); + $html[] = phutil_tag('br'); + $html[] = phutil_tag('br'); + $html = phutil_tag('div', array(), $html); + $html_body .= hsprintf('%s', $html); + } } } From bae9f459ab7f0e1dd0a71d4f4dac27d94d518907 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:06:40 -0800 Subject: [PATCH 41/67] Pass HTML bodies to push email Summary: Depends on D19028. Ref T13053. Fixes T6576. An HTML body was built here, but not passed to the actual mail message. Test Plan: Will verify production push mail. Maniphest Tasks: T13053, T6576 Differential Revision: https://secure.phabricator.com/D19029 --- .../repository/worker/PhabricatorRepositoryPushMailWorker.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php index 5ffaf0a5c2..554c2cf772 100644 --- a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php +++ b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php @@ -123,6 +123,7 @@ final class PhabricatorRepositoryPushMailWorker ->setSubject($subject) ->setFrom($event->getPusherPHID()) ->setBody($body->render()) + ->setHTMLBody($body->renderHTML()) ->setThreadID($event->getPHID(), $is_new = true) ->setIsBulk(true); From 6186f0aa91b6c5d8faa6eecc2a0e98571bb34153 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:19:43 -0800 Subject: [PATCH 42/67] Briefly document mail stamps and remove obsolete header documentation Summary: Fixes T10189. Ref T13053. We haven't sent these headers in a very long time. Briefly mention the new stamps header instead, although I expect to integrate stamp documentation into the UI in a more cohesive way in the future. Test Plan: Read documentation. Maniphest Tasks: T13053, T10189 Differential Revision: https://secure.phabricator.com/D19030 --- .../storage/PhabricatorMetaMTAMail.php | 2 +- src/docs/user/userguide/mail_rules.diviner | 71 +++++-------------- 2 files changed, 20 insertions(+), 53 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 4a9bc68322..f16b50bf11 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -829,7 +829,7 @@ final class PhabricatorMetaMTAMail $stamps = $this->getMailStamps(); if ($stamps) { - $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + $headers[] = array('X-Phabricator-Stamps', implode(' ', $stamps)); } $raw_body = idx($params, 'body', ''); diff --git a/src/docs/user/userguide/mail_rules.diviner b/src/docs/user/userguide/mail_rules.diviner index 3640f5e5a5..61bc3210e9 100644 --- a/src/docs/user/userguide/mail_rules.diviner +++ b/src/docs/user/userguide/mail_rules.diviner @@ -3,7 +3,8 @@ How to effectively manage Phabricator email notifications. -= Overview = +Overview +======== Phabricator uses email as a major notification channel, but the amount of email it sends can seem overwhelming if you're working on an active team. This @@ -13,69 +14,35 @@ By far the best approach to managing mail is to **write mail rules** to categorize mail. Essentially all modern mail clients allow you to quickly write sophisticated rules to route, categorize, or delete email. -= Reducing Email = +Reducing Email +============== You can reduce the amount of email you receive by turning off some types of email in {nav Settings > Email Preferences}. For example, you can turn off email produced by your own actions (like when you comment on a revision), and some types of less-important notifications about events. -= Mail Rules = +Mail Rules +========== The best approach to managing mail is to write mail rules. Simply writing rules to move mail from Differential, Maniphest and Herald to separate folders will vastly simplify mail management. -Phabricator also sets a large number of headers (see below) which can allow you -to write more sophisticated mail rules. +Phabricator also adds mail headers (see below) which can allow you to write +more sophisticated mail rules. -= Mail Headers = +Mail Headers +============ -Phabricator sends a variety of mail headers that can be useful in crafting rules -to route and manage mail. +Phabricator sends various information in mail headers that can be useful in +crafting rules to route and manage mail. To see a full list of headers, use +the "View Raw Message" feature in your mail client. -Headers in plural contain lists. A list containing two items, `1` and -`15` will generally be formatted like this: +The most useful header for routing is generally `X-Phabricator-Stamps`. This +is a list of attributes which describe the object the mail is about and the +actions which the mail informs you about. - X-Header: <1>, <15> - -The intent is to allow you to write a rule which matches against "<1>". If you -just match against "1", you'll incorrectly match "15", but matching "<1>" will -correctly match only "<1>". - -Some other headers use a single value but can be presented multiple times. -It is to support e-mail clients which are not able to create rules using regular -expressions or wildcards (namely Outlook). - -The headers Phabricator adds to mail are: - - - `X-Phabricator-Sent-This-Message`: this is attached to all mail - Phabricator sends. You can use it to differentiate between email from - Phabricator and replies/forwards of Phabricator mail from human beings. - - `X-Phabricator-To`: this is attached to all mail Phabricator sends. - It shows the PHIDs of the original "To" line, before any mutation - by the mailer configuration. - - `X-Phabricator-Cc`: this is attached to all mail Phabricator sends. - It shows the PHIDs of the original "Cc" line, before any mutation by the - mailer configuration. - - `X-Differential-Author`: this is attached to Differential mail and shows - the revision's author. You can use it to filter mail about your revisions - (or other users' revisions). - - `X-Differential-Reviewer`: this is attached to Differential mail and - shows the reviewers. You can use it to filter mail about revisions you - are reviewing, versus revisions you are explicitly CC'd on or CC'd as - a result of Herald rules. - - `X-Differential-Reviewers`: list version of the previous. - - `X-Differential-CC`: this is attached to Differential mail and shows - the CCs on the revision. - - `X-Differential-CCs`: list version of the previous. - - `X-Differential-Explicit-CC`: this is attached to Differential mail and - shows the explicit CCs on the revision (those that were added by humans, - not by Herald). - - `X-Differential-Explicit-CCs`: list version of the previous. - - `X-Phabricator-Mail-Tags`: this is attached to some mail and has - a list of descriptors about the mail. (This is fairly new and subject - to some change.) - - `X-Herald-Rules`: this is attached to some mail and shows Herald rule - IDs which have triggered for the object. You can use this to sort or - categorize mail that has triggered specific rules. +If you use a client which can not perform header matching (like Gmail), you can +change the {nav Settings > Email Format > Send Stamps} setting to include the +stamps in the mail body and then match them with body rules. From bca9c08953bd40c0c534f7d94651e37c3513af85 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:36:47 -0800 Subject: [PATCH 43/67] Add an "Acting user" field to Herald Summary: Ref T13053. Fixes T7804. Adds "Acting user" so you can have "always email me" stuff skip things you did or keep an eye on suspicious interns. For the test console, the current user is the acting user. For pushes, the pusher is the acting user. Test Plan: Wrote acting user rules, triggered them via test console and via multiple actors on real objects. Maniphest Tasks: T13053, T7804 Differential Revision: https://secure.phabricator.com/D19031 --- src/__phutil_library_map__.php | 2 ++ .../engine/DiffusionCommitHookEngine.php | 6 +++- .../herald/adapter/HeraldAdapter.php | 10 ++++++ .../HeraldTestConsoleController.php | 1 + .../herald/field/HeraldActingUserField.php | 32 +++++++++++++++++++ ...habricatorApplicationTransactionEditor.php | 1 + 6 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/applications/herald/field/HeraldActingUserField.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 301459869f..73632d11b4 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1346,6 +1346,7 @@ phutil_register_library_map(array( 'HarbormasterWaitForPreviousBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterWaitForPreviousBuildStepImplementation.php', 'HarbormasterWorker' => 'applications/harbormaster/worker/HarbormasterWorker.php', 'HarbormasterWorkingCopyArtifact' => 'applications/harbormaster/artifact/HarbormasterWorkingCopyArtifact.php', + 'HeraldActingUserField' => 'applications/herald/field/HeraldActingUserField.php', 'HeraldAction' => 'applications/herald/action/HeraldAction.php', 'HeraldActionGroup' => 'applications/herald/action/HeraldActionGroup.php', 'HeraldActionRecord' => 'applications/herald/storage/HeraldActionRecord.php', @@ -6589,6 +6590,7 @@ phutil_register_library_map(array( 'HarbormasterWaitForPreviousBuildStepImplementation' => 'HarbormasterBuildStepImplementation', 'HarbormasterWorker' => 'PhabricatorWorker', 'HarbormasterWorkingCopyArtifact' => 'HarbormasterDrydockLeaseArtifact', + 'HeraldActingUserField' => 'HeraldField', 'HeraldAction' => 'Phobject', 'HeraldActionGroup' => 'HeraldGroup', 'HeraldActionRecord' => 'HeraldDAO', diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index 99df0e54af..a0769a51f0 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -297,7 +297,11 @@ final class DiffusionCommitHookEngine extends Phobject { return; } - $adapter_template->setHookEngine($this); + $viewer = $this->getViewer(); + + $adapter_template + ->setHookEngine($this) + ->setActingAsPHID($viewer->getPHID()); $engine = new HeraldEngine(); $rules = null; diff --git a/src/applications/herald/adapter/HeraldAdapter.php b/src/applications/herald/adapter/HeraldAdapter.php index cc0fdbd3b5..940d604019 100644 --- a/src/applications/herald/adapter/HeraldAdapter.php +++ b/src/applications/herald/adapter/HeraldAdapter.php @@ -40,6 +40,7 @@ abstract class HeraldAdapter extends Phobject { private $forbiddenActions = array(); private $viewer; private $mustEncryptReasons = array(); + private $actingAsPHID; public function getEmailPHIDs() { return array_values($this->emailPHIDs); @@ -49,6 +50,15 @@ abstract class HeraldAdapter extends Phobject { return array_values($this->forcedEmailPHIDs); } + final public function setActingAsPHID($acting_as_phid) { + $this->actingAsPHID = $acting_as_phid; + return $this; + } + + final public function getActingAsPHID() { + return $this->actingAsPHID; + } + public function addEmailPHID($phid, $force) { $this->emailPHIDs[$phid] = $phid; if ($force) { diff --git a/src/applications/herald/controller/HeraldTestConsoleController.php b/src/applications/herald/controller/HeraldTestConsoleController.php index 8a7a94963d..4ddab2669b 100644 --- a/src/applications/herald/controller/HeraldTestConsoleController.php +++ b/src/applications/herald/controller/HeraldTestConsoleController.php @@ -41,6 +41,7 @@ final class HeraldTestConsoleController extends HeraldController { $adapter ->setIsNewObject(false) + ->setActingAsPHID($viewer->getPHID()) ->setViewer($viewer); $rules = id(new HeraldRuleQuery()) diff --git a/src/applications/herald/field/HeraldActingUserField.php b/src/applications/herald/field/HeraldActingUserField.php new file mode 100644 index 0000000000..2245c7e9f7 --- /dev/null +++ b/src/applications/herald/field/HeraldActingUserField.php @@ -0,0 +1,32 @@ +getAdapter()->getActingAsPHID(); + } + + protected function getHeraldFieldStandardType() { + return self::STANDARD_PHID; + } + + protected function getDatasource() { + return new PhabricatorPeopleDatasource(); + } + + public function supportsObject($object) { + return true; + } + + public function getFieldGroupKey() { + return HeraldEditFieldGroup::FIELDGROUPKEY; + } + +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 167aa05fed..8c319fc61e 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -3254,6 +3254,7 @@ abstract class PhabricatorApplicationTransactionEditor $adapter = $this->buildHeraldAdapter($object, $xactions) ->setContentSource($this->getContentSource()) ->setIsNewObject($this->getIsNewObject()) + ->setActingAsPHID($this->getActingAsPHID()) ->setAppliedTransactions($xactions); if ($this->getApplicationEmail()) { From 0402a79e0e5723a54ecffaf5c6bc1c1d89cbe51b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:46:56 -0800 Subject: [PATCH 44/67] Render object remarkup references in a text context as "Dxxx " Summary: Depends on D19031. Fixes T11389. Currently, we render `Dxxx` in a text context (plain text email) as just a URI. Instead, render it like `Dxxx `. This is more faithful to the original intent and preserves `T123/T456` as two separate, usable links. Test Plan: Wrote `T123/T234` in a task, pulled mail for it with `bin/mail show-outbound`, saw separate clickable links. Maniphest Tasks: T11389 Differential Revision: https://secure.phabricator.com/D19032 --- .../markup/rule/PhabricatorObjectRemarkupRule.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php b/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php index 35c0ecfad0..fbbfa36805 100644 --- a/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php +++ b/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php @@ -75,7 +75,7 @@ abstract class PhabricatorObjectRemarkupRule extends PhutilRemarkupRule { } if ($this->getEngine()->isTextMode()) { - return PhabricatorEnv::getProductionURI($href); + return $text.' <'.PhabricatorEnv::getProductionURI($href).'>'; } else if ($this->getEngine()->isHTMLMailMode()) { $href = PhabricatorEnv::getProductionURI($href); return $this->renderObjectTagForMail($text, $href, $handle); From ab04d2179bf1322ec31f03e46c28bebb0334f135 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 10:37:47 -0800 Subject: [PATCH 45/67] Add "Mute/Unmute" for subscribable objects Summary: Ref T13053. See PHI126. Add an explicit "Mute" action to kill mail and notifications for a particular object. Test Plan: Muted and umuted an object while interacting with it. Saw mail route appropriately. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19033 --- resources/celerity/map.php | 6 +- src/__phutil_library_map__.php | 6 ++ .../metamta/query/PhabricatorMetaMTAActor.php | 4 + .../storage/PhabricatorMetaMTAMail.php | 21 +++++ .../PhabricatorSubscriptionsApplication.php | 5 +- ...PhabricatorSubscriptionsMuteController.php | 92 +++++++++++++++++++ ...habricatorSubscriptionsUIEventListener.php | 55 ++++++++--- .../edges/PhabricatorMutedByEdgeType.php | 16 ++++ .../edges/PhabricatorMutedEdgeType.php | 16 ++++ ...habricatorApplicationTransactionEditor.php | 28 ++++++ .../PhabricatorApplicationTransaction.php | 2 + src/view/layout/PhabricatorActionView.php | 10 ++ webroot/rsrc/css/phui/phui-action-list.css | 17 ++-- 13 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php create mode 100644 src/applications/transactions/edges/PhabricatorMutedByEdgeType.php create mode 100644 src/applications/transactions/edges/PhabricatorMutedEdgeType.php diff --git a/resources/celerity/map.php b/resources/celerity/map.php index d9aebf32bc..17138d6604 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -9,7 +9,7 @@ return array( 'names' => array( 'conpherence.pkg.css' => 'e68cf1fa', 'conpherence.pkg.js' => '15191c65', - 'core.pkg.css' => '51debec3', + 'core.pkg.css' => 'ce8c2a58', 'core.pkg.js' => '4c79d74f', 'darkconsole.pkg.js' => '1f9a31bc', 'differential.pkg.css' => '45951e9e', @@ -136,7 +136,7 @@ return array( 'rsrc/css/phui/object-item/phui-oi-flush-ui.css' => '9d9685d6', 'rsrc/css/phui/object-item/phui-oi-list-view.css' => '6ae18df0', 'rsrc/css/phui/object-item/phui-oi-simple-ui.css' => 'a8beebea', - 'rsrc/css/phui/phui-action-list.css' => 'f7f61a34', + 'rsrc/css/phui/phui-action-list.css' => '0bcd9a45', 'rsrc/css/phui/phui-action-panel.css' => 'b4798122', 'rsrc/css/phui/phui-badge.css' => '22c0cf4f', 'rsrc/css/phui/phui-basic-nav-view.css' => '98c11ab3', @@ -766,7 +766,7 @@ return array( 'path-typeahead' => 'f7fc67ec', 'people-picture-menu-item-css' => 'a06f7f34', 'people-profile-css' => '4df76faf', - 'phabricator-action-list-view-css' => 'f7f61a34', + 'phabricator-action-list-view-css' => '0bcd9a45', 'phabricator-busy' => '59a7976a', 'phabricator-chatlog-css' => 'd295b020', 'phabricator-content-source-view-css' => '4b8b05d4', diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 73632d11b4..9342935ca1 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3291,6 +3291,8 @@ phutil_register_library_map(array( 'PhabricatorMultiFactorSettingsPanel' => 'applications/settings/panel/PhabricatorMultiFactorSettingsPanel.php', 'PhabricatorMultimeterApplication' => 'applications/multimeter/application/PhabricatorMultimeterApplication.php', 'PhabricatorMustVerifyEmailController' => 'applications/auth/controller/PhabricatorMustVerifyEmailController.php', + 'PhabricatorMutedByEdgeType' => 'applications/transactions/edges/PhabricatorMutedByEdgeType.php', + 'PhabricatorMutedEdgeType' => 'applications/transactions/edges/PhabricatorMutedEdgeType.php', 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php', @@ -4240,6 +4242,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsHeraldAction.php', 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', 'PhabricatorSubscriptionsMailEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php', + 'PhabricatorSubscriptionsMuteController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSelfHeraldAction.php', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSubscribersHeraldAction.php', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php', @@ -8808,6 +8811,8 @@ phutil_register_library_map(array( 'PhabricatorMultiFactorSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorMultimeterApplication' => 'PhabricatorApplication', 'PhabricatorMustVerifyEmailController' => 'PhabricatorAuthController', + 'PhabricatorMutedByEdgeType' => 'PhabricatorEdgeType', + 'PhabricatorMutedEdgeType' => 'PhabricatorEdgeType', 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost', @@ -9960,6 +9965,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsHeraldAction' => 'HeraldAction', 'PhabricatorSubscriptionsListController' => 'PhabricatorController', 'PhabricatorSubscriptionsMailEngineExtension' => 'PhabricatorMailEngineExtension', + 'PhabricatorSubscriptionsMuteController' => 'PhabricatorController', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', diff --git a/src/applications/metamta/query/PhabricatorMetaMTAActor.php b/src/applications/metamta/query/PhabricatorMetaMTAActor.php index 1f4cc7da12..cf2060a8f7 100644 --- a/src/applications/metamta/query/PhabricatorMetaMTAActor.php +++ b/src/applications/metamta/query/PhabricatorMetaMTAActor.php @@ -21,6 +21,7 @@ final class PhabricatorMetaMTAActor extends Phobject { const REASON_ROUTE_AS_NOTIFICATION = 'route-as-notification'; const REASON_ROUTE_AS_MAIL = 'route-as-mail'; const REASON_UNVERIFIED = 'unverified'; + const REASON_MUTED = 'muted'; private $phid; private $emailAddress; @@ -116,6 +117,7 @@ final class PhabricatorMetaMTAActor extends Phobject { self::REASON_ROUTE_AS_NOTIFICATION => pht('Route as Notification'), self::REASON_ROUTE_AS_MAIL => pht('Route as Mail'), self::REASON_UNVERIFIED => pht('Address Not Verified'), + self::REASON_MUTED => pht('Muted'), ); return idx($names, $reason, pht('Unknown ("%s")', $reason)); @@ -172,6 +174,8 @@ final class PhabricatorMetaMTAActor extends Phobject { 'in Herald.'), self::REASON_UNVERIFIED => pht( 'This recipient does not have a verified primary email address.'), + self::REASON_MUTED => pht( + 'This recipient has muted notifications for this object.'), ); return idx($descriptions, $reason, pht('Unknown Reason ("%s")', $reason)); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f16b50bf11..9dfd6a3eb6 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -160,6 +160,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('exclude', array()); } + public function setMutedPHIDs(array $muted) { + $this->setParam('muted', $muted); + return $this; + } + + private function getMutedPHIDs() { + return $this->getParam('muted', array()); + } + public function setForceHeraldMailRecipientPHIDs(array $force) { $this->setParam('herald-force-recipients', $force); return $this; @@ -1113,6 +1122,18 @@ final class PhabricatorMetaMTAMail } } + // Exclude muted recipients. We're doing this after saving deliverability + // so that Herald "Send me an email" actions can still punch through a + // mute. + + foreach ($this->getMutedPHIDs() as $muted_phid) { + $muted_actor = idx($actors, $muted_phid); + if (!$muted_actor) { + continue; + } + $muted_actor->setUndeliverable(PhabricatorMetaMTAActor::REASON_MUTED); + } + // For the rest of the rules, order matters. We're going to run all the // possible rules in order from weakest to strongest, and let the strongest // matching rule win. The weaker rules leave annotations behind which help diff --git a/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php b/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php index 56759f5dc3..2de2994a92 100644 --- a/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php +++ b/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php @@ -24,7 +24,10 @@ final class PhabricatorSubscriptionsApplication extends PhabricatorApplication { return array( '/subscriptions/' => array( '(?Padd|delete)/'. - '(?P[^/]+)/' => 'PhabricatorSubscriptionsEditController', + '(?P[^/]+)/' => 'PhabricatorSubscriptionsEditController', + 'mute/' => array( + '(?P[^/]+)/' => 'PhabricatorSubscriptionsMuteController', + ), 'list/(?P[^/]+)/' => 'PhabricatorSubscriptionsListController', 'transaction/(?Padd|rem)/(?[^/]+)/' => 'PhabricatorSubscriptionsTransactionController', diff --git a/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php b/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php new file mode 100644 index 0000000000..1369643ffc --- /dev/null +++ b/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php @@ -0,0 +1,92 @@ +getViewer(); + $phid = $request->getURIData('phid'); + + $handle = id(new PhabricatorHandleQuery()) + ->setViewer($viewer) + ->withPHIDs(array($phid)) + ->executeOne(); + + $object = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withPHIDs(array($phid)) + ->executeOne(); + + if (!($object instanceof PhabricatorSubscribableInterface)) { + return new Aphront400Response(); + } + + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $edge_query = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs(array($object->getPHID())) + ->withEdgeTypes(array($muted_type)) + ->withDestinationPHIDs(array($viewer->getPHID())); + + $edge_query->execute(); + + $is_mute = !$edge_query->getDestinationPHIDs(); + $object_uri = $handle->getURI(); + + if ($request->isFormPost()) { + if ($is_mute) { + $xaction_value = array( + '+' => array_fuse(array($viewer->getPHID())), + ); + } else { + $xaction_value = array( + '-' => array_fuse(array($viewer->getPHID())), + ); + } + + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $xaction = id($object->getApplicationTransactionTemplate()) + ->setTransactionType(PhabricatorTransactions::TYPE_EDGE) + ->setMetadataValue('edge:type', $muted_type) + ->setNewValue($xaction_value); + + $editor = id($object->getApplicationTransactionEditor()) + ->setActor($viewer) + ->setContinueOnNoEffect(true) + ->setContinueOnMissingFields(true) + ->setContentSourceFromRequest($request); + + $editor->applyTransactions( + $object->getApplicationTransactionObject(), + array($xaction)); + + return id(new AphrontReloadResponse())->setURI($object_uri); + } + + $dialog = $this->newDialog() + ->addCancelButton($object_uri); + + if ($is_mute) { + $dialog + ->setTitle(pht('Mute Notifications')) + ->appendParagraph( + pht( + 'Mute this object? You will no longer receive notifications or '. + 'email about it.')) + ->addSubmitButton(pht('Mute')); + } else { + $dialog + ->setTitle(pht('Unmute Notifications')) + ->appendParagraph( + pht( + 'Unmute this object? You will receive notifications and email '. + 'again.')) + ->addSubmitButton(pht('Unmute')); + } + + return $dialog; + } + + +} diff --git a/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php b/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php index 5f371d69f3..caf860117e 100644 --- a/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php +++ b/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php @@ -42,6 +42,28 @@ final class PhabricatorSubscriptionsUIEventListener return; } + $src_phid = $object->getPHID(); + $subscribed_type = PhabricatorObjectHasSubscriberEdgeType::EDGECONST; + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $edges = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs(array($src_phid)) + ->withEdgeTypes( + array( + $subscribed_type, + $muted_type, + )) + ->withDestinationPHIDs(array($user_phid)) + ->execute(); + + if ($user_phid) { + $is_subscribed = isset($edges[$src_phid][$subscribed_type][$user_phid]); + $is_muted = isset($edges[$src_phid][$muted_type][$user_phid]); + } else { + $is_subscribed = false; + $is_muted = false; + } + if ($user_phid && $object->isAutomaticallySubscribed($user_phid)) { $sub_action = id(new PhabricatorActionView()) ->setWorkflow(true) @@ -51,22 +73,9 @@ final class PhabricatorSubscriptionsUIEventListener ->setName(pht('Automatically Subscribed')) ->setIcon('fa-check-circle lightgreytext'); } else { - $subscribed = false; - if ($user->isLoggedIn()) { - $src_phid = $object->getPHID(); - $edge_type = PhabricatorObjectHasSubscriberEdgeType::EDGECONST; - - $edges = id(new PhabricatorEdgeQuery()) - ->withSourcePHIDs(array($src_phid)) - ->withEdgeTypes(array($edge_type)) - ->withDestinationPHIDs(array($user_phid)) - ->execute(); - $subscribed = isset($edges[$src_phid][$edge_type][$user_phid]); - } - $can_interact = PhabricatorPolicyFilter::canInteract($user, $object); - if ($subscribed) { + if ($is_subscribed) { $sub_action = id(new PhabricatorActionView()) ->setWorkflow(true) ->setRenderAsForm(true) @@ -89,8 +98,26 @@ final class PhabricatorSubscriptionsUIEventListener } } + $mute_action = id(new PhabricatorActionView()) + ->setWorkflow(true) + ->setHref('/subscriptions/mute/'.$object->getPHID().'/') + ->setDisabled(!$user_phid); + + if (!$is_muted) { + $mute_action + ->setName(pht('Mute Notifications')) + ->setIcon('fa-volume-up'); + } else { + $mute_action + ->setName(pht('Unmute Notifications')) + ->setIcon('fa-volume-off') + ->setColor(PhabricatorActionView::RED); + } + + $actions = $event->getValue('actions'); $actions[] = $sub_action; + $actions[] = $mute_action; $event->setValue('actions', $actions); } diff --git a/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php b/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php new file mode 100644 index 0000000000..1f592239ba --- /dev/null +++ b/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php @@ -0,0 +1,16 @@ +applyOldRecipientLists(); + if ($object instanceof PhabricatorSubscribableInterface) { + $this->mailMutedPHIDs = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorMutedByEdgeType::EDGECONST); + } else { + $this->mailMutedPHIDs = array(); + } + $mail_xactions = $this->getTransactionsForMail($object, $xactions); $stamps = $this->newMailStamps($object, $xactions); foreach ($stamps as $stamp) { @@ -2662,6 +2671,11 @@ abstract class PhabricatorApplicationTransactionEditor $mail_xactions); } + $muted_phids = $this->mailMutedPHIDs; + if (!is_array($muted_phids)) { + $muted_phids = array(); + } + $mail ->setSensitiveContent(false) ->setFrom($this->getActingAsPHID()) @@ -2670,6 +2684,7 @@ abstract class PhabricatorApplicationTransactionEditor ->setThreadID($this->getMailThreadID($object), $this->getIsNewObject()) ->setRelatedPHID($object->getPHID()) ->setExcludeMailRecipientPHIDs($this->getExcludeMailRecipientPHIDs()) + ->setMutedPHIDs($muted_phids) ->setForceHeraldMailRecipientPHIDs($this->heraldForcedEmailPHIDs) ->setMailTags($mail_tags) ->setIsBulk(true) @@ -3186,6 +3201,18 @@ abstract class PhabricatorApplicationTransactionEditor $related_phids = $this->feedRelatedPHIDs; $subscribed_phids = $this->feedNotifyPHIDs; + // Remove muted users from the subscription list so they don't get + // notifications, either. + $muted_phids = $this->mailMutedPHIDs; + if (!is_array($muted_phids)) { + $muted_phids = array(); + } + $subscribed_phids = array_fuse($subscribed_phids); + foreach ($muted_phids as $muted_phid) { + unset($subscribed_phids[$muted_phid]); + } + $subscribed_phids = array_values($subscribed_phids); + $story_type = $this->getFeedStoryType(); $story_data = $this->getFeedStoryData($object, $xactions); @@ -3632,6 +3659,7 @@ abstract class PhabricatorApplicationTransactionEditor 'mustEncrypt', 'mailStamps', 'mailUnexpandablePHIDs', + 'mailMutedPHIDs', ); } diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index d5c28b83bf..d27370cd44 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -643,6 +643,8 @@ abstract class PhabricatorApplicationTransaction case PhabricatorObjectMentionsObjectEdgeType::EDGECONST: case ManiphestTaskHasDuplicateTaskEdgeType::EDGECONST: case ManiphestTaskIsDuplicateOfTaskEdgeType::EDGECONST: + case PhabricatorMutedEdgeType::EDGECONST: + case PhabricatorMutedByEdgeType::EDGECONST: return true; break; case PhabricatorObjectMentionedByObjectEdgeType::EDGECONST: diff --git a/src/view/layout/PhabricatorActionView.php b/src/view/layout/PhabricatorActionView.php index f6de8eca5b..d43cc9428b 100644 --- a/src/view/layout/PhabricatorActionView.php +++ b/src/view/layout/PhabricatorActionView.php @@ -21,6 +21,7 @@ final class PhabricatorActionView extends AphrontView { private $order; private $color; private $type; + private $highlight; const TYPE_DIVIDER = 'type-divider'; const TYPE_LABEL = 'label'; @@ -72,6 +73,15 @@ final class PhabricatorActionView extends AphrontView { return $this->href; } + public function setHighlight($highlight) { + $this->highlight = $highlight; + return $this; + } + + public function getHighlight() { + return $this->highlight; + } + public function setIcon($icon) { $this->icon = $icon; return $this; diff --git a/webroot/rsrc/css/phui/phui-action-list.css b/webroot/rsrc/css/phui/phui-action-list.css index 5e32a1ea0a..e7ee38a8bf 100644 --- a/webroot/rsrc/css/phui/phui-action-list.css +++ b/webroot/rsrc/css/phui/phui-action-list.css @@ -95,15 +95,20 @@ color: {$sky}; } -.device-desktop .phabricator-action-view-href.action-item-red:hover - .phabricator-action-view-item { - background-color: {$sh-redbackground}; - color: {$sh-redtext}; +.phabricator-action-view.action-item-red { + background-color: {$sh-redbackground}; } -.device-desktop .phabricator-action-view-href.action-item-red:hover +.phabricator-action-view.action-item-red .phabricator-action-view-item, +.phabricator-action-view.action-item-red .phabricator-action-view-icon { + color: {$sh-redtext}; +} + +.device-desktop .phabricator-action-view.action-item-red:hover + .phabricator-action-view-item, +.device-desktop .phabricator-action-view.action-item-red:hover .phabricator-action-view-icon { - color: {$red}; + color: {$red}; } .phabricator-action-view-label .phabricator-action-view-item, From 705ff8d33de04b09a31753426850d2c7da9a046a Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 11:40:12 -0800 Subject: [PATCH 46/67] Remove `addHighlight()` action view methods Summary: These didn't actually get used by D19033. Test Plan: Grep. Differential Revision: https://secure.phabricator.com/D19034 --- src/view/layout/PhabricatorActionView.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/view/layout/PhabricatorActionView.php b/src/view/layout/PhabricatorActionView.php index d43cc9428b..f6de8eca5b 100644 --- a/src/view/layout/PhabricatorActionView.php +++ b/src/view/layout/PhabricatorActionView.php @@ -21,7 +21,6 @@ final class PhabricatorActionView extends AphrontView { private $order; private $color; private $type; - private $highlight; const TYPE_DIVIDER = 'type-divider'; const TYPE_LABEL = 'label'; @@ -73,15 +72,6 @@ final class PhabricatorActionView extends AphrontView { return $this->href; } - public function setHighlight($highlight) { - $this->highlight = $highlight; - return $this; - } - - public function getHighlight() { - return $this->highlight; - } - public function setIcon($icon) { $this->icon = $icon; return $this; From d1e273daf62fc12393f107804b7e19c55bbb25d9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 12:41:11 -0800 Subject: [PATCH 47/67] Remove completely pointless load of every repository when viewing a repository URI Summary: See D18176. This query has no effect (other than wasting resources) and the result is unused. `$repository` already has the URI loaded because we load them unconditionally during request initialization. Test Plan: Viewed repository URIs. Subscribers: jmeador Differential Revision: https://secure.phabricator.com/D19036 --- .../controller/DiffusionRepositoryURIViewController.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php b/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php index 91ffbb473f..e923ebfc20 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php @@ -23,14 +23,10 @@ final class DiffusionRepositoryURIViewController return new Aphront404Response(); } - // For display, reload the URI by loading it through the repository. This + // For display, access the URI by loading it through the repository. This // may adjust builtin URIs for repository configuration, so we may end up // with a different view of builtin URIs than we'd see if we loaded them // directly from the database. See T12884. - $repository_with_uris = id(new PhabricatorRepositoryQuery()) - ->setViewer($viewer) - ->needURIs(true) - ->execute(); $repository_uris = $repository->getURIs(); $repository_uris = mpull($repository_uris, null, 'getID'); From f028aa6f60dd6382b59153e681645b478cdd0e62 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 14:20:32 -0800 Subject: [PATCH 48/67] Track closed date and closing user for tasks explicitly Summary: Ref T4434. Although some of the use cases for this data are better fits for Facts, this data is reasonable to track separately. I have an approximate view of it already ("closed, ordered by date modified") that's useful to review things that were fixed recently. This lets us make that view more effective. This just adds (and populates) the storage. Followups will add Conduit, Export, Search, and UI support. This is slightly tricky because merges work oddly (see T13020). Test Plan: - Ran migration, checked database for sensible results. - Created a task in open/closed status, got the right database values. - Modified a task to close/open it, got the right values. - Merged an open task, got updates. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19037 --- .../20180208.maniphest.01.close.sql | 5 ++ .../20180208.maniphest.02.populate.php | 65 +++++++++++++++++++ .../maniphest/storage/ManiphestTask.php | 11 ++++ .../ManiphestTaskMergedIntoTransaction.php | 2 +- .../ManiphestTaskStatusTransaction.php | 2 +- .../xaction/ManiphestTaskTransactionType.php | 23 +++++++ 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 resources/sql/autopatches/20180208.maniphest.01.close.sql create mode 100644 resources/sql/autopatches/20180208.maniphest.02.populate.php diff --git a/resources/sql/autopatches/20180208.maniphest.01.close.sql b/resources/sql/autopatches/20180208.maniphest.01.close.sql new file mode 100644 index 0000000000..856300e9ba --- /dev/null +++ b/resources/sql/autopatches/20180208.maniphest.01.close.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + ADD closedEpoch INT UNSIGNED; + +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + ADD closerPHID VARBINARY(64); diff --git a/resources/sql/autopatches/20180208.maniphest.02.populate.php b/resources/sql/autopatches/20180208.maniphest.02.populate.php new file mode 100644 index 0000000000..16aa2bf57b --- /dev/null +++ b/resources/sql/autopatches/20180208.maniphest.02.populate.php @@ -0,0 +1,65 @@ +establishConnection('w'); +$viewer = PhabricatorUser::getOmnipotentUser(); + +foreach (new LiskMigrationIterator($table) as $task) { + if ($task->getClosedEpoch()) { + // Task already has a closed date. + continue; + } + + $status = $task->getStatus(); + if (!ManiphestTaskStatus::isClosedStatus($status)) { + // Task isn't closed. + continue; + } + + // Look through the transactions from newest to oldest until we find one + // where the task was closed. A merge also counts as a close, even though + // it doesn't currently produce a separate transaction. + + $type_merge = ManiphestTaskStatusTransaction::TRANSACTIONTYPE; + $type_status = ManiphestTaskMergedIntoTransaction::TRANSACTIONTYPE; + + $xactions = id(new ManiphestTransactionQuery()) + ->setViewer($viewer) + ->withObjectPHIDs(array($task->getPHID())) + ->withTransactionTypes( + array( + $type_merge, + $type_status, + )) + ->execute(); + foreach ($xactions as $xaction) { + $old = $xaction->getOldValue(); + $new = $xaction->getNewValue(); + + $type = $xaction->getTransactionType(); + + // If this is a status change, but is not a close, don't use it. + // (We always use merges, even though it's possible to merge a task which + // was previously closed: we can't tell when this happens very easily.) + if ($type === $type_status) { + if (!ManiphestTaskStatus::isClosedStatus($new)) { + continue; + } + + if ($old && ManiphestTaskStatus::isClosedStatus($old)) { + continue; + } + } + + queryfx( + $conn, + 'UPDATE %T SET closedEpoch = %d, closerPHID = %ns + WHERE id = %d', + $table->getTableName(), + $xaction->getDateCreated(), + $xaction->getAuthorPHID(), + $task->getID()); + + break; + } +} diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index e19886d3ff..7ff70abd80 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -44,6 +44,9 @@ final class ManiphestTask extends ManiphestDAO protected $points; protected $subtype; + protected $closedEpoch; + protected $closerPHID; + private $subscriberPHIDs = self::ATTACHABLE; private $groupByProjectPHID = self::ATTACHABLE; private $customFields = self::ATTACHABLE; @@ -90,6 +93,8 @@ final class ManiphestTask extends ManiphestDAO 'points' => 'double?', 'bridgedObjectPHID' => 'phid?', 'subtype' => 'text64', + 'closedEpoch' => 'epoch?', + 'closerPHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_phid' => null, @@ -131,6 +136,12 @@ final class ManiphestTask extends ManiphestDAO 'key_subtype' => array( 'columns' => array('subtype'), ), + 'key_closed' => array( + 'columns' => array('closedEpoch'), + ), + 'key_closer' => array( + 'columns' => array('closerPHID', 'closedEpoch'), + ), ), ) + parent::getConfiguration(); } diff --git a/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php b/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php index cd0cad6a39..630f5190ce 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php +++ b/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php @@ -10,7 +10,7 @@ final class ManiphestTaskMergedIntoTransaction } public function applyInternalEffects($object, $value) { - $object->setStatus(ManiphestTaskStatus::getDuplicateStatus()); + $this->updateStatus($object, ManiphestTaskStatus::getDuplicateStatus()); } public function getActionName() { diff --git a/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php b/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php index dd51a63799..6f4b558e05 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php +++ b/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php @@ -10,7 +10,7 @@ final class ManiphestTaskStatusTransaction } public function applyInternalEffects($object, $value) { - $object->setStatus($value); + $this->updateStatus($object, $value); } public function shouldHide() { diff --git a/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php b/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php index c59de163c6..836e7765b8 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php +++ b/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php @@ -3,4 +3,27 @@ abstract class ManiphestTaskTransactionType extends PhabricatorModularTransactionType { + protected function updateStatus($object, $new_value) { + $old_value = $object->getStatus(); + $object->setStatus($new_value); + + // If this status change closes or opens the task, update the closed + // date and actor PHID. + $old_closed = ManiphestTaskStatus::isClosedStatus($old_value); + $new_closed = ManiphestTaskStatus::isClosedStatus($new_value); + + $is_close = ($new_closed && !$old_closed); + $is_open = (!$new_closed && $old_closed); + + if ($is_close) { + $object + ->setClosedEpoch(PhabricatorTime::getNow()) + ->setCloserPHID($this->getActingAsPHID()); + } else if ($is_open) { + $object + ->setClosedEpoch(null) + ->setCloserPHID(null); + } + } + } From 4c4707e467633032fc56797d6b620b626ff21e18 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 14:48:16 -0800 Subject: [PATCH 49/67] Provide task closed date via Conduit API, data export pipeline, and in list UI Summary: Depends on D19037. Ref T4434. Adds closed date to `maniphest.search` and "Export Data". When a task has been closed, show the closed date with a checkmark in the UI instead of the modified date. Test Plan: - Exported data to CSV, saw close information. - Saw close information in `/maniphest/`. - Queried for close information via `maniphest.search`. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19038 --- .../query/ManiphestTaskSearchEngine.php | 20 ++++++++++++++++++ .../maniphest/storage/ManiphestTask.php | 17 +++++++++++++++ .../maniphest/view/ManiphestTaskListView.php | 21 ++++++++++++++++--- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index ad668db376..a5c98dc202 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -456,6 +456,15 @@ final class ManiphestTaskSearchEngine id(new PhabricatorStringExportField()) ->setKey('statusName') ->setLabel(pht('Status Name')), + id(new PhabricatorEpochExportField()) + ->setKey('dateClosed') + ->setLabel(pht('Date Closed')), + id(new PhabricatorPHIDExportField()) + ->setKey('closerPHID') + ->setLabel(pht('Closer PHID')), + id(new PhabricatorStringExportField()) + ->setKey('closer') + ->setLabel(pht('Closer')), id(new PhabricatorStringExportField()) ->setKey('priority') ->setLabel(pht('Priority')), @@ -492,6 +501,7 @@ final class ManiphestTaskSearchEngine foreach ($tasks as $task) { $phids[] = $task->getAuthorPHID(); $phids[] = $task->getOwnerPHID(); + $phids[] = $task->getCloserPHID(); } $handles = $viewer->loadHandles($phids); @@ -512,6 +522,13 @@ final class ManiphestTaskSearchEngine $owner_name = null; } + $closer_phid = $task->getCloserPHID(); + if ($closer_phid) { + $closer_name = $handles[$closer_phid]->getName(); + } else { + $closer_name = null; + } + $status_value = $task->getStatus(); $status_name = ManiphestTaskStatus::getTaskStatusName($status_value); @@ -534,6 +551,9 @@ final class ManiphestTaskSearchEngine 'title' => $task->getTitle(), 'uri' => PhabricatorEnv::getProductionURI($task->getURI()), 'description' => $task->getDescription(), + 'dateClosed' => $task->getClosedEpoch(), + 'closerPHID' => $closer_phid, + 'closer' => $closer_name, ); } diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index 7ff70abd80..a93fe58c3f 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -513,6 +513,16 @@ final class ManiphestTask extends ManiphestDAO ->setKey('subtype') ->setType('string') ->setDescription(pht('Subtype of the task.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('closerPHID') + ->setType('phid?') + ->setDescription( + pht('User who closed the task, if the task is closed.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('dateClosed') + ->setType('int?') + ->setDescription( + pht('Epoch timestamp when the task was closed.')), ); } @@ -532,6 +542,11 @@ final class ManiphestTask extends ManiphestDAO 'color' => ManiphestTaskPriority::getTaskPriorityColor($priority_value), ); + $closed_epoch = $this->getClosedEpoch(); + if ($closed_epoch !== null) { + $closed_epoch = (int)$closed_epoch; + } + return array( 'name' => $this->getTitle(), 'description' => array( @@ -543,6 +558,8 @@ final class ManiphestTask extends ManiphestDAO 'priority' => $priority_info, 'points' => $this->getPoints(), 'subtype' => $this->getSubtype(), + 'closerPHID' => $this->getCloserPHID(), + 'dateClosed' => $closed_epoch, ); } diff --git a/src/applications/maniphest/view/ManiphestTaskListView.php b/src/applications/maniphest/view/ManiphestTaskListView.php index de6b386ac8..ba17b8e25d 100644 --- a/src/applications/maniphest/view/ManiphestTaskListView.php +++ b/src/applications/maniphest/view/ManiphestTaskListView.php @@ -86,9 +86,24 @@ final class ManiphestTaskListView extends ManiphestView { $item->setStatusIcon($icon.' '.$color, $tooltip); - $item->addIcon( - 'none', - phabricator_datetime($task->getDateModified(), $this->getUser())); + if ($task->isClosed()) { + $closed_epoch = $task->getClosedEpoch(); + + // We don't expect a task to be closed without a closed epoch, but + // recover if we find one. This can happen with older objects or with + // lipsum test data. + if (!$closed_epoch) { + $closed_epoch = $task->getDateModified(); + } + + $item->addIcon( + 'fa-check-square-o grey', + phabricator_datetime($closed_epoch, $this->getUser())); + } else { + $item->addIcon( + 'none', + phabricator_datetime($task->getDateModified(), $this->getUser())); + } if ($this->showSubpriorityControls) { $item->setGrippable(true); From e26a784dcf3e2e90c05229f3d6f71dfd9594348b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 15:13:03 -0800 Subject: [PATCH 50/67] Allow tasks to be filtered and ordered by closed date Summary: Depends on D19038. Fixes T4434. Updates the SearchEngine and Query to handle these fields. Test Plan: Filtered and ordered by date and closer. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19039 --- .../maniphest/query/ManiphestTaskQuery.php | 49 ++++++++++++++++++- .../query/ManiphestTaskSearchEngine.php | 22 +++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php index cfc69722d8..f7c1551be7 100644 --- a/src/applications/maniphest/query/ManiphestTaskQuery.php +++ b/src/applications/maniphest/query/ManiphestTaskQuery.php @@ -23,6 +23,9 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { private $parentTaskIDs; private $subtaskIDs; private $subtypes; + private $closedEpochMin; + private $closedEpochMax; + private $closerPHIDs; private $status = 'status-any'; const STATUS_ANY = 'status-any'; @@ -179,6 +182,17 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { return $this; } + public function withClosedEpochBetween($min, $max) { + $this->closedEpochMin = $min; + $this->closedEpochMax = $max; + return $this; + } + + public function withCloserPHIDs(array $phids) { + $this->closerPHIDs = $phids; + return $this; + } + public function needSubscriberPHIDs($bool) { $this->needSubscriberPHIDs = $bool; return $this; @@ -379,6 +393,27 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { $this->dateModifiedBefore); } + if ($this->closedEpochMin !== null) { + $where[] = qsprintf( + $conn, + 'task.closedEpoch >= %d', + $this->closedEpochMin); + } + + if ($this->closedEpochMax !== null) { + $where[] = qsprintf( + $conn, + 'task.closedEpoch <= %d', + $this->closedEpochMax); + } + + if ($this->closerPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'task.closerPHID IN (%Ls)', + $this->closerPHIDs); + } + if ($this->priorities !== null) { $where[] = qsprintf( $conn, @@ -722,7 +757,11 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'outdated' => array( 'vector' => array('-updated', '-id'), 'name' => pht('Date Updated (Oldest First)'), - ), + ), + 'closed' => array( + 'vector' => array('closed', 'id'), + 'name' => pht('Date Closed (Latest First)'), + ), 'title' => array( 'vector' => array('title', 'id'), 'name' => pht('Title'), @@ -741,6 +780,7 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'outdated', 'newest', 'oldest', + 'closed', 'title', )) + $orders; @@ -790,6 +830,12 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'column' => 'dateModified', 'type' => 'int', ), + 'closed' => array( + 'table' => 'task', + 'column' => 'closedEpoch', + 'type' => 'int', + 'null' => 'tail', + ), ); } @@ -808,6 +854,7 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'status' => $task->getStatus(), 'title' => $task->getTitle(), 'updated' => $task->getDateModified(), + 'closed' => $task->getClosedEpoch(), ); foreach ($keys as $key) { diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index a5c98dc202..565bc7a8f4 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -126,6 +126,17 @@ final class ManiphestTaskSearchEngine id(new PhabricatorSearchDateField()) ->setLabel(pht('Updated Before')) ->setKey('modifiedEnd'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Closed After')) + ->setKey('closedStart'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Closed Before')) + ->setKey('closedEnd'), + id(new PhabricatorUsersSearchField()) + ->setLabel(pht('Closed By')) + ->setKey('closerPHIDs') + ->setAliases(array('closer', 'closerPHID', 'closers')) + ->setDescription(pht('Search for tasks closed by certain users.')), id(new PhabricatorSearchTextField()) ->setLabel(pht('Page Size')) ->setKey('limit'), @@ -153,6 +164,9 @@ final class ManiphestTaskSearchEngine 'createdEnd', 'modifiedStart', 'modifiedEnd', + 'closedStart', + 'closedEnd', + 'closerPHIDs', 'limit', ); } @@ -208,6 +222,14 @@ final class ManiphestTaskSearchEngine $query->withDateModifiedBefore($map['modifiedEnd']); } + if ($map['closedStart'] || $map['closedEnd']) { + $query->withClosedEpochBetween($map['closedStart'], $map['closedEnd']); + } + + if ($map['closerPHIDs']) { + $query->withCloserPHIDs($map['closerPHIDs']); + } + if ($map['hasParents'] !== null) { $query->withOpenParents($map['hasParents']); } From 6ea1b8df9bbb0e6f6f365659b6a6ff14e49168e6 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 15:52:59 -0800 Subject: [PATCH 51/67] Colorize filetree for adds, moves, and deletes Summary: See PHI356. Makes it easier to pick out change types in the filetree view in Differential. Test Plan: Created a diff with adds, copies, moves, deletions, and binary files. Viewed in Differential, had an easier time picking stuff out. Differential Revision: https://secure.phabricator.com/D19040 --- resources/celerity/map.php | 6 +-- .../storage/DifferentialChangeset.php | 45 +++++++++++++++++++ ...rentialChangesetFileTreeSideNavBuilder.php | 10 +++-- .../css/layout/phabricator-filetree-view.css | 12 +++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 17138d6604..70896d400d 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -12,7 +12,7 @@ return array( 'core.pkg.css' => 'ce8c2a58', 'core.pkg.js' => '4c79d74f', 'darkconsole.pkg.js' => '1f9a31bc', - 'differential.pkg.css' => '45951e9e', + 'differential.pkg.css' => '1522c3ad', 'differential.pkg.js' => '19ee9979', 'diffusion.pkg.css' => 'a2d17c7d', 'diffusion.pkg.js' => '6134c5a1', @@ -121,7 +121,7 @@ return array( 'rsrc/css/font/font-awesome.css' => 'e838e088', 'rsrc/css/font/font-lato.css' => 'c7ccd872', 'rsrc/css/font/phui-font-icon-base.css' => '870a7360', - 'rsrc/css/layout/phabricator-filetree-view.css' => 'fccf9f82', + 'rsrc/css/layout/phabricator-filetree-view.css' => 'ea5b30a9', 'rsrc/css/layout/phabricator-source-code-view.css' => 'aea41829', 'rsrc/css/phui/button/phui-button-bar.css' => 'f1ff5494', 'rsrc/css/phui/button/phui-button-simple.css' => '8e1baf68', @@ -784,7 +784,7 @@ return array( 'phabricator-favicon' => '1fe2510c', 'phabricator-feed-css' => 'ecd4ec57', 'phabricator-file-upload' => '680ea2c8', - 'phabricator-filetree-view-css' => 'fccf9f82', + 'phabricator-filetree-view-css' => 'ea5b30a9', 'phabricator-flag-css' => 'bba8f811', 'phabricator-keyboard-shortcut' => '1ae869f2', 'phabricator-keyboard-shortcut-manager' => 'c19dd9b9', diff --git a/src/applications/differential/storage/DifferentialChangeset.php b/src/applications/differential/storage/DifferentialChangeset.php index ebdaeacd0a..dbb06fe72b 100644 --- a/src/applications/differential/storage/DifferentialChangeset.php +++ b/src/applications/differential/storage/DifferentialChangeset.php @@ -221,6 +221,51 @@ final class DifferentialChangeset return $this->assertAttached($this->diff); } + public function newFileTreeIcon() { + $file_type = $this->getFileType(); + $change_type = $this->getChangeType(); + + $change_icons = array( + DifferentialChangeType::TYPE_DELETE => 'fa-file-o', + ); + + if (isset($change_icons[$change_type])) { + $icon = $change_icons[$change_type]; + } else { + $icon = DifferentialChangeType::getIconForFileType($file_type); + } + + $change_colors = array( + DifferentialChangeType::TYPE_ADD => 'green', + DifferentialChangeType::TYPE_DELETE => 'red', + DifferentialChangeType::TYPE_MOVE_AWAY => 'orange', + DifferentialChangeType::TYPE_MOVE_HERE => 'orange', + DifferentialChangeType::TYPE_COPY_HERE => 'orange', + DifferentialChangeType::TYPE_MULTICOPY => 'orange', + ); + + $color = idx($change_colors, $change_type, 'bluetext'); + + return id(new PHUIIconView()) + ->setIcon($icon.' '.$color); + } + + public function getFileTreeClass() { + switch ($this->getChangeType()) { + case DifferentialChangeType::TYPE_ADD: + return 'filetree-added'; + case DifferentialChangeType::TYPE_DELETE: + return 'filetree-deleted'; + case DifferentialChangeType::TYPE_MOVE_AWAY: + case DifferentialChangeType::TYPE_MOVE_HERE: + case DifferentialChangeType::TYPE_COPY_HERE: + case DifferentialChangeType::TYPE_MULTICOPY: + return 'filetree-movecopy'; + } + + return null; + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php index 14050e942c..9781fb0d02 100644 --- a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php +++ b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php @@ -83,6 +83,9 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { while (($path = $path->getNextNode())) { $data = $path->getData(); + $classes = array(); + $classes[] = 'phabricator-filetree-item'; + $name = $path->getName(); $style = 'padding-left: '.(2 + (3 * $path->getDepth())).'px'; @@ -90,8 +93,9 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { if ($data) { $href = '#'.$data->getAnchorName(); $title = $name; - $icon = id(new PHUIIconView()) - ->setIcon('fa-file-text-o bluetext'); + + $icon = $data->newFileTreeIcon(); + $classes[] = $data->getFileTreeClass(); } else { $name .= '/'; $title = $path->getFullPath().'/'; @@ -112,7 +116,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { 'href' => $href, 'style' => $style, 'title' => $title, - 'class' => 'phabricator-filetree-item', + 'class' => implode(' ', $classes), ), array($icon, $name_element)); } diff --git a/webroot/rsrc/css/layout/phabricator-filetree-view.css b/webroot/rsrc/css/layout/phabricator-filetree-view.css index b247f5e4f9..21bbe9f0af 100644 --- a/webroot/rsrc/css/layout/phabricator-filetree-view.css +++ b/webroot/rsrc/css/layout/phabricator-filetree-view.css @@ -54,3 +54,15 @@ background-color: {$hovergrey}; border-left: 4px solid {$sky}; } + +.phabricator-filetree .filetree-added { + background: {$sh-greenbackground}; +} + +.phabricator-filetree .filetree-deleted { + background: {$sh-redbackground}; +} + +.phabricator-filetree .filetree-movecopy { + background: {$sh-orangebackground}; +} From 261a4a0e51fd1be6f5b1a89bba620f4ef9ad7e6d Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 16:55:54 -0800 Subject: [PATCH 52/67] Add inline comment counts to the filetree view Summary: See PHI356. Adds inline comment and done counts to the filetree. Also makes the filetree wider by default. Test Plan: Fiddled with filetrees in different browsers on different revisions. Added inlines, marked them done/undone. Differential Revision: https://secure.phabricator.com/D19041 --- resources/celerity/map.php | 78 +++++++++---------- .../view/DifferentialChangesetDetailView.php | 1 + ...rentialChangesetFileTreeSideNavBuilder.php | 12 ++- .../rsrc/css/aphront/phabricator-nav-view.css | 6 +- .../css/layout/phabricator-filetree-view.css | 33 ++++++-- .../rsrc/js/application/diff/DiffChangeset.js | 74 +++++++++++++++++- .../js/application/diff/DiffChangesetList.js | 5 ++ .../rsrc/js/core/behavior-phabricator-nav.js | 4 + 8 files changed, 164 insertions(+), 49 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 70896d400d..f3ead4de53 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -9,11 +9,11 @@ return array( 'names' => array( 'conpherence.pkg.css' => 'e68cf1fa', 'conpherence.pkg.js' => '15191c65', - 'core.pkg.css' => 'ce8c2a58', - 'core.pkg.js' => '4c79d74f', + 'core.pkg.css' => 'e4f098a5', + 'core.pkg.js' => '3ac6e174', 'darkconsole.pkg.js' => '1f9a31bc', - 'differential.pkg.css' => '1522c3ad', - 'differential.pkg.js' => '19ee9979', + 'differential.pkg.css' => '113e692c', + 'differential.pkg.js' => '5d53d5ce', 'diffusion.pkg.css' => 'a2d17c7d', 'diffusion.pkg.js' => '6134c5a1', 'favicon.ico' => '30672e08', @@ -31,7 +31,7 @@ return array( 'rsrc/css/aphront/multi-column.css' => '84cc6640', 'rsrc/css/aphront/notification.css' => '457861ec', 'rsrc/css/aphront/panel-view.css' => '8427b78d', - 'rsrc/css/aphront/phabricator-nav-view.css' => 'faf6a6fc', + 'rsrc/css/aphront/phabricator-nav-view.css' => '028126f6', 'rsrc/css/aphront/table-view.css' => '8c9bbafe', 'rsrc/css/aphront/tokenizer.css' => '15d5ff71', 'rsrc/css/aphront/tooltip.css' => '173b9431', @@ -121,7 +121,7 @@ return array( 'rsrc/css/font/font-awesome.css' => 'e838e088', 'rsrc/css/font/font-lato.css' => 'c7ccd872', 'rsrc/css/font/phui-font-icon-base.css' => '870a7360', - 'rsrc/css/layout/phabricator-filetree-view.css' => 'ea5b30a9', + 'rsrc/css/layout/phabricator-filetree-view.css' => 'b912ad97', 'rsrc/css/layout/phabricator-source-code-view.css' => 'aea41829', 'rsrc/css/phui/button/phui-button-bar.css' => 'f1ff5494', 'rsrc/css/phui/button/phui-button-simple.css' => '8e1baf68', @@ -395,8 +395,8 @@ return array( 'rsrc/js/application/dashboard/behavior-dashboard-move-panels.js' => '408bf173', 'rsrc/js/application/dashboard/behavior-dashboard-query-panel-select.js' => '453c5375', 'rsrc/js/application/dashboard/behavior-dashboard-tab-panel.js' => 'd4eecc63', - 'rsrc/js/application/diff/DiffChangeset.js' => '99abf4cd', - 'rsrc/js/application/diff/DiffChangesetList.js' => '3b77efdd', + 'rsrc/js/application/diff/DiffChangeset.js' => 'b49b59d6', + 'rsrc/js/application/diff/DiffChangesetList.js' => '1f2e5265', 'rsrc/js/application/diff/DiffInline.js' => 'e83d28f3', 'rsrc/js/application/diff/behavior-preview-link.js' => '051c7832', 'rsrc/js/application/differential/behavior-comment-preview.js' => '51c5ad07', @@ -498,7 +498,7 @@ return array( 'rsrc/js/core/behavior-more.js' => 'a80d0378', 'rsrc/js/core/behavior-object-selector.js' => '77c1f0b0', 'rsrc/js/core/behavior-oncopy.js' => '2926fff2', - 'rsrc/js/core/behavior-phabricator-nav.js' => '947753e0', + 'rsrc/js/core/behavior-phabricator-nav.js' => '81144dfa', 'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => 'acd29eee', 'rsrc/js/core/behavior-read-only-warning.js' => 'ba158207', 'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b', @@ -657,7 +657,7 @@ return array( 'javelin-behavior-phabricator-keyboard-pager' => 'a8da01f0', 'javelin-behavior-phabricator-keyboard-shortcuts' => '01fca1f0', 'javelin-behavior-phabricator-line-linker' => '1499a8cb', - 'javelin-behavior-phabricator-nav' => '947753e0', + 'javelin-behavior-phabricator-nav' => '81144dfa', 'javelin-behavior-phabricator-notification-example' => '8ce821c5', 'javelin-behavior-phabricator-object-selector' => '77c1f0b0', 'javelin-behavior-phabricator-oncopy' => '2926fff2', @@ -775,8 +775,8 @@ return array( 'phabricator-darklog' => 'c8e1ffe3', 'phabricator-darkmessage' => 'c48cccdd', 'phabricator-dashboard-css' => 'fe5b1869', - 'phabricator-diff-changeset' => '99abf4cd', - 'phabricator-diff-changeset-list' => '3b77efdd', + 'phabricator-diff-changeset' => 'b49b59d6', + 'phabricator-diff-changeset-list' => '1f2e5265', 'phabricator-diff-inline' => 'e83d28f3', 'phabricator-drag-and-drop-file-upload' => '58dea2fa', 'phabricator-draggable-list' => 'bea6e7f4', @@ -784,12 +784,12 @@ return array( 'phabricator-favicon' => '1fe2510c', 'phabricator-feed-css' => 'ecd4ec57', 'phabricator-file-upload' => '680ea2c8', - 'phabricator-filetree-view-css' => 'ea5b30a9', + 'phabricator-filetree-view-css' => 'b912ad97', 'phabricator-flag-css' => 'bba8f811', 'phabricator-keyboard-shortcut' => '1ae869f2', 'phabricator-keyboard-shortcut-manager' => 'c19dd9b9', 'phabricator-main-menu-view' => '1802a242', - 'phabricator-nav-view-css' => 'faf6a6fc', + 'phabricator-nav-view-css' => '028126f6', 'phabricator-notification' => '008faf9c', 'phabricator-notification-css' => '457861ec', 'phabricator-notification-menu-css' => '10685bd4', @@ -1044,6 +1044,10 @@ return array( 'javelin-uri', 'javelin-routable', ), + '1f2e5265' => array( + 'javelin-install', + 'phuix-button-view', + ), '1f6794f6' => array( 'javelin-behavior', 'javelin-stratcom', @@ -1143,10 +1147,6 @@ return array( 'javelin-dom', 'javelin-magical-init', ), - '3b77efdd' => array( - 'javelin-install', - 'phuix-button-view', - ), '3cb0b2fc' => array( 'javelin-behavior', 'javelin-dom', @@ -1561,6 +1561,16 @@ return array( '7f243deb' => array( 'javelin-install', ), + '81144dfa' => array( + 'javelin-behavior', + 'javelin-behavior-device', + 'javelin-stratcom', + 'javelin-dom', + 'javelin-magical-init', + 'javelin-vector', + 'javelin-request', + 'javelin-util', + ), '834a1173' => array( 'javelin-behavior', 'javelin-scrollbar', @@ -1648,16 +1658,6 @@ return array( 'javelin-workflow', 'javelin-dom', ), - '947753e0' => array( - 'javelin-behavior', - 'javelin-behavior-device', - 'javelin-stratcom', - 'javelin-dom', - 'javelin-magical-init', - 'javelin-vector', - 'javelin-request', - 'javelin-util', - ), '949c0fe5' => array( 'javelin-install', ), @@ -1678,17 +1678,6 @@ return array( 'javelin-mask', 'phabricator-drag-and-drop-file-upload', ), - '99abf4cd' => array( - 'javelin-dom', - 'javelin-util', - 'javelin-stratcom', - 'javelin-install', - 'javelin-workflow', - 'javelin-router', - 'javelin-behavior-device', - 'javelin-vector', - 'phabricator-diff-inline', - ), '9a6dd75c' => array( 'javelin-behavior', 'javelin-stratcom', @@ -1837,6 +1826,17 @@ return array( 'b3e7d692' => array( 'javelin-install', ), + 'b49b59d6' => array( + 'javelin-dom', + 'javelin-util', + 'javelin-stratcom', + 'javelin-install', + 'javelin-workflow', + 'javelin-router', + 'javelin-behavior-device', + 'javelin-vector', + 'phabricator-diff-inline', + ), 'b59e1e96' => array( 'javelin-behavior', 'javelin-stratcom', diff --git a/src/applications/differential/view/DifferentialChangesetDetailView.php b/src/applications/differential/view/DifferentialChangesetDetailView.php index d4a13745dc..cb697c2e9d 100644 --- a/src/applications/differential/view/DifferentialChangesetDetailView.php +++ b/src/applications/differential/view/DifferentialChangesetDetailView.php @@ -206,6 +206,7 @@ final class DifferentialChangesetDetailView extends AphrontView { 'displayPath' => hsprintf('%s', $display_parts), 'path' => $display_filename, 'icon' => $display_icon, + 'treeNodeID' => 'tree-node-'.$changeset->getAnchorName(), ), 'class' => $class, 'id' => $id, diff --git a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php index 9781fb0d02..1f699be8eb 100644 --- a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php +++ b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php @@ -96,11 +96,20 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { $icon = $data->newFileTreeIcon(); $classes[] = $data->getFileTreeClass(); + + $count = phutil_tag( + 'span', + array( + 'class' => 'filetree-progress-hint', + 'id' => 'tree-node-'.$data->getAnchorName(), + )); } else { $name .= '/'; $title = $path->getFullPath().'/'; $icon = id(new PHUIIconView()) ->setIcon('fa-folder-open blue'); + + $count = null; } $name_element = phutil_tag( @@ -110,6 +119,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { ), $name); + $filetree[] = javelin_tag( $href ? 'a' : 'span', array( @@ -118,7 +128,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { 'title' => $title, 'class' => implode(' ', $classes), ), - array($icon, $name_element)); + array($count, $icon, $name_element)); } $tree->destroy(); diff --git a/webroot/rsrc/css/aphront/phabricator-nav-view.css b/webroot/rsrc/css/aphront/phabricator-nav-view.css index e8081a55e6..f3320e3eae 100644 --- a/webroot/rsrc/css/aphront/phabricator-nav-view.css +++ b/webroot/rsrc/css/aphront/phabricator-nav-view.css @@ -44,7 +44,7 @@ position: fixed; top: 0; bottom: 0; - left: 205px; + left: 410px; width: 7px; cursor: col-resize; @@ -66,7 +66,7 @@ .device-desktop .phabricator-standard-page-body .has-drag-nav .phabricator-nav-content { - margin-left: 212px; + margin-left: 417px; } .device-desktop .phabricator-standard-page-body .has-drag-nav @@ -81,7 +81,7 @@ } .device-desktop .phui-navigation-shell .has-drag-nav .phabricator-nav-local { - width: 205px; + width: 410px; padding: 0; background: transparent; } diff --git a/webroot/rsrc/css/layout/phabricator-filetree-view.css b/webroot/rsrc/css/layout/phabricator-filetree-view.css index 21bbe9f0af..6497c37056 100644 --- a/webroot/rsrc/css/layout/phabricator-filetree-view.css +++ b/webroot/rsrc/css/layout/phabricator-filetree-view.css @@ -50,11 +50,6 @@ background-color: {$hovergrey}; } -.phabricator-filetree .phabricator-active-nav-focus { - background-color: {$hovergrey}; - border-left: 4px solid {$sky}; -} - .phabricator-filetree .filetree-added { background: {$sh-greenbackground}; } @@ -66,3 +61,31 @@ .phabricator-filetree .filetree-movecopy { background: {$sh-orangebackground}; } + +.phabricator-filetree .phabricator-active-nav-focus { + background-color: {$hovergrey}; + border-left: 4px solid {$sky}; +} + +.phabricator-filetree .filetree-progress-hint { + width: 24px; + margin-right: 6px; + display: inline-block; + padding: 0 4px; + border-radius: 4px; + font-size: smaller; + background: {$greybackground}; + text-align: center; + opacity: 0.5; +} + +.phabricator-filetree .filetree-comments-visible { + background: {$lightblue}; + opacity: 0.75; + color: {$darkgreytext}; +} + +.phabricator-filetree .filetree-comments-completed { + background: {$darkgreybackground}; + color: {$greytext}; +} diff --git a/webroot/rsrc/js/application/diff/DiffChangeset.js b/webroot/rsrc/js/application/diff/DiffChangeset.js index 72eeae294a..24d734573d 100644 --- a/webroot/rsrc/js/application/diff/DiffChangeset.js +++ b/webroot/rsrc/js/application/diff/DiffChangeset.js @@ -27,6 +27,7 @@ JX.install('DiffChangeset', { this._highlight = data.highlight; this._encoding = data.encoding; this._loaded = data.loaded; + this._treeNodeID = data.treeNodeID; this._leftID = data.left; this._rightID = data.right; @@ -62,6 +63,7 @@ JX.install('DiffChangeset', { _changesetList: null, _icon: null, + _treeNodeID: null, getLeftChangesetID: function() { return this._leftID; @@ -737,7 +739,8 @@ JX.install('DiffChangeset', { _rebuildAllInlines: function() { var rows = JX.DOM.scry(this._node, 'tr'); - for (var ii = 0; ii < rows.length; ii++) { + var ii; + for (ii = 0; ii < rows.length; ii++) { var row = rows[ii]; if (this._getRowType(row) != 'comment') { continue; @@ -749,6 +752,75 @@ JX.install('DiffChangeset', { } }, + redrawFileTree: function() { + var tree; + try { + tree = JX.$(this._treeNodeID); + } catch (e) { + return; + } + + var inlines = this._inlines; + var done = []; + var undone = []; + var inline; + + for (var ii = 0; ii < inlines.length; ii++) { + inline = inlines[ii]; + + if (inline.isDeleted()) { + continue; + } + + if (inline.isSynthetic()) { + continue; + } + + if (inline.isEditing()) { + continue; + } + + if (!inline.getID()) { + // These are new comments which have been cancelled, and do not + // count as anything. + continue; + } + + if (inline.isDraft()) { + continue; + } + + if (!inline.isDone()) { + undone.push(inline); + } else { + done.push(inline); + } + } + + var total = done.length + undone.length; + + var hint; + var is_visible; + var is_completed; + if (total) { + if (done.length) { + hint = [done.length, '/', total]; + } else { + hint = total; + } + is_visible = true; + is_completed = (done.length == total); + } else { + hint = '-'; + is_visible = false; + is_completed = false; + } + + JX.DOM.setContent(tree, hint); + JX.DOM.alterClass(tree, 'filetree-comments-visible', is_visible); + JX.DOM.alterClass(tree, 'filetree-comments-completed', is_completed); + }, + toggleVisibility: function() { this._visible = !this._visible; diff --git a/webroot/rsrc/js/application/diff/DiffChangesetList.js b/webroot/rsrc/js/application/diff/DiffChangesetList.js index ec0270ac12..e62d2f51dd 100644 --- a/webroot/rsrc/js/application/diff/DiffChangesetList.js +++ b/webroot/rsrc/js/application/diff/DiffChangesetList.js @@ -915,6 +915,11 @@ JX.install('DiffChangesetList', { this._bannerChangeset = null; this._redrawBanner(); + + var changesets = this._changesets; + for (var ii = 0; ii < changesets.length; ii++) { + changesets[ii].redrawFileTree(); + } }, _onscroll: function() { diff --git a/webroot/rsrc/js/core/behavior-phabricator-nav.js b/webroot/rsrc/js/core/behavior-phabricator-nav.js index e37680abf0..74909e447d 100644 --- a/webroot/rsrc/js/core/behavior-phabricator-nav.js +++ b/webroot/rsrc/js/core/behavior-phabricator-nav.js @@ -28,6 +28,10 @@ JX.behavior('phabricator-nav', function(config) { JX.enableDispatch(document.body, 'mousemove'); JX.DOM.listen(drag, 'mousedown', null, function(e) { + if (!e.isNormalMouseEvent()) { + return; + } + dragging = JX.$V(e); // Show the "col-resize" cursor on the whole document while we're From 7d4362690f1c8e616fd9ff6d917f676f163918f6 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:26:01 -0800 Subject: [PATCH 53/67] Fix transposed name/email in Mailgun adapter Summary: Ref T12677. This argument order was swapped. Test Plan: Will push/verify. Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19042 --- .../adapter/PhabricatorMailImplementationMailgunAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index 12c54e0d6a..349dae2d27 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -21,7 +21,7 @@ final class PhabricatorMailImplementationMailgunAdapter if (empty($this->params['reply-to'])) { $this->params['reply-to'] = array(); } - $this->params['reply-to'][] = $this->renderAddress($name, $email); + $this->params['reply-to'][] = $this->renderAddress($email, $name); return $this; } From 09b446b269f5cec0ba154872fddc354db44fd5e1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:48:59 -0800 Subject: [PATCH 54/67] Don't run older mail setup checks if "cluster.mailers" is configured Summary: Ref T12677. Skip these checks if we're doing the new stuff. Also, allow priority to be unspecified. Test Plan: Will deploy. Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19043 --- src/applications/config/check/PhabricatorMailSetupCheck.php | 4 ++++ .../cluster/config/PhabricatorClusterMailersConfigType.php | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/applications/config/check/PhabricatorMailSetupCheck.php b/src/applications/config/check/PhabricatorMailSetupCheck.php index 2b8e4e12d5..b3b6143ad0 100644 --- a/src/applications/config/check/PhabricatorMailSetupCheck.php +++ b/src/applications/config/check/PhabricatorMailSetupCheck.php @@ -7,6 +7,10 @@ final class PhabricatorMailSetupCheck extends PhabricatorSetupCheck { } protected function executeChecks() { + if (PhabricatorEnv::getEnvConfig('cluster.mailers')) { + return; + } + $adapter = PhabricatorEnv::getEnvConfig('metamta.mail-adapter'); switch ($adapter) { diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index 2a7550c419..b3b110298f 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -63,8 +63,8 @@ final class PhabricatorClusterMailersConfigType } $map[$key] = true; - $priority = idx($spec, 'priority', 0); - if ($priority <= 0) { + $priority = idx($spec, 'priority'); + if ($priority !== null && $priority <= 0) { throw $this->newException( pht( 'Mailer configuration ("%s") is invalid: priority must be '. From d45952344bd020b196c635b158fdcc905803865a Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:55:02 -0800 Subject: [PATCH 55/67] Use setOptions() to trigger mailer option validation, not validateOptions() --- .../cluster/config/PhabricatorClusterMailersConfigType.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index b3b110298f..60547eea44 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -85,7 +85,7 @@ final class PhabricatorClusterMailersConfigType $options = idx($spec, 'options', array()); try { - id(clone $adapters[$type])->validateOptions($options); + id(clone $adapters[$type])->setOptions($options); } catch (Exception $ex) { throw $this->newException( pht( From 8de794d3c28a11f1a30e93e7bd74e71031de6ab5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:58:14 -0800 Subject: [PATCH 56/67] Make optional options actually optional in cluster mailer config validation --- .../cluster/config/PhabricatorClusterMailersConfigType.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index 60547eea44..03f30506bd 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -85,6 +85,8 @@ final class PhabricatorClusterMailersConfigType $options = idx($spec, 'options', array()); try { + $defaults = $adapters[$type]->newDefaultOptions(); + $options = $options + $defaults; id(clone $adapters[$type])->setOptions($options); } catch (Exception $ex) { throw $this->newException( From 9386e436fe35c7972116f2df26702a4b390c9985 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 18:14:25 -0800 Subject: [PATCH 57/67] Remove red coloration from "Logout" menu item Summary: I made the red stronger (always visible, not just a hover state) for the "Mute" feature, but this made Logout look a little intense. Just make it normal-colored, logging out isn't a big deal. Test Plan: No longer saw bright red logout action in profile dropdown menu. Differential Revision: https://secure.phabricator.com/D19044 --- .../people/engineextension/PeopleMainMenuBarExtension.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/applications/people/engineextension/PeopleMainMenuBarExtension.php b/src/applications/people/engineextension/PeopleMainMenuBarExtension.php index bed9dde44e..152fb2becf 100644 --- a/src/applications/people/engineextension/PeopleMainMenuBarExtension.php +++ b/src/applications/people/engineextension/PeopleMainMenuBarExtension.php @@ -113,7 +113,6 @@ final class PeopleMainMenuBarExtension ->setName(pht('Log Out %s', $viewer->getUsername())) ->addSigil('logout-item') ->setHref('/logout/') - ->setColor(PhabricatorActionView::RED) ->setWorkflow(true)); return $view; From 0470125d9e5f10ea7f0c3ffacc4faf71fcd40ffe Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 06:14:29 -0800 Subject: [PATCH 58/67] Add skeleton code for webhooks Summary: Ref T11330. Adds general support for webhooks. This is still rough and missing a lot of pieces -- and not yet useful for anything -- but can make HTTP requests. Test Plan: Used `bin/webhook call ...` to complete requests to a test endpoint. Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19045 --- bin/webhook | 1 + .../sql/autopatches/20180209.hook.01.hook.sql | 12 + .../20180209.hook.02.hookxaction.sql | 19 ++ .../20180209.hook.03.hookrequest.sql | 12 + scripts/setup/manage_webhook.php | 21 ++ src/__phutil_library_map__.php | 59 +++++ .../PhabricatorHeraldApplication.php | 14 ++ .../HeraldCreateWebhooksCapability.php | 16 ++ .../herald/controller/HeraldController.php | 19 +- .../controller/HeraldRuleListController.php | 11 + .../controller/HeraldWebhookController.php | 15 ++ .../HeraldWebhookEditController.php | 12 + .../HeraldWebhookListController.php | 26 ++ .../HeraldWebhookTestController.php | 45 ++++ .../HeraldWebhookViewController.php | 160 ++++++++++++ .../herald/editor/HeraldWebhookEditEngine.php | 105 ++++++++ .../herald/editor/HeraldWebhookEditor.php | 22 ++ .../HeraldWebhookCallManagementWorkflow.php | 64 +++++ .../HeraldWebhookManagementWorkflow.php | 4 + .../herald/phid/HeraldWebhookPHIDType.php | 49 ++++ .../phid/HeraldWebhookRequestPHIDType.php | 39 +++ .../herald/query/HeraldWebhookQuery.php | 64 +++++ .../query/HeraldWebhookRequestQuery.php | 126 ++++++++++ .../query/HeraldWebhookSearchEngine.php | 95 ++++++++ .../query/HeraldWebhookTransactionQuery.php | 10 + .../herald/storage/HeraldWebhook.php | 177 ++++++++++++++ .../herald/storage/HeraldWebhookRequest.php | 191 +++++++++++++++ .../storage/HeraldWebhookTransaction.php | 22 ++ .../view/HeraldWebhookRequestListView.php | 78 ++++++ .../herald/worker/HeraldWebhookWorker.php | 229 ++++++++++++++++++ .../xaction/HeraldWebhookNameTransaction.php | 60 +++++ .../HeraldWebhookStatusTransaction.php | 55 +++++ .../xaction/HeraldWebhookTransactionType.php | 4 + .../xaction/HeraldWebhookURITransaction.php | 74 ++++++ 34 files changed, 1896 insertions(+), 14 deletions(-) create mode 120000 bin/webhook create mode 100644 resources/sql/autopatches/20180209.hook.01.hook.sql create mode 100644 resources/sql/autopatches/20180209.hook.02.hookxaction.sql create mode 100644 resources/sql/autopatches/20180209.hook.03.hookrequest.sql create mode 100755 scripts/setup/manage_webhook.php create mode 100644 src/applications/herald/capability/HeraldCreateWebhooksCapability.php create mode 100644 src/applications/herald/controller/HeraldWebhookController.php create mode 100644 src/applications/herald/controller/HeraldWebhookEditController.php create mode 100644 src/applications/herald/controller/HeraldWebhookListController.php create mode 100644 src/applications/herald/controller/HeraldWebhookTestController.php create mode 100644 src/applications/herald/controller/HeraldWebhookViewController.php create mode 100644 src/applications/herald/editor/HeraldWebhookEditEngine.php create mode 100644 src/applications/herald/editor/HeraldWebhookEditor.php create mode 100644 src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php create mode 100644 src/applications/herald/management/HeraldWebhookManagementWorkflow.php create mode 100644 src/applications/herald/phid/HeraldWebhookPHIDType.php create mode 100644 src/applications/herald/phid/HeraldWebhookRequestPHIDType.php create mode 100644 src/applications/herald/query/HeraldWebhookQuery.php create mode 100644 src/applications/herald/query/HeraldWebhookRequestQuery.php create mode 100644 src/applications/herald/query/HeraldWebhookSearchEngine.php create mode 100644 src/applications/herald/query/HeraldWebhookTransactionQuery.php create mode 100644 src/applications/herald/storage/HeraldWebhook.php create mode 100644 src/applications/herald/storage/HeraldWebhookRequest.php create mode 100644 src/applications/herald/storage/HeraldWebhookTransaction.php create mode 100644 src/applications/herald/view/HeraldWebhookRequestListView.php create mode 100644 src/applications/herald/worker/HeraldWebhookWorker.php create mode 100644 src/applications/herald/xaction/HeraldWebhookNameTransaction.php create mode 100644 src/applications/herald/xaction/HeraldWebhookStatusTransaction.php create mode 100644 src/applications/herald/xaction/HeraldWebhookTransactionType.php create mode 100644 src/applications/herald/xaction/HeraldWebhookURITransaction.php diff --git a/bin/webhook b/bin/webhook new file mode 120000 index 0000000000..d320336874 --- /dev/null +++ b/bin/webhook @@ -0,0 +1 @@ +../scripts/setup/manage_webhook.php \ No newline at end of file diff --git a/resources/sql/autopatches/20180209.hook.01.hook.sql b/resources/sql/autopatches/20180209.hook.01.hook.sql new file mode 100644 index 0000000000..58b79227a1 --- /dev/null +++ b/resources/sql/autopatches/20180209.hook.01.hook.sql @@ -0,0 +1,12 @@ +CREATE TABLE {$NAMESPACE}_herald.herald_webhook ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + name VARCHAR(128) NOT NULL COLLATE {$COLLATE_TEXT}, + webhookURI VARCHAR(255) NOT NULL COLLATE {$COLLATE_TEXT}, + viewPolicy VARBINARY(64) NOT NULL, + editPolicy VARBINARY(64) NOT NULL, + status VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + hmacKey VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180209.hook.02.hookxaction.sql b/resources/sql/autopatches/20180209.hook.02.hookxaction.sql new file mode 100644 index 0000000000..8da594f6bd --- /dev/null +++ b/resources/sql/autopatches/20180209.hook.02.hookxaction.sql @@ -0,0 +1,19 @@ +CREATE TABLE {$NAMESPACE}_herald.herald_webhooktransaction ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + authorPHID VARBINARY(64) NOT NULL, + objectPHID VARBINARY(64) NOT NULL, + viewPolicy VARBINARY(64) NOT NULL, + editPolicy VARBINARY(64) NOT NULL, + commentPHID VARBINARY(64) DEFAULT NULL, + commentVersion INT UNSIGNED NOT NULL, + transactionType VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL, + oldValue LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + newValue LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + contentSource LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + metadata LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_phid` (`phid`), + KEY `key_object` (`objectPHID`) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180209.hook.03.hookrequest.sql b/resources/sql/autopatches/20180209.hook.03.hookrequest.sql new file mode 100644 index 0000000000..f20b3a549d --- /dev/null +++ b/resources/sql/autopatches/20180209.hook.03.hookrequest.sql @@ -0,0 +1,12 @@ +CREATE TABLE {$NAMESPACE}_herald.herald_webhookrequest ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + webhookPHID VARBINARY(64) NOT NULL, + objectPHID VARBINARY(64) NOT NULL, + status VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}, + lastRequestResult VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + lastRequestEpoch INT UNSIGNED NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/scripts/setup/manage_webhook.php b/scripts/setup/manage_webhook.php new file mode 100755 index 0000000000..afe662617a --- /dev/null +++ b/scripts/setup/manage_webhook.php @@ -0,0 +1,21 @@ +#!/usr/bin/env php +setTagline(pht('manage webhooks')); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = id(new PhutilClassMapQuery()) + ->setAncestorClass('HeraldWebhookManagementWorkflow') + ->execute(); +$workflows[] = new PhutilHelpArgumentWorkflow(); +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 9342935ca1..bb1706a7ee 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1364,6 +1364,7 @@ phutil_register_library_map(array( 'HeraldContentSourceField' => 'applications/herald/field/HeraldContentSourceField.php', 'HeraldController' => 'applications/herald/controller/HeraldController.php', 'HeraldCoreStateReasons' => 'applications/herald/state/HeraldCoreStateReasons.php', + 'HeraldCreateWebhooksCapability' => 'applications/herald/capability/HeraldCreateWebhooksCapability.php', 'HeraldDAO' => 'applications/herald/storage/HeraldDAO.php', 'HeraldDeprecatedFieldGroup' => 'applications/herald/field/HeraldDeprecatedFieldGroup.php', 'HeraldDifferentialAdapter' => 'applications/differential/herald/HeraldDifferentialAdapter.php', @@ -1440,6 +1441,30 @@ phutil_register_library_map(array( 'HeraldTranscriptSearchEngine' => 'applications/herald/query/HeraldTranscriptSearchEngine.php', 'HeraldTranscriptTestCase' => 'applications/herald/storage/__tests__/HeraldTranscriptTestCase.php', 'HeraldUtilityActionGroup' => 'applications/herald/action/HeraldUtilityActionGroup.php', + 'HeraldWebhook' => 'applications/herald/storage/HeraldWebhook.php', + 'HeraldWebhookCallManagementWorkflow' => 'applications/herald/management/HeraldWebhookCallManagementWorkflow.php', + 'HeraldWebhookController' => 'applications/herald/controller/HeraldWebhookController.php', + 'HeraldWebhookEditController' => 'applications/herald/controller/HeraldWebhookEditController.php', + 'HeraldWebhookEditEngine' => 'applications/herald/editor/HeraldWebhookEditEngine.php', + 'HeraldWebhookEditor' => 'applications/herald/editor/HeraldWebhookEditor.php', + 'HeraldWebhookListController' => 'applications/herald/controller/HeraldWebhookListController.php', + 'HeraldWebhookManagementWorkflow' => 'applications/herald/management/HeraldWebhookManagementWorkflow.php', + 'HeraldWebhookNameTransaction' => 'applications/herald/xaction/HeraldWebhookNameTransaction.php', + 'HeraldWebhookPHIDType' => 'applications/herald/phid/HeraldWebhookPHIDType.php', + 'HeraldWebhookQuery' => 'applications/herald/query/HeraldWebhookQuery.php', + 'HeraldWebhookRequest' => 'applications/herald/storage/HeraldWebhookRequest.php', + 'HeraldWebhookRequestListView' => 'applications/herald/view/HeraldWebhookRequestListView.php', + 'HeraldWebhookRequestPHIDType' => 'applications/herald/phid/HeraldWebhookRequestPHIDType.php', + 'HeraldWebhookRequestQuery' => 'applications/herald/query/HeraldWebhookRequestQuery.php', + 'HeraldWebhookSearchEngine' => 'applications/herald/query/HeraldWebhookSearchEngine.php', + 'HeraldWebhookStatusTransaction' => 'applications/herald/xaction/HeraldWebhookStatusTransaction.php', + 'HeraldWebhookTestController' => 'applications/herald/controller/HeraldWebhookTestController.php', + 'HeraldWebhookTransaction' => 'applications/herald/storage/HeraldWebhookTransaction.php', + 'HeraldWebhookTransactionQuery' => 'applications/herald/query/HeraldWebhookTransactionQuery.php', + 'HeraldWebhookTransactionType' => 'applications/herald/xaction/HeraldWebhookTransactionType.php', + 'HeraldWebhookURITransaction' => 'applications/herald/xaction/HeraldWebhookURITransaction.php', + 'HeraldWebhookViewController' => 'applications/herald/controller/HeraldWebhookViewController.php', + 'HeraldWebhookWorker' => 'applications/herald/worker/HeraldWebhookWorker.php', 'Javelin' => 'infrastructure/javelin/Javelin.php', 'LegalpadController' => 'applications/legalpad/controller/LegalpadController.php', 'LegalpadCreateDocumentsCapability' => 'applications/legalpad/capability/LegalpadCreateDocumentsCapability.php', @@ -6614,6 +6639,7 @@ phutil_register_library_map(array( 'HeraldContentSourceField' => 'HeraldField', 'HeraldController' => 'PhabricatorController', 'HeraldCoreStateReasons' => 'HeraldStateReasons', + 'HeraldCreateWebhooksCapability' => 'PhabricatorPolicyCapability', 'HeraldDAO' => 'PhabricatorLiskDAO', 'HeraldDeprecatedFieldGroup' => 'HeraldFieldGroup', 'HeraldDifferentialAdapter' => 'HeraldAdapter', @@ -6704,6 +6730,39 @@ phutil_register_library_map(array( 'HeraldTranscriptSearchEngine' => 'PhabricatorApplicationSearchEngine', 'HeraldTranscriptTestCase' => 'PhabricatorTestCase', 'HeraldUtilityActionGroup' => 'HeraldActionGroup', + 'HeraldWebhook' => array( + 'HeraldDAO', + 'PhabricatorPolicyInterface', + 'PhabricatorApplicationTransactionInterface', + 'PhabricatorDestructibleInterface', + ), + 'HeraldWebhookCallManagementWorkflow' => 'HeraldWebhookManagementWorkflow', + 'HeraldWebhookController' => 'HeraldController', + 'HeraldWebhookEditController' => 'HeraldWebhookController', + 'HeraldWebhookEditEngine' => 'PhabricatorEditEngine', + 'HeraldWebhookEditor' => 'PhabricatorApplicationTransactionEditor', + 'HeraldWebhookListController' => 'HeraldWebhookController', + 'HeraldWebhookManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'HeraldWebhookNameTransaction' => 'HeraldWebhookTransactionType', + 'HeraldWebhookPHIDType' => 'PhabricatorPHIDType', + 'HeraldWebhookQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', + 'HeraldWebhookRequest' => array( + 'HeraldDAO', + 'PhabricatorPolicyInterface', + 'PhabricatorExtendedPolicyInterface', + ), + 'HeraldWebhookRequestListView' => 'AphrontView', + 'HeraldWebhookRequestPHIDType' => 'PhabricatorPHIDType', + 'HeraldWebhookRequestQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', + 'HeraldWebhookSearchEngine' => 'PhabricatorApplicationSearchEngine', + 'HeraldWebhookStatusTransaction' => 'HeraldWebhookTransactionType', + 'HeraldWebhookTestController' => 'HeraldWebhookController', + 'HeraldWebhookTransaction' => 'PhabricatorModularTransaction', + 'HeraldWebhookTransactionQuery' => 'PhabricatorApplicationTransactionQuery', + 'HeraldWebhookTransactionType' => 'PhabricatorModularTransactionType', + 'HeraldWebhookURITransaction' => 'HeraldWebhookTransactionType', + 'HeraldWebhookViewController' => 'HeraldWebhookController', + 'HeraldWebhookWorker' => 'PhabricatorWorker', 'Javelin' => 'Phobject', 'LegalpadController' => 'PhabricatorController', 'LegalpadCreateDocumentsCapability' => 'PhabricatorPolicyCapability', diff --git a/src/applications/herald/application/PhabricatorHeraldApplication.php b/src/applications/herald/application/PhabricatorHeraldApplication.php index 9160b0e9d9..240aae8dd3 100644 --- a/src/applications/herald/application/PhabricatorHeraldApplication.php +++ b/src/applications/herald/application/PhabricatorHeraldApplication.php @@ -62,6 +62,17 @@ final class PhabricatorHeraldApplication extends PhabricatorApplication { '(?P[1-9]\d*)/' => 'HeraldTranscriptController', ), + 'webhook/' => array( + $this->getQueryRoutePattern() => 'HeraldWebhookListController', + 'view/(?P\d+)/(?:request/(?P[^/]+)/)?' => + 'HeraldWebhookViewController', + $this->getEditRoutePattern('edit/') => 'HeraldWebhookEditController', + 'test/(?P\d+)/' => 'HeraldWebhookTestController', + 'key/' => array( + 'view/(?P\d+)/' => 'HeraldWebhookViewKeyController', + 'cycle/(?P\d+)/' => 'HeraldWebhookCycleKeyController', + ), + ), ), ); } @@ -72,6 +83,9 @@ final class PhabricatorHeraldApplication extends PhabricatorApplication { 'caption' => pht('Global rules can bypass access controls.'), 'default' => PhabricatorPolicies::POLICY_ADMIN, ), + HeraldCreateWebhooksCapability::CAPABILITY => array( + 'default' => PhabricatorPolicies::POLICY_ADMIN, + ), ); } diff --git a/src/applications/herald/capability/HeraldCreateWebhooksCapability.php b/src/applications/herald/capability/HeraldCreateWebhooksCapability.php new file mode 100644 index 0000000000..7537a61900 --- /dev/null +++ b/src/applications/herald/capability/HeraldCreateWebhooksCapability.php @@ -0,0 +1,16 @@ +buildSideNavView()->getMenu(); } - protected function buildApplicationCrumbs() { - $crumbs = parent::buildApplicationCrumbs(); - - $crumbs->addAction( - id(new PHUIListItemView()) - ->setName(pht('Create Herald Rule')) - ->setHref($this->getApplicationURI('create/')) - ->setIcon('fa-plus-square')); - - return $crumbs; - } - public function buildSideNavView() { $viewer = $this->getViewer(); @@ -29,8 +17,11 @@ abstract class HeraldController extends PhabricatorController { ->addNavigationItems($nav->getMenu()); $nav->addLabel(pht('Utilities')) - ->addFilter('test', pht('Test Console')) - ->addFilter('transcript', pht('Transcripts')); + ->addFilter('test', pht('Test Console')) + ->addFilter('transcript', pht('Transcripts')); + + $nav->addLabel(pht('Webhooks')) + ->addFilter('webhook', pht('Webhooks')); $nav->selectFilter(null); diff --git a/src/applications/herald/controller/HeraldRuleListController.php b/src/applications/herald/controller/HeraldRuleListController.php index 490d84212d..846116333f 100644 --- a/src/applications/herald/controller/HeraldRuleListController.php +++ b/src/applications/herald/controller/HeraldRuleListController.php @@ -17,5 +17,16 @@ final class HeraldRuleListController extends HeraldController { return $this->delegateToController($controller); } + protected function buildApplicationCrumbs() { + $crumbs = parent::buildApplicationCrumbs(); + + $crumbs->addAction( + id(new PHUIListItemView()) + ->setName(pht('Create Herald Rule')) + ->setHref($this->getApplicationURI('create/')) + ->setIcon('fa-plus-square')); + + return $crumbs; + } } diff --git a/src/applications/herald/controller/HeraldWebhookController.php b/src/applications/herald/controller/HeraldWebhookController.php new file mode 100644 index 0000000000..6c210640c6 --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookController.php @@ -0,0 +1,15 @@ +addTextCrumb( + pht('Webhooks'), + $this->getApplicationURI('webhook/')); + + return $crumbs; + } + +} diff --git a/src/applications/herald/controller/HeraldWebhookEditController.php b/src/applications/herald/controller/HeraldWebhookEditController.php new file mode 100644 index 0000000000..94c24187ec --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookEditController.php @@ -0,0 +1,12 @@ +setController($this) + ->buildResponse(); + } + +} diff --git a/src/applications/herald/controller/HeraldWebhookListController.php b/src/applications/herald/controller/HeraldWebhookListController.php new file mode 100644 index 0000000000..85c53b2fa6 --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookListController.php @@ -0,0 +1,26 @@ +setController($this) + ->buildResponse(); + } + + protected function buildApplicationCrumbs() { + $crumbs = parent::buildApplicationCrumbs(); + + id(new HeraldWebhookEditEngine()) + ->setViewer($this->getViewer()) + ->addActionToCrumbs($crumbs); + + return $crumbs; + } + +} diff --git a/src/applications/herald/controller/HeraldWebhookTestController.php b/src/applications/herald/controller/HeraldWebhookTestController.php new file mode 100644 index 0000000000..fdc1494af9 --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookTestController.php @@ -0,0 +1,45 @@ +getViewer(); + + $hook = id(new HeraldWebhookQuery()) + ->setViewer($viewer) + ->withIDs(array($request->getURIData('id'))) + ->requireCapabilities( + array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + )) + ->executeOne(); + if (!$hook) { + return new Aphront404Response(); + } + + if ($request->isFormPost()) { + $object = $hook; + + $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) + ->setObjectPHID($object->getPHID()) + ->save(); + + $request->queueCall(); + + $next_uri = $hook->getURI().'request/'.$request->getID().'/'; + + return id(new AphrontRedirectResponse())->setURI($next_uri); + } + + return $this->newDialog() + ->setTitle(pht('New Test Request')) + ->appendParagraph( + pht('This will make a new test request to the configured URI.')) + ->addCancelButton($hook->getURI()) + ->addSubmitButton(pht('Make Request')); + } + + +} diff --git a/src/applications/herald/controller/HeraldWebhookViewController.php b/src/applications/herald/controller/HeraldWebhookViewController.php new file mode 100644 index 0000000000..db52b5bf8f --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookViewController.php @@ -0,0 +1,160 @@ +getViewer(); + + $hook = id(new HeraldWebhookQuery()) + ->setViewer($viewer) + ->withIDs(array($request->getURIData('id'))) + ->executeOne(); + if (!$hook) { + return new Aphront404Response(); + } + + $header = $this->buildHeaderView($hook); + + $warnings = null; + if ($hook->isInErrorBackoff($viewer)) { + $message = pht( + 'Many requests to this webhook have failed recently (at least %s '. + 'errors in the last %s seconds). New requests are temporarily paused.', + $hook->getErrorBackoffThreshold(), + $hook->getErrorBackoffWindow()); + + $warnings = id(new PHUIInfoView()) + ->setSeverity(PHUIInfoView::SEVERITY_WARNING) + ->setErrors( + array( + $message, + )); + } + + $curtain = $this->buildCurtain($hook); + $properties_view = $this->buildPropertiesView($hook); + + $timeline = $this->buildTransactionTimeline( + $hook, + new HeraldWebhookTransactionQuery()); + $timeline->setShouldTerminate(true); + + $requests = id(new HeraldWebhookRequestQuery()) + ->setViewer($viewer) + ->withWebhookPHIDs(array($hook->getPHID())) + ->setLimit(20) + ->execute(); + + $requests_table = id(new HeraldWebhookRequestListView()) + ->setViewer($viewer) + ->setRequests($requests) + ->setHighlightID($request->getURIData('requestID')); + + $requests_view = id(new PHUIObjectBoxView()) + ->setHeaderText(pht('Recent Requests')) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->setTable($requests_table); + + $hook_view = id(new PHUITwoColumnView()) + ->setHeader($header) + ->setMainColumn( + array( + $warnings, + $properties_view, + $requests_view, + $timeline, + )) + ->setCurtain($curtain); + + $crumbs = $this->buildApplicationCrumbs() + ->addTextCrumb(pht('Webhook %d', $hook->getID())) + ->setBorder(true); + + return $this->newPage() + ->setTitle( + array( + pht('Webhook %d', $hook->getID()), + $hook->getName(), + )) + ->setCrumbs($crumbs) + ->setPageObjectPHIDs( + array( + $hook->getPHID(), + )) + ->appendChild($hook_view); + } + + private function buildHeaderView(HeraldWebhook $hook) { + $viewer = $this->getViewer(); + + $title = $hook->getName(); + + $header = id(new PHUIHeaderView()) + ->setHeader($title) + ->setViewer($viewer) + ->setPolicyObject($hook) + ->setHeaderIcon('fa-cloud-upload'); + + return $header; + } + + + private function buildCurtain(HeraldWebhook $hook) { + $viewer = $this->getViewer(); + $curtain = $this->newCurtainView($hook); + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $hook, + PhabricatorPolicyCapability::CAN_EDIT); + + $id = $hook->getID(); + $edit_uri = $this->getApplicationURI("webhook/edit/{$id}/"); + $test_uri = $this->getApplicationURI("webhook/test/{$id}/"); + + $curtain->addAction( + id(new PhabricatorActionView()) + ->setName(pht('Edit Webhook')) + ->setIcon('fa-pencil') + ->setDisabled(!$can_edit) + ->setWorkflow(!$can_edit) + ->setHref($edit_uri)); + + $curtain->addAction( + id(new PhabricatorActionView()) + ->setName(pht('New Test Request')) + ->setIcon('fa-cloud-upload') + ->setDisabled(!$can_edit) + ->setWorkflow(true) + ->setHref($test_uri)); + + return $curtain; + } + + + private function buildPropertiesView(HeraldWebhook $hook) { + $viewer = $this->getViewer(); + + $properties = id(new PHUIPropertyListView()) + ->setViewer($viewer); + + $properties->addProperty( + pht('URI'), + $hook->getWebhookURI()); + + $properties->addProperty( + pht('Status'), + $hook->getStatusDisplayName()); + + return id(new PHUIObjectBoxView()) + ->setHeaderText(pht('Details')) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->appendChild($properties); + } + +} diff --git a/src/applications/herald/editor/HeraldWebhookEditEngine.php b/src/applications/herald/editor/HeraldWebhookEditEngine.php new file mode 100644 index 0000000000..5bca0af542 --- /dev/null +++ b/src/applications/herald/editor/HeraldWebhookEditEngine.php @@ -0,0 +1,105 @@ +getViewer(); + return HeraldWebhook::initializeNewWebhook($viewer); + } + + protected function newObjectQuery() { + return new HeraldWebhookQuery(); + } + + protected function getObjectCreateTitleText($object) { + return pht('Create Webhook'); + } + + protected function getObjectCreateButtonText($object) { + return pht('Create Webhook'); + } + + protected function getObjectEditTitleText($object) { + return pht('Edit Webhook: %s', $object->getName()); + } + + protected function getObjectEditShortText($object) { + return pht('Edit Webhook'); + } + + protected function getObjectCreateShortText() { + return pht('Create Webhook'); + } + + protected function getObjectName() { + return pht('Webhook'); + } + + protected function getEditorURI() { + return '/herald/webhook/edit/'; + } + + protected function getObjectCreateCancelURI($object) { + return '/herald/webhook/'; + } + + protected function getObjectViewURI($object) { + return $object->getURI(); + } + + protected function getCreateNewObjectPolicy() { + return $this->getApplication()->getPolicy( + HeraldCreateWebhooksCapability::CAPABILITY); + } + + protected function buildCustomEditFields($object) { + return array( + id(new PhabricatorTextEditField()) + ->setKey('name') + ->setLabel(pht('Name')) + ->setDescription(pht('Name of the webhook.')) + ->setTransactionType(HeraldWebhookNameTransaction::TRANSACTIONTYPE) + ->setIsRequired(true) + ->setValue($object->getName()), + id(new PhabricatorTextEditField()) + ->setKey('uri') + ->setLabel(pht('URI')) + ->setDescription(pht('URI for the webhook.')) + ->setTransactionType(HeraldWebhookURITransaction::TRANSACTIONTYPE) + ->setIsRequired(true) + ->setValue($object->getWebhookURI()), + id(new PhabricatorSelectEditField()) + ->setKey('status') + ->setLabel(pht('Status')) + ->setDescription(pht('Status mode for the webhook.')) + ->setTransactionType(HeraldWebhookStatusTransaction::TRANSACTIONTYPE) + ->setOptions(HeraldWebhook::getStatusDisplayNameMap()) + ->setValue($object->getStatus()), + + ); + } + +} diff --git a/src/applications/herald/editor/HeraldWebhookEditor.php b/src/applications/herald/editor/HeraldWebhookEditor.php new file mode 100644 index 0000000000..2206a59411 --- /dev/null +++ b/src/applications/herald/editor/HeraldWebhookEditor.php @@ -0,0 +1,22 @@ +setName('call') + ->setExamples('**call** --id __id__') + ->setSynopsis(pht('Call a webhook.')) + ->setArguments( + array( + array( + 'name' => 'id', + 'param' => 'id', + 'help' => pht('Webhook ID to call'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $id = $args->getArg('id'); + if (!$id) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a webhook to call with "--id".')); + } + + $hook = id(new HeraldWebhookQuery()) + ->setViewer($viewer) + ->withIDs(array($id)) + ->executeOne(); + if (!$hook) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to load specified webhook ("%s").', + $id)); + } + + $object = $hook; + + $application_phid = id(new PhabricatorHeraldApplication())->getPHID(); + + $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) + ->setObjectPHID($object->getPHID()) + ->save(); + + PhabricatorWorker::setRunAllTasksInProcess(true); + $request->queueCall(); + + $request->reload(); + + echo tsprintf( + "%s\n", + pht( + 'Success, got HTTP %s from webhook.', + $request->getErrorCode())); + + return 0; + } + +} diff --git a/src/applications/herald/management/HeraldWebhookManagementWorkflow.php b/src/applications/herald/management/HeraldWebhookManagementWorkflow.php new file mode 100644 index 0000000000..4bbfda79fe --- /dev/null +++ b/src/applications/herald/management/HeraldWebhookManagementWorkflow.php @@ -0,0 +1,4 @@ +withPHIDs($phids); + } + + public function loadHandles( + PhabricatorHandleQuery $query, + array $handles, + array $objects) { + + foreach ($handles as $phid => $handle) { + $hook = $objects[$phid]; + + $name = $hook->getName(); + $id = $hook->getID(); + + $handle + ->setName($name) + ->setURI($hook->getURI()) + ->setFullName(pht('Webhook %d %s', $id, $name)); + + if ($hook->isDisabled()) { + $handle->setStatus(PhabricatorObjectHandle::STATUS_CLOSED); + } + } + } + +} diff --git a/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php b/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php new file mode 100644 index 0000000000..8263ba7484 --- /dev/null +++ b/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php @@ -0,0 +1,39 @@ +withPHIDs($phids); + } + + public function loadHandles( + PhabricatorHandleQuery $query, + array $handles, + array $objects) { + + foreach ($handles as $phid => $handle) { + $request = $objects[$phid]; + + // TODO: Fill this in. + } + } + +} diff --git a/src/applications/herald/query/HeraldWebhookQuery.php b/src/applications/herald/query/HeraldWebhookQuery.php new file mode 100644 index 0000000000..ca46880613 --- /dev/null +++ b/src/applications/herald/query/HeraldWebhookQuery.php @@ -0,0 +1,64 @@ +ids = $ids; + return $this; + } + + public function withPHIDs(array $phids) { + $this->phids = $phids; + return $this; + } + + public function withStatuses(array $statuses) { + $this->statuses = $statuses; + return $this; + } + + public function newResultObject() { + return new HeraldWebhook(); + } + + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); + + if ($this->ids !== null) { + $where[] = qsprintf( + $conn, + 'id IN (%Ld)', + $this->ids); + } + + if ($this->phids !== null) { + $where[] = qsprintf( + $conn, + 'phid IN (%Ls)', + $this->phids); + } + + if ($this->statuses !== null) { + $where[] = qsprintf( + $conn, + 'status IN (%Ls)', + $this->statuses); + } + + return $where; + } + + public function getQueryApplicationClass() { + return 'PhabricatorHeraldApplication'; + } + +} diff --git a/src/applications/herald/query/HeraldWebhookRequestQuery.php b/src/applications/herald/query/HeraldWebhookRequestQuery.php new file mode 100644 index 0000000000..4c71d48e05 --- /dev/null +++ b/src/applications/herald/query/HeraldWebhookRequestQuery.php @@ -0,0 +1,126 @@ +ids = $ids; + return $this; + } + + public function withPHIDs(array $phids) { + $this->phids = $phids; + return $this; + } + + public function withWebhookPHIDs(array $phids) { + $this->webhookPHIDs = $phids; + return $this; + } + + public function newResultObject() { + return new HeraldWebhookRequest(); + } + + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + public function withLastRequestEpochBetween($epoch_min, $epoch_max) { + $this->lastRequestEpochMin = $epoch_min; + $this->lastRequestEpochMax = $epoch_max; + return $this; + } + + public function withLastRequestResults(array $results) { + $this->lastRequestResults = $results; + return $this; + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); + + if ($this->ids !== null) { + $where[] = qsprintf( + $conn, + 'id IN (%Ld)', + $this->ids); + } + + if ($this->phids !== null) { + $where[] = qsprintf( + $conn, + 'phid IN (%Ls)', + $this->phids); + } + + if ($this->webhookPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'webhookPHID IN (%Ls)', + $this->webhookPHIDs); + } + + if ($this->lastRequestEpochMin !== null) { + $where[] = qsprintf( + $conn, + 'lastRequestEpoch >= %d', + $this->lastRequestEpochMin); + } + + if ($this->lastRequestEpochMax !== null) { + $where[] = qsprintf( + $conn, + 'lastRequestEpoch <= %d', + $this->lastRequestEpochMax); + } + + if ($this->lastRequestResults !== null) { + $where[] = qsprintf( + $conn, + 'lastRequestResult IN (%Ls)', + $this->lastRequestResults); + } + + return $where; + } + + protected function willFilterPage(array $requests) { + $hook_phids = mpull($requests, 'getWebhookPHID'); + + $hooks = id(new HeraldWebhookQuery()) + ->setViewer($this->getViewer()) + ->setParentQuery($this) + ->withPHIDs($hook_phids) + ->execute(); + $hooks = mpull($hooks, null, 'getPHID'); + + foreach ($requests as $key => $request) { + $hook_phid = $request->getWebhookPHID(); + $hook = idx($hooks, $hook_phid); + + if (!$hook) { + unset($requests[$key]); + $this->didRejectResult($request); + continue; + } + + $request->attachWebhook($hook); + } + + return $requests; + } + + + public function getQueryApplicationClass() { + return 'PhabricatorHeraldApplication'; + } + +} diff --git a/src/applications/herald/query/HeraldWebhookSearchEngine.php b/src/applications/herald/query/HeraldWebhookSearchEngine.php new file mode 100644 index 0000000000..10852f305f --- /dev/null +++ b/src/applications/herald/query/HeraldWebhookSearchEngine.php @@ -0,0 +1,95 @@ +newQuery(); + + if ($map['statuses']) { + $query->withStatuses($map['statuses']); + } + + return $query; + } + + protected function buildCustomSearchFields() { + return array( + id(new PhabricatorSearchCheckboxesField()) + ->setKey('statuses') + ->setLabel(pht('Status')) + ->setDescription( + pht('Search for archived or active pastes.')) + ->setOptions(HeraldWebhook::getStatusDisplayNameMap()), + ); + } + + protected function getURI($path) { + return '/herald/webhook/'.$path; + } + + protected function getBuiltinQueryNames() { + $names = array(); + + $names['active'] = pht('Active'); + $names['all'] = pht('All'); + + return $names; + } + + public function buildSavedQueryFromBuiltin($query_key) { + $query = $this->newSavedQuery(); + $query->setQueryKey($query_key); + + switch ($query_key) { + case 'all': + return $query; + case 'active': + return $query->setParameter( + 'statuses', + array( + HeraldWebhook::HOOKSTATUS_FIREHOSE, + HeraldWebhook::HOOKSTATUS_ENABLED, + )); + } + + return parent::buildSavedQueryFromBuiltin($query_key); + } + + protected function renderResultList( + array $hooks, + PhabricatorSavedQuery $query, + array $handles) { + assert_instances_of($hooks, 'HeraldWebhook'); + + $viewer = $this->requireViewer(); + + $list = id(new PHUIObjectItemListView()) + ->setViewer($viewer); + foreach ($hooks as $hook) { + $item = id(new PHUIObjectItemView()) + ->setObjectName(pht('Hook %d', $hook->getID())) + ->setHeader($hook->getName()) + ->setHref($hook->getURI()); + + $list->addItem($item); + } + + return id(new PhabricatorApplicationSearchResultView()) + ->setObjectList($list) + ->setNoDataString(pht('No webhooks found.')); + } + +} diff --git a/src/applications/herald/query/HeraldWebhookTransactionQuery.php b/src/applications/herald/query/HeraldWebhookTransactionQuery.php new file mode 100644 index 0000000000..b812305e56 --- /dev/null +++ b/src/applications/herald/query/HeraldWebhookTransactionQuery.php @@ -0,0 +1,10 @@ + true, + self::CONFIG_COLUMN_SCHEMA => array( + 'name' => 'text128', + 'webhookURI' => 'text255', + 'status' => 'text32', + 'hmacKey' => 'text32', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_status' => array( + 'columns' => array('status'), + ), + ), + ) + parent::getConfiguration(); + } + + public function getPHIDType() { + return HeraldWebhookPHIDType::TYPECONST; + } + + public static function initializeNewWebhook(PhabricatorUser $viewer) { + return id(new self()) + ->setStatus(self::HOOKSTATUS_ENABLED) + ->setViewPolicy(PhabricatorPolicies::getMostOpenPolicy()) + ->setEditPolicy($viewer->getPHID()) + ->setHmacKey(Filesystem::readRandomCharacters(32)); + } + + public function getURI() { + return '/herald/webhook/view/'.$this->getID().'/'; + } + + public function isDisabled() { + return ($this->getStatus() === self::HOOKSTATUS_DISABLED); + } + + public static function getStatusDisplayNameMap() { + return array( + self::HOOKSTATUS_FIREHOSE => pht('Firehose'), + self::HOOKSTATUS_ENABLED => pht('Enabled'), + self::HOOKSTATUS_DISABLED => pht('Disabled'), + ); + } + + public function getStatusDisplayName() { + $status = $this->getStatus(); + return idx($this->getStatusDisplayNameMap(), $status); + } + + public function getErrorBackoffWindow() { + return phutil_units('5 minutes in seconds'); + } + + public function getErrorBackoffThreshold() { + return 10; + } + + public function isInErrorBackoff(PhabricatorUser $viewer) { + $backoff_window = $this->getErrorBackoffWindow(); + $backoff_threshold = $this->getErrorBackoffThreshold(); + + $now = PhabricatorTime::getNow(); + + $window_start = ($now - $backoff_window); + + $requests = id(new HeraldWebhookRequestQuery()) + ->setViewer($viewer) + ->withWebhookPHIDs(array($this->getPHID())) + ->withLastRequestEpochBetween($window_start, null) + ->withLastRequestResults( + array( + HeraldWebhookRequest::RESULT_FAIL, + )) + ->execute(); + + if (count($requests) >= $backoff_threshold) { + return true; + } + + return false; + } + + +/* -( PhabricatorPolicyInterface )----------------------------------------- */ + + + public function getCapabilities() { + return array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + ); + } + + public function getPolicy($capability) { + switch ($capability) { + case PhabricatorPolicyCapability::CAN_VIEW: + return $this->getViewPolicy(); + case PhabricatorPolicyCapability::CAN_EDIT: + return $this->getEditPolicy(); + } + } + + public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { + return false; + } + + +/* -( PhabricatorApplicationTransactionInterface )------------------------- */ + + + public function getApplicationTransactionEditor() { + return new HeraldWebhookEditor(); + } + + public function getApplicationTransactionObject() { + return $this; + } + + public function getApplicationTransactionTemplate() { + return new HeraldWebhookTransaction(); + } + + public function willRenderTimeline( + PhabricatorApplicationTransactionView $timeline, + AphrontRequest $request) { + return $timeline; + } + + +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + + while (true) { + $requests = id(new HeraldWebhookRequestQuery()) + ->setViewer($engine->getViewer()) + ->withWebhookPHIDs(array($this->getPHID())) + ->setLimit(100) + ->execute(); + + if (!$requests) { + break; + } + + foreach ($requests as $request) { + $request->delete(); + } + } + + $this->delete(); + } + + +} diff --git a/src/applications/herald/storage/HeraldWebhookRequest.php b/src/applications/herald/storage/HeraldWebhookRequest.php new file mode 100644 index 0000000000..19e9bbbc39 --- /dev/null +++ b/src/applications/herald/storage/HeraldWebhookRequest.php @@ -0,0 +1,191 @@ + true, + self::CONFIG_SERIALIZATION => array( + 'properties' => self::SERIALIZATION_JSON, + ), + self::CONFIG_COLUMN_SCHEMA => array( + 'status' => 'text32', + 'lastRequestResult' => 'text32', + 'lastRequestEpoch' => 'epoch', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_ratelimit' => array( + 'columns' => array( + 'webhookPHID', + 'lastRequestResult', + 'lastRequestEpoch', + ), + ), + 'key_collect' => array( + 'columns' => array('dateCreated'), + ), + ), + ) + parent::getConfiguration(); + } + + public function getPHIDType() { + return HeraldWebhookRequestPHIDType::TYPECONST; + } + + public static function initializeNewWebhookRequest(HeraldWebhook $hook) { + return id(new self()) + ->setWebhookPHID($hook->getPHID()) + ->attachWebhook($hook) + ->setStatus(self::STATUS_QUEUED) + ->setRetryMode(self::RETRY_NEVER) + ->setLastRequestResult(self::RESULT_NONE) + ->setLastRequestEpoch(0); + } + + public function getWebhook() { + return $this->assertAttached($this->webhook); + } + + public function attachWebhook(HeraldWebhook $hook) { + $this->webhook = $hook; + return $this; + } + + protected function setProperty($key, $value) { + $this->properties[$key] = $value; + return $this; + } + + protected function getProperty($key, $default = null) { + return idx($this->properties, $key, $default); + } + + public function setRetryMode($mode) { + return $this->setProperty('retry', $mode); + } + + public function getRetryMode() { + return $this->getProperty('retry'); + } + + public function setErrorType($error_type) { + return $this->setProperty('errorType', $error_type); + } + + public function getErrorType() { + return $this->getProperty('errorType'); + } + + public function setErrorCode($error_code) { + return $this->setProperty('errorCode', $error_code); + } + + public function getErrorCode() { + return $this->getProperty('errorCode'); + } + + public function setTransactionPHIDs(array $phids) { + return $this->setProperty('transactionPHIDs', $phids); + } + + public function getTransactionPHIDs() { + return $this->getProperty('transactionPHIDs', array()); + } + + public function queueCall() { + PhabricatorWorker::scheduleTask( + 'HeraldWebhookWorker', + array( + 'webhookRequestPHID' => $this->getPHID(), + ), + array( + 'objectPHID' => $this->getPHID(), + )); + + return $this; + } + + public function newStatusIcon() { + switch ($this->getStatus()) { + case self::STATUS_QUEUED: + $icon = 'fa-refresh'; + $color = 'blue'; + $tooltip = pht('Queued'); + break; + case self::STATUS_SENT: + $icon = 'fa-check'; + $color = 'green'; + $tooltip = pht('Sent'); + break; + case self::STATUS_FAILED: + default: + $icon = 'fa-times'; + $color = 'red'; + $tooltip = pht('Failed'); + break; + + } + + return id(new PHUIIconView()) + ->setIcon($icon, $color) + ->setTooltip($tooltip); + } + + +/* -( PhabricatorPolicyInterface )----------------------------------------- */ + + + public function getCapabilities() { + return array( + PhabricatorPolicyCapability::CAN_VIEW, + ); + } + + public function getPolicy($capability) { + switch ($capability) { + case PhabricatorPolicyCapability::CAN_VIEW: + return PhabricatorPolicies::getMostOpenPolicy(); + } + } + + public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { + return false; + } + + +/* -( PhabricatorExtendedPolicyInterface )--------------------------------- */ + + + public function getExtendedPolicy($capability, PhabricatorUser $viewer) { + return array( + array($this->getWebhook(), PhabricatorPolicyCapability::CAN_VIEW), + ); + } + + + +} diff --git a/src/applications/herald/storage/HeraldWebhookTransaction.php b/src/applications/herald/storage/HeraldWebhookTransaction.php new file mode 100644 index 0000000000..03c8cbb776 --- /dev/null +++ b/src/applications/herald/storage/HeraldWebhookTransaction.php @@ -0,0 +1,22 @@ +requests = $requests; + return $this; + } + + public function setHighlightID($highlight_id) { + $this->highlightID = $highlight_id; + return $this; + } + + public function getHighlightID() { + return $this->highlightID; + } + + public function render() { + $viewer = $this->getViewer(); + $requests = $this->requests; + + $handle_phids = array(); + foreach ($requests as $request) { + $handle_phids[] = $request->getObjectPHID(); + } + $handles = $viewer->loadHandles($handle_phids); + + $highlight_id = $this->getHighlightID(); + + $rows = array(); + $rowc = array(); + foreach ($requests as $request) { + $icon = $request->newStatusIcon(); + + if ($highlight_id == $request->getID()) { + $rowc[] = 'highlighted'; + } else { + $rowc[] = null; + } + + $rows[] = array( + $request->getID(), + $icon, + $handles[$request->getObjectPHID()]->renderLink(), + $request->getErrorType(), + $request->getErrorCode(), + ); + } + + $table = id(new AphrontTableView($rows)) + ->setRowClasses($rowc) + ->setHeaders( + array( + pht('ID'), + '', + pht('Object'), + pht('Type'), + pht('Code'), + )) + ->setColumnClasses( + array( + 'n', + '', + 'wide', + '', + '', + )); + + return $table; + } + +} diff --git a/src/applications/herald/worker/HeraldWebhookWorker.php b/src/applications/herald/worker/HeraldWebhookWorker.php new file mode 100644 index 0000000000..a3d1ab64fc --- /dev/null +++ b/src/applications/herald/worker/HeraldWebhookWorker.php @@ -0,0 +1,229 @@ +getTaskData(); + $request_phid = idx($data, 'webhookRequestPHID'); + + $request = id(new HeraldWebhookRequestQuery()) + ->setViewer($viewer) + ->withPHIDs(array($request_phid)) + ->executeOne(); + if (!$request) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Unable to load webhook request ("%s"). It may have been '. + 'garbage collected.', + $request_phid)); + } + + $status = $request->getStatus(); + if ($status !== HeraldWebhookRequest::STATUS_QUEUED) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Webhook request ("%s") is not in "%s" status (actual '. + 'status is "%s"). Declining call to hook.', + $request_phid, + HeraldWebhookRequest::STATUS_QUEUED, + $status)); + } + + $hook = $request->getWebhook(); + + if ($hook->isDisabled()) { + $this->failRequest($request, 'hook', 'disabled'); + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Associated hook ("%s") for webhook request ("%s") is disabled.', + $hook->getPHID(), + $request_phid)); + } + + $uri = $hook->getWebhookURI(); + try { + PhabricatorEnv::requireValidRemoteURIForFetch( + $uri, + array( + 'http', + 'https', + )); + } catch (Exception $ex) { + $this->failRequest($request, 'hook', 'uri'); + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Associated hook ("%s") for webhook request ("%s") has invalid '. + 'fetch URI: %s', + $hook->getPHID(), + $request_phid, + $ex->getMessage())); + } + + $object_phid = $request->getObjectPHID(); + + $object = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withPHIDs(array($object_phid)) + ->executeOne(); + if (!$object) { + $this->failRequest($request, 'hook', 'object'); + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Unable to load object ("%s") for webhook request ("%s").', + $object_phid, + $request_phid)); + } + + $xaction_query = PhabricatorApplicationTransactionQuery::newQueryForObject( + $object); + $xaction_phids = $request->getTransactionPHIDs(); + if ($xaction_phids) { + $xactions = $xaction_query + ->setViewer($viewer) + ->withObjectPHIDs(array($object_phid)) + ->withPHIDs($xaction_phids) + ->execute(); + $xactions = mpull($xactions, null, 'getPHID'); + } else { + $xactions = array(); + } + + // To prevent thundering herd issues for high volume webhooks (where + // a large number of workers might try to work through a request backlog + // simultaneously, before the error backoff can catch up), we never + // parallelize requests to a particular webhook. + + $lock_key = 'webhook('.$hook->getPHID().')'; + $lock = PhabricatorGlobalLock::newLock($lock_key); + + try { + $lock->lock(); + } catch (Exception $ex) { + phlog($ex); + throw new PhabricatorWorkerYieldException(15); + } + + $caught = null; + try { + $this->callWebhookWithLock($hook, $request, $object, $xactions); + } catch (Exception $ex) { + $caught = $ex; + } + + $lock->unlock(); + + if ($caught) { + throw $caught; + } + } + + private function callWebhookWithLock( + HeraldWebhook $hook, + HeraldWebhookRequest $request, + $object, + array $xactions) { + $viewer = PhabricatorUser::getOmnipotentUser(); + + if ($hook->isInErrorBackoff($viewer)) { + throw new PhabricatorWorkerYieldException($hook->getErrorBackoffWindow()); + } + + $xaction_data = array(); + foreach ($xactions as $xaction) { + $xaction_data[] = array( + 'phid' => $xaction->getPHID(), + ); + } + + $payload = array( + 'triggers' => array(), + 'object' => array( + 'phid' => $object->getPHID(), + ), + 'transactions' => $xaction_data, + ); + + $payload = phutil_json_encode($payload); + $key = $hook->getHmacKey(); + $signature = PhabricatorHash::digestHMACSHA256($payload, $key); + $uri = $hook->getWebhookURI(); + + $future = id(new HTTPSFuture($uri)) + ->setMethod('POST') + ->addHeader('Content-Type', 'application/json') + ->addHeader('X-Phabricator-Webhook-Signature', $signature) + ->setTimeout(15) + ->setData($payload); + + list($status) = $future->resolve(); + + if ($status->isTimeout()) { + $error_type = 'timeout'; + } else { + $error_type = 'http'; + } + $error_code = $status->getStatusCode(); + + $request + ->setErrorType($error_type) + ->setErrorCode($error_code) + ->setLastRequestEpoch(PhabricatorTime::getNow()); + + $retry_forever = HeraldWebhookRequest::RETRY_FOREVER; + if ($status->isTimeout() || $status->isError()) { + $should_retry = ($request->getRetryMode() === $retry_forever); + + $request + ->setLastRequestResult(HeraldWebhookRequest::RESULT_FAIL); + + if ($should_retry) { + $request->save(); + + throw new Exception( + pht( + 'Webhook request ("%s", to "%s") failed (%s / %s). The request '. + 'will be retried.', + $request->getPHID(), + $uri, + $error_type, + $error_code)); + } else { + $request + ->setStatus(HeraldWebhookRequest::STATUS_FAILED) + ->save(); + + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Webhook request ("%s", to "%s") failed (%s / %s). The request '. + 'will not be retried.', + $request->getPHID(), + $uri, + $error_type, + $error_code)); + } + } else { + $request + ->setLastRequestResult(HeraldWebhookRequest::RESULT_OKAY) + ->setStatus(HeraldWebhookRequest::STATUS_SENT) + ->save(); + } + } + + private function failRequest( + HeraldWebhookRequest $request, + $error_type, + $error_code) { + + $request + ->setStatus(HeraldWebhookRequest::STATUS_FAILED) + ->setErrorType($error_type) + ->setErrorCode($error_code) + ->setLastRequestResult(HeraldWebhookRequest::RESULT_NONE) + ->setLastRequestEpoch(0) + ->save(); + } + +} diff --git a/src/applications/herald/xaction/HeraldWebhookNameTransaction.php b/src/applications/herald/xaction/HeraldWebhookNameTransaction.php new file mode 100644 index 0000000000..6224292711 --- /dev/null +++ b/src/applications/herald/xaction/HeraldWebhookNameTransaction.php @@ -0,0 +1,60 @@ +getName(); + } + + public function applyInternalEffects($object, $value) { + $object->setName($value); + } + + public function getTitle() { + return pht( + '%s renamed this webhook from %s to %s.', + $this->renderAuthor(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function getTitleForFeed() { + return pht( + '%s renamed %s from %s to %s.', + $this->renderAuthor(), + $this->renderObject(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function validateTransactions($object, array $xactions) { + $errors = array(); + $viewer = $this->getActor(); + + if ($this->isEmptyTextTransaction($object->getName(), $xactions)) { + $errors[] = $this->newRequiredError( + pht('Webhooks must have a name.')); + return $errors; + } + + $max_length = $object->getColumnMaximumByteLength('name'); + foreach ($xactions as $xaction) { + $old_value = $this->generateOldValue($object); + $new_value = $xaction->getNewValue(); + + $new_length = strlen($new_value); + if ($new_length > $max_length) { + $errors[] = $this->newInvalidError( + pht( + 'Webhook names can be no longer than %s characters.', + new PhutilNumber($max_length))); + } + } + + return $errors; + } + +} diff --git a/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php b/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php new file mode 100644 index 0000000000..4051b6c597 --- /dev/null +++ b/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php @@ -0,0 +1,55 @@ +getStatus(); + } + + public function applyInternalEffects($object, $value) { + $object->setStatus($value); + } + + public function getTitle() { + return pht( + '%s changed hook status from %s to %s.', + $this->renderAuthor(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function getTitleForFeed() { + return pht( + '%s changed %s from %s to %s.', + $this->renderAuthor(), + $this->renderObject(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function validateTransactions($object, array $xactions) { + $errors = array(); + $viewer = $this->getActor(); + + $options = HeraldWebhook::getStatusDisplayNameMap(); + + foreach ($xactions as $xaction) { + $new_value = $xaction->getNewValue(); + + if (!isset($options[$new_value])) { + $errors[] = $this->newInvalidError( + pht( + 'Webhook status "%s" is not valid. Valid statuses are: %s.', + $new_value, + implode(', ', array_keys($options))), + $xaction); + } + } + + return $errors; + } + +} diff --git a/src/applications/herald/xaction/HeraldWebhookTransactionType.php b/src/applications/herald/xaction/HeraldWebhookTransactionType.php new file mode 100644 index 0000000000..d49fcfed86 --- /dev/null +++ b/src/applications/herald/xaction/HeraldWebhookTransactionType.php @@ -0,0 +1,4 @@ +getWebhookURI(); + } + + public function applyInternalEffects($object, $value) { + $object->setWebhookURI($value); + } + + public function getTitle() { + return pht( + '%s changed the URI for this webhook from %s to %s.', + $this->renderAuthor(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function getTitleForFeed() { + return pht( + '%s changed the URI for %s from %s to %s.', + $this->renderAuthor(), + $this->renderObject(), + $this->renderOldValue(), + $this->renderNewValue()); + } + + public function validateTransactions($object, array $xactions) { + $errors = array(); + $viewer = $this->getActor(); + + if ($this->isEmptyTextTransaction($object->getName(), $xactions)) { + $errors[] = $this->newRequiredError( + pht('Webhooks must have a URI.')); + return $errors; + } + + $max_length = $object->getColumnMaximumByteLength('webhookURI'); + foreach ($xactions as $xaction) { + $old_value = $this->generateOldValue($object); + $new_value = $xaction->getNewValue(); + + $new_length = strlen($new_value); + if ($new_length > $max_length) { + $errors[] = $this->newInvalidError( + pht( + 'Webhook URIs can be no longer than %s characters.', + new PhutilNumber($max_length)), + $xaction); + } + + try { + PhabricatorEnv::requireValidRemoteURIForFetch( + $new_value, + array( + 'http', + 'https', + )); + } catch (Exception $ex) { + $errors[] = $this->newInvalidError( + $ex->getMessage(), + $xaction); + } + } + + return $errors; + } + +} From dc2995c4ca3f2d8b01a782f44493295a624213a7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 10:45:18 -0800 Subject: [PATCH 59/67] Refine core webhook implementation somewhat Summary: Depends on D19045. Ref T11330. - View/regenerate HMAC keys. - Pretty JSON. - Readable status transactions. - test, silent, secure flags. - Dates on request view. - More icons. - Can test any object. - GC for requests. Test Plan: Went through each feature poking at it in the web UI and with `bin/webhook call ...` / `bin/garbage collect ...`. Subscribers: ftdysa Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19046 --- src/__phutil_library_map__.php | 5 ++ .../PhabricatorHeraldApplication.php | 6 +- .../controller/HeraldWebhookKeyController.php | 56 +++++++++++++ .../HeraldWebhookTestController.php | 68 +++++++++++++--- .../HeraldWebhookViewController.php | 24 ++++++ .../HeraldWebhookRequestGarbageCollector.php | 29 +++++++ .../HeraldWebhookCallManagementWorkflow.php | 45 ++++++++++- .../query/HeraldWebhookSearchEngine.php | 11 ++- .../herald/storage/HeraldWebhook.php | 81 +++++++++++++++++-- .../herald/storage/HeraldWebhookRequest.php | 24 ++++++ .../view/HeraldWebhookRequestListView.php | 10 +++ .../herald/worker/HeraldWebhookWorker.php | 7 +- .../HeraldWebhookStatusTransaction.php | 20 ++++- 13 files changed, 355 insertions(+), 31 deletions(-) create mode 100644 src/applications/herald/controller/HeraldWebhookKeyController.php create mode 100644 src/applications/herald/garbagecollector/HeraldWebhookRequestGarbageCollector.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index bb1706a7ee..e078e6c10a 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1447,12 +1447,14 @@ phutil_register_library_map(array( 'HeraldWebhookEditController' => 'applications/herald/controller/HeraldWebhookEditController.php', 'HeraldWebhookEditEngine' => 'applications/herald/editor/HeraldWebhookEditEngine.php', 'HeraldWebhookEditor' => 'applications/herald/editor/HeraldWebhookEditor.php', + 'HeraldWebhookKeyController' => 'applications/herald/controller/HeraldWebhookKeyController.php', 'HeraldWebhookListController' => 'applications/herald/controller/HeraldWebhookListController.php', 'HeraldWebhookManagementWorkflow' => 'applications/herald/management/HeraldWebhookManagementWorkflow.php', 'HeraldWebhookNameTransaction' => 'applications/herald/xaction/HeraldWebhookNameTransaction.php', 'HeraldWebhookPHIDType' => 'applications/herald/phid/HeraldWebhookPHIDType.php', 'HeraldWebhookQuery' => 'applications/herald/query/HeraldWebhookQuery.php', 'HeraldWebhookRequest' => 'applications/herald/storage/HeraldWebhookRequest.php', + 'HeraldWebhookRequestGarbageCollector' => 'applications/herald/garbagecollector/HeraldWebhookRequestGarbageCollector.php', 'HeraldWebhookRequestListView' => 'applications/herald/view/HeraldWebhookRequestListView.php', 'HeraldWebhookRequestPHIDType' => 'applications/herald/phid/HeraldWebhookRequestPHIDType.php', 'HeraldWebhookRequestQuery' => 'applications/herald/query/HeraldWebhookRequestQuery.php', @@ -6735,12 +6737,14 @@ phutil_register_library_map(array( 'PhabricatorPolicyInterface', 'PhabricatorApplicationTransactionInterface', 'PhabricatorDestructibleInterface', + 'PhabricatorProjectInterface', ), 'HeraldWebhookCallManagementWorkflow' => 'HeraldWebhookManagementWorkflow', 'HeraldWebhookController' => 'HeraldController', 'HeraldWebhookEditController' => 'HeraldWebhookController', 'HeraldWebhookEditEngine' => 'PhabricatorEditEngine', 'HeraldWebhookEditor' => 'PhabricatorApplicationTransactionEditor', + 'HeraldWebhookKeyController' => 'HeraldWebhookController', 'HeraldWebhookListController' => 'HeraldWebhookController', 'HeraldWebhookManagementWorkflow' => 'PhabricatorManagementWorkflow', 'HeraldWebhookNameTransaction' => 'HeraldWebhookTransactionType', @@ -6751,6 +6755,7 @@ phutil_register_library_map(array( 'PhabricatorPolicyInterface', 'PhabricatorExtendedPolicyInterface', ), + 'HeraldWebhookRequestGarbageCollector' => 'PhabricatorGarbageCollector', 'HeraldWebhookRequestListView' => 'AphrontView', 'HeraldWebhookRequestPHIDType' => 'PhabricatorPHIDType', 'HeraldWebhookRequestQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', diff --git a/src/applications/herald/application/PhabricatorHeraldApplication.php b/src/applications/herald/application/PhabricatorHeraldApplication.php index 240aae8dd3..47e7bd7bfa 100644 --- a/src/applications/herald/application/PhabricatorHeraldApplication.php +++ b/src/applications/herald/application/PhabricatorHeraldApplication.php @@ -68,10 +68,8 @@ final class PhabricatorHeraldApplication extends PhabricatorApplication { 'HeraldWebhookViewController', $this->getEditRoutePattern('edit/') => 'HeraldWebhookEditController', 'test/(?P\d+)/' => 'HeraldWebhookTestController', - 'key/' => array( - 'view/(?P\d+)/' => 'HeraldWebhookViewKeyController', - 'cycle/(?P\d+)/' => 'HeraldWebhookCycleKeyController', - ), + 'key/(?Pview|cycle)/(?P\d+)/' => + 'HeraldWebhookKeyController', ), ), ); diff --git a/src/applications/herald/controller/HeraldWebhookKeyController.php b/src/applications/herald/controller/HeraldWebhookKeyController.php new file mode 100644 index 0000000000..8e8bd03f4a --- /dev/null +++ b/src/applications/herald/controller/HeraldWebhookKeyController.php @@ -0,0 +1,56 @@ +getViewer(); + + $hook = id(new HeraldWebhookQuery()) + ->setViewer($viewer) + ->withIDs(array($request->getURIData('id'))) + ->requireCapabilities( + array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + )) + ->executeOne(); + if (!$hook) { + return new Aphront404Response(); + } + + $action = $request->getURIData('action'); + if ($action === 'cycle') { + if (!$request->isFormPost()) { + return $this->newDialog() + ->setTitle(pht('Regenerate HMAC Key')) + ->appendParagraph( + pht( + 'Regenerate the HMAC key used to sign requests made by this '. + 'webhook?')) + ->appendParagraph( + pht( + 'Requests which are currently authenticated with the old key '. + 'may fail.')) + ->addCancelButton($hook->getURI()) + ->addSubmitButton(pht('Regnerate Key')); + } else { + $hook->regenerateHMACKey()->save(); + } + } + + $form = id(new AphrontFormView()) + ->setViewer($viewer) + ->appendControl( + id(new AphrontFormTextControl()) + ->setLabel(pht('HMAC Key')) + ->setValue($hook->getHMACKey())); + + return $this->newDialog() + ->setTitle(pht('Webhook HMAC Key')) + ->appendForm($form) + ->addCancelButton($hook->getURI(), pht('Done')); + } + + +} diff --git a/src/applications/herald/controller/HeraldWebhookTestController.php b/src/applications/herald/controller/HeraldWebhookTestController.php index fdc1494af9..b791ba783b 100644 --- a/src/applications/herald/controller/HeraldWebhookTestController.php +++ b/src/applications/herald/controller/HeraldWebhookTestController.php @@ -19,26 +19,74 @@ final class HeraldWebhookTestController return new Aphront404Response(); } + $v_object = null; + $e_object = null; + $errors = array(); if ($request->isFormPost()) { - $object = $hook; - $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) - ->setObjectPHID($object->getPHID()) - ->save(); + $v_object = $request->getStr('object'); + if (!strlen($v_object)) { + $object = $hook; + } else { + $objects = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withNames(array($v_object)) + ->execute(); + if ($objects) { + $object = head($objects); + } else { + $e_object = pht('Invalid'); + $errors[] = pht('Specified object could not be loaded.'); + } + } - $request->queueCall(); + if (!$errors) { + $xaction_query = + PhabricatorApplicationTransactionQuery::newQueryForObject($object); - $next_uri = $hook->getURI().'request/'.$request->getID().'/'; + $xactions = $xaction_query + ->withObjectPHIDs(array($object->getPHID())) + ->setViewer($viewer) + ->setLimit(10) + ->execute(); - return id(new AphrontRedirectResponse())->setURI($next_uri); + $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) + ->setObjectPHID($object->getPHID()) + ->setIsTestAction(true) + ->setTransactionPHIDs(mpull($xactions, 'getPHID')) + ->save(); + + $request->queueCall(); + + $next_uri = $hook->getURI().'request/'.$request->getID().'/'; + + return id(new AphrontRedirectResponse())->setURI($next_uri); + } } + $instructions = <<setViewer($viewer) + ->appendControl( + id(new AphrontFormTextControl()) + ->setLabel(pht('Object')) + ->setName('object') + ->setError($e_object) + ->setValue($v_object)); + return $this->newDialog() + ->setErrors($errors) + ->setWidth(AphrontDialogView::WIDTH_FORM) ->setTitle(pht('New Test Request')) - ->appendParagraph( - pht('This will make a new test request to the configured URI.')) + ->appendParagraph(new PHUIRemarkupView($viewer, $instructions)) + ->appendForm($form) ->addCancelButton($hook->getURI()) - ->addSubmitButton(pht('Make Request')); + ->addSubmitButton(pht('Test Webhook')); } diff --git a/src/applications/herald/controller/HeraldWebhookViewController.php b/src/applications/herald/controller/HeraldWebhookViewController.php index db52b5bf8f..9b11f5d433 100644 --- a/src/applications/herald/controller/HeraldWebhookViewController.php +++ b/src/applications/herald/controller/HeraldWebhookViewController.php @@ -94,10 +94,15 @@ final class HeraldWebhookViewController $title = $hook->getName(); + $status_icon = $hook->getStatusIcon(); + $status_color = $hook->getStatusColor(); + $status_name = $hook->getStatusDisplayName(); + $header = id(new PHUIHeaderView()) ->setHeader($title) ->setViewer($viewer) ->setPolicyObject($hook) + ->setStatus($status_icon, $status_color, $status_name) ->setHeaderIcon('fa-cloud-upload'); return $header; @@ -117,6 +122,9 @@ final class HeraldWebhookViewController $edit_uri = $this->getApplicationURI("webhook/edit/{$id}/"); $test_uri = $this->getApplicationURI("webhook/test/{$id}/"); + $key_view_uri = $this->getApplicationURI("webhook/key/view/{$id}/"); + $key_cycle_uri = $this->getApplicationURI("webhook/key/cycle/{$id}/"); + $curtain->addAction( id(new PhabricatorActionView()) ->setName(pht('Edit Webhook')) @@ -133,6 +141,22 @@ final class HeraldWebhookViewController ->setWorkflow(true) ->setHref($test_uri)); + $curtain->addAction( + id(new PhabricatorActionView()) + ->setName(pht('View HMAC Key')) + ->setIcon('fa-key') + ->setDisabled(!$can_edit) + ->setWorkflow(true) + ->setHref($key_view_uri)); + + $curtain->addAction( + id(new PhabricatorActionView()) + ->setName(pht('Regenerate HMAC Key')) + ->setIcon('fa-refresh') + ->setDisabled(!$can_edit) + ->setWorkflow(true) + ->setHref($key_cycle_uri)); + return $curtain; } diff --git a/src/applications/herald/garbagecollector/HeraldWebhookRequestGarbageCollector.php b/src/applications/herald/garbagecollector/HeraldWebhookRequestGarbageCollector.php new file mode 100644 index 0000000000..7626f70f00 --- /dev/null +++ b/src/applications/herald/garbagecollector/HeraldWebhookRequestGarbageCollector.php @@ -0,0 +1,29 @@ +establishConnection('w'); + + queryfx( + $conn_w, + 'DELETE FROM %T WHERE dateCreated < %d LIMIT 100', + $table->getTableName(), + $this->getGarbageEpoch()); + + return ($conn_w->getAffectedRows() == 100); + } + +} diff --git a/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php b/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php index 49904a37b8..abc9a40e17 100644 --- a/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php +++ b/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php @@ -6,7 +6,7 @@ final class HeraldWebhookCallManagementWorkflow protected function didConstruct() { $this ->setName('call') - ->setExamples('**call** --id __id__') + ->setExamples('**call** --id __id__ [--object __object__]') ->setSynopsis(pht('Call a webhook.')) ->setArguments( array( @@ -15,6 +15,19 @@ final class HeraldWebhookCallManagementWorkflow 'param' => 'id', 'help' => pht('Webhook ID to call'), ), + array( + 'name' => 'object', + 'param' => 'object', + 'help' => pht('Submit transactions for a particular object.'), + ), + array( + 'name' => 'silent', + 'help' => pht('Set the "silent" flag on the request.'), + ), + array( + 'name' => 'secure', + 'help' => pht('Set the "secure" flag on the request.'), + ), )); } @@ -39,12 +52,38 @@ final class HeraldWebhookCallManagementWorkflow $id)); } - $object = $hook; + $object_name = $args->getArg('object'); + if ($object_name === null) { + $object = $hook; + } else { + $objects = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withNames(array($object_name)) + ->execute(); + if (!$objects) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to load specified object ("%s").', + $object_name)); + } + $object = head($objects); + } - $application_phid = id(new PhabricatorHeraldApplication())->getPHID(); + $xaction_query = + PhabricatorApplicationTransactionQuery::newQueryForObject($object); + + $xactions = $xaction_query + ->withObjectPHIDs(array($object->getPHID())) + ->setViewer($viewer) + ->setLimit(10) + ->execute(); $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) ->setObjectPHID($object->getPHID()) + ->setIsTestAction(true) + ->setIsSilentAction((bool)$args->getArg('silent')) + ->setIsSecureAction((bool)$args->getArg('secure')) + ->setTransactionPHIDs(mpull($xactions, 'getPHID')) ->save(); PhabricatorWorker::setRunAllTasksInProcess(true); diff --git a/src/applications/herald/query/HeraldWebhookSearchEngine.php b/src/applications/herald/query/HeraldWebhookSearchEngine.php index 10852f305f..84997b60b1 100644 --- a/src/applications/herald/query/HeraldWebhookSearchEngine.php +++ b/src/applications/herald/query/HeraldWebhookSearchEngine.php @@ -80,9 +80,16 @@ final class HeraldWebhookSearchEngine ->setViewer($viewer); foreach ($hooks as $hook) { $item = id(new PHUIObjectItemView()) - ->setObjectName(pht('Hook %d', $hook->getID())) + ->setObjectName(pht('Webhook %d', $hook->getID())) ->setHeader($hook->getName()) - ->setHref($hook->getURI()); + ->setHref($hook->getURI()) + ->addAttribute($hook->getWebhookURI()); + + $item->addIcon($hook->getStatusIcon(), $hook->getStatusDisplayName()); + + if ($hook->isDisabled()) { + $item->setDisabled(true); + } $list->addItem($item); } diff --git a/src/applications/herald/storage/HeraldWebhook.php b/src/applications/herald/storage/HeraldWebhook.php index 1f3da0e0fb..05ec69e194 100644 --- a/src/applications/herald/storage/HeraldWebhook.php +++ b/src/applications/herald/storage/HeraldWebhook.php @@ -5,7 +5,8 @@ final class HeraldWebhook implements PhabricatorPolicyInterface, PhabricatorApplicationTransactionInterface, - PhabricatorDestructibleInterface { + PhabricatorDestructibleInterface, + PhabricatorProjectInterface { protected $name; protected $webhookURI; @@ -44,7 +45,7 @@ final class HeraldWebhook ->setStatus(self::HOOKSTATUS_ENABLED) ->setViewPolicy(PhabricatorPolicies::getMostOpenPolicy()) ->setEditPolicy($viewer->getPHID()) - ->setHmacKey(Filesystem::readRandomCharacters(32)); + ->regenerateHMACKey(); } public function getURI() { @@ -56,16 +57,79 @@ final class HeraldWebhook } public static function getStatusDisplayNameMap() { - return array( - self::HOOKSTATUS_FIREHOSE => pht('Firehose'), - self::HOOKSTATUS_ENABLED => pht('Enabled'), - self::HOOKSTATUS_DISABLED => pht('Disabled'), + $specs = self::getStatusSpecifications(); + return ipull($specs, 'name', 'key'); + } + + private static function getStatusSpecifications() { + $specs = array( + array( + 'key' => self::HOOKSTATUS_FIREHOSE, + 'name' => pht('Firehose'), + 'color' => 'orange', + 'icon' => 'fa-star-o', + ), + array( + 'key' => self::HOOKSTATUS_ENABLED, + 'name' => pht('Enabled'), + 'color' => 'bluegrey', + 'icon' => 'fa-check', + ), + array( + 'key' => self::HOOKSTATUS_DISABLED, + 'name' => pht('Disabled'), + 'color' => 'dark', + 'icon' => 'fa-ban', + ), ); + + return ipull($specs, null, 'key'); + } + + + private static function getSpecificationForStatus($status) { + $specs = self::getStatusSpecifications(); + + if (isset($specs[$status])) { + return $specs[$status]; + } + + return array( + 'key' => $status, + 'name' => pht('Unknown ("%s")', $status), + 'icon' => 'fa-question', + 'color' => 'indigo', + ); + } + + public static function getDisplayNameForStatus($status) { + $spec = self::getSpecificationForStatus($status); + return $spec['name']; + } + + public static function getIconForStatus($status) { + $spec = self::getSpecificationForStatus($status); + return $spec['icon']; + } + + public static function getColorForStatus($status) { + $spec = self::getSpecificationForStatus($status); + return $spec['color']; } public function getStatusDisplayName() { $status = $this->getStatus(); - return idx($this->getStatusDisplayNameMap(), $status); + return self::getDisplayNameForStatus($status); + } + + public function getStatusIcon() { + $status = $this->getStatus(); + return self::getIconForStatus($status); + } + + public function getStatusColor() { + $status = $this->getStatus(); + return self::getColorForStatus($status); } public function getErrorBackoffWindow() { @@ -101,6 +165,9 @@ final class HeraldWebhook return false; } + public function regenerateHMACKey() { + return $this->setHMACKey(Filesystem::readRandomCharacters(32)); + } /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/herald/storage/HeraldWebhookRequest.php b/src/applications/herald/storage/HeraldWebhookRequest.php index 19e9bbbc39..d6e1364aaa 100644 --- a/src/applications/herald/storage/HeraldWebhookRequest.php +++ b/src/applications/herald/storage/HeraldWebhookRequest.php @@ -116,6 +116,30 @@ final class HeraldWebhookRequest return $this->getProperty('transactionPHIDs', array()); } + public function setIsSilentAction($bool) { + return $this->setProperty('silent', $bool); + } + + public function getIsSilentAction() { + return $this->getProperty('silent', false); + } + + public function setIsTestAction($bool) { + return $this->setProperty('test', $bool); + } + + public function getIsTestAction() { + return $this->getProperty('test', false); + } + + public function setIsSecureAction($bool) { + return $this->setProperty('secure', $bool); + } + + public function getIsSecureAction() { + return $this->getProperty('secure', false); + } + public function queueCall() { PhabricatorWorker::scheduleTask( 'HeraldWebhookWorker', diff --git a/src/applications/herald/view/HeraldWebhookRequestListView.php b/src/applications/herald/view/HeraldWebhookRequestListView.php index e44a136d41..4e0f6510b9 100644 --- a/src/applications/herald/view/HeraldWebhookRequestListView.php +++ b/src/applications/herald/view/HeraldWebhookRequestListView.php @@ -44,12 +44,20 @@ final class HeraldWebhookRequestListView $rowc[] = null; } + $last_epoch = $request->getLastRequestEpoch(); + if ($request->getLastRequestEpoch()) { + $last_request = phabricator_datetime($last_epoch, $viewer); + } else { + $last_request = null; + } + $rows[] = array( $request->getID(), $icon, $handles[$request->getObjectPHID()]->renderLink(), $request->getErrorType(), $request->getErrorCode(), + $last_request, ); } @@ -62,6 +70,7 @@ final class HeraldWebhookRequestListView pht('Object'), pht('Type'), pht('Code'), + pht('Requested At'), )) ->setColumnClasses( array( @@ -70,6 +79,7 @@ final class HeraldWebhookRequestListView 'wide', '', '', + '', )); return $table; diff --git a/src/applications/herald/worker/HeraldWebhookWorker.php b/src/applications/herald/worker/HeraldWebhookWorker.php index a3d1ab64fc..fdf4bf7dc1 100644 --- a/src/applications/herald/worker/HeraldWebhookWorker.php +++ b/src/applications/herald/worker/HeraldWebhookWorker.php @@ -143,10 +143,15 @@ final class HeraldWebhookWorker 'object' => array( 'phid' => $object->getPHID(), ), + 'action' => array( + 'test' => $request->getIsTestAction(), + 'silent' => $request->getIsSilentAction(), + 'secure' => $request->getIsSecureAction(), + ), 'transactions' => $xaction_data, ); - $payload = phutil_json_encode($payload); + $payload = id(new PhutilJSON())->encodeFormatted($payload); $key = $hook->getHmacKey(); $signature = PhabricatorHash::digestHMACSHA256($payload, $key); $uri = $hook->getWebhookURI(); diff --git a/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php b/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php index 4051b6c597..951001e0b6 100644 --- a/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php +++ b/src/applications/herald/xaction/HeraldWebhookStatusTransaction.php @@ -14,20 +14,32 @@ final class HeraldWebhookStatusTransaction } public function getTitle() { + $old_value = $this->getOldValue(); + $new_value = $this->getNewValue(); + + $old_status = HeraldWebhook::getDisplayNameForStatus($old_value); + $new_status = HeraldWebhook::getDisplayNameForStatus($new_value); + return pht( '%s changed hook status from %s to %s.', $this->renderAuthor(), - $this->renderOldValue(), - $this->renderNewValue()); + $this->renderValue($old_status), + $this->renderValue($new_status)); } public function getTitleForFeed() { + $old_value = $this->getOldValue(); + $new_value = $this->getNewValue(); + + $old_status = HeraldWebhook::getDisplayNameForStatus($old_value); + $new_status = HeraldWebhook::getDisplayNameForStatus($new_value); + return pht( '%s changed %s from %s to %s.', $this->renderAuthor(), $this->renderObject(), - $this->renderOldValue(), - $this->renderNewValue()); + $this->renderValue($old_status), + $this->renderValue($new_status)); } public function validateTransactions($object, array $xactions) { From 4887c6aa803d2b231b316b5273a11fa98d462547 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 11:46:02 -0800 Subject: [PATCH 60/67] Allow "transaction.search" to be constrained by PHIDs Summary: Depends on D19046. Ref T11330. Supports querying for specific transactions while responding to webhooks. Test Plan: Called `transaction.search` with and without PHID constraints. Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19047 --- .../TransactionSearchConduitAPIMethod.php | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/applications/transactions/conduit/TransactionSearchConduitAPIMethod.php b/src/applications/transactions/conduit/TransactionSearchConduitAPIMethod.php index 43b94874bf..66285547b3 100644 --- a/src/applications/transactions/conduit/TransactionSearchConduitAPIMethod.php +++ b/src/applications/transactions/conduit/TransactionSearchConduitAPIMethod.php @@ -22,6 +22,7 @@ final class TransactionSearchConduitAPIMethod protected function defineParamTypes() { return array( 'objectIdentifier' => 'phid|string', + 'constraints' => 'map', ) + $this->getPagerParamTypes(); } @@ -66,10 +67,23 @@ final class TransactionSearchConduitAPIMethod $xaction_query = PhabricatorApplicationTransactionQuery::newQueryForObject( $object); - $xactions = $xaction_query + $xaction_query ->withObjectPHIDs(array($object->getPHID())) - ->setViewer($viewer) - ->executeWithCursorPager($pager); + ->setViewer($viewer); + + $constraints = $request->getValue('constraints', array()); + PhutilTypeSpec::checkMap( + $constraints, + array( + 'phids' => 'optional list', + )); + + $with_phids = idx($constraints, 'phids'); + if ($with_phids) { + $xaction_query->withPHIDs($with_phids); + } + + $xactions = $xaction_query->executeWithCursorPager($pager); if ($xactions) { $template = head($xactions)->getApplicationTransactionCommentObject(); From 41d28abfcc46f8c139046fd909e148ec23d95135 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 11:48:41 -0800 Subject: [PATCH 61/67] Trigger all "Firehose" webhooks on all transactional edits Summary: Depends on D19047. Ref T11330. Triggers every firehose hook on every edit; prepares for Herald triggers. Test Plan: Configured a firehose hook, edited some objects, saw callbacks. Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19048 --- .../HeraldWebhookTestController.php | 1 + .../herald/editor/HeraldWebhookEditor.php | 9 +++ .../HeraldWebhookCallManagementWorkflow.php | 3 + .../phid/HeraldWebhookRequestPHIDType.php | 3 +- .../herald/storage/HeraldWebhookRequest.php | 8 +++ .../herald/worker/HeraldWebhookWorker.php | 10 +++- ...habricatorApplicationTransactionEditor.php | 57 +++++++++++++++++++ 7 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/applications/herald/controller/HeraldWebhookTestController.php b/src/applications/herald/controller/HeraldWebhookTestController.php index b791ba783b..510e7f3f81 100644 --- a/src/applications/herald/controller/HeraldWebhookTestController.php +++ b/src/applications/herald/controller/HeraldWebhookTestController.php @@ -52,6 +52,7 @@ final class HeraldWebhookTestController $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) ->setObjectPHID($object->getPHID()) + ->setTriggerPHIDs(array($viewer->getPHID())) ->setIsTestAction(true) ->setTransactionPHIDs(mpull($xactions, 'getPHID')) ->save(); diff --git a/src/applications/herald/editor/HeraldWebhookEditor.php b/src/applications/herald/editor/HeraldWebhookEditor.php index 2206a59411..1f138e5028 100644 --- a/src/applications/herald/editor/HeraldWebhookEditor.php +++ b/src/applications/herald/editor/HeraldWebhookEditor.php @@ -19,4 +19,13 @@ final class HeraldWebhookEditor return pht('%s created %s.', $author, $object); } + public function getTransactionTypes() { + $types = parent::getTransactionTypes(); + + $types[] = PhabricatorTransactions::TYPE_VIEW_POLICY; + $types[] = PhabricatorTransactions::TYPE_EDIT_POLICY; + + return $types; + } + } diff --git a/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php b/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php index abc9a40e17..10249cca68 100644 --- a/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php +++ b/src/applications/herald/management/HeraldWebhookCallManagementWorkflow.php @@ -78,11 +78,14 @@ final class HeraldWebhookCallManagementWorkflow ->setLimit(10) ->execute(); + $application_phid = id(new PhabricatorHeraldApplication())->getPHID(); + $request = HeraldWebhookRequest::initializeNewWebhookRequest($hook) ->setObjectPHID($object->getPHID()) ->setIsTestAction(true) ->setIsSilentAction((bool)$args->getArg('silent')) ->setIsSecureAction((bool)$args->getArg('secure')) + ->setTriggerPHIDs(array($application_phid)) ->setTransactionPHIDs(mpull($xactions, 'getPHID')) ->save(); diff --git a/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php b/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php index 8263ba7484..bcf5afb0d3 100644 --- a/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php +++ b/src/applications/herald/phid/HeraldWebhookRequestPHIDType.php @@ -31,8 +31,7 @@ final class HeraldWebhookRequestPHIDType extends PhabricatorPHIDType { foreach ($handles as $phid => $handle) { $request = $objects[$phid]; - - // TODO: Fill this in. + $handle->setName(pht('Webhook Request %d', $request->getID())); } } diff --git a/src/applications/herald/storage/HeraldWebhookRequest.php b/src/applications/herald/storage/HeraldWebhookRequest.php index d6e1364aaa..5db5b2916e 100644 --- a/src/applications/herald/storage/HeraldWebhookRequest.php +++ b/src/applications/herald/storage/HeraldWebhookRequest.php @@ -116,6 +116,14 @@ final class HeraldWebhookRequest return $this->getProperty('transactionPHIDs', array()); } + public function setTriggerPHIDs(array $phids) { + return $this->setProperty('triggerPHIDs', $phids); + } + + public function getTriggerPHIDs() { + return $this->getProperty('triggerPHIDs', array()); + } + public function setIsSilentAction($bool) { return $this->setProperty('silent', $bool); } diff --git a/src/applications/herald/worker/HeraldWebhookWorker.php b/src/applications/herald/worker/HeraldWebhookWorker.php index fdf4bf7dc1..f74268c694 100644 --- a/src/applications/herald/worker/HeraldWebhookWorker.php +++ b/src/applications/herald/worker/HeraldWebhookWorker.php @@ -138,11 +138,19 @@ final class HeraldWebhookWorker ); } + $trigger_data = array(); + foreach ($request->getTriggerPHIDs() as $trigger_phid) { + $trigger_data[] = array( + 'phid' => $trigger_phid, + ); + } + $payload = array( - 'triggers' => array(), 'object' => array( + 'type' => phid_get_type($object->getPHID()), 'phid' => $object->getPHID(), ), + 'triggers' => $trigger_data, 'action' => array( 'test' => $request->getIsTestAction(), 'silent' => $request->getIsSilentAction(), diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 5e6a311a85..09c6af62b7 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -79,6 +79,7 @@ abstract class PhabricatorApplicationTransactionEditor private $mailRemovedPHIDs = array(); private $mailUnexpandablePHIDs = array(); private $mailMutedPHIDs = array(); + private $webhookMap = array(); private $transactionQueue = array(); @@ -1307,6 +1308,8 @@ abstract class PhabricatorApplicationTransactionEditor $mail->save(); } + $this->queueWebhooks($object, $xactions); + return $xactions; } @@ -3660,6 +3663,8 @@ abstract class PhabricatorApplicationTransactionEditor 'mailStamps', 'mailUnexpandablePHIDs', 'mailMutedPHIDs', + 'webhookMap', + 'silent', ); } @@ -4240,4 +4245,56 @@ abstract class PhabricatorApplicationTransactionEditor return $this; } + private function queueWebhooks($object, array $xactions) { + $hook_viewer = PhabricatorUser::getOmnipotentUser(); + + $webhook_map = $this->webhookMap; + if (!is_array($webhook_map)) { + $webhook_map = array(); + } + + // Add any "Firehose" hooks to the list of hooks we're going to call. + $firehose_hooks = id(new HeraldWebhookQuery()) + ->setViewer($hook_viewer) + ->withStatuses( + array( + HeraldWebhook::HOOKSTATUS_FIREHOSE, + )) + ->execute(); + foreach ($firehose_hooks as $firehose_hook) { + // This is "the hook itself is the reason this hook is being called", + // since we're including it because it's configured as a firehose + // hook. + $hook_phid = $firehose_hook->getPHID(); + $webhook_map[$hook_phid][] = $hook_phid; + } + + if (!$webhook_map) { + return; + } + + // NOTE: We're going to queue calls to disabled webhooks, they'll just + // immediately fail in the worker queue. This makes the behavior more + // visible. + + $call_hooks = id(new HeraldWebhookQuery()) + ->setViewer($hook_viewer) + ->withPHIDs(array_keys($webhook_map)) + ->execute(); + + foreach ($call_hooks as $call_hook) { + $trigger_phids = idx($webhook_map, $call_hook->getPHID()); + + $request = HeraldWebhookRequest::initializeNewWebhookRequest($call_hook) + ->setObjectPHID($object->getPHID()) + ->setTransactionPHIDs(mpull($xactions, 'getPHID')) + ->setTriggerPHIDs($trigger_phids) + ->setIsSilentAction((bool)$this->getIsSilent()) + ->setIsSecureAction((bool)$this->getMustEncrypt()) + ->save(); + + $request->queueCall(); + } + } + } From 98c701ffc55f59fd565156b1a83dc9af4b19c634 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 12:18:15 -0800 Subject: [PATCH 62/67] Add a "Call webhooks" action to Herald Summary: Depends on D19048. Fixes T11330. Test Plan: Wrote rules to call webhooks selectively, saw them fire appropriately with correct trigger attribution. Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19049 --- src/__phutil_library_map__.php | 4 ++ .../herald/action/HeraldCallWebhookAction.php | 62 +++++++++++++++++++ .../herald/adapter/HeraldAdapter.php | 14 +++++ .../typeahead/HeraldWebhookDatasource.php | 48 ++++++++++++++ ...habricatorApplicationTransactionEditor.php | 1 + 5 files changed, 129 insertions(+) create mode 100644 src/applications/herald/action/HeraldCallWebhookAction.php create mode 100644 src/applications/herald/typeahead/HeraldWebhookDatasource.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index e078e6c10a..6c3514a2c9 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1357,6 +1357,7 @@ phutil_register_library_map(array( 'HeraldApplyTranscript' => 'applications/herald/storage/transcript/HeraldApplyTranscript.php', 'HeraldBasicFieldGroup' => 'applications/herald/field/HeraldBasicFieldGroup.php', 'HeraldBuildableState' => 'applications/herald/state/HeraldBuildableState.php', + 'HeraldCallWebhookAction' => 'applications/herald/action/HeraldCallWebhookAction.php', 'HeraldCommentAction' => 'applications/herald/action/HeraldCommentAction.php', 'HeraldCommitAdapter' => 'applications/diffusion/herald/HeraldCommitAdapter.php', 'HeraldCondition' => 'applications/herald/storage/HeraldCondition.php', @@ -1444,6 +1445,7 @@ phutil_register_library_map(array( 'HeraldWebhook' => 'applications/herald/storage/HeraldWebhook.php', 'HeraldWebhookCallManagementWorkflow' => 'applications/herald/management/HeraldWebhookCallManagementWorkflow.php', 'HeraldWebhookController' => 'applications/herald/controller/HeraldWebhookController.php', + 'HeraldWebhookDatasource' => 'applications/herald/typeahead/HeraldWebhookDatasource.php', 'HeraldWebhookEditController' => 'applications/herald/controller/HeraldWebhookEditController.php', 'HeraldWebhookEditEngine' => 'applications/herald/editor/HeraldWebhookEditEngine.php', 'HeraldWebhookEditor' => 'applications/herald/editor/HeraldWebhookEditor.php', @@ -6631,6 +6633,7 @@ phutil_register_library_map(array( 'HeraldApplyTranscript' => 'Phobject', 'HeraldBasicFieldGroup' => 'HeraldFieldGroup', 'HeraldBuildableState' => 'HeraldState', + 'HeraldCallWebhookAction' => 'HeraldAction', 'HeraldCommentAction' => 'HeraldAction', 'HeraldCommitAdapter' => array( 'HeraldAdapter', @@ -6741,6 +6744,7 @@ phutil_register_library_map(array( ), 'HeraldWebhookCallManagementWorkflow' => 'HeraldWebhookManagementWorkflow', 'HeraldWebhookController' => 'HeraldController', + 'HeraldWebhookDatasource' => 'PhabricatorTypeaheadDatasource', 'HeraldWebhookEditController' => 'HeraldWebhookController', 'HeraldWebhookEditEngine' => 'PhabricatorEditEngine', 'HeraldWebhookEditor' => 'PhabricatorApplicationTransactionEditor', diff --git a/src/applications/herald/action/HeraldCallWebhookAction.php b/src/applications/herald/action/HeraldCallWebhookAction.php new file mode 100644 index 0000000000..a2003f4f33 --- /dev/null +++ b/src/applications/herald/action/HeraldCallWebhookAction.php @@ -0,0 +1,62 @@ +getAdapter(); + $rule = $effect->getRule(); + $target = $effect->getTarget(); + + foreach ($target as $webhook_phid) { + $adapter->queueWebhook($webhook_phid, $rule->getPHID()); + } + + $this->logEffect(self::DO_WEBHOOK, $target); + } + + public function getHeraldActionStandardType() { + return self::STANDARD_PHID_LIST; + } + + protected function getActionEffectMap() { + return array( + self::DO_WEBHOOK => array( + 'icon' => 'fa-cloud-upload', + 'color' => 'green', + 'name' => pht('Called Webhooks'), + ), + ); + } + + public function renderActionDescription($value) { + return pht('Call webhooks: %s.', $this->renderHandleList($value)); + } + + protected function renderActionEffectDescription($type, $data) { + return pht('Called webhooks: %s.', $this->renderHandleList($data)); + } + + protected function getDatasource() { + return new HeraldWebhookDatasource(); + } + +} diff --git a/src/applications/herald/adapter/HeraldAdapter.php b/src/applications/herald/adapter/HeraldAdapter.php index 940d604019..7764332f11 100644 --- a/src/applications/herald/adapter/HeraldAdapter.php +++ b/src/applications/herald/adapter/HeraldAdapter.php @@ -41,6 +41,7 @@ abstract class HeraldAdapter extends Phobject { private $viewer; private $mustEncryptReasons = array(); private $actingAsPHID; + private $webhookMap = array(); public function getEmailPHIDs() { return array_values($this->emailPHIDs); @@ -1206,4 +1207,17 @@ abstract class HeraldAdapter extends Phobject { return $this->mustEncryptReasons; } + +/* -( Webhooks )----------------------------------------------------------- */ + + + final public function queueWebhook($webhook_phid, $rule_phid) { + $this->webhookMap[$webhook_phid][] = $rule_phid; + return $this; + } + + final public function getWebhookMap() { + return $this->webhookMap; + } + } diff --git a/src/applications/herald/typeahead/HeraldWebhookDatasource.php b/src/applications/herald/typeahead/HeraldWebhookDatasource.php new file mode 100644 index 0000000000..a66431d515 --- /dev/null +++ b/src/applications/herald/typeahead/HeraldWebhookDatasource.php @@ -0,0 +1,48 @@ +getViewer(); + $raw_query = $this->getRawQuery(); + + $hooks = id(new HeraldWebhookQuery()) + ->setViewer($viewer) + ->execute(); + + $handles = id(new PhabricatorHandleQuery()) + ->setViewer($viewer) + ->withPHIDs(mpull($hooks, 'getPHID')) + ->execute(); + + $results = array(); + foreach ($hooks as $hook) { + $handle = $handles[$hook->getPHID()]; + + $result = id(new PhabricatorTypeaheadResult()) + ->setName($handle->getFullName()) + ->setPHID($handle->getPHID()); + + if ($hook->isDisabled()) { + $result->setClosed(pht('Disabled')); + } + + $results[] = $result; + } + + return $results; + } +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 09c6af62b7..20d745555a 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -1156,6 +1156,7 @@ abstract class PhabricatorApplicationTransactionEditor $adapter = $this->getHeraldAdapter(); $this->heraldEmailPHIDs = $adapter->getEmailPHIDs(); $this->heraldForcedEmailPHIDs = $adapter->getForcedEmailPHIDs(); + $this->webhookMap = $adapter->getWebhookMap(); } $xactions = $this->didApplyTransactions($object, $xactions); From 64177cb16e77ad574ec215f523ec3f6b6593cd17 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 12:59:55 -0800 Subject: [PATCH 63/67] Document how webhooks work Summary: Depends on D19049. Ref T11330. Adds some documentation for webhooks. Test Plan: Read the documentation and found it to be exceptionally accurate and helpful. Maniphest Tasks: T11330 Differential Revision: https://secure.phabricator.com/D19050 --- .../PhabricatorHeraldApplication.php | 4 + .../herald/worker/HeraldWebhookWorker.php | 1 + .../bulk/PhabricatorBulkEngine.php | 2 +- ...habricatorApplicationTransactionEditor.php | 1 + src/docs/user/userguide/webhooks.diviner | 212 ++++++++++++++++++ 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 src/docs/user/userguide/webhooks.diviner diff --git a/src/applications/herald/application/PhabricatorHeraldApplication.php b/src/applications/herald/application/PhabricatorHeraldApplication.php index 47e7bd7bfa..753c03b266 100644 --- a/src/applications/herald/application/PhabricatorHeraldApplication.php +++ b/src/applications/herald/application/PhabricatorHeraldApplication.php @@ -28,6 +28,10 @@ final class PhabricatorHeraldApplication extends PhabricatorApplication { 'name' => pht('Herald User Guide'), 'href' => PhabricatorEnv::getDoclink('Herald User Guide'), ), + array( + 'name' => pht('User Guide: Webhooks'), + 'href' => PhabricatorEnv::getDoclink('User Guide: Webhooks'), + ), ); } diff --git a/src/applications/herald/worker/HeraldWebhookWorker.php b/src/applications/herald/worker/HeraldWebhookWorker.php index f74268c694..837ec0bb23 100644 --- a/src/applications/herald/worker/HeraldWebhookWorker.php +++ b/src/applications/herald/worker/HeraldWebhookWorker.php @@ -155,6 +155,7 @@ final class HeraldWebhookWorker 'test' => $request->getIsTestAction(), 'silent' => $request->getIsSilentAction(), 'secure' => $request->getIsSecureAction(), + 'epoch' => (int)$request->getDateCreated(), ), 'transactions' => $xaction_data, ); diff --git a/src/applications/transactions/bulk/PhabricatorBulkEngine.php b/src/applications/transactions/bulk/PhabricatorBulkEngine.php index 534390b518..0091321245 100644 --- a/src/applications/transactions/bulk/PhabricatorBulkEngine.php +++ b/src/applications/transactions/bulk/PhabricatorBulkEngine.php @@ -377,7 +377,7 @@ abstract class PhabricatorBulkEngine extends Phobject { ''))) ->appendChild( id(new AphrontFormSubmitControl()) - ->setValue(pht('Apply Bulk Edit')) + ->setValue(pht('Continue')) ->addCancelButton($cancel_uri)); } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 20d745555a..d1413b49a6 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -4290,6 +4290,7 @@ abstract class PhabricatorApplicationTransactionEditor ->setObjectPHID($object->getPHID()) ->setTransactionPHIDs(mpull($xactions, 'getPHID')) ->setTriggerPHIDs($trigger_phids) + ->setRetryMode(HeraldWebhookRequest::RETRY_FOREVER) ->setIsSilentAction((bool)$this->getIsSilent()) ->setIsSecureAction((bool)$this->getMustEncrypt()) ->save(); diff --git a/src/docs/user/userguide/webhooks.diviner b/src/docs/user/userguide/webhooks.diviner new file mode 100644 index 0000000000..51521b462b --- /dev/null +++ b/src/docs/user/userguide/webhooks.diviner @@ -0,0 +1,212 @@ +@title User Guide: Webhooks +@group userguide + +Guide to configuring webhooks. + + +Overview +======== + +If you'd like to react to events in Phabricator or publish them into external +systems, you can configure webhooks. + +Configure webhooks in {nav Herald > Webhooks}. Users must have the +"Can Create Webhooks" permission to create new webhooks. + + +Triggering Hooks +================ + +Webhooks can be triggered in two ways: + + - Set the hook mode to **Firehose**. In this mode, your hook will be called + for every event. + - Set the hook mode to **Enabled**, then write Herald rules which use the + **Call webhooks** action to choose when the hook is called. This allows + you to choose a narrower range of events to be notified about. + + +Testing Hooks +============= + +To test a webhook, use {nav New Test Request} from the web interface. + +You can also use the command-line tool, which supports a few additional +options: + +``` +phabricator/ $ ./bin/webhook call --id 42 --object D123 +``` + +You can use a tool like [[ https://requestb.in | RequestBin ]] to inspect +the headers and payload for calls to hooks. + + +Verifying Requests +================== + +When your webhook callback URI receives a request, it didn't necessarily come +from Phabricator. An attacker or mischievous user can normally call your hook +directly and pretend to be notifying you of an event. + +To verify that the request is authentic, first retrieve the webhook key from +the web UI with {nav View HMAC Key}. This is a shared secret which will let you +verify that Phabricator originated a request. + +When you receive a request, compute the SHA256 HMAC value of the request body +using the HMAC key as the key. The value should match the value in the +`X-Phabricator-Webhook-Signature` field. + +To compute the SHA256 HMAC of a string in PHP, do this: + +```lang=php +$signature = hash_hmac('sha256', $request_body, $hmac_key); +``` + +To compute the SHA256 HMAC of a string in Python, do this: + +```lang=python +from subprocess import check_output + +signature = check_output( + [ + "php", + "-r", + "echo hash_hmac('sha256', $argv[1], $argv[2]);", + "--", + request_body, + hmac_key + ]) +``` + +Other languages often provide similar support. + +If you somehow disclose the key by accident, use {nav Regenerate HMAC Key} to +throw it away and generate a new one. + + +Request Format +============== + +Webhook callbacks are POST requests with a JSON payload in the body. The +payload looks like this: + +```lang=json +{ + "object": { + "type": "TASK", + "phid": "PHID-TASK-abcd..." + }, + "triggers": [ + { + "phid": "PHID-HRUL-abcd..." + } + ], + "action": { + "test": false, + "silent": false, + "secure": false, + "epoch": 12345 + }, + "transactions": [ + { + "phid": "PHID-XACT-TASK-abcd..." + } + ] +} +``` + +The **object** map describes the object which was edited. + +The **triggers** are a list of reasons why the hook was called. When the hook +is triggered by Herald rules, the specific rules which triggered the call will +be listed. For firehose rules, the rule itself will be listed as the trigger. +For test calls, the user making the request will be listed as a trigger. + +The **action** map has metadata about the action: + + - `test` This was a test call from the web UI or console. + - `silent` This is a silent edit which won't send mail or notifications in + Phabricator. If your hook is doing something like copying events into + a chatroom, it may want to respect this flag. + - `secure` Details about this object should only be transmitted over + secure channels. Your hook may want to respect this flag. + - `epoch` The epoch timestamp when the callback was queued. + +The **transactions** list contains information about the actual changes which +triggered the callback. + + +Responding to Requests +====================== + +Although trivial hooks may not need any more information than this to act, the +information conveyed in the hook body is a minimum set of pointers to relevant +data and likely insufficient for more complex hooks. + +Complex hooks should expect to react to receiving a request by making API +calls to Conduit to retrieve additional information about the object and +transactions. + +Hooks that are interested in reading object state should generally make a call +to a method like `maniphest.search` or `differential.revision.search` using +the PHID from the `object` field to retrieve full details about the object +state. + +Hooks that are interested in changes should generally make a call to +`transaction.search`, passing the transaction PHIDs as a constraint to retrieve +details about the transactions. + +The `phid.query` method can also be used to retrieve generic information about +a list of objects. + + +Retries and Rate Limiting +========================= + +Test requests are never retried: they execute exactly once. + +Live requests are automatically retried. If your endpoint does not return a +HTTP 2XX response, the request will be retried regularly until it suceeds. + +Retries will continue until the request succeeds or is garbage collected. By +default, this is after 7 days. + +If a webhook is disabled, outstanding queued requests will be failed +permanently. Activity which occurs while it is disabled will never be sent to +the callback URI. (Disabling a hook does not "pause" it so that it can be +"resumed" later and pick back up where it left off in the event stream.) + +If a webhook encounters a significant number of errors in a short period of +time, the webhook will be paused for a few minutes before additional requests +are made. The web UI shows a warning indicator when a hook is paused because of +errors. + +Hook requests time out after 10 seconds. Consider offloading response handling +to some kind of worker queue if you expect to routinely require more than 10 +seconds to respond to requests. + +Hook callbacks are single-threaded: you will never receive more than one +simultaneous call to the same webhook from Phabricator. If you have a firehose +hook on an active install, it may be important to respond to requests quickly +to avoid accumulating a backlog. + +Callbacks may be invoked out-of-order. You should not assume that the order +you receive requests in is chronological order. If your hook is order-dependent, +you can ignore the transactions in the callback and use `transaction.search` to +retrieve a consistent list of ordered changes to the object. + +Callbacks may be delayed for an arbitrarily long amount of time, up to the +garbage collection limit. You should not assume that calls are real time. If +your hook is doing something time-sensitive, you can measure the delivery delay +by comparing the current time to the `epoch` value in the `action` field and +ignoring old actions or handling them in some special way. + + +Next Steps +========== + +Continue by: + + - learning more about Herald with @{article:Herald User Guide}; or + - interacting with the Conduit API with @{article:Conduit API Overview}. From 4fef0a6128244b90d0ba60f4758214a6518d4508 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 14:23:39 -0800 Subject: [PATCH 64/67] Allow a wider range of characters in macro names, including emoji MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Fixes T6121. See PHI357. - Allow emoji and other unicode (like Chinese characters) as long as you have at least three of them. - Disallow macros with only latin symbols. These were previously allowed. Test Plan: Created a macro for "🐢🐢🐢", then used it in a comment. Maniphest Tasks: T6121 Differential Revision: https://secure.phabricator.com/D19051 --- src/__phutil_library_map__.php | 2 + .../PhabricatorImageMacroRemarkupRule.php | 2 +- .../PhabricatorMacroNameTransaction.php | 47 ++++++++++++++++--- .../__tests__/PhabricatorMacroTestCase.php | 46 ++++++++++++++++++ 4 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 src/applications/macro/xaction/__tests__/PhabricatorMacroTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 6c3514a2c9..13dd7374d2 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3203,6 +3203,7 @@ phutil_register_library_map(array( 'PhabricatorMacroQuery' => 'applications/macro/query/PhabricatorMacroQuery.php', 'PhabricatorMacroReplyHandler' => 'applications/macro/mail/PhabricatorMacroReplyHandler.php', 'PhabricatorMacroSearchEngine' => 'applications/macro/query/PhabricatorMacroSearchEngine.php', + 'PhabricatorMacroTestCase' => 'applications/macro/xaction/__tests__/PhabricatorMacroTestCase.php', 'PhabricatorMacroTransaction' => 'applications/macro/storage/PhabricatorMacroTransaction.php', 'PhabricatorMacroTransactionComment' => 'applications/macro/storage/PhabricatorMacroTransactionComment.php', 'PhabricatorMacroTransactionQuery' => 'applications/macro/query/PhabricatorMacroTransactionQuery.php', @@ -8751,6 +8752,7 @@ phutil_register_library_map(array( 'PhabricatorMacroQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorMacroReplyHandler' => 'PhabricatorApplicationTransactionReplyHandler', 'PhabricatorMacroSearchEngine' => 'PhabricatorApplicationSearchEngine', + 'PhabricatorMacroTestCase' => 'PhabricatorTestCase', 'PhabricatorMacroTransaction' => 'PhabricatorModularTransaction', 'PhabricatorMacroTransactionComment' => 'PhabricatorApplicationTransactionComment', 'PhabricatorMacroTransactionQuery' => 'PhabricatorApplicationTransactionQuery', diff --git a/src/applications/macro/markup/PhabricatorImageMacroRemarkupRule.php b/src/applications/macro/markup/PhabricatorImageMacroRemarkupRule.php index 8911181a77..4c50322daa 100644 --- a/src/applications/macro/markup/PhabricatorImageMacroRemarkupRule.php +++ b/src/applications/macro/markup/PhabricatorImageMacroRemarkupRule.php @@ -8,7 +8,7 @@ final class PhabricatorImageMacroRemarkupRule extends PhutilRemarkupRule { public function apply($text) { return preg_replace_callback( - '@^\s*([a-zA-Z0-9:_\-]+)$@m', + '@^\s*([a-zA-Z0-9:_\x7f-\xff-]+)$@m', array($this, 'markupImageMacro'), $text); } diff --git a/src/applications/macro/xaction/PhabricatorMacroNameTransaction.php b/src/applications/macro/xaction/PhabricatorMacroNameTransaction.php index 68710605aa..f55de443bf 100644 --- a/src/applications/macro/xaction/PhabricatorMacroNameTransaction.php +++ b/src/applications/macro/xaction/PhabricatorMacroNameTransaction.php @@ -52,12 +52,16 @@ final class PhabricatorMacroNameTransaction new PhutilNumber($max_length))); } - if (!preg_match('/^[a-z0-9:_-]{3,}\z/', $new_value)) { - $errors[] = $this->newInvalidError( - pht('Macro name "%s" be at least three characters long and contain '. - 'only lowercase letters, digits, hyphens, colons and '. - 'underscores.', - $new_value)); + if (!self::isValidMacroName($new_value)) { + // This says "emoji", but the actual rule we implement is "all other + // unicode characters are also fine". + $errors[] = $this->newInvalidError( + pht( + 'Macro name "%s" be: at least three characters long; and contain '. + 'only lowercase letters, digits, hyphens, colons, underscores, '. + 'and emoji; and not be composed entirely of latin symbols.', + $new_value), + $xaction); } // Check name is unique when updating / creating @@ -78,4 +82,35 @@ final class PhabricatorMacroNameTransaction return $errors; } + public static function isValidMacroName($name) { + if (preg_match('/^[:_-]+\z/', $name)) { + return false; + } + + // Accept trivial macro names. + if (preg_match('/^[a-z0-9:_-]{3,}\z/', $name)) { + return true; + } + + // Reject names with fewer than 3 glyphs. + $length = phutil_utf8v_combined($name); + if (count($length) < 3) { + return false; + } + + // Check character-by-character for any symbols that we don't want. + $characters = phutil_utf8v($name); + foreach ($characters as $character) { + if (ord($character[0]) > 0x7F) { + continue; + } + + if (preg_match('/^[^a-z0-9:_-]/', $character)) { + return false; + } + } + + return true; + } + } diff --git a/src/applications/macro/xaction/__tests__/PhabricatorMacroTestCase.php b/src/applications/macro/xaction/__tests__/PhabricatorMacroTestCase.php new file mode 100644 index 0000000000..aa13ec0ac1 --- /dev/null +++ b/src/applications/macro/xaction/__tests__/PhabricatorMacroTestCase.php @@ -0,0 +1,46 @@ + false, + "{$lit}{$lit}" => false, + + // Too short. + 'a' => false, + '' => false, + + // Bad characters. + 'yes!' => false, + "{$lit} {$lit} {$lit}" => false, + "aaa\nbbb" => false, + 'aaa~' => false, + 'aaa`' => false, + + // Special rejections for only latin symbols. + '---' => false, + '___' => false, + '-_-' => false, + ':::' => false, + '-_:' => false, + + "{$lit}{$lit}{$lit}" => true, + 'bwahahaha' => true, + "u{$combining_diaeresis}nt" => true, + 'a-a-a-a' => true, + ); + + foreach ($cases as $input => $expect) { + $this->assertEqual( + $expect, + PhabricatorMacroNameTransaction::isValidMacroName($input), + pht('Validity of macro "%s"', $input)); + } + } +} From 9c8484de320858f2b86c2aa9daee012065a90e0e Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 14:41:45 -0800 Subject: [PATCH 65/67] Document the STMP port option Summary: Ref T12677. This slipped thorugh in the upgrade. Test Plan: Read documentation. Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19052 --- src/docs/user/configuration/configuring_outbound_email.diviner | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 37a344c275..5de8429c13 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -240,6 +240,7 @@ You can use this adapter to send mail via an external SMTP server, like Gmail. To use this mailer, set `type` to `smtp`, then configure these `options`: - `host`: Required string. The hostname of your SMTP server. + - `port`: Optional int. The port to connect to on your SMTP server. - `user`: Optional string. Username used for authentication. - `password`: Optional string. Password for authentication. - `protocol`: Optional string. Set to `tls` or `ssl` if necessary. Use From 9b7d5b74d4a1371583ff48560b40721404d65009 Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 14:47:34 -0800 Subject: [PATCH 66/67] Purge `ssh-auth` key cache after trust/untrust Summary: See PHI358. The `bin/almanac [un]trust-key` workflows don't properly purge the SSH key cache, but should. Test Plan: - Added key `ssh-rsa xyz` to a device. - Used `bin/ssh-auth | grep xyz` to test for the presence of the key. - Before patch: Saw it not present, trusted it, saw it still not present. - After patch: Saw it not present, trusted it, saw it now present. Untrusted it, saw it no longer present. Differential Revision: https://secure.phabricator.com/D19053 --- .../almanac/management/AlmanacManagementTrustKeyWorkflow.php | 2 ++ .../almanac/management/AlmanacManagementUntrustKeyWorkflow.php | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/applications/almanac/management/AlmanacManagementTrustKeyWorkflow.php b/src/applications/almanac/management/AlmanacManagementTrustKeyWorkflow.php index c0bbc59ff0..631cef8c96 100644 --- a/src/applications/almanac/management/AlmanacManagementTrustKeyWorkflow.php +++ b/src/applications/almanac/management/AlmanacManagementTrustKeyWorkflow.php @@ -81,6 +81,8 @@ final class AlmanacManagementTrustKeyWorkflow $key->setIsTrusted(1); $key->save(); + PhabricatorAuthSSHKeyQuery::deleteSSHKeyCache(); + $console->writeOut( "** %s ** %s\n", pht('TRUSTED'), diff --git a/src/applications/almanac/management/AlmanacManagementUntrustKeyWorkflow.php b/src/applications/almanac/management/AlmanacManagementUntrustKeyWorkflow.php index 6ad427eeae..6dc7a21aa3 100644 --- a/src/applications/almanac/management/AlmanacManagementUntrustKeyWorkflow.php +++ b/src/applications/almanac/management/AlmanacManagementUntrustKeyWorkflow.php @@ -43,6 +43,8 @@ final class AlmanacManagementUntrustKeyWorkflow $key->setIsTrusted(0); $key->save(); + PhabricatorAuthSSHKeyQuery::deleteSSHKeyCache(); + $console->writeOut( "** %s ** %s\n", pht('TRUST REVOKED'), From c64aae052ff65e4160c07a185ac472599c78100f Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 9 Feb 2018 17:09:00 -0800 Subject: [PATCH 67/67] Make sure auditors are attached to commits on new pathways Companion change to D19022 for commits. Mentioning and subscribing to commits can load them without audit data. --- .../audit/editor/PhabricatorAuditEditor.php | 24 +++++++++++++++++++ .../storage/PhabricatorRepositoryCommit.php | 4 ++++ 2 files changed, 28 insertions(+) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index d142bd60cd..984e2c1472 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -490,6 +490,8 @@ final class PhabricatorAuditEditor } protected function getMailTo(PhabricatorLiskDAO $object) { + $this->requireAuditors($object); + $phids = array(); if ($object->getAuthorPHID()) { @@ -514,6 +516,8 @@ final class PhabricatorAuditEditor } protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $this->requireAuditors($object); + $phids = array(); foreach ($object->getAudits() as $auditor) { @@ -856,4 +860,24 @@ final class PhabricatorAuditEditor ->executeOne(); } + private function requireAuditors(PhabricatorRepositoryCommit $commit) { + if ($commit->hasAttachedAudits()) { + return; + } + + $with_auditors = id(new DiffusionCommitQuery()) + ->setViewer($this->getActor()) + ->needAuditRequests(true) + ->withPHIDs(array($commit->getPHID())) + ->executeOne(); + if (!$with_auditors) { + throw new Exception( + pht( + 'Failed to reload commit ("%s").', + $commit->getPHID())); + } + + $commit->attachAudits($with_auditors->getAudits()); + } + } diff --git a/src/applications/repository/storage/PhabricatorRepositoryCommit.php b/src/applications/repository/storage/PhabricatorRepositoryCommit.php index 1c5998d583..b3b344312c 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryCommit.php +++ b/src/applications/repository/storage/PhabricatorRepositoryCommit.php @@ -183,6 +183,10 @@ final class PhabricatorRepositoryCommit return $this->assertAttached($this->audits); } + public function hasAttachedAudits() { + return ($this->audits !== self::ATTACHABLE); + } + public function loadAndAttachAuditAuthority( PhabricatorUser $viewer, $actor_phid = null) {