Enable SendGrid Parse API as an inbound email handler

Summary:
Sendmail is seriously difficult to configure; SendGrid is extremely easy. It's
also pretty expensive ($80/mo) but there are a bunch of startups that already
have plans so it's effectively free for them.

Test Plan:
Configured SendGrid and sent reply email through it.

Reviewed By: aran
Reviewers: tuomaspelkonen, jungejason, aran
CC: aran, epriestley
Differential Revision: 376

src/__phutil_library_map__.php

@@ -359,6 +359,7 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTAReceivedListController' => 'applications/metamta/controller/receivedlist',
     'PhabricatorMetaMTAReceivedMail' => 'applications/metamta/storage/receivedmail',
     'PhabricatorMetaMTASendController' => 'applications/metamta/controller/send',
+    'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/sendgridreceive',
     'PhabricatorMetaMTAViewController' => 'applications/metamta/controller/view',
     'PhabricatorOAuthDefaultRegistrationController' => 'applications/auth/controller/oauthregistration/default',
     'PhabricatorOAuthDiagnosticsController' => 'applications/auth/controller/oauthdiagnostics',
@@ -798,6 +799,7 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTAReceivedListController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAReceivedMail' => 'PhabricatorMetaMTADAO',
     'PhabricatorMetaMTASendController' => 'PhabricatorMetaMTAController',
+    'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAViewController' => 'PhabricatorMetaMTAController',
     'PhabricatorOAuthDefaultRegistrationController' => 'PhabricatorOAuthRegistrationController',
     'PhabricatorOAuthDiagnosticsController' => 'PhabricatorAuthController',

src/aphront/default/configuration/AphrontDefaultApplicationConfiguration.php

@@ -126,6 +126,7 @@ class AphrontDefaultApplicationConfiguration
           => 'PhabricatorMetaMTAMailingListEditController',
         'receive/$' => 'PhabricatorMetaMTAReceiveController',
         'received/$' => 'PhabricatorMetaMTAReceivedListController',
+        'sendgrid/$' => 'PhabricatorMetaMTASendGridReceiveController',
       ),
 
       '/login/' => array(

src/applications/metamta/controller/sendgridreceive/PhabricatorMetaMTASendGridReceiveController.php

@@ -0,0 +1,70 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class PhabricatorMetaMTASendGridReceiveController
+  extends PhabricatorMetaMTAController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  public function processRequest() {
+
+    $request = $this->getRequest();
+
+    $raw_headers = $request->getStr('headers');
+    $raw_headers = explode("\n", rtrim($raw_headers));
+    $raw_dict = array();
+    foreach (array_filter($raw_headers) as $header) {
+      list($name, $value) = explode(':', $header, 2);
+      $raw_dict[$name] = ltrim($value);
+    }
+
+    $headers = array(
+      'to'      => $request->getStr('to'),
+      'from'    => $request->getStr('from'),
+      'subject' => $request->getStr('subject'),
+    ) + $raw_dict;
+
+    $received = new PhabricatorMetaMTAReceivedMail();
+    $received->setHeaders($headers);
+    $received->setBodies(array(
+      'text' => $request->getStr('text'),
+      'html' => $request->getStr('from'),
+    ));
+
+    $file_phids = array();
+    foreach ($_FILES as $file_raw) {
+      try {
+        $file = PhabricatorFile::newFromPHPUpload($file_raw);
+        $file_phids[] = $file->getPHID();
+      } catch (Exception $ex) {
+        phlog($ex);
+      }
+    }
+    $received->setAttachments($file_phids);
+    $received->save();
+
+    $received->processReceivedMail();
+
+    $response = new AphrontWebpageResponse();
+    $response->setContent("Got it! Thanks, SendGrid!\n");
+    return $response;
+  }
+
+}

src/applications/metamta/controller/sendgridreceive/__init__.php

@@ -0,0 +1,17 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'aphront/response/webpage');
+phutil_require_module('phabricator', 'applications/files/storage/file');
+phutil_require_module('phabricator', 'applications/metamta/controller/base');
+phutil_require_module('phabricator', 'applications/metamta/storage/receivedmail');
+
+phutil_require_module('phutil', 'error');
+
+
+phutil_require_source('PhabricatorMetaMTASendGridReceiveController.php');

src/applications/metamta/storage/receivedmail/PhabricatorMetaMTAReceivedMail.php

@@ -36,6 +36,16 @@ class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
     ) + parent::getConfiguration();
   }
 
+  public function setHeaders(array $headers) {
+    // Normalize headers to lowercase.
+    $normalized = array();
+    foreach ($headers as $name => $value) {
+      $normalized[strtolower($name)] = $value;
+    }
+    $this->headers = $normalized;
+    return $this;
+  }
+
   public function processReceivedMail() {
     $to = idx($this->headers, 'to');
 

src/docs/configuring_inbound_email.diviner

@@ -6,13 +6,114 @@ may update Differential and Maniphest by replying to messages.
 
 = Preamble =
 
-This is extremely difficult to configure correctly. This is doubly true if
+This can be extremely difficult to configure correctly. This is doubly true if
 you use sendmail.
 
+There are basically a few approaches available:
+
+  - Use SendGrid (<http://sendgrid.com/>), which is very easy but is not free.
+  - Run your own MTA, which can be quite harrowing to configure but is free.
+  - Tell the Phabricator devteam about another service you'd like support for,
+    this stuff is seriously terrible to configure on your own.
+
+= Configuring Phabricator =
+
+By default, Phabricator uses a "noreply@example.com" email address as the 'From'
+(configurable with ##metamta.default-address##) and sets 'Reply-To' to the
+user generating the email (e.g., by making a comment), if the mail was generated
+by a user action. This means that users can reply (or reply-all) to email to
+discuss changes, but the conversation won't be recorded in Phabricator and users
+will not be able to take actions like claiming tasks or requesting changes to
+revisions.
+
+To change this behavior so that users can interact with objects in Phabricator
+over email, set these configuration keys:
+
+  - ##metamta.differential.reply-handler-domain##: enables email replies for
+    Differential.
+  - ##metamta.maniphest.reply-handler-domain##: enables email replies for
+    Maniphest.
+
+Set these keys to some domain which you configure according to the instructions
+below, e.g. "##phabricator.example.com##". You can set these both to the same
+domain, and will generally want to. Once you set these keys, emails will use a
+'Reply-To' like "##T123+273+af310f9220ad@example.com##", which -- when
+configured correctly, according to the instructions below -- will parse incoming
+email and allow users to interact with Maniphest tasks and Differential
+revisions over email.
+
+= Security =
+
+The email reply channel is "somewhat" authenticated. Each reply-to address is
+unique to the recipient and includes a hash of user information and a unique
+object ID, so it can only be used to update that object and only be used to act
+on behalf of the recipient.
+
+However, if an address is leaked (which is fairly easy -- for instance,
+forwarding an email will leak a live reply address, or a user might take a
+screenshot), //anyone// who can send mail to your reply-to domain may interact
+with the object the email relates to as the user who leaked the mail. Because
+the authentication around email has this weakness, some actions (like accepting
+revisions) are not permitted over email.
+
+This implementation is an attempt to balance utility and security, but makes
+some sacrifices on both sides to achieve it because of the difficulty of
+authenticating senders in the general case (e.g., where you are an open source
+project and need to interact with users whose email accounts you have no control
+over).
+
+If you leak a bunch of reply-to addresses by accident, you can change
+##phabricator.mail-key## in your configuration to invalidate all the old hashes.
+
+NOTE: Phabricator does not currently attempt to verify "From" addresses because
+this is technically complex, seems unreasonably difficult in the general case,
+and no installs have had a need for it yet. If you have a specific case where a
+reasonable mechanism exists to provide sender verification (e.g., DKIM
+signatures are sufficient to authenticate the sender under your configuration,
+or you are willing to require all users to sign their email), file a feature
+request.
+
+= Testing =
+
+You can view a log of received mail by going to MetaMTA -> Received in the
+Phabricator web interface. This can help you determine if mail is being
+delivered to Phabricator or not.
+
+You can also use the "Test Receiver" button, but note that this just simulates
+receiving mail and doesn't send any information over the network. It is
+primarily aimed at developing email handlers: it will still work properly
+if your inbound email configuration is incorrect or even disabled.
+
+= SendGrid =
+
+To use SendGrid, you need a SendGrid account with access to the "Parse API" for
+inbound email. Provided you have such an account, configure it like this:
+
+  - Configure an MX record according to SendGrid's instructions, i.e. add
+    ##phabricator.example.com MX 10 mx.sendgrid.net.## or similar.
+  - Go to the "Parse Incoming Emails" page on SendGrid
+    (<http://sendgrid.com/developer/reply>) and add the domain as the
+    "Hostname".
+  - Add the URL ##https://phabricator.example.com/mail/sendgrid/## as the "Url",
+    using your domain (and HTTP instead of HTTPS if you are not configured with
+    SSL).
+  - If you get an error that the hostname "can't be located or verified", it
+    means your MX record is either incorrectly configured or hasn't propagated
+    yet.
+  - Set ##metamta.maniphest.reply-handler-domain## and/or
+    ##metamta.differential.reply-handler-domain## to
+    "##phabricator.example.com##" (whatever you configured the MX record for),
+    depending on whether you want to support email replies for Maniphest,
+    Differential, or both.
+
+That's it! If everything is working properly you should be able to send email
+to ##anything@phabricator.example.com## and it should appear in the "Received"
+tab of MetaMTA within a few seconds.
+
 = Installing Mailparse =
 
-You need to install the PECL mailparse extension. In theory, you can do that
+If you're going to run your own MTA, you need to install the PECL mailparse
-with:
+extension. In theory, you can do that with:
 
   $ sudo pecl install mailparse
 
@@ -32,10 +133,13 @@ If you get a linker error like this:
 mailparse.so. This is not the default if you have individual files in
 ##php.d/##.
 
-= Configuring Sendmail =
+= MTA: Configuring Sendmail =
+
+Before you can configure Sendmail, you need to install Mailparse. See the
+section "Installing Mailparse" above.
 
 Sendmail is very difficult to configure. First, you need to configure it for
-your domain so that mail can be delievered correctly. In broad strokes, this
+your domain so that mail can be delivered correctly. In broad strokes, this
 probably means something like this:
 
   - add an MX record;
@@ -61,5 +165,5 @@ Finally, edit ##/etc/mail/virtusertable## and add an entry like this:
 
 That will forward all mail to @yourdomain.com to the Phabricator processing
 script. Run ##sudo /etc/mail/make## or similar and then restart sendmail with
-##sudo /etc/init.d/sendmail restart##
+##sudo /etc/init.d/sendmail restart##.
 

src/infrastructure/markup/remarkup/markuprule/youtube/__init__.php

@@ -8,6 +8,7 @@
 
 phutil_require_module('phutil', 'markup/engine/remarkup/markuprule/base');
 phutil_require_module('phutil', 'parser/uri');
+phutil_require_module('phutil', 'utils');
 
 
 phutil_require_source('PhabricatorRemarkupRuleYoutube.php');