Begin improving the soundness of received mail

Summary:
We/I broke a couple of things here recently (see D5911) and are doing some work here in general (see D5912, etc.).

Generally, this code is pretty oldschool and not especially well architected for modern application-oriented Phabricator. It hardcodes a lot of stuff which should be applications' responsibilites.

Take the first steps toward making it more solid to reduce the risk here. In particular:

  - Factor out the "self mail" and "duplicate mail" checks and add unit tests.
  - Make Message-ID hash handling automatic.

Test Plan: Ran unit tests.

Reviewers: btrahan, chad

Reviewed By: btrahan

CC: aran

Differential Revision: https://secure.phabricator.com/D5915

resources/sql/patches/20130513.receviedmailstatus.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_metamta.metamta_receivedmail
+  ADD status VARCHAR(32) NOT NULL;

scripts/mail/mail_handler.php

@@ -34,9 +34,6 @@ $received->setBodies(array(
   'text' => $text_body,
   'html' => $parser->getMessageBody('html'),
 ));
-$received->setMessageIDHash(
-  PhabricatorHash::digestForIndex($received->getMessageID())
-);
 
 $attachments = array();
 foreach ($parser->getAttachments() as $attachment) {

src/__phutil_library_map__.php

@@ -653,6 +653,7 @@ phutil_register_library_map(array(
     'ManiphestView' => 'applications/maniphest/view/ManiphestView.php',
     'MetaMTAConstants' => 'applications/metamta/constants/MetaMTAConstants.php',
     'MetaMTANotificationType' => 'applications/metamta/constants/MetaMTANotificationType.php',
+    'MetaMTAReceivedMailStatus' => 'applications/metamta/constants/MetaMTAReceivedMailStatus.php',
     'ObjectHandleLoader' => 'applications/phid/handle/ObjectHandleLoader.php',
     'OwnersPackageReplyHandler' => 'applications/owners/OwnersPackageReplyHandler.php',
     'PHUI' => 'view/phui/PHUI.php',
@@ -1107,6 +1108,8 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTAReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAReceiveController.php',
     'PhabricatorMetaMTAReceivedListController' => 'applications/metamta/controller/PhabricatorMetaMTAReceivedListController.php',
     'PhabricatorMetaMTAReceivedMail' => 'applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php',
+    'PhabricatorMetaMTAReceivedMailProcessingException' => 'applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php',
+    'PhabricatorMetaMTAReceivedMailTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php',
     'PhabricatorMetaMTASendController' => 'applications/metamta/controller/PhabricatorMetaMTASendController.php',
     'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php',
     'PhabricatorMetaMTAViewController' => 'applications/metamta/controller/PhabricatorMetaMTAViewController.php',
@@ -2391,6 +2394,7 @@ phutil_register_library_map(array(
     'ManiphestTransactionType' => 'ManiphestConstants',
     'ManiphestView' => 'AphrontView',
     'MetaMTANotificationType' => 'MetaMTAConstants',
+    'MetaMTAReceivedMailStatus' => 'MetaMTAConstants',
     'OwnersPackageReplyHandler' => 'PhabricatorMailReplyHandler',
     'PHUIBoxExample' => 'PhabricatorUIExample',
     'PHUIBoxView' => 'AphrontTagView',
@@ -2832,6 +2836,8 @@ phutil_register_library_map(array(
     'PhabricatorMetaMTAReceiveController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAReceivedListController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAReceivedMail' => 'PhabricatorMetaMTADAO',
+    'PhabricatorMetaMTAReceivedMailProcessingException' => 'Exception',
+    'PhabricatorMetaMTAReceivedMailTestCase' => 'PhabricatorTestCase',
     'PhabricatorMetaMTASendController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController',
     'PhabricatorMetaMTAViewController' => 'PhabricatorMetaMTAController',

src/applications/metamta/constants/MetaMTAReceivedMailStatus.php

@@ -0,0 +1,9 @@
+<?php
+
+final class MetaMTAReceivedMailStatus
+  extends MetaMTAConstants {
+
+  const STATUS_DUPLICATE        = 'err:duplicate';
+  const STATUS_FROM_PHABRICATOR = 'err:self';
+
+}

src/applications/metamta/controller/PhabricatorMetaMTAReceiveController.php

@@ -10,7 +10,9 @@ final class PhabricatorMetaMTAReceiveController
 
     if ($request->isFormPost()) {
       $received = new PhabricatorMetaMTAReceivedMail();
-      $header_content = array();
+      $header_content = array(
+        'Message-ID' => Filesystem::readRandomBytes(12),
+      );
       $from = $request->getStr('sender');
       $to = $request->getStr('receiver');
       $uri = '/mail/received/';
@@ -42,11 +44,6 @@ final class PhabricatorMetaMTAReceiveController
           'text' => $request->getStr('body'),
         ));
 
-      // Make up some unique value, since this column isn't nullable.
-      $received->setMessageIDHash(
-        PhabricatorHash::digestForIndex(
-          Filesystem::readRandomBytes(12)));
-
       $received->save();
 
       $received->processReceivedMail();

src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php

@@ -39,8 +39,6 @@ final class PhabricatorMetaMTASendGridReceiveController
       'text' => $request->getStr('text'),
       'html' => $request->getStr('from'),
     ));
-    $received->setMessageIDHash(
-      PhabricatorHash::digestForIndex($received->getMessageID()));
 
     $file_phids = array();
     foreach ($_FILES as $file_raw) {

src/applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php

@@ -0,0 +1,20 @@
+<?php
+
+final class PhabricatorMetaMTAReceivedMailProcessingException
+  extends Exception {
+
+  private $statusCode;
+
+  public function getStatusCode() {
+    return $this->statusCode;
+  }
+
+  public function __construct($status_code /* ... */) {
+    $args = func_get_args();
+    $this->statusCode = $args[0];
+
+    $args = array_slice($args, 1);
+    call_user_func_array(array('parent', '__construct'), $args);
+  }
+
+}

src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php

@@ -5,11 +5,12 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
   protected $headers = array();
   protected $bodies = array();
   protected $attachments = array();
+  protected $status = '';
 
   protected $relatedPHID;
   protected $authorPHID;
   protected $message;
-  protected $messageIDHash;
+  protected $messageIDHash = '';
 
   public function getConfiguration() {
     return array(
@@ -25,18 +26,31 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
     // Normalize headers to lowercase.
     $normalized = array();
     foreach ($headers as $name => $value) {
-      $normalized[strtolower($name)] = $value;
+      $name = $this->normalizeMailHeaderName($name);
+      if ($name == 'message-id') {
+        $this->setMessageIDHash(PhabricatorHash::digestForIndex($value));
+      }
+      $normalized[$name] = $value;
     }
     $this->headers = $normalized;
     return $this;
   }
 
+  public function getHeader($key, $default = null) {
+    $key = $this->normalizeMailHeaderName($key);
+    return idx($this->headers, $key, $default);
+  }
+
+  private function normalizeMailHeaderName($name) {
+    return strtolower($name);
+  }
+
   public function getMessageID() {
-    return idx($this->headers, 'message-id');
+    return $this->getHeader('Message-ID');
   }
 
   public function getSubject() {
-    return idx($this->headers, 'subject');
+    return $this->getHeader('Subject');
   }
 
   public function getCCAddresses() {
@@ -156,35 +170,15 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
 
   public function processReceivedMail() {
 
-    // If Phabricator sent the mail, always drop it immediately. This prevents
+    try {
-    // loops where, e.g., the public bug address is also a user email address
+      $this->dropMailFromPhabricator();
-    // and creating a bug sends them an email, which loops.
+      $this->dropMailAlreadyReceived();
-    $is_phabricator_mail = idx(
+    } catch (PhabricatorMetaMTAReceivedMailProcessingException $ex) {
-      $this->headers,
+      $this
-      'x-phabricator-sent-this-message');
+        ->setStatus($ex->getStatusCode())
-    if ($is_phabricator_mail) {
+        ->setMessage($ex->getMessage())
-      $message = "Ignoring email with 'X-Phabricator-Sent-This-Message' ".
+        ->save();
-                 "header to avoid loops.";
+      return $this;
-      return $this->setMessage($message)->save();
-    }
-
-    $message_id_hash = $this->getMessageIDHash();
-    if ($message_id_hash) {
-      $messages = $this->loadAllWhere(
-        'messageIDHash = %s',
-        $message_id_hash);
-      $messages_count = count($messages);
-      if ($messages_count > 1) {
-        $first_message = reset($messages);
-        if ($first_message->getID() != $this->getID()) {
-          $message = sprintf(
-            'Ignoring email with message id hash "%s" that has been seen %d '.
-            'times, including this message.',
-            $message_id_hash,
-            $messages_count);
-          return $this->setMessage($message)->save();
-        }
-      }
     }
 
     list($to,
@@ -460,4 +454,61 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO {
     return $user;
   }
 
+  /**
+   * If Phabricator sent the mail, always drop it immediately. This prevents
+   * loops where, e.g., the public bug address is also a user email address
+   * and creating a bug sends them an email, which loops.
+   */
+  private function dropMailFromPhabricator() {
+    if (!$this->getHeader('x-phabricator-sent-this-message')) {
+      return;
+    }
+
+    throw new PhabricatorMetaMTAReceivedMailProcessingException(
+      MetaMTAReceivedMailStatus::STATUS_FROM_PHABRICATOR,
+      "Ignoring email with 'X-Phabricator-Sent-This-Message' header to avoid ".
+      "loops.");
+  }
+
+  /**
+   * If this mail has the same message ID as some other mail, and isn't the
+   * first mail we we received with that message ID, we drop it as a duplicate.
+   */
+  private function dropMailAlreadyReceived() {
+    $message_id_hash = $this->getMessageIDHash();
+    if (!$message_id_hash) {
+      // No message ID hash, so we can't detect duplicates. This should only
+      // happen with very old messages.
+      return;
+    }
+
+    $messages = $this->loadAllWhere(
+      'messageIDHash = %s ORDER BY id ASC LIMIT 2',
+      $message_id_hash);
+    $messages_count = count($messages);
+    if ($messages_count <= 1) {
+      // If we only have one copy of this message, we're good to process it.
+      return;
+    }
+
+    $first_message = reset($messages);
+    if ($first_message->getID() == $this->getID()) {
+      // If this is the first copy of the message, it is okay to process it.
+      // We may not have been able to to process it immediately when we received
+      // it, and could may have received several copies without processing any
+      // yet.
+      return;
+    }
+
+    $message = sprintf(
+      'Ignoring email with message id hash "%s" that has been seen %d '.
+      'times, including this message.',
+      $message_id_hash,
+      $messages_count);
+
+    throw new PhabricatorMetaMTAReceivedMailProcessingException(
+      MetaMTAReceivedMailStatus::STATUS_DUPLICATE,
+      $message);
+  }
+
 }

src/applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php

@@ -0,0 +1,52 @@
+<?php
+
+final class PhabricatorMetaMTAReceivedMailTestCase extends PhabricatorTestCase {
+
+  protected function getPhabricatorTestCaseConfiguration() {
+    return array(
+      self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
+    );
+  }
+
+  public function testDropSelfMail() {
+    $mail = new PhabricatorMetaMTAReceivedMail();
+    $mail->setHeaders(
+      array(
+        'X-Phabricator-Sent-This-Message' => 'yes',
+      ));
+    $mail->save();
+
+    $mail->processReceivedMail();
+
+    $this->assertEqual(
+      MetaMTAReceivedMailStatus::STATUS_FROM_PHABRICATOR,
+      $mail->getStatus());
+  }
+
+
+  public function testDropDuplicateMail() {
+    $mail_a = new PhabricatorMetaMTAReceivedMail();
+    $mail_a->setHeaders(
+      array(
+        'Message-ID' => 'test@example.com',
+      ));
+    $mail_a->save();
+
+    $mail_b = new PhabricatorMetaMTAReceivedMail();
+    $mail_b->setHeaders(
+      array(
+        'Message-ID' => 'test@example.com',
+      ));
+    $mail_b->save();
+
+    $mail_a->processReceivedMail();
+    $mail_b->processReceivedMail();
+
+    $this->assertEqual(
+      MetaMTAReceivedMailStatus::STATUS_DUPLICATE,
+      $mail_b->getStatus());
+  }
+
+
+
+}

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1294,6 +1294,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'php',
         'name' => $this->getPatchPath('20130508.releephtransactionsmig.php'),
       ),
+      '20130513.receviedmailstatus.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20130513.receviedmailstatus.sql'),
+      ),
     );
   }
 }