Add first unit test for mimemailparser headers

Summary:
Uploaded an example email with a lot of accents called 'test_accents.mbox' and
expected headers in the file 'test_accents.headers.txt'.

Better than nothing.

This change also includes a minor refactor in the library loading.

Ref T15960

Test Plan:
Manually run the new unit test and see green lights:

    arc unit src/applications/metamta/externals/__tests__/PhabricatorExternalMimeMailParserTestCase.php

Double-check that the new class is already recorded:

    arc liberate

Just as extra care, re-apply the same test plan of:

D25839

So, for example, run this, and see no exceptions:

    ./scripts/mail/mail_handler.php < src/applications/metamta/externals/__tests__/data/test_accents.mbox

Reviewers: aklapper, taavi, O1 Blessed Committers

Reviewed By: aklapper, O1 Blessed Committers

Subscribers: tobiaswiese, Matthew, Cigaryno

Maniphest Tasks: T15960

Differential Revision: https://we.phorge.it/D25844

externals/mimemailparser/__init.php

@@ -0,0 +1,11 @@
+<?php
+$root = __DIR__;
+require_once $root.'/Contracts/CharsetManager.php';
+require_once $root.'/Contracts/Middleware.php';
+require_once $root.'/Parser.php';
+require_once $root.'/Charset.php';
+require_once $root.'/Attachment.php';
+require_once $root.'/Exception.php';
+require_once $root.'/Middleware.php';
+require_once $root.'/MiddlewareStack.php';
+require_once $root.'/MimePart.php';

scripts/mail/mail_handler.php

@@ -14,15 +14,7 @@ if ($argc > 1) {
 
 $root = dirname(dirname(dirname(__FILE__)));
 require_once $root.'/scripts/__init_script__.php';
-require_once $root.'/externals/mimemailparser/Contracts/CharsetManager.php';
-require_once $root.'/externals/mimemailparser/Contracts/Middleware.php';
-require_once $root.'/externals/mimemailparser/Parser.php';
-require_once $root.'/externals/mimemailparser/Charset.php';
-require_once $root.'/externals/mimemailparser/Attachment.php';
-require_once $root.'/externals/mimemailparser/Exception.php';
-require_once $root.'/externals/mimemailparser/Middleware.php';
-require_once $root.'/externals/mimemailparser/MiddlewareStack.php';
-require_once $root.'/externals/mimemailparser/MimePart.php';
+require_once $root.'/externals/mimemailparser/__init.php';
 
 $args = new PhutilArgumentParser($argv);
 $args->parseStandardArguments();

src/__phutil_library_map__.php

@@ -3386,6 +3386,7 @@ phutil_register_library_map(array(
     'PhabricatorExternalAccountQuery' => 'applications/auth/query/PhabricatorExternalAccountQuery.php',
     'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php',
     'PhabricatorExternalEditorSettingsPanel' => 'applications/settings/panel/PhabricatorExternalEditorSettingsPanel.php',
+    'PhabricatorExternalMimeMailParserTestCase' => 'applications/metamta/externals/__tests__/PhabricatorExternalMimeMailParserTestCase.php',
     'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php',
     'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php',
     'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php',
@@ -9835,6 +9836,7 @@ phutil_register_library_map(array(
     'PhabricatorExternalAccountQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel',
     'PhabricatorExternalEditorSettingsPanel' => 'PhabricatorEditEngineSettingsPanel',
+    'PhabricatorExternalMimeMailParserTestCase' => 'PhabricatorTestCase',
     'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider',
     'PhabricatorFact' => 'Phobject',

src/applications/metamta/externals/__tests__/PhabricatorExternalMimeMailParserTestCase.php

@@ -0,0 +1,84 @@
+<?php
+
+/**
+ * @phutil-external-symbol class \PhpMimeMailParser\Parser
+ */
+final class PhabricatorExternalMimeMailParserTestCase
+  extends PhabricatorTestCase {
+
+  /**
+   * Be sure to have mimemailparser classes.
+   */
+  private function initMimemailparser() {
+    // Not having this extension is probably a frequent error locally.
+    if (!function_exists('mailparse_msg_create')) {
+      $this->assertSkipped(pht('PHP mailparse extension is not installed'));
+    }
+
+    // Root of Phorge installation
+    $root = dirname(dirname(dirname(dirname(dirname(__DIR__)))));
+
+    // This is safe to be called multiple times.
+    require_once $root.'/externals/mimemailparser/__init.php';
+  }
+
+  public function testMailParse() {
+    $this->initMimemailparser();
+
+    $tests = array(
+      // Test case 0.
+      // Check that no silly "ISO" things are in the headers,
+      // even with esoteric accents.
+      __DIR__.'/data/test_accents',
+    );
+
+    foreach ($tests as $test) {
+      $test_file = $test.'.mbox';
+      $test_file_basename = basename($test_file);
+      $expected_headers_file = $test.'.headers.txt';
+
+      // Unpack the test.
+      $mail_content = Filesystem::readFile($test_file);
+      $expected_headers_raw = Filesystem::readFile($expected_headers_file);
+      $expected_headers = $this->readAssociativeConf($expected_headers_raw);
+
+      // Parse the email.
+      $parser = new \PhpMimeMailParser\Parser();
+      $parser->setText($mail_content);
+
+      // Check email fields headers from the corresponding txt file.
+      $headers = $parser->getHeaders();
+      foreach ($expected_headers as $k => $v) {
+        $this->assertEqual($v, $headers[$k], pht(
+          "Read the header '%s' from the test email %s",
+          $k,
+          $test_file_basename));
+      }
+
+      // If you are creative enough, you can do some tests on the body.
+//      $content = array();
+//      foreach (array('text', 'html') as $part) {
+//        $part_body = $parser->getMessageBody($part);
+//        $content[$part] = $part_body;
+//      }
+    }
+  }
+
+  /**
+   * Get an associative array from "key:value" lines.
+   * @return array
+   */
+  private function readAssociativeConf(string $conf_raw) {
+    $conf = [];
+    $lines = explode("\n", $conf_raw);
+    foreach ($lines as $line) {
+      $line = trim($line);
+      if ($line !== '') {
+        list($k, $v) = explode(':', $line, 2);
+        $conf[$k] = $v;
+      }
+    }
+    return $conf;
+  }
+
+}

src/applications/metamta/externals/__tests__/data/test_accents.headers.txt

@@ -0,0 +1,3 @@
+from:Èxämplæ <myself-ääsdlääl@example.com>
+to:bossò-ääsdlääl@example.com
+subject:È ora di fare review! Aäällright

src/applications/metamta/externals/__tests__/data/test_accents.mbox

@@ -0,0 +1,39 @@
+From myself-ääsdlääl@example.com Tue Dec  3 15:07:14 2024
+Message-ID: <bb2f249a6e74b167d05b8c05977272c4fb78c0b7.camel@reyboz.it>
+Subject: =?ISO-8859-1?Q?=C8?= ora di fare review!
+ =?ISO-8859-1?Q?A=E4=E4llright?=
+From: =?ISO-8859-1?Q?=C8x=E4mpl=E6?= <myself-ääsdlääl@example.com>
+X-Evolution-Identity: df6e2e1cbb42dd3e8553007ad64b060e663ba7d0
+X-Evolution-Fcc: folder://88d77e0e58a4c694494b49c0910f14f16d180d17/Sent
+X-Evolution-Transport: 034c5b726f93fae6b9b88dfdee67e7a519e3ac39
+To: bossò-ääsdlääl@example.com
+X-Evolution-Draft-Folder:
+ folder://88d77e0e58a4c694494b49c0910f14f16d180d17/Drafts
+X-Evolution-Draft-Message: 1444
+X-Evolution-Format: text/html
+X-Evolution-Composer-Mode: text/plain
+Content-Type: multipart/alternative; boundary="=-ljePQpBKYtigz2UvrK3I"
+User-Agent: Evolution 3.46.4-2 
+Date: Tue, 03 Dec 2024 15:07:14 +0100
+MIME-Version: 1.0
+X-Evolution-Source: 88d77e0e58a4c694494b49c0910f14f16d180d17
+
+--=-ljePQpBKYtigz2UvrK3I
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+
+Ottimo! È oraää di fare review. Bella zio.
+
+
+--=-ljePQpBKYtigz2UvrK3I
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+
+<html x-evo-bgcolor="#1b1e20" x-evo-text="#fcfcfc" x-evo-link="#1d99f3" x-evo-vlink="#fcfcfc"><head x-evo-selection="anchorElem=[3] anchorOffset=0 anchorIsElement=1"><style id="x-evo-theme-sheet">html { background-color : #1b1e20; }
+html { color : #fcfcfc; }
+a { color : #1d99f3; }
+a:visited { color : #fcfcfc; }
+</style></head><body data-evo-draft=""><div style="width: 71ch;">Ottimo! È oraää di fare review. Bella zio.</div><div style="width: 71ch;"><span class="-x-evo-signature" id="none"></span></div><div style="width: 71ch;"><span class="-x-evo-signature" id="none"></span></div><div style="width: 71ch;"><br></div><div style="width: 71ch;" class="-x-evo-signature-wrapper"><span class="-x-evo-signature" id="none"></span></div></body></html>
+
+--=-ljePQpBKYtigz2UvrK3I--
+