remail: Simplify send_mail()

1) Use EmailMessage

   EmailMessage with the default policy handles header encoding correctly by
   default and uses '\n' line separators, which works for both as_string() and
   smtplib.send_message() correctly. The latter flattens the message with the
   RFC conform '\r\n' line separators unconditionally.

   Aside of that EmailMessage provides defects reporting which is useful to
   ensure that the outgoing mails are correct. A check for that will be added
   later

2) Simplify header handling

   Remove all existing headers from the cloned message first and add those
   which are required either as new headers or copied from the original
   message which was handed in to send_mail()

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

remail/mail.py

@@ -5,11 +5,12 @@
 # Mail message related code
 
 from email.utils import make_msgid, formatdate, parseaddr
-from email.header import Header
 from email import message_from_string, message_from_bytes
+from email.parser import BytesFeedParser
 from email.generator import Generator
 from email.message import Message, EmailMessage
 from email.policy import EmailPolicy
+from email.policy import default as DefaultPolicy
 
 import smtplib
 import mailbox
@@ -55,44 +56,6 @@ class sender_info(object):
             else:
                 msg.add_attachment(data, filename=fname)
 
-def sanitize_headers(msg):
-    '''
-    Sanitize headers by keeping only the ones which are interesting
-    and order them as gmail is picky about that for no good reason.
-    '''
-    headers_order = [
-        'Return-Path',
-        'Date',
-        'From',
-        'To',
-        'Subject',
-        'In-Reply-To',
-        'References',
-        'User-Agent',
-        'MIME-Version',
-        'Charset',
-        'Message-ID',
-        'List-Id',
-        'List-Post',
-        'List-Owner',
-        'Content-Type',
-        'Content-Disposition',
-        'Content-Transfer-Encoding',
-        'Content-Language',
-        'Envelope-to',
-    ]
-
-    # Get all headers and remove them from the message
-    hdrs = msg.items()
-    for k in msg.keys():
-        del msg[k]
-
-    # Add the headers back in proper order
-    for h in headers_order:
-        for k, v in hdrs:
-            if k.lower() == h.lower():
-                msg[k] = v
-
 def send_smtp(msg, to, sender):
     '''
     A dumb localhost only SMTP delivery mechanism. No point in trying
@@ -106,32 +69,56 @@ def send_smtp(msg, to, sender):
     server.send_message(msg, sender, [to])
     server.quit()
 
+def msg_copy_headers(msgout, msg, headers):
+    for key in headers:
+        val = msg.get(key)
+        if val:
+            msgout[key] = val
+
 def send_mail(msg, account, mfrom, sender, listheaders, use_smtp):
     '''
     Send mail to the account. Make sure that the message is correct and all
     required headers and only necessary headers are in the outgoing mail.
     '''
 
+    # Convert to EmailMessage with default policy (utf-8=false) so both
+    # smptlib.send_message() and the stdout dump keep the headers properly
+    # encoded.
+    parser = BytesFeedParser(policy=DefaultPolicy)
+    parser.feed(msg.as_bytes())
+    msgout = parser.close()
+
+    # Remove all mail headers
+    for k in msgout.keys():
+        del msgout[k]
+
+    # Add the required headers in the proper order
+    msgout['Return-path'] = sender
+    msg_copy_headers(msgout, msg, ['Date'])
+    msgout['From'] = mfrom
+    msgout['To'] = account.addr
+
+    msg_copy_headers(msgout, msg, ['Subject', 'In-Reply-To', 'References',
+                                   'User-Agent', 'MIME-Version', 'Charset',
+                                   'Message-ID'])
+
     # Add the list headers
     for key, val in listheaders.items():
-        msg_set_header(msg, key, val)
+        msgout[key] = val
 
-    msg_set_header(msg, 'From', encode_addr(mfrom))
-    msg_set_header(msg, 'To', encode_addr(account.addr))
-    msg_set_header(msg, 'Return-path', sender)
-    msg_set_header(msg, 'Envelope-to', get_raw_email_addr(account.addr))
-
-    sanitize_headers(msg)
+    msg_copy_headers(msgout, msg, ['Content-Type', 'Content-Disposition',
+                                   'Content-Transfer-Encoding',
+                                   'Content-Language'])
+    msgout['Envelope-To'] = get_raw_email_addr(account.addr)
 
     # Set unixfrom with the current date/time
-    msg.set_unixfrom('From remail ' + time.ctime(time.time()))
+    msgout.set_unixfrom('From remail ' + time.ctime(time.time()))
 
     # Send it out
-    mout = msg_from_string(msg.as_string().replace('\r\n', '\n'))
     if use_smtp:
-        send_smtp(mout, account.addr, sender)
+        send_smtp(msgout, account.addr, sender)
     else:
-        print(msg.as_string())
+        print(msgout.as_string())
 
 # Minimal check for a valid email address
 re_mail = re.compile('^\w+([\.-]?\w+)*@\w+([\.-]?\w+)*(\.\w{2,})+$')
@@ -145,24 +132,6 @@ def get_raw_email_addr(addr):
     '''
     return parseaddr(addr)[1]
 
-re_noquote = re.compile('[a-zA-Z0-9_\- ]+')
-
-def encode_addr(fulladdr):
-    try:
-        name, addr = fulladdr.split('<', 1)
-        name = name.strip()
-    except:
-        return fulladdr
-
-    try:
-        name = txt.encode('ascii').decode()
-        if not re_noquote.fullmatch(name):
-            name = '"%s"' %name.replace('"', '')
-    except:
-        name = Header(name).encode()
-
-    return name + ' <' + addr
-
 def msg_from_string(txt):
     policy = EmailPolicy(utf8=True)
     return message_from_string(txt, policy=policy)