commit 60f6698e525747341d4842dac20b7e84bdfb391d Author: Thomas Gleixner Date: Sun Sep 22 22:38:59 2019 +0200 remail: Initial import Signed-off-by: Thomas Gleixner diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f6ab00 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Normal rules +*.bz2 +*.diff +*.gz +*.patch +*.orig +*.rej +*.tar +*.xz + +# Editor artifacts +*~ +\#*# + +# Quilt's files +patches +series +.pc/ + +# Python artifacts +*.pyc + +# Documentation artifacts +Documentation/output/ diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..45e34ee --- /dev/null +++ b/COPYING @@ -0,0 +1,14 @@ +remail is provided under: + + SPDX-License-Identifier: GPL-2.0-only + +Being under the terms of the GNU General Public License version 2 only, +according with: + + LICENSES/GPL-2.0 + +In addition, other licenses may also apply. Please see: + + Documentation/license-rules.rst + +for more details. diff --git a/Documentation/Makefile b/Documentation/Makefile new file mode 100644 index 0000000..e57a9d3 --- /dev/null +++ b/Documentation/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = remail +SOURCEDIR = . +BUILDDIR = output + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/Documentation/conf.py b/Documentation/conf.py new file mode 100644 index 0000000..fe8e401 --- /dev/null +++ b/Documentation/conf.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# +# remail documentation build configuration file, created by +# sphinx-quickstart on Tue May 15 16:12:59 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import glob + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['sphinx_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'remail' +copyright = u'2018, Thomas Gleixner ' +author = u'Thomas Gleixner ' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'' +# The full version, including alpha/beta/rc tags. +release = u'' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['sphinx_static'] + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'remaildoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'remail.tex', u'remail Documentation', + u'Thomas Gleixner \\textless{}tglx@linutronix.de\\textgreater{}', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + # (master_doc, 'remail', u'remail Documentation', + # [author], 1), +] + +for f in glob.glob('man1/*.rst'): + src = f.replace('.rst', '') + dst = src.split('/')[1] + man_pages.append((src, dst, u'%s Documentation' % dst, [author], 1)) + +for f in glob.glob('man5/*.rst'): + src = f.replace('.rst', '') + dst = src.split('/')[1] + man_pages.append((src, dst, u'%s Documentation' % dst, [author], 5)) + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'remail', u'remail Documentation', + author, 'remail', 'One line description of project.', + 'Miscellaneous'), +] diff --git a/Documentation/configuration.rst b/Documentation/configuration.rst new file mode 100644 index 0000000..24d4f8d --- /dev/null +++ b/Documentation/configuration.rst @@ -0,0 +1,176 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_configuration: + +remail configuration guide +========================== + +Introduction +------------ + +The configuration files are simple yaml files which only use the very basic +yaml components. The yaml file does not have a schema yet, but the +configuration parser is doing extensive verification. + +See :ref:`remail_daemon_config_man` for a detailed explanation of the +configuration files and the configuration items. + +The examples directory in Documentation contains a full dummy configuration +for two lists including fully documented yaml files. + +Work directory +-------------- + +The work directory structure is fixed and is documented in the man page in +the section :ref:`config_dir_struct`. + +A comfortable way to manage the configuration is git. The base +configuration directory and the actual list configuration directories are +in separate git trees so that they can be maintained by different people +with different permissions. + + +Key considerations +------------------ + +Private keys +^^^^^^^^^^^^ + +In order to encrypt incoming mail and to sign outgoing mail the list daemon +needs access to the private GPG and S/MIME keys of each list. + +The keys must be stored without password. + +This is not a security issue because if the keys would have passwords then +the password needs to be part of the configuration file. + +If the list configuration or the list server is compromised all bets are +off. Having a password on the keys would not make any difference. + +Public keys +^^^^^^^^^^^ + +The public GPG keys of the subscribers in the mailing list specific keyring +are always trusted by remail. The list administrator is responsible for +verifying the authenticity of the subscribers key, so extensive trust +management does not make sense and just complicates the handling. + +remail does not try to update the keyring at any time. This is in the +responsibility of the list administrator. Automatic updates are not a +really good idea in the light of the recent attacks on the PGP +infrastructure. + +The public S/MIME certificates are verified against the CA certificates +which are provided and managed by the remail base administrator as part of +the base configuration. + +Key storage format +------------------ + +GPG +^^^ + +GPG keys are stored using the storage format of gnugp. Create and manage +the private keys and the subscriber keyrings with the GPG tool of your +choice. + +S/MIME +^^^^^^ + +S/MIME .key and .crt files are stored in pem format and have to follow +the following naming convention. + +Private keys (list keys): + email@address.domain.key + +Public certificates (list and subscribers): + email@address.domain.crt + +Using git for configuration management +-------------------------------------- + +As seen above the base configuration directory contains the private keys +for the lists. So the administrator has to ensure that the git repository +which is used to push the configuration to is properly protected. Also the +machine used for updating the base configuration repository should not be +accessible by unpriviledged users. + +The incomplete list of things to avoid under all circumstances: + + - Push the base configuration to a public hosted repository server like + github, gitlab etc. + + - Keep the base configuration on an easy accessible machine if the data is + not sufficiently protected by encryption + + - Keep the base configuration unencrypted on a backup medium. + +The list configuration does not contain strictly secret information, but +the pure existance of an incident list and the list of involved people +might give a hint in which area the handled issue might be. So in general +it is recommended to hide a list configuration repository from public and +unpriviledged access as well, but in case of an leak the potential damage +is way smaller than the one of leaking the base configuration which +contains the private keys. + + +Base configuration +------------------ + +To configure the list daemon the following configuration files need to be +created or modified: + + - remail.yaml + +remail.yaml: + + The main configuration file for the remail daemon. See + :ref:`remail_daemon_config_man` for a detailed information of the file. + +List configuration +------------------ + +For each enabled mailing list the following configuration files need to be +created or modified: + + - list.yaml + - template_welcome + - template_admin + +list.yaml: + + The list specific configuration file for the mailing list. It contains + the subscriber list. See :ref:`remail_daemon_config_man` for a detailed + information of the file. + +template_admin: + + The mail template for mail to the list admin(s). + +template_welcome: + + The mail template for mail to welcome new subscribers. + + +Public mail +----------- + +Request or configure a user account for receiving list mail. This mail +account acts as a catch all for the list, the list-owner and list-bounce +addresses. If more than one mailing list is served by the list daemon then +the user mail account can receive all mails for the lists. remail finds the +appropriate mailing list for the various addresses. + +remail does not retrieve mail from a public mail server as this is outside +the scope of remail and depends on the particular setup. remail expects the +incoming mails to be delivered into a maildir. Tools like getmail, +fetchmail can handle that as well as SMTP servers. + + +Local mail transport +-------------------- + +remail delivers the outgoing mail to the local SMTP server. The +configuration for the SMTP server to relay the mails to the public facing +machines is outside the scope of this documentation and depends on the SMTP +server variant you are using. diff --git a/Documentation/examples/conf/.certs/admin1@admin.domain.crt b/Documentation/examples/conf/.certs/admin1@admin.domain.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.certs/admin1@admin.domain.crt @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/.certs/admin1@admin.domain.key b/Documentation/examples/conf/.certs/admin1@admin.domain.key new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.certs/admin1@admin.domain.key @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/.certs/cacert.pem b/Documentation/examples/conf/.certs/cacert.pem new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.certs/cacert.pem @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST1_40_CHARACTER_PGP_FINGERPRINT.key b/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST1_40_CHARACTER_PGP_FINGERPRINT.key new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST1_40_CHARACTER_PGP_FINGERPRINT.key @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST2_40_CHARACTER_PGP_FINGERPRINT.key b/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST2_40_CHARACTER_PGP_FINGERPRINT.key new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.gnupg/private-keys-v1.d/LIST2_40_CHARACTER_PGP_FINGERPRINT.key @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/.gnupg/pubring.kbx b/Documentation/examples/conf/.gnupg/pubring.kbx new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/.gnupg/pubring.kbx @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list1/.certs/admin1@admin.domain.crt b/Documentation/examples/conf/lists/list1/.certs/admin1@admin.domain.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/.certs/admin1@admin.domain.crt @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list1/.certs/dev1@some.domain.crt b/Documentation/examples/conf/lists/list1/.certs/dev1@some.domain.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/.certs/dev1@some.domain.crt @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list1/.gnupg/pubring.kbx b/Documentation/examples/conf/lists/list1/.gnupg/pubring.kbx new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/.gnupg/pubring.kbx @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list1/list.yaml b/Documentation/examples/conf/lists/list1/list.yaml new file mode 100644 index 0000000..f9c0907 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/list.yaml @@ -0,0 +1,41 @@ +# +# Recrypting mailing list configuration file for list1 +# + +# The list subscribers +subscribers: + + # Format: + # email address: + # name: Real name of the subscriber + # enabled: Subscriber is enabled (if omitted defaults to False) + # use_smime: True/False (Use S/MIME for encryption. If omitted defaults to False) + # fingerprint: GPG fingerprint (Not required when use_smime == True) + # gpg_plain: Plain text inline GPG encryption (If omitted defaults to False) + # aliases: List of alias addresses which are valid for posting (moderated list) + # + # Note: Use spaces for spacing not tabs + + dev1@some.domain: + name: Developer One + enabled: True + use_smime: True + + dev2@some.domain: + name: Developer Two + enabled: False + + dev3@other.domain: + name: Developer Three + enabled: True + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + aliases: + - dev3@dev3.domain + - hckr@dev3.domain + + dev4@other.domain: + name: Developer Four + enabled: True + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + gpg_plain: True + diff --git a/Documentation/examples/conf/lists/list1/template_admin b/Documentation/examples/conf/lists/list1/template_admin new file mode 100644 index 0000000..2b97050 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/template_admin @@ -0,0 +1,6 @@ +Subject: [LIST1] Remail failure report +User-Agent: Remail +Content-Type: text/plain +Content-Disposition: inline + +The following problems have been encountered with the list list1: diff --git a/Documentation/examples/conf/lists/list1/template_welcome b/Documentation/examples/conf/lists/list1/template_welcome new file mode 100644 index 0000000..42dc647 --- /dev/null +++ b/Documentation/examples/conf/lists/list1/template_welcome @@ -0,0 +1,26 @@ +Subject: Welcome to the list1 mailing-list +User-Agent: Remail +Content-Type: text/plain +Content-Disposition: inline + +Dear $NAME! + +This is an automated email from your friendly mailing-list update bot. This +email is sent to you because you got (re-)subscribed to the list1 +mailing-list. + +This email to you is signed either with the list's PGP key or the list's +S/MIME certificate depending on your encryption preference, so your +email-client should have already retrieved the PGP key or the S/MIME +certificate for you. + +The list's email address is: list1@list.domain + +To send email to the list, encrypt it either with the list's PGP key or the +list's S/MIME certificate. Note, that the subject line of the email is not +encrypted, so you should be careful about that. + +If you have questions, please contact the list administrator by email. The +administrators email address is: list1-owner@list.domain + + diff --git a/Documentation/examples/conf/lists/list2/.certs/dev1@some.domain.crt b/Documentation/examples/conf/lists/list2/.certs/dev1@some.domain.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lists/list2/.certs/dev1@some.domain.crt @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list2/.gnupg/pubring.kbx b/Documentation/examples/conf/lists/list2/.gnupg/pubring.kbx new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lists/list2/.gnupg/pubring.kbx @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/lists/list2/list.yaml b/Documentation/examples/conf/lists/list2/list.yaml new file mode 100644 index 0000000..ac15ae9 --- /dev/null +++ b/Documentation/examples/conf/lists/list2/list.yaml @@ -0,0 +1,41 @@ +# +# Recrypting mailing list configuration file for list2 +# + +# The list subscribers +subscribers: + + # Format: + # email address: + # name: Real name of the subscriber + # enabled: Subscriber is enabled (if omitted defaults to False) + # use_smime: True/False (Use S/MIME for encryption. If omitted defaults to False) + # fingerprint: GPG fingerprint (Not required when use_smime == True) + # gpg_plain: Plain text inline GPG encryption (If omitted defaults to False) + # aliases: List of alias addresses which are valid for posting (moderated list) + # + # Note: Use spaces for spacing not tabs + + dev1@some.domain: + name: Developer One + enabled: True + use_smime: True + + dev2@some.domain: + name: Developer Two + enabled: False + + dev3@other.domain: + name: Developer Three + enabled: True + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + aliases: + - dev3@dev3.domain + - hckr@dev3.domain + + dev4@other.domain: + name: Developer Four + enabled: True + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + gpg_plain: True + diff --git a/Documentation/examples/conf/lists/list2/template_admin b/Documentation/examples/conf/lists/list2/template_admin new file mode 100644 index 0000000..760a0a9 --- /dev/null +++ b/Documentation/examples/conf/lists/list2/template_admin @@ -0,0 +1,6 @@ +Subject: [LIST2] Remail failure report +User-Agent: Remail +Content-Type: text/plain +Content-Disposition: inline + +The following problems have been encountered with the list list2: diff --git a/Documentation/examples/conf/lists/list2/template_welcome b/Documentation/examples/conf/lists/list2/template_welcome new file mode 100644 index 0000000..d74a8d7 --- /dev/null +++ b/Documentation/examples/conf/lists/list2/template_welcome @@ -0,0 +1,26 @@ +Subject: Welcome to the list2 mailing-list +User-Agent: Remail +Content-Type: text/plain +Content-Disposition: inline + +Dear $NAME! + +This is an automated email from your friendly mailing-list update bot. This +email is sent to you because you got (re-)subscribed to the list2 +mailing-list. + +This email to you is signed either with the list's PGP key or the list's +S/MIME certificate depending on your encryption preference, so your +email-client should have already retrieved the PGP key or the S/MIME +certificate for you. + +The list's email address is: list2@list.domain + +To send email to the list, encrypt it either with the list's PGP key or the +list's S/MIME certificate. Note, that the subject line of the email is not +encrypted, so you should be careful about that. + +If you have questions, please contact the list administrator by email. The +administrators email address is: list2-owner@list.domain + + diff --git a/Documentation/examples/conf/lock b/Documentation/examples/conf/lock new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Documentation/examples/conf/lock @@ -0,0 +1 @@ + diff --git a/Documentation/examples/conf/remail.yaml b/Documentation/examples/conf/remail.yaml new file mode 100644 index 0000000..4f9f094 --- /dev/null +++ b/Documentation/examples/conf/remail.yaml @@ -0,0 +1,116 @@ +# +# Remail base configuration file for the remail list server +# +# Note: YAML requires spaces. Do not use TABs +# + +# Enabled. If False all other items are ignored. Set to true to get it starting +enabled: True + +# Set to True to enable SMTP delivery. Set only to False for testing or +# troubleshooting +use_smtp: True + +# S/MIME +smime: + # Verify CA certs. Only disable for troubleshooting + verify: True + +# GPG +gpg: + # Trust your keys always. That avoids manual fiddling with trust-db + always_trust: True + # List mail should always be signed with the lists key + sign: True + +# The mailing lists +lists: + + # The list name + list1: + + # Set to True if the list is enabled + enabled: True + + # Set to True if the list should be moderated + # If set, only subscribers can post to the list + # Non subscriber mails are forwarded to the list admins + moderated: True + + archive: + use_maildir: True + # Set to true if the incoming mails should be archived + # locally in a mailbox/dir unmodified before decryption + archive_incoming: True + # Set to true if the incoming mails should be archived + # locally in a mailbox/dir after decrypting them + archive_plain: True + + # The list account itself + listaccount: + list1@list.domain: + name: list1 + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + + # The list admins + admins: + # Format: + # email address: + # name: Real name of the subscriber + # enabled: Subscriber is enabled (if omitted defaults to False) + # use_smime: True/False (Use S/MIME for encryption. If omitted defaults to False) + # fingerprint: GPG fingerprint (Not required when use_smime == True) + # gpg_plain: Plain text inline GPG encryption (If omitted defaults to False) + admin1@admin.domain: + name: Admin one + use_smime: True + enabled: True + + admin2@admin2.domain: + name: Admin2 + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + gpg_plain: True + enabled: True + + # The list name + list2: + # Set to True if the list is enabled + enabled: True + + # Set to True if the list should be moderated + # If set, only subscribers can post to the list + # Non subscriber mails are forwarded to the list admins + moderated: True + + archive: + use_maildir: True + # Set to true if the incoming mails should be archived + # locally in a mailbox/dir unmodified before decryption + archive_incoming: True + # Set to true if the incoming mails should be archived + # locally in a mailbox/dir after decrypting them + archive_plain: False + + # The list account itself + listaccount: + list1@list.domain: + name: list1 + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + + # The list admins + admins: + # Format: + # email address: + # name: Real name of the subscriber + # enabled: Subscriber is enabled (if omitted defaults to False) + # use_smime: True/False (Use S/MIME for encryption. If omitted defaults to False) + # fingerprint: GPG fingerprint (Not required when use_smime == True) + # gpg_plain: Plain text inline GPG encryption (If omitted defaults to False) + admin2@admin2.domain: + name: Admin2 + fingerprint: 40_CHARACTER_PGP_FINGERPRINT + enabled: True + + admin3@admin3.domain: + name: Admin three + enabled: False diff --git a/Documentation/index.rst b/Documentation/index.rst new file mode 100644 index 0000000..07896aa --- /dev/null +++ b/Documentation/index.rst @@ -0,0 +1,80 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. remail documentation master file + +Welcome to remail's documentation! +================================== + +.. toctree:: + :maxdepth: 3 + :caption: Contents: + +Introduction +------------ + +.. toctree:: + :maxdepth: 3 + + introduction.rst + + +Licensing +--------- + +Describes the licensing rules for the remail source code. + +.. toctree:: + :maxdepth: 3 + + license-rules.rst + + +Installation +------------ + +Installation guidelines + +.. toctree:: + :maxdepth: 3 + + installation.rst + + +Configuration +------------- + +Configuration guidelines + +.. toctree:: + :maxdepth: 3 + + configuration.rst + +Operation +---------- + +Information about operation + +.. toctree:: + :maxdepth: 3 + + operation.rst + + +Usage +----- + +Usage guidelines (man pages) + +.. toctree:: + :maxdepth: 3 + + man1/remail_daemon.rst + man1/remail_chkcfg.rst + man5/remail_daemon.config.rst + + +Indices and tables +================== + +* :ref:`genindex` diff --git a/Documentation/installation.rst b/Documentation/installation.rst new file mode 100644 index 0000000..7f227c9 --- /dev/null +++ b/Documentation/installation.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_installation: + +remail installation guide +========================= + +General +------- + +The recommended installation procedure is via the package management system +of your distribution. Manual installation is surely possible and the +various bits and pieces including their default or recommended locations on +the target are documented in the packaging files. + +By default the daemon is disabled as it requires configuration. + + +Protecting the system +--------------------- + +Run the list daemon on a machine which is not publicly accessible and +ensure that no untrusted users can access it. It can be run in a VM with +the sole purpose of running the encrypted mailing list. As the private keys +of the list are stored on that machine the setup requires deep +understanding of IT security. It's outside the scope of this documentation +to provide guidance on this. + + +Dependencies +------------ + +remail requires Python 3 and depends on the following python packages: + + - python3-M2Crypto + - python3-gnupg + - python3-yaml + +Note, that on Debian 10 python3-M2Crypto is not available, but can be +installed as a Debian package from the testing repository. + +Further the systems needs to have: + + - A mail transport agent which handles the transport of the mails to an + outgoing SMTP server. + + - A mail retriever agent or some other mechanism which can fetch or + handle incoming mail and deliver it to remails maildir. + +The initial deployment of the list daemon which was used to handle +development of mitigations for embargoed hardware security vulnerabilities +uses postfix and getmail, but that's only one of various possibilites. + +As the choice of tools depends on the setup and the situation under which +the system is deployed, there is no documentation provided here. It's a +prerequisite that an administrator who wants to deploy remail is familiar +with these kind of tools. + diff --git a/Documentation/introduction.rst b/Documentation/introduction.rst new file mode 100644 index 0000000..7a96164 --- /dev/null +++ b/Documentation/introduction.rst @@ -0,0 +1,95 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_introduction: + +Introduction +============ + +remail is a very simplistic mailing list tool which provides encryption. + +Why yet another mailing list tool? +---------------------------------- + +The handling of the embargoed hardware security issues made it necessary +to have a encrypted mailing list. Managing Cc lists manually and dealing +with the odd GPG integrations in the mail clients is just not a workable +solution. + +There are only a few Open Source implementations of encrypted mailing lists +available. Some of them are abandoned projects. The alive ones have their +own set of shortcomings. One of them supports only S/MIME. The other +supports only PGP, but did not survive testing in a real world +deployment. Repairing ruby code and dis-tangling the thing from its weird +daemons and other over-engineered features was certainly not a project for +the weekend. + +After thinking about it for a while, I recognized that for the purpose at +hand the tool can be very simplistic. Using existing tools like getmail and +SMTP servers plus the knowledge about the gory details of emails which I +gained by writing mail handling scripts in python for my daily work, made +it possible to implement it in a rather short time. + + +What it does +------------ + +remail reads mail from a maildir, decrypts it with the mailing list private +key and re-encrypts it for every enabled subscriber. The resulting mails are +delivered to the localhost's SMTP server. + +remail supports S/MIME and PGP on both ends. + +No MUA functionality and no complicated SMTP delivery mechanisms are +required. They all exist in well maintained tools already. + +The list configuration is a simple yaml file and is maintained manually +with your favorite text editor. Building tools around that is not part of +this project and there are many existing ways to handle that conveniently. + +The proven in use mechanism is to have the configuration files in git and +let the mailing list system either check for updates regularly or get some +external notification that a new configuration is available. + + +What it does not +---------------- + +It does not care about mail transport on the receiving side, no POP or IMAP +support as this can be handled by tools like getmail. + +It does not care about complex mail transport on the sending side as this +can be handled by SMTP servers like postfix or exim. + +It has no integration into SMTP server transports or filters because it +makes no sense to deploy such a sensitive mechanism on a public facing +machine. + +It does not come with GUI and managenent tools. A crypto mailing list is +subscribers only for obvious reasons and not meant to handle a large amount +of subscribers. It's meant for secure communications of a small group of +people without having the hassle of keeping Cc lists up to date and +encrypting for every recipient. + + +How it works +------------ + +.. image:: remail.svg + +The mailing list has the usual post, bounce and owner email addresses on a +public mail server. These addresses are aliased to a user account. The +mails are stored in the users inbox or forwarded to a protected machine. + +The mailing list software runs on a protected and separated machine behind +firewalls and access barriers like any other sensitive application. + +The mail is retrieved from the public facing machine by any of the existing +mechanisms, e.g. getmail, fetchmail or any MTA which can deliver mail to a +maildir. remail does not implement any transport on the incoming side as +there are good tools available which can handle the requirements of a +particular setup. + +The list daemon retrieves the mails from the maildir and selects the +appropriate list via the 'To' header. It then decrypts the incoming mail, +re-encrypts it for each subscriber and delivers it to the local SMTP server +which takes care of relaying it to the public facing SMTP server. diff --git a/Documentation/license-rules.rst b/Documentation/license-rules.rst new file mode 100644 index 0000000..caf86b0 --- /dev/null +++ b/Documentation/license-rules.rst @@ -0,0 +1,161 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_licensing_rules: + +remail licensing rules +====================== + +The remail project is provided under the terms of the GNU General Public +License version 2 only (GPL-2.0-only), as provided in LICENSES/GPL-2.0. + +This documentation file provides a description of how each source file +should be annotated to make its license clear and unambiguous. +It doesn't replace the projects license. + +The license described in the COPYING file applies to the project source +as a whole, though individual source files can have a different license +which is required to be compatible with the GPL-2.0-only:: + + GPL-2.0-or-later : GNU General Public License v2.0 or later + +Aside from that, individual files can be provided under a dual license, +e.g. one of the compatible GPL variants and alternatively under a +permissive license like BSD, MIT etc. + +The common way of expressing the license of a source file is to add the +matching boilerplate text into the top comment of the file. Due to +formatting, typos etc. these "boilerplates" are hard to validate for +tools which are used in the context of license compliance. + +An alternative to boilerplate text is the use of Software Package Data +Exchange (SPDX) license identifiers in each source file. SPDX license +identifiers are machine parsable and precise shorthands for the license +under which the content of the file is contributed. SPDX license +identifiers are managed by the SPDX Workgroup at the Linux Foundation and +have been agreed on by partners throughout the industry, tool vendors, and +legal teams. For further information see https://spdx.org/ + +The remail prokect requires the precise SPDX identifier in all source +files. The valid identifiers used in the remail project are explained in +the section `License identifiers`_ and have been retrieved from the +official SPDX license list at https://spdx.org/licenses/ along with the +license texts. + +License identifier syntax +------------------------- + +1. Placement: + + The SPDX license identifier in source files shall be added at the first + possible line in a file which can contain a comment. For the majority + or files this is the first line, except for executable scripts which + require the '#!PATH_TO_INTERPRETER' in the first line. For those + scripts the SPDX identifier goes into the second line. + +| + +2. Style: + + The SPDX license identifier is added in form of a comment. The comment + style depends on the file type:: + + scripts: # SPDX-License-Identifier: + .py: # SPDX-License-Identifier: + .rst: .. SPDX-License-Identifier: + +| + +3. Syntax: + + A is either an SPDX short form license + identifier found on the SPDX License List. When multiple licenses apply, + an expression consists of keywords "AND", "OR" separating + sub-expressions and surrounded by "(", ")" . + + License identifiers for licenses like [L]GPL with the 'or later' option + are constructed by using a "-or-later" for indicating the 'or later' + option.:: + + # SPDX-License-Identifier: GPL-2.0-or-later + + OR should be used if the file is dual licensed and only one license is + to be selected. For example:: + + # SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + + AND should be used if the file has multiple licenses whose terms all + apply to use the file. For example, if code is inherited from another + project and permission has been given to put it in the remail project, + but the original license terms need to remain in effect:: + + # SPDX-License-Identifier: GPL-2.0-only AND MIT + + +License identifiers +------------------- + +The licenses currently used, as well as the licenses for code added to the +project can be found in the LICENSES directory. + +Whenever possible these licenses should be used as they are known to be +fully compatible and widely used. + +The files in this directory contain the full license text and metatags. +The file names are identical to the SPDX license identifier which shall be +used for the license in source files. + +Examples:: + + LICENSES/GPL-2.0 + +Contains the GPL version 2 license text and the required metatags: + +Metatags: + + The following meta tags must be available in a license file: + + - Valid-License-Identifier: + + One or more lines which declare which License Identifiers are valid + inside the project to reference this particular license text. Usually + this is a single valid identifier, but e.g. for licenses with the 'or + later' options two identifiers are valid. + + - SPDX-URL: + + The URL of the SPDX page which contains additional information related + to the license. + + - Usage-Guidance: + + Freeform text for usage advice. The text must include correct examples + for the SPDX license identifiers as they should be put into source + files according to the `License identifier syntax`_ guidelines. + + - License-Text: + + All text after this tag is treated as the original license text + + File format examples:: + + Valid-License-Identifier: GPL-2.0-only + Valid-License-Identifier: GPL-2.0-or-later + SPDX-URL: https://spdx.org/licenses/GPL-2.0-only.html + Usage-Guide: + To use this license in source code, put one of the following SPDX + tag/value pairs into a comment according to the placement + guidelines in the licensing rules documentation. + For 'GNU General Public License (GPL) version 2 only' use: + SPDX-License-Identifier: GPL-2.0-only + For 'GNU General Public License (GPL) version 2 or any later version' use: + SPDX-License-Identifier: GPL-2.0-or-later + License-Text: + Full license text + +| + +All SPDX license identifiers must have a corresponding file in the LICENSE +subdirectory. This is required to allow tool verification and to have the +licenses ready to read and extract right from the source, which is +recommended by various FOSS organizations, e.g. the `FSFE REUSE initiative +`_. diff --git a/Documentation/man1/remail_chkcfg.rst b/Documentation/man1/remail_chkcfg.rst new file mode 100644 index 0000000..26c3f43 --- /dev/null +++ b/Documentation/man1/remail_chkcfg.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_chkcfg_man: + + +remail_chkcfg manual page +========================= + +Synopsis +-------- + +**remail_chkcfg** [*options*] config_file + +Description +----------- + +:program:`remail_chkcfg`, A program to check and display show the +configuration of a crypto mailing list + +It reads the configuration file, verifies for correctness and displays the +resulting aggregated configuration in simple form. + +Options +------- + +-h, --help + Show this help message and exit + +-e, --enabled + Show a pretty printed tabular list of names and email addresses of all + enabled subscribers in a list specific configuration file. Implies -lnq. + +-l, --list + Configuration file is a list specific configuration which contains only + the subscribers + +-n, --nokeys + Do not check for GPG and S/MIME keys and certs + +-q, --quiet + Quiet mode. Do not show the configuration. Only check for correctness + +-v, --verbose + Enable verbose logging. + +-V, --version + Display version information + + +Invocation +---------- + +The program expects a properly populated configuration directory at the +place where the config_file is. If the program is invoked from outside the +configuration directory the program changes into the configuration +directory according to the directory part of the config_file argument. + +If the configuration file is the base configuration file the program +expects the list configuration files of the enabled lists to be available +in the lists subdirectory. + +If it is invoked with a list specific configuration file it only checks and +shows this part. If not disabled it checks for the availability and validity +of the keys for each subscriber. + + +See also +-------- +:manpage:`remail_daemon.config(5)` diff --git a/Documentation/man1/remail_daemon.rst b/Documentation/man1/remail_daemon.rst new file mode 100644 index 0000000..6c50a17 --- /dev/null +++ b/Documentation/man1/remail_daemon.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_daemon_man: + +remail_daemon manual page +========================= + +Synopsis +-------- + +**remail_daemon** [*options*] config_file + +Description +----------- + +:program:`remail_daemon`, The daemon for running an encrypted mailing +list. It reads the configuration file from the command line. + + +Options +------- + +-h, --help + Show this help message and exit + +-s syslog, --syslog + Use syslog for logging. Default is stderr + +-v, --verbose + Enable verbose logging. + +-V, --version + Display version information + + +Configuration file +------------------ + +remail_daemon reads the configuration file which was handed in on the +command line. The configuration file is a simple yaml file. Non-mandatory +configuration options which are not in the configuration file are set to +the default values. + +See the configuration file man page for detailed information. + + +Work directory +-------------- + +remail daemon assumes that the configuration file is in the work directory +which has a defined layout and content. The directory structure is +documented in the full remail documentation along with hints how to manage +documentation. + + +See also +-------- +:manpage:`remail_daemon.config(5)` diff --git a/Documentation/man5/remail_daemon.config.rst b/Documentation/man5/remail_daemon.config.rst new file mode 100644 index 0000000..ebe3959 --- /dev/null +++ b/Documentation/man5/remail_daemon.config.rst @@ -0,0 +1,430 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_daemon_config_man: + +remail_daemon.config manual page +================================ + +Synopsis +-------- + +remail.yaml +list.yaml + +Description +----------- + +This manual describes the configuration file format for the remail daemon. + +The configuration files are using yaml syntax. There are two types of +configuration files: + +1) Base configuration file + + The default configuration file is located in the base configuration + directory which is also the base work directory. + +2) List configuration file + + Per mailing list configuration file. + +Non-mandatory options are set to the builtin defaults if they are not +available in the configuration files. + +.. _config_dir_struct: + +Configuration directory structure +--------------------------------- + +The configuration directory structure is fixed and looks like this:: + + ├── .certs + │   ├── cacert.pem + │   ├── list1@your.domain.key + │   ├── list2@your.domain.key + │   ├── admin1@some.domain.crt + │   ├── admin2@other.domain.crt + ├── .git + ├── .gnupg + │   ├── private-keys-v1.d + │   │   ├── XYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYX1.key + │   │   ├── YXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXYXY2.key + │   ├── pubring.kbx + ├── lists + │   ├── list1 + │   │   ├── .certs + │   │   │   ├── list1@your.domain.crt + │   │   │   ├── subscr1-1@subscr1.domain.crt + │   │   │   ├── subscr1-2@subscr2.domain.crt + │   │   │   ├── admin1@some.domain.crt + │   │   ├── .git + │   │   ├── .gnupg + │   │   │   └── pubring.kbx + │   │   ├── list.yaml + │   │   ├── template_admin + │   │   └── template_welcome + │   ├── list2 + │   │   ├── .certs + │   │   │   ├── list2@your.domain.crt + │   │   │   ├── subscr2-1@subscr1.domain.crt + │   │   │   ├── subscr2-2@subscr2.domain.crt + │   │   │   ├── admin2@other.domain.crt + │   │   ├── .git + │   │   ├── .gnupg + │   │   │   └── pubring.kbx + │   │   ├── list.yaml + │   │   ├── template_admin + │   │   └── template_welcome + ├── lock + ├── maildir + │   ├── cur + │   ├── new + │   └── tmp + ├── maildir.frozen + │   ├── cur + │   ├── new + │   └── tmp + └── remail.yaml + +The base directory contains: + + - The main configuration file (remail.yaml). + + - The S/MIME directory (.certs). + + .. warning:: It contains the private S/MIME keys of the lists. + + - The GPG directory (.gnupg). + + .. warning:: It contains the private GPG keys of the lists. + + - The maildir directory to which the incoming mail gets delivered by a + MTA or MRA. + + - The maildir.frozen directory to which unprocessed or failed mail + gets moved to. + + - A lock file which can be used to protect a reload against a concurrent + configuration update. + + - The lists directory under which the list specific configuration directories + are located. + +The list directories contain: + + - The list configuration file (list.yaml) + + - The S/MIME directory (.certs). It contains the public S/MIME keys of the + list subscribers and of the list itself. + + - The GPG directory (.gnupg). It contains the public GPG keys of the list + subscribers and of the list itself. + + +Base configuration items +------------------------ + +The structure of the base configuration file is:: + + .. code-block:: yaml + + enabled: False + use_smtp: False + smime: + ... + gpg: + ... + lists: + list1: + enabled: True + moderated: True + archive: + ... + listaccount: + ... + admins: + ad1@min.domain: + ... + adN@min.domain: + ... + listN: + ... + +Base items: +^^^^^^^^^^^ + + .. code-block:: yaml + + enabled: False + use_smtp: False + + enabled: + + Optional item to enable or disable the daemon. If the item is set to + False then no other options are evaluated and the daemon sleeps waiting + for termination or reconfiguration. If True, the rest or the options is + evaluated. + + Optional item which defaults to False + + use_smtp: + + Set to True to enable mail delivery via SMTP to the SMTP server on + localhost. The SMTP server is responsible for relaying the mails to a + public mail server. remail does not implement any other transport for + outgoing mail and the target server is therefore not configurable. + + If False the encrypted mails are delivered to stdout. That's mainly a + development option which is not meant for production use. + + Optional time which defaults to False + + +S/MIME options: +^^^^^^^^^^^^^^^ + + .. code-block:: yaml + + smime: + verify: True + sign: True + + verify: + + When handling S/MIME encrypted mail then the validity of the senders key + is by default verified against the CA certs. If set to False this + verification is disabled. Disable this only in extreme situations and + consider the consequences. + + sign: + + Sign the mails sent to S/MIME recipients with the lists key. Enabled by + default as this is the recommended way to send S/MIME mail. If disabled + then the public certificate of the list is not part of the welcome + message which is sent to new recipients. + +GPG options: +^^^^^^^^^^^^ + + .. code-block:: yaml + + gpg: + always_trust: True + sign: True + + always_trust: + + The public keyring of a list is managed by the list administrator. To + avoid having to manually tweak the trust DB, it's possible to force + trust mode on the keyring with this option. Defaults to True as the + trust establishment is the responsibility of the list administrator + anyway and setting this to True avoids a lot of pointless manual + operations. + + sign: + + Sign the mails sent to GPG recipients with the lists key. Enabled by + default as this is the recommended way to send GPG mail. If disabled then + the public key of the list is not part of the welcome message which is + sent to new recipients. + +The mailing lists collection: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + .. code-block:: yaml + + lists: + listname: + ... + listname: + + lists: + + The opening of the lists map. + +The list base configuration: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The list base configuration for each list consists of the following items: + + .. code-block:: yaml + + listname: + enabled: True + moderated: True + archive: + ... + listaccount: + ... + admins: + ... + +The list base items: +"""""""""""""""""""" + + .. code-block:: yaml + + listname: + enabled: True + moderated: True + + enabled: + + If False, the list configuration is disabled. No mail is delivered to this + list. If True, the list is enabled. + + moderated: + + Optional item to set the list to moderated. It True only subscribers + are allowed to post to the list either from their subscription address + or from one of the optional alias mail addresses which are associated + with a subscriber. Mails from non-subscribers are not delivered to the + list, they are delivered to the list administrator + +The archive section: +"""""""""""""""""""" + + .. code-block:: yaml + + archive: + use_maildir: True + archive_incoming: True + archive_plain: False + + use_maildir: + + If True, maildir is the storage format for enabled archives. If False, + mbox is used. + + archive_incoming: + + If True archive the incoming encrypted mails in the selected storage + format. The maildir folder name is archive_incoming/. The mbox name is + archive_incoming.mbox. These files/directories are located in the per + mailing list configuration/work directory. + + archive_plain: + + If True archive the decrypted mails in the selected storage format. The + mails are archived in two stores: + + - archive_admin[.mbox] for mails which are directed to the list admins + either directly or through bounce catching, moderation etc. + + - archive_list[.mbox] for mails which are delivered to the list + +The list account section: +""""""""""""""""""""""""" + + .. code-block:: yaml + + listaccount: + list@mail.domain: + name: Clear text name + fingerprint: 40CHARACTERFINGERPRINT + + The list account's e-mail address is the key item for the name and + fingerprint options. + + name: + + A clear text name for the list, e.g. incident-17 or whatever sensible + name is selected. This name is used in From mangling when a list post + is sent to the subscribers: + + incident-17 for Joe Poster + + From rewriting is used to ensure that replies go only to the list and + not to some other place. The Reply-To field could be used as well but + that is not correctly handled by mail clients and users can force + reply to all nevertheless. + + fingerprint: + + The full 40 character PGP fingerprint of the list key. + +.. _list_admin_section: + +The list administrators section: +"""""""""""""""""""""""""""""""" + + .. code-block:: yaml + + admins: + admin1@some.domain: + name: Clear text name + fingerprint: 40CHARACTERFINGERPRINT + enabled: True + use_smime: False + gpg_plain: False + admin2@other.domain: + + name: + + The real user name. Mandatory field + + fingerprint: + + The full 40 character PGP fingerprint of the administrators + key. Mandatory if the use_smime option is not set. + + enabled: + + Switch to enable/disable the account. Mandatory item. + + use_smime: + + Send S/MIME encrypted mail to the admin if True. Otherwise use + PGP. Optional, defaults to False. + + gpg_plain: + + If False send mail in the application/pgp-encrypted format. If True + use the plain/text embedded PGP variant if possible. The latter does + not work for mails with attachments but for normal plain/text + conversation this can be requested by a recipient because that's + better supported in some mail clients. Optional, defaults to False. + +List configuration items +------------------------ + +The structure of a list specific configuration file is:: + + .. code-block:: yaml + + subscribers: + subscriber1@some.domain: + ... + subscriberN@other.domain: + ... + + +The configuration of the subscribers is identical to the configuration of +:ref:`the list administrators section ` above, but it +allows one additional field: + + .. code-block:: yaml + + subscribers: + subscriber1@some.domain: + ... + aliases: + - subscr1@some.domain + - subscriber1@other.domain + + aliases: + + The optional aliases item is a list of alias email addresses for a + subscriber. List mail is always delivered to the subscriber e-mail + address, but people have often several e-mail addresses covered by + the same PGP key and post from various addresses. If the list is + moderated, then the aliases allow posting for subscribers from their + registered alias addresses. If moderation is disabled the alias list + is not used at all. + + +See also +-------- +:manpage:`remail_daemon(1)` +:manpage:`remail_chkcfg(1)` + diff --git a/Documentation/operation.rst b/Documentation/operation.rst new file mode 100644 index 0000000..88af480 --- /dev/null +++ b/Documentation/operation.rst @@ -0,0 +1,97 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _remail_operation: + +remail operation related information +==================================== + +Mails to list administrators +---------------------------- + +remail sends mail to the list administrators in case of failures or email +related issues. The mails are either incoming mails or mails composed from +the template_admin file. + +The mail subject is marked with a prefix depending on the nature of the +issue: + + - [MODERATED] + + The mail which is sent to the moderator is an incoming list mail from a + non-subscribed sender. Subject and content are kept intact. The mail is + sent encrypted. + + - [AUTOREPLY] + + Auto-replies like out of office notifications are catched and forwarded + to the list administrator. There is no policy to disable subscribers + built in. This has to be handled by the administrator. + + - [TEMPFAIL] + + Temporary delivery failures which are bounced back from a MTA. The + affected subscriber account is not disabled. This is informational. + + - [FROZEN] + + A permanent delivery failure or a encryption problem happened. + + The affected subscriber account is frozen due to that. See + :ref:`subscriber_status` below. + + - Subject taken from the template_admin file + + Mails which are composed from the template file contain internal error + information about issues which the mail processing triggered. Likely + causes are encryption/decryption failures. + +Fatal issues in the list daemon like failures to connect to the local host +SMTP daemon, file system errors or bugs in the list code which trigger an +exception are not sent to the administrators of a mailing list. These +issues are reported verbosely in syslog and the base administrator needs to +take care of them. + +.. _subscriber_status: + +Subscriber status +----------------- + +remail has subscriber state tracking for each mailing list. The tracking +status is maintained in the file tracking.yaml which is managed by remail +in the mailing list specific directory. + +The states are stored in a subscriber email-address mapping. The possible +states are: + + - DISABLED + + The subscriber is disabled in the configuration file. + + - REGISTERED + + The subscriber is enabled in the configuration file, but no welcome + mail has been sent to the subscriber. + + - ENABLED + + The subscriber is enabled in the configuration file. The welcome + mail has been sent to the subscriber. + + - FROZEN + + The subscriber is enabled in the configuration file, but the account is + frozen due to a permanent delivery failure or an internal encryption + problem. + +Unfreeze +^^^^^^^^ + +To unfreeze a subscriber the subscriber account has to be disabled in the +list configuration file and the list daemon configuration has to be +reloaded. After this the subscriber state is set to DISABLED. Re-enabling +the subscriber in the configuration file and reloading the daemon +configuration re-enables the account. + +This is a bit tedious and could be handled by a command mail which is sent +from the list administrator to the list, but that's not yet implemented. + diff --git a/Documentation/remail.svg b/Documentation/remail.svg new file mode 100644 index 0000000..765b09e --- /dev/null +++ b/Documentation/remail.svg @@ -0,0 +1,921 @@ + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + maildir + + + + + + + + + remaildaemon + + + + + + + + + (Mail RetrievalAgent) + + + + + + + + + SMTP + + + + + + + + + Public Mailserver + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FIREWALL + + + + + + + Secured Listserver + + + + + + + + + \ No newline at end of file diff --git a/LICENSES/GPL-2.0 b/LICENSES/GPL-2.0 new file mode 100644 index 0000000..0a29508 --- /dev/null +++ b/LICENSES/GPL-2.0 @@ -0,0 +1,266 @@ +Valid-License-Identifier: GPL-2.0-only +Valid-License-Identifier: GPL-2.0-or-later +SPDX-URL: https://spdx.org/licenses/GPL-2.0-only.html +Usage-Guide: + To use this license in source code, put one of the following SPDX + tag/value pairs into a comment according to the placement + guidelines in the licensing rules documentation. + For 'GNU General Public License (GPL) version 2 only' use: + SPDX-License-Identifier: GPL-2.0-only + For 'GNU General Public License (GPL) version 2 or any later version' use: + SPDX-License-Identifier: GPL-2.0-or-later +License-Text: + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + + Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to most +of the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for this service if you wish), +that you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs; and that you know you +can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for +a fee, you must give the recipients all the rights that you have. You must make +sure that they, too, receive or can get the source code. And you must show them +these terms so they know their rights. +We protect your rights with two steps: (1) copyright the software, and (2) +offer you this license which gives you legal permission to copy, distribute +and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced +by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We wish +to avoid the danger that redistributors of a free program will individually +obtain patent licenses, in effect making the program proprietary. To prevent +this, we have made it clear that any patent must be licensed for everyone's +free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included without +limitation in the term "modification".) Each licensee is addressed as "you". +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is not +restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such modifications +or work under the terms of Section 1 above, provided that you also meet all of +these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole + or in part contains or is derived from the Program or any part thereof, + to be licensed as a whole at no charge to all third parties under the + terms of this License. + + c) If the modified program normally reads commands interactively when + run, you must cause it, when started running for such interactive use in + the most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a + copy of this License. (Exception: if the Program itself is interactive + but does not normally print such an announcement, your work based on the + Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +works. But when you distribute the same sections as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all the +source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code distributed +need not include anything that is normally distributed (in either source or +binary form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component itself +accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. + +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. +It is not the purpose of this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In such +case, this License incorporates the limitation as if written in the body of +this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. +Each version is given a distinguishing version number. If the Program specifies +a version number of this License which applies to it and "any later version", +you have the option of following the terms and conditions either of that +version or of any later version published by the Free Software Foundation. If +the Program does not specify a version number of this License, you may choose +any version ever published by the Free Software Foundation. +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of +all derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU +ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..238ea4f --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# remail - A set of tools for crypted mailing lists + +A simplistic but powerfull crypted mailing list tool set. The following +tools are available: + + remail_daemon - The crypto mailing list processor + remail_chkcfg - Tool to check the configuration + +## Install + +It's recommended to generate the appropriate package for your system and +install it via the package manager. + +## Usage + +See Documentation and man pages + diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/remail/__init__.py b/remail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/remail/config.py b/remail/config.py new file mode 100644 index 0000000..16422d0 --- /dev/null +++ b/remail/config.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# Configuration items + +from remail.mail import email_addr_valid + +import os + + +class RemailException(Exception): + pass + +class RemailConfigException(RemailException): + pass + +class RemailListConfigException(RemailException): + pass + +def get_mandatory(key, cfgdict, base): + res = cfgdict.get(key) + if not res: + txt = 'Missing config entry: %s.%s' % (base, key) + raise RemailListConfigException(txt) + return res + +def get_optional(key, defdict, cfgdict): + return cfgdict.get(key, defdict[key]) + +def set_defaults(obj, defdict, cfgdict): + if not cfgdict: + cfgdict = {} + for key, val in defdict.items(): + val = cfgdict.get(key, val) + setattr(obj, key, val) + +def show_attrs(obj, attrdict, indent): + for attr in attrdict: + print('%*s%-40s: %s' %(indent, '', attr, getattr(obj, attr))) + +account_defaults = { + 'enabled' : False, + 'fingerprint' : None, + 'use_smime' : False, + 'gpg_plain' : False, +} + +class account_config(object): + def __init__(self, cfgdict, addr, base): + base = base + '.addr' + # Do at least minimal checks for a valid email address + if not email_addr_valid(addr): + txt = 'Invalid email address: %s' % base + raise RemailListConfig_Exception(txt) + + self.addr = addr + self.name = get_mandatory('name', cfgdict, base) + set_defaults(self, account_defaults, cfgdict) + + # Get the optional aliases to allow sending from + # different accounts when the list is moderated + aliases = cfgdict.get('aliases') + if not aliases: + self.aliases = [] + else: + self.aliases = aliases + + def show(self, indent, all=True): + print('%*s%-40s: %s' %(indent, '', self.name, self.addr)) + indent += 2 + if all: + show_attrs(self, account_defaults, indent) + txt = '' + for alias in self.aliases: + txt += '%s ' % alias + print('%*s%-40s: %s' %(indent, '', 'aliases', txt)) + else: + print('%*s%-40s: %s' %(indent, '', 'fingerprint', self.fingerprint)) + +class accounts_config(object): + def __init__(self, cfgdict, base): + self.accounts = {} + for addr, cfg in cfgdict.items(): + account = account_config(cfg, addr, base) + self.accounts[addr] = account + + def __len__(self): + return len(self.accounts) + + def has_account(self, addr): + for account in self.accounts.values(): + if account.addr == addr: + return True + if addr in account.aliases: + return True + return False + + def get(self, addr): + return self.accounts.get(addr) + + def keys(self): + return self.accounts.keys() + + def values(self): + return self.accounts.values() + + def show(self, indent): + for addr, acc in self.accounts.items(): + acc.show(indent) + + def pop(self): + addr, acc = self.accounts.popitem() + return acc + +def list_account_config(cfgdict, base): + laccs = accounts_config(cfgdict, base) + if len(laccs) != 1: + txt = '% entry for %s' % base + raise RemailListConfigException(txt) + return laccs.pop() + +def build_listheaders(mailaddr): + addr, domain = mailaddr.split('@') + headers = {} + headers['List-Id'] = mailaddr + headers['List-Owner'] = '' % (addr, domain) + headers['List-Post'] = '' % mailaddr + return headers + +class destination(object): + def __init__(self, toadmin, accounts): + self.toadmin = toadmin + self.accounts = accounts + +class listaddrs(object): + ''' + Build an object with the valid list addresses + addr@domain, addr-owner@domain, addr-bounce@domain + ''' + def __init__(self, mailaddr): + addr, domain = mailaddr.split('@') + self.post = mailaddr + self.owner = '%s-owner@%s' % (addr, domain) + self.bounce = '%s-bounce@%s' % (addr, domain) + # Dictionary to lookup an incoming address + # and to establish to which target it goes + # Entry is true if the target is admins + self.addrs = {} + self.addrs[self.post] = False + self.addrs[self.owner] = True + self.addrs[self.bounce] = True + + def get_destination(self, msgto, admins, subscribers): + if not msgto in self.addrs.keys(): + return None + + if self.addrs[msgto]: + return destination(True, admins) + return destination(False, subscribers) + + def show(self, indent): + print('%*slistaddrs:' %(indent, '')) + indent += 2 + print('%*s%-40s: %s' %(indent, '', 'post', self.post)) + print('%*s%-40s: %s' %(indent, '', 'owner', self.owner)) + print('%*s%-40s: %s' %(indent, '', 'bounce', self.bounce)) + +class archive_config(object): + def __init__(self, aopts, listdir): + self.incoming = aopts.incoming + self.plain = aopts.plain + self.mdir = aopts.use_maildir + if self.mdir: + self.m_encr = os.path.join(listdir, 'archive_encr/') + self.m_admin = os.path.join(listdir, 'archive_admin/') + self.m_list = os.path.join(listdir, 'archive_list/') + else: + self.m_encr = os.path.join(listdir, 'archive_encr.mbox') + self.m_admin = os.path.join(listdir, 'archive_admin.mbox') + self.m_list = os.path.join(listdir, 'archive_list.mbox') + + def show(self, indent): + print('%*s%-40s: %s' % (indent, '', 'use_maildir', self.mdir)) + if self.incoming: + print('%*s%-40s: %s' % (indent, '', 'incoming', self.m_encr)) + if self.plain: + print('%*s%-40s: %s' % (indent, '', 'plain_admin', self.m_admin)) + print('%*s%-40s: %s' % (indent, '', 'plain_list', self.m_list)) + +smime_defaults = { + 'verify' : True, + 'sign' : True, +} + +class smime_config(object): + def __init__(self, listdir, cfgdict): + set_defaults(self, smime_defaults, cfgdict) + self.global_certs = '.certs' + self.ca_certs = os.path.join(self.global_certs, 'cacert.pem') + self.list_certs = os.path.normpath(os.path.join(listdir, '.certs')) + + def show(self, indent): + print('%*sS/MIME:' % (indent, '')) + indent += 2 + show_attrs(self, smime_defaults, indent) + print('%*s%-40s: %s' %(indent, '', 'global_certs', self.global_certs)) + print('%*s%-40s: %s' %(indent, '', 'ca_certs', self.ca_certs)) + print('%*s%-40s: %s' %(indent, '', 'list_certs', self.list_certs)) + +gpg_defaults = { + 'always_trust' : True, + 'sign' : True, +} + +class gpg_config(object): + def __init__(self, listdir, cfgdict): + set_defaults(self, gpg_defaults, cfgdict) + self.armor = True + self.home = '.gnupg' + self.keyring = os.path.join(listdir, '.gnupg', 'pubring.kbx') + self.keyring = os.path.normpath(self.keyring) + + def show(self, indent): + print('%*sGPG:' % (indent, '')) + indent += 2 + show_attrs(self, gpg_defaults, indent) + print('%*s%-40s: %s' %(indent, '', 'home', self.home)) + print('%*s%-40s: %s' %(indent, '', 'keyring', self.keyring)) + +class tracking_config(object): + def __init__(self, listdir): + self.tracking_file = os.path.join(listdir, 'tracking.yaml') + + def show(self, indent): + print('%*s%-40s: %s' % (indent, '', 'tracking_file', self.tracking_file)) + +class template_config(object): + def __init__(self, listdir): + self.welcome = os.path.join(listdir, 'template_welcome') + self.admin = os.path.join(listdir, 'template_admin') + + def show(self, indent): + print('%*stemplates:' % (indent, '')) + indent += 2 + print('%*s%-40s: %s' %(indent, '', 'welcome', self.welcome)) + print('%*s%-40s: %s' %(indent, '', 'admin', self.admin)) + +archive_defaults = { + 'incoming' : True, + 'plain' : True, + 'use_maildir' : False, +} + +class archive_options(object): + def __init__(self, cfgdict): + set_defaults(self, archive_defaults, cfgdict) + +list_defaults = { + 'enabled' : False, + 'moderated' : False, +} + +class list_config(object): + def __init__(self, name, cfgdict): + base = 'base.lists.%s' %name + self.base = base + self.name = name + set_defaults(self, list_defaults, cfgdict) + + self.listdir = os.path.join('lists', name) + self.listcfg = os.path.join(self.listdir, 'list.yaml') + self.tracking = tracking_config(self.listdir) + self.templates = template_config(self.listdir) + aopts = archive_options(cfgdict.get('archive', {})) + self.archives = archive_config(aopts, self.listdir) + + acc = get_mandatory('listaccount', cfgdict, base) + self.listaccount = list_account_config(acc, base + '.listaccount') + + self.listaddrs = listaddrs(self.listaccount.addr) + self.listheaders = build_listheaders(self.listaccount.addr) + + self.smime = smime_config(self.listdir, None) + self.gpg = gpg_config(self.listdir, None) + + self.admins = accounts_config(get_mandatory('admins', cfgdict, base), + base) + self.subscribers = accounts_config({}, base) + + def config_subscribers(self, cfgdict): + self.subscribers = accounts_config(cfgdict, self.base) + + def show(self, indent): + print('%*s%s' %(indent, '', self.name)) + indent += 2 + show_attrs(self, list_defaults, indent) + print('%*s%-40s: %s' %(indent, '', 'listdir', self.listdir)) + print('%*s%-40s: %s' %(indent, '', 'listcfg', self.listcfg)) + self.tracking.show(indent) + print('%*sarchive:' %(indent, '')) + self.archives.show(indent + 2) + self.smime.show(indent) + self.gpg.show(indent) + print('%*slistaccount:' %(indent, '')) + self.listaccount.show(indent + 2, all=False) + self.listaddrs.show(indent) + print('%*slistheaders:' %(indent, '')) + for h, v in self.listheaders.items(): + print('%*s%-40s: %s' % (indent + 2, '', h, v)) + + print('%*sadmins:' %(indent, '')) + self.admins.show(indent + 2) + print('%*ssubscribers:' %(indent, '')) + self.subscribers.show(indent + 2) + +main_defaults = { + 'enabled' : False, + 'use_smtp' : False, +} + +class main_config(object): + def __init__(self, cfgdict): + set_defaults(self, main_defaults, cfgdict) + self.maildir = 'maildir' + self.mailfrozen = 'maildir.frozen' + self.lockfile = 'lock' + + self.smime = smime_config('.', cfgdict.get('smime')) + self.gpg = gpg_config('.', cfgdict.get('gpg')) + + # Configure the lists + self.lists = [] + for name, l in cfgdict.get('lists', {}).items(): + self.lists.append(list_config(name, l)) + + def show(self): + show_attrs(self, main_defaults, 2) + print(' %-40s: %s' %('maildir', self.maildir)) + self.smime.show(4) + self.gpg.show(4) + print(' lists:') + for ml in self.lists: + ml.show(4) diff --git a/remail/gpg.py b/remail/gpg.py new file mode 100644 index 0000000..be6c033 --- /dev/null +++ b/remail/gpg.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# GPG mail encryption/decryption + +from remail.mail import msg_set_payload, msg_set_header +from remail.mail import msg_get_payload_as_string +from remail.mail import msg_from_string, msg_set_gpg_payload + +import hashlib +import string +import gnupg +import time +import os + +class RemailGPGException(Exception): + pass + +class RemailGPGKeyException(Exception): + pass + +class gpg_crypt(object): + def __init__(self, gpgcfg, account, checkkey=True): + self.config = gpgcfg + self.account = account + self.gpg = gnupg.GPG(gnupghome=self.config.home, + keyring=self.config.keyring) + self.keys = self.gpg.list_keys() + + if not checkkey: + return + + self._check_key(self.keys, self.account) + # Verify that the private key is there as well + self._check_key(self.gpg.list_keys(True), self.account, private=True) + + def _check_key_addr(self, key, addr): + maddr = '<%s>' %addr + maddr = maddr.lower() + for uid in key['uids']: + uid = uid.lower() + if uid.find('<') >= 0: + if uid.find(maddr) >= 0: + return + elif uid == addr: + return + + txt = 'Account address/alias %s' % addr + txt += ' not in GPG key %s\nuids: ' % key['fingerprint'] + for uid in key['uids']: + txt += ' %s' %uid + raise RemailGPGKeyException(txt) + + def _check_key_expiry(self, key, addr): + exp = key['expires'] + if not exp: + return + now = time.time() + if now < float(exp): + return + + txt = 'Account %s' % addr + txt += ' GPG key %s expired' % key['fingerprint'] + raise RemailGPGKeyException(txt) + + def _check_key(self, keys, account, private=False): + for key in keys: + if account.fingerprint != key['fingerprint']: + continue + + # Validate that it's the right one + self._check_key_addr(key, account.addr) + for alias in account.aliases: + self._check_key_addr(key, alias) + self._check_key_expiry(key, account.addr) + return + + if private: + txt = 'No private key found for %s' % account.addr + else: + txt = 'No public key found for %s' % account.addr + raise RemailGPGKeyException(txt) + + def check_key(self, account): + self._check_key(self.keys, account) + + def do_encrypt(self, payload, fingerprints): + ''' Common encryption helper''' + + enc = self.gpg.encrypt(payload, fingerprints, armor=self.config.armor, + always_trust=self.config.always_trust, + sign=self.config.sign) + if enc.ok: + return str(enc) + raise RemailGPGException('Encryption fail: %s' % enc.status) + + def gpg_encrypt(self, msg, account): + ''' Encrypt a message for a subscriber. Depending on the + subscribers preference, inline it or use the enveloped + version. + ''' + + fingerprints = [str(account.fingerprint)] + payload = msg.get_payload() + + # GPG inline encryption magic + + # Use plain GPG if requested by accoung and possible + isplain = msg.get_content_type() == 'text/plain' + if account.gpg_plain and type(payload) == str and isplain: + encpl = self.do_encrypt(payload, fingerprints) + msg_set_payload(msg, msg_from_string(encpl)) + return msg + + # Extract payload for encryption + payload = msg_get_payload_as_string(msg) + encpl = self.do_encrypt(payload, fingerprints) + msg_set_gpg_payload(msg, encpl, account.addr) + return msg + + def gpg_decrypt_plain(self, msg): + ''' + Try to decrypt inline plain/text + + If gpg decrypt returns 'no data was provided' treat the + message as unencrypted plain text. + ''' + pl = msg.get_payload(decode=True) + plain = self.gpg.decrypt(pl, always_trust=self.config.always_trust) + if plain.ok: + msg_set_payload(msg, msg_from_string(str(plain))) + if plain.signature_id: + msg_set_header(msg, 'Signature-Id', plain.username) + elif plain.status != 'no data was provided': + # Check for an empty return path which is a good indicator + # for a mail server message. + rp = msg.get('Return-path') + if rp and rp != '<>': + raise RemailGPGException('Decryption failed: %s' % plain.status) + return msg + + def gpg_decrypt_enveloped(self, msg): + ''' + Try to decrypt an enveloped mail + ''' + contents = msg.get_payload() + proto = msg.get_param('protocol') + if proto != 'application/pgp-encrypted': + raise RemailGPGException('PGP wrong protocol %s' % proto) + if len(contents) != 2: + raise RemailGPGException('PGP content length %d' % len(contents)) + ct = contents[0].get_content_type() + if ct != 'application/pgp-encrypted': + raise RemailGPGException('PGP wrong app type %s' % ct) + ct = contents[1].get_content_type() + if ct != 'application/octet-stream': + raise RemailGPGException('PGP wrong app type %s' % ct) + + plain = self.gpg.decrypt(contents[1].get_payload(decode=True), + always_trust=self.config.always_trust) + if not plain.ok: + raise RemailGPGException('Decryption failed: %s' % plain.status) + + if plain.signature_id: + msg_set_header(msg, 'Signature-Id', plain.username) + + pl = msg_from_string(str(plain)) + msg_set_payload(msg, pl) + return msg + + def decrypt(self, msg): + ''' + Try to handle received mail with PGP. Return decoded or plain mail + ''' + msgout = msg_from_string(msg.as_string()) + ct = msg.get_content_type() + + if ct == 'text/plain': + return self.gpg_decrypt_plain(msgout) + + elif ct == 'multipart/encrypted': + return self.gpg_decrypt_enveloped(msgout) + + # There might be inline PGP with no mentioning in the content type + if not msg.is_multipart(): + return msgout + + payloads = msgout.get_payload() + payldecr = [] + for pl in payloads: + pl = self.decrypt(pl) + payldecr.append(pl) + msgout.set_payload(payldecr) + return msgout + + def encrypt(self, msg, account): + ''' + Encrypt a message for a recipient + ''' + return self.gpg_encrypt(msg, account) + diff --git a/remail/mail.py b/remail/mail.py new file mode 100644 index 0000000..90198b5 --- /dev/null +++ b/remail/mail.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# Mail message related code + +from email.utils import make_msgid, formatdate +from email.header import Header, decode_header +from email import message_from_string, message_from_bytes +from email.generator import Generator +from email.message import Message, EmailMessage +from email.policy import EmailPolicy + +import smtplib +import mailbox +import hashlib +import quopri +import base64 +import time +import sys +import re + +def sanitize_headers(msg): + ''' + Sanitize headers by keeping only the ones which are interesting + and order them as gmail is picky about that for no good reason. + ''' + headers_order = [ + 'Return-Path', + 'Date', + 'From', + 'To', + 'Subject', + 'In-Reply-To', + 'References', + 'User-Agent', + 'MIME-Version', + 'Charset', + 'Message-ID', + 'List-Id', + 'List-Post', + 'List-Owner', + 'Content-Type', + 'Content-Disposition', + 'Content-Transfer-Encoding', + 'Content-Language', + 'Envelope-to', + ] + + # Get all headers and remove them from the message + hdrs = msg.items() + for k in msg.keys(): + del msg[k] + + # Add the headers back in proper order + for h in headers_order: + for k, v in hdrs: + if k.lower() == h.lower(): + msg[k] = v + +def send_smtp(msg, to, sender): + ''' + A dumb localhost only SMTP delivery mechanism. No point in trying + to implement the world of SMTP again. KISS rules! + + Any exception from the smtp transport is propagated to the caller + ''' + to = msg['To'] + server = smtplib.SMTP('localhost') + server.ehlo() + server.send_message(msg, sender, [to]) + server.quit() + +def msg_deliver(msg, account, mfrom, sender, use_smtp): + ''' + Deliver the message. Replace or set the mandatory headers, sanitize + and order them properly to make gmail happy. + ''' + msg_set_header(msg, 'From', encode_addr(mfrom)) + msg_set_header(msg, 'To', encode_addr(account.addr)) + msg_set_header(msg, 'Return-path', sender) + msg_set_header(msg, 'Envelope-to', get_raw_email_addr(account.addr)) + + sanitize_headers(msg) + + # Set unixfrom with the current date/time + msg.set_unixfrom('From remail ' + time.ctime(time.time())) + + # Send it out + mout = msg_from_string(msg.as_string().replace('\r\n', '\n')) + if use_smtp: + send_smtp(mout, account.addr, sender) + else: + print(msg.as_string()) + +def send_mail(msg_out, account, mfrom, sender, listheaders, use_smtp): + ''' + Send mail to the account. Make sure that the message + is correct and all required headers and only necessary + headers are in the outgoing mail. + ''' + # Add the list headers + for key, val in listheaders.items(): + msg_out[key] = val + + msg_deliver(msg_out, account, mfrom, sender, use_smtp) + +# Minimal check for a valid email address +re_mail = re.compile('^\w+([\.-]?\w+)*@\w+([\.-]?\w+)*(\.\w{2,3})+$') + +def email_addr_valid(addr): + return re_mail.match(addr) + +def get_raw_email_addr(addr): + ''' + Return the raw mail address, name and brackets stripped off. + ''' + try: + return addr.split('<')[1].split('>')[0].strip() + except: + return addr + +re_compress_space = re.compile('\s+') + +def decode_hdr(hdr): + ''' + Decode a mail header with encoding + ''' + elm = decode_header(hdr.strip()) + res = '' + for txt, enc in elm: + # Groan .... + if enc: + res += ' ' + txt.decode(enc) + elif isinstance(txt, str): + res += ' ' + txt + else: + res += ' ' + txt.decode('ascii') + return re_compress_space.sub(' ', res).strip() + +def decode_addrs(hdr): + ''' + Decode mail addresses from a header and handle encondings + ''' + addrs = [] + if not hdr: + return addrs + parts = re_compress_space.sub(' ', hdr).split(',') + for p in parts: + addr = decode_hdr(p) + addrs.append(addr) + return addrs + +def decode_from(msg): + ''' + Decode the From header and return it as the topmost element + ''' + addrs = decode_addrs(str(msg['From'])) + return addrs.get_first() + +re_noquote = re.compile('[a-zA-Z0-9_\- ]+') + +def encode_hdr(txt): + try: + return txt.encode('ascii').decode() + except: + txt = txt.encode('UTF-8').decode() + +def encode_addr(fulladdr): + try: + name, addr = fulladdr.split('<', 1) + name = name.strip() + except: + return fulladdr + + try: + name = txt.encode('ascii').decode() + if not re_noquote.fullmatch(name): + name = '"%s"' %name.replace('"', '') + except: + name = Header(name).encode() + + return name + ' <' + addr + +def msg_from_string(txt): + policy = EmailPolicy(utf8=True) + return message_from_string(txt, policy=policy) + +def msg_from_bytes(txt): + policy = EmailPolicy(utf8=True) + return message_from_bytes(txt, policy=policy) + +def msg_force_msg_id(msg, name): + # Make sure this has a message ID + id = msg.get('Message-ID', None) + if not id: + id = make_msgid(name.split('@')[0]) + msg_set_header(msg, 'Message-ID', id) + + +def msg_set_header(msg, hdr, txt): + ''' + Set new or replace a message header + ''' + for k in msg.keys(): + if hdr.lower() == k.lower(): + msg.replace_header(k, txt) + return + # Not found set new + msg[hdr] = txt + +payload_valid_mime_headers = [ + 'Content-Description', + 'Content-Transfer-Encoding', + 'Content-Disposition', + 'Content-Language', + 'Content-Type', + 'Charset', + 'Mime-Version', +] + +def is_payload_header(hdr): + for h in payload_valid_mime_headers: + if h.lower() == hdr.lower(): + return True + return False + +def msg_set_payload(msg, payload): + ''' + Set the payload of a message. + ''' + msg.clear_content() + + if payload.get_content_type() == 'text/plain': + pl = payload.get_content() + msg.set_content(pl) + else: + pl = payload.get_payload() + for h, val in payload.items(): + if is_payload_header(h): + msg_set_header(msg, h, val) + msg.set_payload(pl) + +def msg_get_payload_as_string(msg): + ''' + Get the payload with the associated and relevant headers + ''' + payload = EmailMessage() + payload.set_payload(msg.get_payload()) + + for h in payload_valid_mime_headers: + if h in msg.keys(): + payload[h] = msg[h] + elif h == 'Content-Type': + # Force Content-Type if not set + # to avoid confusing gmail + payload[h] = 'text/plain' + + return payload.as_string() + +def msg_set_gpg_payload(msg, encpl, bseed, addpgp=False): + # Create the message boundary + boundary = hashlib.sha1('.'.join(bseed).encode()).hexdigest() + '-' * 3 + + content = '--%s\n' % boundary + content += 'Content-Type: application/pgp-encrypted\n' + content += 'Content-Disposition: attachment\n\n' + content += 'Version: 1\n\n' + content += '--%s\n' % boundary + content += 'Content-Type: application/octet-stream\n' + content += 'Content-Disposition: attachment; filename="msg.asc"\n\n' + if addpgp: + content += '-----BEGIN PGP MESSAGE-----\n\n' + content += encpl + '\n' + if addpgp: + content += '-----END PGP MESSAGE-----\n\n' + + msg_set_payload(msg, msg_from_string(content)) + msg_set_header(msg, 'Mime-Version', '1') + msg_set_header(msg, 'Content-Type', + 'multipart/encrypted; protocol="application/pgp-encrypted";boundary="%s"' % (boundary)) + msg_set_header(msg, 'Content-Disposition', 'inline') + +def msg_strip_signature(msg): + ''' + Strip signature from msg for now. The formats are horribly different + and proper encrypted mails are signed as part of the encryption. + ''' + ct = msg.get_content_type() + if ct != 'multipart/signed': + return msg + + boundary = msg.get_boundary(None) + payload = msg.get_payload() + stripped = False + + for m in payload: + if m.get_content_type() == 'application/pgp-signature': + payload.remove(m) + stripped = True + + # If no signature found return unmodified msg + if not stripped: + return + + if len(payload) == 1: + # If the remaining message is only a single item set it as payload + msg_set_payload(msg, payload[0]) + else: + # Recreate the multipart message + content = 'Content-type: multipart/mixed; boundary="%s"\n\n' % boundary + for m in payload: + content += '--%s\n' % boundary + content += m.as_string() + content += '\n' + content += '--%s\n' % boundary + msg_set_payload(msg, msg_from_string(content)) + +def msg_strip_html(msg): + ''' + Strip html from msg + ''' + + ct = msg.get_content_type() + if ct != 'multipart/alternative': + return + + boundary = msg.get_boundary(None) + payload = msg.get_payload() + stripped = False + + for m in payload: + if m.get_content_type() == 'text/html': + payload.remove(m) + stripped = True + + # If no html found return + if not stripped: + return + + if len(payload) == 1: + # If the remaining message is only a single item set it as payload + msg_set_payload(msg, payload[0]) + else: + # Recreate the multipart message + content = 'Content-type: multipart/mixed; boundary="%s"\n\n' % boundary + for m in payload: + content += '--%s\n' % boundary + content += m.as_string() + content += '\n' + content += '--%s\n' % boundary + msg_set_payload(msg, msg_from_string(content)) + +def msg_sanitize_outlook(msg): + ''' + Oh well ... + ''' + ct = msg.get_content_type() + if ct != 'multipart/mixed': + return + + # The bogus outlook mails consist of a text/plain and an attachment + payload = msg.get_payload() + if len(payload) != 2: + return + + if payload[0].get_content_type() != 'text/plain': + return + + if payload[1].get_content_type() != 'application/octet-stream': + return + + fname = payload[1].get_filename(None) + if not fname: + return + + if fname != 'msg.gpg' and fname != 'msg.asc': + return + + encpl = payload[1].get_payload() + msg_set_gpg_payload(msg, encpl, 'outlook', addpgp=True) + +def decode_base64(msg): + # + # Decode base64 encoded text/plain sections + # + if msg.get('Content-Transfer-Encoding', '') == 'base64': + dec = base64.decodestring(msg.get_payload().encode()) + msg.set_payload(dec) + del msg['Content-Transfer-Encoding'] + +def decode_alternative(msg): + ''' + Deal with weird MUAs which put the GPG encrypted text/plain + part into a multipart/alternative mail. + ''' + ct = msg.get_content_type() + if ct == 'multipart/alternative': + payloads = msg.get_payload() + payldec = [] + for pl in payloads: + ct = pl.get_content_type() + if ct == 'text/plain': + pl = decode_base64(pl) + payldec.append(pl) + msg.set_payload(payldec) + elif ct == 'text/plain': + decode_base64(msg) + +def msg_sanitize_incoming(msg): + ''' + Get rid of HMTL, outlook, alternatives etc. + ''' + # Strip html multipart first + msg_strip_html(msg) + + # Sanitize outlook crappola + msg_sanitize_outlook(msg) + + # Handle mutlipart/alternative and base64 encodings + decode_alternative(msg) + +re_noreply = re.compile('^no.?reply@') + +def msg_is_autoreply(msg): + ''' + Check whether a message is an autoreply + ''' + # RFC 3834 + ar = msg.get('Auto-Submitted') + if ar and ar != 'no': + return True + + # Microsoft ... + ar = msg.get('X-Auto-Response-Suppress') + if ar in ('DR', 'AutoReply', 'All'): + return True + + # precedence auto-reply + if msg.get('Precedence', '') == 'auto-reply': + return True + + # Reply-To is empty + rt = msg.get('Reply-To') + if rt: + rt = rt.strip() + if len(rt) == 0: + return True + + if rt == '<>': + return True + + # Reply-To matches no_reply, no-reply, noreply + if re_noreply.search(rt): + return True + + # Catch empty return path + rp = msg.get('Return-Path') + if not rp or rp == '<>': + return True + + return False diff --git a/remail/maillist.py b/remail/maillist.py new file mode 100644 index 0000000..6b1b699 --- /dev/null +++ b/remail/maillist.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# Mailing list related code + +from remail.mail import msg_set_header, msg_force_msg_id, send_mail +from remail.mail import msg_sanitize_incoming, msg_is_autoreply +from remail.mail import get_raw_email_addr, decode_addrs +from remail.mail import msg_from_string + +from remail.smime import smime_crypt, RemailSmimeException +from remail.gpg import gpg_crypt, RemailGPGException +from remail.tracking import account_tracking +from remail.config import accounts_config, gpg_config, smime_config + +from email.utils import make_msgid, formatdate +from email.policy import EmailPolicy +from flufl.bounce import all_failures + +import mailbox +import yaml +import os + +class maillist(object): + ''' + A class representing a mailing list + + The list is configured by a preconfigured config item. + ''' + def __init__(self, listcfg, logger, use_smtp): + self.logger = logger + self.config = listcfg + self.enabled = listcfg.enabled + self.use_smtp = use_smtp + + self.smime = smime_crypt(self.config.smime, self.config.listaccount) + self.gpg = gpg_crypt(self.config.gpg, self.config.listaccount) + + self.tracking = account_tracking(self.config.tracking, logger) + + def get_name(self): + return self.config.name + + def start_list(self): + if not self.config.enabled: + return + + # Initilize the tracker and get the accounts which want + # a welcome mail sent to them + welcome = self.tracking.tracking_init(self.config.subscribers) + for acc in welcome: + self.send_welcome_mail(acc) + # Failure to encrypt to the receipient disables the account + # temporarily + if acc.enabled: + self.tracking.enable_account(acc) + + # If exceptions happened tell the administrator + self.handle_log() + + def send_plain_mail(self, msg, account): + ''' + Only ever use for admin mails which contain no content! + ''' + send_mail(msg, account, self.config.listaddrs.owner, + self.config.listaddrs.bounce, {}, self.use_smtp) + + def encrypt(self, msg_plain, account): + ''' + Encrypt plain text message for the account + ''' + msg = msg_from_string(msg_plain.as_string()) + if account.use_smime: + self.smime.encrypt(msg, account) + else: + self.gpg.encrypt(msg, account) + return msg + + def send_encrypted_mail(self, msg_plain, account, mfrom): + try: + msg_out = self.encrypt(msg_plain, account) + send_mail(msg_out, account, mfrom, self.config.listaddrs.bounce, + self.config.listheaders, self.use_smtp) + except (RemailGPGException, RemailSmimeException) as ex: + ''' + GPG and S/MIME exceptions are not fatal. If they happen + then in most cases the key/cert is not valid. Freeze the + subscribers account. The log handling will inform the + administrator about the problem. + ''' + txt = 'Failed to encrypt mail to %s' % (account.addr) + self.logger.log_exception(txt, ex) + if account in self.config.subscribers.values(): + txt = 'Account frozen: %s\n' % (account.addr) + self.logger.log_warn(txt) + account.enabled = False + self.tracking.freeze_account(account) + + def prepare_mail_msg(self, txt): + ''' + Prepare a mail message from a string. Used for sending + template based mails to subscribers and admins. + ''' + msg = msg_from_string(txt) + # Force a message id and set the date header + msg_force_msg_id(msg, self.config.listaddrs.post) + msg_set_header(msg, 'Date', formatdate()) + return msg + + def send_welcome_mail(self, account): + # Read the template and replace the optional $NAME + # placeholder with the subscribers name + txt = open(self.config.templates.welcome).read() + txt = txt.replace('$NAME', account.name) + msg = self.prepare_mail_msg(txt) + self.send_encrypted_mail(msg, account, self.config.listaddrs.post) + + def archive_mail(self, msg, incoming=False, admin=False): + ''' + Archive mail depending on configuration. + ''' + if incoming and self.config.archives.incoming: + f = self.config.archives.m_encr + elif admin and self.config.archives.plain: + f = self.config.archives.m_admin + elif not admin and self.config.archives.plain: + f = self.config.archives.m_list + else: + return + if self.config.archives.mdir: + mbox = mailbox.Maildir(f, create=True) + else: + mbox = mailbox.mbox(f, create=True) + mbox.add(msg) + mbox.close() + + def decrypt_mail(self, msg): + ''' + Decrypt mail after sanitizing it from HTML and outlook magic + and decoding base64 transport. + ''' + msg_sanitize_incoming(msg) + + msg_plain = self.smime.decrypt(msg) + if not msg_plain: + msg_plain = self.gpg.decrypt(msg) + return msg_plain + + def disable_subscribers(self, addrs, msgid): + ''' + Disable subscribers in @addrs. Emit a warning when an address is + newly disabled. Update state tracking. + Admins are informed via log handling + ''' + for addr in addrs: + addr = addr.decode() + acc = self.config.subscribers.get(addr) + if acc: + if acc.enabled: + acc.enabled = False + txt = 'Freezing account due to permanent failure %s\n' % addr + self.logger.log_warn('txt') + self.tracking.freeze_account(acc) + else: + txt = 'Trying to freeze non existing account %s.\n' % addr + txt += ' Message-ID: %s\n' % msgid + self.logger.log_warn(txt) + + def modsubject(self, msg, prefix): + ''' + Add a prefix to the subject. Used for administrator mails to inform + about the nature of the information, e.g. Moderation, Bounces etc. + ''' + subj = prefix + msg['Subject'] + msg_set_header(msg, 'Subject', subj) + + def moderate(self, msg, dest): + ''' + If the list is moderated make sure that the sender of a mail + is subscribed or an administrator. This checks also aliases. + ''' + if not self.config.moderated: + return False + + mfrom = get_raw_email_addr(msg.get('From')) + r = self.config.subscribers.has_account(mfrom) + r = r or self.config.admins.has_account(mfrom) + if not r: + self.modsubject(msg, '[MODERATED] ') + dest.toadmins = True + dest.accounts = self.config.admins + + def check_bounces(self, msg, dest): + ''' + Catch bounces and autoreply messages. + ''' + temp_fail, perm_fail = all_failures(msg) + + # Disable all permanent failing addresses + self.disable_subscribers(perm_fail, msg.get('Message-Id')) + + # If this is a bounce, send it to the admins + if len(temp_fail) or len(perm_fail): + if len(temp_fail): + self.modsubject(msg, '[TEMPFAIL] ') + if len(perm_fail): + self.modsubject(msg, '[FROZEN] ') + dest.toadmins = True + dest.accounts = self.config.admins + + if msg_is_autoreply(msg): + self.modsubject(msg, '[AUTOREPLY] ') + dest.toadmins = True + dest.accounts = self.config.admins + + def mangle_from(self, msg): + ''' + Build 'From' string so the original 'From' is 'visible': + From: $LISTNAME for $ORIGINAL_FROM <$LISTADDRESS> + + If $ORIGINAL_FROM does not contain a name, mangle the email + address by replacing @ with _at_ + ''' + mfrom = msg.get('From').split('<')[0].replace('@', '_at_').strip() + return '%s for %s <%s>' % (self.config.name, mfrom, + self.config.listaddrs.post) + + def do_process_mail(self, msg, dest): + # Archive the incoming mail + self.archive_mail(msg, incoming=True) + # Destination was already established. Check for bounces first + self.check_bounces(msg, dest) + # Check for moderation + self.moderate(msg, dest) + + msgid = msg.get('Message-Id', '') + msgto = msg.get('To') + + try: + msg_plain = self.decrypt_mail(msg) + except Exception as ex: + txt = 'Failed to decrypt incoming %s to %s\n' %(msgid, msgto) + self.logger.log_exception(txt, ex) + return False + + self.archive_mail(msg_plain, admin=dest.toadmin) + + mfrom = self.mangle_from(msg) + + for account in dest.accounts.values(): + if not account.enabled: + continue + self.send_encrypted_mail(msg_plain, account, mfrom) + return True + + def handle_log(self): + ''' + If the logger captured warnings send them to the list admin(s) + ''' + if not len(self.logger.warnings): + return + + txt = open(self.config.templates.admin).read() + txt += '\n\n%s' %self.logger.warnings + msg = self.prepare_mail_msg(txt) + + self.logger.log_debug('Sending warnings to admins\n') + + for account in self.config.admins.values(): + if not account.enabled: + continue + # Use the bounce address as from ... + self.send_plain_mail(msg, account) + self.logger.warnings = '' + + def process_mail(self, msg, dest): + txt = 'Processing %s mail: %s\n' %(self.config.name, msg.get('Message-ID')) + self.logger.log_debug(txt) + res = self.do_process_mail(msg, dest) + # Send out any warning which might have happened to the admins + self.handle_log() + return res + + def get_destination(self, msg): + # Handle the case where someone put several addresses on To: + addrs = decode_addrs(msg['To']) + + for addr in addrs: + to = get_raw_email_addr(addr) + dest = self.config.listaddrs.get_destination(to, self.config.admins, + self.config.subscribers) + if dest: + msg_set_header(msg, 'To', to) + return dest + return None + + def check_keys(self): + ''' + Check and validate subscriber keys + ''' + for account in self.config.subscribers.values(): + if not account.enabled: + continue + if not account.use_smime: + self.gpg.check_key(account) + else: + self.smime.check_cert(account) + +class maillist_checker(object): + ''' + Trivial wrapper around the list to check and show the subscriber + configuration. + ''' + def __init__(self, configfile, logger): + self.logger = logger + try: + cfgdict = yaml.load(open(configfile)) + except Exception as ex: + txt = 'Failed to load list configfile %s' %configfile + logger.log_exception(txt, ex) + return + + self.accounts = accounts_config(cfgdict.get('subscribers', {}), '') + + def show_config(self): + if self.accounts: + print('Subscribers:') + self.accounts.show(2) + + def show_enabled(self): + if not self.accounts: + return + + subs = {} + for account in self.accounts.values(): + if not account.enabled: + continue + subs[account.name] = account.addr + + for name in sorted(subs.keys()): + print('%-40s %s' %(name, subs[name])) + + def check_keys(self): + if not self.accounts: + return + + gpgcfg = gpg_config(os.getcwd(), {}) + gpg = gpg_crypt(gpgcfg, None, checkkey=False) + + smimecfg = smime_config(os.getcwd(), {}) + smime = smime_crypt(smimecfg, None, checkkey=False) + + for account in self.accounts.values(): + if not account.enabled: + continue + try: + if not account.use_smime: + gpg.check_key(account) + else: + smime.check_cert(account) + except Exception as ex: + self.logger.log(str(ex) + '\n') diff --git a/remail/remaild.py b/remail/remaild.py new file mode 100644 index 0000000..b087f4b --- /dev/null +++ b/remail/remaild.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# + +from remail.config import RemailConfigException +from remail.config import main_config +from remail.maillist import maillist + +from email import message_from_binary_file +from email.policy import EmailPolicy +from ruamel.yaml import YAML +import pyinotify +import mailbox +import pathlib +import signal +import fcntl +import os + +class EventHandler(pyinotify.ProcessEvent): + """ + inotify event handler which invokes the mailer function + to enqueue new mail into the mail process queue + """ + def __init__(self, mailer): + self.mailer = mailer + pass + + def process_IN_CREATE(self, event): + self.mailer.queue_newmail(event.pathname) + pass + +class remaild(object): + """ + The remail daemon. + """ + def __init__(self, cfgfile, logger): + self.logger = logger + self.enabled = False + + self.cfgfile = cfgfile + self.yaml = YAML() + + # Maildir related data + self.inotifier = None + self.newmails = [] + self.failedmails = [] + + self.policy = EmailPolicy(utf8=True) + + # Mailing lists + self.mailinglists = [] + + # Signals and related data + self._should_stop = False + self._should_reload = False + self.siginstall() + + # Signals + def term_handler(self, signum, frame): + ''' + SIGTERM/SIGINT handler. Sets the should stop flag and stops the + inotifier which brings the inotify watch out of the wait loop. + ''' + self._should_stop = True + self.stop_inotifier() + + def reload_handler(self, signum, frame): + ''' + SIGTERM handler. Sets the should reload flag and stops the inotifier + which brings the inotify watch out of the wait loop. + ''' + self._should_reload = True + self.stop_inotifier() + + def siginstall(self): + self.sigset = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) + signal.signal(signal.SIGINT, self.term_handler) + signal.signal(signal.SIGTERM, self.term_handler) + signal.signal(signal.SIGHUP, self.reload_handler) + + def sigblock(self): + ''' + Blocks the relevant signals during mail processing. After each + processed mail sigpending() is checked to handle the signals + gracefully without too much delay. + ''' + signal.pthread_sigmask(signal.SIG_BLOCK, self.sigset) + + def sigunblock(self): + ''' + Unblock the signals again + ''' + signal.pthread_sigmask(signal.SIG_UNBLOCK, self.sigset) + + # The remail internal queueing + def queue_newmail(self, path): + ''' + new mail queue. Contains mails which have not been processed. + Scanned from the 'new' folder in the maildir after setting + up the async inotifier to make sure that existing entries are + collected. When the inotify based processing runs new entries + are added when inotify.process_events() is invoked. + + Duplicate entries are prevented. Also mails in failedmails + are ignored. + ''' + if path not in self.failedmails and path not in self.newmails: + self.newmails.append(path) + + def scan_maildir(self): + ''' + Scan all existing mails in maildir/new and maildir/cur (not + in use yet). The scanning happens after the inotifier was + installed and before the inotify events are processed. Duplicate + entries are prevented in the enqeueing function + ''' + mdir = pathlib.Path(self.config.maildir) / 'cur' + for p in mdir.iterdir(): + self.queue_newmail(str(p.resolve())) + mdir = pathlib.Path(self.config.maildir) / 'new' + for p in mdir.iterdir(): + self.queue_newmail(str(p.resolve())) + + # Inotify related functions + def install_inotifier(self): + ''' + Install an async inotifier on maildir/new to avoid polling + the maildir. + ''' + wm = pyinotify.WatchManager() + self.inotifier = pyinotify.AsyncNotifier(wm, EventHandler(self)) + ndir = pathlib.Path(self.config.maildir) / 'new' + wm.add_watch(str(ndir.resolve()), pyinotify.IN_CREATE) + + def stop_inotifier(self): + ''' + Stop the inotifier. Called from signal handlers to stop waiting + and processing either for termination or reconfiguration. + ''' + if self.inotifier: + self.inotifier.stop() + self.inotifier = None + + def process_events(self): + try: + # Can't disable signals here as this can block + if self.inotifier.check_events(): + self.inotifier.read_events(); + + self.inotifier.process_events() + + except AttributeError as ex: + # Handle the case gracefully where the inotifier died + # There is surely a better way to do that :) + txt = '%s' %ex + if txt.find('check_events') > 0: + pass + elif txt.find('read_events') > 0: + pass + elif txt.find('process_events') > 0: + pass + else: + raise ex + + def move_frozen(self, mailfile): + try: + fname = os.path.basename(mailfile) + fpath = os.path.join(self.config.mailfrozen, fname) + if os.path.isfile(mailfile): + os.link(mailfile, fpath) + os.unlink(mailfile) + except: + pass + + # The actual mail processing + def process_mail(self, queue): + ''' + Process the mails which are enqueued on the internal list. Invoked + with signals disabled. After each processed mail the signals are + checked. + ''' + while len(queue) and not self._should_stop and not signal.sigpending(): + mailfile = queue.pop() + + # Try to read the mail + try: + msg = message_from_binary_file(open(mailfile, 'rb'), + policy=self.policy) + except Exception as ex: + self.move_frozen(mailfile) + txt = 'Failed to read mail file %s. Moved to frozen\n' %(mailfile) + self.logger.log_exception(txt, ex) + continue + + # Check whether one of the lists will take it + processed = False + for ml in self.mailinglists: + if not ml.enabled: + continue + dest = ml.get_destination(msg) + if not dest: + continue + try: + processed = ml.process_mail(msg, dest) + break + except Exception as ex: + self.failedmails.append(mailfile) + txt = 'Failed to process mail file %s\n' %(mailfile) + self.logger.log_exception(txt, ex) + break + + if processed: + os.unlink(mailfile) + else: + self.move_frozen(mailfile) + txt = 'Failed to process mail file %s.' %(mailfile) + txt += ' Moved to frozen\n.' + self.logger.log_warn(txt) + + def process_mails(self): + ''' + Block signals for mail processing to simplify the mail processing + as some of the invoked functions are not restarting their syscalls. + Blocking the syscalls avoids dealing with those exceptions all over + the place. The mail processing checks for pending signals after each + mail so the delay for handling them is minimal. + ''' + self.sigblock() + self.process_mail(self.newmails) + self.sigunblock() + + # Configuration processing + def config_list(self, ml): + ''' + Configure a mailing list from the list config file + ''' + try: + cfgdict = self.yaml.load(open(ml.listcfg)) + except Exception as ex: + txt = 'Failed to load list configfile %s' %ml.listcfg + self.logger.log_exception(txt, ex) + txt = 'Disabling list. Waiting for reconfiguration' + self.logger.log_warn(txt) + ml.enabled = False + + # If the list is disabled, nothing else to do than wait + if not ml.enabled: + return + + try: + subscr = cfgdict.get('subscribers', {}) + ml.config_subscribers(subscr) + except RemailConfigException as ex: + txt = 'list %s disabled. Waiting for reconfiguration' % ml.name + self.logger.log_exception(txt, ex) + self.log_warn(txt) + ml.enabled = False + + def read_config(self): + ''' + Read the main configuration file and analyze it. + ''' + try: + cfgdict = self.yaml.load(open(self.cfgfile)) + except Exception as ex: + txt = 'Failed to load configfile %s.' %self.cfgfile + self.logger.log_exception(txt, ex) + txt = 'Waiting for reconfiguration' + self.logger.log_warn(txt) + cfgdict = {} + + + # If remail is disabled, nothing else to do than wait + self.enabled = cfgdict.get('enabled', False) + if not self.enabled: + return + + # Read the configuration + try: + self.config = main_config(cfgdict) + except RemailConfigException as ex: + txt = 'remail disabled. Waiting for reconfiguration' + self.logger.log_exception(txt, ex) + self.enabled = False + + # If remail is disabled, nothing else to do than wait + if not self.enabled: + return + + # Read the mailing list subscribers + for l in self.config.lists: + self.config_list(l) + + # Now set up the real mailing lists + self.mailinglists = [] + for l in self.config.lists: + ml = maillist(l, self.logger, self.config.use_smtp) + self.mailinglists.append(ml) + + def show_config(self): + ''' + Show the configuration of this remail instance including + the mailing list configurations on stdout. + ''' + self.read_config() + if not self.enabled: + print('Disabled') + return + + self.config.show() + + def check_keys(self): + self.read_config() + if not self.enabled: + return + for ml in self.mailinglists: + ml.check_keys() + + def reconfigure(self): + """ + Check for reconfiguration request. + """ + if not self._should_reload: + return + self._should_reload = False + + # Prevent concurrent updates from a notification + # mechanism or a cronjob. + fd = open('lock', 'w') + fcntl.flock(fd, fcntl.LOCK_EX) + + self.read_config() + + if not self.enabled: + return + + for ml in self.mailinglists: + try: + ml.start_list() + except Exception as ex: + txt = 'Failed to start list %s' % (ml.get_name()) + self.logger.log_exception(txt, ex) + ml.enabled = False + + fcntl.flock(fd, fcntl.LOCK_UN) + + def should_stop(self): + return self._should_stop + + def check_config(self): + # Invoke reconfiguration if requested + self.reconfigure() + + # Wait if configuration is disabled + while not self.enabled and not self.should_stop(): + signal.pause() + if not self.should_stop(): + self.reconfigure() + + return self.should_stop() + + # The runner + def run(self): + + # Force configuration reload + self._should_reload = True + + while not self.should_stop(): + + # Check configuration request and eventually wait if disabled + if self.check_config(): + continue + + # Install inotify watch on the maildir + self.install_inotifier() + # Scan for existing mail + self.scan_maildir() + self.process_mails() + + while not self._should_stop and not self._should_reload: + try: + self.process_events() + self.process_mails() + except Exception as ex: + self.stop_inotifier() + self.logger.log_exception('', ex) + self._should_stop = True diff --git a/remail/smime.py b/remail/smime.py new file mode 100644 index 0000000..840f736 --- /dev/null +++ b/remail/smime.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# S/MIME decrypt/encrypt functionality + +from remail.mail import msg_set_payload, get_raw_email_addr +from remail.mail import msg_get_payload_as_string, msg_set_header +from remail.mail import msg_from_string, msg_from_bytes + +from M2Crypto import SMIME, BIO, Rand, X509 +import time +import os + +class RemailSmimeException(Exception): + pass + +class smime_crypt(object): + def __init__(self, smime_cfg, account, checkkey=True): + self.config = smime_cfg + self.smime = SMIME.SMIME() + self.account = account + if self.config.verify: + self.ca_verify = 0 + else: + self.ca_verify = SMIME.PKCS7_NOVERIFY + + if not checkkey: + return + + # Make it explode right away if the account key is missing, broken ... + try: + self.load_account_key() + except Exception as ex: + # SMIME Exceptions are undecodable + txt = 'key or crt of %s not loadable. %s' % (self.account.addr, ex) + raise RemailSmimeException(txt) + + def check_cert(self, account): + addr = account.addr + crt = os.path.join(self.config.list_certs, addr + '.crt') + try: + x509 = X509.load_cert(crt) + subj = x509.get_subject() + nbef = x509.get_not_before() + naft = x509.get_not_after() + except Exception as ex: + txt = 'Account %s. ' % account.addr + txt += 'S/MIME cert %s not loadable' % crt + raise RemailSmimeException(txt) + + txt = '/emailAddress=%s' %account.addr + if str(subj) != txt: + txt = 'Account %s. ' % account.addr + txt += 'S/MIME cert %s is not matching: %s' % (crt, subj) + raise RemailSmimeException(txt) + + val = nbef.get_datetime().timestamp() + now = time.time() + if now < val: + txt = 'Account %s. ' % account.addr + txt += 'S/MIME cert %s not yet valid: %s' % (crt, nbef) + raise RemailSmimeException(txt) + + val = naft.get_datetime().timestamp() + now = time.time() + if now >= val: + txt = 'Account %s. ' % account.addr + txt += 'S/MIME cert %s expired: %s' % (crt, nbef) + raise RemailSmimeException(txt) + + def load_account_key(self): + addr = self.account.addr + key = os.path.join(self.config.global_certs, addr + '.key') + crt = os.path.join(self.config.list_certs, addr + '.crt') + self.smime.load_key(key, crt) + + def smime_is_multipart_signed(self, msg): + ''' + Check whether the message is signed must be verified and decoded + ''' + ct = msg.get_content_type() + if ct == 'multipart/signed': + proto = msg.get_param('protocol', '') + if proto == 'application/x-pkcs7-signature': + return True + if proto == 'application/pkcs7-signature': + return True + return False + + def smime_must_verify(self, msg): + ''' + Check whether the message is signed and must be verified and decoded + ''' + ct = msg.get_content_type() + if ct == 'application/x-pkcs7-mime' or ct == 'application/pkcs7-mime': + if msg.get_param('smime-type', '') == 'signed-data': + return True + else: + return self.smime_is_multipart_signed(msg) + return False + + def smime_verify(self, msg): + ''' + Verify SMIME signed message and return the payload as email.message + ''' + mfrom = get_raw_email_addr(msg['From']) + + crt = os.path.join(self.config.list_certs, mfrom + '.crt') + x509 = X509.load_cert(crt) + sk = X509.X509_Stack() + sk.push(x509) + self.smime.set_x509_stack(sk) + store = X509.X509_Store() + store.load_info(self.config.ca_certs) + self.smime.set_x509_store(store) + p7_bio = BIO.MemoryBuffer(msg.as_bytes()) + p7, data = SMIME.smime_load_pkcs7_bio(p7_bio) + + msgout = self.smime.verify(p7, data, flags=self.ca_verify) + msg_set_header(msg, 'Signature-Id', mfrom) + return msg_from_bytes(msgout) + + def smime_decrypt(self, msg): + ''' + Decrypt SMIME message and replace the payload of the original message + ''' + self.load_account_key() + bio = BIO.MemoryBuffer(msg.as_bytes()) + p7, data = SMIME.smime_load_pkcs7_bio(bio) + msg_plain = msg_from_bytes(self.smime.decrypt(p7)) + + # If the message is signed as well get the content + if self.smime_must_verify(msg_plain): + msg_set_payload(msg, msg_plain) + msg_plain = self.smime_verify(msg) + + msg_set_payload(msg, msg_plain) + + def do_decrypt(self, msg): + ''' + Try to handle received mail with S/MIME. Return the decoded mail or None + ''' + if self.smime_is_multipart_signed(msg): + payload = self.smime_verify(msg) + msg_set_payload(msg, payload) + + ct = msg.get_content_type() + if ct == 'application/pkcs7-mime' or ct == 'application/x-pkcs7-mime': + msgout = msg_from_string(msg.as_string()) + self.smime_decrypt(msgout) + return msgout + + elif self.smime_must_verify(msg): + msgout = msg_from_string(msg.as_string()) + payload = self.smime_verify(msgout) + msg_set_payload(msgout, payload) + return msgout + + return None + + def decrypt(self, msg): + try: + envto = msg.get('To', None) + msgid = msg.get('Message-Id', None) + return self.do_decrypt(msg) + except SMIME.PKCS7_Error as ex: + # SMIME Exceptions are undecodable + txt = 'PKCS7 error when decrypting message ' + txt += '%s to %s: %s' % (msgid, envto, ex) + raise RemailSmimeException(txt) + except SMIME.SMIME_Error as ex: + # SMIME Exceptions are undecodable + txt = 'SMIME error when decrypting message ' + txt += '%s to %s: %s' % (msgid, envto, ex) + raise RemailSmimeException(txt) + except Exception as ex: + txt = 'Error when decrypting message ' + txt += '%s to %s: %s' % (msgid, envto, ex) + raise RemailSmimeException(txt) + + def smime_encrypt(self, msg, to): + ''' + Encrypt a message for a recipient + ''' + # Extract payload for encryption + payload = msg_get_payload_as_string(msg).encode() + + # Sign the content if a signer is set + if self.config.sign: + self.load_account_key() + + sbuf = BIO.MemoryBuffer(payload) + # Sigh. sign() defaults to sha1... + p7 = self.smime.sign(sbuf, algo='sha256') + sbuf.close() + + buf = BIO.MemoryBuffer() + self.smime.write(buf, p7) + else: + buf = BIO.MemoryBuffer(payload) + + # Load target cert to encrypt to. + key = os.path.join(self.config.list_certs, to + '.crt') + x509 = X509.load_cert(key) + sk = X509.X509_Stack() + sk.push(x509) + self.smime.set_x509_stack(sk) + self.smime.set_cipher(SMIME.Cipher('aes_256_cbc')) + + # Encrypt the buffer. + p7 = self.smime.encrypt(buf) + buf.close() + out = BIO.MemoryBuffer() + self.smime.write(out, p7) + encmsg = msg_from_bytes(out.read()) + out.close() + + # Get rid of the old style x-pkcs7 type + ct = encmsg['Content-Type'] + ct = ct.replace('application/x-pkcs7-mime', 'application/pkcs7-mime') + msg_set_header(encmsg, 'Content-Type', ct) + + msg_set_payload(msg, encmsg) + + def encrypt(self, msg, account): + ''' + Encrypt a message for a recipient + ''' + try: + msgid = msg.get('Message-Id', None) + self.smime_encrypt(msg, account.addr) + except SMIME.PKCS7_Error as ex: + # SMIME Exceptions are undecodable + txt = 'PKCS7 error when encrypting message ' + txt += '%s to %s: %s' % (msgid, account.addr, ex) + raise RemailSmimeException(txt) + except SMIME.SMIME_Error as ex: + # SMIME Exceptions are undecodable + txt = 'SMIME error when encrypting message ' + txt += '%s to %s: %s' % (msgid, account.addr, ex) + raise RemailSmimeException(txt) + except Exception as ex: + txt = 'Error when encrypting message ' + txt += '%s to %s: %s' % (msgid, account.addr, ex) + raise RemailSmimeException(txt) + diff --git a/remail/tracking.py b/remail/tracking.py new file mode 100644 index 0000000..66fbc1c --- /dev/null +++ b/remail/tracking.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# Account state tracking + +from ruamel.yaml import YAML + +class RemailTrackerException(Exception): + pass + +class account_tracking(object): + '''Subscriber account tracking in a yaml file which is not part of the + configuration as the configuration is git based and updates to it + cannot be pushed back into the repository without creating a major + trainwreck. + + The local states are: 'DISABLED', 'REGISTERED', 'FROZEN', 'ENABLED' + + The FROZEN state is set by bounce processing and can only be removed by + two consecutive configuration updates (disable/enable) for now. Not + the most pretty mechanism, but everything else turned out to be even + more horrible. + ''' + def __init__(self, trcfg, logger): + self.logger = logger + self.config = trcfg + self.yaml = YAML() + try: + self.tracked = self.yaml.load(open(self.config.tracking_file)) + except: + self.tracked = {} + + def tracking_init(self, subscribers): + ''' + Initialize the state tracker and safe current state + after consolidating it with the configuration file + + Returns a lost of accounts which require to be sent a + welcome mail. + ''' + welcome = [] + for acc in subscribers.values(): + # Check if the account is tracked already + if acc.addr in self.tracked: + # Check the state for consistency + state = self.tracked[acc.addr] + + # Last state was disabled. + if state == 'DISABLED': + # If enabled in the config file set the state to + # registered + if acc.enabled: + state = 'REGISTERED' + welcome.append(acc) + + # Last state was frozen (Bounce processing...) + elif state == 'FROZEN': + # If it's disabled in the config, remove the frozen + # state. Otherwise disable the account in the in memory + # configuration until it is really disabled in the real + # configuration file. See above. + if not acc.enabled: + state = 'DISABLED' + else: + acc.enabled = False + txt = 'Account %s frozen by tracking' % acc.addr + self.logger.log_warn(txt) + + # Last state was enabled or registered + elif state == 'ENABLED' or state == 'REGISTERED': + if not acc.enabled: + state = 'DISABLED' + elif state == 'REGISTERED': + welcome.append(acc) + + # Ooops. Should not happen! + else: + if not acc.enabled: + state = 'DISABLED' + else: + state = 'REGISTERED' + welcome.append(acc) + pass + + # Untracked account + else: + if not acc.enabled: + state = 'DISABLED' + else: + state = 'REGISTERED' + welcome.append(acc) + + # Update the tracking dict + self.tracked[acc.addr] = state + + # Remove all accounts from the tracker which are not longer + # in the subscriber list + for addr in list(self.tracked): + if addr not in subscribers.keys(): + del self.tracked[addr] + + # Update the tracking file + self.dump_tracker() + return welcome + + def dump_tracker(self): + try: + self.yaml.dump(self.tracked, open(self.config.tracking_file, 'w')) + except Exception as ex: + txt = 'Failed to dump to %s: %s' %(self.config.tracking_file, ex) + raise RemailTrackerException(txt) + + def freeze_account(self, acc): + self.tracked[acc.addr] = 'FROZEN' + self.dump_tracker() + + def enable_account(self, acc): + self.tracked[acc.addr] = 'ENABLED' + self.dump_tracker() + + def get_state(self, acc): + return self.tracked[acc.addr] diff --git a/remail/utils.py b/remail/utils.py new file mode 100644 index 0000000..4b0bf29 --- /dev/null +++ b/remail/utils.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# +# Utilities, RemailException and assorted stuff + +import traceback +import syslog +import time +import sys +import os + +class RemailException(Exception): + pass + +class logger(object): + ''' + Logger implementation which can log on stderr or syslog + and provides a verbose mode to emit debug log messages + + Warnings and exceptions are recorded in an internal buffer + which can be sent via mail to the admin(s) + ''' + def __init__(self, use_syslog=False, verbose=False): + self.use_syslog = use_syslog + self.verbose = verbose + self.warnings = '' + self.exceptions = '' + self.syslog_warn = syslog.LOG_MAIL | syslog.LOG_WARNING + self.syslog_info = syslog.LOG_MAIL | syslog.LOG_INFO + self.syslog_debug = syslog.LOG_MAIL | syslog.LOG_DEBUG + + def log_debug(self, txt): + ''' + Debug log. Only active if verbose mode is enabled. + Content is not recorded. + ''' + if self.verbose: + if self.use_syslog: + syslog.syslog(self.syslog_debug, txt) + else: + sys.stderr.write(txt) + + def log(self, txt): + ''' + Regular info log. Content is not recorded. + ''' + if self.use_syslog: + syslog.syslog(self.syslog_info, txt) + else: + sys.stderr.write(txt) + + def log_warn(self, txt): + ''' + Warning log. Content is recorded. + ''' + self.warnings += txt + if self.use_syslog: + syslog.syslog(self.syslog_warn, txt) + else: + sys.stderr.write(txt) + + def log_exception(self, txt, ex, verbose=False): + ''' + Exception log. Content is recorded. In verbose mode the + traceback is recorded as well. + ''' + etxt = '%s: %s' % (type(ex).__name__, ex) + txt = 'REMAIL: %s: %s\n' % (txt, etxt) + if self.verbose or verbose: + txt += '%s\n' % (traceback.format_exc()) + self.exceptions += txt + self.log_warn(txt) + +def mergeconfig(key, cfgdict, defcfg, default): + ''' + Return a merged configuration item. + + @key: key to look up in @cfgdict and @defcfg + @cfgdict: Dictionary which can contain @key + @defcfg: If not None, it can contain an attr named @key + @default: Used if neither @cfgdict nor @defcfg provide an answer + + If @defcfg is not None, then the attr named @key is queried. If + available it replaces @default. + + If @key is in @cfgdict, then the value from @cnfdict is returned + otherwise the default + ''' + if defcfg: + default = getattr(defcfg, key, default) + return cfgdict.get(key, default) + +def makepath(base, path): + ''' + Returns a path built from @base and @path. + + If @path is an absolute path, it is returned unmodified + If @path is a relative path, it is appended to @base + + Both @base and @path are user expanded, i.e ~/ or ~user/ are + expanded to absolute pathes + ''' + path = os.path.expanduser(path) + if os.path.isabs(path): + return path + + base = os.path.expanduser(base) + if not os.path.isabs(base): + base = os.path.abspath(base) + return os.path.join(base, path) diff --git a/remail/version.py b/remail/version.py new file mode 100644 index 0000000..be47fbe --- /dev/null +++ b/remail/version.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner + +__version__ = '0.3' diff --git a/remail_chkcfg b/remail_chkcfg new file mode 100755 index 0000000..e9a6771 --- /dev/null +++ b/remail_chkcfg @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# + +from remail.utils import logger +from remail.version import __version__ +from remail.remaild import remaild +from remail.maillist import maillist_checker + +from argparse import ArgumentParser + +import sys +import os + +if __name__ == '__main__': + parser = ArgumentParser(description='remail daemon') + parser.add_argument('config', help='Config file') + parser.add_argument('--enabled', '-e', dest='enabled', action='store_true', + help='Show alist of enabled accounts. Implies -lnq') + parser.add_argument('--list', '-l', dest='islist', action='store_true', + help='List configuration check') + parser.add_argument('--nokeys', '-n', dest='nokeys', action='store_true', + help='Do not check keys') + parser.add_argument('--quiet', '-q', dest='quiet', action='store_true', + help='Quiet mode. Do not output the parsed config') + parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', + help='Verbose logging') + parser.add_argument('--version', '-V', action='version', + version='%(prog)s {version}'.format(version=__version__)) + args = parser.parse_args() + + if args.enabled: + args.islist = True + args.quiet = True + args.nokeys = True + + logger = logger(use_syslog=False, verbose=args.verbose) + + # Change into the directory in which the config file resides + wdir = os.path.dirname(args.config) + if len(wdir): + os.chdir(wdir) + args.config = os.path.basename(args.config) + + try: + if not args.islist: + rd = remaild(args.config, logger) + else: + rd = maillist_checker(args.config, logger) + except Exception as ex: + ''' + Exceptions which reach here are fatal + ''' + logger.log_exception('', ex) + sys.exit(11) + + try: + if not args.quiet: + rd.show_config() + if not args.nokeys: + rd.check_keys() + if args.enabled: + rd.show_enabled() + res = 0 + except Exception as ex: + ''' + Exceptions which reach here are fatal + ''' + logger.log_exception('', ex) + res = 12 + + sys.exit(res) + diff --git a/remail_daemon b/remail_daemon new file mode 100755 index 0000000..5711cae --- /dev/null +++ b/remail_daemon @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner +# + +from remail.utils import logger +from remail.version import __version__ +from remail.remaild import remaild + +from argparse import ArgumentParser + +import sys +import os + +def exit_daemon(logger, res): + logger.log('Stopped\n') + sys.exit(res) + +if __name__ == '__main__': + parser = ArgumentParser(description='remail daemon') + parser.add_argument('config', help='Config file') + parser.add_argument('--syslog', '-s', dest='syslog', action='store_true', + help='Use syslog for logging. Default is stderr') + parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', + help='Verbose logging') + parser.add_argument('--version', '-V', action='version', + version='%(prog)s {version}'.format(version=__version__)) + args = parser.parse_args() + + logger = logger(use_syslog=args.syslog, verbose=args.verbose) + logger.log("Started\n") + + # Change into the directory in which the config file resides + wdir = os.path.dirname(args.config) + if len(wdir): + os.chdir(wdir) + args.config = os.path.basename(args.config) + + try: + rd = remaild(args.config, logger) + except Exception as ex: + ''' + Exceptions which reach here are fatal + ''' + logger.log_exception('', ex, verbose=True) + exit_daemon(logger, 11) + + try: + res = rd.run() + except Exception as ex: + ''' + Exceptions which reach here are fatal + ''' + logger.log_exception('', ex, verbose=True) + res = 12 + + exit_daemon(logger, res)