Added and modified various options regarding DMARC. See the NEWS file.

11 files changed, 236 insertions, 48 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 3de69d74..c04ba8fa 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -108,10 +108,6 @@ ALLOW_SITE_ADMIN_COOKIES = No
 # expire that many seconds following their last use.
 AUTHENTICATION_COOKIE_LIFETIME = 0
 
-# The following must be set to Yes to enable the 'from is list' feature.
-# See DEFAULT_FROM_IS_LIST below.
-ALLOW_FROM_IS_LIST = No
-
 # Form lifetime is set against Cross Site Request Forgery.
 FORM_LIFETIME = hours(1)
 
@@ -1064,6 +1060,20 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No
 # moderators?
 DEFAULT_FORWARD_AUTO_DISCARDS = Yes
 
+# Shall dmarc_moderation_action be applied to messages From: domains with
+# a DMARC policy of quarantine as well as reject?
+DMARC_QUARANTINE_MODERATION_ACTION = Yes
+
+# Default action for posts whose From: address domain has a DMARC policy of
+# reject or quarantine.  See DEFAULT_FROM_IS_LIST below.  Whatever is set as
+# the default here precludes the list owner from setting a lower value.
+# 0 = Accept
+# 1 = Munge From
+# 2 = Wrap Message
+# 3 = Reject
+# 4 = Discard
+DEFAULT_DMARC_MODERATION_ACTION = 0
+
 # What shold happen to non-member posts which are do not match explicit
 # non-member actions?
 # 0 = Accept
@@ -1101,7 +1111,9 @@ DEFAULT_SEND_WELCOME_MSG = Yes
 # Send goodbye messages to unsubscribed members?
 DEFAULT_SEND_GOODBYE_MSG = Yes
 
-# The following is a three way setting.
+# The following is a three way setting.  It sets the default for the list's
+# from_is_list policy which is applied to all posts except those for which a
+# dmarc_moderation_action other than accept applies.
 # 0 -> Do not rewrite the From: or wrap the message.
 # 1 -> Rewrite the From: header of posts replacing the posters address with
 #      that of the list.  Also see REMOVE_DKIM_HEADERS above.
diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py
index 24bc009a..a917642c 100644
--- a/Mailman/Gui/General.py
+++ b/Mailman/Gui/General.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -153,25 +153,27 @@ class General(GUIBase):
              directive. eg.; [listname %%d] -> [listname 123]
                             (listname %%05d) -> (listname 00123)
              """)),
-          ]
 
-        if mm_cfg.ALLOW_FROM_IS_LIST:
-            rtn.append(
-                ('from_is_list', mm_cfg.Radio,
-                 (_('No'), _('Mung From'), _('Wrap Message')), 0,
-                 _("""Replace the sender with the list address to conform with
-                 policies like ADSP and DMARC.  It replaces the poster's
-                 address in the From: header with the list address and adds the
-                 poster to the Reply-To: header, but the anonymous_list and
-                 Reply-To: header munging settings below take priority.  If
-                 setting this to Yes, it is advised to set the MTA to DKIM sign
-                 all emails.""") +
-                 _("""<br>If this is set to Wrap Message, just wrap the message
-                 in an outer message From: the list with Content-Type:
-                 message/rfc822."""))
-              )
-
-        rtn.extend([
+            ('from_is_list', mm_cfg.Radio,
+             (_('No'), _('Munge From'), _('Wrap Message')), 0,
+             _("""Replace the sender with the list address to conform with
+             policies like DMARC."""),
+             _("""Replace the sender with the list address to conform with
+             policies like ADSP and DMARC.  It replaces the poster's
+             address in the From: header with the list address and adds the
+             poster to the Reply-To: header, but the anonymous_list and
+             Reply-To: header munging settings below take priority.  If
+             setting this to Yes, it is advised to set the MTA to DKIM sign
+             all emails.""") +
+             _("""<p>If this is set to Wrap Message, just wrap the message
+             in an outer message From: the list with Content-Type:
+             message/rfc822.""") +
+             _("""<p>If <a
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
+             dmarc_moderation_action</a> applies to this message with an
+             action other than Accept, that action rather than this is
+             applied""")),
+
             ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0,
              _("""Hide the sender of a message, replacing it with the list
              address (Removes From, Sender and Reply-To fields)""")),
@@ -392,7 +394,7 @@ class General(GUIBase):
              useful for selecting among alternative names of a host that has
              multiple addresses.""")),
 
-          ])
+          ]
 
         if mm_cfg.ALLOW_RFC2369_OVERRIDES:
             rtn.append(
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
index 75eff2b5..5dcc3f48 100644
--- a/Mailman/Gui/Privacy.py
+++ b/Mailman/Gui/Privacy.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -158,6 +158,11 @@ class Privacy(GUIBase):
             ]
 
         adminurl = mlist.GetScriptURL('admin', absolute=1)
+    
+        if mm_cfg.DMARC_QUARANTINE_MODERATION_ACTION:
+            quarantine = _('/Quarantine')
+        else:
+            quarantine = ''
         sender_rtn = [
             _("""When a message is posted to the list, a series of
             moderation steps are taken to decide whether a moderator must
@@ -235,6 +240,42 @@ class Privacy(GUIBase):
              >rejection notice</a> to
              be sent to moderated members who post to this list.""")),
 
+            ('dmarc_moderation_action', mm_cfg.Radio,
+             (_('Accept'), _('Wrap Message'), _('Munge From'), _('Reject'),
+                 _('Discard')), 0,
+             _("""Action to take when anyone posts to the
+             list from a domain with a DMARC Reject%(quarantine)s Policy."""),
+
+             _("""<ul><li><b>Wrap Message</b> -- applies the <a
+             href="?VARHELP=general/from_is_list">from_is _list Wrap
+             Message</a> transformation to these messages.
+
+             <p><li><b>Munge From</b> -- applies the <a
+             href="?VARHELP=general/from_is_list">from_is _list Munge From</a>
+             transformation to these messages.
+
+             <p><li><b>Reject</b> -- this automatically rejects the message by
+             sending a bounce notice to the post's author.  The text of the
+             bounce notice can be <a
+             href="?VARHELP=privacy/sender/dmarc_moderation_notice"
+             >configured by you</a>.
+
+             <p><li><b>Discard</b> -- this simply discards the message, with
+             no notice sent to the post's author.
+             </ul>
+
+             <p>This setting takes precedence over the <a
+             href="?VARHELP=general/from_is_list"> from_is_list</a> setting
+             if the message is From: an affected domain and the setting is
+             other than Accept.""")),
+
+            ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1,
+             _("""Text to include in any
+             <a href="?VARHELP=privacy/sender/dmarc_moderation_action"
+             >rejection notice</a> to
+             be sent to anyone who posts to this list from a domain
+             with DMARC Reject/Quarantine Policy.""")),
+
             _('Non-member filters'),
 
             ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1,
@@ -442,6 +483,11 @@ class Privacy(GUIBase):
         # an option.
         if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE:
             val += 1
+        if (property == 'dmarc_moderation_action' and
+                val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION):
+            doc.addError(_("""dmarc_moderation_action must be >= the configured
+                           default value."""))
+            val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
         setattr(mlist, property, val)
 
     # We need to handle the header_filter_rules widgets specially, but
diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py
index 0df2d97f..5a83a2e0 100644
--- a/Mailman/Handlers/CleanseDKIM.py
+++ b/Mailman/Handlers/CleanseDKIM.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -31,11 +31,12 @@ from Mailman import mm_cfg
 def process(mlist, msg, msgdata):
     if not mm_cfg.REMOVE_DKIM_HEADERS:
         return
-    if (mm_cfg.ALLOW_FROM_IS_LIST and
-            mm_cfg.REMOVE_DKIM_HEADERS == 1 and
-            mlist.from_is_list != 1):
-        return
-    del msg['domainkey-signature']
-    del msg['dkim-signature']
-    del msg['authentication-results']
+    if (mm_cfg.REMOVE_DKIM_HEADERS == 1 and
+           (msgdata.get('from_is_list') == 1 or
+            (mlist.from_is_list == 1 and msgdata.get('from_is_list') != 2)
+           )
+       ):
+        del msg['domainkey-signature']
+        del msg['dkim-signature']
+        del msg['authentication-results']
 
diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
index 110302ea..83f278b4 100755
--- a/Mailman/Handlers/CookHeaders.py
+++ b/Mailman/Handlers/CookHeaders.py
@@ -65,8 +65,8 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None):
     return Header(s, charset, maxlinelen, header_name, continuation_ws)
 
 def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True):
-    if (mm_cfg.ALLOW_FROM_IS_LIST and
-        mlist.from_is_list == 2 and
+    if ((msgdata.get('from_is_list') == 2 or
+        (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and 
         not msgdata.get('_fasttrack')
        ):
         msgdata.setdefault('add_header', {})[name] = value
@@ -119,7 +119,7 @@ def process(mlist, msg, msgdata):
     change_header('Precedence', 'list',
                   mlist, msg, msgdata, repl=False)
     # Do we change the from so the list takes ownership of the email
-    if mm_cfg.ALLOW_FROM_IS_LIST and mlist.from_is_list and not fasttrack:
+    if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack:
         realname, email = parseaddr(msg['from'])
         if not realname:
             if mlist.isMember(email):
@@ -200,8 +200,8 @@ def process(mlist, msg, msgdata):
         # is already in From and Reply-To in this case and similarly for
         # a 'from is list' list.
         if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \
-           and not mlist.anonymous_list and not (mlist.from_is_list and
-                                                 mm_cfg.ALLOW_FROM_IS_LIST):
+           and not mlist.anonymous_list and not (mlist.from_is_list or
+                                                 msgdata.get('from_is_list')):
             # Watch out for existing Cc headers, merge, and remove dups.  Note
             # that RFC 2822 says only zero or one Cc header is allowed.
             new = []
diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py
index 42dffb88..d901eb59 100644
--- a/Mailman/Handlers/Moderate.py
+++ b/Mailman/Handlers/Moderate.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,6 +21,7 @@
 import re
 from email.MIMEMessage import MIMEMessage
 from email.MIMEText import MIMEText
+from email.Utils import parseaddr
 
 from Mailman import mm_cfg
 from Mailman import Utils
@@ -49,7 +50,32 @@ class ModeratedMemberPost(Hold.ModeratedPost):
 def process(mlist, msg, msgdata):
     if msgdata.get('approved'):
         return
-    # First of all, is the poster a member or not?
+    # Before anything else, check DMARC.
+    msgdata['from_is_list'] = 0
+    dn, addr = parseaddr(msg.get('from'))
+    if addr:
+        if Utils.IsDMARCProhibited(addr):
+            # Note that for dmarc_moderation_action, 0 = Accept, 
+            #    1 = Wrap, 2 = Munge, 3 = Reject, 4 = Discard
+            if mlist.dmarc_moderation_action == 1:
+                msgdata['from_is_list'] = 2
+            elif mlist.dmarc_moderation_action == 2:
+                msgdata['from_is_list'] = 1
+            elif mlist.dmarc_moderation_action == 3:
+                # Reject
+                text = mlist.dmarc_moderation_notice
+                if text:
+                    text = Utils.wrap(text)
+                else:
+                    text = Utils.wrap(_(
+"""You are not allowed to post to this mailing list From: a domain which
+publishes a DMARC policy of reject or quarantine, and your message has been
+automatically rejected.  If you think that your messages are being rejected in
+error, contact the mailing list owner at %(listowner)s."""))
+                raise Errors.RejectMessage, text
+            elif mlist.dmarc_moderation_action == 4:
+                raise Errors.DiscardMessage
+    # Then, is the poster a member or not?
     for sender in msg.get_senders():
         if mlist.isMember(sender):
             break
diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py
index 68c89ff2..de981dd6 100644
--- a/Mailman/Handlers/WrapMessage.py
+++ b/Mailman/Handlers/WrapMessage.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 by the Free Software Foundation, Inc.
+# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -35,7 +35,8 @@ KEEPERS = ('to',
 
 
 def process(mlist, msg, msgdata):
-    if not mm_cfg.ALLOW_FROM_IS_LIST or mlist.from_is_list != 2:
+    if not (msgdata.get('from_is_list') == 2 or
+            (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
         return
 
     # There are various headers in msg that we don't want, so we basically
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index cdebb507..bc771f4c 100755
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -389,6 +389,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         # 2==Discard
         self.member_moderation_action = 0
         self.member_moderation_notice = ''
+        self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
+        self.dmarc_moderation_notice = ''
         self.accept_these_nonmembers = []
         self.hold_these_nonmembers = []
         self.reject_these_nonmembers = []
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 0a20423a..89b7975e 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -35,6 +35,7 @@ import errno
 import base64
 import random
 import urlparse
+import collections
 import htmlentitydefs
 import email.Header
 import email.Iterators
@@ -71,6 +72,13 @@ except NameError:
     True = 1
     False = 0
 
+try:
+    import dns.resolver
+    from dns.exception import DNSException
+    dns_resolver = True
+except ImportError:
+    dns_resolver = False
+
 EMPTYSTRING = ''
 UEMPTYSTRING = u''
 NL = '\n'
@@ -1058,3 +1066,90 @@ def suspiciousHTML(html):
     else:
         return False
 
+
+# This takes an email address, and returns True if DMARC policy is p=reject
+# or possibly quarantine.
+def IsDMARCProhibited(email):
+    if not dns_resolver:
+         return False
+
+    email = email.lower()
+    at_sign = email.find('@')
+    if at_sign < 1:
+        return False
+    dmarc_domain = '_dmarc.' + email[at_sign+1:]
+
+    try:
+        resolver = dns.resolver.Resolver()
+        resolver.timeout = 3
+        resolver.lifetime = 5
+        txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
+    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+        return False
+    except DNSException, e:
+        syslog('error',
+               'DNSException: Unable to query DMARC policy for %s (%s). %s',
+              email, dmarc_domain, e.__class__)
+        return False
+    else:
+# people are already being dumb, don't trust them to provide honest DNS
+# where the answer section only contains what was asked for, nor to include
+# CNAMEs before the values they point to.
+        full_record = ""
+        results_by_name = collections.defaultdict(list)
+        cnames = {}
+        want_names = set([dmarc_domain + '.'])
+        for txt_rec in txt_recs.response.answer:
+            if txt_rec.rdtype == dns.rdatatype.CNAME:
+                cnames[txt_rec.name.to_text()] = (
+                    txt_rec.items[0].target.to_text())
+            if txt_rec.rdtype != dns.rdatatype.TXT:
+                continue
+            results_by_name[txt_rec.name.to_text()].append(
+                "".join(txt_rec.items[0].strings))
+        expands = list(want_names)
+        seen = set(expands)
+        while expands:
+            item = expands.pop(0)
+            if item in cnames:
+                if cnames[item] in seen:
+                    continue # cname loop
+                expands.append(cnames[item])
+                seen.add(cnames[item])
+                want_names.add(cnames[item])
+                want_names.discard(item)
+
+        if len(want_names) != 1:
+            syslog('error',
+                   """multiple DMARC entries in results for %s,
+                   processing each to be strict""",
+                   dmarc_domain)
+        for name in want_names:
+            if name not in results_by_name:
+                continue
+            dmarcs = filter(lambda n: n.startswith('v=DMARC1;'),
+                            results_by_name[name])
+            if len(dmarcs) == 0:
+                return False
+            if len(dmarcs) > 1:
+                syslog('error',
+                       """RRset of TXT records for %s has %d v=DMARC1 entries;
+                       testing them all""",
+                        dmarc_domain, len(dmarc))
+            for entry in dmarcs:
+                if re.search(r'\bp=reject\b', entry, re.IGNORECASE):
+#                   syslog('info',
+#                       'DMARC lookup for %s (%s) found p=reject in %s = %s',
+#                       email, dmarc_domain, name, entry)
+                    return True
+
+                if (mm_cfg.DMARC_QUARANTINE_MODERATION_ACTION and
+                    re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)):
+#                   syslog('info',
+#                     'DMARC lookup for %s (%s) found p=quarantine in %s = %s',
+#                           email, dmarc_domain, name, entry)
+                    return True
+
+    return False
+
+
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 8897164f..185ff888 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 102
+DATA_FILE_VERSION = 103
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 31c4f470..a568e8e6 100755
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -388,6 +388,9 @@ def NewVars(l):
     # the current GUI description model.  So, 0==Hold, 1==Reject, 2==Discard
     add_only_if_missing('member_moderation_action', 0)
     add_only_if_missing('member_moderation_notice', '')
+    add_only_if_missing('dmarc_moderation_action', 
+                        mm_cfg.DEFAULT_DMARC_MODERATION_ACTION)
+    add_only_if_missing('dmarc_moderation_notice', '')
     add_only_if_missing('new_member_options',
                         mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS)
     # Emergency moderation flag