diff options
| -rwxr-xr-x | Mailman/Defaults.py.in | 22 | ||||
| -rw-r--r-- | Mailman/Gui/General.py | 42 | ||||
| -rw-r--r-- | Mailman/Gui/Privacy.py | 48 | ||||
| -rw-r--r-- | Mailman/Handlers/CleanseDKIM.py | 17 | ||||
| -rwxr-xr-x | Mailman/Handlers/CookHeaders.py | 10 | ||||
| -rw-r--r-- | Mailman/Handlers/Moderate.py | 30 | ||||
| -rw-r--r-- | Mailman/Handlers/WrapMessage.py | 5 | ||||
| -rwxr-xr-x | Mailman/MailList.py | 4 | ||||
| -rw-r--r-- | Mailman/Utils.py | 97 | ||||
| -rw-r--r-- | Mailman/Version.py | 4 | ||||
| -rwxr-xr-x | Mailman/versions.py | 5 | ||||
| -rwxr-xr-x | NEWS | 29 |
12 files changed, 265 insertions, 48 deletions
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 3de69d74..c04ba8fa 100755 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -108,10 +108,6 @@ ALLOW_SITE_ADMIN_COOKIES = No # expire that many seconds following their last use. AUTHENTICATION_COOKIE_LIFETIME = 0 -# The following must be set to Yes to enable the 'from is list' feature. -# See DEFAULT_FROM_IS_LIST below. -ALLOW_FROM_IS_LIST = No - # Form lifetime is set against Cross Site Request Forgery. FORM_LIFETIME = hours(1) @@ -1064,6 +1060,20 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No # moderators? DEFAULT_FORWARD_AUTO_DISCARDS = Yes +# Shall dmarc_moderation_action be applied to messages From: domains with +# a DMARC policy of quarantine as well as reject? +DMARC_QUARANTINE_MODERATION_ACTION = Yes + +# Default action for posts whose From: address domain has a DMARC policy of +# reject or quarantine. See DEFAULT_FROM_IS_LIST below. Whatever is set as +# the default here precludes the list owner from setting a lower value. +# 0 = Accept +# 1 = Munge From +# 2 = Wrap Message +# 3 = Reject +# 4 = Discard +DEFAULT_DMARC_MODERATION_ACTION = 0 + # What shold happen to non-member posts which are do not match explicit # non-member actions? # 0 = Accept @@ -1101,7 +1111,9 @@ DEFAULT_SEND_WELCOME_MSG = Yes # Send goodbye messages to unsubscribed members? DEFAULT_SEND_GOODBYE_MSG = Yes -# The following is a three way setting. +# The following is a three way setting. It sets the default for the list's +# from_is_list policy which is applied to all posts except those for which a +# dmarc_moderation_action other than accept applies. # 0 -> Do not rewrite the From: or wrap the message. # 1 -> Rewrite the From: header of posts replacing the posters address with # that of the list. Also see REMOVE_DKIM_HEADERS above. diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py index 24bc009a..a917642c 100644 --- a/Mailman/Gui/General.py +++ b/Mailman/Gui/General.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -153,25 +153,27 @@ class General(GUIBase): directive. eg.; [listname %%d] -> [listname 123] (listname %%05d) -> (listname 00123) """)), - ] - if mm_cfg.ALLOW_FROM_IS_LIST: - rtn.append( - ('from_is_list', mm_cfg.Radio, - (_('No'), _('Mung From'), _('Wrap Message')), 0, - _("""Replace the sender with the list address to conform with - policies like ADSP and DMARC. It replaces the poster's - address in the From: header with the list address and adds the - poster to the Reply-To: header, but the anonymous_list and - Reply-To: header munging settings below take priority. If - setting this to Yes, it is advised to set the MTA to DKIM sign - all emails.""") + - _("""<br>If this is set to Wrap Message, just wrap the message - in an outer message From: the list with Content-Type: - message/rfc822.""")) - ) - - rtn.extend([ + ('from_is_list', mm_cfg.Radio, + (_('No'), _('Munge From'), _('Wrap Message')), 0, + _("""Replace the sender with the list address to conform with + policies like DMARC."""), + _("""Replace the sender with the list address to conform with + policies like ADSP and DMARC. It replaces the poster's + address in the From: header with the list address and adds the + poster to the Reply-To: header, but the anonymous_list and + Reply-To: header munging settings below take priority. If + setting this to Yes, it is advised to set the MTA to DKIM sign + all emails.""") + + _("""<p>If this is set to Wrap Message, just wrap the message + in an outer message From: the list with Content-Type: + message/rfc822.""") + + _("""<p>If <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a> applies to this message with an + action other than Accept, that action rather than this is + applied""")), + ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Hide the sender of a message, replacing it with the list address (Removes From, Sender and Reply-To fields)""")), @@ -392,7 +394,7 @@ class General(GUIBase): useful for selecting among alternative names of a host that has multiple addresses.""")), - ]) + ] if mm_cfg.ALLOW_RFC2369_OVERRIDES: rtn.append( diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py index 75eff2b5..5dcc3f48 100644 --- a/Mailman/Gui/Privacy.py +++ b/Mailman/Gui/Privacy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -158,6 +158,11 @@ class Privacy(GUIBase): ] adminurl = mlist.GetScriptURL('admin', absolute=1) + + if mm_cfg.DMARC_QUARANTINE_MODERATION_ACTION: + quarantine = _('/Quarantine') + else: + quarantine = '' sender_rtn = [ _("""When a message is posted to the list, a series of moderation steps are taken to decide whether a moderator must @@ -235,6 +240,42 @@ class Privacy(GUIBase): >rejection notice</a> to be sent to moderated members who post to this list.""")), + ('dmarc_moderation_action', mm_cfg.Radio, + (_('Accept'), _('Wrap Message'), _('Munge From'), _('Reject'), + _('Discard')), 0, + _("""Action to take when anyone posts to the + list from a domain with a DMARC Reject%(quarantine)s Policy."""), + + _("""<ul><li><b>Wrap Message</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is _list Wrap + Message</a> transformation to these messages. + + <p><li><b>Munge From</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is _list Munge From</a> + transformation to these messages. + + <p><li><b>Reject</b> -- this automatically rejects the message by + sending a bounce notice to the post's author. The text of the + bounce notice can be <a + href="?VARHELP=privacy/sender/dmarc_moderation_notice" + >configured by you</a>. + + <p><li><b>Discard</b> -- this simply discards the message, with + no notice sent to the post's author. + </ul> + + <p>This setting takes precedence over the <a + href="?VARHELP=general/from_is_list"> from_is_list</a> setting + if the message is From: an affected domain and the setting is + other than Accept.""")), + + ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1, + _("""Text to include in any + <a href="?VARHELP=privacy/sender/dmarc_moderation_action" + >rejection notice</a> to + be sent to anyone who posts to this list from a domain + with DMARC Reject/Quarantine Policy.""")), + _('Non-member filters'), ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1, @@ -442,6 +483,11 @@ class Privacy(GUIBase): # an option. if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE: val += 1 + if (property == 'dmarc_moderation_action' and + val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION): + doc.addError(_("""dmarc_moderation_action must be >= the configured + default value.""")) + val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION setattr(mlist, property, val) # We need to handle the header_filter_rules widgets specially, but diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py index 0df2d97f..5a83a2e0 100644 --- a/Mailman/Handlers/CleanseDKIM.py +++ b/Mailman/Handlers/CleanseDKIM.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006-2013 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -31,11 +31,12 @@ from Mailman import mm_cfg def process(mlist, msg, msgdata): if not mm_cfg.REMOVE_DKIM_HEADERS: return - if (mm_cfg.ALLOW_FROM_IS_LIST and - mm_cfg.REMOVE_DKIM_HEADERS == 1 and - mlist.from_is_list != 1): - return - del msg['domainkey-signature'] - del msg['dkim-signature'] - del msg['authentication-results'] + if (mm_cfg.REMOVE_DKIM_HEADERS == 1 and + (msgdata.get('from_is_list') == 1 or + (mlist.from_is_list == 1 and msgdata.get('from_is_list') != 2) + ) + ): + del msg['domainkey-signature'] + del msg['dkim-signature'] + del msg['authentication-results'] diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py index 110302ea..83f278b4 100755 --- a/Mailman/Handlers/CookHeaders.py +++ b/Mailman/Handlers/CookHeaders.py @@ -65,8 +65,8 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): return Header(s, charset, maxlinelen, header_name, continuation_ws) def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True): - if (mm_cfg.ALLOW_FROM_IS_LIST and - mlist.from_is_list == 2 and + if ((msgdata.get('from_is_list') == 2 or + (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and not msgdata.get('_fasttrack') ): msgdata.setdefault('add_header', {})[name] = value @@ -119,7 +119,7 @@ def process(mlist, msg, msgdata): change_header('Precedence', 'list', mlist, msg, msgdata, repl=False) # Do we change the from so the list takes ownership of the email - if mm_cfg.ALLOW_FROM_IS_LIST and mlist.from_is_list and not fasttrack: + if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack: realname, email = parseaddr(msg['from']) if not realname: if mlist.isMember(email): @@ -200,8 +200,8 @@ def process(mlist, msg, msgdata): # is already in From and Reply-To in this case and similarly for # a 'from is list' list. if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \ - and not mlist.anonymous_list and not (mlist.from_is_list and - mm_cfg.ALLOW_FROM_IS_LIST): + and not mlist.anonymous_list and not (mlist.from_is_list or + msgdata.get('from_is_list')): # Watch out for existing Cc headers, merge, and remove dups. Note # that RFC 2822 says only zero or one Cc header is allowed. new = [] diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py index 42dffb88..d901eb59 100644 --- a/Mailman/Handlers/Moderate.py +++ b/Mailman/Handlers/Moderate.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,6 +21,7 @@ import re from email.MIMEMessage import MIMEMessage from email.MIMEText import MIMEText +from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Utils @@ -49,7 +50,32 @@ class ModeratedMemberPost(Hold.ModeratedPost): def process(mlist, msg, msgdata): if msgdata.get('approved'): return - # First of all, is the poster a member or not? + # Before anything else, check DMARC. + msgdata['from_is_list'] = 0 + dn, addr = parseaddr(msg.get('from')) + if addr: + if Utils.IsDMARCProhibited(addr): + # Note that for dmarc_moderation_action, 0 = Accept, + # 1 = Wrap, 2 = Munge, 3 = Reject, 4 = Discard + if mlist.dmarc_moderation_action == 1: + msgdata['from_is_list'] = 2 + elif mlist.dmarc_moderation_action == 2: + msgdata['from_is_list'] = 1 + elif mlist.dmarc_moderation_action == 3: + # Reject + text = mlist.dmarc_moderation_notice + if text: + text = Utils.wrap(text) + else: + text = Utils.wrap(_( +"""You are not allowed to post to this mailing list From: a domain which +publishes a DMARC policy of reject or quarantine, and your message has been +automatically rejected. If you think that your messages are being rejected in +error, contact the mailing list owner at %(listowner)s.""")) + raise Errors.RejectMessage, text + elif mlist.dmarc_moderation_action == 4: + raise Errors.DiscardMessage + # Then, is the poster a member or not? for sender in msg.get_senders(): if mlist.isMember(sender): break diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py index 68c89ff2..de981dd6 100644 --- a/Mailman/Handlers/WrapMessage.py +++ b/Mailman/Handlers/WrapMessage.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 by the Free Software Foundation, Inc. +# Copyright (C) 2013-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -35,7 +35,8 @@ KEEPERS = ('to', def process(mlist, msg, msgdata): - if not mm_cfg.ALLOW_FROM_IS_LIST or mlist.from_is_list != 2: + if not (msgdata.get('from_is_list') == 2 or + (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)): return # There are various headers in msg that we don't want, so we basically diff --git a/Mailman/MailList.py b/Mailman/MailList.py index cdebb507..bc771f4c 100755 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -389,6 +389,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # 2==Discard self.member_moderation_action = 0 self.member_moderation_notice = '' + self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION + self.dmarc_moderation_notice = '' self.accept_these_nonmembers = [] self.hold_these_nonmembers = [] self.reject_these_nonmembers = [] diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 0a20423a..89b7975e 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -35,6 +35,7 @@ import errno import base64 import random import urlparse +import collections import htmlentitydefs import email.Header import email.Iterators @@ -71,6 +72,13 @@ except NameError: True = 1 False = 0 +try: + import dns.resolver + from dns.exception import DNSException + dns_resolver = True +except ImportError: + dns_resolver = False + EMPTYSTRING = '' UEMPTYSTRING = u'' NL = '\n' @@ -1058,3 +1066,90 @@ def suspiciousHTML(html): else: return False + +# This takes an email address, and returns True if DMARC policy is p=reject +# or possibly quarantine. +def IsDMARCProhibited(email): + if not dns_resolver: + return False + + email = email.lower() + at_sign = email.find('@') + if at_sign < 1: + return False + dmarc_domain = '_dmarc.' + email[at_sign+1:] + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = 3 + resolver.lifetime = 5 + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return False + except DNSException, e: + syslog('error', + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__class__) + return False + else: +# people are already being dumb, don't trust them to provide honest DNS +# where the answer section only contains what was asked for, nor to include +# CNAMEs before the values they point to. + full_record = "" + results_by_name = collections.defaultdict(list) + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name[txt_rec.name.to_text()].append( + "".join(txt_rec.items[0].strings)) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + syslog('error', + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = filter(lambda n: n.startswith('v=DMARC1;'), + results_by_name[name]) + if len(dmarcs) == 0: + return False + if len(dmarcs) > 1: + syslog('error', + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarc)) + for entry in dmarcs: + if re.search(r'\bp=reject\b', entry, re.IGNORECASE): +# syslog('info', +# 'DMARC lookup for %s (%s) found p=reject in %s = %s', +# email, dmarc_domain, name, entry) + return True + + if (mm_cfg.DMARC_QUARANTINE_MODERATION_ACTION and + re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)): +# syslog('info', +# 'DMARC lookup for %s (%s) found p=quarantine in %s = %s', +# email, dmarc_domain, name, entry) + return True + + return False + + diff --git a/Mailman/Version.py b/Mailman/Version.py index 8897164f..185ff888 100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) # config.pck schema version number -DATA_FILE_VERSION = 102 +DATA_FILE_VERSION = 103 # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 diff --git a/Mailman/versions.py b/Mailman/versions.py index 31c4f470..a568e8e6 100755 --- a/Mailman/versions.py +++ b/Mailman/versions.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -388,6 +388,9 @@ def NewVars(l): # the current GUI description model. So, 0==Hold, 1==Reject, 2==Discard add_only_if_missing('member_moderation_action', 0) add_only_if_missing('member_moderation_notice', '') + add_only_if_missing('dmarc_moderation_action', + mm_cfg.DEFAULT_DMARC_MODERATION_ACTION) + add_only_if_missing('dmarc_moderation_notice', '') add_only_if_missing('new_member_options', mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS) # Emergency moderation flag @@ -7,6 +7,35 @@ Here is a history of user visible changes to Mailman. 2.1.18 (xx-xxx-xxxx) + Dependencies + + - There is a new dependency associated with the new Privacy options -> + Sender filters -> dmarc_moderation_action feature discussed below. + This requires that the dnspython <http://www.dnspython.org/> package + be available in Python. + + New Features + + - The from_is_list feature introduced in 2.1.16 is now unconditionally + available to list owners. There is also, a new Privacy options -> + Sender filters -> dmarc_moderation_action feature which applies to list + messages where the From: address is in a domain which publishes a DMARC + policy of reject or possibly quarantine. This is a list setting with + values of Accept, Wrap Message, Munge From, Reject or Discard. There is + a new DEFAULT_DMARC_MODERATION_ACTION configuration setting to set the + default for this, and the list admin UI is not able to set an action + which is 'less' than the default. The prior ALLOW_FROM_IS_LIST setting + has been removed and is effectively always Yes. There is a new + DMARC_QUARANTINE_MODERATION_ACTION configuration setting which defaults + to Yes but can be set to No to exclude domains with DMARC policy of + quarantine from dmarc_moderation_action. + + dmarc_moderation_action and from_is_list interact in the following way. + If the message is From: a domain to which dmarc_moderation_action applies + and if dmarc_moderation_action is other than Accept, + dmarc_moderation_action applies to that message. Otherwise the + from_is_list action applies. + i18n - Added missing <mm-digest-question-start> tag to French listinfo template. |