Update to lp:mailman/2.1 rev 1629 (no difference against the original branch

9 files changed, 167 insertions, 29 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 8c5d9e7b..04d7db8a 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -136,7 +136,7 @@ SUBSCRIBE_FORM_MIN_TIME = seconds(5)
 # in the installation.  This supplements the individual list's ban_list.
 # For example, to ban xxx@aol.com and any @gmail.com address beginning with
 # yyy, set
-# GLOBAL_BAN_LIST = ['xxx@aol.com', '^yyy.*@gmail\.com']
+# GLOBAL_BAN_LIST = ['xxx@aol.com', '^yyy.*@gmail\.com$']
 GLOBAL_BAN_LIST = []
 
 # Command that is used to convert text/html parts into plain text.  This
@@ -153,6 +153,13 @@ ACCEPTABLE_LISTNAME_CHARACTERS = '[-+_.=a-z0-9]'
 # in list rosters?  Defaults to No to preserve prior behavior.
 ROSTER_DISPLAY_REALNAME = No
 
+# Beginning in Mailman 2.1.21, localized help and some other output from
+# Mailman's bin/ commands is converted to the character set of the user's
+# workstation (LC_CTYPE) if different from the character set of the language.
+# This is not well tested over a wide range of locales, so if it causes
+# problems, it can be disabled by setting the following to Yes.
+DISABLE_COMMAND_LOCALE_CSET = No
+
 
 
 #####
@@ -1130,6 +1137,13 @@ DMARC_RESOLVER_TIMEOUT = seconds(3)
 # The total time to spend trying to get an answer to the question.
 DMARC_RESOLVER_LIFETIME = seconds(5)
 
+# A URL from which to retrieve the data for the algorithm that computes
+# Organizational Domains for DMARC policy lookup purposes.  This can be
+# anything handled by the Python urllib2.urlopen function.  See
+# https://publicsuffix.org/list/ for info.
+DMARC_ORGANIZATIONAL_DOMAIN_DATA_URL = \
+'https://publicsuffix.org/list/public_suffix_list.dat'
+
 # Should the list server auto-moderate members who post too frequently
 # This is intended to stop people who join a list and then use a bot to
 # send many spam messages in a short interval.  These are default settings
diff --git a/Mailman/Handlers/Cleanse.py b/Mailman/Handlers/Cleanse.py
index c684cd19..5270bb5a 100644
--- a/Mailman/Handlers/Cleanse.py
+++ b/Mailman/Handlers/Cleanse.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -60,6 +60,9 @@ def process(mlist, msg, msgdata):
     del msg['x-approve']
     # Also remove this header since it can contain a password
     del msg['urgent']
+    # If we're anonymizing, we need to save the sender here, and we may as
+    # well do it for all.
+    msgdata['original_sender'] = msg.get_sender()
     # We remove other headers from anonymous lists
     if mlist.anonymous_list:
         syslog('post', 'post to %s from %s anonymized',
diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
index 8b371b77..59eb67b7 100755
--- a/Mailman/Handlers/CookHeaders.py
+++ b/Mailman/Handlers/CookHeaders.py
@@ -98,7 +98,9 @@ def process(mlist, msg, msgdata):
     # message, we want to save some of the information in the msgdata
     # dictionary for later.  Specifically, the sender header will get waxed,
     # but we need it for the Acknowledge module later.
-    msgdata['original_sender'] = msg.get_sender()
+    # We may have already saved it; if so, don't clobber it here.
+    if 'original_sender' not in msgdata:
+        msgdata['original_sender'] = msg.get_sender()
     # VirginRunner sets _fasttrack for internally crafted messages.
     fasttrack = msgdata.get('_fasttrack')
     if not msgdata.get('isdigest') and not fasttrack:
diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py
index ab7483ba..691a6e85 100644
--- a/Mailman/Handlers/MimeDel.py
+++ b/Mailman/Handlers/MimeDel.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2011 by the Free Software Foundation, Inc.
+# Copyright (C) 2002-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -210,6 +210,11 @@ def recast_multipart(msg):
     # If we're left with a multipart message with only one sub-part, recast
     # the message to just the sub-part, but not if the part is message/rfc822
     # because we don't want to lose the headers.
+    # Also, if this is a multipart/signed part, stop now as the original part
+    # may have had a multipart sub-part with only one sub-sub-part, the sig
+    # may still be valid and going further may break it.  (LP: #1551075)
+    if msg.get_content_type() == 'multipart/signed':
+        return
     if msg.is_multipart():
         if (len(msg.get_payload()) == 1 and
                 msg.get_content_type() <> 'message/rfc822'):
diff --git a/Mailman/MTA/Manual.py b/Mailman/MTA/Manual.py
index 92e1c03c..14158263 100644
--- a/Mailman/MTA/Manual.py
+++ b/Mailman/MTA/Manual.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2005 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -25,7 +25,7 @@ from Mailman import mm_cfg
 from Mailman import Message
 from Mailman import Utils
 from Mailman.Queue.sbcache import get_switchboard
-from Mailman.i18n import _
+from Mailman.i18n import _, C_
 from Mailman.MTA.Utils import makealiases
 
 try:
@@ -74,12 +74,12 @@ Here are the entries for the /etc/aliases file:
         outfp = sfp
     else:
         if not quiet:
-            print _("""\
+            print C_("""\
 To finish creating your mailing list, you must edit your /etc/aliases (or
 equivalent) file by adding the following lines, and possibly running the
 `newaliases' program:
 """)
-        print _("""\
+        print C_("""\
 ## %(listname)s mailing list""")
         outfp = sys.stdout
     # Common path
@@ -120,7 +120,7 @@ Here are the entries in the /etc/aliases file that should be removed:
 """)
         outfp = sfp
     else:
-        print _("""
+        print C_("""
 To finish removing your mailing list, you must edit your /etc/aliases (or
 equivalent) file by removing the following lines, and possibly running the
 `newaliases' program:
diff --git a/Mailman/MTA/Postfix.py b/Mailman/MTA/Postfix.py
index 3f5c9984..8860459e 100644
--- a/Mailman/MTA/Postfix.py
+++ b/Mailman/MTA/Postfix.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2014 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,7 +27,7 @@ from stat import *
 from Mailman import mm_cfg
 from Mailman import Utils
 from Mailman import LockFile
-from Mailman.i18n import _
+from Mailman.i18n import C_
 from Mailman.MTA.Utils import makealiases
 from Mailman.Logging.Syslog import syslog
 
@@ -358,7 +358,7 @@ def checkperms(state):
     targetmode = S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
     for file in ALIASFILE, VIRTFILE:
         if state.VERBOSE:
-            print _('checking permissions on %(file)s')
+            print C_('checking permissions on %(file)s')
         stat = None
         try:
             stat = os.stat(file)
@@ -368,9 +368,9 @@ def checkperms(state):
         if stat and (stat[ST_MODE] & targetmode) <> targetmode:
             state.ERRORS += 1
             octmode = oct(stat[ST_MODE])
-            print _('%(file)s permissions must be 066x (got %(octmode)s)'),
+            print C_('%(file)s permissions must be 066x (got %(octmode)s)'),
             if state.FIX:
-                print _('(fixing)')
+                print C_('(fixing)')
                 os.chmod(file, stat[ST_MODE] | targetmode)
             else:
                 print
@@ -386,7 +386,7 @@ def checkperms(state):
                 raise
             continue
         if state.VERBOSE:
-            print _('checking ownership of %(dbfile)s')
+            print C_('checking ownership of %(dbfile)s')
         user = mm_cfg.MAILMAN_USER
         ownerok = stat[ST_UID] == pwd.getpwnam(user)[2]
         if not ownerok:
@@ -394,10 +394,11 @@ def checkperms(state):
                 owner = pwd.getpwuid(stat[ST_UID])[0]
             except KeyError:
                 owner = 'uid %d' % stat[ST_UID]
-            print _('%(dbfile)s owned by %(owner)s (must be owned by %(user)s'),
+            print C_(
+                '%(dbfile)s owned by %(owner)s (must be owned by %(user)s'),
             state.ERRORS += 1
             if state.FIX:
-                print _('(fixing)')
+                print C_('(fixing)')
                 uid = pwd.getpwnam(user)[2]
                 gid = grp.getgrnam(mm_cfg.MAILMAN_GROUP)[2]
                 os.chown(dbfile, uid, gid)
@@ -406,9 +407,9 @@ def checkperms(state):
         if stat and (stat[ST_MODE] & targetmode) <> targetmode:
             state.ERRORS += 1
             octmode = oct(stat[ST_MODE])
-            print _('%(dbfile)s permissions must be 066x (got %(octmode)s)'),
+            print C_('%(dbfile)s permissions must be 066x (got %(octmode)s)'),
             if state.FIX:
-                print _('(fixing)')
+                print C_('(fixing)')
                 os.chmod(dbfile, stat[ST_MODE] | targetmode)
             else:
                 print
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 1aa49954..f821f13a 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -34,6 +34,7 @@ import time
 import errno
 import base64
 import random
+import urllib2
 import urlparse
 import htmlentitydefs
 import email.Header
@@ -1156,6 +1157,79 @@ def suspiciousHTML(html):
         return False
 
 
+# The next functions read data from
+# https://publicsuffix.org/list/public_suffix_list.dat and implement the
+# algorithm at https://publicsuffix.org/list/ to find the "Organizational
+# Domain corresponding to a From: domain.
+
+s_dict = {}
+
+def get_suffixes(url):
+    """This loads and parses the data from the url argument into s_dict for
+    use by get_org_dom."""
+    global s_dict
+    if s_dict:
+        return
+    try:
+        d = urllib2.urlopen(url)
+    except urllib2.URLError, e:
+        syslog('error',
+               'Unable to retrieve data from %s: %s',
+               url, e)
+        return
+    for line in d.readlines():
+        if not line.strip() or line.startswith(' ') or line.startswith('//'):
+            continue
+        line = re.sub(' .*', '', line.strip())
+        if not line:
+            continue
+        parts = line.lower().split('.')
+        if parts[0].startswith('!'):
+            exc = True
+            parts = [parts[0][1:]] + parts[1:]
+        else:
+            exc = False
+        parts.reverse()
+        k = '.'.join(parts)
+        s_dict[k] = exc
+
+def _get_dom(d, l):
+    """A helper to get a domain name consisting of the first l+1 labels
+    in d."""
+    dom = d[:min(l+1, len(d))]
+    dom.reverse()
+    return '.'.join(dom)
+
+def get_org_dom(domain):
+    """Given a domain name, this returns the corresponding Organizational
+    Domain which may be the same as the input."""
+    global s_dict
+    if not s_dict:
+        get_suffixes(mm_cfg.DMARC_ORGANIZATIONAL_DOMAIN_DATA_URL)
+    hits = []
+    d = domain.lower().split('.')
+    d.reverse()
+    for k in s_dict.keys():
+        ks = k.split('.')
+        if len(d) >= len(ks):
+            for i in range(len(ks)-1):
+                if d[i] != ks[i] and ks[i] != '*':
+                    break
+            else:
+                if d[len(ks)-1] == ks[-1] or ks[-1] == '*':
+                    hits.append(k)
+    if not hits:
+        return _get_dom(d, 1)
+    l = 0
+    for k in hits:
+        if s_dict[k]:
+            # It's an exception
+            return _get_dom(d, len(k.split('.'))-1)
+        if len(k.split('.')) > l:
+            l = len(k.split('.'))
+    return _get_dom(d, l)
+
+
 # This takes an email address, and returns True if DMARC policy is p=reject
 # or possibly quarantine.
 def IsDMARCProhibited(mlist, email):
@@ -1170,7 +1244,18 @@ def IsDMARCProhibited(mlist, email):
     at_sign = email.find('@')
     if at_sign < 1:
         return False
-    dmarc_domain = '_dmarc.' + email[at_sign+1:]
+    f_dom = email[at_sign+1:]
+    x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom)
+    if x != 'continue':
+        return x
+    o_dom = get_org_dom(f_dom)
+    if o_dom != f_dom:
+        x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom)
+        if x != 'continue':
+            return x
+    return False
+
+def _DMARCProhibited(mlist, email, dmarc_domain):
 
     try:
         resolver = dns.resolver.Resolver()
@@ -1178,12 +1263,12 @@ def IsDMARCProhibited(mlist, email):
         resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME)
         txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
     except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-        return False
+        return 'continue'
     except DNSException, e:
         syslog('error',
                'DNSException: Unable to query DMARC policy for %s (%s). %s',
               email, dmarc_domain, e.__class__)
-        return False
+        return 'continue'
     else:
 # people are already being dumb, don't trust them to provide honest DNS
 # where the answer section only contains what was asked for, nor to include
@@ -1223,7 +1308,7 @@ def IsDMARCProhibited(mlist, email):
             dmarcs = filter(lambda n: n.startswith('v=DMARC1;'),
                             results_by_name[name])
             if len(dmarcs) == 0:
-                return False
+                return 'continue'
             if len(dmarcs) > 1:
                 syslog('error',
                        """RRset of TXT records for %s has %d v=DMARC1 entries;
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 091e0c54..132648a8 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -16,7 +16,7 @@
 # USA.
 
 # Mailman version
-VERSION = '2.1.21rc2'
+VERSION = '2.1.21'
 
 # And as a hex number in the manner of PY_VERSION_HEX
 ALPHA = 0xa
@@ -29,9 +29,9 @@ FINAL = 0xf
 MAJOR_REV = 2
 MINOR_REV = 1
 MICRO_REV = 21
-REL_LEVEL = GAMMA
+REL_LEVEL = FINAL
 # at most 15 beta releases!
-REL_SERIAL = 2
+REL_SERIAL = 0
 
 HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
diff --git a/Mailman/i18n.py b/Mailman/i18n.py
index 5f926b77..b8b527e0 100644
--- a/Mailman/i18n.py
+++ b/Mailman/i18n.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000-2010 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -17,6 +17,7 @@
 
 import sys
 import time
+import locale
 import gettext
 from types import StringType, UnicodeType
 
@@ -25,6 +26,16 @@ from Mailman.SafeDict import SafeDict
 
 _translation = None
 
+
+def _get_ctype_charset():
+    old = locale.setlocale(locale.LC_CTYPE, '')
+    charset = locale.nl_langinfo(locale.CODESET)
+    locale.setlocale(locale.LC_CTYPE, old)
+    return charset
+
+if not mm_cfg.DISABLE_COMMAND_LOCALE_CSET:
+    _ctype_charset = _get_ctype_charset()
+
 
 
 def set_language(language=None):
@@ -54,7 +65,7 @@ if _translation is None:
 
 
 
-def _(s):
+def _(s, frame=1):
     if s == '':
         return s
     assert s
@@ -70,7 +81,7 @@ def _(s):
     # original string is 1) locals dictionary, 2) globals dictionary.
     #
     # First, get the frame of the caller
-    frame = sys._getframe(1)
+    frame = sys._getframe(frame)
     # A `safe' dictionary is used so we won't get an exception if there's a
     # missing key in the dictionary.
     dict = SafeDict(frame.f_globals.copy())
@@ -95,6 +106,23 @@ def _(s):
 
 
 
+def tolocale(s):
+    global _ctype_charset
+    if isinstance(s, UnicodeType):
+        return s
+    source = _translation.charset ()
+    if not source:
+        return s
+    return unicode(s, source, 'replace').encode(_ctype_charset, 'replace')
+
+if mm_cfg.DISABLE_COMMAND_LOCALE_CSET:
+    C_ = _
+else:
+    def C_(s):
+        return tolocale(_(s, 2))
+
+    
+
 def ctime(date):
     # Don't make these module globals since we have to do runtime translation
     # of the strings anyway.