Merge lp:mailman/2.1 up to 1686

5 files changed, 66 insertions, 11 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 9ecdbe62..fb2b26b6 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -666,7 +666,6 @@ GLOBAL_PIPELINE = [
 OWNER_PIPELINE = [
     'SpamDetect',
     'Replybot',
-    'CleanseDKIM',
     'OwnerRecips',
     'ToOutgoing',
     ]
diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
index 59eb67b7..84d3032d 100755
--- a/Mailman/Handlers/CookHeaders.py
+++ b/Mailman/Handlers/CookHeaders.py
@@ -29,6 +29,7 @@ from email.Header import Header, decode_header, make_header
 from email.Utils import parseaddr, formataddr, getaddresses
 from email.Errors import HeaderParseError
 
+from Mailman import i18n
 from Mailman import mm_cfg
 from Mailman import Utils
 from Mailman.i18n import _
@@ -154,12 +155,37 @@ def process(mlist, msg, msgdata):
                 realname = email
         # Remove domain from realname if it looks like an email address
         realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
-        # RFC 2047 encode realname if necessary.
-        realname = str(uheader(mlist, realname))
-        lrn = mlist.real_name
+        # Make a display name and RFC 2047 encode it if necessary.  This is
+        # difficult and kludgy. If the realname came from From: it should be
+        # ascii or RFC 2047 encoded. If it came from the list, it should be
+        # in the charset of the list's preferred language or possibly unicode.
+        # if it's from the email address, it should be ascii. In any case,
+        # make it a unicode.
+        if isinstance(realname, unicode):
+            urn = realname
+        else:
+            rn, cs = ch_oneline(realname)
+            urn = unicode(rn, cs, errors='replace')
+        # likewise, the list's real_name which should be ascii, but use the
+        # charset of the list's preferred_language which should be a superset.
+        lcs = Utils.GetCharSet(mlist.preferred_language)
+        ulrn = unicode(mlist.real_name, lcs, errors='replace')
+        # get translated 'via' with dummy replacements
+        realname = '%(realname)s'
+        lrn = '%(lrn)s'
+        # We want the i18n context to be the list's preferred_language.  It
+        # could be the poster's.
+        otrans = i18n.get_translation()
+        i18n.set_language(mlist.preferred_language)
+        via = _('%(realname)s via %(lrn)s')
+        i18n.set_translation(otrans)
+        uvia = unicode(via, lcs, errors='replace')
+        # Replace the dummy replacements.
+        uvia = re.sub(u'%\(lrn\)s', ulrn, re.sub(u'%\(realname\)s', urn, uvia))
+        # And get an RFC 2047 encoded header string.
+        dn = str(Header(uvia, lcs))
         change_header('From',
-                      formataddr((_('%(realname)s via %(lrn)s'),
-                                 mlist.GetListEmail())),
+                      formataddr((dn, mlist.GetListEmail())),
                       mlist, msg, msgdata)
     else:
         # Use this as a flag
@@ -400,7 +426,12 @@ def prefix_subject(mlist, msg, msgdata):
     # At this point, subject may become null if someone post mail with
     # subject: [subject prefix]
     if subject.strip() == '':
+        # We want the i18n context to be the list's preferred_language.  It
+        # could be the poster's.
+        otrans = i18n.get_translation()
+        i18n.set_language(mlist.preferred_language)
         subject = _('(no subject)')
+        i18n.set_translation(otrans)
         cset = Utils.GetCharSet(mlist.preferred_language)
         subject = unicode(subject, cset)
     # and substitute %d in prefix with post_id
diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py
index 2027a46c..02965f82 100644
--- a/Mailman/Handlers/ToDigest.py
+++ b/Mailman/Handlers/ToDigest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2013 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -68,6 +68,17 @@ except NameError:
 
 
 
+def to_cset_out(text, lcset):
+    # Convert text from unicode or lcset to output cset.
+    ocset = Charset(lcset).get_output_charset() or lcset
+    if isinstance(text, unicode):
+        return text.encode(ocset, errors='replace')
+    else:
+        return text.decode(lcset, errors='replace').encode(ocset,
+                                                           errors='replace')
+
+
+
 def process(mlist, msg, msgdata):
     # Short circuit non-digestable lists.
     if not mlist.digestable or msgdata.get('isdigest'):
@@ -299,7 +310,7 @@ def send_i18n_digests(mlist, mboxfp):
     if msgcount == 0:
         # Why did we even get here?
         return
-    toctext = toc.getvalue()
+    toctext = to_cset_out(toc.getvalue(), lcset)
     # MIME
     tocpart = MIMEText(toctext, _charset=lcset)
     tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
@@ -412,7 +423,7 @@ def send_i18n_digests(mlist, mboxfp):
                     listname=mlist.internal_name(),
                     isdigest=True)
     # RFC 1153
-    rfc1153msg.set_payload(plainmsg.getvalue(), lcset)
+    rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
     virginq.enqueue(rfc1153msg,
                     recips=plainrecips,
                     listname=mlist.internal_name(),
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 99cbbd2f..d1dc17a4 100755
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -634,6 +634,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
             if e.errno <> errno.ENOENT: raise
             # The file doesn't exist yet
             return None, e
+        now = int(time.time())
         try:
             try:
                 dict = loadfunc(fp)
@@ -645,8 +646,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         finally:
             fp.close()
         # Update the timestamp.  We use current time here rather than mtime
-        # so the test above might succeed the next time.
-        self.__timestamp = int(time.time())
+        # so the test above might succeed the next time.  And we get the time
+        # before unpickling in case it takes more than a second.  (LP: #266464)
+        self.__timestamp = now
         return dict, None
 
     def Load(self, check_version=True):
diff --git a/NEWS b/NEWS
index 5ea05a7d..2e89898e 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,18 @@ Here is a history of user visible changes to Mailman.
 
   Bug fixes and other patches
 
+    - The CleanseDKIM handler has been removed from OWNER_PIPELINE.  It isn't
+      needed there and has adverse DMARC implications for messages to -owner
+      of an anonymous list.  (LP: #1645901)
+
+    - Fixed an issue with properly RFC 2047 encoding the display name in the
+      From: header for messages with DMARC mitigations.  (LP: #1643210)
+
+    - Fixed an issue causing UnicodeError in sending digests following a
+      change of a list's preferred_language.  (LP: #1644356)
+
+    - Enhanced the fix for race conditions in MailList().Load().  (LP: #266464)
+
     - Fixed a typo in Utils.py that could have resulted in a NameError in
       logging an unlikely occurrence.  (LP: #1637745)