sync merge lp:mailman/2.1 up to 1871

10 files changed, 83 insertions, 7 deletions

diff --git a/Mailman/Bouncer.py b/Mailman/Bouncer.py
index 5077f84c..909f58b0 100644
--- a/Mailman/Bouncer.py
+++ b/Mailman/Bouncer.py
@@ -226,12 +226,14 @@ class Bouncer:
         if self.bounce_notify_owner_on_disable:
             self.__sendAdminBounceNotice(member, msg)
 
-    def __sendAdminBounceNotice(self, member, msg, did=_('disabled')):
+    def __sendAdminBounceNotice(self, member, msg, did=None):
         # BAW: This is a bit kludgey, but we're not providing as much
         # information in the new admin bounce notices as we used to (some of
         # it was of dubious value).  However, we'll provide empty, strange, or
         # meaningless strings for the unused %()s fields so that the language
         # translators don't have to provide new templates.
+        if did is None:
+            did = _('disabled')
         siteowner = Utils.get_site_email(self.host_name)
         text = Utils.maketext(
             'bounce.txt',
diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py
index 6de9a858..b7889d21 100644
--- a/Mailman/Bouncers/SimpleMatch.py
+++ b/Mailman/Bouncers/SimpleMatch.py
@@ -199,6 +199,10 @@ PATTERNS = [
     (_c(r'wasn\'t able to deliver the following message'),
      _c(r'---Below this line is a copy of the message.'),
      _c(r'To: (?P<addr>[^\s@]+@[^\s@]+)')),
+    # From some unknown MTA
+    (_c(r'This is a delivery failure notification message'),
+     _c(r'The problem appears to be'),
+     _c(r'-- (?P<addr>[^\s@]+@[^\s@]+)')),
     # Next one goes here...
     ]
 
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index fea5dcf1..765d705c 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -841,7 +841,10 @@ VERP_PROBES = No
 
 # A perfect opportunity for doing VERP is the password reminders, which are
 # already addressed individually to each recipient.  Set this to Yes to enable
-# VERPs on all password reminders.
+# VERPs on all password reminders.  However, because password reminders are
+# sent from the site list and site list bounces aren't processed but are just
+# forwarded to the site list admins, this isn't too useful.  See comments at
+# lines 70-84 of Mailman/Queue/BounceRunner.py for why we don't process them.
 VERP_PASSWORD_REMINDERS = No
 
 # Another good opportunity is when regular delivery is personalized.  Here
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index cecd11fb..a498f814 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -90,6 +90,9 @@ def guess_extension(ctype, ext):
     if ctype.lower == 'application/octet-stream':
         # For this type, all[0] is '.obj'. '.bin' is better.
         return '.bin'
+    if ctype.lower == 'text/plain':
+        # For this type, all[0] is '.ksh'. '.txt' is better.
+        return '.txt'
     return all and all[0]
 
 
@@ -196,8 +199,11 @@ def process(mlist, msg, msgdata=None):
                 format = part.get_param('format')
                 delsp = part.get_param('delsp')
             # TK: if part is attached then check charset and scrub if none
-            if part.get('content-disposition') and \
-               not part.get_content_charset():
+            # MAS: Content-Disposition is not a good test for 'attached'.
+            # RFC 2183 sec. 2.10 allows Content-Disposition on the main body.
+            # Make it specifically 'attachment'.
+            if (part.get('content-disposition', '').lower() == 'attachment'
+                    and not part.get_content_charset()):
                 omask = os.umask(002)
                 try:
                     url = save_attachment(mlist, part, dir)
diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py
index 0ee08cb1..3aef64a2 100644
--- a/Mailman/Handlers/WrapMessage.py
+++ b/Mailman/Handlers/WrapMessage.py
@@ -63,13 +63,19 @@ def process(mlist, msg, msgdata):
     # make a copy of the msg, then delete almost everything and set/copy
     # what we want.
     omsg = copy.deepcopy(msg)
+    # If CookHeaders didn't change the Subject: we need to keep it too.
+    # Get a fresh list.
+    keepers = list(KEEPERS)
+    if 'subject' not in [key.lower() for key in
+                         msgdata.get('add_header', {}).keys()]:
+        keepers.append('subject')
     for key in msg.keys():
-        if key.lower() not in KEEPERS:
+        if key.lower() not in keepers:
             del msg[key]
     msg['MIME-Version'] = '1.0'
     msg['Message-ID'] = Utils.unique_message_id(mlist)
     # Add the headers from CookHeaders.
-    for k, v in msgdata['add_header'].items():
+    for k, v in msgdata.get('add_header', {}).items():
         msg[k] = v
     # Are we including dmarc_wrapped_message_text?  I.e., do we have text and
     # are we wrapping because of dmarc_moderation_action?
diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py
index 6ea50255..3c21065f 100644
--- a/Mailman/Queue/CommandRunner.py
+++ b/Mailman/Queue/CommandRunner.py
@@ -100,6 +100,11 @@ class Results:
             # E.g the outer Content-Type: was text/html
             return
         body = part.get_payload(decode=True)
+        if (part.get_content_charset(None)):
+            body = unicode(body, part.get_content_charset(),
+                           errors='replace').encode(
+                           Utils.GetCharSet(self.msgdata['lang']),
+                           errors='replace')
         # text/plain parts better have string payloads
         assert isinstance(body, StringType) or isinstance(body, UnicodeType)
         lines = body.splitlines()
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 2615229b..c61f7e2c 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -1380,8 +1380,9 @@ def _DMARCProhibited(mlist, email, dmarc_domain, org=False):
             if len(dmarcs) > 1:
                 syslog('error',
                        """RRset of TXT records for %s has %d v=DMARC1 entries;
-                       testing them all""",
+                       ignoring them per RFC 7849""",
                         dmarc_domain, len(dmarcs))
+                return False
             for entry in dmarcs:
                 mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE)
                 if org and mo:
diff --git a/NEWS b/NEWS
index 24bd861b..31c6925b 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,26 @@ Copyright (C) 1998-2020 by the Free Software Foundation, Inc.
 
 Here is a history of user visible changes to Mailman.
 
+2.1.35 (xx-xxx-xxxx)
+
+  Bug Fixes and other patches
+
+    - Fixed an issue where sometimes the wrapper message for DMARC mitigation
+      Wrap Message has no Subject:.  (LP: #1915655)
+
+    - Plain text message bodies with Content-Disposition: and no declared
+      charset are no longer scrubbed.  (LP: #1917968)
+
+    - CommandRunner now recodes message bodies in the charset of the user's
+      or list's language to avoid a possible UnicodeError when including the
+      message body in the reply.  (LP: #1921682)
+
+    - Delivery disabled by bounce notices to admins now have 'disabled'
+      properly translated.  (LP: #1922843)
+
+    - DMARC policy discovery ignores domains with multiple DMARC records per
+      RFC 7849,  (LP: 1931029)
+
 2.1.34 (26-Jun-2020)
 
   i18n
diff --git a/tests/bounces/simple_44.txt b/tests/bounces/simple_44.txt
new file mode 100644
index 00000000..832ca5f9
--- /dev/null
+++ b/tests/bounces/simple_44.txt
@@ -0,0 +1,28 @@
+Subject: [Postmaster] Email Delivery Failure
+Message-Id: <111195647-1608276241173@example.com>
+Date: Fri, 18 Dec 2020 09:24:01 +0200
+From: "Mail Delivery System" <mailer-daemon@example.com>
+To: "dns-operations" <dns-operations-bounces@example.net>
+X-Mimecast-Originator: <>
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+X-MIME-Autoconverted: from quoted-printable to 8bit by example.net id 0BI7O1sk026573
+
+This is a delivery failure notification message indicating that
+an email dns-operations-bounces@example.net addressed to email address :
+-- user@example.com
+
+With subject Re: [dns-operations] Monitoring for impending expiration of domains? sent on the Mon, 14 Dec 2020 11:30:37 +0100 (CET)
+could not be delivered. The problem appears to be :
+-- Recipient server unavailable or busy
+
+Additional information follows :
+-- 4.4.4 Temporary server error. Please try again later ATTR5 [VE1EUR02FT049.eop-EUR02.prod.protection.outlook.com]
+
+This condition occurred after 30 attempt(s) to deliver over
+a period of 92 hour(s) and 52 minute(s).
+
+If you sent the email to multiple recipients, you will receive one
+of these messages for each one which failed delivery,  otherwise
+they have been sent.
+
diff --git a/tests/test_bounces.py b/tests/test_bounces.py
index 3ddde6b6..78527f8b 100644
--- a/tests/test_bounces.py
+++ b/tests/test_bounces.py
@@ -82,6 +82,7 @@ class BounceTest(unittest.TestCase):
         ('SimpleMatch', 'simple_38.txt', ['userx@example.com']),
         ('SimpleMatch', 'simple_39.txt', ['userx@example.ru']),
         ('SimpleMatch', 'simple_41.txt', ['userx@example.com']),
+        ('SimpleMatch', 'simple_44.txt', ['user@example.com']),
         ('SimpleMatch', 'bounce_02.txt', ['userx@example.com']),
         ('SimpleMatch', 'bounce_03.txt', ['userx@example.uk']),
         ('SimpleMatch', 'yahoo_12.txt', ['user@yahoo.com']),