From 66778c53a61f9c09fa5d1c7bb3d2e8866f8a246c Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Fri, 18 Dec 2020 12:04:50 -0800 Subject: Added recognition for a non-compliant DSN from an unknown MTA. --- Mailman/Bouncers/SimpleMatch.py | 4 ++++ tests/bounces/simple_44.txt | 28 ++++++++++++++++++++++++++++ tests/test_bounces.py | 1 + 3 files changed, 33 insertions(+) create mode 100644 tests/bounces/simple_44.txt diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py index 6de9a858..b7889d21 100644 --- a/Mailman/Bouncers/SimpleMatch.py +++ b/Mailman/Bouncers/SimpleMatch.py @@ -199,6 +199,10 @@ PATTERNS = [ (_c(r'wasn\'t able to deliver the following message'), _c(r'---Below this line is a copy of the message.'), _c(r'To: (?P[^\s@]+@[^\s@]+)')), + # From some unknown MTA + (_c(r'This is a delivery failure notification message'), + _c(r'The problem appears to be'), + _c(r'-- (?P[^\s@]+@[^\s@]+)')), # Next one goes here... ] diff --git a/tests/bounces/simple_44.txt b/tests/bounces/simple_44.txt new file mode 100644 index 00000000..832ca5f9 --- /dev/null +++ b/tests/bounces/simple_44.txt @@ -0,0 +1,28 @@ +Subject: [Postmaster] Email Delivery Failure +Message-Id: <111195647-1608276241173@example.com> +Date: Fri, 18 Dec 2020 09:24:01 +0200 +From: "Mail Delivery System" +To: "dns-operations" +X-Mimecast-Originator: <> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +X-MIME-Autoconverted: from quoted-printable to 8bit by example.net id 0BI7O1sk026573 + +This is a delivery failure notification message indicating that +an email dns-operations-bounces@example.net addressed to email address : +-- user@example.com + +With subject Re: [dns-operations] Monitoring for impending expiration of domains? sent on the Mon, 14 Dec 2020 11:30:37 +0100 (CET) +could not be delivered. The problem appears to be : +-- Recipient server unavailable or busy + +Additional information follows : +-- 4.4.4 Temporary server error. Please try again later ATTR5 [VE1EUR02FT049.eop-EUR02.prod.protection.outlook.com] + +This condition occurred after 30 attempt(s) to deliver over +a period of 92 hour(s) and 52 minute(s). + +If you sent the email to multiple recipients, you will receive one +of these messages for each one which failed delivery, otherwise +they have been sent. + diff --git a/tests/test_bounces.py b/tests/test_bounces.py index 3ddde6b6..78527f8b 100644 --- a/tests/test_bounces.py +++ b/tests/test_bounces.py @@ -82,6 +82,7 @@ class BounceTest(unittest.TestCase): ('SimpleMatch', 'simple_38.txt', ['userx@example.com']), ('SimpleMatch', 'simple_39.txt', ['userx@example.ru']), ('SimpleMatch', 'simple_41.txt', ['userx@example.com']), + ('SimpleMatch', 'simple_44.txt', ['user@example.com']), ('SimpleMatch', 'bounce_02.txt', ['userx@example.com']), ('SimpleMatch', 'bounce_03.txt', ['userx@example.uk']), ('SimpleMatch', 'yahoo_12.txt', ['user@yahoo.com']), -- cgit v1.2.3 From 7651d375b035cdfde969aaa4556b13f3caab8781 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Sun, 14 Feb 2021 19:36:36 -0800 Subject: Fix missing Subject: in some Wrap Message wrappers. --- Mailman/Handlers/WrapMessage.py | 10 ++++++++-- NEWS | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py index 0ee08cb1..3aef64a2 100644 --- a/Mailman/Handlers/WrapMessage.py +++ b/Mailman/Handlers/WrapMessage.py @@ -63,13 +63,19 @@ def process(mlist, msg, msgdata): # make a copy of the msg, then delete almost everything and set/copy # what we want. omsg = copy.deepcopy(msg) + # If CookHeaders didn't change the Subject: we need to keep it too. + # Get a fresh list. + keepers = list(KEEPERS) + if 'subject' not in [key.lower() for key in + msgdata.get('add_header', {}).keys()]: + keepers.append('subject') for key in msg.keys(): - if key.lower() not in KEEPERS: + if key.lower() not in keepers: del msg[key] msg['MIME-Version'] = '1.0' msg['Message-ID'] = Utils.unique_message_id(mlist) # Add the headers from CookHeaders. - for k, v in msgdata['add_header'].items(): + for k, v in msgdata.get('add_header', {}).items(): msg[k] = v # Are we including dmarc_wrapped_message_text? I.e., do we have text and # are we wrapping because of dmarc_moderation_action? diff --git a/NEWS b/NEWS index 24bd861b..d9b6da1e 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,13 @@ Copyright (C) 1998-2020 by the Free Software Foundation, Inc. Here is a history of user visible changes to Mailman. +2.1.35 (xx-xxx-xxxx) + + Bug Fixes and other patches + + - Fixed an issue where sometimes the wrapper message for DMARC mitigation + Wrap Message has no Subject:. (LP: #1915655) + 2.1.34 (26-Jun-2020) i18n -- cgit v1.2.3 From d2ca3472e083e435e812cbc3247cccf372cba906 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Fri, 5 Mar 2021 18:24:12 -0800 Subject: Thghten conditions for scrubbing text/plain. --- Mailman/Handlers/Scrubber.py | 8 +++++++- NEWS | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py index cecd11fb..16ce3b02 100644 --- a/Mailman/Handlers/Scrubber.py +++ b/Mailman/Handlers/Scrubber.py @@ -90,6 +90,9 @@ def guess_extension(ctype, ext): if ctype.lower == 'application/octet-stream': # For this type, all[0] is '.obj'. '.bin' is better. return '.bin' + if ctype.lower == 'text/plain': + # For this type, all[0] is '.ksh'. '.txt' is better. + return '.txt' return all and all[0] @@ -196,7 +199,10 @@ def process(mlist, msg, msgdata=None): format = part.get_param('format') delsp = part.get_param('delsp') # TK: if part is attached then check charset and scrub if none - if part.get('content-disposition') and \ + # MAS: Content-Disposition is not a good test for 'attached'. + # RFC 2183 sec. 2.10 allows Content-Disposition on the main body. + # Make it specifically 'attachment'. + if part.get('content-disposition').lower() == 'attachment' and \ not part.get_content_charset(): omask = os.umask(002) try: diff --git a/NEWS b/NEWS index d9b6da1e..7fb1f609 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,9 @@ Here is a history of user visible changes to Mailman. - Fixed an issue where sometimes the wrapper message for DMARC mitigation Wrap Message has no Subject:. (LP: #1915655) + - Plain text message bodies with Content-Disposition: and no declared + charset are no longer scrubbed. (LP: #1917968) + 2.1.34 (26-Jun-2020) i18n -- cgit v1.2.3 From 416d2e6ef003869486dcf6d212133511114767be Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Sat, 6 Mar 2021 11:11:23 -0800 Subject: Fix bug in prior commit. --- Mailman/Handlers/Scrubber.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py index 16ce3b02..a498f814 100644 --- a/Mailman/Handlers/Scrubber.py +++ b/Mailman/Handlers/Scrubber.py @@ -202,8 +202,8 @@ def process(mlist, msg, msgdata=None): # MAS: Content-Disposition is not a good test for 'attached'. # RFC 2183 sec. 2.10 allows Content-Disposition on the main body. # Make it specifically 'attachment'. - if part.get('content-disposition').lower() == 'attachment' and \ - not part.get_content_charset(): + if (part.get('content-disposition', '').lower() == 'attachment' + and not part.get_content_charset()): omask = os.umask(002) try: url = save_attachment(mlist, part, dir) -- cgit v1.2.3 From e1f509efe72b433fdedea7ef24526b09934768d2 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Tue, 30 Mar 2021 12:32:42 -0700 Subject: Decode message bodies for replies in CommandRunner. --- Mailman/Queue/CommandRunner.py | 3 +++ NEWS | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index 6ea50255..a2a9dab7 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -100,6 +100,9 @@ class Results: # E.g the outer Content-Type: was text/html return body = part.get_payload(decode=True) + if (part.get_content_charset(None)): + body = unicode(body, part.get_content_charset()).encode( + Utils.GetCharSet(self.msgdata['lang'])) # text/plain parts better have string payloads assert isinstance(body, StringType) or isinstance(body, UnicodeType) lines = body.splitlines() diff --git a/NEWS b/NEWS index 7fb1f609..e944a11d 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,10 @@ Here is a history of user visible changes to Mailman. - Plain text message bodies with Content-Disposition: and no declared charset are no longer scrubbed. (LP: #1917968) + - CommandRunner now recodes message bodies in the charset of the user's + or list's language to avoid a possible UnicodeError when including the + message body in the reply. (LP: #1921682) + 2.1.34 (26-Jun-2020) i18n -- cgit v1.2.3 From ed0adf071ced3aef5799ab9d08503142553703af Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Wed, 31 Mar 2021 14:56:21 -0700 Subject: Improve fix for lp:1921682. --- Mailman/Queue/CommandRunner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index a2a9dab7..3c21065f 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -101,8 +101,10 @@ class Results: return body = part.get_payload(decode=True) if (part.get_content_charset(None)): - body = unicode(body, part.get_content_charset()).encode( - Utils.GetCharSet(self.msgdata['lang'])) + body = unicode(body, part.get_content_charset(), + errors='replace').encode( + Utils.GetCharSet(self.msgdata['lang']), + errors='replace') # text/plain parts better have string payloads assert isinstance(body, StringType) or isinstance(body, UnicodeType) lines = body.splitlines() -- cgit v1.2.3 From 5a9f8b6ae30f93bdff8f7a198f59d9ef74165faa Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Wed, 7 Apr 2021 21:34:54 -0700 Subject: Translate 'disabled' when used. --- Mailman/Bouncer.py | 4 +++- NEWS | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Mailman/Bouncer.py b/Mailman/Bouncer.py index 5077f84c..909f58b0 100644 --- a/Mailman/Bouncer.py +++ b/Mailman/Bouncer.py @@ -226,12 +226,14 @@ class Bouncer: if self.bounce_notify_owner_on_disable: self.__sendAdminBounceNotice(member, msg) - def __sendAdminBounceNotice(self, member, msg, did=_('disabled')): + def __sendAdminBounceNotice(self, member, msg, did=None): # BAW: This is a bit kludgey, but we're not providing as much # information in the new admin bounce notices as we used to (some of # it was of dubious value). However, we'll provide empty, strange, or # meaningless strings for the unused %()s fields so that the language # translators don't have to provide new templates. + if did is None: + did = _('disabled') siteowner = Utils.get_site_email(self.host_name) text = Utils.maketext( 'bounce.txt', diff --git a/NEWS b/NEWS index e944a11d..a1cf751f 100644 --- a/NEWS +++ b/NEWS @@ -19,6 +19,9 @@ Here is a history of user visible changes to Mailman. or list's language to avoid a possible UnicodeError when including the message body in the reply. (LP: #1921682) + - Delivery disabled by bounce notices to admins now have 'disabled' + properly translated. (LP: #1922843) + 2.1.34 (26-Jun-2020) i18n -- cgit v1.2.3 From 61bc9dd01f3bafa9c5f9687b34fb20e9beace5cd Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Sun, 6 Jun 2021 10:55:49 -0700 Subject: DMARC policy ignores domains with multiple DMARC records. --- Mailman/Utils.py | 3 ++- NEWS | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 2615229b..c61f7e2c 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -1380,8 +1380,9 @@ def _DMARCProhibited(mlist, email, dmarc_domain, org=False): if len(dmarcs) > 1: syslog('error', """RRset of TXT records for %s has %d v=DMARC1 entries; - testing them all""", + ignoring them per RFC 7849""", dmarc_domain, len(dmarcs)) + return False for entry in dmarcs: mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE) if org and mo: diff --git a/NEWS b/NEWS index a1cf751f..31c6925b 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,9 @@ Here is a history of user visible changes to Mailman. - Delivery disabled by bounce notices to admins now have 'disabled' properly translated. (LP: #1922843) + - DMARC policy discovery ignores domains with multiple DMARC records per + RFC 7849, (LP: 1931029) + 2.1.34 (26-Jun-2020) i18n -- cgit v1.2.3 From 6cc831d90d9e22091c05f49e54502d188f7f0712 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Fri, 1 Oct 2021 20:37:34 -0700 Subject: Improve doc for VERP_PASSWORD_REMINDERS. --- Mailman/Defaults.py.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index fea5dcf1..765d705c 100755 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -841,7 +841,10 @@ VERP_PROBES = No # A perfect opportunity for doing VERP is the password reminders, which are # already addressed individually to each recipient. Set this to Yes to enable -# VERPs on all password reminders. +# VERPs on all password reminders. However, because password reminders are +# sent from the site list and site list bounces aren't processed but are just +# forwarded to the site list admins, this isn't too useful. See comments at +# lines 70-84 of Mailman/Queue/BounceRunner.py for why we don't process them. VERP_PASSWORD_REMINDERS = No # Another good opportunity is when regular delivery is personalized. Here -- cgit v1.2.3