merge lp:mailman/2.1 up to rev 1805

12 files changed, 61 insertions, 11 deletions

diff --git a/Mailman/Bouncers/DSN.py b/Mailman/Bouncers/DSN.py
index 701617db..b316c696 100644
--- a/Mailman/Bouncers/DSN.py
+++ b/Mailman/Bouncers/DSN.py
@@ -56,7 +56,8 @@ def check(msg):
             # Some MTAs have been observed that put comments on the action.
             if action.startswith('delayed'):
                 return Stop
-            if not action.startswith('fail'):
+            # opensmtpd uses non-compliant Action: error.
+            if not (action.startswith('fail') or action.startswith('error')):
                 # Some non-permanent failure, so ignore this block
                 continue
             params = []
diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py
index 8dd39aff..1175b81a 100644
--- a/Mailman/Cgi/confirm.py
+++ b/Mailman/Cgi/confirm.py
@@ -327,6 +327,12 @@ def subscription_cancel(mlist, doc, cookie):
     try:
         # Discard this cookie
         userdesc = mlist.pend_confirm(cookie)[1]
+    except TypeError:
+        # See comment about TypeError in subscription_confirm.
+        # Give a generic message.  It doesn't much matter what since it's a
+        # bot anyway.
+        doc.AddItem(_('Error'))
+        return
     finally:
         mlist.Unlock()
     lang = userdesc.language
@@ -362,6 +368,10 @@ def subscription_confirm(mlist, doc, cookie, cgidata):
             else:
                 digest = None
             userdesc = mlist.pend_confirm(cookie, expunge=False)[1]
+            # There is a potential race condition if two (robotic?) clients try
+            # to confirm the same token simultaneously.  If they both succeed in
+            # retrieving the data above, when the second gets here, the cookie
+            # is gone and TypeError is thrown.  Catch it below.
             fullname = cgidata.getfirst('realname', None)
             if fullname is not None:
                 fullname = Utils.canonstr(fullname, lang)
@@ -379,7 +389,7 @@ def subscription_confirm(mlist, doc, cookie, cgidata):
             the list moderator before you will be subscribed.  Your request
             has been forwarded to the list moderator, and you will be notified
             of the moderator's decision."""))
-        except Errors.NotAMemberError:
+        except (Errors.NotAMemberError, TypeError):
             bad_confirmation(doc, _('''Invalid confirmation string.  It is
             possible that you are attempting to confirm a request for an
             address that has already been unsubscribed.'''))
@@ -444,7 +454,8 @@ def unsubscription_confirm(mlist, doc, cookie):
             i18n.set_language(lang)
             doc.set_language(lang)
             op, addr = mlist.ProcessConfirmation(cookie)
-        except Errors.NotAMemberError:
+        # See comment about TypeError in subscription_confirm.
+        except (Errors.NotAMemberError, TypeError):
             bad_confirmation(doc, _('''Invalid confirmation string.  It is
             possible that you are attempting to confirm a request for an
             address that has already been unsubscribed.'''))
@@ -533,7 +544,8 @@ def addrchange_confirm(mlist, doc, cookie):
             i18n.set_language(lang)
             doc.set_language(lang)
             op, oldaddr, newaddr = mlist.ProcessConfirmation(cookie)
-        except Errors.NotAMemberError:
+        # See comment about TypeError in subscription_confirm.
+        except (Errors.NotAMemberError, TypeError):
             bad_confirmation(doc, _('''Invalid confirmation string.  It is
             possible that you are attempting to confirm a request for an
             address that has already been unsubscribed.'''))
@@ -657,7 +669,8 @@ def heldmsg_confirm(mlist, doc, cookie):
             # Discard the message
             mlist.HandleRequest(id, mm_cfg.DISCARD,
                                 _('Sender discarded message via web.'))
-        except (Errors.LostHeldMessage, KeyError):
+        # See comment about TypeError in subscription_confirm.
+        except (Errors.LostHeldMessage, KeyError, TypeError):
             bad_confirmation(doc, _('''The held message with the Subject:
             header <em>%(subject)s</em> could not be found.  The most likely
             reason for this is that the list moderator has already approved or
@@ -770,7 +783,8 @@ def reenable_confirm(mlist, doc, cookie):
             i18n.set_language(lang)
             doc.set_language(lang)
             op, addr = mlist.ProcessConfirmation(cookie)
-        except Errors.NotAMemberError:
+        # See comment about TypeError in subscription_confirm.
+        except (Errors.NotAMemberError, TypeError):
             bad_confirmation(doc, _('''Invalid confirmation string.  It is
             possible that you are attempting to confirm a request for an
             address that has already been unsubscribed.'''))
diff --git a/Mailman/Cgi/create.py b/Mailman/Cgi/create.py
index ebb211ae..d72e6967 100644
--- a/Mailman/Cgi/create.py
+++ b/Mailman/Cgi/create.py
@@ -162,6 +162,13 @@ def process_request(doc, cgidata):
         if not ok:
             ok = Utils.check_global_password(auth)
     if not ok:
+        remote = os.environ.get('HTTP_FORWARDED_FOR',
+                 os.environ.get('HTTP_X_FORWARDED_FOR',
+                 os.environ.get('REMOTE_ADDR',
+                                'unidentified origin')))
+        syslog('security',
+               'Authorization failed (create): list=%s: remote=%s',
+               listname, remote)
         request_creation(
             doc, cgidata,
             _('You are not authorized to create new mailing lists'))
diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py
index 34a7718e..3a3b7841 100644
--- a/Mailman/Cgi/options.py
+++ b/Mailman/Cgi/options.py
@@ -296,7 +296,7 @@ def main():
                      os.environ.get('REMOTE_ADDR',
                                     'unidentified origin')))
             syslog('security',
-                 'Authorization failed (private): user=%s: list=%s: remote=%s',
+                 'Authorization failed (options): user=%s: list=%s: remote=%s',
                    user, listname, remote)
             # So as not to allow membership leakage, prompt for the email
             # address and the password here.
diff --git a/Mailman/Cgi/rmlist.py b/Mailman/Cgi/rmlist.py
index 4472c1c5..4c37a15d 100644
--- a/Mailman/Cgi/rmlist.py
+++ b/Mailman/Cgi/rmlist.py
@@ -127,6 +127,13 @@ def process_request(doc, cgidata, mlist):
                            mm_cfg.AuthListAdmin,
                            mm_cfg.AuthSiteAdmin),
                           password) == mm_cfg.UnAuthorized:
+        remote = os.environ.get('HTTP_FORWARDED_FOR',
+                 os.environ.get('HTTP_X_FORWARDED_FOR',
+                 os.environ.get('REMOTE_ADDR',
+                                'unidentified origin')))
+        syslog('security',
+               'Authorization failed (rmlist): list=%s: remote=%s',
+               mlist.internal_name(), remote)
         request_deletion(
             doc, mlist,
             _('You are not authorized to delete this mailing list'))
diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py
index abf87e08..eddd697b 100644
--- a/Mailman/Cgi/roster.py
+++ b/Mailman/Cgi/roster.py
@@ -123,8 +123,8 @@ def main():
                  os.environ.get('REMOTE_ADDR',
                                 'unidentified origin')))
         syslog('security',
-               'Authorization failed (roster): list=%s: remote=%s',
-               listname, remote)
+               'Authorization failed (roster): user=%s: list=%s: remote=%s',
+               addr, listname, remote)
         return
 
     # The document and its language
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
index 4df63da1..04e1b4d9 100644
--- a/Mailman/Gui/Privacy.py
+++ b/Mailman/Gui/Privacy.py
@@ -356,6 +356,21 @@ class Privacy(GUIBase):
              be sent to anyone who posts to this list from a domain
              with a DMARC Reject%(quarantine)s Policy.""")),
              
+            ('dmarc_moderation_addresses', mm_cfg.EmailListEx, (10, WIDTH), 1,
+             _("""List of addresses (or regexps) whose posts should always apply
+             <a href="?VARHELP=privacy/sender/dmarc_moderation_action"
+             >dmarc_moderation_action</a> 
+             regardless of any domain specific DMARC Policy."""),
+
+             _("""Postings from any of these addresses will automatically
+             apply any DMARC action mitigation.  This can be utilized to
+             automatically wrap or munge postings from known addresses or
+             domains that might have policies rejecting external mail From:
+             themselves.
+
+             <p>Add member addresses one per line; start the line with a ^
+             character to designate a regular expression match.""")),
+             
             ('dmarc_wrapped_message_text', mm_cfg.Text, (10, WIDTH), 1,
              _("""If dmarc_moderation_action applies and is Wrap Message,
              and this text is provided, the text will be placed in a
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index 429312be..97e443b7 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -87,6 +87,9 @@ def guess_extension(ctype, ext):
     all = guess_all_extensions(ctype, strict=False)
     if ext in all:
         return ext
+    if ctype.lower == 'application/octet-stream':
+        # For this type, all[0] is '.obj'. '.bin' is better.
+        return '.bin'
     return all and all[0]
 
 
diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py
index 7e035184..cf41303f 100644
--- a/Mailman/Handlers/SpamDetect.py
+++ b/Mailman/Handlers/SpamDetect.py
@@ -109,7 +109,8 @@ def process(mlist, msg, msgdata):
         msgdata['from_is_list'] = 0
         dn, addr = parseaddr(msg.get('from'))
         if addr and mlist.dmarc_moderation_action > 0:
-            if Utils.IsDMARCProhibited(mlist, addr):
+            if (mlist.GetPattern(addr, mlist.dmarc_moderation_addresses) or
+                Utils.IsDMARCProhibited(mlist, addr)):
                 # Note that for dmarc_moderation_action, 0 = Accept, 
                 #    1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard
                 if mlist.dmarc_moderation_action == 1:
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 8e2518c5..d74978af 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -424,6 +424,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         self.dmarc_none_moderation_action = (
             mm_cfg.DEFAULT_DMARC_NONE_MODERATION_ACTION)
         self.dmarc_moderation_notice = ''
+	self.dmarc_moderation_addresses = []
         self.dmarc_wrapped_message_text = (
             mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT)
         self.equivalent_domains = (
diff --git a/Mailman/Version.py b/Mailman/Version.py
index da704882..f607c126 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 110
+DATA_FILE_VERSION = 111
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 428bb0af..d317a46d 100644
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -497,6 +497,7 @@ def NewVars(l):
     add_only_if_missing('dmarc_none_moderation_action',
                        mm_cfg.DEFAULT_DMARC_NONE_MODERATION_ACTION)
     add_only_if_missing('dmarc_moderation_notice', '')
+    add_only_if_missing('dmarc_moderation_addresses', [])
     add_only_if_missing('dmarc_wrapped_message_text',
                        mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT)
     add_only_if_missing('member_verbosity_threshold',