Merge lp:mailman/2.1 up to rev 1649

10 files changed, 139 insertions, 28 deletions

diff --git a/Mailman/Archiver/pipermail.py b/Mailman/Archiver/pipermail.py
index 9c54bbd9..c03d43b3 100644
--- a/Mailman/Archiver/pipermail.py
+++ b/Mailman/Archiver/pipermail.py
@@ -16,6 +16,7 @@ __version__ = '0.09 (Mailman edition)'
 VERSION = __version__
 CACHESIZE = 100    # Number of slots in the cache
 
+from Mailman import mm_cfg
 from Mailman import Errors
 from Mailman.Mailbox import ArchiverMailbox
 from Mailman.Logging.Syslog import syslog
@@ -230,21 +231,30 @@ class Article:
         self.body = s.readlines()
 
     def _set_date(self, message):
-        def floatdate(header):
-            missing = []
-            datestr = message.get(header, missing)
-            if datestr is missing:
+        def floatdate(datestr):
+            if not datestr:
                 return None
             date = parsedate_tz(datestr)
             try:
-                return mktime_tz(date)
+                date = mktime_tz(date)
+                if (date < 0 or
+                    date - time.time() >
+                        mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW
+                   ):
+                    return None
+                return date
             except (TypeError, ValueError, OverflowError):
                 return None
-        date = floatdate('date')
+        date = floatdate(message.get('date'))
+        if date is None:
+            date = floatdate(message.get('x-list-received-date'))
+        if date is None:
+            date = floatdate(re.sub(r'^.*;\s*', '',
+                                    message.get('received', ''), flags=re.S))
         if date is None:
-            date = floatdate('x-list-received-date')
+            date = floatdate(re.sub(r'From \s*\S+\s+', '',
+                                    message.get_unixfrom() or '' ))
         if date is None:
-            # What's left to try?
             date = self._last_article_time + 1
         self._last_article_time = date
         self.date = '%011i' % date
diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py
index af3f46d1..2cc348e6 100644
--- a/Mailman/Cgi/admindb.py
+++ b/Mailman/Cgi/admindb.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -213,9 +213,10 @@ def main():
             doc.AddItem(Link(admindburl,
                              _('Click here to reload this page.')))
             # Put 'Logout' link before the footer
+            doc.AddItem('\n<div align="right"><font size="+2">')
             doc.AddItem(Link('%s/logout' % admindburl,
-                '<div align="right"><font size="+2"><b>%s</b></font></div>' %
-                _('Logout')))
+                '<b>%s</b>' % _('Logout')))
+            doc.AddItem('</font></div>\n')
             doc.AddItem(mlist.GetMailmanFooter())
             print doc.Format()
             mlist.Save()
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 04d7db8a..3569cc07 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -469,6 +469,12 @@ PUBLIC_MBOX = No
 #DELIVERY_MODULE = 'Sendmail'
 DELIVERY_MODULE = 'SMTPDirect'
 
+# Sometimes there are 'low level' smtplib failures that are difficult to
+# debug.  To enable very verbose debugging info from smtplib to Mailman's
+# error log, set the following to 1.  This will only work if
+# DELIVERY_MODULE = 'SMTPDirect' and Python is >= 2.4.
+SMTPLIB_DEBUG_LEVEL = 0
+
 # MTA should name a module in Mailman/MTA which provides the MTA specific
 # functionality for creating and removing lists.  Some MTAs like Exim can be
 # configured to automatically recognize new lists, in which case the MTA
@@ -556,6 +562,22 @@ SMTPPORT = 0                                      # default from smtplib
 # when DELIVERY_MODULE is 'Sendmail'.
 SENDMAIL_CMD = '/usr/lib/sendmail'
 
+# SMTP authentication for DELIVERY_MODULE = 'SMTPDirect'.  To enable SASL
+# authentication for SMTPDirect, set SMTP_AUTH = Yes and provide appropriate
+# settings for SMTP_USER and SMTP_PASSWD.
+SMTP_AUTH = No
+SMTP_USER = ''
+SMTP_PASSWD = ''
+
+# If using SASL authentication (SMTP_AUTH = Yes), set the following to Yes
+# to also use TLS. This has no effect if SMTP_AUTH = No.
+SMTP_USE_TLS = No
+
+# When using TLS the following should be set to the hostname that should be
+# used in order to identify Mailman to the SMTP server.  By default, it
+# uses DEFAULT_URL_HOST. Normally, you should not change this.
+SMTP_HELO_HOST = DEFAULT_URL_HOST
+
 # Set these variables if you need to authenticate to your NNTP server for
 # Usenet posting or reading.  If no authentication is necessary, specify None
 # for both variables.
diff --git a/Mailman/Handlers/Hold.py b/Mailman/Handlers/Hold.py
index 5452d06a..2faebae1 100644
--- a/Mailman/Handlers/Hold.py
+++ b/Mailman/Handlers/Hold.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -220,7 +220,12 @@ def hold_for_approval(mlist, msg, msgdata, exc):
     # We need to send both the reason and the rejection notice through the
     # translator again, because of the games we play above
     reason = Utils.wrap(exc.reason_notice())
-    msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist))
+    if isinstance(exc, NonMemberPost) and mlist.nonmember_rejection_notice:
+        msgdata['rejection_notice'] = Utils.wrap(
+                                  mlist.nonmember_rejection_notice.replace(
+                                      '%(listowner)s', owneraddr))
+    else:
+        msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist))
     id = mlist.HoldMessage(msg, reason, msgdata)
     # Now we need to craft and send a message to the list admin so they can
     # deal with the held message.
diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py
index 1d11d19a..3b489c2f 100644
--- a/Mailman/Handlers/SMTPDirect.py
+++ b/Mailman/Handlers/SMTPDirect.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -61,7 +61,38 @@ class Connection:
 
     def __connect(self):
         self.__conn = smtplib.SMTP()
+        self.__conn.set_debuglevel(mm_cfg.SMTPLIB_DEBUG_LEVEL)
         self.__conn.connect(mm_cfg.SMTPHOST, mm_cfg.SMTPPORT)
+        if mm_cfg.SMTP_AUTH:
+            if mm_cfg.SMTP_USE_TLS:
+                try:
+                    self.__conn.starttls()
+                except SMTPException, e:
+                    syslog('smtp-failure', 'SMTP TLS error: %s', e)
+                    self.quit()
+                    raise
+                try:
+                    self.__conn.ehlo(mm_cfg.SMTP_HELO_HOST)
+                except SMTPException, e:
+                    syslog('smtp-failure', 'SMTP EHLO error: %s', e)
+                    self.quit()
+                    raise
+            try:
+                self.__conn.login(mm_cfg.SMTP_USER, mm_cfg.SMTP_PASSWD)
+            except smtplib.SMTPHeloError, e:
+                syslog('smtp-failure', 'SMTP HELO error: %s', e)
+                self.quit()
+                raise
+            except smtplib.SMTPAuthenticationError, e:
+                syslog('smtp-failure', 'SMTP AUTH error: %s', e)
+                self.quit()
+                raise
+            except smtplib.SMTPException, e:
+                syslog('smtp-failure',
+                       'SMTP - no suitable authentication method found: %s', e)
+                self.quit()
+                raise
+
         self.__numsessions = mm_cfg.SMTP_MAX_SESSIONS_PER_CONNECTION
 
     def sendmail(self, envsender, recips, msgtext):
diff --git a/Mailman/MTA/Postfix.py b/Mailman/MTA/Postfix.py
index 8860459e..d18d850b 100644
--- a/Mailman/MTA/Postfix.py
+++ b/Mailman/MTA/Postfix.py
@@ -28,6 +28,7 @@ from Mailman import mm_cfg
 from Mailman import Utils
 from Mailman import LockFile
 from Mailman.i18n import C_
+from Mailman.MailList import MailList
 from Mailman.MTA.Utils import makealiases
 from Mailman.Logging.Syslog import syslog
 
@@ -132,6 +133,11 @@ def _addvirtual(mlist, fp):
     sitedest = Utils.ParseEmail(siteaddr)[0]
     if mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN:
         loopdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN
+        sitedest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN
+    # If the site list's host_name is a virtual domain, adding it to the
+    # SITE ADDRESSES will duplicate the list posting entry, so comment it.
+    if _isvirtual(MailList(mm_cfg.MAILMAN_SITE_LIST, lock=False)):
+        siteaddr = '#' + siteaddr
     # Seek to the end of the text file, but if it's empty write the standard
     # disclaimer, and the loop catch address and site address.
     fp.seek(0, 2)
@@ -179,6 +185,9 @@ def _check_for_virtual_loopaddr(mlist, filename):
     loopdest = Utils.ParseEmail(loopaddr)[0]
     siteaddr = Utils.get_site_email(mlist.host_name)
     sitedest = Utils.ParseEmail(siteaddr)[0]
+    if mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN:
+        loopdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN
+        sitedest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN
     infp = open(filename)
     omask = os.umask(007)
     try:
@@ -230,7 +239,7 @@ def _check_for_virtual_loopaddr(mlist, filename):
                 print >> outfp, '%s\t%s' % (siteaddr, sitedest)
                 outfp.write(line)
                 break
-            elif line.startswith(siteaddr):
+            elif line.startswith(siteaddr) or line.startswith('#' + siteaddr):
                 # We just found it
                 outfp.write(line)
                 break
diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py
index c5cc3d94..a9f6f000 100644
--- a/Mailman/Queue/CommandRunner.py
+++ b/Mailman/Queue/CommandRunner.py
@@ -134,7 +134,8 @@ class Results:
             handler = sys.modules[modname]
         # ValueError can be raised if cmd has dots in it.
         # and KeyError if cmd is otherwise good but ends with a dot.
-        except (ImportError, ValueError, KeyError):
+        # and TypeError if cmd has a null byte.
+        except (ImportError, ValueError, KeyError, TypeError):
             # If we're on line zero, it was the Subject: header that didn't
             # contain a command.  It's possible there's a Re: prefix (or
             # localized version thereof) on the Subject: line that's messing
diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py
index 449532fb..fe693f28 100644
--- a/Mailman/Queue/NewsRunner.py
+++ b/Mailman/Queue/NewsRunner.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -12,7 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
 """NNTP queue runner."""
 
@@ -109,7 +110,11 @@ def prepare_message(mlist, msg, msgdata):
                        or msgdata.get('origsubj')
     if not mlist.news_prefix_subject_too and stripped_subject is not None:
         del msg['subject']
-        msg['subject'] = stripped_subject
+        msg['Subject'] = stripped_subject
+    # Make sure we have a non-blank subject.
+    if not msg.get('subject', ''):
+        del msg['subject']
+        msg['Subject'] = '(no subject)'
     # Add the appropriate Newsgroups: header
     if msg['newsgroups'] is not None:
         # This message is gated from our list to it's associated usnet group.
@@ -125,6 +130,9 @@ def prepare_message(mlist, msg, msgdata):
     # isn't ours with one of ours, so we need to parse it to be sure we're not
     # looping.
     #
+    # We also add the original Message-ID: to References: to try to help with
+    # threading issues and create another header for documentation.
+    #
     # Our Message-ID format is <mailman.secs.pid.listname@hostname>
     msgid = msg['message-id']
     hackmsgid = True
@@ -137,6 +145,18 @@ def prepare_message(mlist, msg, msgdata):
     if hackmsgid:
         del msg['message-id']
         msg['Message-ID'] = Utils.unique_message_id(mlist)
+        if msgid:
+            msg['X-Mailman-Original-Message-ID'] = msgid
+            refs = msg['references']
+            del msg['references']
+            if not refs:
+                refs = msg.get('in-reply-to', '')
+            else:
+                msg['X-Mailman-Original-References'] = refs
+            if refs:
+                msg['References'] = '\n '.join([refs, msgid])
+            else:
+                msg['References'] = msgid
     # Lines: is useful
     if msg['Lines'] is None:
         # BAW: is there a better way?
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index f821f13a..2dbaef0b 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -1170,6 +1170,8 @@ def get_suffixes(url):
     global s_dict
     if s_dict:
         return
+    if not url:
+        return
     try:
         d = urllib2.urlopen(url)
     except urllib2.URLError, e:
@@ -1241,7 +1243,8 @@ def IsDMARCProhibited(mlist, email):
         return False
 
     email = email.lower()
-    at_sign = email.find('@')
+    # Scan from the right in case quoted local part has an '@'.
+    at_sign = email.rfind('@')
     if at_sign < 1:
         return False
     f_dom = email[at_sign+1:]
@@ -1250,12 +1253,12 @@ def IsDMARCProhibited(mlist, email):
         return x
     o_dom = get_org_dom(f_dom)
     if o_dom != f_dom:
-        x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom)
+        x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True)
         if x != 'continue':
             return x
     return False
 
-def _DMARCProhibited(mlist, email, dmarc_domain):
+def _DMARCProhibited(mlist, email, dmarc_domain, org=False):
 
     try:
         resolver = dns.resolver.Resolver()
@@ -1267,7 +1270,7 @@ def _DMARCProhibited(mlist, email, dmarc_domain):
     except DNSException, e:
         syslog('error',
                'DNSException: Unable to query DMARC policy for %s (%s). %s',
-              email, dmarc_domain, e.__class__)
+              email, dmarc_domain, e.__doc__)
         return 'continue'
     else:
 # people are already being dumb, don't trust them to provide honest DNS
@@ -1315,14 +1318,23 @@ def _DMARCProhibited(mlist, email, dmarc_domain):
                        testing them all""",
                         dmarc_domain, len(dmarc))
             for entry in dmarcs:
-                if re.search(r'\bp=reject\b', entry, re.IGNORECASE):
+                mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE)
+                if org and mo:
+                    policy = mo.group(1).lower()
+                else:
+                    mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE)
+                    if mo:
+                        policy = mo.group(1).lower()
+                    else:
+                        continue
+                if policy == 'reject':
                     syslog('vette',
                       '%s: DMARC lookup for %s (%s) found p=reject in %s = %s',
                       mlist.real_name,  email, dmarc_domain, name, entry)
                     return True
 
                 if (mlist.dmarc_quarantine_moderation_action and
-                    re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)):
+                    policy == 'quarantine'):
                     syslog('vette',
                   '%s: DMARC lookup for %s (%s) found p=quarantine in %s = %s',
                           mlist.real_name,  email, dmarc_domain, name, entry)
@@ -1331,7 +1343,7 @@ def _DMARCProhibited(mlist, email, dmarc_domain):
                 if (mlist.dmarc_none_moderation_action and
                     mlist.dmarc_quarantine_moderation_action and
                     mlist.dmarc_moderation_action in (1, 2) and
-                    re.search(r'\bp=none\b', entry, re.IGNORECASE)):
+                    policy == 'none'):
                     syslog('vette',
                   '%s: DMARC lookup for %s (%s) found p=none in %s = %s',
                           mlist.real_name,  email, dmarc_domain, name, entry)
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 132648a8..413ac531 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -16,7 +16,7 @@
 # USA.
 
 # Mailman version
-VERSION = '2.1.21'
+VERSION = '2.1.22'
 
 # And as a hex number in the manner of PY_VERSION_HEX
 ALPHA = 0xa
@@ -28,7 +28,7 @@ FINAL = 0xf
 
 MAJOR_REV = 2
 MINOR_REV = 1
-MICRO_REV = 21
+MICRO_REV = 22
 REL_LEVEL = FINAL
 # at most 15 beta releases!
 REL_SERIAL = 0