merge lp:mailman/2.1 up to rev 1808

13 files changed, 168 insertions, 141 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 5e158e5b..fabd95bd 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -166,6 +166,15 @@ HTML_TO_PLAIN_TEXT_COMMAND = '/usr/bin/lynx -dump %(filename)s'
 # character that doesn't match this class.  Do not include '/' in this list.
 ACCEPTABLE_LISTNAME_CHARACTERS = '[-+_.=a-z0-9]'
 
+# The number of characters in the longest listname in the installation.  The
+# fix for LP: #1780874 truncates list names in web URLs to this length to avoid
+# a content spoofing vulnerability.  If this is left at its default value of
+# 0, the length of the longest listname is calculated on every web access.
+# This can have performance implications in installations with a very large
+# number of lists.  To use this feature to avoid the calculation, set this to
+# a number equal to the length of the longest expected valid list name.
+MAX_LISTNAME_LENGTH = 0
+
 # Shall the user's real names be displayed along with their email addresses
 # in list rosters?  Defaults to No to preserve prior behavior.
 ROSTER_DISPLAY_REALNAME = No
diff --git a/Mailman/Digester.py b/Mailman/Digester.py
index 8a65043b..3dc7ce49 100644
--- a/Mailman/Digester.py
+++ b/Mailman/Digester.py
@@ -31,20 +31,20 @@ from Mailman.i18n import _
 
 class Digester:
     def InitVars(self):
-	# Configurable
-	self.digestable = mm_cfg.DEFAULT_DIGESTABLE
-	self.digest_is_default = mm_cfg.DEFAULT_DIGEST_IS_DEFAULT
-	self.mime_is_default_digest = mm_cfg.DEFAULT_MIME_IS_DEFAULT_DIGEST
-	self.digest_size_threshhold = mm_cfg.DEFAULT_DIGEST_SIZE_THRESHHOLD
-	self.digest_send_periodic = mm_cfg.DEFAULT_DIGEST_SEND_PERIODIC
-	self.next_post_number = 1
-	self.digest_header = mm_cfg.DEFAULT_DIGEST_HEADER
-	self.digest_footer = mm_cfg.DEFAULT_DIGEST_FOOTER
+        # Configurable
+        self.digestable = mm_cfg.DEFAULT_DIGESTABLE
+        self.digest_is_default = mm_cfg.DEFAULT_DIGEST_IS_DEFAULT
+        self.mime_is_default_digest = mm_cfg.DEFAULT_MIME_IS_DEFAULT_DIGEST
+        self.digest_size_threshhold = mm_cfg.DEFAULT_DIGEST_SIZE_THRESHHOLD
+        self.digest_send_periodic = mm_cfg.DEFAULT_DIGEST_SEND_PERIODIC
+        self.next_post_number = 1
+        self.digest_header = mm_cfg.DEFAULT_DIGEST_HEADER
+        self.digest_footer = mm_cfg.DEFAULT_DIGEST_FOOTER
         self.digest_volume_frequency = mm_cfg.DEFAULT_DIGEST_VOLUME_FREQUENCY
-	# Non-configurable.
+        # Non-configurable.
         self.one_last_digest = {}
-	self.digest_members = {}
-	self.next_digest_number = 1
+        self.digest_members = {}
+        self.next_digest_number = 1
         self.digest_last_sent_at = 0
 
     def send_digest_now(self):
diff --git a/Mailman/Gui/Archive.py b/Mailman/Gui/Archive.py
index fc313c00..8167a001 100644
--- a/Mailman/Gui/Archive.py
+++ b/Mailman/Gui/Archive.py
@@ -27,18 +27,18 @@ class Archive(GUIBase):
     def GetConfigInfo(self, mlist, category, subcat=None):
         if category <> 'archive':
             return None
-	return [
+        return [
             _("List traffic archival policies."),
 
-	    ('archive', mm_cfg.Toggle, (_('No'), _('Yes')), 0, 
-	     _('Archive messages?')),
+            ('archive', mm_cfg.Toggle, (_('No'), _('Yes')), 0, 
+             _('Archive messages?')),
 
-	    ('archive_private', mm_cfg.Radio, (_('public'), _('private')), 0,
+            ('archive_private', mm_cfg.Radio, (_('public'), _('private')), 0,
              _('Is archive file source for public or private archival?')),
 
- 	    ('archive_volume_frequency', mm_cfg.Radio, 
+            ('archive_volume_frequency', mm_cfg.Radio, 
              (_('Yearly'), _('Monthly'), _('Quarterly'),
               _('Weekly'), _('Daily')),
              0,
- 	     _('How often should a new archive volume be started?')),
-	    ]
+             _('How often should a new archive volume be started?')),
+            ]
diff --git a/Mailman/Gui/Digest.py b/Mailman/Gui/Digest.py
index 55cee19d..9eafb1d2 100644
--- a/Mailman/Gui/Digest.py
+++ b/Mailman/Gui/Digest.py
@@ -40,37 +40,37 @@ class Digest(GUIBase):
             return None
         WIDTH = mm_cfg.TEXTFIELDWIDTH
 
-	info = [
+        info = [
             _("Batched-delivery digest characteristics."),
 
-	    ('digestable', mm_cfg.Toggle, (_('No'), _('Yes')), 1,
-	     _('Can list members choose to receive list traffic '
-	       'bunched in digests?')),
+            ('digestable', mm_cfg.Toggle, (_('No'), _('Yes')), 1,
+             _('Can list members choose to receive list traffic '
+               'bunched in digests?')),
 
-	    ('digest_is_default', mm_cfg.Radio, 
-	     (_('Regular'), _('Digest')), 0,
-	     _('Which delivery mode is the default for new users?')),
+            ('digest_is_default', mm_cfg.Radio, 
+             (_('Regular'), _('Digest')), 0,
+             _('Which delivery mode is the default for new users?')),
 
-	    ('mime_is_default_digest', mm_cfg.Radio, 
-	     (_('Plain'), _('MIME')), 0,
-	     _('When receiving digests, which format is default?')),
+            ('mime_is_default_digest', mm_cfg.Radio, 
+             (_('Plain'), _('MIME')), 0,
+             _('When receiving digests, which format is default?')),
 
-	    ('digest_size_threshhold', mm_cfg.Number, 3, 0,
-	     _('How big in Kb should a digest be before it gets sent out?'
+            ('digest_size_threshhold', mm_cfg.Number, 3, 0,
+             _('How big in Kb should a digest be before it gets sent out?'
                '  0 implies no maximum size.')),
 
- 	    ('digest_send_periodic', mm_cfg.Radio, (_('No'), _('Yes')), 1,
-	     _('Should a digest be dispatched daily when the size threshold '
-	       "isn't reached?")),
+            ('digest_send_periodic', mm_cfg.Radio, (_('No'), _('Yes')), 1,
+             _('Should a digest be dispatched daily when the size threshold '
+               "isn't reached?")),
 
             ('digest_header', mm_cfg.Text, (4, WIDTH), 0,
-	     _('Header added to every digest'),
+             _('Header added to every digest'),
              _("Text attached (as an initial message, before the table"
                " of contents) to the top of digests. ")
              + Utils.maketext('headfoot.html', raw=1, mlist=mlist)),
 
-	    ('digest_footer', mm_cfg.Text, (4, WIDTH), 0,
-	     _('Footer added to every digest'),
+            ('digest_footer', mm_cfg.Text, (4, WIDTH), 0,
+             _('Footer added to every digest'),
              _("Text attached (as a final message) to the bottom of digests. ")
              + Utils.maketext('headfoot.html', raw=1, mlist=mlist)),
 
@@ -89,7 +89,7 @@ class Digest(GUIBase):
             ('_send_digest_now', mm_cfg.Toggle, (_('No'), _('Yes')), 0,
              _('''Should Mailman send the next digest right now, if it is not
              empty?''')),
-	    ]
+            ]
 
 ##        if mm_cfg.OWNERS_CAN_ENABLE_PERSONALIZATION:
 ##            info.extend([
diff --git a/Mailman/LockFile.py b/Mailman/LockFile.py
index d7cd9252..45d92cd6 100644
--- a/Mailman/LockFile.py
+++ b/Mailman/LockFile.py
@@ -195,7 +195,7 @@ class LockFile:
         self.__logprefix = os.path.split(self.__lockfile)[1]
         # For transferring ownership across a fork.
         self.__owned = True
-	
+
     def __repr__(self):
         return '<LockFile %s: %s [%s: %ssec] pid=%s>' % (
             id(self), self.__lockfile,
diff --git a/Mailman/Logging/StampedLogger.py b/Mailman/Logging/StampedLogger.py
index 2657d7fc..5d259e38 100644
--- a/Mailman/Logging/StampedLogger.py
+++ b/Mailman/Logging/StampedLogger.py
@@ -42,14 +42,14 @@ class StampedLogger(Logger):
     """
     def __init__(self, category, label=None, manual_reprime=0, nofail=1,
                  immediate=1):
-	"""If specified, optional label is included after timestamp.
+        """If specified, optional label is included after timestamp.
         Other options are passed to the Logger class initializer.
         """
-	self.__label = label
+        self.__label = label
         self.__manual_reprime = manual_reprime
         self.__primed = 1
         self.__bol = 1
-	Logger.__init__(self, category, nofail, immediate)
+        Logger.__init__(self, category, nofail, immediate)
 
     def reprime(self):
         """Reset so timestamp will be included with next write."""
@@ -77,13 +77,13 @@ class StampedLogger(Logger):
             self.__bol = 0
 
     def writelines(self, lines):
-	first = 1
-	for l in lines:
-	    if first:
-		self.write(l)
-		first = 0
-	    else:
-		if l and l[0] not in [' ', '\t', '\n']:
-		    Logger.write(self, ' ' + l)
-		else:
-		    Logger.write(self, l)
+        first = 1
+        for l in lines:
+            if first:
+                self.write(l)
+                first = 0
+            else:
+                if l and l[0] not in [' ', '\t', '\n']:
+                    Logger.write(self, ' ' + l)
+                else:
+                    Logger.write(self, l)
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index d74978af..f4b38b49 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -424,7 +424,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         self.dmarc_none_moderation_action = (
             mm_cfg.DEFAULT_DMARC_NONE_MODERATION_ACTION)
         self.dmarc_moderation_notice = ''
-	self.dmarc_moderation_addresses = []
+        self.dmarc_moderation_addresses = []
         self.dmarc_wrapped_message_text = (
             mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT)
         self.equivalent_domains = (
diff --git a/Mailman/Message.py b/Mailman/Message.py
index 2d68fd8f..f4ca20c7 100644
--- a/Mailman/Message.py
+++ b/Mailman/Message.py
@@ -236,7 +236,7 @@ class Message(email.Message.Message):
         Mailman.Message.Generator.
 
         Operates like email.Message.Message.as_string, only
-	using Mailman's Message.Generator class. Only the top headers will
+        using Mailman's Message.Generator class. Only the top headers will
         get folded.
         """
         fp = StringIO()
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 605d0976..10629fc4 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -292,11 +292,16 @@ def GetPathPieces(envar='PATH_INFO'):
                    remote)
         # Check for listname injections that won't be websafed.
         pieces = [p for p in path.split('/') if p]
-        # Get the longest listname or 20 if none.
-        if list_names():
-            longest = max([len(x) for x in list_names()])
+        # Get the longest listname or 20 if none or use MAX_LISTNAME_LENGTH if
+        # provided > 0.
+        if mm_cfg.MAX_LISTNAME_LENGTH > 0:
+            longest = mm_cfg.MAX_LISTNAME_LENGTH
         else:
-            longest = 20
+            lst_names = list_names()
+            if lst_names:
+                longest = max([len(x) for x in lst_names])
+            else:
+                longest = 20
         if pieces and len(pieces[0]) > longest:
             syslog('mischief',
                'Hostile listname: listname=%s: remote=%s', pieces[0], remote)
@@ -1529,7 +1534,7 @@ def banned_ip(ip):
             ptr = ipaddress.ip_address(uip).reverse_pointer
         except ValueError:
             return False
-	lookup = '{0}.zen.spamhaus.org'.format('.'.join(ptr.split('.')[:-2]))
+        lookup = '{0}.zen.spamhaus.org'.format('.'.join(ptr.split('.')[:-2]))
     else:
         parts = ip.split('.')
         if len(parts) != 4:
diff --git a/NEWS b/NEWS
index cfca39a2..85b15ff2 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,13 @@ Here is a history of user visible changes to Mailman.
       From: addresses listed or matching listed regexps.  This can be used
       to modify mail to addresses that don't accept external mail From:
       themselves.
+
+    - There is a new MAX_LISTNAME_LENGTH setting.  The fix for LP: #1780874
+      obtains a list of the names of all the all the lists in the installation
+      in order to determine the maximum length of a legitimate list name.  It
+      does this on every web access and on sites with a very large number of
+      lists, this can have performance implications.  See the description in
+      Defaults.py for more information.
   
   Bug Fixes and other patches
   
@@ -28,6 +35,9 @@ Here is a history of user visible changes to Mailman.
 
     - Corrected and augmented some security log messages.  (LP: #1810098)
 
+    - Implemented use of QRUNNER_SLEEP_TIME for bin/qrunner --runner=All.
+      (LP: #1818205)
+
 2.1.29 (24-Jul-2018)
 
   Bug Fixes
diff --git a/bin/qrunner b/bin/qrunner
index 0ca2f993..911276e3 100644
--- a/bin/qrunner
+++ b/bin/qrunner
@@ -75,6 +75,7 @@ operation.  It is only useful for debugging if it is run separately.
 import sys
 import getopt
 import signal
+import time
 
 import paths
 from Mailman import mm_cfg
@@ -268,6 +269,8 @@ def main():
                 qrunner.run()
             if once:
                 break
+            if mm_cfg.QRUNNER_SLEEP_TIME > 0:
+                time.sleep(mm_cfg.QRUNNER_SLEEP_TIME)
         syslog('qrunner', 'Main qrunner loop exiting.')
     # All done
     sys.exit(loop.status)
diff --git a/contrib/courier-to-mailman.py b/contrib/courier-to-mailman.py
index 922401fb..320a6e94 100644
--- a/contrib/courier-to-mailman.py
+++ b/contrib/courier-to-mailman.py
@@ -57,69 +57,69 @@ MailmanOwner = "postmaster@localhost"; # Postmaster and abuse mail recepient.
 import sys, os, re, string
 
 def main():
-	os.nice(5)  # Handle mailing lists at non-interactive priority.
+        os.nice(5)  # Handle mailing lists at non-interactive priority.
 
-	os.chdir(MailmanVar + "/lists")
+        os.chdir(MailmanVar + "/lists")
 
-	try:
-		local = string.lower(os.environ["LOCAL"])
-	except:
-		# This might happen if we're not using qmail.
-		sys.stderr.write("LOCAL not set in environment?\n")
-		sys.exit(112)
+        try:
+                local = string.lower(os.environ["LOCAL"])
+        except:
+                # This might happen if we're not using qmail.
+                sys.stderr.write("LOCAL not set in environment?\n")
+                sys.exit(112)
 
-	names = ("root", "postmaster", "mailer-daemon", "mailman-owner", "owner",
-			 "abuse")
-	for i in names:
-		if i == local:
-			os.execv("/usr/bin/sendmail",
-					 ("/usr/bin/sendmail", MailmanOwner))
-			sys.exit(0)
+        names = ("root", "postmaster", "mailer-daemon", "mailman-owner", "owner",
+                         "abuse")
+        for i in names:
+                if i == local:
+                        os.execv("/usr/bin/sendmail",
+                                         ("/usr/bin/sendmail", MailmanOwner))
+                        sys.exit(0)
 
-	type = "post"
-	listname = string.lower(local)
-	types = (("-admin$", "admin"),
-			 ("-bounces$", "bounces"),
-			 ("-bounces\+.*$", "bounces"),		# for VERP
-			 ("-confirm$", "confirm"),
-			 ("-confirm\+.*$", "confirm"),
-			 ("-join$", "join"),
-			 ("-leave$", "leave"),
-			 ("-owner$", "owner"),
-			 ("-request$", "request"),
-			 ("-subscribe$", "subscribe"),
-			 ("-unsubscribe$", "unsubscribe"))
+        type = "post"
+        listname = string.lower(local)
+        types = (("-admin$", "admin"),
+                         ("-bounces$", "bounces"),
+                         ("-bounces\+.*$", "bounces"),          # for VERP
+                         ("-confirm$", "confirm"),
+                         ("-confirm\+.*$", "confirm"),
+                         ("-join$", "join"),
+                         ("-leave$", "leave"),
+                         ("-owner$", "owner"),
+                         ("-request$", "request"),
+                         ("-subscribe$", "subscribe"),
+                         ("-unsubscribe$", "unsubscribe"))
 
-	for i in types:
-		if re.search(i[0],local):
-			type = i[1]
-			listname = re.sub(i[0],"",local)
+        for i in types:
+                if re.search(i[0],local):
+                        type = i[1]
+                        listname = re.sub(i[0],"",local)
 
-	if os.path.exists(listname):
-		os.execv(MailmanHome + "/mail/mailman",
-				 (MailmanHome + "/mail/mailman", type, listname))
-	else:
-		bounce()
+        if os.path.exists(listname):
+                os.execv(MailmanHome + "/mail/mailman",
+                                 (MailmanHome + "/mail/mailman", type, listname))
+        else:
+                bounce()
 
-	sys.exit(111)
+        sys.exit(111)
 
 def bounce():
-	bounce_message = """\
+        bounce_message = """\
 TO ACCESS THE MAILING LIST SYSTEM: Start your web browser on
 http://%s/
 That web page will help you subscribe or unsubscribe, and will
 give you directions on how to post to each mailing list.\n"""
-	sys.stderr.write(bounce_message % (os.environ["HOST"]))
-	sys.exit(100)
+        sys.stderr.write(bounce_message % (os.environ["HOST"]))
+        sys.exit(100)
 
 try:
-	sys.exit(main())
+        sys.exit(main())
 except SystemExit, argument:
-	sys.exit(argument)
+        sys.exit(argument)
 
 except Exception, argument:
-	info = sys.exc_info()
-	trace = info[2]
-	sys.stderr.write("%s %s\n" % (sys.exc_type, argument))
-	sys.stderr.write("LINE %d\n" % (trace.tb_lineno))
-	sys.exit(111) # Soft failure, try again later.
+        info = sys.exc_info()
+        trace = info[2]
+        sys.stderr.write("%s %s\n" % (sys.exc_type, argument))
+        sys.stderr.write("LINE %d\n" % (trace.tb_lineno))
+        sys.exit(111) # Soft failure, try again later.
diff --git a/contrib/rotatelogs.py b/contrib/rotatelogs.py
index 0341d811..b28638b1 100644
--- a/contrib/rotatelogs.py
+++ b/contrib/rotatelogs.py
@@ -37,7 +37,7 @@ and the logs are rotated.
 #  Please direct questions on this to the above address.
 #
 
-showLines = 100		#  lines of log messages to display before truncating
+showLines = 100         #  lines of log messages to display before truncating
 
 import sys, os, string, time, errno
 import paths
@@ -59,45 +59,45 @@ text.append('')
 logDate = time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time()))
 textSend = 0
 for log in ( 'error', 'smtp-failures' ):
-	fileName = os.path.join(mm_cfg.LOG_DIR, log)
+        fileName = os.path.join(mm_cfg.LOG_DIR, log)
 
-	#  rotate file if it contains any data
-	stats = os.stat(fileName)
-	if stats[stat.ST_SIZE] < 1: continue
-	fileNameNew = '%s.%s' % ( fileName, logDate )
-	newLogfiles.append(fileNameNew)
-	os.rename(fileName, fileNameNew)
-	open(fileName, 'w')
-	os.chmod(fileName, stat.S_IMODE(stats[stat.ST_MODE]))
-	try: os.chown(fileName, stats[stat.ST_UID], stats[stat.ST_GID])
-	except OSError: pass    #  permission denied, DOH!
+        #  rotate file if it contains any data
+        stats = os.stat(fileName)
+        if stats[stat.ST_SIZE] < 1: continue
+        fileNameNew = '%s.%s' % ( fileName, logDate )
+        newLogfiles.append(fileNameNew)
+        os.rename(fileName, fileNameNew)
+        open(fileName, 'w')
+        os.chmod(fileName, stat.S_IMODE(stats[stat.ST_MODE]))
+        try: os.chown(fileName, stats[stat.ST_UID], stats[stat.ST_GID])
+        except OSError: pass    #  permission denied, DOH!
 
-	textSend = 1
-	tmp = '#  FILE: %s  #' % fileNameNew
-	text.append('#' * len(tmp))
-	text.append(tmp)
-	text.append('#' * len(tmp))
-	text.append('')
+        textSend = 1
+        tmp = '#  FILE: %s  #' % fileNameNew
+        text.append('#' * len(tmp))
+        text.append(tmp)
+        text.append('#' * len(tmp))
+        text.append('')
 
-	linesLeft = showLines	#  e-mail first linesLeft of log files
-	for line in fileinput.input(fileNameNew):
-		if linesLeft == 0:
-			text.append('[... truncated ...]')
-			break
-		linesLeft = linesLeft - 1
-		line = string.rstrip(line)
-		text.append(line)
-	text.append('')
+        linesLeft = showLines   #  e-mail first linesLeft of log files
+        for line in fileinput.input(fileNameNew):
+                if linesLeft == 0:
+                        text.append('[... truncated ...]')
+                        break
+                linesLeft = linesLeft - 1
+                line = string.rstrip(line)
+                text.append(line)
+        text.append('')
 
 #  send message if we've actually found anything
 if textSend:
-	text = string.join(text, '\n') + '\n'
-	siteowner = Utils.get_site_email()
-	Utils.SendTextToUser(
-		'Mailman Log Report -- %s' % time.ctime(time.time()),
-		text, siteowner, siteowner)
+        text = string.join(text, '\n') + '\n'
+        siteowner = Utils.get_site_email()
+        Utils.SendTextToUser(
+                'Mailman Log Report -- %s' % time.ctime(time.time()),
+                text, siteowner, siteowner)
 
 #  compress any log-files we made
 if hasattr(mm_cfg, 'COMPRESS_LOGFILES_WITH') and mm_cfg.COMPRESS_LOGFILES_WITH:
-	for file in newLogfiles:
-		os.system(mm_cfg.COMPRESS_LOGFILES_WITH % file)
+        for file in newLogfiles:
+                os.system(mm_cfg.COMPRESS_LOGFILES_WITH % file)