- CGI/admin.py

   The email address which forms a part of the various CGI data keys
   in the admin membership list is now urllib.quote()ed. This allows
   changing options for and unsubbing an address which contains a
   double-quote character.

- CGI/admindb.py
   Added additional test to not display "Database Updated ..." when
   coming from the login page.

- CGI/roster.py, HTMLFormatter.py
   Changed to show hidden members when authorization is site or list's
   admin or moterator password.  Patch 1587651.

- Defaults.py.in, Handlers/Cleanse_DKIM.py
   Added a new REMOVE_DKIM_HEADERS Defaults.py/mm_cfg.py setting (default
   = No) to control removing dkim/domainkey signatures from posts and
   mail to -owner.

- Handlers/Decorate.py, Handlers/Scrubber.py
   Changed to preserve format=flowed and delsp=yes in the Content-Type:
   of the body when adding header/footer and when scrubbing attachments
   and to remove trailing spaces from the header/footer lines so they
   won't be flowed. Bug 1495122.

   Fixed a scrubber issue where the i18n translated 'next part' separator
   can be garbled if the list charset is different from the message.

- Queue/Runner.py. Queue/Switchboard.py
   Now that we have .bak queue entries for recovery, it is no longer the
   case that an unparseable message is lost.  In this case, and in case
   of other exceptions when dequeueing, I added a preservation feature
   to move the .bak file to qfiles/shunt as a .psv file and write an
   appropriate log entry.  It is also possible for an attempt to shunt
   a message to fail.  One example that occurred in practice (bug 1656289)
   was caused by a huge message that threw a MemoryError in processing and
   then threw another MemoryError in the attempt to pickle the message for
   the shunt queue.  In this case as well, I log and attempt to preserve
   the original queue entry by renaming.

3 files changed, 44 insertions, 6 deletions

diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py
index 850e3668..0c548a9a 100644
--- a/Mailman/Handlers/CleanseDKIM.py
+++ b/Mailman/Handlers/CleanseDKIM.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -25,9 +25,12 @@ and it will also give the MTA the opportunity to regenerate valid keys
 originating at the Mailman server for the outgoing message.
 """
 
+from Mailman import mm_cfg
+
 
-
 def process(mlist, msg, msgdata):
+    if not mm_cfg.REMOVE_DKIM_HEADERS:
+        return
     del msg['domainkey-signature']
     del msg['dkim-signature']
 
diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py
index d6b20391..4c00e34d 100644
--- a/Mailman/Handlers/Decorate.py
+++ b/Mailman/Handlers/Decorate.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -17,6 +17,8 @@
 
 """Decorate a message by sticking the header and footer around it."""
 
+import re
+
 from types import ListType
 from email.MIMEText import MIMEText
 
@@ -115,9 +117,15 @@ def process(mlist, msg, msgdata):
                     payload = payload.encode(mcset)
                     newcset = mcset
                     # if this fails, fallback to outer try and wrap=true
+            format = msg.get_param('format')
+            delsp = msg.get_param('delsp')
             del msg['content-transfer-encoding']
             del msg['content-type']
             msg.set_payload(payload, newcset)
+            if format:
+                msg.set_param('Format', format)
+            if delsp:
+                msg.set_param('DelSp', delsp)
             wrap = False
         except (LookupError, UnicodeError):
             pass
@@ -211,7 +219,7 @@ def decorate(mlist, template, what, extradict={}):
         template = Utils.to_percent(template)
     # Interpolate into the template
     try:
-        text = (template % d).replace('\r\n', '\n')
+        text = re.sub(r' *\r?\n', r'\n', template % d)
     except (ValueError, TypeError), e:
         syslog('error', 'Exception while calculating %s:\n%s', what, e)
         what = what.upper()
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index 4a4a3c59..fd35cbdd 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2006 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -187,6 +187,7 @@ def process(mlist, msg, msgdata=None):
     lcset = Utils.GetCharSet(mlist.preferred_language)
     lcset_out = Charset(lcset).output_charset or lcset
     # Now walk over all subparts of this message and scrub out various types
+    format = delsp = None
     for part in msg.walk():
         ctype = part.get_type(part.get_default_type())
         # If the part is text/plain, we leave it alone
@@ -194,8 +195,21 @@ def process(mlist, msg, msgdata=None):
             # We need to choose a charset for the scrubbed message, so we'll
             # arbitrarily pick the charset of the first text/plain part in the
             # message.
+            # MAS: Also get the RFC 3676 stuff from this part. This seems to
+            # work OK for scrub_nondigest.  It will also work as far as
+            # scrubbing messages for the archive is concerned, but pipermail
+            # doesn't pay any attention to the RFC 3676 parameters.  The plain
+            # format digest is going to be a disaster in any case as some of
+            # messages will be format="flowed" and some not.  ToDigest creates
+            # its own Content-Type: header for the plain digest which won't
+            # have RFC 3676 parameters. If the message Content-Type: headers
+            # are retained for display in the digest, the parameters will be
+            # there for information, but not for the MUA. This is the best we
+            # can do without having get_payload() process the parameters.
             if charset is None:
                 charset = part.get_content_charset(lcset)
+                format = part.get_param('format')
+                delsp = part.get_param('delsp')
             # TK: if part is attached then check charset and scrub if none
             if part.get('content-disposition') and \
                not part.get_content_charset():
@@ -380,7 +394,18 @@ Url : %(url)s
                 text.append(t)
         # Now join the text and set the payload
         sep = _('-------------- next part --------------\n')
+        # The i18n separator is in the list's charset. Coerce it to the
+        # message charset.
+        try:
+            s = unicode(sep, lcset, 'replace')
+            sep = s.encode(charset, 'replace')
+        except (UnicodeError, LookupError, ValueError):
+            pass
         replace_payload_by_text(msg, sep.join(text), charset)
+        if format:
+            msg.set_param('Format', format)
+        if delsp:
+            msg.set_param('DelSp', delsp)
     return msg
 
 
@@ -513,5 +538,7 @@ def save_attachment(mlist, msg, dir, filter_html=True):
         baseurl += '/'
     # A trailing space in url string may save users who are using
     # RFC-1738 compliant MUA (Not Mozilla).
-    url = baseurl + '%s/%s%s%s ' % (dir, filebase, extra, ext)
+    # Trailing space will definitely be a problem with format=flowed.
+    # Bracket the URL instead.
+    url = '<' + baseurl + '%s/%s%s%s>' % (dir, filebase, extra, ext)
     return url