- CGI/admin.py

   The email address which forms a part of the various CGI data keys
   in the admin membership list is now urllib.quote()ed. This allows
   changing options for and unsubbing an address which contains a
   double-quote character.

- CGI/admindb.py
   Added additional test to not display "Database Updated ..." when
   coming from the login page.

- CGI/roster.py, HTMLFormatter.py
   Changed to show hidden members when authorization is site or list's
   admin or moterator password.  Patch 1587651.

- Defaults.py.in, Handlers/Cleanse_DKIM.py
   Added a new REMOVE_DKIM_HEADERS Defaults.py/mm_cfg.py setting (default
   = No) to control removing dkim/domainkey signatures from posts and
   mail to -owner.

- Handlers/Decorate.py, Handlers/Scrubber.py
   Changed to preserve format=flowed and delsp=yes in the Content-Type:
   of the body when adding header/footer and when scrubbing attachments
   and to remove trailing spaces from the header/footer lines so they
   won't be flowed. Bug 1495122.

   Fixed a scrubber issue where the i18n translated 'next part' separator
   can be garbled if the list charset is different from the message.

- Queue/Runner.py. Queue/Switchboard.py
   Now that we have .bak queue entries for recovery, it is no longer the
   case that an unparseable message is lost.  In this case, and in case
   of other exceptions when dequeueing, I added a preservation feature
   to move the .bak file to qfiles/shunt as a .psv file and write an
   appropriate log entry.  It is also possible for an attempt to shunt
   a message to fail.  One example that occurred in practice (bug 1656289)
   was caused by a huge message that threw a MemoryError in processing and
   then threw another MemoryError in the attempt to pickle the message for
   the shunt queue.  In this case as well, I log and attempt to preserve
   the original queue entry by renaming.

3 files changed, 29 insertions, 24 deletions

diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 718bb0c8..d1a255d3 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -982,15 +982,16 @@ def membership_options(mlist, subcat, cgidata, doc, form):
                   }
     # Now populate the rows
     for addr in members:
+        qaddr = urllib.quote(addr)
         link = Link(mlist.GetOptionsURL(addr, obscure=1),
                     mlist.getMemberCPAddress(addr))
         fullname = Utils.uncanonstr(mlist.getMemberName(addr),
                                     mlist.preferred_language)
-        name = TextBox(addr + '_realname', fullname, size=longest).Format()
-        cells = [Center(CheckBox(addr + '_unsub', 'off', 0).Format()),
+        name = TextBox(qaddr + '_realname', fullname, size=longest).Format()
+        cells = [Center(CheckBox(qaddr + '_unsub', 'off', 0).Format()),
                  link.Format() + '<br>' +
                  name +
-                 Hidden('user', urllib.quote(addr)).Format(),
+                 Hidden('user', qaddr).Format(),
                  ]
         # Do the `mod' option
         if mlist.getMemberOption(addr, mm_cfg.Moderate):
@@ -999,7 +1000,7 @@ def membership_options(mlist, subcat, cgidata, doc, form):
         else:
             value = 'off'
             checked = 0
-        box = CheckBox('%s_mod' % addr, value, checked)
+        box = CheckBox('%s_mod' % qaddr, value, checked)
         cells.append(Center(box).Format())
         for opt in ('hide', 'nomail', 'ack', 'notmetoo', 'nodupes'):
             extra = ''
@@ -1018,23 +1019,23 @@ def membership_options(mlist, subcat, cgidata, doc, form):
             else:
                 value = 'off'
                 checked = 0
-            box = CheckBox('%s_%s' % (addr, opt), value, checked)
+            box = CheckBox('%s_%s' % (qaddr, opt), value, checked)
             cells.append(Center(box.Format() + extra))
         # This code is less efficient than the original which did a has_key on
         # the underlying dictionary attribute.  This version is slower and
         # less memory efficient.  It points to a new MemberAdaptor interface
         # method.
         if addr in mlist.getRegularMemberKeys():
-            cells.append(Center(CheckBox(addr + '_digest', 'off', 0).Format()))
+            cells.append(Center(CheckBox(qaddr + '_digest', 'off', 0).Format()))
         else:
-            cells.append(Center(CheckBox(addr + '_digest', 'on', 1).Format()))
+            cells.append(Center(CheckBox(qaddr + '_digest', 'on', 1).Format()))
         if mlist.getMemberOption(addr, mm_cfg.OPTINFO['plain']):
             value = 'on'
             checked = 1
         else:
             value = 'off'
             checked = 0
-        cells.append(Center(CheckBox('%s_plain' % addr, value, checked)))
+        cells.append(Center(CheckBox('%s_plain' % qaddr, value, checked)))
         # User's preferred language
         langpref = mlist.getMemberLanguage(addr)
         langs = mlist.GetAvailableLanguages()
@@ -1043,7 +1044,7 @@ def membership_options(mlist, subcat, cgidata, doc, form):
             selected = langs.index(langpref)
         except ValueError:
             selected = 0
-        cells.append(Center(SelectOptions(addr + '_language', langs,
+        cells.append(Center(SelectOptions(qaddr + '_language', langs,
                                           langdescs, selected)).Format())
         usertable.AddRow(cells)
     # Add the usertable and a legend
@@ -1427,7 +1428,8 @@ def change_options(mlist, category, subcat, cgidata, doc):
         errors = []
         removes = []
         for user in users:
-            if cgidata.has_key('%s_unsub' % user):
+            quser = urllib.quote(user)
+            if cgidata.has_key('%s_unsub' % quser):
                 try:
                     mlist.ApprovedDeleteMember(user, whence='member mgt page')
                     removes.append(user)
@@ -1438,7 +1440,7 @@ def change_options(mlist, category, subcat, cgidata, doc):
                 doc.addError(_('Ignoring changes to deleted member: %(user)s'),
                              tag=_('Warning: '))
                 continue
-            value = cgidata.has_key('%s_digest' % user)
+            value = cgidata.has_key('%s_digest' % quser)
             try:
                 mlist.setMemberOption(user, mm_cfg.Digests, value)
             except (Errors.AlreadyReceivingDigests,
@@ -1448,28 +1450,28 @@ def change_options(mlist, category, subcat, cgidata, doc):
                 # BAW: Hmm...
                 pass
 
-            newname = cgidata.getvalue(user+'_realname', '')
+            newname = cgidata.getvalue(quser+'_realname', '')
             newname = Utils.canonstr(newname, mlist.preferred_language)
             mlist.setMemberName(user, newname)
 
-            newlang = cgidata.getvalue(user+'_language')
+            newlang = cgidata.getvalue(quser+'_language')
             oldlang = mlist.getMemberLanguage(user)
             if Utils.IsLanguage(newlang) and newlang <> oldlang:
                 mlist.setMemberLanguage(user, newlang)
 
-            moderate = not not cgidata.getvalue(user+'_mod')
+            moderate = not not cgidata.getvalue(quser+'_mod')
             mlist.setMemberOption(user, mm_cfg.Moderate, moderate)
 
             # Set the `nomail' flag, but only if the user isn't already
             # disabled (otherwise we might change BYUSER into BYADMIN).
-            if cgidata.has_key('%s_nomail' % user):
+            if cgidata.has_key('%s_nomail' % quser):
                 if mlist.getDeliveryStatus(user) == MemberAdaptor.ENABLED:
                     mlist.setDeliveryStatus(user, MemberAdaptor.BYADMIN)
             else:
                 mlist.setDeliveryStatus(user, MemberAdaptor.ENABLED)
             for opt in ('hide', 'ack', 'notmetoo', 'nodupes', 'plain'):
                 opt_code = mm_cfg.OPTINFO[opt]
-                if cgidata.has_key('%s_%s' % (user, opt)):
+                if cgidata.has_key('%s_%s' % (quser, opt)):
                     mlist.setMemberOption(user, opt_code, 1)
                 else:
                     mlist.setMemberOption(user, opt_code, 0)
diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py
index e5fd2ade..6e8b58f8 100644
--- a/Mailman/Cgi/admindb.py
+++ b/Mailman/Cgi/admindb.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -154,9 +154,9 @@ def main():
         signal.signal(signal.SIGTERM, sigterm_handler)
 
         realname = mlist.real_name
-        if not cgidata.keys():
+        if not cgidata.keys() or cgidata.has_key('admlogin'):
             # If this is not a form submission (i.e. there are no keys in the
-            # form), then we don't need to do much special.
+            # form) or it's a login, then we don't need to do much special.
             doc.SetTitle(_('%(realname)s Administrative Database'))
         elif not details:
             # This is a form submission
diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py
index a67e5100..b53e5912 100644
--- a/Mailman/Cgi/roster.py
+++ b/Mailman/Cgi/roster.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -71,13 +71,17 @@ def main():
     # "admin"-only, then we try to cookie authenticate the user, and failing
     # that, we check roster-email and roster-pw fields for a valid password.
     # (also allowed: the list moderator, the list admin, and the site admin).
+    password = cgidata.getvalue('roster-pw', '')
+    list_hidden = mlist.WebAuthenticate((mm_cfg.AuthListModerator,
+                                         mm_cfg.AuthListAdmin,
+                                         mm_cfg.AuthSiteAdmin),
+                                        password)
     if mlist.private_roster == 0:
         # No privacy
         ok = 1
     elif mlist.private_roster == 1:
         # Members only
         addr = cgidata.getvalue('roster-email', '')
-        password = cgidata.getvalue('roster-pw', '')
         ok = mlist.WebAuthenticate((mm_cfg.AuthUser,
                                     mm_cfg.AuthListModerator,
                                     mm_cfg.AuthListAdmin,
@@ -85,7 +89,6 @@ def main():
                                    password, addr)
     else:
         # Admin only, so we can ignore the address field
-        password = cgidata.getvalue('roster-pw', '')
         ok = mlist.WebAuthenticate((mm_cfg.AuthListModerator,
                                     mm_cfg.AuthListAdmin,
                                     mm_cfg.AuthSiteAdmin),
@@ -103,7 +106,7 @@ def main():
     doc = HeadlessDocument()
     doc.set_language(lang)
 
-    replacements = mlist.GetAllReplacements(lang)
+    replacements = mlist.GetAllReplacements(lang, list_hidden)
     replacements['<mm-displang-box>'] = mlist.FormatButton(
         'displang-button',
         text = _('View this page in'))