Backporting from the trunk.

19 files changed, 364 insertions, 209 deletions

diff --git a/Mailman/Archiver/HyperArch.py b/Mailman/Archiver/HyperArch.py
index 4633228d..ea09b877 100644
--- a/Mailman/Archiver/HyperArch.py
+++ b/Mailman/Archiver/HyperArch.py
@@ -888,40 +888,43 @@ class HyperArchive(pipermail.T):
             return time.strftime("%Y-%B",datetuple)
 
 
-    def volNameToDate(self,volname):
+    def volNameToDate(self, volname):
         volname = volname.strip()
         for each in self._volre.keys():
-            match=re.match(self._volre[each],volname)
+            match = re.match(self._volre[each],volname)
             if match:
-                year=int(match.group('year'))
-                month=1
+                year = int(match.group('year'))
+                month = 1
                 day = 1
                 if each == 'quarter':
-                    q=int(match.group('quarter'))
-                    month=(q*3)-2
+                    q = int(match.group('quarter'))
+                    month = (q * 3) - 2
                 elif each == 'month':
-                    monthstr=match.group('month').lower()
-                    m=[]
+                    monthstr = match.group('month').lower()
+                    m = []
                     for i in range(1,13):
                         m.append(
                             time.strftime("%B",(1999,i,1,0,0,0,0,1,0)).lower())
                     try:
-                        month=m.index(monthstr)+1
+                        month = m.index(monthstr) + 1
                     except ValueError:
                         pass
                 elif each == 'week' or each == 'day':
                     month = int(match.group("month"))
                     day = int(match.group("day"))
-                return time.mktime((year,month,1,0,0,0,0,1,-1))
+                try:
+                    return time.mktime((year,month,1,0,0,0,0,1,-1))
+                except OverflowError:
+                    return 0.0
         return 0.0
 
     def sortarchives(self):
-        def sf(a,b,s=self):
-            al=s.volNameToDate(a)
-            bl=s.volNameToDate(b)
-            if al>bl:
+        def sf(a, b):
+            al = self.volNameToDate(a)
+            bl = self.volNameToDate(b)
+            if al > bl:
                 return 1
-            elif al<bl:
+            elif al < bl:
                 return -1
             else:
                 return 0
diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 49c6efbf..1c629c10 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -1376,7 +1376,7 @@ def change_options(mlist, category, subcat, cgidata, doc):
 
             newlang = cgidata.getvalue(user+'_language')
             oldlang = mlist.getMemberLanguage(user)
-            if newlang and newlang <> oldlang:
+            if Utils.IsLanguage(newlang) and newlang <> oldlang:
                 mlist.setMemberLanguage(user, newlang)
 
             moderate = not not cgidata.getvalue(user+'_mod')
diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py
index e6b71cda..49007fb6 100644
--- a/Mailman/Cgi/admindb.py
+++ b/Mailman/Cgi/admindb.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Produce and process the pending-approval items for a list."""
@@ -111,7 +111,7 @@ def main():
     # Set up the results document
     doc = Document()
     doc.set_language(mlist.preferred_language)
-    
+
     # See if we're requesting all the messages for a particular sender, or if
     # we want a specific held message.
     sender = None
@@ -307,7 +307,7 @@ def show_pending_subs(mlist, form):
         form.AddItem(table)
     return num
 
-    
+
 
 def show_pending_unsubs(mlist, form):
     # Add the pending unsubscription request section
diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py
index 2348b0b6..abb0ac29 100644
--- a/Mailman/Cgi/confirm.py
+++ b/Mailman/Cgi/confirm.py
@@ -1,17 +1,17 @@
-# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Confirm a pending action via URL."""
@@ -183,7 +183,7 @@ def ask_for_cookie(mlist, doc, extra=''):
     if extra:
         table.AddRow([Bold(FontAttr(extra, size='+1'))])
         table.AddCellInfo(table.GetCurrentRowIndex(), 0, colspan=2)
-        
+
     # Add cookie entry box
     table.AddRow([_("""Please enter the confirmation string
     (i.e. <em>cookie</em>) that you received in your email message, in the box
@@ -313,6 +313,8 @@ def subscription_confirm(mlist, doc, cookie, cgidata):
             # Some pending values may be overridden in the form.  email of
             # course is hardcoded. ;)
             lang = cgidata.getvalue('language')
+            if not Utils.IsLanguage(lang):
+                lang = mlist.preferred_language
             i18n.set_language(lang)
             doc.set_language(lang)
             if cgidata.has_key('digests'):
@@ -368,7 +370,7 @@ def subscription_confirm(mlist, doc, cookie, cgidata):
         mlist.Save()
     finally:
         mlist.Unlock()
-    
+
 
 
 def unsubscription_cancel(mlist, doc, cookie):
@@ -456,7 +458,7 @@ def unsubscription_prompt(mlist, doc, cookie, addr):
 
     form.AddItem(table)
     doc.AddItem(form)
-    
+
 
 
 def addrchange_cancel(mlist, doc, cookie):
diff --git a/Mailman/Cgi/listinfo.py b/Mailman/Cgi/listinfo.py
index d9e4d266..5244d75c 100644
--- a/Mailman/Cgi/listinfo.py
+++ b/Mailman/Cgi/listinfo.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Produce listinfo page, primary web entry-point to mailing lists.
@@ -54,7 +54,9 @@ def main():
 
     # See if the user want to see this page in other language
     cgidata = cgi.FieldStorage()
-    language = cgidata.getvalue('language', mlist.preferred_language)
+    language = cgidata.getvalue('language')
+    if not Utils.IsLanguage(language):
+        language = mlist.preferred_language
     i18n.set_language(language)
     list_listinfo(mlist, language)
 
@@ -192,7 +194,7 @@ def list_listinfo(mlist, lang):
     else:
         displang = mlist.FormatButton('displang-button',
                                       text = _("View this page in"))
-    replacements['<mm-displang-box>'] = displang 
+    replacements['<mm-displang-box>'] = displang
     replacements['<mm-lang-form-start>'] = mlist.FormatFormStart('listinfo')
     replacements['<mm-fullname-box>'] = mlist.FormatBox('fullname', size=30)
 
diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py
index ef080a68..2f9e9afa 100644
--- a/Mailman/Cgi/options.py
+++ b/Mailman/Cgi/options.py
@@ -82,7 +82,9 @@ def main():
     # we might have a 'language' key in the cgi data.  That was an explicit
     # preference to view the page in, so we should honor that here.  If that's
     # not available, use the list's default language.
-    language = cgidata.getvalue('language', mlist.preferred_language)
+    language = cgidata.getvalue('language')
+    if not Utils.IsLanguage(language):
+        language = mlist.preferred_language
     i18n.set_language(language)
     doc.set_language(language)
 
@@ -94,7 +96,7 @@ def main():
             # button UserOptions; we can use that as the descriminator.
             if not cgidata.getvalue('UserOptions'):
                 doc.addError(_('No address given'))
-            loginpage(mlist, doc, None, cgidata)
+            loginpage(mlist, doc, None, language)
             print doc.Format()
             return
     else:
@@ -102,11 +104,18 @@ def main():
 
     # Avoid cross-site scripting attacks
     safeuser = Utils.websafe(user)
-    # Sanity check the user, but be careful about leaking membership
-    # information when we're using private rosters.
+    try:
+        Utils.ValidateEmail(user)
+    except Errors.EmailAddressError:
+        doc.addError(_('Illegal Email Address: %(safeuser)s'))
+        loginpage(mlist, doc, None, language)
+        print doc.Format()
+        return
+    # Sanity check the user, but only give the "no such member" error when
+    # using public rosters, otherwise, we'll leak membership information.
     if not mlist.isMember(user) and mlist.private_roster == 0:
         doc.addError(_('No such member: %(safeuser)s.'))
-        loginpage(mlist, doc, None, cgidata)
+        loginpage(mlist, doc, None, language)
         print doc.Format()
         return
 
@@ -123,7 +132,9 @@ def main():
     # And now we know the user making the request, so set things up to for the
     # user's stored preferred language, overridden by any form settings for
     # their new language preference.
-    userlang = cgidata.getvalue('language', mlist.getMemberLanguage(user))
+    userlang = cgidata.getvalue('language')
+    if not Utils.IsLanguage(userlang):
+        userlang = mlist.getMemberLanguage(user)
     doc.set_language(userlang)
     i18n.set_language(userlang)
 
@@ -159,7 +170,7 @@ def main():
                        user)
                 doc.addError(_('The confirmation email has been sent.'),
                              tag='')
-        loginpage(mlist, doc, user, cgidata)
+        loginpage(mlist, doc, user, language)
         print doc.Format()
         return
 
@@ -182,7 +193,7 @@ def main():
                 doc.addError(
                     _('A reminder of your password has been emailed to you.'),
                     tag='')
-        loginpage(mlist, doc, user, cgidata)
+        loginpage(mlist, doc, user, language)
         print doc.Format()
         return
 
@@ -205,7 +216,7 @@ def main():
                        'Login failure with private rosters: %s',
                        user)
                 user = None
-        loginpage(mlist, doc, user, cgidata)
+        loginpage(mlist, doc, user, language)
         print doc.Format()
         return
 
@@ -215,7 +226,7 @@ def main():
 
     if cgidata.has_key('logout'):
         print mlist.ZapCookie(mm_cfg.AuthUser, user)
-        loginpage(mlist, doc, user, cgidata)
+        loginpage(mlist, doc, user, language)
         print doc.Format()
         return
 
@@ -229,7 +240,7 @@ def main():
 
     if cgidata.has_key('othersubs'):
         hostname = mlist.host_name
-        title = _('List subscriptions for %(user)s on %(hostname)s')
+        title = _('List subscriptions for %(safeuser)s on %(hostname)s')
         doc.SetTitle(title)
         doc.AddItem(Header(2, title))
         doc.AddItem(_('''Click on a link to visit your options page for the
@@ -302,7 +313,7 @@ def main():
 The new address you requested %(newaddr)s is already a member of the
 %(listname)s mailing list, however you have also requested a global change of
 address.  Upon confirmation, any other mailing list containing the address
-%(user)s will be changed. """)
+%(safeuser)s will be changed. """)
                     # Don't return
                 else:
                     options_page(
@@ -743,20 +754,20 @@ You are subscribed to this list with the case-preserved address
 
 
 
-def loginpage(mlist, doc, user, cgidata):
+def loginpage(mlist, doc, user, lang):
     realname = mlist.real_name
     actionurl = mlist.GetScriptURL('options')
     if user is None:
         title = _('%(realname)s list: member options login page')
         extra = _('email address and ')
     else:
-        title = _('%(realname)s list: member options for user %(user)s')
+        safeuser = Utils.websafe(user)
+        title = _('%(realname)s list: member options for user %(safeuser)s')
         obuser = Utils.ObscureEmail(user)
         extra = ''
     # Set up the title
     doc.SetTitle(title)
     # We use a subtable here so we can put a language selection box in
-    lang = cgidata.getvalue('language', mlist.preferred_language)
     table = Table(width='100%', border=0, cellspacing=4, cellpadding=5)
     # If only one language is enabled for this mailing list, omit the choice
     # buttons.
diff --git a/Mailman/Cgi/private.py b/Mailman/Cgi/private.py
index 6b7af70a..5fa5398e 100644
--- a/Mailman/Cgi/private.py
+++ b/Mailman/Cgi/private.py
@@ -1,25 +1,26 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Provide a password-interface wrapper around private archives.
 """
 
-import sys
 import os
+import sys
 import cgi
+import mimetypes
 
 from Mailman import mm_cfg
 from Mailman import Utils
@@ -43,12 +44,11 @@ def true_path(path):
     return path[1:]
 
 
-def content_type(path):
-    if path[-3:] == '.gz':
-        path = path[:-3]
-    if path[-4:] == '.txt':
-        return 'text/plain'
-    return 'text/html'
+
+def guess_type(url, strict):
+    if hasattr(mimetypes, 'common_types'):
+        return mimetypes.guess_type(url, strict)
+    return mimetypes.guess_type(url)
 
 
 
@@ -140,12 +140,14 @@ def main():
 
     # Authorization confirmed... output the desired file
     try:
-        ctype = content_type(path)
+        ctype, enc = guess_type(path, strict=0)
+        if ctype is None:
+            ctype = 'text/html'
         if mboxfile:
             f = open(os.path.join(mlist.archive_dir() + '.mbox',
                                   mlist.internal_name() + '.mbox'))
             ctype = 'text/plain'
-        elif true_filename[-3:] == '.gz':
+        elif true_filename.endswith('.gz'):
             import gzip
             f = gzip.open(true_filename, 'r')
         else:
diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py
index 71c06240..2dc0c98d 100644
--- a/Mailman/Cgi/roster.py
+++ b/Mailman/Cgi/roster.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Produce subscriber roster, using listinfo form data, roster.html template.
@@ -21,7 +21,7 @@ Takes listname in PATH_INFO.
 
 
 # We don't need to lock in this script, because we're never going to change
-# data. 
+# data.
 
 import sys
 import os
@@ -61,11 +61,9 @@ def main():
     cgidata = cgi.FieldStorage()
 
     # messages in form should go in selected language (if any...)
-    if cgidata.has_key('language'):
-        lang = cgidata['language'].value
-    else:
+    lang = cgidata.getvalue('language')
+    if not Utils.IsLanguage(lang):
         lang = mlist.preferred_language
-
     i18n.set_language(lang)
 
     # Perform authentication for protected rosters.  If the roster isn't
diff --git a/Mailman/Cgi/subscribe.py b/Mailman/Cgi/subscribe.py
index c2dfe5cd..d0a477d7 100644
--- a/Mailman/Cgi/subscribe.py
+++ b/Mailman/Cgi/subscribe.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Process subscription or roster requests from listinfo form."""
@@ -50,7 +50,7 @@ def main():
         doc.AddItem(Bold(_('Invalid options to CGI script')))
         print doc.Format()
         return
-        
+
     listname = parts[0].lower()
     try:
         mlist = MailList.MailList(listname, lock=0)
@@ -66,7 +66,9 @@ def main():
     # See if the form data has a preferred language set, in which case, use it
     # for the results.  If not, use the list's preferred language.
     cgidata = cgi.FieldStorage()
-    language = cgidata.getvalue('language', mlist.preferred_language)
+    language = cgidata.getvalue('language')
+    if not Utils.IsLanguage(language):
+        language = mlist.preferred_language
     i18n.set_language(language)
     doc.set_language(language)
 
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index a9d11f63..4286e468 100644
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -1,6 +1,6 @@
 # -*- python -*-
 
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -939,9 +939,24 @@ DEFAULT_DIGEST_IS_DEFAULT = 0
 DEFAULT_MIME_IS_DEFAULT_DIGEST = 0
 DEFAULT_DIGEST_SIZE_THRESHHOLD = 30     # KB
 DEFAULT_DIGEST_SEND_PERIODIC = 1
-DEFAULT_PLAIN_DIGEST_KEEP_HEADERS = ['message', 'date', 'from',
-                                     'subject', 'to', 'cc',
-                                     'reply-to', 'organization']
+
+# Headers which should be kept in both RFC 1153 (plain) and MIME digests.  RFC
+# 1153 also specifies these headers in this exact order, so order matters.
+MIME_DIGEST_KEEP_HEADERS = [
+    'Date', 'From', 'To', 'Cc', 'Subject', 'Message-ID', 'Keywords',
+    # I believe we should also keep these headers though.
+    'In-Reply-To', 'References', 'Content-Type', 'MIME-Version',
+    'Content-Transfer-Encoding', 'Precedence', 'Reply-To',
+    # Mailman 2.0 adds these headers
+    'Message',
+    ]
+
+PLAIN_DIGEST_KEEP_HEADERS = [
+    'Message', 'Date', 'From',
+    'Subject', 'To', 'Cc',
+    'Message-ID', 'Keywords',
+    'Content-Type',
+    ]
 
 
 
diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
index 40eddd66..c4ad06ab 100644
--- a/Mailman/Handlers/CookHeaders.py
+++ b/Mailman/Handlers/CookHeaders.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -39,7 +39,7 @@ MAXLINELEN = 78
 def _isunicode(s):
     return isinstance(s, UnicodeType)
 
-def uheader(mlist, s, header_name=None):
+def uheader(mlist, s, header_name=None, continuation_ws='\t'):
     # Get the charset to encode the string in.  If this is us-ascii, we'll use
     # iso-8859-1 instead, just to get a little extra coverage, and because the
     # Header class tries us-ascii first anyway.
@@ -54,7 +54,8 @@ def uheader(mlist, s, header_name=None):
         codec = charset.input_codec or 'ascii'
         s = unicode(s, codec, 'replace')
     # We purposefully leave no space b/w prefix and subject!
-    return Header(s, charset, header_name=header_name)    
+    return Header(s, charset, header_name=header_name,
+                  continuation_ws=continuation_ws)
 
 
 
@@ -218,7 +219,15 @@ def prefix_subject(mlist, msg, msgdata):
     # tracked (e.g. internally crafted, delivered to a single user such as the
     # list admin).
     prefix = mlist.subject_prefix
-    subject = msg['subject']
+    subject = msg.get('subject', '')
+    # Try to figure out what the continuation_ws is for the header
+    if isinstance(subject, Header):
+        lines = str(subject).splitlines()
+    else:
+        lines = subject.splitlines()
+    ws = '\t'
+    if len(lines) > 1 and lines[1] and lines[1][0] in ' \t':
+        ws = lines[1][0]
     msgdata['origsubj'] = subject
     # The header may be multilingual; decode it from base64/quopri and search
     # each chunk for the prefix.  BAW: Note that if the prefix contains spaces
@@ -235,7 +244,7 @@ def prefix_subject(mlist, msg, msgdata):
     if not subject:
         subject = _('(no subject)')
     # Get the header as a Header instance, with proper unicode conversion
-    h = uheader(mlist, prefix, 'Subject')
+    h = uheader(mlist, prefix, 'Subject', continuation_ws=ws)
     for s, c in headerbits:
         # Once again, convert the string to unicode.
         if c is None:
diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py
index fd64f6f1..4724c3a1 100644
--- a/Mailman/Handlers/SMTPDirect.py
+++ b/Mailman/Handlers/SMTPDirect.py
@@ -25,6 +25,7 @@ Note: This file only handles single threaded delivery.  See SMTPThreaded.py
 for a threaded implementation.
 """
 
+import copy
 import time
 import socket
 import smtplib
@@ -268,12 +269,20 @@ def verpdeliver(mlist, msg, msgdata, envsender, failures, conn):
         # they missed due to bouncing.  Neat idea.
         msgdata['recips'] = [recip]
         # Make a copy of the message and decorate + delivery that
-        msgcopy = email.message_from_string(msg.as_string())
+        msgcopy = copy.deepcopy(msg)
         Decorate.process(mlist, msgcopy, msgdata)
         # Calculate the envelope sender, which we may be VERPing
         if msgdata.get('verp'):
             bmailbox, bdomain = Utils.ParseEmail(envsender)
             rmailbox, rdomain = Utils.ParseEmail(recip)
+            if rdomain is None:
+                # The recipient address is not fully-qualified.  We can't
+                # deliver it to this person, nor can we craft a valid verp
+                # header.  I don't think there's much we can do except ignore
+                # this recipient.
+                syslog('smtp', 'Skipping VERP delivery to unqual recip: %s',
+                       recip)
+                continue
             d = {'bounces': bmailbox,
                  'mailbox': rmailbox,
                  'host'   : DOT.join(rdomain),
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index 024832a4..b5be73df 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -17,6 +17,8 @@
 """Cleanse a message for archiving.
 """
 
+from __future__ import nested_scopes
+
 import os
 import re
 import sha
@@ -24,7 +26,6 @@ import time
 import errno
 import binascii
 import tempfile
-import mimetypes
 from cStringIO import StringIO
 from types import IntType
 
@@ -51,6 +52,35 @@ dre = re.compile(r'^\.*')
 BR = '<br>\n'
 SPACE = ' '
 
+try:
+    from mimetypes import guess_all_extensions
+except ImportError:
+    import mimetypes
+    def guess_all_extensions(ctype, strict=1):
+        # BAW: sigh, guess_all_extensions() is new in Python 2.3
+        all = []
+        def check(map):
+            for e, t in map.items():
+                if t == ctype:
+                    all.append(e)
+        check(mimetypes.types_map)
+        # Python 2.1 doesn't have common_types.  Sigh, sigh.
+        if not strict and hasattr(mimetypes, 'common_types'):
+            check(mimetypes.common_types)
+        return all
+
+
+
+def guess_extension(ctype, ext):
+    # mimetypes maps multiple extensions to the same type, e.g. .doc, .dot,
+    # and .wiz are all mapped to application/msword.  This sucks for finding
+    # the best reverse mapping.  If the extension is one of the giving
+    # mappings, we'll trust that, otherwise we'll just guess. :/
+    all = guess_all_extensions(ctype, strict=0)
+    if ext in all:
+        return ext
+    return all and all[0]
+
 
 
 # We're using a subclass of the standard Generator because we want to suppress
@@ -131,6 +161,7 @@ def process(mlist, msg, msgdata=None):
         msgdata = {}
     dir = calculate_attachments_dir(mlist, msg, msgdata)
     charset = None
+    lcset = Utils.GetCharSet(mlist.preferred_language)
     # Now walk over all subparts of this message and scrub out various types
     for part in msg.walk():
         ctype = part.get_type(part.get_default_type())
@@ -140,13 +171,16 @@ def process(mlist, msg, msgdata=None):
             # arbitrarily pick the charset of the first text/plain part in the
             # message.
             if charset is None:
-                charset = part.get_content_charset(charset)
+                charset = part.get_content_charset(lcset)
         elif ctype == 'text/html' and isinstance(sanitize, IntType):
             if sanitize == 0:
                 if outer:
                     raise DiscardMessage
-                part.set_payload(_('HTML attachment scrubbed and removed'))
-                part.set_type('text/plain')
+                del part['content-type']
+                part.set_payload(_('HTML attachment scrubbed and removed'),
+                                 # Adding charset arg and removing content-tpe
+                                 # sets content-type to text/plain
+                                 lcset)
             elif sanitize == 2:
                 # By leaving it alone, Pipermail will automatically escape it
                 pass
@@ -159,11 +193,11 @@ def process(mlist, msg, msgdata=None):
                     url = save_attachment(mlist, part, dir, filter_html=0)
                 finally:
                     os.umask(omask)
+                del part['content-type']
                 part.set_payload(_("""\
 An HTML attachment was scrubbed...
 URL: %(url)s
-"""))
-                part.set_type('text/plain')
+"""), lcset)
             else:
                 # HTML-escape it and store it as an attachment, but make it
                 # look a /little/ bit prettier. :(
@@ -185,11 +219,11 @@ URL: %(url)s
                     url = save_attachment(mlist, part, dir, filter_html=0)
                 finally:
                     os.umask(omask)
+                del part['content-type']
                 part.set_payload(_("""\
 An HTML attachment was scrubbed...
 URL: %(url)s
-"""))
-                part.set_type('text/plain')
+"""), lcset)
         elif ctype == 'message/rfc822':
             # This part contains a submessage, so it too needs scrubbing
             submsg = part.get_payload(0)
@@ -202,6 +236,7 @@ URL: %(url)s
             date = submsg.get('date', _('no date'))
             who = submsg.get('from', _('unknown sender'))
             size = len(str(submsg))
+            del part['content-type']
             part.set_payload(_("""\
 An embedded message was scrubbed...
 From: %(who)s
@@ -209,13 +244,12 @@ Subject: %(subject)s
 Date: %(date)s
 Size: %(size)s
 Url: %(url)s
-"""))
-            part.set_type('text/plain')
+"""), lcset)
         # If the message isn't a multipart, then we'll strip it out as an
         # attachment that would have to be separately downloaded.  Pipermail
         # will transform the url into a hyperlink.
         elif not part.is_multipart():
-            payload = part.get_payload()
+            payload = part.get_payload(decode=1)
             ctype = part.get_type()
             size = len(payload)
             omask = os.umask(002)
@@ -225,6 +259,8 @@ Url: %(url)s
                 os.umask(omask)
             desc = part.get('content-description', _('not available'))
             filename = part.get_filename(_('not available'))
+            del part['content-type']
+            del part['content-transfer-encoding']
             part.set_payload(_("""\
 A non-text attachment was scrubbed...
 Name: %(filename)s
@@ -232,8 +268,7 @@ Type: %(ctype)s
 Size: %(size)d bytes
 Desc: %(desc)s
 Url : %(url)s
-"""))
-            part.set_type('text/plain')
+"""), lcset)
         outer = 0
     # We still have to sanitize multipart messages to flat text because
     # Pipermail can't handle messages with list payloads.  This is a kludge;
@@ -242,8 +277,8 @@ Url : %(url)s
         # By default we take the charset of the first text/plain part in the
         # message, but if there was none, we'll use the list's preferred
         # language's charset.
-        if charset is None:
-            charset = Utils.GetCharSet(mlist.preferred_language)
+        if charset is None or charset == 'us-ascii':
+            charset = lcset
         # We now want to concatenate all the parts which have been scrubbed to
         # text/plain, into a single text/plain payload.  We need to make sure
         # all the characters in the concatenated string are in the same
@@ -261,20 +296,26 @@ Url : %(url)s
                 t = part.get_payload(decode=1)
             except binascii.Error:
                 t = part.get_payload()
-            partcharset = part.get_charset()
+            partcharset = part.get_content_charset()
             if partcharset and partcharset <> charset:
                 try:
                     t = unicode(t, partcharset, 'replace')
-                    # Should use HTML-Escape, or try generalizing to UTF-8
-                    t = t.encode(charset, 'replace')
-                except UnicodeError:
+                except (UnicodeError, LookupError):
                     # Replace funny characters
                     t = unicode(t, 'ascii', 'replace').encode('ascii')
+                try:
+                    # Should use HTML-Escape, or try generalizing to UTF-8
+                    t = t.encode(charset, 'replace')
+                except (UnicodeError, LookupError):
+                    t = t.encode(lcset, 'replace')
+            # Separation is useful
+            if not t.endswith('\n'):
+                t += '\n'
             text.append(t)
         # Now join the text and set the payload
         sep = _('-------------- next part --------------\n')
+        del msg['content-type']
         msg.set_payload(sep.join(text), charset)
-        msg.set_type('text/plain')
         del msg['content-transfer-encoding']
         msg.add_header('Content-Transfer-Encoding', '8bit')
     return msg
@@ -285,13 +326,13 @@ def makedirs(dir):
     # Create all the directories to store this attachment in
     try:
         os.makedirs(dir, 02775)
+        # Unfortunately, FreeBSD seems to be broken in that it doesn't honor
+        # the mode arg of mkdir().
+        def twiddle(arg, dirname, names):
+            os.chmod(dirname, 02775)
+        os.path.walk(dir, twiddle, None)
     except OSError, e:
         if e.errno <> errno.EEXIST: raise
-    # Unfortunately, FreeBSD seems to be broken in that it doesn't honor the
-    # mode arg of mkdir().
-    def twiddle(arg, dirname, names):
-        os.chmod(dirname, 02775)
-    os.path.walk(dir, twiddle, None)
 
 
 
@@ -303,13 +344,15 @@ def save_attachment(mlist, msg, dir, filter_html=1):
     # BAW: mimetypes ought to handle non-standard, but commonly found types,
     # e.g. image/jpg (should be image/jpeg).  For now we just store such
     # things as application/octet-streams since that seems the safest.
-    ext = mimetypes.guess_extension(msg.get_type())
+    ctype = msg.get_content_type()
+    fnext = os.path.splitext(msg.get_filename(''))[1]
+    ext = guess_extension(ctype, fnext)
     if not ext:
         # We don't know what it is, so assume it's just a shapeless
         # application/octet-stream, unless the Content-Type: is
         # message/rfc822, in which case we know we'll coerce the type to
         # text/plain below.
-        if msg.get_type() == 'message/rfc822':
+        if ctype == 'message/rfc822':
             ext = '.txt'
         else:
             ext = '.bin'
@@ -361,7 +404,7 @@ def save_attachment(mlist, msg, dir, filter_html=1):
     # ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be
     # here), then send the attachment through the filter program for
     # sanitization
-    if filter_html and msg.get_type() == 'text/html':
+    if filter_html and ctype == 'text/html':
         base, ext = os.path.splitext(path)
         tmppath = base + '-tmp' + ext
         fp = open(tmppath, 'w')
@@ -384,7 +427,7 @@ def save_attachment(mlist, msg, dir, filter_html=1):
         ext = '.txt'
         path = base + '.txt'
     # Is it a message/rfc822 attachment?
-    elif msg.get_type() == 'message/rfc822':
+    elif ctype == 'message/rfc822':
         submsg = msg.get_payload()
         # BAW: I'm sure we can eventually do better than this. :(
         decodedpayload = Utils.websafe(str(submsg))
diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py
index d735cd69..79090051 100644
--- a/Mailman/Handlers/ToDigest.py
+++ b/Mailman/Handlers/ToDigest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -37,6 +37,7 @@ from email.MIMEBase import MIMEBase
 from email.MIMEText import MIMEText
 from email.MIMEMessage import MIMEMessage
 from email.Utils import getaddresses
+from email.Header import decode_header, make_header, Header
 
 from Mailman import mm_cfg
 from Mailman import Utils
@@ -46,19 +47,13 @@ from Mailman.MemberAdaptor import ENABLED
 from Mailman.Handlers.Decorate import decorate
 from Mailman.Queue.sbcache import get_switchboard
 from Mailman.Mailbox import Mailbox
+from Mailman.Handlers.Scrubber import process as scrubber
+from Mailman.Logging.Syslog import syslog
 
 _ = i18n._
 
-
-# rfc1153 says we should keep only these headers, and present them in this
-# exact order.
-KEEP = ['Date', 'From', 'To', 'Cc', 'Subject', 'Message-ID', 'Keywords',
-        # I believe we should also keep these headers though.
-        'In-Reply-To', 'References', 'Content-Type', 'MIME-Version',
-        'Content-Transfer-Encoding', 'Precedence', 'Reply-To',
-        # Mailman 2.0 adds these headers, but they don't need to be kept from
-        # the original message: Message
-        ]
+UEMPTYSTRING = u''
+EMPTYSTRING = ''
 
 
 
@@ -73,7 +68,7 @@ def process(mlist, msg, msgdata):
     finally:
         os.umask(omask)
     g = Generator(mboxfp)
-    g(msg, unixfrom=1)
+    g.flatten(msg, unixfrom=1)
     # Calculate the current size of the accumulation file.  This will not tell
     # us exactly how big the MIME, rfc1153, or any other generated digest
     # message will be, but it's the most easily available metric to decide
@@ -135,24 +130,26 @@ def send_i18n_digests(mlist, mboxfp):
     mbox = Mailbox(mboxfp)
     # Prepare common information
     lang = mlist.preferred_language
+    lcset = Utils.GetCharSet(lang)
     realname = mlist.real_name
     volume = mlist.volume
     issue = mlist.next_digest_number
     digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
+    digestsubj = Header(digestid, lcset, header_name='Subject')
     # Set things up for the MIME digest.  Only headers not added by
     # CookHeaders need be added here.
     mimemsg = Message.Message()
     mimemsg['Content-Type'] = 'multipart/mixed'
     mimemsg['MIME-Version'] = '1.0'
     mimemsg['From'] = mlist.GetRequestEmail()
-    mimemsg['Subject'] = digestid
+    mimemsg['Subject'] = digestsubj
     mimemsg['To'] = mlist.GetListEmail()
     mimemsg['Reply-To'] = mlist.GetListEmail()
     # Set things up for the rfc1153 digest
     plainmsg = StringIO()
     rfc1153msg = Message.Message()
     rfc1153msg['From'] = mlist.GetRequestEmail()
-    rfc1153msg['Subject'] = digestid
+    rfc1153msg['Subject'] = digestsubj
     rfc1153msg['To'] = mlist.GetListEmail()
     rfc1153msg['Reply-To'] = mlist.GetListEmail()
     separator70 = '-' * 70
@@ -170,20 +167,20 @@ def send_i18n_digests(mlist, mboxfp):
          'got_owner_email':   mlist.GetOwnerEmail(),
          }, mlist=mlist)
     # MIME
-    masthead = MIMEText(mastheadtxt, _charset=Utils.GetCharSet(lang))
+    masthead = MIMEText(mastheadtxt, _charset=lcset)
     masthead['Content-Description'] = digestid
     mimemsg.attach(masthead)
-    # rfc1153
+    # RFC 1153
     print >> plainmsg, mastheadtxt
     print >> plainmsg
     # Now add the optional digest header
     if mlist.digest_header:
         headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
         # MIME
-        header = MIMEText(headertxt)
+        header = MIMEText(headertxt, _charset=lcset)
         header['Content-Description'] = _('Digest Header')
         mimemsg.attach(header)
-        # rfc1153
+        # RFC 1153
         print >> plainmsg, headertxt
         print >> plainmsg
     # Now we have to cruise through all the messages accumulated in the
@@ -196,7 +193,7 @@ def send_i18n_digests(mlist, mboxfp):
     toc = StringIO()
     print >> toc, _("Today's Topics:\n")
     # Now cruise through all the messages in the mailbox of digest messages,
-    # building the MIME payload and core of the rfc1153 digest.  We'll also
+    # building the MIME payload and core of the RFC 1153 digest.  We'll also
     # accumulate Subject: headers and authors for the table-of-contents.
     messages = []
     msgcount = 0
@@ -208,23 +205,26 @@ def send_i18n_digests(mlist, mboxfp):
         msgcount += 1
         messages.append(msg)
         # Get the Subject header
-        subject = msg.get('subject', _('(no subject)'))
+        msgsubj = msg.get('subject', _('(no subject)'))
+        subject = oneline(msgsubj, lcset)
         # Don't include the redundant subject prefix in the toc
         mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
                       subject, re.IGNORECASE)
         if mo:
             subject = subject[:mo.start(2)] + subject[mo.end(2):]
-        addresses = getaddresses([msg.get('From', '')])
         username = ''
+        addresses = getaddresses([oneline(msg.get('from', ''), lcset)])
         # Take only the first author we find
-        if type(addresses) is ListType and len(addresses) > 0:
+        if isinstance(addresses, ListType) and addresses:
             username = addresses[0][0]
+            if not username:
+                username = addresses[0][1]
         if username:
             username = ' (%s)' % username
-        # Wrap the toc subject line
-        wrapped = Utils.wrap('%2d. %s' % (msgcount, subject))
-        # Split by lines and see if the username can fit on the last line
+        # Put count and Wrap the toc subject line
+        wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
         slines = wrapped.split('\n')
+        # See if the user's name can fit on the last line
         if len(slines[-1]) + len(username) > 70:
             slines.append(username)
         else:
@@ -236,20 +236,26 @@ def send_i18n_digests(mlist, mboxfp):
                 print >> toc, ' ', line
                 first = 0
             else:
-                print >> toc, '     ', line
+                print >> toc, '     ', line.lstrip()
         # We do not want all the headers of the original message to leak
-        # through in the digest messages.  For simplicity, we'll leave the
-        # same set of headers in both digests, i.e. those required in rfc1153
+        # through in the digest messages.  For this phase, we'll leave the
+        # same set of headers in both digests, i.e. those required in RFC 1153
         # plus a couple of other useful ones.  We also need to reorder the
-        # headers according to rfc1153.
+        # headers according to RFC 1153.  Later, we'll strip out headers for
+        # for the specific MIME or plain digests.
         keeper = {}
-        for keep in KEEP:
+        all_keepers = {}
+        for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
+                       mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
+            all_keepers[header] = 1
+        all_keepers = all_keepers.keys()
+        for keep in all_keepers:
             keeper[keep] = msg.get_all(keep, [])
         # Now remove all unkempt headers :)
         for header in msg.keys():
             del msg[header]
-        # And add back the kept header in the rfc1153 designated order
-        for keep in KEEP:
+        # And add back the kept header in the RFC 1153 designated order
+        for keep in all_keepers:
             for field in keeper[keep]:
                 msg[keep] = field
         # And a bit of extra stuff
@@ -263,13 +269,13 @@ def send_i18n_digests(mlist, mboxfp):
         return
     toctext = toc.getvalue()
     # MIME
-    tocpart = MIMEText(toctext)
+    tocpart = MIMEText(toctext, _charset=lcset)
     tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
     mimemsg.attach(tocpart)
-    # rfc1153
+    # RFC 1153
     print >> plainmsg, toctext
     print >> plainmsg
-    # For rfc1153 digests, we now need the standard separator
+    # For RFC 1153 digests, we now need the standard separator
     print >> plainmsg, separator70
     print >> plainmsg
     # Now go through and add each message
@@ -285,20 +291,28 @@ def send_i18n_digests(mlist, mboxfp):
         else:
             print >> plainmsg, separator30
             print >> plainmsg
-        g = Generator(plainmsg)
-        g(msg, unixfrom=0)
+        # Use Mailman.Handlers.Scrubber.process() to get plain text
+        msg = scrubber(mlist, msg)
+        # Honor the default setting
+        for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
+            if msg[h]:
+                uh = Utils.wrap('%s: %s' % (h, oneline(msg[h], lcset)))
+                uh = '\n\t'.join(uh.split('\n'))
+                print >> plainmsg, uh
+        print >> plainmsg
+        print >> plainmsg, msg.get_payload(decode=1)
     # Now add the footer
     if mlist.digest_footer:
         footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
         # MIME
-        footer = MIMEText(footertxt)
+        footer = MIMEText(footertxt, _charset=lcset)
         footer['Content-Description'] = _('Digest Footer')
         mimemsg.attach(footer)
-        # rfc1153
-        # BAW: This is not strictly conformant rfc1153.  The trailer is only
+        # RFC 1153
+        # BAW: This is not strictly conformant RFC 1153.  The trailer is only
         # supposed to contain two lines, i.e. the "End of ... Digest" line and
         # the row of asterisks.  If this screws up MUAs, the solution is to
-        # add the footer as the last message in the rfc1153 digest.  I just
+        # add the footer as the last message in the RFC 1153 digest.  I just
         # hate the way that VM does that and I think it's confusing to users,
         # so don't do it unless there's a clamor.
         print >> plainmsg, separator30
@@ -343,9 +357,22 @@ def send_i18n_digests(mlist, mboxfp):
                     recips=mimerecips,
                     listname=mlist.internal_name(),
                     isdigest=1)
-    # rfc1153
-    rfc1153msg.set_payload(plainmsg.getvalue())
+    # RFC 1153
+    rfc1153msg.set_payload(plainmsg.getvalue(), lcset)
     virginq.enqueue(rfc1153msg,
                     recips=plainrecips,
                     listname=mlist.internal_name(),
                     isdigest=1)
+
+
+
+def oneline(s, cset):
+    # Decode header string in one line and convert into specified charset
+    try:
+        h = make_header(decode_header(s))
+        ustr = h.__unicode__()
+        oneline = UEMPTYSTRING.join(ustr.splitlines())
+        return oneline.encode(cset, 'replace')
+    except (LookupError, UnicodeError):
+        # possibly charset problem. return with undecoded string in one line.
+        return EMPTYSTRING.join(s.splitlines())
diff --git a/Mailman/ListAdmin.py b/Mailman/ListAdmin.py
index 82eedc80..d4d72375 100644
--- a/Mailman/ListAdmin.py
+++ b/Mailman/ListAdmin.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Mixin class for MailList which handles administrative requests.
@@ -95,19 +95,25 @@ class ListAdmin:
             # fullname data field.
             type, version = self.__db.get('version', (IGN, None))
             if version is None:
-                # No previous revisiont number, must be upgrading to 2.1a3 or
+                # No previous revision number, must be upgrading to 2.1a3 or
                 # beyond from some unknown earlier version.
                 for id, (type, data) in self.__db.items():
-                    if id == IGN:
+                    if type == IGN:
                         pass
-                    elif id == HELDMSG and len(data) == 5:
+                    elif type == HELDMSG and len(data) == 5:
                         # tack on a msgdata dictionary
                         self.__db[id] = data + ({},)
-                    elif id == SUBSCRIPTION and len(data) == 5:
-                        # a fullname field was added
-                        stime, addr, password, digest, lang = data
-                        self.__db[id] = stime, addr, '', password, digest, lang
-                        
+                    elif type == SUBSCRIPTION:
+                        if len(data) == 4:
+                            # fullname and lang was added
+                            stime, addr, password, digest = data
+                            lang = self.preferred_language
+                            data = stime, addr, '', password, digest, lang
+                        elif len(data) == 5:
+                            # a fullname field was added
+                            stime, addr, password, digest, lang = data
+                            data = stime, addr, '', password, digest, lang
+                        self.__db[id] = type, data
 
     def __closedb(self):
         if self.__db is not None:
@@ -130,9 +136,9 @@ class ListAdmin:
             os.rename(tmpfile, self.__filename())
 
     def __request_id(self):
-	id = self.next_request_id
-	self.next_request_id += 1
-	return id
+        id = self.next_request_id
+        self.next_request_id += 1
+        return id
 
     def SaveRequestsDb(self):
         self.__closedb()
@@ -351,7 +357,7 @@ class ListAdmin:
             fmsg.attach(copy)
             fmsg.send(self)
         # Log the rejection
-	if rejection:
+        if rejection:
             note = '''%(listname)s: %(rejection)s posting:
 \tFrom: %(sender)s
 \tSubject: %(subject)s''' % {
@@ -374,7 +380,7 @@ class ListAdmin:
                 # and inform of this status.
                 return LOST
         return status
-            
+
     def HoldSubscription(self, addr, fullname, password, digest, lang):
         # Assure that the database is open for writing
         self.__opendb()
@@ -390,7 +396,7 @@ class ListAdmin:
         # the subscriber's address
         # the subscriber's selected password (TBD: is this safe???)
         # the digest flag
-	# the user's preferred language
+        # the user's preferred language
         #
         data = time.time(), addr, fullname, password, digest, lang
         self.__db[id] = (SUBSCRIPTION, data)
@@ -493,7 +499,7 @@ class ListAdmin:
         # to his/her language choice, if they are a member.  Otherwise use the
         # list's preferred language.
         realname = self.real_name
-	if lang is None:
+        if lang is None:
             lang = self.getMemberLanguage(recip)
         text = Utils.maketext(
             'refuse.txt',
diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py
index 303d4c52..785511b3 100644
--- a/Mailman/Queue/CommandRunner.py
+++ b/Mailman/Queue/CommandRunner.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -104,7 +104,8 @@ class Results:
         try:
             __import__(modname)
             handler = sys.modules[modname]
-        except ImportError:
+        # ValueError can be raised if cmd has dots in it.
+        except (ImportError, ValueError):
             # If we're on line zero, it was the Subject: header that didn't
             # contain a command.  It's possible there's a Re: prefix (or
             # localized version thereof) on the Subject: line that's messing
diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py
index 4a60ceb9..e85cc764 100644
--- a/Mailman/Queue/IncomingRunner.py
+++ b/Mailman/Queue/IncomingRunner.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -126,11 +126,12 @@ class IncomingRunner(Runner):
         # used.  Final fallback is the global pipeline.
         try:
             pipeline = self._get_pipeline(mlist, msg, msgdata)
-            status = self._dopipeline(mlist, msg, msgdata, pipeline)
-            if status:
-                msgdata['pipeline'] = pipeline
+            msgdata['pipeline'] = pipeline
+            more = self._dopipeline(mlist, msg, msgdata, pipeline)
+            if not more:
+                del msgdata['pipeline']
             mlist.Save()
-            return status
+            return more
         finally:
             mlist.Unlock()
 
@@ -166,5 +167,10 @@ class IncomingRunner(Runner):
             except Errors.RejectMessage, e:
                 mlist.BounceMessage(msg, msgdata, e)
                 return 0
+            except:
+                # Push this pipeline module back on the stack, then re-raise
+                # the exception.
+                pipeline.insert(0, handler)
+                raise
         # We've successfully completed handling of this message
         return 0
diff --git a/Mailman/SecurityManager.py b/Mailman/SecurityManager.py
index 8b65738e..af8e6b07 100644
--- a/Mailman/SecurityManager.py
+++ b/Mailman/SecurityManager.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -47,11 +47,12 @@
 # also relies on the security of SHA1.
 
 import os
-import time
+import re
 import sha
+import time
+import Cookie
 import marshal
 import binascii
-import Cookie
 from types import StringType, TupleType
 from urlparse import urlparse
 
@@ -269,14 +270,12 @@ class SecurityManager:
         cookiedata = os.environ.get('HTTP_COOKIE')
         if not cookiedata:
             return 0
-        # Treat the cookie data as simple strings, and do application level
-        # decoding as necessary.  By using SimpleCookie, we prevent any kind
-        # of security breach due to untrusted cookie data being unpickled
-        # (which is quite unsafe).
-        try:
-            c = Cookie.SimpleCookie(cookiedata)
-        except Cookie.CookieError:
-            return 0
+        # We can't use the Cookie module here because it isn't liberal in what
+        # it accepts.  Feed it a MM2.0 cookie along with a MM2.1 cookie and
+        # you get a CookieError. :(.  All we care about is accessing the
+        # cookie data via getitem, so we'll use our own parser, which returns
+        # a dictionary.
+        c = parsecookie(cookiedata)
         # If the user was not supplied, but the authcontext is AuthUser, we
         # can try to glean the user address from the cookie key.  There may be
         # more than one matching key (if the user has multiple accounts
@@ -316,7 +315,7 @@ class SecurityManager:
         # simply request reauthorization, resulting in a new cookie being
         # returned to the client.
         try:
-            data = marshal.loads(binascii.unhexlify(c[key].value))
+            data = marshal.loads(binascii.unhexlify(c[key]))
             issued, received_mac = data
         except (EOFError, ValueError, TypeError, KeyError):
             return 0
@@ -331,3 +330,18 @@ class SecurityManager:
             return 0
         # Authenticated!
         return 1
+
+
+
+splitter = re.compile(';\s*')
+
+def parsecookie(s):
+    c = {}
+    for p in splitter.split(s):
+        try:
+            k, v = p.split('=', 1)
+        except ValueError:
+            pass
+        else:
+            c[k] = v
+    return c
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index b814f3d0..92262684 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
@@ -160,7 +160,7 @@ def wrap(text, column=70, honor_leading_ws=1):
         # end for text in lines
     # the last two newlines are bogus
     return wrapped[:-2]
-    
+
 
 
 def QuotePeriods(text):
@@ -232,7 +232,7 @@ def ScriptURL(target, web_page_url=None, absolute=0):
         fullpath = os.environ.get('SCRIPT_NAME', '') + \
                    os.environ.get('PATH_INFO', '')
     baseurl = urlparse.urlparse(web_page_url)[2]
-    if not absolute and fullpath[:len(baseurl)] == baseurl:
+    if not absolute and fullpath.endswith(baseurl):
         # Use relative addressing
         fullpath = fullpath[len(baseurl):]
         i = fullpath.find('?')
@@ -254,7 +254,7 @@ def GetPossibleMatchingAddrs(name):
     For Example, given scott@pobox.com, return ['scott@pobox.com'],
     given scott@blackbox.pobox.com return ['scott@blackbox.pobox.com',
                                            'scott@pobox.com']"""
-    
+
     name = name.lower()
     user, domain = ParseEmail(name)
     res = [name]
@@ -322,7 +322,7 @@ def set_global_password(pw, siteadmin=1):
         fp.close()
     finally:
         os.umask(omask)
-    
+
 
 def get_global_password(siteadmin=1):
     if siteadmin:
@@ -469,8 +469,10 @@ def maketext(templatefile, dict=None, raw=0, lang=None, mlist=None):
                 # Try again after coercing the template to unicode
                 utemplate = unicode(template, GetCharSet(lang), 'replace')
                 text = sdict.interpolate(utemplate)
-        except (TypeError, ValueError):
+        except (TypeError, ValueError), e:
             # The template is really screwed up
+            from Mailman.Logging.Syslog import syslog
+            syslog('error', 'broken template: %s\n%s', filename, e)
             pass
     if raw:
         return text
@@ -533,7 +535,7 @@ def is_administrivia(msg):
             return 1
     return 0
 
-        
+
 
 def GetRequestURI(fallback=None, escape=1):
     """Return the full virtual path this CGI script was invoked with.
@@ -590,6 +592,9 @@ def GetLanguageDescr(lang):
 def GetCharSet(lang):
     return mm_cfg.LC_DESCRIPTIONS[lang][1]
 
+def IsLanguage(lang):
+    return mm_cfg.LC_DESCRIPTIONS.has_key(lang)
+
 
 
 def get_domain():