Backported several bug fixes from the 2.2 branch.

8 files changed, 76 insertions, 42 deletions

diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 3d790b2e..33aec336 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -232,7 +232,7 @@ def admin_overview(msg=''):
         mlist = MailList.MailList(name, lock=0)
         if mlist.advertised:
             if mm_cfg.VIRTUAL_HOST_OVERVIEW and \
-                   mlist.web_page_url.find(hostname) == -1:
+                   mlist.web_page_url.find('/%s/' % hostname) == -1:
                 # List is for different identity of this host - skip it.
                 continue
             else:
diff --git a/Mailman/Cgi/listinfo.py b/Mailman/Cgi/listinfo.py
index abbf570b..5b965909 100644
--- a/Mailman/Cgi/listinfo.py
+++ b/Mailman/Cgi/listinfo.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -12,7 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
 """Produce listinfo page, primary web entry-point to mailing lists.
 """
@@ -87,7 +88,7 @@ def listinfo_overview(msg=''):
         mlist = MailList.MailList(name, lock=0)
         if mlist.advertised:
             if mm_cfg.VIRTUAL_HOST_OVERVIEW and \
-                   mlist.web_page_url.find(hostname) == -1:
+                   mlist.web_page_url.find('/%s/' % hostname) == -1:
                 # List is for different identity of this host - skip it.
                 continue
             else:
diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py
index 523b3dfc..a1a20205 100644
--- a/Mailman/Handlers/MimeDel.py
+++ b/Mailman/Handlers/MimeDel.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2002-2009 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -183,7 +183,7 @@ def collapse_multipart_alternatives(msg):
             try:
                 firstalt = subpart.get_payload(0)
                 newpayload.append(firstalt)
-            except IndexError:
+            except (IndexError, TypeError):
                 pass
         else:
             newpayload.append(subpart)
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index 64b46eaf..a990d721 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -262,7 +262,7 @@ URL: %(url)s
                 # mono-space font.  Still looks hideous to me, but then I'd
                 # just as soon discard them.
                 def doreplace(s):
-                    return s.replace(' ', ' ').replace('\t', '&nbsp'*8)
+                    return s.expandtabs(8).replace(' ', ' ')
                 lines = [doreplace(s) for s in payload.split('\n')]
                 payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n'
                 part.set_payload(payload)
diff --git a/Mailman/Pending.py b/Mailman/Pending.py
index e6372eda..191ead1a 100644
--- a/Mailman/Pending.py
+++ b/Mailman/Pending.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -12,7 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
 """Track pending actions which require confirmation."""
 
@@ -23,6 +24,7 @@ import random
 import cPickle
 
 from Mailman import mm_cfg
+from Mailman import UserDesc
 from Mailman.Utils import sha_new
 
 # Types of pending records
@@ -174,9 +176,15 @@ def _update(olddb):
         # know that the only things that were kept in the old format were
         # subscription requests.  Also, the old request format didn't have the
         # subscription language.  Best we can do here is use the server
-        # default.
-        db[cookie] = (SUBSCRIPTION,) + data[:-1] + \
-                     (mm_cfg.DEFAULT_SERVER_LANGUAGE,)
+        # default.  We also need a fullname because confirmation processing
+        # references all those UserDesc attributes.
+        ud = UserDesc.UserDesc(address=data[0],
+                               fullname='',
+                               password=data[1],
+                               digest=data[2],
+                               lang=mm_cfg.DEFAULT_SERVER_LANGUAGE,
+                               )
+        db[cookie] = (SUBSCRIPTION, ud)
         # The old database format kept the timestamp as the time the request
         # was made.  The new format keeps it as the time the request should be
         # evicted.
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 847d1a82..b0eb2dd0 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -808,12 +808,25 @@ def canonstr(s, lang=None):
     newparts = []
     parts = re.split(r'&(?P<ref>[^;]+);', s)
     def appchr(i):
-        if i < 256:
-            newparts.append(chr(i))
+        # do everything in unicode
+        newparts.append(unichr(i))
+    def tounicode(s):
+        # We want the default fallback to be iso-8859-1 even if the language
+        # is English (us-ascii).  This seems like a practical compromise so
+        # that non-ASCII characters in names can be used in English lists w/o
+        # having to change the global charset for English from us-ascii (which
+        # I superstitiously think may have unintended consequences).
+        if isinstance(s, unicode):
+            return s
+        if lang is None:
+            charset = 'iso-8859-1'
         else:
-            newparts.append(unichr(i))
+            charset = GetCharSet(lang)
+            if charset == 'us-ascii':
+                charset = 'iso-8859-1'
+        return unicode(s, charset, 'replace')
     while True:
-        newparts.append(parts.pop(0))
+        newparts.append(tounicode(parts.pop(0)))
         if not parts:
             break
         ref = parts.pop(0)
@@ -822,28 +835,16 @@ def canonstr(s, lang=None):
                 appchr(int(ref[1:]))
             except ValueError:
                 # Non-convertable, stick with what we got
-                newparts.append('&'+ref+';')
+                newparts.append(tounicode('&'+ref+';'))
         else:
             c = htmlentitydefs.entitydefs.get(ref, '?')
             if c.startswith('#') and c.endswith(';'):
                 appchr(int(ref[1:-1]))
             else:
-                newparts.append(c)
+                newparts.append(tounicode(c))
     newstr = EMPTYSTRING.join(newparts)
-    if isinstance(newstr, UnicodeType):
-        return newstr
-    # We want the default fallback to be iso-8859-1 even if the language is
-    # English (us-ascii).  This seems like a practical compromise so that
-    # non-ASCII characters in names can be used in English lists w/o having to
-    # change the global charset for English from us-ascii (which I
-    # superstitiously think may have unintended consequences).
-    if lang is None:
-        charset = 'iso-8859-1'
-    else:
-        charset = GetCharSet(lang)
-        if charset == 'us-ascii':
-            charset = 'iso-8859-1'
-    return unicode(newstr, charset, 'replace')
+    # newstr is unicode
+    return newstr
 
 
 # The opposite of canonstr() -- sorta.  I.e. it attempts to encode s in the
diff --git a/NEWS b/NEWS
index d8006b22..bae87f49 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,27 @@ Here is a history of user visible changes to Mailman.
 
   Bug Fixes and other patches
 
+    - Scrubbed HTML attachments containing tab characters would get the tabs
+      replaced by a string of '&nbsp' without a semicolon.  Fixed.
+
+    - Caught a TypeError in content filtering, collapse alternatives that
+      occurred with a malformed message if a multipart/alternative part
+      wasn't multi-part.  Reported in comments to bug #266230.
+
+    - Fixed a few things in bin/update:
+      - Changed some old messages for more current meaning.
+      - Fixed qfiles update to not lose metadata from 2.1.5+ format entries.
+      - Fixed 2.0.x template migration to not die if the templates/ tree
+        contains subdirectories from a version control system.
+
+    - Fixed a bug that would show a list on the admin and listinfo overview
+      pages if its web_page_url host contained the current host as a
+      substring.  Bug #342162.
+
+    - Fixed a bug in Utils.canonstr() that would throw a UnicodeDecodeError
+      if the string contained an HTML entity > 255 and also characters in the
+      128-255 range.  Bug #341594.
+
     - Added recognition for more bounces.
 
     - Updated contrib/mmdsr to report preserved messages.
diff --git a/bin/update b/bin/update
index e36c1c40..d74cae92 100755
--- a/bin/update
+++ b/bin/update
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -14,7 +14,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
 """Perform all necessary upgrades.
 
@@ -126,8 +127,8 @@ def move_language_templates(mlist):
         try:
             fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate))
         except IOError, e:
-            if e.errno <> errno.ENOENT: raise
-            # No global template
+            if e.errno not in (errno.ENOENT, errno.EISDIR): raise
+            # No global template or maybe a VCS directory
             continue
 
         gcksum = Utils.md5_new(fp.read()).digest()
@@ -298,9 +299,7 @@ script.
         %(newname)s""")
         else:
             # directory
-            print _("""\
-    looks like you have a really recent CVS installation...
-    you're either one brave soul, or you already ran me""")
+            print _('Nothing to do.')
 
 
     #
@@ -321,9 +320,7 @@ script.
     to
         %(newname)s""")
         else: # directory
-            print _("""\
-    looks like you have a really recent CVS installation...
-    you're either one brave soul, or you already ran me""")
+            print _('Nothing to do.')
 
     #
     # move the html archives there
@@ -510,6 +507,12 @@ def dequeue(filebase):
         try:
             msgfp = open(pckfile)
             msg = cPickle.load(msgfp)
+            if not data:
+                # There was no .db file. Is this a post 2.1.5 .pck?
+                try:
+                    data = cPickle.load(msgfp)
+                except EOFError:
+                    pass
             os.unlink(pckfile)
         except EnvironmentError, e:
             if e.errno <> errno.ENOENT: raise