Backporting from trunk

26 files changed, 445 insertions, 211 deletions

diff --git a/Mailman/Bouncers/DSN.py b/Mailman/Bouncers/DSN.py
index 3e040bef..6c32f0ff 100644
--- a/Mailman/Bouncers/DSN.py
+++ b/Mailman/Bouncers/DSN.py
@@ -1,20 +1,24 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
-"""Parse RFC 1894 (i.e. DSN) bounce formats."""
+"""Parse RFC 3464 (i.e. DSN) bounce formats.
+
+RFC 3464 obsoletes 1894 which was the old DSN standard.  This module has not
+been audited for differences between the two.
+"""
 
 from email.Iterators import typed_subpart_iterator
 from email.Utils import parseaddr
diff --git a/Mailman/Bouncers/Microsoft.py b/Mailman/Bouncers/Microsoft.py
index 65d49cc1..f14268a9 100644
--- a/Mailman/Bouncers/Microsoft.py
+++ b/Mailman/Bouncers/Microsoft.py
@@ -1,23 +1,24 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Microsoft's `SMTPSVC' nears I kin tell."""
 
 import re
 from cStringIO import StringIO
+from types import ListType
 
 scre = re.compile(r'transcript of session follows', re.IGNORECASE)
 
@@ -32,7 +33,11 @@ def process(msg):
     except IndexError:
         # The message *looked* like a multipart but wasn't
         return None
-    body = StringIO(subpart.get_payload())
+    data = subpart.get_payload()
+    if isinstance(data, ListType):
+        # The message is a multi-multipart, so not a matching bounce
+        return None
+    body = StringIO(data)
     state = 0
     addrs = []
     while 1:
diff --git a/Mailman/Bouncers/Postfix.py b/Mailman/Bouncers/Postfix.py
index fb1a1233..447e326c 100644
--- a/Mailman/Bouncers/Postfix.py
+++ b/Mailman/Bouncers/Postfix.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Parse bounce messages generated by Postfix.
@@ -20,12 +20,10 @@ This also matches something called `Keftamail' which looks just like Postfix
 bounces with the word Postfix scratched out and the word `Keftamail' written
 in in crayon.
 
-It also matches something claiming to be `The BNS Postfix program'.
-/Everybody's/ gotta be different, huh?
-
+It also matches something claiming to be `The BNS Postfix program', and
+`SMTP_Gateway'.  Everybody's gotta be different, huh?
 """
 
-
 import re
 from cStringIO import StringIO
 
@@ -42,7 +40,8 @@ def flatten(msg, leaves):
 
 
 # are these heuristics correct or guaranteed?
-pcre = re.compile(r'[ \t]*the\s*(bns)?\s*(postfix|keftamail)', re.IGNORECASE)
+pcre = re.compile(r'[ \t]*the\s*(bns)?\s*(postfix|keftamail|smtp_gateway)',
+                  re.IGNORECASE)
 rcre = re.compile(r'failure reason:$', re.IGNORECASE)
 acre = re.compile(r'<(?P<addr>[^>]*)>:')
 
diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py
index ccc8d6ed..9cb0832b 100644
--- a/Mailman/Bouncers/SimpleMatch.py
+++ b/Mailman/Bouncers/SimpleMatch.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -70,6 +70,10 @@ PATTERNS = [
     (_c('Undeliverable Address:\s*(?P<addr>.*)$'),
      _c('Original message attached'),
      _c('Undeliverable Address:\s*(?P<addr>.*)$')),
+    # Another demon.co.uk format
+    (_c('This message was created automatically by mail delivery'),
+     _c('^---- START OF RETURNED MESSAGE ----'),
+     _c("addressed to '(?P<addr>[^']*)'")),
     # Next one goes here...
     ]
 
diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 1c629c10..17b3919f 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -1234,7 +1234,7 @@ def change_options(mlist, category, subcat, cgidata, doc):
         # Default is to subscribe
         subscribe_or_invite = safeint('subscribe_or_invite', 0)
         invitation = cgidata.getvalue('invitation', '')
-        digest = 0
+        digest = mlist.digest_is_default
         if not mlist.digestable:
             digest = 0
         if not mlist.nondigestable:
diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py
index abb0ac29..23a92740 100644
--- a/Mailman/Cgi/confirm.py
+++ b/Mailman/Cgi/confirm.py
@@ -348,6 +348,11 @@ def subscription_confirm(mlist, doc, cookie, cgidata):
             address that has already been unsubscribed.'''))
         except Errors.MMAlreadyAMember:
             doc.addError(_("You are already a member of this mailing list!"))
+        except Errors.HostileSubscriptionError:
+            doc.addError(_("""\
+            You were not invited to this mailing list.  The invitation has
+            been discarded, and both list administrators have been
+            alerted."""))
         else:
             # Use the user's preferred language
             i18n.set_language(lang)
diff --git a/Mailman/Commands/cmd_confirm.py b/Mailman/Commands/cmd_confirm.py
index 5e4fc701..f93e7b9b 100644
--- a/Mailman/Commands/cmd_confirm.py
+++ b/Mailman/Commands/cmd_confirm.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2002-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
 """
     confirm <confirmation-string>
         Confirm an action.  The confirmation-string is required and should be
-        supplied with in mailback confirmation notice.
+        supplied by a mailback confirmation notice.
 """
 
 from Mailman import mm_cfg
@@ -63,6 +63,10 @@ Your request has been forwarded to the list moderator for approval."""))
         res.results.append(_("""\
 You are not current a member.  Have you already unsubscribed or changed
 your email address?"""))
+    except Errors.HostileSubscriptionError:
+        res.results.append(_("""\
+You were not invited to this mailing list.  The invitation has been discarded,
+and both list administrators have been alerted."""))
     else:
         if ((results[0] == Pending.SUBSCRIPTION and mlist.send_welcome_msg)
             or
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 4286e468..06b510e0 100644
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -507,13 +507,13 @@ SMTP_LOG_EVERY_MESSAGE = (
 # Mutually exclusive with SMTP_LOG_REFUSED.
 SMTP_LOG_SUCCESS = (
     'post',
-    'post to %(listname)s from %(sender)s, size=%(size)d, success')
+    'post to %(listname)s from %(sender)s, size=%(size)d, message-id=%(msg_message-id)s, success')
 
 # This will only be printed if there were any addresses which encountered an
 # immediate smtp failure.  Mutually exclusive with SMTP_LOG_SUCCESS.
 SMTP_LOG_REFUSED = (
     'post',
-    'post to %(listname)s from %(sender)s, size=%(size)d, %(#refused)d failures')
+    'post to %(listname)s from %(sender)s, size=%(size)d, message-id=%(msg_message-id)s, %(#refused)d failures')
 
 # This will be logged for each specific recipient failure.  Additional %()s
 # keys are:
@@ -1032,6 +1032,9 @@ PENDING_REQUEST_LIFE = days(3)
 # will be dequeued and those recipients will never receive the message.
 DELIVERY_RETRY_PERIOD = days(5)
 
+# How long should we wait before we retry a temporary delivery failure?
+DELIVERY_RETRY_WAIT = hours(1)
+
 
 
 #####
@@ -1062,6 +1065,25 @@ LIST_LOCK_LIFETIME = hours(5)
 # the message will be re-queued for later delivery.
 LIST_LOCK_TIMEOUT = seconds(10)
 
+# Set this to true to turn on lock debugging messages for the pending requests
+# database, which will be written to logs/locks.  If you think you're having
+# lock problems, or just want to tune the locks for your system, turn on lock
+# debugging.
+PENDINGDB_LOCK_DEBUGGING = 0
+
+# This variable specifies how long an attempt will be made to acquire a
+# pendingdb lock by the incoming qrunner process.  If the lock acquisition
+# times out, the message will be re-queued for later delivery.
+PENDINGDB_LOCK_TIMEOUT = seconds(30)
+
+# The pendingdb is shared among all lists, and handles all list
+# (un)subscriptions, admin approvals and otherwise held messages, so it is
+# potentially locked a lot more often than single lists.  Mailman deals with
+# this by re-trying any attempts to alter the pendingdb that failed because of
+# locking errors.  This variable indicates how many attempt should be made
+# before abandoning all hope.
+PENDINGDB_LOCK_ATTEMPTS = 10
+
 
 
 #####
diff --git a/Mailman/Deliverer.py b/Mailman/Deliverer.py
index 983a67d5..be5ddfe9 100644
--- a/Mailman/Deliverer.py
+++ b/Mailman/Deliverer.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -24,9 +24,17 @@ from Mailman import mm_cfg
 from Mailman import Errors
 from Mailman import Utils
 from Mailman import Message
-from Mailman.i18n import _
+from Mailman import i18n
 from Mailman.Logging.Syslog import syslog
 
+_ = i18n._
+
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 class Deliverer:
@@ -54,9 +62,10 @@ your membership administrative address, %(addr)s.'''))
              'welcome'     : welcome,
              'umbrella'    : umbrella,
              'emailaddr'   : self.GetListEmail(),
-             'listinfo_url': self.GetScriptURL('listinfo', absolute=1),
-             'optionsurl'  : self.GetOptionsURL(name, absolute=1),
+             'listinfo_url': self.GetScriptURL('listinfo', absolute=True),
+             'optionsurl'  : self.GetOptionsURL(name, absolute=True),
              'password'    : password,
+             'user'        : self.getMemberCPAddress(name),
              }, lang=pluser, mlist=self)
         if digest:
             digmode = _(' (Digest mode)')
@@ -109,7 +118,7 @@ your membership administrative address, %(addr)s.'''))
              'listname'   : self.real_name,
              'fqdn_lname' : self.GetListEmail(),
              'password'   : self.getMemberPassword(user),
-             'options_url': self.GetOptionsURL(user, absolute=1),
+             'options_url': self.GetOptionsURL(user, absolute=True),
              'requestaddr': requestaddr,
              'owneraddr'  : self.GetOwnerEmail(),
             }, lang=self.getMemberLanguage(user), mlist=self)
@@ -118,7 +127,7 @@ your membership administrative address, %(addr)s.'''))
         msg['X-No-Archive'] = 'yes'
         msg.send(self, verp=mm_cfg.VERP_PERSONALIZED_DELIVERIES)
 
-    def ForwardMessage(self, msg, text=None, subject=None, tomoderators=1):
+    def ForwardMessage(self, msg, text=None, subject=None, tomoderators=True):
         # Wrap the message as an attachment
         if text is None:
             text = _('No reason given')
@@ -134,3 +143,41 @@ your membership administrative address, %(addr)s.'''))
         notice.attach(text)
         notice.attach(attachment)
         notice.send(self)
+
+    def SendHostileSubscriptionNotice(self, listname, address):
+        # Some one was invited to one list but tried to confirm to a different
+        # list.  We inform both list owners of the bogosity, but be careful
+        # not to reveal too much information.
+        selfname = self.internal_name()
+        syslog('mischief', '%s was invited to %s but confirmed to %s',
+               address, listname, selfname)
+        # First send a notice to the attacked list
+        msg = Message.OwnerNotification(
+            self,
+            _('Hostile subscription attempt detected'),
+            Utils.wrap(_("""%(address)s was invited to a different mailing
+list, but in a deliberate malicious attempt they tried to confirm the
+invitation to your list.  We just thought you'd like to know.  No further
+action by you is required.""")))
+        msg.send(self)
+        # Now send a notice to the invitee list
+        try:
+            # Avoid import loops
+            from Mailman.MailList import MailList
+            mlist = MailList(listname, lock=False)
+        except Errors.MMListError:
+            # Oh well
+            return
+        otrans = i18n.get_translation()
+        i18n.set_language(mlist.preferred_language)
+        try:
+            msg = Message.OwnerNotification(
+                mlist,
+                _('Hostile subscription attempt detected'),
+                Utils.wrap(_("""You invited %(address)s to your list, but in a
+deliberate malicious attempt, they tried to confirm the invitation to a
+different list.  We just thought you'd like to know.  No further action by you
+is required.""")))
+            msg.send(mlist)
+        finally:
+            i18n.set_translation(otrans)
diff --git a/Mailman/Errors.py b/Mailman/Errors.py
index ce1868cc..a2329729 100644
--- a/Mailman/Errors.py
+++ b/Mailman/Errors.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
@@ -145,3 +145,10 @@ class RejectMessage(HandlerError):
 
     def notice(self):
         return self.__notice
+
+
+# Additional exceptions
+class HostileSubscriptionError(MailmanError):
+    """A cross-subscription attempt was made."""
+    # This exception gets raised when an invitee attempts to use the
+    # invitation to cross-subscribe to some other mailing list.
diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
index c4ad06ab..9ef5550c 100644
--- a/Mailman/Handlers/CookHeaders.py
+++ b/Mailman/Handlers/CookHeaders.py
@@ -229,6 +229,8 @@ def prefix_subject(mlist, msg, msgdata):
     if len(lines) > 1 and lines[1] and lines[1][0] in ' \t':
         ws = lines[1][0]
     msgdata['origsubj'] = subject
+    if not subject:
+        subject = _('(no subject)')
     # The header may be multilingual; decode it from base64/quopri and search
     # each chunk for the prefix.  BAW: Note that if the prefix contains spaces
     # and each word of the prefix is encoded in a different chunk in the
@@ -241,8 +243,6 @@ def prefix_subject(mlist, msg, msgdata):
                 # The subject's already got the prefix, so don't change it
                 return
     del msg['subject']
-    if not subject:
-        subject = _('(no subject)')
     # Get the header as a Header instance, with proper unicode conversion
     h = uheader(mlist, prefix, 'Subject', continuation_ws=ws)
     for s, c in headerbits:
diff --git a/Mailman/Handlers/Hold.py b/Mailman/Handlers/Hold.py
index 15223959..c3a6d6f8 100644
--- a/Mailman/Handlers/Hold.py
+++ b/Mailman/Handlers/Hold.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -197,6 +197,7 @@ def hold_for_approval(mlist, msg, msgdata, exc):
         exc = exc()
     listname = mlist.real_name
     sender = msgdata.get('sender', msg.get_sender())
+    message_id = msg.get('message-id', 'n/a')
     owneraddr = mlist.GetOwnerEmail()
     adminaddr = mlist.GetBouncesEmail()
     requestaddr = mlist.GetRequestEmail()
@@ -274,7 +275,8 @@ also appear in the first line of the body of the reply.""")),
         finally:
             i18n.set_translation(otranslation)
     # Log the held message
-    syslog('vette', '%s post from %s held: %s', listname, sender, reason)
+    syslog('vette', '%s post from %s held, message-id=%s: %s',
+           listname, sender, message_id, reason)
     # raise the specific MessageHeld exception to exit out of the message
     # delivery pipeline
     raise exc
diff --git a/Mailman/Handlers/Replybot.py b/Mailman/Handlers/Replybot.py
index 8a9be5cb..30fbb512 100644
--- a/Mailman/Handlers/Replybot.py
+++ b/Mailman/Handlers/Replybot.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -71,8 +71,8 @@ def process(mlist, msg, msgdata):
     # Okay, we know we're going to auto-respond to this sender, craft the
     # message, send it, and update the database.
     realname = mlist.real_name
-    subject = _('Auto-response for your message to ') + \
-              msg.get('to',  _('the "%(realname)s" mailing list'))
+    subject = _(
+        'Auto-response for your message to the "%(realname)s" mailing list')
     # Do string interpolation
     d = SafeDict({'listname'    : realname,
                   'listurl'     : mlist.GetScriptURL('listinfo'),
diff --git a/Mailman/ListAdmin.py b/Mailman/ListAdmin.py
index d4d72375..18ef945a 100644
--- a/Mailman/ListAdmin.py
+++ b/Mailman/ListAdmin.py
@@ -205,7 +205,9 @@ class ListAdmin:
         self.__opendb()
         # get the next unique id
         id = self.__request_id()
-        assert not self.__db.has_key(id)
+        while self.__db.has_key(id):
+            # Shouldn't happen unless the db has gone odd, but let's cope.
+            id = self.__request_id()
         # get the message sender
         sender = msg.get_sender()
         # calculate the file name for the message text and write it to disk
diff --git a/Mailman/LockFile.py b/Mailman/LockFile.py
index 796a81eb..f8813839 100644
--- a/Mailman/LockFile.py
+++ b/Mailman/LockFile.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Portable, NFS-safe file locking with timeouts.
@@ -271,14 +271,15 @@ class LockFile:
                 elif e.errno <> errno.EEXIST:
                     # Something very bizarre happened.  Clean up our state and
                     # pass the error on up.
-                    self.__writelog('unexpected link error: %s' % e)
+                    self.__writelog('unexpected link error: %s' % e,
+                                    important=1)
                     os.unlink(self.__tmpfname)
                     raise
                 elif self.__linkcount() <> 2:
                     # Somebody's messin' with us!  Log this, and try again
                     # later.  TBD: should we raise an exception?
                     self.__writelog('unexpected linkcount: %d' %
-                                    self.__linkcount())
+                                    self.__linkcount(), important=1)
                 elif self.__read() == self.__tmpfname:
                     # It was us that already had the link.
                     self.__writelog('already locked')
@@ -296,7 +297,8 @@ class LockFile:
             if time.time() > self.__releasetime() + CLOCK_SLOP:
                 # Yes, so break the lock.
                 self.__break()
-                self.__writelog('lifetime has expired, breaking')
+                self.__writelog('lifetime has expired, breaking',
+                                important=1)
             # Okay, someone else has the lock, our claim hasn't timed out yet,
             # and the expected lock lifetime hasn't expired yet.  So let's
             # wait a while for the owner of the lock to give it up.
@@ -402,8 +404,8 @@ class LockFile:
     # Private interface
     #
 
-    def __writelog(self, msg):
-        if self.__withlogging:
+    def __writelog(self, msg, important=0):
+        if self.__withlogging or important:
             logf = _get_logfile()
             logf.write('%s %s\n' % (self.__logprefix, msg))
             traceback.print_stack(file=logf)
@@ -560,7 +562,7 @@ def _onetest():
             except KeyboardInterrupt:
                 pass
             os._exit(0)
-    
+
 
 def _reap(kids):
     if not kids:
diff --git a/Mailman/MTA/Manual.py b/Mailman/MTA/Manual.py
index dd9127cc..db4161e0 100644
--- a/Mailman/MTA/Manual.py
+++ b/Mailman/MTA/Manual.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -26,6 +26,12 @@ from Mailman.Queue.sbcache import get_switchboard
 from Mailman.i18n import _
 from Mailman.MTA.Utils import makealiases
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 # no-ops for interface compliance
@@ -33,7 +39,7 @@ def makelock():
     class Dummy:
         def lock(self):
             pass
-        def unlock(self, unconditionally=0):
+        def unlock(self, unconditionally=False):
             pass
     return Dummy()
 
@@ -44,7 +50,7 @@ def clear():
 
 
 # nolock argument is ignored, but exists for interface compliance
-def create(mlist, cgi=0, nolock=0):
+def create(mlist, cgi=False, nolock=False, quiet=False):
     if mlist is None:
         return
     listname = mlist.internal_name()
@@ -54,7 +60,8 @@ def create(mlist, cgi=0, nolock=0):
         # an email message to mailman-owner requesting that the proper aliases
         # be installed.
         sfp = StringIO()
-        print >> sfp, _("""\
+        if not quiet:
+            print >> sfp, _("""\
 The mailing list `%(listname)s' has been created via the through-the-web
 interface.  In order to complete the activation of this mailing list, the
 proper /etc/aliases (or equivalent) file must be updated.  The program
@@ -64,11 +71,13 @@ Here are the entries for the /etc/aliases file:
 """)
         outfp = sfp
     else:
-        print _("""
+        if not quiet:
+            print _("""\
 To finish creating your mailing list, you must edit your /etc/aliases (or
 equivalent) file by adding the following lines, and possibly running the
 `newaliases' program:
-
+""")
+        print _("""\
 ## %(listname)s mailing list""")
         outfp = sys.stdout
     # Common path
@@ -92,7 +101,7 @@ equivalent) file by adding the following lines, and possibly running the
 
 
 
-def remove(mlist, cgi=0):
+def remove(mlist, cgi=False):
     listname = mlist.internal_name()
     fieldsz = len(listname) + len('-unsubscribe')
     if cgi:
diff --git a/Mailman/MTA/Postfix.py b/Mailman/MTA/Postfix.py
index 84718e5f..929ed1b5 100644
--- a/Mailman/MTA/Postfix.py
+++ b/Mailman/MTA/Postfix.py
@@ -18,10 +18,10 @@
 """
 
 import os
-import time
-import errno
 import pwd
 import grp
+import time
+import errno
 from stat import *
 
 from Mailman import mm_cfg
@@ -35,6 +35,12 @@ LOCKFILE = os.path.join(mm_cfg.LOCK_DIR, 'creator')
 ALIASFILE = os.path.join(mm_cfg.DATA_DIR, 'aliases')
 VIRTFILE = os.path.join(mm_cfg.DATA_DIR, 'virtual-mailman')
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 def _update_maps():
@@ -160,7 +166,7 @@ def _check_for_virtual_loopaddr(mlist, filename):
         os.umask(omask)
     try:
         # Find the start of the loop address block
-        while 1:
+        while True:
             line = infp.readline()
             if not line:
                 break
@@ -168,7 +174,7 @@ def _check_for_virtual_loopaddr(mlist, filename):
             if line.startswith('# LOOP ADDRESSES START'):
                 break
         # Now see if our domain has already been written
-        while 1:
+        while True:
             line = infp.readline()
             if not line:
                 break
@@ -212,8 +218,8 @@ def _do_create(mlist, textfile, func):
         _check_for_virtual_loopaddr(mlist, textfile)
 
 
-def create(mlist, cgi=0, nolock=0):
-    # Acquire the global list database lock
+def create(mlist, cgi=False, nolock=False, quiet=False):
+    # Acquire the global list database lock.  quiet flag is ignored.
     lock = None
     if not nolock:
         lock = makelock()
@@ -226,7 +232,7 @@ def create(mlist, cgi=0, nolock=0):
         _update_maps()
     finally:
         if lock:
-            lock.unlock(unconditionally=1)
+            lock.unlock(unconditionally=True)
 
 
 
@@ -247,7 +253,7 @@ def _do_remove(mlist, textfile, virtualp):
             outfp = open(textfile + '.tmp', 'w')
         finally:
             os.umask(omask)
-        filteroutp = 0
+        filteroutp = False
         start = '# STANZA START: ' + listname
         end = '# STANZA END: ' + listname
         while 1:
@@ -260,7 +266,7 @@ def _do_remove(mlist, textfile, virtualp):
             # marker.
             if filteroutp:
                 if line.strip() == end:
-                    filteroutp = 0
+                    filteroutp = False
                     # Discard the trailing blank line, but don't worry if
                     # we're at the end of the file.
                     infp.readline()
@@ -268,7 +274,7 @@ def _do_remove(mlist, textfile, virtualp):
             else:
                 if line.strip() == start:
                     # Filter out this stanza
-                    filteroutp = 1
+                    filteroutp = True
                 else:
                     outfp.write(line)
     # Close up shop, and rotate the files
@@ -278,18 +284,18 @@ def _do_remove(mlist, textfile, virtualp):
     os.rename(textfile+'.tmp', textfile)
 
 
-def remove(mlist, cgi=0):
+def remove(mlist, cgi=False):
     # Acquire the global list database lock
     lock = makelock()
     lock.lock()
     try:
-        _do_remove(mlist, ALIASFILE, 0)
+        _do_remove(mlist, ALIASFILE, False)
         if mlist.host_name in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS:
-            _do_remove(mlist, VIRTFILE, 1)
+            _do_remove(mlist, VIRTFILE, True)
         # Regenerate the alias and map files
         _update_maps()
     finally:
-        lock.unlock(unconditionally=1)
+        lock.unlock(unconditionally=True)
 
 
 
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 335a7d71..67f0329c 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -66,11 +66,19 @@ from Mailman.OldStyleMemberships import OldStyleMemberships
 from Mailman import Message
 from Mailman import Pending
 from Mailman import Site
-from Mailman.i18n import _
+from Mailman import i18n
 from Mailman.Logging.Syslog import syslog
 
+_ = i18n._
+
 EMPTYSTRING = ''
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 # Use mixins here just to avoid having any one chunk be too large.
@@ -93,25 +101,27 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         self.InitTempVars(name)
         # Default membership adaptor class
         self._memberadaptor = OldStyleMemberships(self)
-        if name:
-            if lock:
-                # This will load the database.
-                self.Lock()
-            else:
-                self.Load()
-            # This extension mechanism allows list-specific overrides of any
-            # method (well, except __init__(), InitTempVars(), and InitVars()
-            # I think).
-            filename = os.path.join(self.fullpath(), 'extend.py')
-            dict = {}
-            try:
-                execfile(filename, dict)
-            except IOError, e:
-                if e.errno <> errno.ENOENT: raise
-            else:
-                func = dict.get('extend')
-                if func:
-                    func(self)
+        # This extension mechanism allows list-specific overrides of any
+        # method (well, except __init__(), InitTempVars(), and InitVars()
+        # I think).  Note that fullpath() will return None when we're creating
+        # the list, which will only happen when name is None.
+        if name is None:
+            return
+        filename = os.path.join(self.fullpath(), 'extend.py')
+        dict = {}
+        try:
+            execfile(filename, dict)
+        except IOError, e:
+            if e.errno <> errno.ENOENT: raise
+        else:
+            func = dict.get('extend')
+            if func:
+                func(self)
+        if lock:
+            # This will load the database.
+            self.Lock()
+        else:
+            self.Load()
 
     def __getattr__(self, name):
         # Because we're using delegation, we want to be sure that attribute
@@ -677,8 +687,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         # Hack alert!  Squirrel away a flag that only invitations have, so
         # that we can do something slightly different when an invitation
         # subscription is confirmed.  In those cases, we don't need further
-        # admin approval, even if the list is so configured
-        userdesc.invitation = 1
+        # admin approval, even if the list is so configured.  The flag is the
+        # list name to prevent invitees from cross-subscribing.
+        userdesc.invitation = self.internal_name()
         cookie = Pending.new(Pending.SUBSCRIPTION, userdesc)
         confirmurl = '%s/%s' % (self.GetScriptURL('confirm', absolute=1),
                                 cookie)
@@ -890,17 +901,25 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
             self.SendSubscribeAck(email, self.getMemberPassword(email),
                                   digest, text)
         if admin_notif:
-            realname = self.real_name
-            subject = _('%(realname)s subscription notification')
+            lang = self.preferred_language
+            otrans = i18n.get_translation()
+            i18n.set_language(lang)
+            try:
+                realname = self.real_name
+                subject = _('%(realname)s subscription notification')
+            finally:
+                i18n.set_translation(otrans)
+            if isinstance(name, UnicodeType):
+                name = name.encode(Utils.GetCharSet(lang), 'replace')
             text = Utils.maketext(
                 "adminsubscribeack.txt",
-                {"listname" : self.real_name,
+                {"listname" : realname,
                  "member"   : formataddr((name, email)),
                  }, mlist=self)
             msg = Message.OwnerNotification(self, subject, text)
             msg.send(self)
 
-    def DeleteMember(self, name, whence=None, admin_notif=0, userack=1):
+    def DeleteMember(self, name, whence=None, admin_notif=None, userack=True):
         realname, email = parseaddr(name)
         if self.unsubscribe_policy == 0:
             self.ApprovedDeleteMember(name, whence, admin_notif, userack)
@@ -1059,11 +1078,18 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
             except ValueError:
                 raise Errors.MMBadConfirmation, 'bad subscr data %s' % (data,)
             # Hack alert!  Was this a confirmation of an invitation?
-            invitation = getattr(userdesc, 'invitation', 0)
+            invitation = getattr(userdesc, 'invitation', False)
             # We check for both 2 (approval required) and 3 (confirm +
             # approval) because the policy could have been changed in the
             # middle of the confirmation dance.
-            if not invitation and self.subscribe_policy in (2, 3):
+            if invitation:
+                if invitation <> self.internal_name():
+                    # Not cool.  The invitee was trying to subscribe to a
+                    # different list than they were invited to.  Alert both
+                    # list administrators.
+                    self.SendHostileSubscriptionNotice(invitation, addr)
+                    raise Errors.HostileSubscriptionError
+            elif self.subscribe_policy in (2, 3):
                 self.HoldSubscription(addr, fullname, password, digest, lang)
                 name = self.real_name
                 raise Errors.MMNeedApproval, _(
diff --git a/Mailman/MemberAdaptor.py b/Mailman/MemberAdaptor.py
index dc24ea08..edcb659c 100644
--- a/Mailman/MemberAdaptor.py
+++ b/Mailman/MemberAdaptor.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -42,8 +42,8 @@ email interfaces.  Updating membership information in that case is the
 backend's responsibility.  Adaptors are allowed to support parts of the
 writeable interface.
 
-For any writeable method not supported, a NotImplemented exception should be
-raised.
+For any writeable method not supported, a NotImplementedError exception should
+be raised.
 """
 
 # Delivery statuses
@@ -61,15 +61,15 @@ class MemberAdaptor:
     #
     def getMembers(self):
         """Get the LCE for all the members of the mailing list."""
-        raise NotImplemented
+        raise NotImplementedError
 
     def getRegularMemberKeys(self):
         """Get the LCE for all regular delivery members (i.e. non-digest)."""
-        raise NotImplemented
+        raise NotImplementedError
 
     def getDigestMemberKeys(self):
         """Get the LCE for all digest delivery members."""
-        raise NotImplemented
+        raise NotImplementedError
 
     def isMember(self, member):
         """Return 1 if member KEY/LCE is a valid member, otherwise 0."""
@@ -79,14 +79,14 @@ class MemberAdaptor:
 
         If member does not refer to a valid member, raise NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberCPAddress(self, member):
         """Return the CPE for the member KEY/LCE.
 
         If member does not refer to a valid member, raise NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberCPAddresses(self, members):
         """Return a sequence of CPEs for the given sequence of members.
@@ -96,7 +96,7 @@ class MemberAdaptor:
         in the returned sequence will be None (i.e. NotAMemberError is never
         raised).
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def authenticateMember(self, member, response):
         """Authenticate the member KEY/LCE with the given response.
@@ -106,14 +106,14 @@ class MemberAdaptor:
         password, but it will be used to craft a session cookie, so it should
         be persistent for the life of the session.
 
-        If the authentication failed return 0.  If member did not refer to a
-        valid member, raise NotAMemberError.
+        If the authentication failed return False.  If member did not refer to
+        a valid member, raise NotAMemberError.
 
         Normally, the response will be the password typed into a web form or
         given in an email command, but it needn't be.  It is up to the adaptor
         to compare the typed response to the user's authentication token.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberPassword(self, member):
         """Return the member's password.
@@ -121,7 +121,7 @@ class MemberAdaptor:
         If the member KEY/LCE is not a member of the list, raise
         NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberLanguage(self, member):
         """Return the preferred language for the member KEY/LCE.
@@ -132,7 +132,7 @@ class MemberAdaptor:
         If member does not refer to a valid member, the list's default
         language is returned instead of raising a NotAMemberError error.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberOption(self, member, flag):
         """Return the boolean state of the member option for member KEY/LCE.
@@ -141,7 +141,7 @@ class MemberAdaptor:
 
         If member does not refer to a valid member, raise NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberName(self, member):
         """Return the full name of the member KEY/LCE.
@@ -151,14 +151,14 @@ class MemberAdaptor:
         characters in the name.  NotAMemberError is raised if member does not
         refer to a valid member.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getMemberTopics(self, member):
         """Return the list of topics this member is interested in.
 
         The return value is a list of strings which name the topics.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getDeliveryStatus(self, member):
         """Return the delivery status of this member.
@@ -176,7 +176,7 @@ class MemberAdaptor:
 
         If member is not a member of the list, raise NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getDeliveryStatusChangeTime(self, member):
         """Return the time of the last disabled delivery status change.
@@ -185,7 +185,7 @@ class MemberAdaptor:
         be zero.  If member is not a member of the list, raise
         NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getDeliveryStatusMembers(self,
                                  status=(UNKNOWN, BYUSER, BYADMIN, BYBOUNCE)):
@@ -195,7 +195,7 @@ class MemberAdaptor:
         of ENABLED, UNKNOWN, BYUSER, BYADMIN, or BYBOUNCE.  The members whose
         delivery status is in this sequence are returned.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getBouncingMembers(self):
         """Return the list of members who have outstanding bounce information.
@@ -204,7 +204,7 @@ class MemberAdaptor:
         getDeliveryStatusMembers() since getBouncingMembers() will return
         member who have bounced but not yet reached the disable threshold.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def getBounceInfo(self, member):
         """Return the member's bounce information.
@@ -217,7 +217,7 @@ class MemberAdaptor:
 
         If member is not a member of the list, raise NotAMemberError.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
 
     #
@@ -245,14 +245,14 @@ class MemberAdaptor:
         Raise AlreadyAMemberError it the member is already subscribed to the
         list.  Raises ValueError if **kws contains an invalid option.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def removeMember(self, memberkey):
         """Unsubscribes the member from the mailing list.
 
         Raise NotAMemberError if member is not subscribed to the list.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def changeMemberAddress(self, memberkey, newaddress, nodelete=0):
         """Change the address for the member KEY.
@@ -267,7 +267,7 @@ class MemberAdaptor:
 
         If nodelete flag is true, then the old membership is not removed.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setMemberPassword(self, member, password):
         """Set the password for member LCE/KEY.
@@ -275,9 +275,8 @@ class MemberAdaptor:
         If member does not refer to a valid member, raise NotAMemberError.
         Also raise BadPasswordError if the password is illegal (e.g. too
         short or easily guessed via a dictionary attack).
-
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setMemberLanguage(self, member, language):
         """Set the language for the member LCE/KEY.
@@ -286,7 +285,7 @@ class MemberAdaptor:
         Also raise BadLanguageError if the language is invalid (e.g. the list
         is not configured to support the given language).
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setMemberOption(self, member, flag, value):
         """Set the option for the given member to value.
@@ -298,7 +297,7 @@ class MemberAdaptor:
         Also raise BadOptionError if the flag does not refer to a valid
         option.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setMemberName(self, member, realname):
         """Set the member's full name.
@@ -307,7 +306,7 @@ class MemberAdaptor:
         be a Unicode string if there are non-ASCII characters in the name.
         NotAMemberError is raised if member does not refer to a valid member.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setMemberTopics(self, member, topics):
         """Add list of topics to member's interest.
@@ -316,7 +315,7 @@ class MemberAdaptor:
         NotAMemberError is raised if member does not refer to a valid member.
         topics must be a sequence of strings.
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setDeliveryStatus(self, member, status):
         """Set the delivery status of the member's address.
@@ -337,7 +336,7 @@ class MemberAdaptor:
         ENABLED, then the change time information will be deleted.  This value
         is retrievable via getDeliveryStatusChangeTime().
         """
-        raise NotImplemented
+        raise NotImplementedError
 
     def setBounceInfo(self, member, info):
         """Set the member's bounce information.
@@ -347,4 +346,4 @@ class MemberAdaptor:
         Bounce info is opaque to the MemberAdaptor.  It is set by this method
         and returned by getBounceInfo() without modification.
         """
-        raise NotImplemented
+        raise NotImplementedError
diff --git a/Mailman/Message.py b/Mailman/Message.py
index b82ddf81..e8dd5953 100644
--- a/Mailman/Message.py
+++ b/Mailman/Message.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Standard Mailman message object.
@@ -20,6 +20,7 @@ This is a subclass of mimeo.Message but provides a slightly extended interface
 which is more convenient for use inside Mailman.
 """
 
+import re
 import email
 import email.Message
 import email.Utils
@@ -33,7 +34,8 @@ from Mailman import Utils
 
 COMMASPACE = ', '
 
-VERSION = tuple([int(s) for s in email.__version__.split('.')])
+mo = re.match(r'([\d.]+)', email.__version__)
+VERSION = tuple([int(s) for s in mo.group().split('.')])
 
 
 
@@ -200,7 +202,8 @@ class UserNotification(Message):
             self.set_payload(text, charset)
         if subject is None:
             subject = '(no subject)'
-        self['Subject'] = Header(subject, charset, header_name='Subject')
+        self['Subject'] = Header(subject, charset, header_name='Subject',
+                                 errors='replace')
         self['From'] = sender
         if isinstance(recip, ListType):
             self['To'] = COMMASPACE.join(recip)
diff --git a/Mailman/OldStyleMemberships.py b/Mailman/OldStyleMemberships.py
index 8ade3565..44adc9bf 100644
--- a/Mailman/OldStyleMemberships.py
+++ b/Mailman/OldStyleMemberships.py
@@ -113,8 +113,11 @@ class OldStyleMemberships(MemberAdaptor.MemberAdaptor):
             raise Errors.NotAMemberError, member
 
     def getMemberLanguage(self, member):
-        return self.__mlist.language.get(member.lower(),
-                                         self.__mlist.preferred_language)
+        lang = self.__mlist.language.get(
+            member.lower(), self.__mlist.preferred_language)
+        if lang in self.__mlist.GetAvailableLanguages():
+            return lang
+        return self.__mlist.preferred_language
 
     def getMemberOption(self, member, flag):
         self.__assertIsMember(member)
@@ -168,7 +171,7 @@ class OldStyleMemberships(MemberAdaptor.MemberAdaptor):
     def addNewMember(self, member, **kws):
         assert self.__mlist.Locked()
         # Make sure this address isn't already a member
-        if self.__mlist.isMember(member):
+        if self.isMember(member):
             raise Errors.MMAlreadyAMember, member
         # Parse the keywords
         digest = 0
@@ -335,8 +338,8 @@ class OldStyleMemberships(MemberAdaptor.MemberAdaptor):
         self.__assertIsMember(member)
         member = member.lower()
         if status == MemberAdaptor.ENABLED:
+            # Enable by resetting their bounce info.
             self.setBounceInfo(member, None)
-            # Otherwise, nothing to do
         else:
             self.__mlist.delivery_status[member] = (status, time.time())
 
diff --git a/Mailman/Pending.py b/Mailman/Pending.py
index be1c6cac..0d6986cf 100644
--- a/Mailman/Pending.py
+++ b/Mailman/Pending.py
@@ -1,17 +1,17 @@
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """ Track pending confirmation of subscriptions.
@@ -56,29 +56,48 @@ def new(*content):
     # It's a programming error if this assertion fails!  We do it this way so
     # the assert test won't fail if the sequence is empty.
     assert content[:1] in _ALLKEYS
-    # Acquire the pending database lock, letting TimeOutError percolate up.
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
+
+    # Get a lock handle now, but only lock inside the loop.
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    # We try the main loop several times. If we get a lock error somewhere
+    # (for instance because someone broke the lock) we simply try again.
+    retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS
     try:
-        # Load the current database
-        db = _load()
-        # Calculate a unique cookie
-        while 1:
-            n = random.random()
-            now = time.time()
-            hashfood = str(now) + str(n) + str(content)
-            cookie = sha.new(hashfood).hexdigest()
-            if not db.has_key(cookie):
-                break
-        # Store the content, plus the time in the future when this entry will
-        # be evicted from the database, due to staleness.
-        db[cookie] = content
-        evictions = db.setdefault('evictions', {})
-        evictions[cookie] = now + mm_cfg.PENDING_REQUEST_LIFE
-        _save(db)
-        return cookie
+        while retries:
+            retries -= 1
+            if not lock.locked():
+                try:
+                    lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
+                except LockFile.TimeOutError:
+                    continue
+            # Load the current database
+            db = _load()
+            # Calculate a unique cookie
+            while 1:
+                n = random.random()
+                now = time.time()
+                hashfood = str(now) + str(n) + str(content)
+                cookie = sha.new(hashfood).hexdigest()
+                if not db.has_key(cookie):
+                    break
+            # Store the content, plus the time in the future when this entry
+            # will be evicted from the database, due to staleness.
+            db[cookie] = content
+            evictions = db.setdefault('evictions', {})
+            evictions[cookie] = now + mm_cfg.PENDING_REQUEST_LIFE
+            try:
+                _save(db, lock)
+            except LockFile.NotLockedError:
+                continue
+            return cookie
+        else:
+            # We failed to get the lock or keep it long enough to save the
+            # data!
+            raise LockFile.TimeOutError
     finally:
-        lock.unlock()
+        if lock.locked():
+            lock.unlock()
 
 
 
@@ -88,30 +107,53 @@ def confirm(cookie, expunge=1):
     If optional expunge is true (the default), the record is also removed from
     the database.
     """
-    # Acquire the pending database lock, letting TimeOutError percolate up.
-    # BAW: we perhaps shouldn't acquire the lock if expunge==0.
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
-    try:
-        # Load the database
+    if not expunge:
         db = _load()
         missing = []
         content = db.get(cookie, missing)
         if content is missing:
             return None
-        # Remove the entry from the database
-        if expunge:
+        return content
+
+    # Get a lock handle now, but only lock inside the loop.
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    # We try the main loop several times. If we get a lock error somewhere
+    # (for instance because someone broke the lock) we simply try again.
+    retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS
+    try:
+        while retries:
+            retries -= 1
+            if not lock.locked():
+               try:
+                   lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
+               except LockFile.TimeOutError:
+                   continue
+            # Load the database
+            db = _load()
+            missing = []
+            content = db.get(cookie, missing)
+            if content is missing:
+                return None
             del db[cookie]
             del db['evictions'][cookie]
-            _save(db)
-        return content
+            try:
+                _save(db, lock)
+            except LockFile.NotLockedError:
+                continue
+            return content
+        else:
+            # We failed to get the lock and keep it long enough to save the
+            # data!
+            raise LockFile.TimeOutError
     finally:
-        lock.unlock()
+        if lock.locked():
+            lock.unlock()
 
 
 
 def _load():
-    # The list's lock must be acquired.
+    # The list's lock must be acquired if you wish to alter data and save.
     #
     # First try to load the pickle file
     fp = None
@@ -134,8 +176,10 @@ def _load():
             fp.close()
 
 
-def _save(db):
-    # Lock must be acquired.
+def _save(db, lock):
+    # Lock must be acquired before loading the data that is now being saved.
+    if not lock.locked():
+        raise LockFile.NotLockedError
     evictions = db['evictions']
     now = time.time()
     for cookie, data in db.items():
@@ -154,13 +198,17 @@ def _save(db):
     omask = os.umask(007)
     # Always save this as a pickle (safely), and after that succeeds, blow
     # away any old marshal file.
-    tmpfile = PCKFILE + '.tmp'
+    tmpfile = '%s.tmp.%d.%d' % (PCKFILE, os.getpid(), now)
     fp = None
     try:
         fp = open(tmpfile, 'w')
         cPickle.dump(db, fp)
         fp.close()
         fp = None
+        if not lock.locked():
+            # Our lock was broken?
+            os.remove(tmpfile)
+            raise LockFile.NotLockedError
         os.rename(tmpfile, PCKFILE)
         if os.path.exists(DBFILE):
             os.remove(DBFILE)
@@ -173,8 +221,9 @@ def _save(db):
 
 def _update(olddb):
     # Update an old pending_subscriptions.db database to the new format
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
     try:
         # We don't need this entry anymore
         if olddb.has_key('lastculltime'):
@@ -199,6 +248,7 @@ def _update(olddb):
             # request was made.  The new format keeps it as the time the
             # request should be evicted.
             evictions[cookie] = data[-1] + mm_cfg.PENDING_REQUEST_LIFE
-        _save(db)
+        _save(db, lock)
     finally:
-        lock.unlock()
+        if lock.locked():
+            lock.unlock()
diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py
index 785511b3..5bc1599b 100644
--- a/Mailman/Queue/CommandRunner.py
+++ b/Mailman/Queue/CommandRunner.py
@@ -140,13 +140,25 @@ Attached is your original message.
         if unprocessed:
             resp.append(_('\n- Unprocessed:'))
             resp.extend(indent(unprocessed))
+        if not unprocessed and not self.results:
+            # The user sent an empty message; return a helpful one.
+            resp.append(Utils.wrap(_("""\
+No commands were found in this message.
+To obtain instructions, send a message containing just the word "help".
+""")))
         if self.ignored:
             resp.append(_('\n- Ignored:'))
             resp.extend(indent(self.ignored))
         resp.append(_('\n- Done.\n\n'))
-        results = MIMEText(
-            NL.join(resp),
-            _charset=Utils.GetCharSet(self.mlist.preferred_language))
+        # Encode any unicode strings into the list charset, so we don't try to
+        # join unicode strings and invalid ASCII.
+        charset = Utils.GetCharSet(self.mlist.preferred_language)
+        encoded_resp = []
+        for item in resp:
+            if isinstance(item, UnicodeType):
+                item = item.encode(charset, 'replace')
+            encoded_resp.append(item)
+        results = MIMEText(NL.join(encoded_resp), _charset=charset)
         # Safety valve for mail loops with misconfigured email 'bots.  We
         # don't respond to commands sent with "Precedence: bulk|junk|list"
         # unless they explicitly "X-Ack: yes", but not all mail 'bots are
diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py
index aed8dcb9..11c94dfe 100644
--- a/Mailman/Queue/OutgoingRunner.py
+++ b/Mailman/Queue/OutgoingRunner.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -16,8 +16,9 @@
 
 """Outgoing queue runner."""
 
-import sys
 import os
+import sys
+import copy
 import time
 import socket
 
@@ -31,9 +32,15 @@ from Mailman.Queue.Runner import Runner
 from Mailman.Logging.Syslog import syslog
 
 # This controls how often _doperiodic() will try to deal with deferred
-# permanent failures.
+# permanent failures.  It is a count of calls to _doperiodic()
 DEAL_WITH_PERMFAILURES_EVERY = 1
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 class OutgoingRunner(Runner):
@@ -51,9 +58,13 @@ class OutgoingRunner(Runner):
         # This prevents smtp server connection problems from filling up the
         # error log.  It gets reset if the message was successfully sent, and
         # set if there was a socket.error.
-        self.__logged = 0
+        self.__logged = False
 
     def _dispose(self, mlist, msg, msgdata):
+        # See if we should retry delivery of this message again.
+        deliver_after = msgdata.get('deliver_after', 0)
+        if time.time() < deliver_after:
+            return True
         # Make sure we have the most up-to-date state
         mlist.Load()
         try:
@@ -63,7 +74,7 @@ class OutgoingRunner(Runner):
             if pid <> os.getpid():
                 syslog('error', 'child process leaked thru: %s', modname)
                 os._exit(1)
-            self.__logged = 0
+            self.__logged = False
         except socket.error:
             # There was a problem connecting to the SMTP server.  Log this
             # once, but crank up our sleep time so we don't fill the error
@@ -75,8 +86,8 @@ class OutgoingRunner(Runner):
             if not self.__logged:
                 syslog('error', 'Cannot connect to SMTP server %s on port %s',
                        mm_cfg.SMTPHOST, port)
-                self.__logged = 1
-            return 1
+                self.__logged = True
+            return True
         except Errors.SomeRecipientsFailed, e:
             # The delivery module being used (SMTPDirect or Sendmail) failed
             # to deliver the message to one or all of the recipients.
@@ -88,14 +99,14 @@ class OutgoingRunner(Runner):
             # handling.  I'm not sure this is necessary, or the right thing to
             # do.
             pcnt = len(e.permfailures)
-            copy = email.message_from_string(str(msg))
+            msgcopy = copy.deepcopy(msg)
             self._permfailures.setdefault(mlist, []).extend(
-                zip(e.permfailures, [copy] * pcnt))
+                zip(e.permfailures, [msgcopy] * pcnt))
             # Temporary failures
             if not e.tempfailures:
                 # Don't need to keep the message queued if there were only
                 # permanent failures.
-                return 0
+                return False
             now = time.time()
             recips = e.tempfailures
             last_recip_count = msgdata.get('last_recip_count', 0)
@@ -104,17 +115,18 @@ class OutgoingRunner(Runner):
                 # We didn't make any progress, so don't attempt delivery any
                 # longer.  BAW: is this the best disposition?
                 if now > deliver_until:
-                    return 0
+                    return False
             else:
-                # Keep trying to delivery this for 3 days
+                # Keep trying to delivery this message for a while
                 deliver_until = now + mm_cfg.DELIVERY_RETRY_PERIOD
             msgdata['last_recip_count'] = len(recips)
             msgdata['deliver_until'] = deliver_until
+            msgdata['deliver_after'] = now + mm_cfg.DELIVERY_RETRY_WAIT
             msgdata['recips'] = recips
             # Requeue
-            return 1
+            return True
         # We've successfully completed handling of this message
-        return 0
+        return False
 
     def _doperiodic(self):
         # Periodically try to acquire the list lock and clear out the
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 92262684..57c87c36 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -192,7 +192,7 @@ def LCDomain(addr):
 
 
 # TBD: what other characters should be disallowed?
-_badchars = re.compile('[][()<>|;^,/]')
+_badchars = re.compile(r'[][()<>|;^,/\200-\377]')
 
 def ValidateEmail(s):
     """Verify that the an email address isn't grossly evil."""
diff --git a/Mailman/i18n.py b/Mailman/i18n.py
index d38eba85..c5853438 100644
--- a/Mailman/i18n.py
+++ b/Mailman/i18n.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
 import sys
 import time
 import gettext
-from types import StringType
+from types import StringType, UnicodeType
 
 from Mailman import mm_cfg
 from Mailman.SafeDict import SafeDict
@@ -74,8 +74,19 @@ def _(s):
     # missing key in the dictionary.
     dict = SafeDict(frame.f_globals.copy())
     dict.update(frame.f_locals)
-    # Translate the string, then interpolate into it.
-    return _translation.gettext(s) % dict
+    # Translating the string returns an encoded 8-bit string.  Rather than
+    # turn that into a Unicode, we turn any Unicodes in the dictionary values
+    # into encoded 8-bit strings.  BAW: Returning a Unicode here broke too
+    # much other stuff and _() has many tentacles.  Eventually I think we want
+    # to use Unicode everywhere.
+    tns = _translation.gettext(s)
+    charset = _translation.charset()
+    if not charset:
+        charset = 'us-ascii'
+    for k, v in dict.items():
+        if isinstance(v, UnicodeType):
+            dict[k] = v.encode(charset, 'replace')
+    return tns % dict