Backporting from the HEAD -- bin and cron scripts

9 files changed, 122 insertions, 35 deletions

diff --git a/bin/Makefile.in b/bin/Makefile.in
index bcbcb40c..091c2cc1 100644
--- a/bin/Makefile.in
+++ b/bin/Makefile.in
@@ -48,7 +48,7 @@ SCRIPTS=	mmsitepass newlist rmlist add_members \
 		version config_list list_lists dumpdb cleanarch \
 		list_admins genaliases change_pw mailmanctl qrunner inject \
 		unshunt fix_url.py convert.py transcheck b4b5-archfix \
-		list_owners msgfmt.py
+		list_owners msgfmt.py show_qfiles discard
 
 BUILDDIR=	../build/bin
 
diff --git a/bin/dumpdb b/bin/dumpdb
index be04385b..ec31cb0f 100644
--- a/bin/dumpdb
+++ b/bin/dumpdb
@@ -1,19 +1,19 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Dump the contents of any Mailman `database' file.
@@ -49,7 +49,7 @@ import sys
 import os
 import getopt
 import pprint
-import cPickle
+import cPickle as pickle
 
 import paths
 # Import this /after/ paths so that the sys.path is properly hacked
@@ -123,7 +123,7 @@ def main():
             pp.pprint(d)
         return d
     else:
-        m = cPickle.load(open(filename))
+        m = pickle.load(open(filename))
         if doprint:
             pp.pprint(m)
         return m
diff --git a/bin/list_members b/bin/list_members
index b4fdbb27..33eb8db2 100755
--- a/bin/list_members
+++ b/bin/list_members
@@ -44,11 +44,18 @@ Where:
     --fullnames / -f
         Include the full names in the output.
 
-    --preserve
-    -p
+    --preserve / -p
         Output member addresses case preserved the way they were added to the
         list.  Otherwise, addresses are printed in all lowercase.
 
+    --invalid / -i
+        Print only the addresses in the membership list that are invalid.
+        Ignores -r, -d, -n.
+
+    --unicode / -u
+        Print addresses which are stored as Unicode objects instead of normal
+        string objects.  Ignores -r, -d, -n.
+
     --help
     -h
         Print this help message and exit.
@@ -65,6 +72,7 @@ from types import UnicodeType
 
 import paths
 from Mailman import mm_cfg
+from Mailman import Utils
 from Mailman import MailList
 from Mailman import Errors
 from Mailman import MemberAdaptor
@@ -74,6 +82,14 @@ from email.Utils import formataddr
 
 PROGRAM = sys.argv[0]
 ENC = sys.getdefaultencoding()
+COMMASPACE = ', '
+
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 WHYCHOICES = {'enabled' : MemberAdaptor.ENABLED,
               'unknown' : MemberAdaptor.UNKNOWN,
@@ -103,6 +119,17 @@ def safe(s):
     return unicode(s, ENC, 'replace').encode(ENC, 'replace')
 
 
+def isinvalid(addr):
+    try:
+        Utils.ValidateEmail(addr)
+        return False
+    except Errors.EmailAddressError:
+        return True
+
+def isunicode(addr):
+    return isinstance(addr, UnicodeType)
+
+
 
 def whymatches(mlist, addr, why):
     # Return true if the `why' matches the reason the address is enabled, or
@@ -124,14 +151,16 @@ def main():
     nomail = None
     why = None
     kind = None
-    fullnames = 0
+    fullnames = False
+    invalidonly = False
+    unicodeonly = False
 
     # Throw away the first (program) argument
     args = sys.argv[1:]
     if not args:
         usage(0)
 
-    while 1:
+    while True:
         try:
             opt = args.pop(0)
         except IndexError:
@@ -139,38 +168,42 @@ def main():
         if opt in ('-h', '--help'):
             usage(0)
         elif opt in ('-f', '--fullnames'):
-            fullnames = 1
+            fullnames = True
         elif opt in ('-p', '--preserve'):
-            preserve = 1
+            preserve = True
         elif opt in ('-r', '--regular'):
-            regular = 1
+            regular = True
         elif opt in ('-o', '--output'):
             try:
                 outfile = args.pop(0)
             except IndexError:
                 usage(1)
         elif opt == '-n':
-            nomail = 1
+            nomail = True
             if args and args[0] in WHYCHOICES.keys():
                 why = args.pop(0)
         elif opt.startswith('--nomail'):
-            nomail = 1
+            nomail = True
             i = opt.find('=')
             if i >= 0:
                 why = opt[i+1:]
                 if why not in WHYCHOICES.keys():
                     usage(1, _('Bad --nomail option: %(why)s'))
         elif opt == '-d':
-            digest = 1
+            digest = True
             if args and args[0] in ('mime', 'plain'):
                 kind = args.pop(0)
         elif opt.startswith('--digest'):
-            digest = 1
+            digest = True
             i = opt.find('=')
             if i >= 0:
                 kind = opt[i+1:]
                 if kind not in ('mime', 'plain'):
                     usage(1, _('Bad --digest option: %(kind)s'))
+        elif opt in ('-i', '--invalid'):
+            invalidonly = True
+        elif opt in ('-u', '--unicode'):
+            unicodeonly = True
         else:
             # No more options left, push the last one back on the list
             args.insert(0, opt)
@@ -182,7 +215,7 @@ def main():
     listname = args[0].lower().strip()
 
     if regular is None and digest is None:
-        regular = digest = 1
+        regular = digest = True
 
     if outfile:
         try:
@@ -194,7 +227,7 @@ def main():
         fp = sys.stdout
 
     try:
-        mlist = MailList.MailList(listname, lock=0)
+        mlist = MailList.MailList(listname, lock=False)
     except Errors.MMListError, e:
         print >> sys.stderr, _('No such list: %(listname)s')
         sys.exit(1)
@@ -208,6 +241,19 @@ def main():
         rmembers = mlist.getMemberCPAddresses(rmembers)
         dmembers = mlist.getMemberCPAddresses(dmembers)
 
+    if invalidonly or unicodeonly:
+        all = rmembers + dmembers
+        all.sort()
+        for addr in all:
+            name = fullnames and mlist.getMemberName(addr) or ''
+            showit = False
+            if invalidonly and isinvalid(addr):
+                showit = True
+            if unicodeonly and isunicode(addr):
+                showit = True
+            if showit:
+                print >> fp, formataddr((safe(name), addr))
+        return
     if regular:
         rmembers.sort()
         for addr in rmembers:
diff --git a/bin/mailmanctl b/bin/mailmanctl
index 1243dbd4..4ebeb836 100644
--- a/bin/mailmanctl
+++ b/bin/mailmanctl
@@ -250,9 +250,10 @@ def start_runner(qrname, slice, count):
     #
     # Craft the command line arguments for the exec() call.
     rswitch = '--runner=%s:%d:%d' % (qrname, slice, count)
-    # BAW: should argv[0] be `python'?
     exe = os.path.join(mm_cfg.BIN_DIR, 'qrunner')
-    os.execl(mm_cfg.PYTHON, 'qrunner', exe, rswitch, '-s')
+    # mm_cfg.PYTHON, which is the absolute path to the Python interpreter,
+    # must be given as argv[0] due to Python's library search algorithm.
+    os.execl(mm_cfg.PYTHON, mm_cfg.PYTHON, exe, rswitch, '-s')
     # Should never get here
     raise RuntimeError, 'os.execl() failed'
 
diff --git a/bin/msgfmt.py b/bin/msgfmt.py
index 411e3aef..8a2d4e66 100755
--- a/bin/msgfmt.py
+++ b/bin/msgfmt.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-
+# -*- coding: iso-8859-1 -*-
 # Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
 
 """Generate binary message catalog from textual translation description.
@@ -82,8 +82,8 @@ def generate():
         koffsets += [l1, o1+keystart]
         voffsets += [l2, o2+valuestart]
     offsets = koffsets + voffsets
-    output = struct.pack("iiiiiii",
-                         0x950412de,        # Magic
+    output = struct.pack("Iiiiiii",
+                         0x950412deL,       # Magic
                          0,                 # Version
                          len(keys),         # # of entries
                          7*4,               # start of key index
diff --git a/bin/pygettext.py b/bin/pygettext.py
index d6e1b4f7..84421ee8 100755
--- a/bin/pygettext.py
+++ b/bin/pygettext.py
@@ -200,8 +200,8 @@ def make_escapes(pass_iso8859):
     global escapes
     if pass_iso8859:
         # Allow iso-8859 characters to pass through so that e.g. 'msgid
-        # "Höhe"' would result not result in 'msgid "H\366he"'.  Otherwise we
-        # escape any character outside the 32..126 range.
+        # "H[o-umlaut]he"' would result not result in 'msgid "H\366he"'.
+        # Otherwise we escape any character outside the 32..126 range.
         mod = 128
     else:
         mod = 256
diff --git a/cron/disabled b/cron/disabled
index dcf05f25..75972e55 100644
--- a/cron/disabled
+++ b/cron/disabled
@@ -1,19 +1,19 @@
 #! @PYTHON@
 #
-# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 """Process disabled members, recommended once per day.
@@ -73,6 +73,7 @@ from Mailman import Utils
 from Mailman import MailList
 from Mailman import Pending
 from Mailman import MemberAdaptor
+from Mailman import Errors
 from Mailman.Bouncer import _BounceInfo
 from Mailman.Logging.Syslog import syslog
 from Mailman.i18n import _
@@ -198,7 +199,16 @@ def main():
             for member in notify:
                 syslog('bounce', 'Notifying disabled member %s for list: %s',
                        member, mlist.internal_name())
-                mlist.sendNextNotification(member)
+                try:
+                    mlist.sendNextNotification(member)
+                except Errors.NotAMemberError:
+                    # There must have been some problem with the data we have
+                    # on this member.  Most likely it's that they don't have a
+                    # password assigned.  Log this and delete the member.
+                    syslog('bounce',
+                           'NotAMemberError when sending disabled notice: %s',
+                           member)
+                    mlist.ApprovedDeleteMember(member, 'cron/disabled')
             mlist.Save()
         finally:
             mlist.Unlock()
diff --git a/cron/gate_news b/cron/gate_news
index 3fe466d4..19ccb2c6 100755
--- a/cron/gate_news
+++ b/cron/gate_news
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -67,6 +67,13 @@ class _ContinueLoop(Exception):
     pass
 
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
+
 
 def usage(status, msg=''):
     if code:
@@ -83,12 +90,15 @@ def usage(status, msg=''):
 _hostcache = {}
 
 def open_newsgroup(mlist):
+    # Split host:port if given
+    nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
     # Open up a "mode reader" connection to nntp server.  This will be shared
     # for all the gated lists having the same nntp_host.
     conn = _hostcache.get(mlist.nntp_host)
     if conn is None:
         try:
-            conn = nntplib.NNTP(mlist.nntp_host, readermode=1,
+            conn = nntplib.NNTP(nntp_host, nntp_port,
+                                readermode=True,
                                 user=mm_cfg.NNTP_USERNAME,
                                 password=mm_cfg.NNTP_PASSWORD)
         except (socket.error, nntplib.NNTPError, IOError), e:
diff --git a/cron/mailpasswds b/cron/mailpasswds
index 348ca336..86f49d18 100755
--- a/cron/mailpasswds
+++ b/cron/mailpasswds
@@ -42,6 +42,7 @@ import sys
 import os
 import errno
 import getopt
+from types import UnicodeType
 
 import paths
 # mm_cfg must be imported before the other modules, due to the side-effect of
@@ -78,6 +79,13 @@ def usage(code, msg=''):
 
 
 
+def tounicode(s, enc):
+    if isinstance(s, UnicodeType):
+        return s
+    return unicode(s, enc, 'replace')
+
+
+
 def main():
     try:
         opts, args = getopt.getopt(sys.argv[1:], 'l:h',
@@ -142,7 +150,15 @@ def main():
                     continue
                 # Group by the lower-cased address, since Mailman always
                 # treates person@dom.ain the same as PERSON@dom.ain.
-                password = mlist.getMemberPassword(member)
+                try:
+                    password = mlist.getMemberPassword(member)
+                except Errors.NotAMemberError:
+                    # Here's a member with no passwords, which I think was
+                    # possible in older versions of Mailman.  Log this and
+                    # move on.
+                    syslog('error', 'password-less member %s for list %s',
+                           member, mlist.internal_name())
+                    continue
                 optionsurl = mlist.GetOptionsURL(member)
                 lang = mlist.getMemberLanguage(member)
                 info = (listaddr, password, optionsurl, lang)
@@ -176,6 +192,7 @@ def main():
                 if cnt > langcnt:
                     poplang = lang
                     langcnt = cnt
+            enc = Utils.GetCharSet(poplang)
             # Craft the table header
             header = '%-40s %-10s\n%-40s %-10s' % (
                 _('List'), _('Password // URL'), '----', '--------')
@@ -190,6 +207,9 @@ def main():
                  'exreq'   : sitereq,
                  'owner'   : siteowner,
                  }, lang=poplang)
+            # Coerce everything to Unicode
+            text = tounicode(text, enc)
+            table = [tounicode(_t, enc) for _t in table]
             # Add the table to the end so it doesn't get wrapped/filled
             text += (header + '\n' + NL.join(table))
             # Translate the message and headers to user's suggested lang