This commit was manufactured by cvs2svn to create branch

'Release_2_1-maint'.

10 files changed, 1282 insertions, 0 deletions

diff --git a/cron/.cvsignore b/cron/.cvsignore
new file mode 100644
index 00000000..d2278ab6
--- /dev/null
+++ b/cron/.cvsignore
@@ -0,0 +1,2 @@
+crontab.in
+Makefile
diff --git a/cron/Makefile.in b/cron/Makefile.in
new file mode 100644
index 00000000..2f596751
--- /dev/null
+++ b/cron/Makefile.in
@@ -0,0 +1,75 @@
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+# NOTE: Makefile.in is converted into Makefile by the configure script
+# in the parent directory.  Once configure has run, you can recreate
+# the Makefile by running just config.status.
+
+# Variables set by configure
+
+VPATH=		@srcdir@
+srcdir= 	@srcdir@
+bindir= 	@bindir@
+prefix=   	@prefix@
+exec_prefix=	@exec_prefix@
+
+CC=		@CC@
+CHMOD=  	@CHMOD@
+INSTALL=	@INSTALL@
+
+DEFS=   	@DEFS@
+
+# Customizable but not set by configure
+
+OPT=		@OPT@
+CFLAGS=		$(OPT) $(DEFS)
+CRONDIR= 	$(prefix)/cron
+
+SHELL=		/bin/sh
+
+PROGRAMS=	checkdbs mailpasswds senddigests gate_news \
+		nightly_gzip bumpdigests disabled
+FILES=  	crontab.in
+
+BUILDDIR=	../build/cron
+
+# Modes for directories and executables created by the install
+# process.  Default to group-writable directories but
+# user-only-writable for executables.
+EXEMODE=	755
+FILEMODE=	644
+
+
+# Rules
+
+all:
+
+install:
+	for f in $(FILES); \
+	do \
+	    $(INSTALL) -m $(FILEMODE) $$f $(CRONDIR); \
+	done
+	for f in $(PROGRAMS); \
+	do \
+	    $(INSTALL) -m $(EXEMODE) $(BUILDDIR)/$$f $(CRONDIR); \
+	done
+
+finish:
+
+clean:
+
+distclean:
+	-rm Makefile crontab.in
diff --git a/cron/bumpdigests b/cron/bumpdigests
new file mode 100644
index 00000000..3636fc6e
--- /dev/null
+++ b/cron/bumpdigests
@@ -0,0 +1,96 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Increment the digest volume number and reset the digest number to one.
+
+Usage: %(PROGRAM)s [options] [listname ...]
+
+Options:
+
+    --help/-h
+        Print this message and exit.
+
+The lists named on the command line are bumped.  If no list names are given,
+all lists are bumped.
+"""
+
+import sys
+import getopt
+
+import paths
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+from Mailman import Errors
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+
+    if args:
+        listnames = args
+    else:
+        listnames = Utils.list_names()
+
+    if not listnames:
+        print _('Nothing to do.')
+        sys.exit(0)
+
+    for listname in listnames:
+        try:
+            # be sure the list is locked
+            mlist = MailList.MailList(listname)
+        except Errors.MMListError, e:
+            usage(1, _('No such list: %(listname)s'))
+        try:
+            mlist.bump_digest_volume()
+        finally:
+            mlist.Save()
+            mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cron/checkdbs b/cron/checkdbs
new file mode 100755
index 00000000..46883cf0
--- /dev/null
+++ b/cron/checkdbs
@@ -0,0 +1,136 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Invoked by cron, this checks for pending moderation requests and mails the
+list moderators if necessary.
+"""
+
+import sys
+import time
+from types import UnicodeType
+
+import paths
+
+# Import this after paths so we get Mailman's copy of the email package
+from email.Charset import Charset
+
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+from Mailman import Message
+from Mailman import i18n
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+NL = '\n'
+
+_ = i18n._
+i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE)
+
+def _isunicode(s):
+    return isinstance(s, UnicodeType)
+
+
+
+def main():
+    for name in Utils.list_names():
+        # the list must be locked in order to open the requests database
+        mlist = MailList.MailList(name)
+        try:
+            count = mlist.NumRequestsPending()
+            # While we're at it, let's evict yesterday's autoresponse data
+            midnightToday = Utils.midnight()
+            evictions = []
+            for sender in mlist.hold_and_cmd_autoresponses.keys():
+                date, respcount = mlist.hold_and_cmd_autoresponses[sender]
+                if Utils.midnight(date) < midnightToday:
+                    evictions.append(sender)
+            if evictions:
+                for sender in evictions:
+                    del mlist.hold_and_cmd_autoresponses[sender]
+                # Only here have we changed the list's database
+                mlist.Save()
+            if count:
+                i18n.set_language(mlist.preferred_language)
+                realname = mlist.real_name
+                text = Utils.maketext(
+                    'checkdbs.txt',
+                    {'count'    : count,
+                     'host_name': mlist.host_name,
+                     'adminDB'  : mlist.GetScriptURL('admindb', absolute=1),
+                     'real_name': realname,
+                     }, mlist=mlist)
+                text += '\n' + pending_requests(mlist)
+                subject = _(
+                    '%(count)d %(realname)s moderator request(s) waiting')
+                msg = Message.UserNotification(mlist.GetOwnerEmail(),
+                                               mlist.GetBouncesEmail(),
+                                               subject, text,
+                                               mlist.preferred_language)
+                msg.send(mlist, **{'tomoderators': 1})
+        finally:
+            mlist.Unlock()
+
+
+
+
+def pending_requests(mlist):
+    # Must return a byte string
+    pending = []
+    first = 1
+    for id in mlist.GetSubscriptionIds():
+        if first:
+            pending.append(_('Pending subscriptions:'))
+            first = 0
+        when, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
+        if fullname:
+            fullname = ' (%s)' % fullname
+        pending.append('    %s%s %s' % (addr, fullname, time.ctime(when)))
+    first = 1
+    for id in mlist.GetHeldMessageIds():
+        if first:
+            pending.append(_('\nPending posts:'))
+            first = 0
+        info = mlist.GetRecord(id)
+        when, sender, subject, reason, text, msgdata = mlist.GetRecord(id)
+        date = time.ctime(when)
+        pending.append(_("""\
+From: %(sender)s on %(date)s
+Subject: %(subject)s
+Cause: %(reason)s"""))
+        pending.append('')
+    # Make sure that the text we return from here can be encoded to a byte
+    # string in the charset of the list's language.  This could fail if for
+    # example, the request was pended while the list's language was French,
+    # but then it was changed to English before checkdbs ran.
+    text = NL.join(pending)
+    charset = Charset(Utils.GetCharSet(mlist.preferred_language))
+    incodec = charset.input_codec or 'ascii'
+    outcodec = charset.output_codec or 'ascii'
+    if _isunicode(text):
+        return text.encode(outcodec, 'replace')
+    # Be sure this is a byte string encodeable in the list's charset
+    utext = unicode(text, incodec, 'replace')
+    return utext.encode(outcodec, 'replace')
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cron/crontab.in.in b/cron/crontab.in.in
new file mode 100755
index 00000000..49f27c72
--- /dev/null
+++ b/cron/crontab.in.in
@@ -0,0 +1,24 @@
+# At 8AM every day, mail reminders to admins as to pending requests.
+# They are less likely to ignore these reminders if they're mailed
+# early in the morning, but of course, this is local time... ;)
+0 8 * * * @PYTHON@ -S @prefix@/cron/checkdbs
+#
+# At 9AM, send notifications to disabled members that are due to be
+# reminded to re-enable their accounts.
+0 9 * * * @PYTHON@ -S @prefix@/cron/disabled
+#
+# Noon, mail digests for lists that do periodic as well as threshhold delivery.
+0 12 * * * @PYTHON@ -S @prefix@/cron/senddigests
+#
+# 5 AM on the first of each month, mail out password reminders.
+0 5 1 * * @PYTHON@ -S @prefix@/cron/mailpasswds
+#
+# Every 5 mins, try to gate news to mail.  You can comment this one out
+# if you don't want to allow gating, or don't have any going on right now,
+# or want to exclusively use a callback strategy instead of polling.
+0,5,10,15,20,25,30,35,40,45,50,55 * * * * @PYTHON@ -S @prefix@/cron/gate_news
+#
+# At 3:27am every night, regenerate the gzip'd archive file.  Only
+# turn this on if the internal archiver is used and
+# GZIP_ARCHIVE_TXT_FILES is false in mm_cfg.py
+27 3 * * * @PYTHON@ -S @prefix@/cron/nightly_gzip
diff --git a/cron/disabled b/cron/disabled
new file mode 100644
index 00000000..dcf05f25
--- /dev/null
+++ b/cron/disabled
@@ -0,0 +1,209 @@
+#! @PYTHON@
+#
+# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Process disabled members, recommended once per day.
+
+This script cruises through every mailing list looking for members whose
+delivery is disabled.  If they have been disabled due to bounces, they will
+receive another notification, or they may be removed if they've received the
+maximum number of notifications.
+
+Use the --byadmin, --byuser, and --unknown flags to also send notifications to
+members whose accounts have been disabled for those reasons.  Use --all to
+send the notification to all disabled members.
+
+Usage: %(PROGRAM)s [options]
+
+Options:
+    -h / --help
+        Print this message and exit.
+
+    -o / --byadmin
+        Also send notifications to any member disabled by the list
+        owner/administrator.
+
+    -m / --byuser
+        Also send notifications to any member disabled by themselves.
+
+    -u / --unknown
+        Also send notifications to any member disabled for unknown reasons
+        (usually a legacy disabled address).
+
+    -b / --notbybounce
+        Don't send notifications to members disabled because of bounces (the
+        default is to notify bounce disabled members).
+
+    -a / --all
+        Send notifications to all disabled members.
+
+    -f / --force
+        Send notifications to disabled members even if they're not due a new
+        notification yet.
+
+    -l listname
+    --listname=listname
+        Process only the given list, otherwise do all lists.
+"""
+
+import sys
+import time
+import getopt
+
+import paths
+# mm_cfg must be imported before the other modules, due to the side-effect of
+# it hacking sys.paths to include site-packages.  Without this, running this
+# script from cron with python -S will fail.
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+from Mailman import Pending
+from Mailman import MemberAdaptor
+from Mailman.Bouncer import _BounceInfo
+from Mailman.Logging.Syslog import syslog
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(
+            sys.argv[1:], 'hl:omubaf',
+            ['byadmin', 'byuser', 'unknown', 'notbybounce', 'all',
+             'listname=', 'help', 'force'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    if args:
+        usage(1)
+
+    force = 0
+    listnames = []
+    who = [MemberAdaptor.BYBOUNCE]
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-l', '--list'):
+            listnames.append(arg)
+        elif opt in ('-o', '--byadmin'):
+            who.append(MemberAdaptor.BYADMIN)
+        elif opt in ('-m', '--byuser'):
+            who.append(MemberAdaptor.BYUSER)
+        elif opt in ('-u', '--unknown'):
+            who.append(MemberAdaptor.UNKNOWN)
+        elif opt in ('-b', '--notbybounce'):
+            try:
+                who.remove(MemberAdaptor.BYBOUNCE)
+            except ValueError:
+                # Already removed
+                pass
+        elif opt in ('-a', '--all'):
+            who = [MemberAdaptor.BYBOUNCE, MemberAdaptor.BYADMIN,
+                   MemberAdaptor.BYUSER, MemberAdaptor.UNKNOWN]
+        elif opt in ('-f', '--force'):
+            force = 1
+
+    who = tuple(who)
+
+    if not listnames:
+        listnames = Utils.list_names()
+
+    msg = _('[disabled by periodic sweep and cull, no message available]')
+    today = time.mktime(time.localtime()[:3] + (0,) * 6)
+    for listname in listnames:
+        # List of members to notify
+        notify = []
+        mlist = MailList.MailList(listname)
+        try:
+            interval = mlist.bounce_you_are_disabled_warnings_interval
+            # Find all the members who are currently bouncing and see if
+            # they've reached the disable threshold but haven't yet been
+            # disabled.  This is a sweep through the membership catching
+            # situations where they've bounced a bunch, then the list admin
+            # lowered the threshold, but we haven't (yet) seen more bounces
+            # from the member.  Note: we won't worry about stale information
+            # or anything else since the normal bounce processing code will
+            # handle that.
+            disables = []
+            for member in mlist.getBouncingMembers():
+                if mlist.getDeliveryStatus(member) <> MemberAdaptor.ENABLED:
+                    continue
+                info = mlist.getBounceInfo(member)
+                if info.score >= mlist.bounce_score_threshold:
+                    disables.append((member, info))
+            if disables:
+                for member, info in disables:
+                    mlist.disableBouncingMember(member, info, msg)
+            # Go through all the members who have delivery disabled, and find
+            # those that are due to have another notification.  If they are
+            # disabled for another reason than bouncing, and we're processing
+            # them (because of the command line switch) then they won't have a
+            # bounce info record.  We can piggyback on that for all disable
+            # purposes.
+            members = mlist.getDeliveryStatusMembers(who)
+            for member in members:
+                info = mlist.getBounceInfo(member)
+                if not info:
+                    # See if they are bounce disabled, or disabled for some
+                    # other reason.
+                    status = mlist.getDeliveryStatus(member)
+                    if status == MemberAdaptor.BYBOUNCE:
+                        syslog(
+                            'error',
+                            '%s disabled BYBOUNCE lacks bounce info, list: %s',
+                            member, mlist.internal_name())
+                        continue
+                    info = _BounceInfo(
+                        member, 0, today,
+                        mlist.bounce_you_are_disabled_warnings,
+                        Pending.new(Pending.RE_ENABLE, mlist.internal_name(),
+                                    member))
+                    mlist.setBounceInfo(member, info)
+                lastnotice = time.mktime(info.lastnotice + (0,) * 6)
+                if force or today >= lastnotice + interval:
+                    notify.append(member)
+            # Now, send notifications to anyone who is due
+            for member in notify:
+                syslog('bounce', 'Notifying disabled member %s for list: %s',
+                       member, mlist.internal_name())
+                mlist.sendNextNotification(member)
+            mlist.Save()
+        finally:
+            mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cron/gate_news b/cron/gate_news
new file mode 100755
index 00000000..3fe466d4
--- /dev/null
+++ b/cron/gate_news
@@ -0,0 +1,274 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Poll the NNTP servers for messages to be gatewayed to mailing lists.
+
+Usage: gate_news [options]
+
+Where options are
+
+    --help
+    -h
+        Print this text and exit.
+
+"""
+
+import sys
+import os
+import time
+import getopt
+import socket
+import nntplib
+
+import paths
+# Import this /after/ paths so that the sys.path is properly hacked
+import email.Errors
+from email.Parser import Parser
+
+from Mailman import mm_cfg
+from Mailman import MailList
+from Mailman import Utils
+from Mailman import Message
+from Mailman import LockFile
+from Mailman.i18n import _
+from Mailman.Queue.sbcache import get_switchboard
+from Mailman.Logging.Utils import LogStdErr
+from Mailman.Logging.Syslog import syslog
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+GATENEWS_LOCK_FILE = os.path.join(mm_cfg.LOCK_DIR, 'gate_news.lock')
+
+LogStdErr('error', 'gate_news', manual_reprime=0)
+
+LOCK_LIFETIME = mm_cfg.hours(2)
+NL = '\n'
+
+# Continues inside try: block are not allowed in Python versions before 2.1.
+# This exception is used to work around that.
+class _ContinueLoop(Exception):
+    pass
+
+
+
+def usage(status, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+_hostcache = {}
+
+def open_newsgroup(mlist):
+    # Open up a "mode reader" connection to nntp server.  This will be shared
+    # for all the gated lists having the same nntp_host.
+    conn = _hostcache.get(mlist.nntp_host)
+    if conn is None:
+        try:
+            conn = nntplib.NNTP(mlist.nntp_host, readermode=1,
+                                user=mm_cfg.NNTP_USERNAME,
+                                password=mm_cfg.NNTP_PASSWORD)
+        except (socket.error, nntplib.NNTPError, IOError), e:
+            syslog('fromusenet',
+                   'error opening connection to nntp_host: %s\n%s',
+                   mlist.nntp_host, e)
+            raise
+        _hostcache[mlist.nntp_host] = conn
+    # Get the GROUP information for the list, but we're only really interested
+    # in the first article number and the last article number
+    r,c,f,l,n = conn.group(mlist.linked_newsgroup)
+    return conn, int(f), int(l)
+
+
+def clearcache():
+    reverse = {}
+    for conn in _hostcache.values():
+        reverse[conn] = 1
+    for conn in reverse.keys():
+        conn.quit()
+    _hostcache.clear()
+
+
+
+# This function requires the list to be locked.
+def poll_newsgroup(mlist, conn, first, last, glock):
+    listname = mlist.internal_name()
+    # NEWNEWS is not portable and has synchronization issues.
+    for num in range(first, last):
+        glock.refresh()
+        try:
+            headers = conn.head(`num`)[3]
+            found_to = 0
+            beenthere = 0
+            for header in headers:
+                i = header.find(':')
+                value = header[:i].lower()
+                if i > 0 and value == 'to':
+                    found_to = 1
+                if value <> 'x-beenthere':
+                    continue
+                if header[i:] == ': %s' % mlist.GetListEmail():
+                    beenthere = 1
+                    break
+            if not beenthere:
+                body = conn.body(`num`)[3]
+                # Usenet originated messages will not have a Unix envelope
+                # (i.e. "From " header).  This breaks Pipermail archiving, so
+                # we will synthesize one.  Be sure to use the format searched
+                # for by mailbox.UnixMailbox._isrealfromline().  BAW: We use
+                # the -bounces address here in case any downstream clients use
+                # the envelope sender for bounces; I'm not sure about this,
+                # but it's the closest to the old semantics.
+                lines = ['From %s  %s' % (mlist.GetBouncesEmail(),
+                                          time.ctime(time.time()))]
+                lines.extend(headers)
+                lines.append('')
+                lines.extend(body)
+                lines.append('')
+                p = Parser(Message.Message)
+                try:
+                    msg = p.parsestr(NL.join(lines))
+                except email.Errors.MessageError, e:
+                    syslog('fromusenet',
+                           'email package exception for %s:%d\n%s',
+                           mlist.linked_newsgroup, num, e)
+                    raise _ContinueLoop
+                if found_to:
+                    del msg['X-Originally-To']
+                    msg['X-Originally-To'] = msg['To']
+                    del msg['To']
+                msg['To'] = mlist.GetListEmail()
+                # Post the message to the locked list
+                inq = get_switchboard(mm_cfg.INQUEUE_DIR)
+                inq.enqueue(msg,
+                            listname = mlist.internal_name(),
+                            fromusenet = 1)
+                syslog('fromusenet',
+                       'posted to list %s: %7d' % (listname, num))
+        except nntplib.NNTPError, e:
+            syslog('fromusenet',
+                   'NNTP error for list %s: %7d' % (listname, num))
+            syslog('fromusenet', str(e))
+        except _ContinueLoop:
+            continue
+        # Even if we don't post the message because it was seen on the
+        # list already, update the watermark
+        mlist.usenet_watermark = num
+
+
+
+def process_lists(glock):
+    for listname in Utils.list_names():
+        glock.refresh()
+        # Open the list unlocked just to check to see if it is gating news to
+        # mail.  If not, we're done with the list.  Otherwise, lock the list
+        # and gate the group.
+        mlist = MailList.MailList(listname, lock=0)
+        if not mlist.gateway_to_mail:
+            continue
+        # Get the list's watermark, i.e. the last article number that we gated
+        # from news to mail.  `None' means that this list has never polled its
+        # newsgroup and that we should do a catch up.
+        watermark = getattr(mlist, 'usenet_watermark', None)
+        # Open the newsgroup, but let most exceptions percolate up.
+        try:
+            conn, first, last = open_newsgroup(mlist)
+        except (socket.error, nntplib.NNTPError):
+            break
+        syslog('fromusenet', '%s: [%d..%d]' % (listname, first, last))
+        try:
+            try:
+                if watermark is None:
+                    mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
+                    # This is the first time we've tried to gate this
+                    # newsgroup.  We essentially do a mass catch-up, otherwise
+                    # we'd flood the mailing list.
+                    mlist.usenet_watermark = last
+                    syslog('fromusenet', '%s caught up to article %d' %
+                           (listname, last))
+                else:
+                    # The list has been polled previously, so now we simply
+                    # grab all the messages on the newsgroup that have not
+                    # been seen by the mailing list.  The first such article
+                    # is the maximum of the lowest article available in the
+                    # newsgroup and the watermark.  It's possible that some
+                    # articles have been expired since the last time gate_news
+                    # has run.  Not much we can do about that.
+                    start = max(watermark+1, first)
+                    if start > last:
+                        syslog('fromusenet', 'nothing new for list %s' %
+                               listname)
+                    else:
+                        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
+                        syslog('fromusenet', 'gating %s articles [%d..%d]' %
+                               (listname, start, last))
+                        # Use last+1 because poll_newsgroup() employes a for
+                        # loop over range, and this will not include the last
+                        # element in the list.
+                        poll_newsgroup(mlist, conn, start, last+1, glock)
+            except LockFile.TimeOutError:
+                syslog('fromusenet', 'Could not acquire list lock: %s' %
+                       listname)
+        finally:
+            if mlist.Locked():
+                mlist.Save()
+                mlist.Unlock()
+        syslog('fromusenet', '%s watermark: %d' %
+               (listname, mlist.usenet_watermark))
+
+
+
+def main():
+    lock = LockFile.LockFile(GATENEWS_LOCK_FILE,
+                             # it's okay to hijack this
+                             lifetime=LOCK_LIFETIME)
+    try:
+        lock.lock(timeout=0.5)
+    except LockFile.TimeOutError:
+        syslog('fromusenet', 'Could not acquire gate_news lock')
+        return
+    try:
+        process_lists(lock)
+    finally:
+        clearcache()
+        lock.unlock(unconditionally=1)
+
+
+
+if __name__ == '__main__':
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    if args:
+        usage(1, 'No args are expected')
+
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+
+    main()
diff --git a/cron/mailpasswds b/cron/mailpasswds
new file mode 100755
index 00000000..a009e92b
--- /dev/null
+++ b/cron/mailpasswds
@@ -0,0 +1,216 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Send password reminders for all lists to all users.
+
+This program scans all mailing lists and collects users and their passwords,
+grouped by the list's host_name if mm_cfg.VIRTUAL_HOST_OVERVIEW is true.  Then
+one email message is sent to each unique user (per-virtual host) containing
+the list passwords and options url for the user.  The password reminder comes
+from the mm_cfg.MAILMAN_SITE_LIST, which must exist.
+
+Usage: %(PROGRAM)s [options]
+
+Options:
+    -l listname
+    --listname=listname
+        Send password reminders for the named list only.  If omitted,
+        reminders are sent for all lists.  Multiple -l/--listname options are
+        allowed.
+
+    -h/--help
+        Print this message and exit.
+"""
+
+# This puppy should probably do lots of logging.
+import sys
+import os
+import errno
+import getopt
+
+import paths
+# mm_cfg must be imported before the other modules, due to the side-effect of
+# it hacking sys.paths to include site-packages.  Without this, running this
+# script from cron with python -S will fail.
+from Mailman import mm_cfg
+from Mailman import MailList
+from Mailman import Errors
+from Mailman import Utils
+from Mailman import Message
+from Mailman import i18n
+from Mailman.Logging.Syslog import syslog
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+NL = '\n'
+PROGRAM = sys.argv[0]
+
+_ = i18n._
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'l:h',
+                                   ['listname=', 'help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    if args:
+        usage(1)
+
+    listnames = None
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        if opt in ('-l', '--listname'):
+            if listnames is None:
+                listnames = [arg]
+            else:
+                listnames.append(arg)
+
+    if listnames is None:
+        listnames = Utils.list_names()
+
+    # This is the list that all the reminders will look like they come from,
+    # but with the host name coerced to the virtual host we're processing.
+    try:
+        sitelist = MailList.MailList(mm_cfg.MAILMAN_SITE_LIST, lock=0)
+    except Errors.MMUnknownListError:
+        # Do it this way for I18n's _()
+        sitelistname = mm_cfg.MAILMAN_SITE_LIST
+        print >> sys.stderr, _('Site list is missing: %(sitelistname)s')
+        syslog('error', 'Site list is missing: %s', mm_cfg.MAILMAN_SITE_LIST)
+        sys.exit(1)
+
+    # Group lists by host_name if VIRTUAL_HOST_OVERVIEW is true, otherwise
+    # there's only one key in this dictionary: mm_cfg.DEFAULT_EMAIL_HOST.  The
+    # values are lists of the unlocked MailList instances.
+    byhost = {}
+    for listname in listnames:
+        mlist = MailList.MailList(listname, lock=0)
+        if not mlist.send_reminders:
+            continue
+        if mm_cfg.VIRTUAL_HOST_OVERVIEW:
+            host = mlist.host_name
+        else:
+            # See the note in Defaults.py concerning DEFAULT_HOST_NAME
+            # vs. DEFAULT_EMAIL_HOST.
+            host = mm_cfg.DEFAULT_HOST_NAME or mm_cfg.DEFAULT_EMAIL_HOST
+        byhost.setdefault(host, []).append(mlist)
+
+    # Now for each virtual host, collate the user information.  Each user
+    # entry has the form (listaddr, password, optionsurl)
+    for host in byhost.keys():
+        # Site owner is `mailman@dom.ain'
+        userinfo = {}
+        for mlist in byhost[host]:
+            listaddr = mlist.GetListEmail()
+            for member in mlist.getMembers():
+                # BAW: we group by cpaddress because although it's highly
+                # likely, there's no guarantee that person@list1 is the same
+                # as PERSON@list2.  Sigh.
+                cpaddress = mlist.getMemberCPAddress(member)
+                password = mlist.getMemberPassword(member)
+                optionsurl = mlist.GetOptionsURL(member)
+                lang = mlist.getMemberLanguage(member)
+                info = (listaddr, password, optionsurl, lang)
+                userinfo.setdefault(cpaddress, []).append(info)
+        # Now that we've collected user information for this host, send each
+        # user the password reminder.
+        for addr in userinfo.keys():
+            # If the person is on more than one list, it is possible that they
+            # have different preferred languages, and there's no good way to
+            # know which one they want their password reminder in.  Pick the
+            # most popular, and break the tie randomly.
+            #
+            # Also, we need an example -request address for cronpass.txt and
+            # again, there's no clear winner.  Just take the first one in this
+            # case.
+            table = []
+            langs = {}
+            for listaddr, password, optionsurl, lang in userinfo[addr]:
+                langs[lang] = langs.get(lang, 0) + 1
+                # If the list address is really long, break it across two
+                # lines.
+                if len(listaddr) > 39:
+                    fmt = '%s\n           %-10s\n%s\n'
+                else:
+                    fmt = '%-40s %-10s\n%s\n'
+                table.append(fmt % (listaddr, password, optionsurl))
+            # Figure out which language to use
+            langcnt = 0
+            poplang = None
+            for lang, cnt in langs.items():
+                if cnt > langcnt:
+                    poplang = lang
+                    langcnt = cnt
+            # Craft the table header
+            header = '%-40s %-10s\n%-40s %-10s' % (
+                _('List'), _('Password // URL'), '----', '--------')
+            # Now we're finally ready to send the email!
+            siteowner = Utils.get_site_email(host, 'owner')
+            sitereq = Utils.get_site_email(host, 'request')
+            sitebounce = Utils.get_site_email(host, 'bounces')
+            text = Utils.maketext(
+                'cronpass.txt',
+                {'hostname': host,
+                 'useraddr': addr,
+                 'exreq'   : sitereq,
+                 'owner'   : siteowner,
+                 }, lang=poplang)
+            # Add the table to the end so it doesn't get wrapped/filled
+            text += (header + '\n' + NL.join(table))
+            # Translate the message and headers to user's suggested lang
+            otrans = i18n.get_translation()
+            try:
+                i18n.set_language(poplang)
+                msg = Message.UserNotification(
+                    addr, siteowner,
+                    _('%(host)s mailing list memberships reminder'),
+                    text, poplang)
+            finally:
+                i18n.set_translation(otrans)
+            msg['X-No-Archive'] = 'yes'
+            # We want to make this look like it's coming from the siteowner's
+            # list, but we also want to be sure that the apparent host name is
+            # the current virtual host.  Look in CookHeaders.py for why this
+            # trick works.  Blarg.
+            msg.send(sitelist, **{'errorsto': sitebounce,
+                                  '_nolist' : 1,
+                                  'verp'    : mm_cfg.VERP_PASSWORD_REMINDERS,
+                                  })
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cron/nightly_gzip b/cron/nightly_gzip
new file mode 100644
index 00000000..61b64184
--- /dev/null
+++ b/cron/nightly_gzip
@@ -0,0 +1,156 @@
+#! @PYTHON@
+# 
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+"""Re-generate the Pipermail gzip'd archive flat files.
+
+This script should be run nightly from cron.  When run from the command line,
+the following usage is understood:
+
+Usage: %(program)s [-v] [-h] [listnames]
+
+Where:
+    --verbose
+    -v
+        print each file as it's being gzip'd
+
+    --help
+    -h
+        print this message and exit
+
+    listnames
+        Optionally, only compress the .txt files for the named lists.  Without 
+        this, all archivable lists are processed.
+
+"""
+
+import sys
+import os
+import time
+from stat import *
+import getopt
+
+try:
+    import gzip
+except ImportError:
+    gzip = None
+
+import paths
+# mm_cfg must be imported before the other modules, due to the side-effect of
+# it hacking sys.paths to include site-packages.  Without this, running this
+# script from cron with python -S will fail.
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+
+
+
+program = sys.argv[0]
+VERBOSE = 0
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__) % globals()
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def compress(txtfile):
+    if VERBOSE:
+        print "gzip'ing:", txtfile
+    infp = open(txtfile)
+    outfp = gzip.open(txtfile+'.gz', 'wb', 6)
+    outfp.write(infp.read())
+    outfp.close()
+    infp.close()
+
+
+
+def main():
+    global VERBOSE
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'vh', ['verbose', 'help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    # defaults
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-v', '--verbose'):
+            VERBOSE = 1
+
+    # limit to the specified lists?
+    if args:
+        listnames = args
+    else:
+        listnames = Utils.list_names()
+
+    # process all the specified lists
+    for name in listnames:
+        mlist = MailList.MailList(name, lock=0)
+        if not mlist.archive:
+            continue
+        dir = mlist.archive_dir()
+        try:
+            allfiles = os.listdir(dir)
+        except os.error:
+            # has the list received any messages?  if not, last_post_time will
+            # be zero, so it's not really a bogus archive dir.
+            if mlist.last_post_time > 0:
+                print 'List', name, 'has a bogus archive_directory:', dir
+            continue
+        if VERBOSE:
+            print 'Processing list:', name
+        files = []
+        for f in allfiles:
+            if f[-4:] <> '.txt':
+                continue
+            # stat both the .txt and .txt.gz files and append them only if 
+            # the former is newer than the latter.
+            txtfile = os.path.join(dir, f)
+            gzpfile = txtfile + '.gz'
+            txt_mtime = os.stat(txtfile)[ST_MTIME]
+            try:
+                gzp_mtime = os.stat(gzpfile)[ST_MTIME]
+            except os.error:
+                gzp_mtime = -1
+            if txt_mtime > gzp_mtime:
+                files.append(txtfile)
+        for f in files:
+            compress(f)
+
+        
+
+if __name__ == '__main__' and \
+   gzip is not None and \
+   mm_cfg.ARCHIVE_TO_MBOX in (1, 2) and \
+   not mm_cfg.GZIP_ARCHIVE_TXT_FILES:
+    # we're only going to run the nightly archiver if messages are archived to
+    # the mbox, and the gzip file is not created on demand (i.e. for every
+    # individual post).  This is the normal mode of operation.  Also, be sure
+    # we can actually import the gzip module!
+    omask = os.umask(002)
+    try:
+        main()
+    finally:
+        os.umask(omask)
diff --git a/cron/senddigests b/cron/senddigests
new file mode 100755
index 00000000..5f03606b
--- /dev/null
+++ b/cron/senddigests
@@ -0,0 +1,94 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software 
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Dispatch digests for lists w/pending messages and digest_send_periodic set.
+
+Usage: %(PROGRAM)s [options]
+
+Options:
+    -h / --help
+        Print this message and exit.
+
+    -l listname
+    --listname=listname
+        Send the digest for the given list only, otherwise the digests for all
+        lists are sent out.
+"""
+
+import sys
+import getopt
+
+import paths
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'hl:', ['help', 'listname='])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    if args:
+        usage(1)
+
+    listnames = []
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-l', '--listname'):
+            listnames.append(arg)
+
+    if not listnames:
+        listnames = Utils.list_names()
+
+    for listname in listnames:
+        mlist = MailList.MailList(listname, lock=0)
+        if mlist.digest_send_periodic:
+            mlist.Lock()
+            try:
+                mlist.send_digest_now()
+                mlist.Save()
+            finally:
+                mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()