Merged the cull_bad_shunt branch.

8 files changed, 166 insertions, 2 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 8f8c64f0..1631a714 100644
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -725,6 +725,26 @@ QRUNNER_SLEEP_TIME = seconds(1)
 # them in qfiles/bad subdirectory.
 QRUNNER_SAVE_BAD_MESSAGES = Yes
 
+# Depending on the above setting, the queue entries with messages which can't
+# be parsed may be saved in qfiles/bad.  Certain other exceptions which occur
+# during unpickling of a queue entry also cause the entry to be saved in
+# qfiles/bad.  Various exceptions which occur during message processing cause
+# the message to be shunted (saved in qfiles/shunt) where they can be
+# reprocessed with bin/unshunt after the underlying problem is fixed.  The
+# cull_bad_shunt cron job normally runs daily to remove and possibly archive
+# stale entries in qfiles/bad and qfiles/shunt.  The following settings
+# control this.
+
+# The length of time after which a qfiles/bad or qfiles/shunt file is
+# considered to be stale.  Set to zero to disable culling of qfiles/bad and
+# qfiles/shunt entries.
+BAD_SHUNT_STALE_AFTER = days(7)
+
+# The pathname of a directory (searchable and writable by the Mailman cron
+# user) to which the culled qfiles/bad and qfiles/shunt entries will be
+# moved.  Set to None to simply delete the culled entries.
+BAD_SHUNT_ARCHIVE_DIRECTORY = None
+
 # This flag causes Mailman to fsync() its data files after writing and
 # flushing its contents.  While this ensures the data is written to disk,
 # avoiding data loss, it may be a performance killer.  Note that this flag
diff --git a/NEWS b/NEWS
index 298352c1..c7e435e3 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,15 @@ Here is a history of user visible changes to Mailman.
 
 2.x.xx (xx-xxx-xxxx)
 
+  New Features
+
+    - Added a new cron/cull_bad_shunt script to cull and optionally
+      archive old entries from the bad and shunt queues. This is controlled
+      by new Defaults.py/mm_cfg.py settings BAD_SHUNT_STALE_AFTER (default
+      7 days) and BAD_SHUNT_ARCHIVE_DIRECTORY (default None) which determine
+      how long to keep bad and shunt queue entries and optionally, where to
+      archive removed entries.
+
   Bug fixes and other patches
 
     - Changed the preservation of unparseable messages to be conditional on
@@ -23,6 +32,9 @@ Here is a history of user visible changes to Mailman.
     - Changed Utils.ValidateEmail to not allow specials (particularly ':')
       in unquoted local parts (1956393).
 
+    - Changed bin/update to remove .bak files erroneously left behind in
+      qfiles/*/ by a 2.1.9 bug.
+
   Miscellaneous
 
     - Brad Knowles' mailman daily status report script updated to 0.0.18.
diff --git a/bin/update b/bin/update
index ac83094a..bb4682d3 100755
--- a/bin/update
+++ b/bin/update
@@ -442,6 +442,10 @@ def update_qfiles():
             for filename in os.listdir(dirpath):
                 filepath = os.path.join(dirpath, filename)
                 filebase, ext = os.path.splitext(filepath)
+                # A bug in Mailman 2.1.9 left .bak files behind in some
+                # circumstances. It should be safe to remove them.
+                if ext == '.bak':
+                    os.remove(filepath)
                 # Handle the .db metadata files as part of the handling of the
                 # .pck or .msg message files.
                 if ext not in ('.pck', '.msg'):
diff --git a/configure b/configure
index 3f05770b..c5831a96 100755
--- a/configure
+++ b/configure
@@ -4250,6 +4250,7 @@ build/contrib/qmail-to-mailman.py:contrib/qmail-to-mailman.py \
 build/contrib/rotatelogs.py:contrib/rotatelogs.py \
 build/cron/bumpdigests:cron/bumpdigests \
 build/cron/checkdbs:cron/checkdbs \
+build/cron/cull_bad_shunt:cron/cull_bad_shunt \
 build/cron/disabled:cron/disabled \
 build/cron/gate_news:cron/gate_news \
 build/cron/mailpasswds:cron/mailpasswds \
diff --git a/configure.in b/configure.in
index b97df6d2..a3aabfdf 100644
--- a/configure.in
+++ b/configure.in
@@ -593,6 +593,7 @@ contrib/qmail-to-mailman.py \
 contrib/rotatelogs.py \
 cron/bumpdigests \
 cron/checkdbs \
+cron/cull_bad_shunt \
 cron/disabled \
 cron/gate_news \
 cron/mailpasswds \
diff --git a/cron/Makefile.in b/cron/Makefile.in
index afb03370..2a7c8fd2 100644
--- a/cron/Makefile.in
+++ b/cron/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -42,7 +42,7 @@ CRONDIR= 	$(prefix)/cron
 SHELL=		/bin/sh
 
 PROGRAMS=	checkdbs mailpasswds senddigests gate_news \
-		nightly_gzip bumpdigests disabled
+		nightly_gzip bumpdigests disabled cull_bad_shunt
 FILES=  	crontab.in
 
 BUILDDIR=	../build/cron
diff --git a/cron/crontab.in.in b/cron/crontab.in.in
index 49f27c72..540dfc1d 100644
--- a/cron/crontab.in.in
+++ b/cron/crontab.in.in
@@ -22,3 +22,6 @@
 # turn this on if the internal archiver is used and
 # GZIP_ARCHIVE_TXT_FILES is false in mm_cfg.py
 27 3 * * * @PYTHON@ -S @prefix@/cron/nightly_gzip
+#
+# At 4:30AM daily, cull old entries from the 'bad' and 'shunt' queues.
+30 4 * * * @PYTHON@ -S @prefix@/cron/cull_bad_shunt
diff --git a/cron/cull_bad_shunt b/cron/cull_bad_shunt
new file mode 100755
index 00000000..ce0dfb0f
--- /dev/null
+++ b/cron/cull_bad_shunt
@@ -0,0 +1,123 @@
+#! @PYTHON@
+#
+# Copyright (C) 2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Cull bad and shunt queues, recommended once per day.
+
+This script goes through the 'bad' and 'shunt' queue directories and,
+if mm_cfg.BAD_SHUNT_STALE_AFTER is > 0, it removes all files more than
+that many seconds old.
+
+If mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY is a writable directory, the old
+files are moved there. Otherwise they are deleted.
+
+Only regular files immediately subordinate to the 'bad' and 'shunt'
+directories are processed. Anything else is skipped.
+
+Usage: %(PROGRAM)s [options]
+
+Options:
+    -h / --help
+        Print this message and exit.
+"""
+
+import os
+import sys
+import stat
+import time
+import errno
+import getopt
+
+import paths
+# mm_cfg must be imported before the other modules, due to the side-effect of
+# it hacking sys.paths to include site-packages.  Without this, running this
+# script from cron with python -S will fail.
+from Mailman import mm_cfg
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(
+            sys.argv[1:], 'h', ['help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    if args:
+        usage(1)
+
+    if mm_cfg.BAD_SHUNT_STALE_AFTER <= 0:
+        # Nothing to do
+        return
+    now = time.time()
+    old = now - float(mm_cfg.BAD_SHUNT_STALE_AFTER)
+    os.stat_float_times(True)
+    adir = mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY
+    if (adir and os.access(adir, os.W_OK | os.X_OK)
+             and stat.S_ISDIR(os.stat(adir).st_mode)):
+        pass
+    else:
+        if adir:
+            print >>sys.stderr, '%s: archive directory %s not usable.' % (
+                    PROGRAM, adir)
+        adir = None
+    for qdir in (mm_cfg.BADQUEUE_DIR, mm_cfg.SHUNTQUEUE_DIR):
+        try:
+            names = os.listdir(qdir)
+        except OSError, e:
+            if e.errno <> errno.ENOENT:
+                # OK if qdir doesn't exist, else
+                raise
+            continue
+        for name in names:
+            pathname = os.path.join(qdir, name)
+            pstat = os.stat(pathname)
+            if not stat.S_ISREG(pstat.st_mode):
+                # Not a regular file
+                continue
+            if pstat.st_mtime > old:
+                # File is new
+                continue
+            if adir:
+                os.rename(pathname, os.path.join(adir, name))
+            else:
+                os.remove(pathname)
+
+
+
+if __name__ == '__main__':
+    main()