diff options
| -rw-r--r-- | Mailman/Defaults.py.in | 20 | ||||
| -rw-r--r-- | NEWS | 12 | ||||
| -rwxr-xr-x | bin/update | 4 | ||||
| -rwxr-xr-x | configure | 1 | ||||
| -rw-r--r-- | configure.in | 1 | ||||
| -rw-r--r-- | cron/Makefile.in | 4 | ||||
| -rw-r--r-- | cron/crontab.in.in | 3 | ||||
| -rwxr-xr-x | cron/cull_bad_shunt | 123 |
8 files changed, 166 insertions, 2 deletions
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 8f8c64f0..1631a714 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -725,6 +725,26 @@ QRUNNER_SLEEP_TIME = seconds(1) # them in qfiles/bad subdirectory. QRUNNER_SAVE_BAD_MESSAGES = Yes +# Depending on the above setting, the queue entries with messages which can't +# be parsed may be saved in qfiles/bad. Certain other exceptions which occur +# during unpickling of a queue entry also cause the entry to be saved in +# qfiles/bad. Various exceptions which occur during message processing cause +# the message to be shunted (saved in qfiles/shunt) where they can be +# reprocessed with bin/unshunt after the underlying problem is fixed. The +# cull_bad_shunt cron job normally runs daily to remove and possibly archive +# stale entries in qfiles/bad and qfiles/shunt. The following settings +# control this. + +# The length of time after which a qfiles/bad or qfiles/shunt file is +# considered to be stale. Set to zero to disable culling of qfiles/bad and +# qfiles/shunt entries. +BAD_SHUNT_STALE_AFTER = days(7) + +# The pathname of a directory (searchable and writable by the Mailman cron +# user) to which the culled qfiles/bad and qfiles/shunt entries will be +# moved. Set to None to simply delete the culled entries. +BAD_SHUNT_ARCHIVE_DIRECTORY = None + # This flag causes Mailman to fsync() its data files after writing and # flushing its contents. While this ensures the data is written to disk, # avoiding data loss, it may be a performance killer. Note that this flag @@ -6,6 +6,15 @@ Here is a history of user visible changes to Mailman. 2.x.xx (xx-xxx-xxxx) + New Features + + - Added a new cron/cull_bad_shunt script to cull and optionally + archive old entries from the bad and shunt queues. This is controlled + by new Defaults.py/mm_cfg.py settings BAD_SHUNT_STALE_AFTER (default + 7 days) and BAD_SHUNT_ARCHIVE_DIRECTORY (default None) which determine + how long to keep bad and shunt queue entries and optionally, where to + archive removed entries. + Bug fixes and other patches - Changed the preservation of unparseable messages to be conditional on @@ -23,6 +32,9 @@ Here is a history of user visible changes to Mailman. - Changed Utils.ValidateEmail to not allow specials (particularly ':') in unquoted local parts (1956393). + - Changed bin/update to remove .bak files erroneously left behind in + qfiles/*/ by a 2.1.9 bug. + Miscellaneous - Brad Knowles' mailman daily status report script updated to 0.0.18. @@ -442,6 +442,10 @@ def update_qfiles(): for filename in os.listdir(dirpath): filepath = os.path.join(dirpath, filename) filebase, ext = os.path.splitext(filepath) + # A bug in Mailman 2.1.9 left .bak files behind in some + # circumstances. It should be safe to remove them. + if ext == '.bak': + os.remove(filepath) # Handle the .db metadata files as part of the handling of the # .pck or .msg message files. if ext not in ('.pck', '.msg'): @@ -4250,6 +4250,7 @@ build/contrib/qmail-to-mailman.py:contrib/qmail-to-mailman.py \ build/contrib/rotatelogs.py:contrib/rotatelogs.py \ build/cron/bumpdigests:cron/bumpdigests \ build/cron/checkdbs:cron/checkdbs \ +build/cron/cull_bad_shunt:cron/cull_bad_shunt \ build/cron/disabled:cron/disabled \ build/cron/gate_news:cron/gate_news \ build/cron/mailpasswds:cron/mailpasswds \ diff --git a/configure.in b/configure.in index b97df6d2..a3aabfdf 100644 --- a/configure.in +++ b/configure.in @@ -593,6 +593,7 @@ contrib/qmail-to-mailman.py \ contrib/rotatelogs.py \ cron/bumpdigests \ cron/checkdbs \ +cron/cull_bad_shunt \ cron/disabled \ cron/gate_news \ cron/mailpasswds \ diff --git a/cron/Makefile.in b/cron/Makefile.in index afb03370..2a7c8fd2 100644 --- a/cron/Makefile.in +++ b/cron/Makefile.in @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -42,7 +42,7 @@ CRONDIR= $(prefix)/cron SHELL= /bin/sh PROGRAMS= checkdbs mailpasswds senddigests gate_news \ - nightly_gzip bumpdigests disabled + nightly_gzip bumpdigests disabled cull_bad_shunt FILES= crontab.in BUILDDIR= ../build/cron diff --git a/cron/crontab.in.in b/cron/crontab.in.in index 49f27c72..540dfc1d 100644 --- a/cron/crontab.in.in +++ b/cron/crontab.in.in @@ -22,3 +22,6 @@ # turn this on if the internal archiver is used and # GZIP_ARCHIVE_TXT_FILES is false in mm_cfg.py 27 3 * * * @PYTHON@ -S @prefix@/cron/nightly_gzip +# +# At 4:30AM daily, cull old entries from the 'bad' and 'shunt' queues. +30 4 * * * @PYTHON@ -S @prefix@/cron/cull_bad_shunt diff --git a/cron/cull_bad_shunt b/cron/cull_bad_shunt new file mode 100755 index 00000000..ce0dfb0f --- /dev/null +++ b/cron/cull_bad_shunt @@ -0,0 +1,123 @@ +#! @PYTHON@ +# +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Cull bad and shunt queues, recommended once per day. + +This script goes through the 'bad' and 'shunt' queue directories and, +if mm_cfg.BAD_SHUNT_STALE_AFTER is > 0, it removes all files more than +that many seconds old. + +If mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY is a writable directory, the old +files are moved there. Otherwise they are deleted. + +Only regular files immediately subordinate to the 'bad' and 'shunt' +directories are processed. Anything else is skipped. + +Usage: %(PROGRAM)s [options] + +Options: + -h / --help + Print this message and exit. +""" + +import os +import sys +import stat +import time +import errno +import getopt + +import paths +# mm_cfg must be imported before the other modules, due to the side-effect of +# it hacking sys.paths to include site-packages. Without this, running this +# script from cron with python -S will fail. +from Mailman import mm_cfg +from Mailman.i18n import _ + +# Work around known problems with some RedHat cron daemons +import signal +signal.signal(signal.SIGCHLD, signal.SIG_DFL) + +PROGRAM = sys.argv[0] + + + +def usage(code, msg=''): + if code: + fd = sys.stderr + else: + fd = sys.stdout + print >> fd, _(__doc__) + if msg: + print >> fd, msg + sys.exit(code) + + + +def main(): + try: + opts, args = getopt.getopt( + sys.argv[1:], 'h', ['help']) + except getopt.error, msg: + usage(1, msg) + + if args: + usage(1) + + if mm_cfg.BAD_SHUNT_STALE_AFTER <= 0: + # Nothing to do + return + now = time.time() + old = now - float(mm_cfg.BAD_SHUNT_STALE_AFTER) + os.stat_float_times(True) + adir = mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY + if (adir and os.access(adir, os.W_OK | os.X_OK) + and stat.S_ISDIR(os.stat(adir).st_mode)): + pass + else: + if adir: + print >>sys.stderr, '%s: archive directory %s not usable.' % ( + PROGRAM, adir) + adir = None + for qdir in (mm_cfg.BADQUEUE_DIR, mm_cfg.SHUNTQUEUE_DIR): + try: + names = os.listdir(qdir) + except OSError, e: + if e.errno <> errno.ENOENT: + # OK if qdir doesn't exist, else + raise + continue + for name in names: + pathname = os.path.join(qdir, name) + pstat = os.stat(pathname) + if not stat.S_ISREG(pstat.st_mode): + # Not a regular file + continue + if pstat.st_mtime > old: + # File is new + continue + if adir: + os.rename(pathname, os.path.join(adir, name)) + else: + os.remove(pathname) + + + +if __name__ == '__main__': + main() |