#!/bin/sh
###############################################################################
# mmdsr -- Mailman Daily Status Report (cron job, run at 23:59)
###############################################################################
# Copyright (c) 2005, Brad Knowles
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#
# The name of the author or other contributors may not be used
# to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
###############################################################################
# Version history
###############################################################################
# 0.0.1 Initial creation by Brad Knowles <brad@stop.mail-abuse.org>
# Created on: Tue Feb 15 04:01:20 PST 2005
#
# 0.0.2 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Wed Feb 16 18:55:52 CET 2005
# Feedback from Tom G. Christensen (tgc99):
# > The current UID grab command doesn't work on Solaris
# > (2.6 & 8 tested).
# >
# > I'd recommend this instead:
# > ps -o user -p $$|tail -1
# >
# > This is tested and works on RH 6.2, RH 7.3, RHEL 2.1,
# > RHEL3, FC3, FreeBSD 4.9, Solaris 2.6, 8.
#
# 0.0.3 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Sun Mar 13 01:15:24 CET 2005
# Noted errors from grep when trying to search a nonexistant
# file. Added "-s" option to compensate, and extra code to
# notify the admin if the log file doesn't exist.
#
# 0.0.4 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Sun Mar 13 01:58:37 CET 2005
# Eliminate some extra cruft from the "fromusenet" logs.
#
# 0.0.5 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Wed Apr 13 00:54:42 CEST 2005
# Eliminate a lot of extra cruft from the "error" & "vette" logs.
# Eliminate more cruft from the "fromusenet" logs.
#
# 0.0.6 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Thu Apr 14 03:07:08 CEST 2005
# Eliminate even more cruft from the "fromusenet" logs.
# Eliminate more cruft from the "vette" logs.
# Eliminate cruft from the "mischief" logs.
#
# 0.0.7 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Sun May 1 22:29:13 CEST 2005 [guess]
# The "error" log "no such list" errors didn't need to be
# quite so "compressed", open them up to be more readable.
# Also eliminate all use of "xargs", use "fmt -w 75" instead.
#
# 0.0.8 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Sun May 1 23:09:13 CEST 2005
# Clean up code for queue subdirectories to check.
#
# 0.0.9 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Wed May 11 01:23:40 CEST 2005
# Eliminate date/time stamps from the rest of the logs,
# then pipe through `sort | uniq -c | sort -nr`
# so that all possible duplicates are eliminated.
#
# 0.0.10 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Tue Jul 12 16:53:49 CDT 2005
# Add code to do summary of normal list activity
#
# 0.0.11 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Tue Sep 6 15:44:19 CEST 2005
# Tweaked display of Mailman/qfiles subdirectories
# so as to avoid displaying more than $MAX_LINES
# output, which really helps if you have thousands
# of bounces sitting around, etc....
# Also slightly tweaked display of fromusenet logs
# to split summary from errors.
#
# 0.0.12 Update by Brad Knowles <brad@stop.mail-abuse.org>
# Updated on: Thu Sep 22 22:37:35 CEST 2005
# Bugs found by Mark Sapiro and Adrian Wells.
# Mark suggested splitting the Mailman log directory
# from the rest of the Mailman source and queues,
# as well as making sure to clean up all temp files.
# Adrian discovered that there was a log file format
# change between Mailman 2.1.5 and 2.1.6, which broke
# hourly statistics.
###############################################################################
# Set up locations of standard commands, directories, etc....
# You may need to modify these for your platform or installation.
###############################################################################
AWK="/usr/bin/awk"
BASENAME="/usr/bin/basename"
CAT="/bin/cat"
DATE="/bin/date"
EGREP="/bin/egrep"
FMT="/usr/bin/fmt"
GREP="/bin/grep"
HEAD="/usr/bin/head"
LS="/bin/ls"
PS="/bin/ps"
RM="/bin/rm"
SED="/bin/sed"
SENDMAIL="/usr/sbin/sendmail"
SLEEP="/bin/sleep"
SORT="/usr/bin/sort"
TAIL="/usr/bin/tail"
TOUCH="/bin/touch"
TR="/usr/bin/tr"
UNIQ="/usr/bin/uniq"
XARGS="/usr/bin/xargs"
WC="/usr/bin/wc"
###############################################################################
# Directory for temporary files
###############################################################################
TMPDIR="/tmp"
###############################################################################
# Mailman queue directory
###############################################################################
QUEUEDIR="/usr/local/mailman/qfiles"
###############################################################################
# Mailman log directory
###############################################################################
LOGDIR="/usr/local/mailman/logs"
###############################################################################
# Maximum number of subdirectory entries to display in report
###############################################################################
MAX_LINES=20
###############################################################################
# Mailman Log files to check for errors.
# No need to specify path, only log file name.
###############################################################################
ERR_LOGS="error fromusenet locks mischief post qrunner smtp-failure vette"
###############################################################################
# Mailman Log files to summarize.
# No need to specify path, only log file name.
###############################################################################
SUM_LOGS="fromusenet post smtp"
###############################################################################
# Mailman queue subdirectories to check. No need to specify path, only name.
###############################################################################
SUBDIRS="archive bounces commands in news out retry shunt virgin"
###############################################################################
# Specify recipients for report. If none, then simply print to "STDOUT".
# Specify sender address for report. Not used if recipient list is empty.
###############################################################################
SENDER="put@your.address.here"
RCPTS="put@the.address.of,your@recipients.here"
###############################################################################
# If you run this program in cron at 23:59:00, you need to sleep sixty
# seconds to make sure that you capture all the logs for the previous day.
# Comment this line out to disable sleeping altogether.
###############################################################################
SLEEPTIME=60
###############################################################################
# What user id is Mailman installed under?
# How do we determine what UID this program is running as?
# NB: There will probably be SysV vs. BSD semantic differences here.
# Be careful. Make sure the command you specify actually works.
# The command string specified here may seem arcane, but should pull
# out the information we need, and throw away everything else. If
# there is an easier cross-platform way to do it, please let me know.
###############################################################################
MYUID=`$PS -o user -p $$ | $TAIL -1`
RUNAS="mailman"
###############################################################################
# No modifications below this line should be required.
###############################################################################
DAY=`$DATE '+%b %d'`
YEAR=`$DATE '+%Y'`
PROG=`$BASENAME $0`
if [ "${MYUID}" != "${RUNAS}" -a "${MYUID}" != "root" ] ; then
echo "ERROR: You must be root or ${RUNAS} to run $PROG."
exit 1
fi
if [ "${SLEEPTIME}" != "" -a "${RCPTS}x" != "${SENDER}x" ] ; then
if [ $SLEEPTIME -gt 0 ] ; then
$SLEEP $SLEEPTIME
fi
fi
TMP="$TMPDIR/.$PROG.$$"
TMPLOG="$TMP.log"
$RM -f $TMP
$TOUCH $TMP
if [ "${RCPTS}x" != "x" ] ; then
echo "From: ${SENDER}" >> $TMP
echo "To: ${RCPTS}" >> $TMP
echo "Subject: Mailman Daily Status Report: $DAY $YEAR" >> $TMP
echo "" >> $TMP
fi
echo " Mailman Daily Status Report: $DAY $YEAR" >> $TMP
echo "" >> $TMP
echo "" >> $TMP
echo "******************************" >> $TMP
echo "Log File Summary" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP
for LOG in $SUM_LOGS
do
$RM -f $TMPLOG
$TOUCH $TMPLOG
echo "Log file: $LOG" >> $TMP
echo "==============================" >> $TMP
$GREP -si "^$DAY [0-9][0-9:]* $YEAR" $LOGDIR/$LOG >> $TMPLOG
if [ -f "$LOGDIR/${LOG}" ] ; then
if [ "${LOG}" = "post" ] ; then
echo "" >> $TMP
echo "Hourly Summary of Posts" >> $TMP
echo "-----------------------" >> $TMP
$SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.*$//' $TMPLOG | $UNIQ -c | $SORT -n +1 | awk '{ printf( "%8d %02d:00-%02d:59\n", $1, $2, $2 ) }' >> $TMP
echo "" >> $TMP
echo "Post Count by List" >> $TMP
echo "------------------" >> $TMP
$SED -e 's/^.* post to //' -e 's/ .*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Post Count by Sender" >> $TMP
echo "--------------------" >> $TMP
$SED -e 's/^.* from //' -e 's/,.*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP
elif [ "${LOG}" = "fromusenet" ] ; then
GRPS=`$EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -i ' gating ' | $AWK '{ print $7 }' | $SORT -u | $FMT -w 75`
for GRP in $GRPS
do
echo "" >> $TMP
echo "$GRP Article #'s Gated:" >> $TMP
echo "------------------------------" >> $TMP
$EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -i " gating " | $GREP -i " $GRP " | $SED -e 's/^.*\[//' -e 's/\]$//' -e 's/\.\./ /' | $TR ' ' '\n' | $SORT -u | $FMT -w 75 >> $TMP
done
elif [ "${LOG}" = "smtp" ] ; then
echo "" >> $TMP
echo "Hourly Summary of Messages Sent" >> $TMP
echo "-------------------------------" >> $TMP
$SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.* for / /' -e 's/ recips,.*$//' $TMPLOG | awk '{ val=int($1); sum[val]+=$2 } END { for (i=0; i<24; i++) { printf "%8d %02d:00-%02d:59\n", sum[i], i, i } }' >> $TMP
else
$SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP
fi
else
echo " ### Log file ${LOG} does not exist ### " >> $TMP
fi
echo "" >> $TMP
done
echo "******************************" >> $TMP
echo "Log File Squawks" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP
for LOG in $ERR_LOGS
do
$RM -f $TMPLOG
$TOUCH $TMPLOG
echo "Log file: $LOG" >> $TMP
echo "==============================" >> $TMP
$GREP -si "^$DAY [0-9][0-9:]* $YEAR" $LOGDIR/$LOG >> $TMPLOG
if [ -f "$LOGDIR/${LOG}" ] ; then
if [ "${LOG}" = "error" ] ; then
echo "Uncaught Runner Exceptions:" >> $TMP
echo "------------------------------" >> $TMP
$GREP 'Uncaught runner exception' $TMPLOG | $SED 's/^.*exception: //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "No Such List:" >> $TMP
echo "------------------------------" >> $TMP
$GREP 'No such list' $TMPLOG | $SED -e 's/^.* "//' -e 's/".*$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
CNT=`$GREP -i 'shunting' $TMPLOG | $WC -l`
if [ "${CNT}x" != "x" ] ; then
if [ ${CNT} -gt 0 ] ; then
echo "" >> $TMP
echo "Shunting Count: $CNT" >> $TMP
echo "------------------------------" >> $TMP
fi
fi
echo "" >> $TMP
echo "Other Errors:" >> $TMP
echo "------------------------------" >> $TMP
$EGREP -vi '(Uncaught runner exception|No such list|Ignoring unparseable message|Traceback|shunting)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
elif [ "${LOG}" = "fromusenet" ] ; then
$EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -vi " gating " | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
elif [ "${LOG}" = "mischief" ] ; then
echo "" >> $TMP
echo "Login failure with private rosters (by user):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Login failure with private rosters' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Unsub attempt of non-member (by user):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Unsub attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Reminder attempt of non-member (by user):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Reminder attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Other Messages:" >> $TMP
echo "------------------------------" >> $TMP
$EGREP -vi '(Login failure with private rosters|Unsub attempt of non-member|Reminder attempt of non-member)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
elif [ "${LOG}" = "post" ] ; then
$GREP -vi 'success' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
elif [ "${LOG}" = "vette" ] ; then
echo "" >> $TMP
echo "Message held -- Post by non-member (by list):" >> $TMP
echo "------------------------------" >> $TMP
$GREP 'Post by non-member' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Message held -- Suspicious header (by list):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'suspicious header' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Discarded posting (by list):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Discarded posting' $TMPLOG | $AWK '{ print $6 }' | $SED 's/:$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Bulk/Junk message discarded (by list):" >> $TMP
echo "------------------------------" >> $TMP
$EGREP -i '(bulk|junk) message discarded' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Implicit destination (by list):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Message has implicit destination' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Post to moderated newsgroup (by list):" >> $TMP
echo "------------------------------" >> $TMP
$GREP -i 'Posting to a moderated newsgroup' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP
echo "Other Errors:" >> $TMP
echo "------------------------------" >> $TMP
$EGREP -vi '(Post by non-member|suspicious header|message approved|Discarded posting|bulk message discarded|junk message discarded|Message has implicit destination|Posting to a moderated newsgroup|Message discarded, msgid)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
else
$SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP
fi
else
echo " ### Log file ${LOG} does not exist ### " >> $TMP
fi
echo "" >> $TMP
done
echo "******************************" >> $TMP
echo "Queue Directory Contents" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP
cd $QUEUEDIR
for DIR in $SUBDIRS
do
$RM -f $TMPLOG
$TOUCH $TMPLOG
$LS -la $DIR >> $TMPLOG
LINES=`wc -l < $TMPLOG`
LINES=`expr $LINES - 3`
echo "Subdirectory: $DIR" >> $TMP
echo "==============================" >> $TMP
echo "Entries: $LINES" >> $TMP
echo "------------------------------" >> $TMP
if [ $LINES -le $MAX_LINES ] ; then
$CAT $TMPLOG >> $TMP
else
$HEAD $TMPLOG >> $TMP
echo "" >> $TMP
echo " More than $MAX_LINES total entries, skipping ..." >> $TMP
echo "" >> $TMP
$TAIL $TMPLOG >> $TMP
fi
echo "" >> $TMP
done
if [ "${RCPTS}x" != "x" ] ; then
$SENDMAIL -t -f$SENDER < $TMP
else
$CAT $TMP
fi
$RM $TMP $TMPLOG
