path: root/contrib/mmdsr

#!/bin/sh
###############################################################################
# mmdsr -- Mailman Daily Status Report (cron job, run at 23:59)
###############################################################################
# Copyright (c) 2005-2008 Brad Knowles
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 
#     Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 
#     Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer
#     in the documentation and/or other materials provided with the
#     distribution.
# 
#     The name of the author or other contributors may not be used
#     to endorse or promote products derived from this software without
#     specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################

###############################################################################
# Version history
###############################################################################
# 0.0.1		Initial creation by Brad Knowles <brad@shub-internet.org>
#		Created on: Tue Feb 15 04:01:20 PST 2005
#
# 0.0.2         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Wed Feb 16 18:55:52 CET 2005
#               Feedback from Tom G. Christensen (tgc99):
#               > The current UID grab command doesn't work on Solaris 
#               > (2.6 & 8 tested).      
#               >
#               > I'd recommend this instead:
#               > ps -o user -p $$|tail -1
#               >
#               > This is tested and works on RH 6.2, RH 7.3, RHEL 2.1,
#               > RHEL3, FC3, FreeBSD 4.9, Solaris 2.6, 8.
#
# 0.0.3         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Sun Mar 13 01:15:24 CET 2005
#               Noted errors from grep when trying to search a nonexistant
#               file.  Added "-s" option to compensate, and extra code to
#		notify the admin if the log file doesn't exist.
#
# 0.0.4         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Sun Mar 13 01:58:37 CET 2005
#               Eliminate some extra cruft from the "fromusenet" logs.
#
# 0.0.5         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Wed Apr 13 00:54:42 CEST 2005
#		Eliminate a lot of extra cruft from the "error" & "vette" logs.
#               Eliminate more cruft from the "fromusenet" logs.
#
# 0.0.6         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Thu Apr 14 03:07:08 CEST 2005
#               Eliminate even more cruft from the "fromusenet" logs.
#               Eliminate more cruft from the "vette" logs.
#               Eliminate cruft from the "mischief" logs.
#
# 0.0.7         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Sun May  1 22:29:13 CEST 2005   [guess]
#		The "error" log "no such list" errors didn't need to be
#		quite so "compressed", open them up to be more readable.
#		Also eliminate all use of "xargs", use "fmt -w 75" instead.
#
# 0.0.8         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Sun May  1 23:09:13 CEST 2005
#		Clean up code for queue subdirectories to check.
#
# 0.0.9         Update by Brad Knowles <brad@shub-internet.org>
#               Updated on: Wed May 11 01:23:40 CEST 2005
#		Eliminate date/time stamps from the rest of the logs,
#		then pipe through `sort | uniq -c | sort -nr`
#		so that all possible duplicates are eliminated.
#
# 0.0.10	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Tue Jul 12 16:53:49 CDT 2005
#		Add code to do summary of normal list activity
#
# 0.0.11	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Tue Sep  6 15:44:19 CEST 2005
#		Tweaked display of Mailman/qfiles subdirectories
#		so as to avoid displaying more than $MAX_LINES
#		output, which really helps if you have thousands
#		of bounces sitting around, etc....
#		Also slightly tweaked display of fromusenet logs
#		to split summary from errors.
#
# 0.0.12	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Thu Sep 22 22:37:35 CEST 2005
#		Bugs found by Mark Sapiro and Adrian Wells.
#		Mark suggested splitting the Mailman log directory
#		from the rest of the Mailman source and queues,
#		as well as making sure to clean up all temp files.
#		Adrian discovered that there was a log file format
#		change between Mailman 2.1.5 and 2.1.6, which broke
#		hourly statistics.
#
# 0.0.13	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Mon Dec 26 05:54:27 CET 2005
#		Bugs found by Tom G. Christensen (tgc99):
#		> ps output on solaris is full of whitespace but a further
#		> echo get's rid of it.
#		> The lines in the smtp log are sometimes broken up by a
#		> newline (right before the msgid) which throws of the
#		> summary. Piping it through sed first will rejoin the broken
#		> lines.
#		> Use $AWK instead of awk.
#		Enhancements from Mark Sapiro:
#		> The vette log summary lists posts held for moderation
#		> individually under "Other Errors:". The following patch
#		> (watch out for wrapped lines) summarizes them by list instead.
#
# 0.0.14	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Thu Dec 29 08:17:38 CET 2005
#		Added code to check /usr/local/mailman/data for
#		moderation hold queue
#
# 0.0.15	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Thu Jan 26 02:39:38 CET 2006
#		Tweaked display of summary data in the "smtp-failure" log,
#		and "other" category in the "vette" log, so as to reduce
#		the spewage when things go wonky, mostly by removing
#		unique-ifying data like message-id or sender address.
#		Also tweaked moderation queue information, to tell us
#		how many Python pickle files are in the data directory,
#		as opposed to just doing a directory listing and skipping
#		the files in the middle if there are too many -- now we
#		know how many are being skipped.
#
# 0.0.16	Update by Brad Knowles <brad@shub-internet.org>
#		Updated on: Sun Jan 29 11:45:58 CET 2006
#		Mark found a couple of typos that I somehow let slip
#		through.  Thanks!
#
# 0.0.17	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Thu Jan  3 18:25:06 PST 2008
#		Enabled processing the prior generation of a rotated log.
#		Added SORT_FIELD to the user modifiable parameters.
#
# 0.0.18	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Tue May 13 08:58:26 PDT 2008
#		Changes for Mailman 2.1.10 --
#		Added 'bad' to the list of queues.
#		Made listing of a queue directory conditional on existence.
#		Sumarized unparseable messages and preserved queue entries
#		in the Other Errors section.
#
# 0.0.19	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Sun Nov 30 10:26:15 PST 2008
#		Changed processing of fromusenet posts.  Previously, mmdsr
#		reported as gated only the first and last of a gated range.
#		Now mmdsr reports every post from the 'posted to' entries.
#
# 0.0.20	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Sun Jul 26 08:56:41 PDT 2009
#		Added report of files in the preserved messages directory.
#
# 0.0.21	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Sun Sep 27 12:45:44 PDT 2009
#		Refactored to use mktemp to create temp files
#
# 0.0.22        Update by Mark Sapiro <mark@msapiro.net>
#               Updated on: Fri Mar 21 19:47:19 PDT 2014
#               Updated the "No such list" error report to include the newly
#               added name of the CGI module.
#
# 0.0.23        Update by Mark Sapiro <mark@msapiro.net>
#               Updated on: Sat Jul 25 07:52:55 PDT 2015
#               Updated "Login failure with private rosters" report to
#               account for newly added IP in log message.
#
# 0.0.24        Update by Mark Sapiro <mark@msapiro.net>
#               Updated on: Thu Oct  1 15:15:39 PDT 2015
#               Report vette log held and banned subscriptions as their
#               own categories instead of in other errors.
#               Replaced s/\+\d+@/+<digits>@/ in vette log
#               banned subscription messages due to flood of such.
#
# 0.0.25        Update by Mark Sapiro <mark@msapiro.net>
#               Updated on: Sun Jan 31 07:13:47 PST 2016
#               Report vette log DMARC lookups as their own
#               category instead of in other errors.
#
# 0.0.26        Update by Mark Sapiro <mark@msapiro.net>
#               Updated on: Sat Feb 20 21:17:14 PST 2016
#               Properly display No such list names that contain ".
#
# 0.0.27	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Sun Jun  4 17:12:54 PDT 2017
#		Added -a to default (e)grep to account for logs that may
#		have non-ascii and be seen as binary.
#
# 0.0.28	Update by Mark Sapiro <mark@msapiro.net>
#		Updated on: Fri Jun 15 20:20:36 PDT 2018
#		Added report of new security log.
#		Augmented 0.0.24 to do <letters> as well as <digits>.

###############################################################################
# Set up locations of standard commands, directories, etc....
# You may need to modify these for your platform or installation.
###############################################################################

AWK="/usr/bin/awk"
BASENAME="/usr/bin/basename"
CAT="/bin/cat"
DATE="/bin/date"
EGREP="/bin/egrep -a"
FMT="/usr/bin/fmt"
GREP="/bin/grep -a"
HEAD="/usr/bin/head"
LS="/bin/ls"
MKTEMP="/bin/mktemp"
PS="/bin/ps"
RM="/bin/rm"
SED="/bin/sed"
SENDMAIL="/usr/sbin/sendmail"
SLEEP="/bin/sleep"
SORT="/usr/bin/sort"
TAIL="/usr/bin/tail"
TR="/usr/bin/tr"
UNIQ="/usr/bin/uniq"
XARGS="/usr/bin/xargs"
WC="/usr/bin/wc"

###############################################################################
# Arguments for your mktemp command to specify directory and/or create file.
# For example, HPUX mktemp requires "-c -d <dir>"; FreeBSD requires "-t <dir>".
###############################################################################
TMPDIR="-p /tmp"

###############################################################################
# Mailman queue directory
###############################################################################
QUEUEDIR="/usr/local/mailman/qfiles"

###############################################################################
# Mailman log directory
###############################################################################
LOGDIR="/usr/local/mailman/logs"

###############################################################################
# Mailman data directory
###############################################################################
DATADIR="/usr/local/mailman/data"

###############################################################################
# Mailman preserved messages directory
###############################################################################
SPAMDIR="/usr/local/mailman/spam"

###############################################################################
# Maximum number of subdirectory entries to display in report
###############################################################################
MAX_QUEUE_LINES=20

###############################################################################
# Maximum number of moderation queue pickle files to display in report
###############################################################################
MAX_DATA_LINES=100

###############################################################################
# Mailman Log files to check for errors.
# No need to specify path, only log file name.
###############################################################################
ERR_LOGS="error fromusenet locks mischief security post qrunner smtp-failure vette"

###############################################################################
# Mailman Log files to summarize.
# No need to specify path, only log file name.
###############################################################################
SUM_LOGS="fromusenet post smtp"

###############################################################################
# Mailman queue subdirectories to check.  No need to specify path, only name.
###############################################################################
SUBDIRS="archive bad bounces commands in news out retry shunt virgin"

###############################################################################
# Specify the field specifier used by your sort command to select the second
# field.  This will normally be "+1" or "-k 2".
###############################################################################
SORT_FIELD="+1"

###############################################################################
# Specify recipients for report.  If none, then simply print to "STDOUT".
# Specify sender address for report.  Not used if recipient list is empty.
###############################################################################

SENDER="INSERT.YOUR.SENDER.ADDRESS@HERE"
RCPTS="INSERT.YOUR.RECIPIENTS.ADDRESSES@HERE"

###############################################################################
# If you run this program in cron at 23:59:00, you need to sleep sixty
# seconds to make sure that you capture all the logs for the previous day.
# Comment this line out to disable sleeping altogether.
###############################################################################

SLEEPTIME=60

###############################################################################
# What user id is Mailman installed under?
# How do we determine what UID this program is running as?
# NB:	There will probably be SysV vs. BSD semantic differences here.
#	Be careful.  Make sure the command you specify actually works.
#	The command string specified here may seem arcane, but should pull
#	out the information we need, and throw away everything else.  If
#	there is an easier cross-platform way to do it, please let me know.
###############################################################################

GRABUID=`$PS -o user -p $$ | $TAIL -1`
MYUID=`echo $GRABUID`
RUNAS="mailman"

###############################################################################
# No modifications below this line should be required.
###############################################################################

DAY=`$DATE '+%b %d'`
YEAR=`$DATE '+%Y'`

PROG=`$BASENAME $0`

if [ "${MYUID}" != "${RUNAS}" -a "${MYUID}" != "root" ] ; then
    echo "ERROR: You must be root or ${RUNAS} to run $PROG."
    exit 1
fi

if [ "${SLEEPTIME}" != "" -a "${RCPTS}x" != "${SENDER}x" ] ; then
    if [ $SLEEPTIME -gt 0 ] ; then
	$SLEEP $SLEEPTIME
    fi
fi

TMP=`$MKTEMP $TMPDIR`
TMPLOG=`$MKTEMP $TMPDIR`

if [ "${RCPTS}x" != "x" ] ; then
    echo "From: ${SENDER}" >> $TMP
    echo "To: ${RCPTS}" >> $TMP
    echo "Subject: Mailman Daily Status Report: $DAY $YEAR" >> $TMP
    echo "" >> $TMP
fi

echo "		Mailman Daily Status Report: $DAY $YEAR" >> $TMP
echo "" >> $TMP
echo "" >> $TMP

echo "******************************" >> $TMP
echo "Log File Summary" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP

for LOG in $SUM_LOGS
do

    echo "Log file: $LOG" >> $TMP
    echo "==============================" >> $TMP

    if [ -f "$LOGDIR/${LOG}" ] ; then
      if [ -f "$LOGDIR/${LOG}.1" ] ; then
        $SED -e :a -e '$!N;s/\n //;ta' -e 'P;D' ${LOGDIR}/${LOG}.1 $LOGDIR/$LOG | $GREP -si "^$DAY [0-9][0-9:]* $YEAR" > $TMPLOG
      else
        $SED -e :a -e '$!N;s/\n //;ta' -e 'P;D' $LOGDIR/$LOG | $GREP -si "^$DAY [0-9][0-9:]* $YEAR" > $TMPLOG
      fi

	if [ "${LOG}" = "post" ] ; then

	    echo "" >> $TMP
	    echo "Hourly Summary of Posts" >> $TMP
	    echo "-----------------------" >> $TMP

	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.*$//' $TMPLOG | $UNIQ -c | $SORT -n $SORT_FIELD | $AWK '{ printf( "%8d  %02d:00-%02d:59\n", $1, $2, $2 ) }' >> $TMP

	    echo "" >> $TMP
	    echo "Post Count by List" >> $TMP
	    echo "------------------" >> $TMP

	    $SED -e 's/^.* post to //' -e 's/ .*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Post Count by Sender" >> $TMP
	    echo "--------------------" >> $TMP

	    $SED -e 's/^.* from //' -e 's/,.*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "fromusenet" ] ; then

	    GRPS=`$EGREP -vi '(watermark:|nothing new| gating |: \[[0-9\.]*\])' $TMPLOG | $GREP -i ' posted to ' | $AWK '{ print $9 }' | $SED -e 's/:$//' | $SORT -u | $FMT -w 75`

	    for GRP in $GRPS
	    do
		echo "" >> $TMP
		echo "$GRP Article #'s Gated:" >> $TMP
		echo "------------------------------" >> $TMP
		$EGREP -vi '(watermark:|nothing new| gating |: \[[0-9\.]*\])' $TMPLOG | $GREP -i " posted to " | $GREP -i " $GRP: " | $AWK '{print $10}' |  $SORT -u | $FMT -w 75 >> $TMP
	    done

	elif [ "${LOG}" = "smtp" ] ; then

	    echo "" >> $TMP
	    echo "Hourly Summary of Messages Sent" >> $TMP
	    echo "-------------------------------" >> $TMP
	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.* for / /' -e 's/ recips,.*$//' $TMPLOG | $AWK '{ val=int($1); sum[val]+=$2 } END { for (i=0; i<24; i++) { printf "%8d %02d:00-%02d:59\n", sum[i], i, i } }' >> $TMP

	else

	    $SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	fi

    else

    	echo " ### Log file ${LOG} does not exist ### " >> $TMP

    fi

    echo "" >> $TMP

done

echo "******************************" >> $TMP
echo "Log File Squawks" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP

for LOG in $ERR_LOGS
do
    echo "Log file: $LOG" >> $TMP
    echo "==============================" >> $TMP

    if [ -f "$LOGDIR/${LOG}" ] ; then
      if [ -f "$LOGDIR/${LOG}.1" ] ; then
        $GREP -si "^$DAY [0-9][0-9:]* $YEAR" ${LOGDIR}/${LOG}.1 ${LOGDIR}/${LOG} | $SED "s/^[^:]*://" > $TMPLOG
      else
        $GREP -si "^$DAY [0-9][0-9:]* $YEAR" ${LOGDIR}/${LOG} > $TMPLOG
      fi

	if [ "${LOG}" = "error" ] ; then

	    echo "Uncaught Runner Exceptions:" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP 'Uncaught runner exception' $TMPLOG | $SED 's/^.*exception: //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "No Such List:" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP 'No such list' $TMPLOG | $SED -e 's/^[^)]*) //' -e 's/No such list "//' -e 's/"[^"]*$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    CNT=`$GREP -i 'shunting' $TMPLOG | $WC -l`
	    if [ "${CNT}x" != "x" ] ; then
		if [ ${CNT} -gt 0 ] ; then
		    echo "" >> $TMP
		    echo "Shunting Count: $CNT" >> $TMP
		    echo "------------------------------" >> $TMP
		fi
	    fi

	    echo "" >> $TMP
	    echo "Other Errors:" >> $TMP
	    echo "------------------------------" >> $TMP
	    $EGREP -vi '(Uncaught runner exception|No such list|Traceback|shunting)' $TMPLOG | $SED -e 's/^.* ([0-9]*) //' -e 's/unparseable message.*/unparseable message/' -e 's/preserving file.*/preserving file/' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "fromusenet" ] ; then

	    $EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -vi " gating " | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "mischief" ] ; then

	    echo "" >> $TMP
	    echo "Login failure with private rosters (by user):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Login failure with private rosters' $TMPLOG | $AWK '{ print $(NF-2) " " $(NF-1) " " $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Unsub attempt of non-member (by user):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Unsub attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Reminder attempt of non-member (by user):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Reminder attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Other Messages:" >> $TMP
	    echo "------------------------------" >> $TMP
	    $EGREP -vi '(Login failure with private rosters|Unsub attempt of non-member|Reminder attempt of non-member)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "security" ] ; then

	    $GREP 'Authorization failed' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "post" ] ; then

	    $GREP -vi 'success' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	elif [ "${LOG}" = "vette" ] ; then

	    echo "" >> $TMP
	    echo "Message held -- Post by non-member (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP 'Post by non-member' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Message held -- Suspicious header (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'suspicious header' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Discarded posting (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Discarded posting' $TMPLOG | $AWK '{ print $6 }' | $SED 's/:$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Bulk/Junk message discarded (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $EGREP -i '(bulk|junk) message discarded' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Implicit destination (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Message has implicit destination' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Post to moderated newsgroup (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Posting to a moderated newsgroup' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Post to moderated list (by list):" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'Post to moderated list' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Held Subscriptions" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'held subscription request' $TMPLOG | $AWK '{ print $6 " " $11 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Banned Subscriptions" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i 'banned subscription' $TMPLOG | $AWK '{ print $6 " " $9 }' | $SED -e 's/\+[0-9][0-9]*@/+<digits>@/' -e 's/\+[a-z][a-z]*@/+<letters>@/' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "DMARC lookups" >> $TMP
	    echo "------------------------------" >> $TMP
	    $GREP -i "DMARC lookup for" $TMPLOG | $AWK '{ print $6 " " $10 " " $13 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	    echo "" >> $TMP
	    echo "Other Errors:" >> $TMP
	    echo "------------------------------" >> $TMP
	    $EGREP -vi '(Post by non-member|suspicious header|message approved|Discarded posting|bulk message discarded|junk message discarded|Message has implicit destination|Posting to a moderated newsgroup|Post to moderated list|Message discarded, msgid|held subscription request|banned subscription|dmarc lookup for)' $TMPLOG | $SED -e 's/^.* ([0-9]*) //' -e 's/, message-id=<[^> ]*>:/:/' | $SORT | $UNIQ -c | $SORT -nr >> $TMP 

	elif [ "${LOG}" = "smtp-failure" ] ; then

	    $SED 's/^.* ([0-9]*) //' $TMPLOG | $SED 's/delivery to [^@ ]*@[^@ ]* failed with code/delivery failed with code/g' | $SORT | $UNIQ -c | $SORT -nr >> $TMP 

	else

	    $SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP

	fi
    else

    	echo " ### Log file ${LOG} does not exist ### " >> $TMP

    fi
    echo "" >> $TMP

done

echo "******************************" >> $TMP
echo "Queue Directory Contents" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP

cd $QUEUEDIR
for DIR in $SUBDIRS
do

  if [ -d $DIR ] ; then
    $LS -la $DIR > $TMPLOG
    LINES=`$WC -l < $TMPLOG`
    LINES=`expr $LINES - 3`

    echo "Subdirectory: $DIR" >> $TMP
    echo "==============================" >> $TMP
    echo "Entries: $LINES" >> $TMP
    echo "------------------------------" >> $TMP

    if [ $LINES -le $MAX_QUEUE_LINES ] ; then
    	$CAT $TMPLOG >> $TMP
    else
    	$HEAD $TMPLOG >> $TMP
	echo "" >> $TMP
	echo "	More than $MAX_QUEUE_LINES total entries, skipping ..." >> $TMP
	echo "" >> $TMP
    	$TAIL $TMPLOG >> $TMP
    fi
    echo "" >> $TMP
  fi

done

echo "******************************" >> $TMP
echo "Data Directory Contents" >> $TMP
echo "	(Python pickle files only)" >> $TMP
echo "******************************" >> $TMP

cd $DATADIR

$LS -la | $GREP -i '\.pck$' > $TMPLOG
LINES=`$WC -l < $TMPLOG`

echo "Python pickle files: $LINES" >> $TMP
echo "==============================" >> $TMP
echo "" >> $TMP

if [ $LINES -le $MAX_DATA_LINES ] ; then
    $CAT $TMPLOG >> $TMP
else
    $HEAD $TMPLOG >> $TMP
    echo "" >> $TMP
    echo "	More than $MAX_DATA_LINES total entries, skipping ..." >> $TMP
    echo "" >> $TMP
    $TAIL $TMPLOG >> $TMP
fi
echo "" >> $TMP

echo "******************************" >> $TMP
echo "Moderation Queue Summary" >> $TMP
echo "******************************" >> $TMP
echo "" >> $TMP

$AWK '{ print $NF }' $TMPLOG | $GREP -i '^heldmsg-' | $SED -e 's/^heldmsg-//' -e 's/-[0-9]*\.pck$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
echo "" >> $TMP

if [ -d $SPAMDIR ] ; then
    cd $SPAMDIR
    $LS -la | $GREP -i '\.msg$' > $TMPLOG
    LINES=`$WC -l < $TMPLOG`
    
    echo "******************************" >> $TMP
    echo "Preserved Messages Directory Contents" >> $TMP
    echo "        (.msg files only)" >> $TMP
    echo "******************************" >> $TMP
    echo "Files: $LINES" >> $TMP
    echo "==============================" >> $TMP
    echo "" >> $TMP
    
    if [ $LINES -le $MAX_DATA_LINES ] ; then
        $CAT $TMPLOG >> $TMP
    else
        $HEAD $TMPLOG >> $TMP
        echo "" >> $TMP
        echo "  More than $MAX_DATA_LINES total entries, skipping ..." >> $TMP
        echo "" >> $TMP
        $TAIL $TMPLOG >> $TMP
    fi
    echo "" >> $TMP
fi

###############################################################################
# Okay, we're done with all the code stuff, now handle the output
###############################################################################

if [ "${RCPTS}x" != "x" ] ; then
    $SENDMAIL -t -f$SENDER < $TMP
else
    $CAT $TMP
fi

$RM $TMP $TMPLOG