Update Brad Knowles' mailman daily status report.

1 files changed, 103 insertions, 11 deletions

diff --git a/contrib/mmdsr b/contrib/mmdsr
index 77cd3c4c..3b6e49fd 100644
--- a/contrib/mmdsr
+++ b/contrib/mmdsr
@@ -112,6 +112,38 @@
 #		Adrian discovered that there was a log file format
 #		change between Mailman 2.1.5 and 2.1.6, which broke
 #		hourly statistics.
+#
+# 0.0.13	Update by Brad Knowles <brad@stop.mail-abuse.org>
+#		Updated on: Mon Dec 26 05:54:27 CET 2005
+#		Bugs found by Tom G. Christensen (tgc99):
+#		> ps output on solaris is full of whitespace but a further
+#		> echo get's rid of it.
+#		> The lines in the smtp log are sometimes broken up by a
+#		> newline (right before the msgid) which throws of the
+#		> summary. Piping it through sed first will rejoin the broken
+#		> lines.
+#		> Use $AWK instead of awk.
+#		Enhancements from Mark Sapiro:
+#		> The vette log summary lists posts held for moderation
+#		> individually under "Other Errors:". The following patch
+#		> (watch out for wrapped lines) summarizes them by list instead.
+#
+# 0.0.14	Update by Brad Knowles <brad@stop.mail-abuse.org>
+#		Updated on: Thu Dec 29 08:17:38 CET 2005
+#		Added code to check /usr/local/mailman/data for
+#		moderation hold queue
+#
+# 0.0.15	Update by Brad Knowles <brad@stop.mail-abuse.org>
+#		Updated on: Thu Jan 26 02:39:38 CET 2006
+#		Tweaked display of summary data in the "smtp-failure" log,
+#		and "other" category in the "vette" log, so as to reduce
+#		the spewage when things go wonky, mostly by removing
+#		unique-ifying data like message-id or sender address.
+#		Also tweaked moderation queue information, to tell us
+#		how many Python pickle files are in the data directory,
+#		as opposed to just doing a directory listing and skipping
+#		the files in the middle if there are too many -- now we
+#		know how many are being skipped.
 
 ###############################################################################
 # Set up locations of standard commands, directories, etc....
@@ -156,9 +188,19 @@ QUEUEDIR="/usr/local/mailman/qfiles"
 LOGDIR="/usr/local/mailman/logs"
 
 ###############################################################################
+# Mailman data directory
+###############################################################################
+DATADIR="/usr/local/mailman/data"
+
+###############################################################################
 # Maximum number of subdirectory entries to display in report
 ###############################################################################
-MAX_LINES=20
+MAX_QUEUE_LINES=20
+
+###############################################################################
+# Maximum number of moderation queue pickle files to display in report
+###############################################################################
+MAX_DATA_LINES=100
 
 ###############################################################################
 # Mailman Log files to check for errors.
@@ -182,8 +224,8 @@ SUBDIRS="archive bounces commands in news out retry shunt virgin"
 # Specify sender address for report.  Not used if recipient list is empty.
 ###############################################################################
 
-SENDER="put@your.address.here"
-RCPTS="put@the.address.of,your@recipients.here"
+SENDER="INSERT.YOUR.SENDER.ADDRESS@HERE"
+RCPTS="INSERT.YOUR.RECIPIENTS.ADDRESSES@HERE"
 
 ###############################################################################
 # If you run this program in cron at 23:59:00, you need to sleep sixty
@@ -203,7 +245,8 @@ SLEEPTIME=60
 #	there is an easier cross-platform way to do it, please let me know.
 ###############################################################################
 
-MYUID=`$PS -o user -p $$ | $TAIL -1`
+GRABUID=`$PS -o user -p $$ | $TAIL -1`
+MYUID=`echo $GRABUID`
 RUNAS="mailman"
 
 ###############################################################################
@@ -254,7 +297,7 @@ do
     $TOUCH $TMPLOG
     echo "Log file: $LOG" >> $TMP
     echo "==============================" >> $TMP
-    $GREP -si "^$DAY [0-9][0-9:]* $YEAR" $LOGDIR/$LOG >> $TMPLOG
+    $SED -e :a -e '$!N;s/\n //;ta' -e 'P;D' $LOGDIR/$LOG | $GREP -si "^$DAY [0-9][0-9:]* $YEAR" >> $TMPLOG
 
     if [ -f "$LOGDIR/${LOG}" ] ; then
 
@@ -264,7 +307,7 @@ do
 	    echo "Hourly Summary of Posts" >> $TMP
 	    echo "-----------------------" >> $TMP
 
-	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.*$//' $TMPLOG | $UNIQ -c | $SORT -n +1 | awk '{ printf( "%8d  %02d:00-%02d:59\n", $1, $2, $2 ) }' >> $TMP
+	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.*$//' $TMPLOG | $UNIQ -c | $SORT -n +1 | $AWK '{ printf( "%8d  %02d:00-%02d:59\n", $1, $2, $2 ) }' >> $TMP
 
 	    echo "" >> $TMP
 	    echo "Post Count by List" >> $TMP
@@ -295,7 +338,7 @@ do
 	    echo "" >> $TMP
 	    echo "Hourly Summary of Messages Sent" >> $TMP
 	    echo "-------------------------------" >> $TMP
-	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.* for / /' -e 's/ recips,.*$//' $TMPLOG | awk '{ val=int($1); sum[val]+=$2 } END { for (i=0; i<24; i++) { printf "%8d %02d:00-%02d:59\n", sum[i], i, i } }' >> $TMP
+	    $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.* for / /' -e 's/ recips,.*$//' $TMPLOG | $AWK '{ val=int($1); sum[val]+=$2 } END { for (i=0; i<24; i++) { printf "%8d %02d:00-%02d:59\n", sum[i], i, i } }' >> $TMP
 
 	else
 
@@ -416,9 +459,19 @@ do
 	    $GREP -i 'Posting to a moderated newsgroup' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
 
 	    echo "" >> $TMP
+	    echo "Post to moderated list (by list):" >> $TMP
+	    echo "------------------------------" >> $TMP
+	    $GREP -i 'Post to moderated list' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
+
+	    echo "" >> $TMP
 	    echo "Other Errors:" >> $TMP
 	    echo "------------------------------" >> $TMP
-	    $EGREP -vi '(Post by non-member|suspicious header|message approved|Discarded posting|bulk message discarded|junk message discarded|Message has implicit destination|Posting to a moderated newsgroup|Message discarded, msgid)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP 
+MYUID=`$PS -o user -p $$ | $TAIL -1`
+	    $EGREP -vi '(Post by non-member|suspicious header|message approved|Discarded posting|bulk message discarded|junk message discarded|Message has implicit destination|Posting to a moderated newsgroup|Post to moderated list|Message discarded, msgid)' $TMPLOG | $SED -e 's/^.* ([0-9]*) //' -e 's/, message-id=<[^> ]*>:/:/' | $SORT | $UNIQ -c | $SORT -nr >> $TMP 
+
+	elif [ "${LOG}" = "smtp-failure" ] ; then
+
+	    $SED 's/^.* ([0-9]*) //' $TMPLOG | $SED 's/delivery to [^@ ]*@[^@ ]* failed with code/delivery failed with code/g' | $SORT | $UNIQ -c | $SORT -nr >> $TMP 
 
 	else
 
@@ -446,7 +499,7 @@ do
     $RM -f $TMPLOG
     $TOUCH $TMPLOG
     $LS -la $DIR >> $TMPLOG
-    LINES=`wc -l < $TMPLOG`
+    LINES=`$WC -l < $TMPLOG`
     LINES=`expr $LINES - 3`
 
     echo "Subdirectory: $DIR" >> $TMP
@@ -454,12 +507,12 @@ do
     echo "Entries: $LINES" >> $TMP
     echo "------------------------------" >> $TMP
 
-    if [ $LINES -le $MAX_LINES ] ; then
+    if [ $LINES -le $MAX_QUEUE_LINES ] ; then
     	$CAT $TMPLOG >> $TMP
     else
     	$HEAD $TMPLOG >> $TMP
 	echo "" >> $TMP
-	echo "	More than $MAX_LINES total entries, skipping ..." >> $TMP
+	echo "	More than $MAX_QUEUE_LINES total entries, skipping ..." >> $TMP
 	echo "" >> $TMP
     	$TAIL $TMPLOG >> $TMP
     fi
@@ -467,6 +520,45 @@ do
 
 done
 
+echo "******************************" >> $TMP
+echo "Data Directory Contents" >> $TMP
+echo "	(Python pickle files only)" >> $TMP
+echo "******************************" >> $TMP
+
+cd $DATADIR
+
+$RM -f $TMPLOG
+$TOUCH $TMPLOG
+$LS -la | $GREP -i '\.pck$' >> $TMPLOG
+LINES=`$WC -l < $TMPLOG`
+
+echo "Python pickle files: $LINES" >> $TMP
+echo "==============================" >> $TMP
+echo "" >> $TMP
+
+if [ $LINES -le $MAX_DATA_LINES ] ; then
+    $CAT $TMPLOG >> $TMP
+else
+    $HEAD $TMPLOG >> $TMP
+    echo "" >> $TMP
+    echo "	More than $MAX_DATA_LINES total entries, skipping ..." >> $TMP
+    echo "" >> $TMP
+    $TAIL $TMPLOG >> $TMP
+fi
+echo "" >> $TMP
+
+echo "******************************" >> $TMP
+echo "Moderation Queue Summary" >> $TMP
+echo "******************************" >> $TMP
+echo "" >> $TMP
+
+$AWK '{ print $NF }' $TMPLOG | $GREP -i '^heldmsg-' | $SED -e 's/^heldmsg-//' -e 's/-[0-9]*\.pck$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP
+echo "" >> $TMP
+
+###############################################################################
+# Okay, we're done with all the code stuff, now handle the output
+###############################################################################
+
 if [ "${RCPTS}x" != "x" ] ; then
     $SENDMAIL -t -f$SENDER < $TMP
 else