Merged and tweaked Jim P's mailman-auto-mod-verbose-members branch.

6 files changed, 132 insertions, 16 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 0e376e1c..a4b63285 100755
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -1118,6 +1118,25 @@ DMARC_RESOLVER_TIMEOUT = seconds(3)
 # The total time to spend trying to get an answer to the question.
 DMARC_RESOLVER_LIFETIME = seconds(5)
 
+# Should the list server auto-moderate members who post too frequently
+# This is intended to stop people who join a list and then use a bot to
+# send many spam messages in a short interval.  These are default settings
+# for new lists.  See the web admin Privacy options -> Sender filters page
+# and the Details for member_verbosity_threshold and member_verbosity_interval
+# links for more information.
+# DEFAULT_MEMBER_VERBOSITY_INTERVAL = number of seconds to track posts
+# DEFAULT_MEMBER_VERBOSITY_THRESHOLD = number of allowed posts per interval
+#                                     (0 to disable).
+DEFAULT_MEMBER_VERBOSITY_INTERVAL = 300
+DEFAULT_MEMBER_VERBOSITY_THRESHOLD = 0
+ 
+# This controls how often to clean old post time entries from the dictionary
+# used to implement the member verbosity feature. This is a compromise between
+# using resources for cleaning and allowing the dictionary to grow very large.
+# The setting is the number of passes through the code before the dictionary
+# is cleaned.
+VERBOSE_CLEAN_LIMIT = 1000
+
 # What domains should be considered equivalent when testing list membership
 # for posting/moderation.
 # If two poster addresses with the same local part but
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
index 3af5d8ef..a9ce0cde 100644
--- a/Mailman/Gui/Privacy.py
+++ b/Mailman/Gui/Privacy.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -61,7 +61,7 @@ class Privacy(GUIBase):
                             _('Confirm and approve')),
                            0,
                            _('What steps are required for subscription?<br>'),
-                           _('''None - no verification steps (<em>Not
+                           _("""None - no verification steps (<em>Not
                            Recommended </em>)<br>
                            Confirm (*) - email confirmation step required <br>
                            Require approval - require list administrator
@@ -75,7 +75,7 @@ class Privacy(GUIBase):
 
                            This prevents mischievous (or malicious) people
                            from creating subscriptions for others without
-                           their consent.'''))
+                           their consent."""))
         else:
             sub_cfentry = ('subscribe_policy', mm_cfg.Radio,
                            # choices
@@ -84,7 +84,7 @@ class Privacy(GUIBase):
                             _('Confirm and approve')),
                            1,
                            _('What steps are required for subscription?<br>'),
-                           _('''Confirm (*) - email confirmation required <br>
+                           _("""Confirm (*) - email confirmation required <br>
                            Require approval - require list administrator
                            approval for subscriptions <br>
                            Confirm and approve - both confirm and approve
@@ -94,7 +94,7 @@ class Privacy(GUIBase):
                            subscription request number that they must reply to
                            in order to subscribe.<br> This prevents
                            mischievous (or malicious) people from creating
-                           subscriptions for others without their consent.'''))
+                           subscriptions for others without their consent."""))
 
         # some helpful values
         admin = mlist.GetScriptURL('admin')
@@ -108,8 +108,8 @@ class Privacy(GUIBase):
 
             _('Subscribing'),
             ('advertised', mm_cfg.Radio, (_('No'), _('Yes')), 0,
-             _('''Advertise this list when people ask what lists are on this
-             machine?''')),
+             _("""Advertise this list when people ask what lists are on this
+             machine?""")),
 
             sub_cfentry,
 
@@ -155,8 +155,8 @@ class Privacy(GUIBase):
              (_('Anyone'), _('List members'), _('List admin only')), 0,
              _('Who can view subscription list?'),
 
-             _('''When set, the list of subscribers is protected by member or
-             admin password authentication.''')),
+             _("""When set, the list of subscribers is protected by member or
+             admin password authentication.""")),
 
             ('obscure_addresses', mm_cfg.Radio, (_('No'), _('Yes')), 0,
              _("""Show member addresses so they're not directly recognizable
@@ -229,6 +229,37 @@ class Privacy(GUIBase):
              <a href="%(adminurl)s/members">membership management
              screens</a>.""")),
 
+            ('member_verbosity_threshold', mm_cfg.Number, 5, 0,
+             _("""Ceiling on acceptable number of member posts, per interval,
+               before automatic moderation."""),
+
+             _("""If a member posts this many times, within a period of time
+               the member is automatically moderated.  Use 0 to disable.  See
+               <a href="?VARHELP=privacy/sender/member_verbosity_interval"
+               >member_verbosity_interval</a> for details on the time period.
+
+               <p>This is intended to stop people who join a list or lists and
+               then use a bot to send many spam messages in a short interval.
+
+               <p>Be careful when using this setting.  If it is set too low,
+               this can be triggered by a single post cross-posted to
+               multiple lists or by a single post to an umbrella list.""")),
+
+            ('member_verbosity_interval', mm_cfg.Number, 5, 0,
+             _("""Number of seconds to keep posts to this list to determine
+               member_verbosity_threshold for automatic moderation of a
+               member."""),
+
+             _("""If a member's total posts to all lists in this installation
+               with member_verbosity_threshold enabled exceeds this list's
+               member_verbosity_threshold, the member is automatically
+               moderated on this list.
+
+               <p>Posts which are counted towards this list's
+               member_verbosity_threshold are all posts to any list with
+               member_verbosity_threshold enabled that arrived within that
+               list's member_verbosity_interval.""")),
+
             ('member_moderation_action', mm_cfg.Radio,
              (_('Hold'), _('Reject'), _('Discard')), 0,
              _("""Action to take when a moderated member posts to the
@@ -484,8 +515,8 @@ class Privacy(GUIBase):
             ('max_num_recipients', mm_cfg.Number, 5, 0,
              _('Ceiling on acceptable number of recipients for a posting.'),
 
-             _('''If a posting has this number, or more, of recipients, it is
-             held for admin approval.  Use 0 for no ceiling.''')),
+             _("""If a posting has this number, or more, of recipients, it is
+             held for admin approval.  Use 0 for no ceiling.""")),
             ]
 
         spam_rtn = [
diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py
index d85cc6a6..c7b8d9e7 100644
--- a/Mailman/Handlers/SpamDetect.py
+++ b/Mailman/Handlers/SpamDetect.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -122,6 +122,21 @@ error, contact the mailing list owner at %(listowner)s."""))
                     raise Errors.RejectMessage, text
                 elif mlist.dmarc_moderation_action == 4:
                     raise Errors.DiscardMessage
+
+        # Get member address if any.
+        for sender in msg.get_senders():
+            if mlist.isMember(sender):
+                break
+        else:
+            sender = msg.get_sender()
+        if (mlist.member_verbosity_threshold > 0 and
+            Utils.IsVerboseMember(mlist, sender)
+           ):
+             mlist.setMemberOption(sender, mm_cfg.Moderate, 1)
+             syslog('vette',
+                    '%s: Automatically Moderated %s for verbose postings.',
+                     mlist.real_name, sender) 
+
     if msgdata.get('approved'):
         return
     # First do site hard coded header spam checks
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index f22e45b4..2404c445 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -1246,6 +1246,53 @@ def IsDMARCProhibited(mlist, email):
     return False
 
 
+# Check a known list in order to auto-moderate verbose members
+# dictionary to remember recent posts.
+recentMemberPostings = {}
+# counter of times through
+clean_count = 0
+def IsVerboseMember(mlist, email):
+    """For lists that request it, we keep track of recent posts by address.
+A message from an address to a list, if the list requests it, is remembered
+for a specified time whether or not the address is a list member, and if the
+address is a member and the member is over the threshold for the list, that
+fact is returned."""
+
+    global clean_count
+
+    if mlist.member_verbosity_threshold == 0:
+        return False
+
+    email = email.lower()
+
+    now = time.time()
+    recentMemberPostings.setdefault(email,[]).append(now +
+                                       float(mlist.member_verbosity_interval)
+                                   )
+    x = range(len(recentMemberPostings[email]))
+    x.reverse()
+    for i in x:
+        if recentMemberPostings[email][i] < now:
+            del recentMemberPostings[email][i]
+
+    clean_count += 1
+    if clean_count >= mm_cfg.VERBOSE_CLEAN_LIMIT:
+        clean_count = 0
+        for addr in recentMemberPostings.keys():
+            x = range(len(recentMemberPostings[addr]))
+            x.reverse()
+            for i in x:
+                if recentMemberPostings[addr][i] < now:
+                    del recentMemberPostings[addr][i]
+            if not recentMemberPostings[addr]:
+                del recentMemberPostings[addr]
+    if not mlist.isMember(email):
+        return False
+    return (len(recentMemberPostings.get(email, [])) >=
+                mlist.member_verbosity_threshold
+           )
+
+
 def check_eq_domains(email, domains_list):
     """The arguments are an email address and a string representing a
     list of lists in a form like 'a,b,c;1,2' representing [['a', 'b',
diff --git a/Mailman/Version.py b/Mailman/Version.py
index cdc8a043..0695cd30 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 108
+DATA_FILE_VERSION = 109
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index da33dc10..66775f05 100755
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2015 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -497,6 +497,10 @@ def NewVars(l):
     add_only_if_missing('dmarc_moderation_notice', '')
     add_only_if_missing('dmarc_wrapped_message_text',
                        mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT)
+    add_only_if_missing('member_verbosity_threshold', 
+                       mm_cfg.DEFAULT_MEMBER_VERBOSITY_THRESHOLD)
+    add_only_if_missing('member_verbosity_interval',
+                       mm_cfg.DEFAULT_MEMBER_VERBOSITY_INTERVAL)
     add_only_if_missing('equivalent_domains', 
                        mm_cfg.DEFAULT_EQUIVALENT_DOMAINS)
     add_only_if_missing('new_member_options',