Merging SF patches:

 [ 904850 ] Scrubber in regular delivery per list
 [ 1027882 ] filter attachments by filename extensions
and a patch in mailman-dev mailing list
 http://mail.python.org/pipermail/mailman-developers/2004-September/017246.html

8 files changed, 121 insertions, 7 deletions

diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index c8f48ff6..bc1c4538 100644
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -261,6 +261,11 @@ ARCHIVE_SCRUBBER = 'Mailman.Handlers.Scrubber'
 # this True.
 SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = False
 
+# Use of attachment filename extension per se is may be dangerous because
+# virus fakes it. You can set this True if you filter the attachment by
+# filename extension
+SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION = False
+
 # This variable defines what happens to text/html subparts.  They can be
 # stripped completely, escaped, or filtered through an external program.  The
 # legal values are:
@@ -458,6 +463,7 @@ GLOBAL_PIPELINE = [
     'Moderate',
     'Hold',
     'MimeDel',
+    'Scrubber',
     'Emergency',
     'Tagger',
     'CalcRecips',
@@ -800,6 +806,9 @@ DEFAULT_MSG_FOOTER = """_______________________________________________
 %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
 """
 
+# Scrub regular delivery
+DEFAULT_SCRUB_NONDIGEST = False
+
 # Mail command processor will ignore mail command lines after designated max.
 DEFAULT_MAIL_COMMANDS_MAX_LINES = 25
 
@@ -922,11 +931,24 @@ DEFAULT_FILTER_CONTENT = No
 # types regardless of subtype (jpeg, gif, etc.).
 DEFAULT_FILTER_MIME_TYPES = []
 
-# DEFAULT_PASS_MIME_TYPES is a list of MIME types to be passed through.  Format is the same as DEFAULT_FILTER_MIME_TYPES
+# DEFAULT_PASS_MIME_TYPES is a list of MIME types to be passed through.
+# Format is the same as DEFAULT_FILTER_MIME_TYPES
 DEFAULT_PASS_MIME_TYPES = ['multipart/mixed',
                            'multipart/alternative',
                            'text/plain']
 
+# DEFAULT_FILTER_FILENAME_EXTENSIONS is a list of filename extensions to be
+# removed. It is useful because many viruses fake their content-type as
+# harmless ones while keep their extension as executable and expect to be
+# executed when victims 'open' them.
+DEFAULT_FILTER_FILENAME_EXTENSIONS = [
+    'exe', 'bat', 'cmd', 'com', 'pif', 'scr', 'vbs', 'cpl'
+    ]
+
+# DEFAULT_PASS_FILENAME_EXTENSIONS is a list of filename extensions to be
+# passed through. Format is the same as DEFAULT_FILTER_FILENAME_EXTENSIONS.
+DEFAULT_PASS_FILENAME_EXTENSIONS = []
+
 # Whether text/html should be converted to text/plain after content filtering
 # is performed.  Conversion is done according to HTML_TO_PLAIN_TEXT_COMMAND
 DEFAULT_CONVERT_HTML_TO_PLAINTEXT = Yes
diff --git a/Mailman/Gui/ContentFilter.py b/Mailman/Gui/ContentFilter.py
index cb7ed95c..d681d6df 100644
--- a/Mailman/Gui/ContentFilter.py
+++ b/Mailman/Gui/ContentFilter.py
@@ -100,6 +100,15 @@ class ContentFilter(GUIBase):
              <tt>multipart</tt> to this list, any messages with attachments
              will be rejected by the pass filter.""")),
 
+            ('filter_filename_extensions', mm_cfg.Text, (10, WIDTH), 0,
+             _("""Remove message attachments that have a matching filename
+             extension."""),),
+
+            ('pass_filename_extensions', mm_cfg.Text, (10, WIDTH), 0,
+             _("""Remove message attachments that don't have a matching
+             filename extension.  Leave this field blank to skip this filter
+             test."""),),
+
             ('convert_html_to_plaintext', mm_cfg.Radio, (_('No'), _('Yes')), 0,
              _("""Should Mailman convert <tt>text/html</tt> parts to plain
              text?  This conversion happens after MIME attachments have been
@@ -158,6 +167,15 @@ class ContentFilter(GUIBase):
                 mlist.filter_mime_types = types
 	    elif property == 'pass_mime_types':
                 mlist.pass_mime_types = types
+        elif property in ('filter_filename_extensions',
+                          'pass_filename_extensions'):
+            fexts = []
+            for ext in [s.strip() for s in val.splitlines()]:
+                fexts.append(ext.lower())
+            if property == 'filter_filename_extensions':
+                mlist.filter_filename_extensions = fexts
+            elif property == 'pass_filename_extensions':
+                mlist.pass_filename_extensions = fexts
         else:
             GUIBase._setValue(self, mlist, property, val, doc)
 
@@ -166,4 +184,8 @@ class ContentFilter(GUIBase):
             return NL.join(mlist.filter_mime_types)
         if property == 'pass_mime_types':
             return NL.join(mlist.pass_mime_types)
+        if property == 'filter_filename_extensions':
+            return NL.join(mlist.filter_filename_extensions)
+        if property == 'pass_filename_extensions':
+            return NL.join(mlist.pass_filename_extensions)
         return None
diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py
index b66c5c4f..6926632d 100644
--- a/Mailman/Gui/NonDigest.py
+++ b/Mailman/Gui/NonDigest.py
@@ -134,6 +134,15 @@ and footers:
              _('''Text appended to the bottom of every immediately-delivery
              message. ''') + headfoot + extra),
             ])
+
+        info.extend([
+            ('scrub_nondigest', mm_cfg.Toggle, (_('No'), _('Yes')), 0,
+             _('Scrub attachments of regular delivery message?'),
+             _('''When you scrub attachments, they are stored in archive
+             area and links are made in the message so that the member can
+             access via web browser. If you want the attachments totally
+             disappear, you can use content filter options.''')),
+            ])
         return info
 
     def _setValue(self, mlist, property, val, doc):
diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py
index 3bcdaffa..79efa620 100644
--- a/Mailman/Handlers/MimeDel.py
+++ b/Mailman/Handlers/MimeDel.py
@@ -26,6 +26,7 @@ contents.
 import os
 import errno
 import tempfile
+from os.path import splitext
 
 from email.Iterators import typed_subpart_iterator
 
@@ -36,6 +37,7 @@ from Mailman.Queue.sbcache import get_switchboard
 from Mailman.Logging.Syslog import syslog
 from Mailman.Version import VERSION
 from Mailman.i18n import _
+from Mailman.Utils import oneline
 
 
 
@@ -59,12 +61,23 @@ def process(mlist, msg, msgdata):
     if passtypes and not (ctype in passtypes or mtype in passtypes):
         dispose(mlist, msg, msgdata,
                 _("The message's content type was not explicitly allowed"))
+    # Filter by file extensions
+    filterexts = mlist.filter_filename_extensions
+    passexts = mlist.pass_filename_extensions
+    fext = get_file_ext(msg)
+    if fext:
+        if fext in filterexts:
+            dispose(mlist, msg, msgdata,
+                 _("The message's file extension was explicitly disallowed"))
+        if passexts and not (fext in passexts):
+            dispose(mlist, msg, msgdata,
+                 _("The message's file extension was not explicitly allowed"))
     numparts = len([subpart for subpart in msg.walk()])
     # If the message is a multipart, filter out matching subparts
     if msg.is_multipart():
         # Recursively filter out any subparts that match the filter list
         prelen = len(msg.get_payload())
-        filter_parts(msg, filtertypes, passtypes)
+        filter_parts(msg, filtertypes, passtypes, filterexts, passexts)
         # If the outer message is now an empty multipart (and it wasn't
         # before!) then, again it gets discarded.
         postlen = len(msg.get_payload())
@@ -121,7 +134,7 @@ def reset_payload(msg, subpart):
 
 
 
-def filter_parts(msg, filtertypes, passtypes):
+def filter_parts(msg, filtertypes, passtypes, filterexts, passexts):
     # Look at all the message's subparts, and recursively filter
     if not msg.is_multipart():
         return 1
@@ -129,7 +142,8 @@ def filter_parts(msg, filtertypes, passtypes):
     prelen = len(payload)
     newpayload = []
     for subpart in payload:
-        keep = filter_parts(subpart, filtertypes, passtypes)
+        keep = filter_parts(subpart, filtertypes, passtypes,
+                            filterexts, passexts)
         if not keep:
             continue
         ctype = subpart.get_content_type()
@@ -140,6 +154,13 @@ def filter_parts(msg, filtertypes, passtypes):
         if passtypes and not (ctype in passtypes or mtype in passtypes):
             # Throw this subpart away
             continue
+        # check file extension
+        fext = get_file_ext(subpart)
+        if fext:
+            if fext in filterexts:
+                continue
+            if passexts and not (fext in passexts):
+                continue
         newpayload.append(subpart)
     # Check to see if we discarded all the subparts
     postlen = len(newpayload)
@@ -218,3 +239,18 @@ are receiving the only remaining copy of the discarded message.
         badq.enqueue(msg, msgdata)
     # Most cases also discard the message
     raise Errors.DiscardMessage
+
+def get_file_ext(m):
+    """
+    Get filename extension. Caution: some virus don't put filename
+    in 'Content-Disposition' header.
+"""
+    fext = ''
+    filename = m.get_filename('') or m.get_param('name', '')
+    if filename:
+        fext = splitext(oneline(filename,'utf-8'))[1]
+        if len(fext) > 1:
+            fext = fext[1:]
+        else:
+            fext = ''
+    return fext
diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py
index 8c1124ec..7429c0b4 100644
--- a/Mailman/Handlers/Scrubber.py
+++ b/Mailman/Handlers/Scrubber.py
@@ -168,6 +168,12 @@ def process(mlist, msg, msgdata=None):
     outer = True
     if msgdata is None:
         msgdata = {}
+    if msgdata:
+        # msgdata is available if it is in GLOBAL_PIPELINE
+        # ie. not in digest or archiver
+        # check if the list owner want to scrub regular delivery
+        if not mlist.scrub_nondigest:
+            return
     dir = calculate_attachments_dir(mlist, msg, msgdata)
     charset = None
     lcset = Utils.GetCharSet(mlist.preferred_language)
@@ -389,8 +395,16 @@ def save_attachment(mlist, msg, dir, filter_html=True):
     # e.g. image/jpg (should be image/jpeg).  For now we just store such
     # things as application/octet-streams since that seems the safest.
     ctype = msg.get_content_type()
-    fnext = os.path.splitext(msg.get_filename(''))[1]
-    ext = guess_extension(ctype, fnext)
+    # i18n file name is encoded
+    lcset = Utils.GetCharSet(mlist.preferred_language)
+    filename = Utils.oneline(msg.get_filename(''), lcset)
+    fnext = os.path.splitext(filename)[1]
+    # For safety, we should confirm this is valid ext for content-type
+    # but we can use fnext if we introduce fnext filtering
+    if mm_cfg.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION:
+        ext = fnext
+    else:
+        ext = guess_extension(ctype, fnext)
     if not ext:
         # We don't know what it is, so assume it's just a shapeless
         # application/octet-stream, unless the Content-Type: is
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 9308ecc6..7eee3524 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -335,6 +335,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         self.include_list_post_header = 1
         self.filter_mime_types = mm_cfg.DEFAULT_FILTER_MIME_TYPES
         self.pass_mime_types = mm_cfg.DEFAULT_PASS_MIME_TYPES
+        self.filter_filename_extensions = \
+            mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS
+        self.pass_filename_extensions = mm_cfg.DEFAULT_PASS_FILENAME_EXTENSIONS
         self.filter_content = mm_cfg.DEFAULT_FILTER_CONTENT
         self.convert_html_to_plaintext = \
             mm_cfg.DEFAULT_CONVERT_HTML_TO_PLAINTEXT
@@ -382,6 +385,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
             self.encode_ascii_prefixes = 0
         else:
             self.encode_ascii_prefixes = 2
+        # scrub regular delivery
+        self.scrub_nondigest = mm_cfg.DEFAULT_SCRUB_NONDIGEST
 
 
     #
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 535f453f..eebd4094 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -36,7 +36,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 90
+DATA_FILE_VERSION = 91
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 4f31ea77..02a16e7e 100644
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -394,6 +394,12 @@ def NewVars(l):
     add_only_if_missing('encode_ascii_prefixes', encode)
     add_only_if_missing('news_moderation', 0)
     add_only_if_missing('header_filter_rules', [])
+    # Scrubber in regular delivery
+    add_only_if_missing('scrub_nondigest', 0)
+    # ContentFilter by file extensions
+    add_only_if_missing('filter_filename_extensions',
+        mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS)
+    add_only_if_missing('pass_filename_extensions', [])