diff options
| -rw-r--r-- | Mailman/Cgi/admin.py | 36 | ||||
| -rw-r--r-- | Mailman/Cgi/admindb.py | 6 | ||||
| -rw-r--r-- | Mailman/Cgi/roster.py | 11 | ||||
| -rw-r--r-- | Mailman/Defaults.py.in | 11 | ||||
| -rw-r--r-- | Mailman/HTMLFormatter.py | 12 | ||||
| -rw-r--r-- | Mailman/Handlers/CleanseDKIM.py | 7 | ||||
| -rw-r--r-- | Mailman/Handlers/Decorate.py | 12 | ||||
| -rw-r--r-- | Mailman/Handlers/Scrubber.py | 31 | ||||
| -rw-r--r-- | Mailman/Queue/Runner.py | 40 | ||||
| -rw-r--r-- | Mailman/Queue/Switchboard.py | 23 | ||||
| -rw-r--r-- | NEWS | 23 |
11 files changed, 157 insertions, 55 deletions
diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py index 718bb0c8..d1a255d3 100644 --- a/Mailman/Cgi/admin.py +++ b/Mailman/Cgi/admin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -982,15 +982,16 @@ def membership_options(mlist, subcat, cgidata, doc, form): } # Now populate the rows for addr in members: + qaddr = urllib.quote(addr) link = Link(mlist.GetOptionsURL(addr, obscure=1), mlist.getMemberCPAddress(addr)) fullname = Utils.uncanonstr(mlist.getMemberName(addr), mlist.preferred_language) - name = TextBox(addr + '_realname', fullname, size=longest).Format() - cells = [Center(CheckBox(addr + '_unsub', 'off', 0).Format()), + name = TextBox(qaddr + '_realname', fullname, size=longest).Format() + cells = [Center(CheckBox(qaddr + '_unsub', 'off', 0).Format()), link.Format() + '<br>' + name + - Hidden('user', urllib.quote(addr)).Format(), + Hidden('user', qaddr).Format(), ] # Do the `mod' option if mlist.getMemberOption(addr, mm_cfg.Moderate): @@ -999,7 +1000,7 @@ def membership_options(mlist, subcat, cgidata, doc, form): else: value = 'off' checked = 0 - box = CheckBox('%s_mod' % addr, value, checked) + box = CheckBox('%s_mod' % qaddr, value, checked) cells.append(Center(box).Format()) for opt in ('hide', 'nomail', 'ack', 'notmetoo', 'nodupes'): extra = '' @@ -1018,23 +1019,23 @@ def membership_options(mlist, subcat, cgidata, doc, form): else: value = 'off' checked = 0 - box = CheckBox('%s_%s' % (addr, opt), value, checked) + box = CheckBox('%s_%s' % (qaddr, opt), value, checked) cells.append(Center(box.Format() + extra)) # This code is less efficient than the original which did a has_key on # the underlying dictionary attribute. This version is slower and # less memory efficient. It points to a new MemberAdaptor interface # method. if addr in mlist.getRegularMemberKeys(): - cells.append(Center(CheckBox(addr + '_digest', 'off', 0).Format())) + cells.append(Center(CheckBox(qaddr + '_digest', 'off', 0).Format())) else: - cells.append(Center(CheckBox(addr + '_digest', 'on', 1).Format())) + cells.append(Center(CheckBox(qaddr + '_digest', 'on', 1).Format())) if mlist.getMemberOption(addr, mm_cfg.OPTINFO['plain']): value = 'on' checked = 1 else: value = 'off' checked = 0 - cells.append(Center(CheckBox('%s_plain' % addr, value, checked))) + cells.append(Center(CheckBox('%s_plain' % qaddr, value, checked))) # User's preferred language langpref = mlist.getMemberLanguage(addr) langs = mlist.GetAvailableLanguages() @@ -1043,7 +1044,7 @@ def membership_options(mlist, subcat, cgidata, doc, form): selected = langs.index(langpref) except ValueError: selected = 0 - cells.append(Center(SelectOptions(addr + '_language', langs, + cells.append(Center(SelectOptions(qaddr + '_language', langs, langdescs, selected)).Format()) usertable.AddRow(cells) # Add the usertable and a legend @@ -1427,7 +1428,8 @@ def change_options(mlist, category, subcat, cgidata, doc): errors = [] removes = [] for user in users: - if cgidata.has_key('%s_unsub' % user): + quser = urllib.quote(user) + if cgidata.has_key('%s_unsub' % quser): try: mlist.ApprovedDeleteMember(user, whence='member mgt page') removes.append(user) @@ -1438,7 +1440,7 @@ def change_options(mlist, category, subcat, cgidata, doc): doc.addError(_('Ignoring changes to deleted member: %(user)s'), tag=_('Warning: ')) continue - value = cgidata.has_key('%s_digest' % user) + value = cgidata.has_key('%s_digest' % quser) try: mlist.setMemberOption(user, mm_cfg.Digests, value) except (Errors.AlreadyReceivingDigests, @@ -1448,28 +1450,28 @@ def change_options(mlist, category, subcat, cgidata, doc): # BAW: Hmm... pass - newname = cgidata.getvalue(user+'_realname', '') + newname = cgidata.getvalue(quser+'_realname', '') newname = Utils.canonstr(newname, mlist.preferred_language) mlist.setMemberName(user, newname) - newlang = cgidata.getvalue(user+'_language') + newlang = cgidata.getvalue(quser+'_language') oldlang = mlist.getMemberLanguage(user) if Utils.IsLanguage(newlang) and newlang <> oldlang: mlist.setMemberLanguage(user, newlang) - moderate = not not cgidata.getvalue(user+'_mod') + moderate = not not cgidata.getvalue(quser+'_mod') mlist.setMemberOption(user, mm_cfg.Moderate, moderate) # Set the `nomail' flag, but only if the user isn't already # disabled (otherwise we might change BYUSER into BYADMIN). - if cgidata.has_key('%s_nomail' % user): + if cgidata.has_key('%s_nomail' % quser): if mlist.getDeliveryStatus(user) == MemberAdaptor.ENABLED: mlist.setDeliveryStatus(user, MemberAdaptor.BYADMIN) else: mlist.setDeliveryStatus(user, MemberAdaptor.ENABLED) for opt in ('hide', 'ack', 'notmetoo', 'nodupes', 'plain'): opt_code = mm_cfg.OPTINFO[opt] - if cgidata.has_key('%s_%s' % (user, opt)): + if cgidata.has_key('%s_%s' % (quser, opt)): mlist.setMemberOption(user, opt_code, 1) else: mlist.setMemberOption(user, opt_code, 0) diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py index e5fd2ade..6e8b58f8 100644 --- a/Mailman/Cgi/admindb.py +++ b/Mailman/Cgi/admindb.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -154,9 +154,9 @@ def main(): signal.signal(signal.SIGTERM, sigterm_handler) realname = mlist.real_name - if not cgidata.keys(): + if not cgidata.keys() or cgidata.has_key('admlogin'): # If this is not a form submission (i.e. there are no keys in the - # form), then we don't need to do much special. + # form) or it's a login, then we don't need to do much special. doc.SetTitle(_('%(realname)s Administrative Database')) elif not details: # This is a form submission diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py index a67e5100..b53e5912 100644 --- a/Mailman/Cgi/roster.py +++ b/Mailman/Cgi/roster.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -71,13 +71,17 @@ def main(): # "admin"-only, then we try to cookie authenticate the user, and failing # that, we check roster-email and roster-pw fields for a valid password. # (also allowed: the list moderator, the list admin, and the site admin). + password = cgidata.getvalue('roster-pw', '') + list_hidden = mlist.WebAuthenticate((mm_cfg.AuthListModerator, + mm_cfg.AuthListAdmin, + mm_cfg.AuthSiteAdmin), + password) if mlist.private_roster == 0: # No privacy ok = 1 elif mlist.private_roster == 1: # Members only addr = cgidata.getvalue('roster-email', '') - password = cgidata.getvalue('roster-pw', '') ok = mlist.WebAuthenticate((mm_cfg.AuthUser, mm_cfg.AuthListModerator, mm_cfg.AuthListAdmin, @@ -85,7 +89,6 @@ def main(): password, addr) else: # Admin only, so we can ignore the address field - password = cgidata.getvalue('roster-pw', '') ok = mlist.WebAuthenticate((mm_cfg.AuthListModerator, mm_cfg.AuthListAdmin, mm_cfg.AuthSiteAdmin), @@ -103,7 +106,7 @@ def main(): doc = HeadlessDocument() doc.set_language(lang) - replacements = mlist.GetAllReplacements(lang) + replacements = mlist.GetAllReplacements(lang, list_hidden) replacements['<mm-displang-box>'] = mlist.FormatButton( 'displang-button', text = _('View this page in')) diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index ae33c857..a18f3a93 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1,6 +1,6 @@ # -*- python -*- -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -460,6 +460,15 @@ NNTP_REWRITE_DUPLICATE_HEADERS = [ ('mime-version', 'X-MIME-Version'), ] +# Some list posts and mail to the -owner address may contain DomainKey or +# DomainKeys Identified Mail (DKIM) signature headers <http://www.dkim.org/>. +# Various list transformations to the message such as adding a list header or +# footer or scrubbing attachments or even reply-to munging can break these +# signatures. It is generally felt that these signatures have value, even if +# broken and even if the outgoing message is resigned. However, some sites +# may wish to remove these headers by setting this to Yes. +REMOVE_DKIM_HEADERS = No + # All `normal' messages which are delivered to the entire list membership go # through this pipeline of handler modules. Lists themselves can override the # global pipeline by defining a `pipeline' attribute. diff --git a/Mailman/HTMLFormatter.py b/Mailman/HTMLFormatter.py index 99ed96da..2a316e3a 100644 --- a/Mailman/HTMLFormatter.py +++ b/Mailman/HTMLFormatter.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -60,7 +60,7 @@ class HTMLFormatter: _('Overview of all %(hostname)s mailing lists')), '<p>', MailmanLogo()))).Format() - def FormatUsers(self, digest, lang=None): + def FormatUsers(self, digest, lang=None, list_hidden=False): if lang is None: lang = self.preferred_language conceal_sub = mm_cfg.ConcealSubscription @@ -74,7 +74,7 @@ class HTMLFormatter: else: members = self.getRegularMemberKeys() for m in members: - if not self.getMemberOption(m, conceal_sub): + if list_hidden or not self.getMemberOption(m, conceal_sub): people.append(m) num_concealed = len(members) - len(people) if num_concealed == 1: @@ -410,7 +410,7 @@ class HTMLFormatter: d['<mm-favicon>'] = mm_cfg.IMAGE_LOGOS + mm_cfg.SHORTCUT_ICON return d - def GetAllReplacements(self, lang=None): + def GetAllReplacements(self, lang=None, list_hidden=False): """ returns standard replaces plus formatted user lists in a dict just like GetStandardReplacements. @@ -418,8 +418,8 @@ class HTMLFormatter: if lang is None: lang = self.preferred_language d = self.GetStandardReplacements(lang) - d.update({"<mm-regular-users>": self.FormatUsers(0, lang), - "<mm-digest-users>": self.FormatUsers(1, lang)}) + d.update({"<mm-regular-users>": self.FormatUsers(0, lang, list_hidden), + "<mm-digest-users>": self.FormatUsers(1, lang, list_hidden)}) return d def GetLangSelectBox(self, lang=None, varname='language'): diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py index 850e3668..0c548a9a 100644 --- a/Mailman/Handlers/CleanseDKIM.py +++ b/Mailman/Handlers/CleanseDKIM.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,9 +25,12 @@ and it will also give the MTA the opportunity to regenerate valid keys originating at the Mailman server for the outgoing message. """ +from Mailman import mm_cfg + - def process(mlist, msg, msgdata): + if not mm_cfg.REMOVE_DKIM_HEADERS: + return del msg['domainkey-signature'] del msg['dkim-signature'] diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py index d6b20391..4c00e34d 100644 --- a/Mailman/Handlers/Decorate.py +++ b/Mailman/Handlers/Decorate.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,6 +17,8 @@ """Decorate a message by sticking the header and footer around it.""" +import re + from types import ListType from email.MIMEText import MIMEText @@ -115,9 +117,15 @@ def process(mlist, msg, msgdata): payload = payload.encode(mcset) newcset = mcset # if this fails, fallback to outer try and wrap=true + format = msg.get_param('format') + delsp = msg.get_param('delsp') del msg['content-transfer-encoding'] del msg['content-type'] msg.set_payload(payload, newcset) + if format: + msg.set_param('Format', format) + if delsp: + msg.set_param('DelSp', delsp) wrap = False except (LookupError, UnicodeError): pass @@ -211,7 +219,7 @@ def decorate(mlist, template, what, extradict={}): template = Utils.to_percent(template) # Interpolate into the template try: - text = (template % d).replace('\r\n', '\n') + text = re.sub(r' *\r?\n', r'\n', template % d) except (ValueError, TypeError), e: syslog('error', 'Exception while calculating %s:\n%s', what, e) what = what.upper() diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py index 4a4a3c59..fd35cbdd 100644 --- a/Mailman/Handlers/Scrubber.py +++ b/Mailman/Handlers/Scrubber.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -187,6 +187,7 @@ def process(mlist, msg, msgdata=None): lcset = Utils.GetCharSet(mlist.preferred_language) lcset_out = Charset(lcset).output_charset or lcset # Now walk over all subparts of this message and scrub out various types + format = delsp = None for part in msg.walk(): ctype = part.get_type(part.get_default_type()) # If the part is text/plain, we leave it alone @@ -194,8 +195,21 @@ def process(mlist, msg, msgdata=None): # We need to choose a charset for the scrubbed message, so we'll # arbitrarily pick the charset of the first text/plain part in the # message. + # MAS: Also get the RFC 3676 stuff from this part. This seems to + # work OK for scrub_nondigest. It will also work as far as + # scrubbing messages for the archive is concerned, but pipermail + # doesn't pay any attention to the RFC 3676 parameters. The plain + # format digest is going to be a disaster in any case as some of + # messages will be format="flowed" and some not. ToDigest creates + # its own Content-Type: header for the plain digest which won't + # have RFC 3676 parameters. If the message Content-Type: headers + # are retained for display in the digest, the parameters will be + # there for information, but not for the MUA. This is the best we + # can do without having get_payload() process the parameters. if charset is None: charset = part.get_content_charset(lcset) + format = part.get_param('format') + delsp = part.get_param('delsp') # TK: if part is attached then check charset and scrub if none if part.get('content-disposition') and \ not part.get_content_charset(): @@ -380,7 +394,18 @@ Url : %(url)s text.append(t) # Now join the text and set the payload sep = _('-------------- next part --------------\n') + # The i18n separator is in the list's charset. Coerce it to the + # message charset. + try: + s = unicode(sep, lcset, 'replace') + sep = s.encode(charset, 'replace') + except (UnicodeError, LookupError, ValueError): + pass replace_payload_by_text(msg, sep.join(text), charset) + if format: + msg.set_param('Format', format) + if delsp: + msg.set_param('DelSp', delsp) return msg @@ -513,5 +538,7 @@ def save_attachment(mlist, msg, dir, filter_html=True): baseurl += '/' # A trailing space in url string may save users who are using # RFC-1738 compliant MUA (Not Mozilla). - url = baseurl + '%s/%s%s%s ' % (dir, filebase, extra, ext) + # Trailing space will definitely be a problem with format=flowed. + # Bracket the URL instead. + url = '<' + baseurl + '%s/%s%s%s>' % (dir, filebase, extra, ext) return url diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py index e229043e..1724f043 100644 --- a/Mailman/Queue/Runner.py +++ b/Mailman/Queue/Runner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -98,16 +98,17 @@ class Runner: # Ask the switchboard for the message and metadata objects # associated with this filebase. msg, msgdata = self._switchboard.dequeue(filebase) - except email.Errors.MessageParseError, e: - # It's possible to get here if the message was stored in the - # pickle in plain text, and the metadata had a _parsemsg key - # that was true, /and/ if the message had some bogosity in - # it. It's almost always going to be spam or bounced spam. - # There's not much we can do (and we didn't even get the - # metadata, so just log the exception and continue. + except Exception, e: + # This used to just catch email.Errors.MessageParseError, + # but other problems can occur in message parsing, e.g. + # ValueError, and exceptions can occur in unpickling too. + # We don't want the runner to die, so we just log and skip + # this entry, but preserve it for analysis. self._log(e) - syslog('error', 'Ignoring unparseable message: %s', filebase) - self._switchboard.finish(filebase) + syslog('error', + 'Skipping and preserving unparseable message: %s', + filebase) + self._switchboard.finish(filebase, preserve=True) continue try: self._onefile(msg, msgdata) @@ -122,9 +123,22 @@ class Runner: self._log(e) # Put a marker in the metadata for unshunting msgdata['whichq'] = self._switchboard.whichq() - new_filebase = self._shunt.enqueue(msg, msgdata) - syslog('error', 'SHUNTING: %s', new_filebase) - self._switchboard.finish(filebase) + # It is possible that shunting can throw an exception, e.g. a + # permissions problem or a MemoryError due to a really large + # message. Try to be graceful. + try: + new_filebase = self._shunt.enqueue(msg, msgdata) + syslog('error', 'SHUNTING: %s', new_filebase) + self._switchboard.finish(filebase) + except Exception, e: + # The message wasn't successfully shunted. Log the + # exception and try to preserve the original queue entry + # for possible analysis. + self._log(e) + syslog('error', + 'SHUNTING FAILED, preserving original entry: %s', + filebase) + self._switchboard.finish(filebase, preserve=True) # Other work we want to do each time through the loop Utils.reap(self._kids, once=True) self._doperiodic() diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index 9a45280d..84e8a5e3 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -164,12 +164,27 @@ class Switchboard: msg = email.message_from_string(msg, Message.Message) return msg, data - def finish(self, filebase): + def finish(self, filebase, preserve=False): bakfile = os.path.join(self.__whichq, filebase + '.bak') try: - os.unlink(bakfile) + if preserve: + psvfile = os.path.join(mm_cfg.SHUNTQUEUE_DIR, filebase + '.psv') + # Create the directory if it doesn't yet exist. + # Copied from __init__. + omask = os.umask(0) # rwxrws--- + try: + try: + os.mkdir(mm_cfg.SHUNTQUEUE_DIR, 0770) + except OSError, e: + if e.errno <> errno.EEXIST: raise + finally: + os.umask(omask) + os.rename(bakfile, psvfile) + else: + os.unlink(bakfile) except EnvironmentError, e: - syslog('error', 'Failed to unlink backup file: %s', bakfile) + syslog('error', 'Failed to unlink/preserve backup file: %s', + bakfile) def files(self, extension='.pck'): times = {} @@ -12,7 +12,9 @@ Here is a history of user visible changes to Mailman. - Changed cmd_who.py to list all members if authorization is with the list's admin or moderator password and to accept the password if the - roster is public. + roster is public. Also changed the web roster to show hidden members + when authorization is by site or list's admin or moderator password + (1587651). - Fixed OldStyleMemberships.py to preserve delivery statuses BYADMIN and BYUSER on a straight change of address (1642388). Also fixed a @@ -21,6 +23,25 @@ Here is a history of user visible changes to Mailman. - Fixed bin/withlist so that -r can take a full package path to a callable. + - Removal of DomainKey/DKIM signatures is now controlled by Defaults.py + mm_cfg.py variable REMOVE_DKIM_HEADERS (default = No). + + - format=flowed and delsp=yes are now preserved for message bodies when + message headers/footers are added and attachments are scrubbed + (1495122). + + - Queue runner processing is improved to log and preserve for analysis in + the shunt queue certain bad queue entries that were previously logged + but lost. Also, entries are preserved when an attempt to shunt throws + an exception (1656289). + + - The admin Membership List pages have been changed in that the email + address which forms a part of the various CGI data keys is now + urllib.quote()ed. This allows changing options for and unsubbing an + address which contains double-quote character, but it may require + changes to scripts that screen-scrape the web admin interface to + produce a membership list so they will report an unquoted address. + 2.1.9 (12-Sep-2006) Security |