diff options
Diffstat (limited to 'Mailman')
57 files changed, 1987 insertions, 367 deletions
diff --git a/Mailman/Archiver/HyperArch.py b/Mailman/Archiver/HyperArch.py index 33a77f0b..0c0e3356 100644 --- a/Mailman/Archiver/HyperArch.py +++ b/Mailman/Archiver/HyperArch.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -226,9 +226,9 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None): Utils.GetCharSet(lang), 'replace') text = sdict.interpolate(utemplate) - except (TypeError, ValueError): + except (TypeError, ValueError), e: # The template is really screwed up - pass + syslog('error', 'broken template: %s\n%s', filepath, e) # Make sure the text is in the given character set, or html-ify any bogus # characters. return Utils.uncanonstr(text, lang) @@ -271,6 +271,8 @@ class Article(pipermail.Article): if result: i = result.end(0) self.subject = self.subject[i:] + if self.subject == '': + self.subject = _('No subject') else: i = -1 # Useful to keep around @@ -471,6 +473,7 @@ class Article(pipermail.Article): d["email_html"] = self.quote(self.email) d["title"] = self.quote(self.subject) d["subject_html"] = self.quote(self.subject) + d["message_id"] = self.quote(self._message_id) # TK: These two _url variables are used to compose a response # from the archive web page. So, ... d["subject_url"] = url_quote('Re: ' + self.subject) @@ -506,7 +509,7 @@ class Article(pipermail.Article): subject = self._get_subject_enc(self.prev) prev = ('<LINK REL="Previous" HREF="%s">' % (url_quote(self.prev.filename))) - prev_wsubj = ('<LI>' + _('Previous message:') + + prev_wsubj = ('<LI>' + _('Previous message (by thread):') + ' <A HREF="%s">%s\n</A></li>' % (url_quote(self.prev.filename), self.quote(subject))) @@ -528,7 +531,7 @@ class Article(pipermail.Article): subject = self._get_subject_enc(self.next) next = ('<LINK REL="Next" HREF="%s">' % (url_quote(self.next.filename))) - next_wsubj = ('<LI>' + _('Next message:') + + next_wsubj = ('<LI>' + _('Next message (by thread):') + ' <A HREF="%s">%s\n</A></li>' % (url_quote(self.next.filename), self.quote(subject))) diff --git a/Mailman/Archiver/pipermail.py b/Mailman/Archiver/pipermail.py index 939602ba..9c54bbd9 100644 --- a/Mailman/Archiver/pipermail.py +++ b/Mailman/Archiver/pipermail.py @@ -552,6 +552,8 @@ class T: if start is None: start = 0 counter = 0 + if start: + mbox.skipping(True) while counter < start: try: m = mbox.next() @@ -560,6 +562,8 @@ class T: if m is None: return counter += 1 + if start: + mbox.skipping(False) while 1: try: pos = input.tell() diff --git a/Mailman/Bouncer.py b/Mailman/Bouncer.py index 29fe4ea5..2a1cb539 100644 --- a/Mailman/Bouncer.py +++ b/Mailman/Bouncer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -89,6 +89,8 @@ class Bouncer: mm_cfg.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL self.bounce_unrecognized_goes_to_list_owner = \ mm_cfg.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER + self.bounce_notify_owner_on_bounce_increment = \ + mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT self.bounce_notify_owner_on_disable = \ mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE self.bounce_notify_owner_on_removal = \ @@ -135,6 +137,7 @@ class Bouncer: siblist.Unlock() return info = self.getBounceInfo(member) + first_today = True if day is None: # Use today's date day = time.localtime()[:3] @@ -155,6 +158,7 @@ class Bouncer: return elif info.date == day: # We've already scored any bounces for this day, so ignore it. + first_today = False syslog('bounce', '%s: %s already scored a bounce for date %s', self.internal_name(), member, time.strftime('%d-%b-%Y', day + (0,0,0,0,1,0))) @@ -189,6 +193,9 @@ class Bouncer: info.reset(0, info.date, info.noticesleft) else: self.disableBouncingMember(member, info, msg) + elif self.bounce_notify_owner_on_bounce_increment and first_today: + self.__sendAdminBounceNotice(member, msg, + did=_('bounce score incremented')) # We've set/changed bounce info above. We now need to tell the # MemberAdaptor to set/update it. We do it here in case the # MemberAdaptor stores bounce info externally to the list object to @@ -218,7 +225,7 @@ class Bouncer: if self.bounce_notify_owner_on_disable: self.__sendAdminBounceNotice(member, msg) - def __sendAdminBounceNotice(self, member, msg): + def __sendAdminBounceNotice(self, member, msg, did=_('disabled')): # BAW: This is a bit kludgey, but we're not providing as much # information in the new admin bounce notices as we used to (some of # it was of dubious value). However, we'll provide empty, strange, or @@ -230,7 +237,7 @@ class Bouncer: {'listname' : self.real_name, 'addr' : member, 'negative' : '', - 'did' : _('disabled'), + 'did' : did, 'but' : '', 'reenable' : '', 'owneraddr': siteowner, diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py index 0607ce86..2aa082a2 100644 --- a/Mailman/Bouncers/SimpleMatch.py +++ b/Mailman/Bouncers/SimpleMatch.py @@ -42,7 +42,7 @@ PATTERNS = [ # sz-sb.de, corridor.com, nfg.nl (_c('the following addresses had'), _c('transcript of session follows'), - _c(r'<(?P<fulladdr>[^>]*)>|\(expanded from: <?(?P<addr>[^>)]*)>?\)')), + _c(r'^ *(\(expanded from: )?<?(?P<addr>[^\s@]+@[^\s@>]+?)>?\)?\s*$')), # robanal.demon.co.uk (_c('this message was created automatically by mail delivery software'), _c('original message follows'), @@ -184,6 +184,10 @@ PATTERNS = [ _c( 'Your message to (?P<addr>[^\s@]+@[^\s@]+) was automatically rejected' )), + # mail.ru + (_c('A message that you sent was rejected'), + _c('This is a copy of your message'), + _c('\s(?P<addr>[^\s@]+@[^\s@]+)')), # Next one goes here... ] diff --git a/Mailman/Bouncers/SimpleWarning.py b/Mailman/Bouncers/SimpleWarning.py index ab8d6aa2..4f5958ea 100644 --- a/Mailman/Bouncers/SimpleWarning.py +++ b/Mailman/Bouncers/SimpleWarning.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -57,6 +57,10 @@ patterns = [ (_c('We will continue to try to deliver'), _c('.+'), _c('(?P<addr>.+)')), + # kundenserver.de + (_c('not yet been delivered'), + _c('No action is required on your part'), + _c(r'\s*<?(?P<addr>\S+@[^>\s]+)>?\s*')), # Next one goes here... ] diff --git a/Mailman/Bouncers/Yahoo.py b/Mailman/Bouncers/Yahoo.py index b3edf4fa..47fedce2 100644 --- a/Mailman/Bouncers/Yahoo.py +++ b/Mailman/Bouncers/Yahoo.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Yahoo! has its own weird format for bounces.""" @@ -20,9 +21,15 @@ import re import email from email.Utils import parseaddr -tcre = re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE) +tcre = (re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE), + re.compile(r'Sorry, we were unable to deliver your message to ' + r'the following address(\(es\))?\.', + re.IGNORECASE), + ) acre = re.compile(r'<(?P<addr>[^>]*)>:') -ecre = re.compile(r'--- Original message follows') +ecre = (re.compile(r'--- Original message follows'), + re.compile(r'--- Below this line is a copy of the message'), + ) @@ -36,18 +43,26 @@ def process(msg): # simple state machine # 0 == nothing seen # 1 == tag line seen + # 2 == end line seen state = 0 for line in email.Iterators.body_line_iterator(msg): line = line.strip() - if state == 0 and tcre.match(line): - state = 1 + if state == 0: + for cre in tcre: + if cre.match(line): + state = 1 + break elif state == 1: mo = acre.match(line) if mo: addrs.append(mo.group('addr')) continue - mo = ecre.match(line) - if mo: - # we're at the end of the error response - break + for cre in ecre: + mo = cre.match(line) + if mo: + # we're at the end of the error response + state = 2 + break + elif state == 2: + break return addrs diff --git a/Mailman/CSRFcheck.py b/Mailman/CSRFcheck.py index a3b6885a..d531ffc2 100644 --- a/Mailman/CSRFcheck.py +++ b/Mailman/CSRFcheck.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 by the Free Software Foundation, Inc. +# Copyright (C) 2011-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -55,8 +55,9 @@ def csrf_check(mlist, token): try: issued, keymac = marshal.loads(binascii.unhexlify(token)) key, received_mac = keymac.split(':', 1) - klist, key = key.split('+', 1) - assert klist == mlist.internal_name() + if not key.startswith(mlist.internal_name() + '+'): + return False + key = key[len(mlist.internal_name()) + 1:] if '+' in key: key, user = key.split('+', 1) else: diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py index f3284e17..370a2507 100644 --- a/Mailman/Cgi/admin.py +++ b/Mailman/Cgi/admin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -32,6 +32,7 @@ from email.Utils import unquote, parseaddr, formataddr from Mailman import mm_cfg from Mailman import Utils +from Mailman import Message from Mailman import MailList from Mailman import Errors from Mailman import MemberAdaptor @@ -77,8 +78,8 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' admin_overview(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'admin.py access for non-existent list: %s', - listname) + syslog('error', 'admin: No such list "%s": %s\n', + listname, e) return # Now that we know what list has been requested, all subsequent admin # pages are shown in that list's preferred language. @@ -88,7 +89,8 @@ def main(): # CSRF check safe_params = ['VARHELP', 'adminpw', 'admlogin', - 'letter', 'chunk', 'findmember'] + 'letter', 'chunk', 'findmember', + 'legend'] params = cgidata.keys() if set(params) - set(safe_params): csrf_checked = csrf_check(mlist, cgidata.getvalue('csrf_token')) @@ -522,7 +524,7 @@ def show_results(mlist, doc, category, subcat, cgidata): if category == 'members': # Figure out which subcategory we should display subcat = Utils.GetPathPieces()[-1] - if subcat not in ('list', 'add', 'remove'): + if subcat not in ('list', 'add', 'remove', 'change'): subcat = 'list' # Add member category specific tables form.AddItem(membership_options(mlist, subcat, cgidata, doc, form)) @@ -876,6 +878,13 @@ def membership_options(mlist, subcat, cgidata, doc, form): container.AddItem(header) mass_remove(mlist, container) return container + if subcat == 'change': + header.AddRow([Center(Header(2, _('Address Change')))]) + header.AddCellInfo(header.GetCurrentRowIndex(), 0, colspan=2, + bgcolor=mm_cfg.WEB_HEADER_COLOR) + container.AddItem(header) + address_change(mlist, container) + return container # Otherwise... header.AddRow([Center(Header(2, _('Membership List')))]) header.AddCellInfo(header.GetCurrentRowIndex(), 0, colspan=2, @@ -902,6 +911,15 @@ def membership_options(mlist, subcat, cgidata, doc, form): all.sort(lambda x, y: cmp(x.lower(), y.lower())) # See if the query has a regular expression regexp = cgidata.getvalue('findmember', '').strip() + try: + regexp = regexp.decode(Utils.GetCharSet(mlist.preferred_language)) + except UnicodeDecodeError: + # This is probably a non-ascii character and an English language + # (ascii) list. Even if we didn't throw the UnicodeDecodeError, + # the input may have contained mnemonic or numeric HTML entites mixed + # with other characters. Trying to grok the real meaning out of that + # is complex and error prone, so we don't try. + pass if regexp: try: cre = re.compile(regexp, re.IGNORECASE) @@ -1151,7 +1169,8 @@ def membership_options(mlist, subcat, cgidata, doc, form): continue start = chunkmembers[i*chunksz] end = chunkmembers[min((i+1)*chunksz, last)-1] - link = Link(url + 'chunk=%d' % i, _('from %(start)s to %(end)s')) + link = Link(url + 'chunk=%d' % i + findfrag, + _('from %(start)s to %(end)s')) buttons.append(link) buttons = UnorderedList(*buttons) container.AddItem(footer + buttons.Format() + '<p>') @@ -1167,7 +1186,8 @@ def mass_subscribe(mlist, container): Label(_('Subscribe these users now or invite them?')), RadioButtonArray('subscribe_or_invite', (_('Subscribe'), _('Invite')), - 0, values=(0, 1)) + mm_cfg.DEFAULT_SUBSCRIBE_OR_INVITE, + values=(0, 1)) ]) table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) @@ -1241,6 +1261,38 @@ def mass_remove(mlist, container): +def address_change(mlist, container): + # ADDRESS CHANGE + GREY = mm_cfg.WEB_ADMINITEM_COLOR + table = Table(width='90%') + table.AddRow([Italic(_("""To change a list member's address, enter the + member's current and new addresses below. Use the check boxes to send + notice of the change to the old and/or new address(es)."""))]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, colspan=3) + table.AddRow([ + Label(_("Member's current address")), + TextBox(name='change_from'), + CheckBox('notice_old', 'yes', 0).Format() + + ' ' + + _('Send notice') + ]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 2, bgcolor=GREY) + table.AddRow([ + Label(_('Address to change to')), + TextBox(name='change_to'), + CheckBox('notice_new', 'yes', 0).Format() + + ' ' + + _('Send notice') + ]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 2, bgcolor=GREY) + container.AddItem(Center(table)) + + + def password_inputs(mlist): adminurl = mlist.GetScriptURL('admin', absolute=1) table = Table(cellspacing=3, cellpadding=4) @@ -1436,10 +1488,12 @@ def change_options(mlist, category, subcat, cgidata, doc): removals += cgidata['unsubscribees_upload'].value if removals: names = filter(None, [n.strip() for n in removals.splitlines()]) - send_unsub_notifications = int( - cgidata['send_unsub_notifications_to_list_owner'].value) - userack = int( - cgidata['send_unsub_ack_to_this_batch'].value) + send_unsub_notifications = safeint( + 'send_unsub_notifications_to_list_owner', + mlist.admin_notify_mchanges) + userack = safeint( + 'send_unsub_ack_to_this_batch', + mlist.send_goodbye_msg) unsubscribe_errors = [] unsubscribe_success = [] for addr in names: @@ -1461,13 +1515,77 @@ def change_options(mlist, category, subcat, cgidata, doc): color='#ff0000', size='+2')).Format())) doc.AddItem(UnorderedList(*unsubscribe_errors)) doc.AddItem('<p>') + # Address Changes + if cgidata.has_key('change_from'): + change_from = cgidata.getvalue('change_from', '') + change_to = cgidata.getvalue('change_to', '') + schange_from = Utils.websafe(change_from) + schange_to = Utils.websafe(change_to) + success = False + msg = None + if not (change_from and change_to): + msg = _('You must provide both current and new addresses.') + elif change_from == change_to: + msg = _('Current and new addresses must be different.') + elif mlist.isMember(change_to): + # ApprovedChangeMemberAddress will just delete the old address + # and we don't want that here. + msg = _('%(schange_to)s is already a list member.') + else: + try: + Utils.ValidateEmail(change_to) + except (Errors.MMBadEmailError, Errors.MMHostileAddress): + msg = _('%(schange_to)s is not a valid email address.') + if msg: + doc.AddItem(Header(3, msg)) + doc.AddItem('<p>') + return + try: + mlist.ApprovedChangeMemberAddress(change_from, change_to, False) + except Errors.NotAMemberError: + msg = _('%(schange_from)s is not a member') + except Errors.MMAlreadyAMember: + msg = _('%(schange_to)s is already a member') + except Errors.MembershipIsBanned, pat: + spat = Utils.websafe(str(pat)) + msg = _('%(schange_to)s matches banned pattern %(spat)s') + else: + msg = _('Address %(schange_from)s changed to %(schange_to)s') + success = True + doc.AddItem(Header(3, msg)) + lang = mlist.getMemberLanguage(change_to) + otrans = i18n.get_translation() + i18n.set_language(lang) + list_name = mlist.getListAddress() + text = Utils.wrap(_("""The member address %(change_from)s on the +%(list_name)s list has been changed to %(change_to)s. +""")) + subject = _('%(list_name)s address change notice.') + i18n.set_translation(otrans) + if success and cgidata.getvalue('notice_old', '') == 'yes': + # Send notice to old address. + msg = Message.UserNotification(change_from, + mlist.GetOwnerEmail(), + text=text, + subject=subject, + lang=lang + ) + msg.send(mlist) + doc.AddItem(Header(3, _('Notification sent to %(schange_from)s.'))) + if success and cgidata.getvalue('notice_new', '') == 'yes': + # Send notice to new address. + msg = Message.UserNotification(change_to, + mlist.GetOwnerEmail(), + text=text, + subject=subject, + lang=lang + ) + msg.send(mlist) + doc.AddItem(Header(3, _('Notification sent to %(schange_to)s.'))) + doc.AddItem('<p>') # See if this was a moderation bit operation if cgidata.has_key('allmodbit_btn'): - val = cgidata.getvalue('allmodbit_val') - try: - val = int(val) - except VallueError: - val = None + val = safeint('allmodbit_val') if val not in (0, 1): doc.addError(_('Bad moderation flag value')) else: diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py index d1873321..af3f46d1 100644 --- a/Mailman/Cgi/admindb.py +++ b/Mailman/Cgi/admindb.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -50,16 +50,38 @@ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) EXCERPT_HEIGHT = 10 EXCERPT_WIDTH = 76 +SSENDER = mm_cfg.SSENDER +SSENDERTIME = mm_cfg.SSENDERTIME +STIME = mm_cfg.STIME +if mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS in (SSENDERTIME, STIME): + ssort = mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS +else: + ssort = SSENDER -def helds_by_sender(mlist): +def helds_by_skey(mlist, ssort=SSENDER): heldmsgs = mlist.GetHeldMessageIds() - bysender = {} + byskey = {} for id in heldmsgs: + ptime = mlist.GetRecord(id)[0] sender = mlist.GetRecord(id)[1] - bysender.setdefault(sender, []).append(id) - return bysender + if ssort in (SSENDER, SSENDERTIME): + skey = (0, sender) + else: + skey = (ptime, sender) + byskey.setdefault(skey, []).append((ptime, id)) + # Sort groups by time + for k, v in byskey.items(): + if len(v) > 1: + v.sort() + byskey[k] = v + if ssort == SSENDERTIME: + # Rekey with time + newkey = (v[0][0], k[1]) + del byskey[k] + byskey[newkey] = v + return byskey def hacky_radio_buttons(btnname, labels, values, defaults, spacing=3): @@ -76,6 +98,7 @@ def hacky_radio_buttons(btnname, labels, values, defaults, spacing=3): def main(): + global ssort # Figure out which list is being requested parts = Utils.GetPathPieces() if not parts: @@ -91,7 +114,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' handle_no_list(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'admindb: No such list "%s": %s\n', listname, e) return # Now that we know which list to use, set the system's language to it. @@ -213,9 +236,11 @@ def main(): nomessages = not mlist.GetHeldMessageIds() if not (details or sender or msgid or nomessages): form.AddItem(Center( + '<label>' + CheckBox('discardalldefersp', 0).Format() + ' ' + - _('Discard all messages marked <em>Defer</em>') + _('Discard all messages marked <em>Defer</em>') + + '</label>' )) # Add a link back to the overview, if we're not viewing the overview! adminurl = mlist.GetScriptURL('admin', absolute=1) @@ -253,7 +278,7 @@ def main(): raw=1, mlist=mlist)) num = show_pending_subs(mlist, form) num += show_pending_unsubs(mlist, form) - num += show_helds_overview(mlist, form) + num += show_helds_overview(mlist, form, ssort) addform = num > 0 # Finish up the document, adding buttons to the form if addform: @@ -261,9 +286,11 @@ def main(): form.AddItem('<hr>') if not (details or sender or msgid or nomessages): form.AddItem(Center( + '<label>' + CheckBox('discardalldefersp', 0).Format() + ' ' + - _('Discard all messages marked <em>Defer</em>') + _('Discard all messages marked <em>Defer</em>') + + '</label>' )) form.AddItem(Center(SubmitButton('submit', _('Submit All Data')))) # Put 'Logout' link before the footer @@ -314,10 +341,10 @@ def show_pending_subs(mlist, form): for id in pendingsubs: addr = mlist.GetRecord(id)[1] byaddrs.setdefault(addr, []).append(id) - addrs = byaddrs.keys() + addrs = byaddrs.items() addrs.sort() num = 0 - for addr, ids in byaddrs.items(): + for addr, ids in addrs: # Eliminate duplicates for id in ids[1:]: mlist.HandleRequest(id, mm_cfg.DISCARD) @@ -334,8 +361,10 @@ def show_pending_subs(mlist, form): mm_cfg.DISCARD), checked=0).Format() if addr not in mlist.ban_list: - radio += '<br>' + CheckBox('ban-%d' % id, 1).Format() + \ - ' ' + _('Permanently ban from this list') + radio += ('<br>' + '<label>' + + CheckBox('ban-%d' % id, 1).Format() + + ' ' + _('Permanently ban from this list') + + '</label>') # While the address may be a unicode, it must be ascii paddr = addr.encode('us-ascii', 'replace') table.AddRow(['%s<br><em>%s</em>' % (paddr, Utils.websafe(fullname)), @@ -365,10 +394,10 @@ def show_pending_unsubs(mlist, form): for id in pendingunsubs: addr = mlist.GetRecord(id) byaddrs.setdefault(addr, []).append(id) - addrs = byaddrs.keys() + addrs = byaddrs.items() addrs.sort() num = 0 - for addr, ids in byaddrs.items(): + for addr, ids in addrs: # Eliminate duplicates for id in ids[1:]: mlist.HandleRequest(id, mm_cfg.DISCARD) @@ -402,20 +431,29 @@ def show_pending_unsubs(mlist, form): -def show_helds_overview(mlist, form): - # Sort the held messages by sender - bysender = helds_by_sender(mlist) - if not bysender: +def show_helds_overview(mlist, form, ssort=SSENDER): + # Sort the held messages. + byskey = helds_by_skey(mlist, ssort) + if not byskey: return 0 form.AddItem('<hr>') form.AddItem(Center(Header(2, _('Held Messages')))) + # Add the sort sequence choices if wanted + if mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS: + form.AddItem(Center(_('Show this list grouped/sorted by'))) + form.AddItem(Center(hacky_radio_buttons( + 'summary_sort', + (_('sender/sender'), _('sender/time'), _('ungrouped/time')), + (SSENDER, SSENDERTIME, STIME), + (ssort == SSENDER, ssort == SSENDERTIME, ssort == STIME)))) # Add the by-sender overview tables admindburl = mlist.GetScriptURL('admindb', absolute=1) table = Table(border=0) form.AddItem(table) - senders = bysender.keys() - senders.sort() - for sender in senders: + skeys = byskey.keys() + skeys.sort() + for skey in skeys: + sender = skey[1] qsender = quote_plus(sender) esender = Utils.websafe(sender) senderurl = admindburl + '?sender=' + qsender @@ -434,15 +472,19 @@ def show_helds_overview(mlist, form): left.AddRow([btns]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ + '<label>' + CheckBox('senderpreserve-' + qsender, 1).Format() + ' ' + - _('Preserve messages for the site administrator') + _('Preserve messages for the site administrator') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ + '<label>' + CheckBox('senderforward-' + qsender, 1).Format() + ' ' + - _('Forward messages (individually) to:') + _('Forward messages (individually) to:') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ @@ -458,9 +500,11 @@ def show_helds_overview(mlist, form): if mlist.isMember(sender): if mlist.getMemberOption(sender, mm_cfg.Moderate): left.AddRow([ + '<label>' + CheckBox('senderclearmodp-' + qsender, 1).Format() + ' ' + - _("Clear this member's <em>moderate</em> flag") + _("Clear this member's <em>moderate</em> flag") + + '</label>' ]) else: left.AddRow( @@ -471,9 +515,11 @@ def show_helds_overview(mlist, form): mlist.reject_these_nonmembers + mlist.discard_these_nonmembers): left.AddRow([ + '<label>' + CheckBox('senderfilterp-' + qsender, 1).Format() + ' ' + - _('Add <b>%(esender)s</b> to one of these sender filters:') + _('Add <b>%(esender)s</b> to one of these sender filters:') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) btns = hacky_radio_buttons( @@ -485,10 +531,11 @@ def show_helds_overview(mlist, form): left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) if sender not in mlist.ban_list: left.AddRow([ + '<label>' + CheckBox('senderbanp-' + qsender, 1).Format() + ' ' + _("""Ban <b>%(esender)s</b> from ever subscribing to this - mailing list""")]) + mailing list""") + '</label>']) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) right = Table(border=0) right.AddRow([ @@ -499,7 +546,7 @@ def show_helds_overview(mlist, form): right.AddCellInfo(right.GetCurrentRowIndex(), 0, colspan=2) right.AddRow([' ', ' ']) counter = 1 - for id in bysender[sender]: + for ptime, id in byskey[skey]: info = mlist.GetRecord(id) ptime, sender, subject, reason, filename, msgdata = info # BAW: This is really the size of the message pickle, which should @@ -540,13 +587,14 @@ def show_helds_overview(mlist, form): def show_sender_requests(mlist, form, sender): - bysender = helds_by_sender(mlist) - if not bysender: + byskey = helds_by_skey(mlist, SSENDER) + if not byskey: return - sender_ids = bysender.get(sender) + sender_ids = byskey.get((0, sender)) if sender_ids is None: # BAW: should we print an error message? return + sender_ids = [x[1] for x in sender_ids] total = len(sender_ids) count = 1 for id in sender_ids: @@ -623,13 +671,11 @@ def show_post_requests(mlist, id, info, total, count, form): for line in email.Iterators.body_line_iterator(msg, decode=True): lines.append(line) chars += len(line) - if chars > limit > 0: + if chars >= limit > 0: break - # Negative values mean display the entire message, regardless of size - if limit > 0: - body = EMPTYSTRING.join(lines)[:mm_cfg.ADMINDB_PAGE_TEXT_LIMIT] - else: - body = EMPTYSTRING.join(lines) + # We may have gone over the limit on the last line, but keep the full line + # anyway to avoid losing part of a multibyte character. + body = EMPTYSTRING.join(lines) # Get message charset and try encode in list charset # We get it from the first text part. # We need to replace invalid characters here or we can throw an uncaught @@ -644,7 +690,7 @@ def show_post_requests(mlist, id, info, total, count, form): lcset = Utils.GetCharSet(mlist.preferred_language) if mcset <> lcset: try: - body = unicode(body, mcset).encode(lcset, 'replace') + body = unicode(body, mcset, 'replace').encode(lcset, 'replace') except (LookupError, UnicodeError, ValueError): pass hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()]) @@ -677,12 +723,16 @@ def show_post_requests(mlist, id, info, total, count, form): t.AddRow([Bold(_('Action:')), buttons]) t.AddCellInfo(row+3, col-1, align='right') t.AddRow([' ', + '<label>' + CheckBox('preserve-%d' % id, 'on', 0).Format() + - ' ' + _('Preserve message for site administrator') + ' ' + _('Preserve message for site administrator') + + '</label>' ]) t.AddRow([' ', + '<label>' + CheckBox('forward-%d' % id, 'on', 0).Format() + ' ' + _('Additionally, forward this message to: ') + + '</label>' + TextBox('forward-addr-%d' % id, size=47, value=mlist.GetOwnerEmail()).Format() ]) @@ -709,7 +759,9 @@ def show_post_requests(mlist, id, info, total, count, form): def process_form(mlist, doc, cgidata): + global ssort senderactions = {} + badaddrs = [] # Sender-centric actions for k in cgidata.keys(): for prefix in ('senderaction-', 'senderpreserve-', 'senderforward-', @@ -729,6 +781,8 @@ def process_form(mlist, doc, cgidata): discardalldefersp = cgidata.getvalue('discardalldefersp', 0) except ValueError: discardalldefersp = 0 + # Get the summary sequence + ssort = int(cgidata.getvalue('summary_sort', SSENDER)) for sender in senderactions.keys(): actions = senderactions[sender] # Handle what to do about all this sender's held messages @@ -743,8 +797,8 @@ def process_form(mlist, doc, cgidata): preserve = actions.get('senderpreserve', 0) forward = actions.get('senderforward', 0) forwardaddr = actions.get('senderforwardto', '') - bysender = helds_by_sender(mlist) - for id in bysender.get(sender, []): + byskey = helds_by_skey(mlist, SSENDER) + for ptime, id in byskey.get((0, sender), []): if id not in senderactions[sender]['message_ids']: # It arrived after the page was displayed. Skip it. continue @@ -762,20 +816,27 @@ def process_form(mlist, doc, cgidata): # Now see if this sender should be added to one of the nonmember # sender filters. if actions.get('senderfilterp', 0): + # Check for an invalid sender address. try: - which = int(actions.get('senderfilter')) - except ValueError: - # Bogus form - which = 'ignore' - if which == mm_cfg.ACCEPT: - mlist.accept_these_nonmembers.append(sender) - elif which == mm_cfg.HOLD: - mlist.hold_these_nonmembers.append(sender) - elif which == mm_cfg.REJECT: - mlist.reject_these_nonmembers.append(sender) - elif which == mm_cfg.DISCARD: - mlist.discard_these_nonmembers.append(sender) - # Otherwise, it's a bogus form, so ignore it + Utils.ValidateEmail(sender) + except Errors.EmailAddressError: + # Don't check for dups. Report it once for each checked box. + badaddrs.append(sender) + else: + try: + which = int(actions.get('senderfilter')) + except ValueError: + # Bogus form + which = 'ignore' + if which == mm_cfg.ACCEPT: + mlist.accept_these_nonmembers.append(sender) + elif which == mm_cfg.HOLD: + mlist.hold_these_nonmembers.append(sender) + elif which == mm_cfg.REJECT: + mlist.reject_these_nonmembers.append(sender) + elif which == mm_cfg.DISCARD: + mlist.discard_these_nonmembers.append(sender) + # Otherwise, it's a bogus form, so ignore it # And now see if we're to clear the member's moderation flag. if actions.get('senderclearmodp', 0): try: @@ -785,8 +846,15 @@ def process_form(mlist, doc, cgidata): pass # And should this address be banned? if actions.get('senderbanp', 0): - if sender not in mlist.ban_list: - mlist.ban_list.append(sender) + # Check for an invalid sender address. + try: + Utils.ValidateEmail(sender) + except Errors.EmailAddressError: + # Don't check for dups. Report it once for each checked box. + badaddrs.append(sender) + else: + if sender not in mlist.ban_list: + mlist.ban_list.append(sender) # Now, do message specific actions banaddrs = [] erroraddrs = [] @@ -836,6 +904,8 @@ def process_form(mlist, doc, cgidata): if cgidata.getvalue(bankey): sender = mlist.GetRecord(request_id)[1] if sender not in mlist.ban_list: + # We don't need to validate the sender. An invalid address + # can't get here. mlist.ban_list.append(sender) # Handle the request id try: @@ -854,7 +924,14 @@ def process_form(mlist, doc, cgidata): doc.AddItem(Header(2, _('Database Updated...'))) if erroraddrs: for addr in erroraddrs: + addr = Utils.websafe(addr) doc.AddItem(`addr` + _(' is already a member') + '<br>') if banaddrs: for addr, patt in banaddrs: + addr = Utils.websafe(addr) doc.AddItem(_('%(addr)s is banned (matched: %(patt)s)') + '<br>') + if badaddrs: + for addr in badaddrs: + addr = Utils.websafe(addr) + doc.AddItem(`addr` + ': ' + _('Bad/Invalid email address') + + '<br>') diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py index 607f1784..97297e10 100644 --- a/Mailman/Cgi/confirm.py +++ b/Mailman/Cgi/confirm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -64,7 +64,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'confirm: No such list "%s": %s', listname, e) return # Set the language for the list @@ -99,8 +99,9 @@ def main(): %(safecookie)s. <p>Note that confirmation strings expire approximately - %(days)s days after the initial subscription request. If your - confirmation has expired, please try to re-submit your subscription. + %(days)s days after the initial request. They also expire if the + request has already been handled in some way. If your confirmation + has expired, please try to re-submit your request. Otherwise, <a href="%(confirmurl)s">re-enter</a> your confirmation string.''') @@ -258,7 +259,8 @@ def subscription_prompt(mlist, doc, cookie, userdesc): <p>Or hit <em>Cancel my subscription request</em> if you no longer want to subscribe to this list.""") + '<p><hr>' - if mlist.subscribe_policy in (2, 3): + if (mlist.subscribe_policy in (2, 3) and + not getattr(userdesc, 'invitation', False)): # Confirmation is required result = _("""Your confirmation is required in order to continue with the subscription request to the mailing list <em>%(listname)s</em>. diff --git a/Mailman/Cgi/edithtml.py b/Mailman/Cgi/edithtml.py index ee1ccd04..d01ff889 100644 --- a/Mailman/Cgi/edithtml.py +++ b/Mailman/Cgi/edithtml.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -72,7 +72,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'edithtml: No such list "%s": %s', listname, e) return # Now that we have a valid list, set the language to its default diff --git a/Mailman/Cgi/listinfo.py b/Mailman/Cgi/listinfo.py index 8aaae14c..c13fdb26 100644 --- a/Mailman/Cgi/listinfo.py +++ b/Mailman/Cgi/listinfo.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -22,6 +22,7 @@ import os import cgi +import time from Mailman import mm_cfg from Mailman import Utils @@ -52,7 +53,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' listinfo_overview(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'listinfo: No such list "%s": %s', listname, e) return # See if the user want to see this page in other language @@ -184,6 +185,30 @@ def list_listinfo(mlist, lang): replacements['<mm-confirm-password>'] = mlist.FormatSecureBox('pw-conf') replacements['<mm-subscribe-form-start>'] = mlist.FormatFormStart( 'subscribe') + if mm_cfg.SUBSCRIBE_FORM_SECRET: + now = str(int(time.time())) + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'w.x.y.z'))) + # Try to accept a range in case of load balancers, etc. (LP: #1447445) + if remote.find('.') >= 0: + # ipv4 - drop last octet + remote = remote.rsplit('.', 1)[0] + else: + # ipv6 - drop last 16 (could end with :: in which case we just + # drop one : resulting in an invalid format, but it's only + # for our hash so it doesn't matter. + remote = remote.rsplit(':', 1)[0] + replacements['<mm-subscribe-form-start>'] += ( + '<input type="hidden" name="sub_form_token" value="%s:%s">\n' + % (now, Utils.sha_new(mm_cfg.SUBSCRIBE_FORM_SECRET + + now + + mlist.internal_name() + + remote + ).hexdigest() + ) + ) # Roster form substitutions replacements['<mm-roster-form-start>'] = mlist.FormatFormStart('roster') replacements['<mm-roster-option>'] = mlist.FormatRosterOptionForUser(lang) diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py index 9a2389a9..cdc2bef3 100644 --- a/Mailman/Cgi/options.py +++ b/Mailman/Cgi/options.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,6 +17,7 @@ """Produce and handle the member options.""" +import re import sys import os import cgi @@ -33,8 +34,12 @@ from Mailman import i18n from Mailman.htmlformat import * from Mailman.Logging.Syslog import syslog +OR = '|' SLASH = '/' SETLANGUAGE = -1 +DIGRE = re.compile( + '<!--Start-Digests-Delete-->.*<!--End-Digests-Delete-->', + re.DOTALL) # Set up i18n _ = i18n._ @@ -52,6 +57,18 @@ def main(): doc = Document() doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + method = Utils.GetRequestMethod() + if method.lower() not in ('get', 'post'): + title = _('CGI script error') + doc.SetTitle(title) + doc.AddItem(Header(2, title)) + doc.addError(_('Invalid request method: %(method)s')) + doc.AddItem('<hr>') + doc.AddItem(MailmanLogo()) + print 'Status: 405 Method Not Allowed' + print doc.Format() + return + parts = Utils.GetPathPieces() lenparts = parts and len(parts) if not parts or lenparts < 1: @@ -81,7 +98,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'options: No such list "%s": %s\n', listname, e) return # The total contents of the user's response @@ -112,6 +129,14 @@ def main(): return else: user = Utils.LCDomain(Utils.UnobscureEmail(SLASH.join(parts[1:]))) + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(user, list) or isinstance(user, tuple): + if len(user) == 0: + user = '' + else: + user = user[-1] # Avoid cross-site scripting attacks safeuser = Utils.websafe(user) @@ -164,6 +189,9 @@ def main(): return # Are we processing an unsubscription request from the login screen? + msgc = _('If you are a list member, a confirmation email has been sent.') + msga = _("""If you are a list member, your unsubscription request has been + forwarded to the list administrator for approval.""") if cgidata.has_key('login-unsub'): # Because they can't supply a password for unsubscribing, we'll need # to do the confirmation dance. @@ -175,14 +203,14 @@ def main(): # be held. Otherwise, send a confirmation. if mlist.unsubscribe_policy: mlist.HoldUnsubscription(user) - doc.addError(_("""Your unsubscription request has been - forwarded to the list administrator for approval."""), - tag='') + doc.addError(msga, tag='') else: - ip = os.environ.get('REMOTE_ADDR') + ip = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) mlist.ConfirmUnsubscription(user, userlang, remote=ip) - doc.addError(_('The confirmation email has been sent.'), - tag='') + doc.addError(msgc, tag='') mlist.Save() finally: mlist.Unlock() @@ -195,19 +223,21 @@ def main(): syslog('mischief', 'Unsub attempt of non-member w/ private rosters: %s', user) - doc.addError(_('The confirmation email has been sent.'), - tag='') + if mlist.unsubscribe_policy: + doc.addError(msga, tag='') + else: + doc.addError(msgc, tag='') loginpage(mlist, doc, user, language) print doc.Format() return # Are we processing a password reminder from the login screen? + msg = _("""If you are a list member, + your password has been emailed to you.""") if cgidata.has_key('login-remind'): if mlist.isMember(user): mlist.MailUserPassword(user) - doc.addError( - _('A reminder of your password has been emailed to you.'), - tag='') + doc.addError(msg, tag='') else: # Not a member if mlist.private_roster == 0: @@ -217,9 +247,7 @@ def main(): syslog('mischief', 'Reminder attempt of non-member w/ private rosters: %s', user) - doc.addError( - _('A reminder of your password has been emailed to you.'), - tag='') + doc.addError(msg, tag='') loginpage(mlist, doc, user, language) print doc.Format() return @@ -251,9 +279,13 @@ def main(): # So as not to allow membership leakage, prompt for the email # address and the password here. if mlist.private_roster <> 0: + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) syslog('mischief', - 'Login failure with private rosters: %s', - user) + 'Login failure with private rosters: %s from %s', + user, remote) user = None # give an HTTP 401 for authentication failure print 'Status: 401 Unauthorized' @@ -265,6 +297,14 @@ def main(): # options. The first set of checks does not require the list to be # locked. + # However, if a form is submitted for a user who has been asynchronously + # unsubscribed, uncaught NotAMemberError exceptions can be thrown. + + if not mlist.isMember(user): + loginpage(mlist, doc, user, language) + print doc.Format() + return + if cgidata.has_key('logout'): print mlist.ZapCookie(mm_cfg.AuthUser, user) loginpage(mlist, doc, user, language) @@ -506,6 +546,13 @@ address. Upon confirmation, any other mailing list containing the address user, 'via the member options page', userack=1) except Errors.MMNeedApproval: needapproval = True + except Errors.NotAMemberError: + # MAS This except should really be in the outer try so we + # don't save the list redundantly, but except and finally in + # the same try requires Python >= 2.5. + # Setting a switch and making the Save() conditional doesn't + # seem worth it as the Save() won't change anything. + pass mlist.Save() finally: mlist.Unlock() @@ -846,8 +893,10 @@ You are subscribed to this list with the case-preserved address else: replacements['<mm-case-preserved-user>'] = '' - doc.AddItem(mlist.ParseTags('options.html', replacements, userlang)) - + page_text = mlist.ParseTags('options.html', replacements, userlang) + if not (mlist.digestable or mlist.getMemberOption(user, mm_cfg.Digests)): + page_text = DIGRE.sub('', page_text) + doc.AddItem(page_text) def loginpage(mlist, doc, user, lang): @@ -1049,7 +1098,8 @@ def topic_details(mlist, doc, user, cpuser, userlang, varhelp): table.AddRow([Bold(Label(_('Name:'))), Utils.websafe(name)]) table.AddRow([Bold(Label(_('Pattern (as regexp):'))), - '<pre>' + Utils.websafe(pattern) + '</pre>']) + '<pre>' + Utils.websafe(OR.join(pattern.splitlines())) + + '</pre>']) table.AddRow([Bold(Label(_('Description:'))), Utils.websafe(description)]) # Make colors look nice diff --git a/Mailman/Cgi/private.py b/Mailman/Cgi/private.py index b0358285..36cacee4 100644..100755 --- a/Mailman/Cgi/private.py +++ b/Mailman/Cgi/private.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -111,7 +111,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'private: No such list "%s": %s\n', listname, e) return i18n.set_language(mlist.preferred_language) @@ -135,6 +135,27 @@ def main(): message = Bold(FontSize('+1', _('Authorization failed.'))).Format() # give an HTTP 401 for authentication failure print 'Status: 401 Unauthorized' + # Are we processing a password reminder from the login screen? + if cgidata.has_key('login-remind'): + if username: + message = Bold(FontSize('+1', _("""If you are a list member, + your password has been emailed to you."""))).Format() + else: + message = Bold(FontSize('+1', + _('Please enter your email address'))).Format() + if mlist.isMember(username): + mlist.MailUserPassword(username) + elif username: + # Not a member + if mlist.private_roster == 0: + # Public rosters + safeuser = Utils.websafe(username) + message = Bold(FontSize('+1', + _('No such member: %(safeuser)s.'))).Format() + else: + syslog('mischief', + 'Reminder attempt of non-member w/ private rosters: %s', + username) # Output the password form charset = Utils.GetCharSet(mlist.preferred_language) print 'Content-type: text/html; charset=' + charset + '\n\n' diff --git a/Mailman/Cgi/rmlist.py b/Mailman/Cgi/rmlist.py index 8988dc42..db588121 100644 --- a/Mailman/Cgi/rmlist.py +++ b/Mailman/Cgi/rmlist.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -62,7 +62,7 @@ def main(): # Avoid cross-site scripting attacks safelistname = Utils.websafe(listname) title = _('No such list <em>%(safelistname)s</em>') - doc.SetTitle(title) + doc.SetTitle(_('No such list %(safelistname)s')) doc.AddItem( Header(3, Bold(FontAttr(title, color='#ff0000', size='+2')))) @@ -71,7 +71,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'rmlist: No such list "%s": %s\n', listname, e) return # Now that we have a valid mailing list, set the language @@ -188,7 +188,7 @@ def process_request(doc, cgidata, mlist): def request_deletion(doc, mlist, errmsg=None): realname = mlist.real_name title = _('Permanently remove mailing list <em>%(realname)s</em>') - doc.SetTitle(title) + doc.SetTitle(_('Permanently remove mailing list %(realname)s')) table = Table(border=0, width='100%') table.AddRow([Center(Bold(FontAttr(title, size='+1')))]) diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py index 6260c973..6c64925b 100644 --- a/Mailman/Cgi/roster.py +++ b/Mailman/Cgi/roster.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -57,7 +57,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' error_page(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'roster: no such list "%s": %s', listname, e) + syslog('error', 'roster: No such list "%s": %s', listname, e) return cgidata = cgi.FieldStorage() diff --git a/Mailman/Cgi/subscribe.py b/Mailman/Cgi/subscribe.py index 7c49c51c..ab5c7cd8 100644..100755 --- a/Mailman/Cgi/subscribe.py +++ b/Mailman/Cgi/subscribe.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -20,6 +20,7 @@ import sys import os import cgi +import time import signal from Mailman import mm_cfg @@ -63,7 +64,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'subscribe: No such list "%s": %s\n', listname, e) return # See if the form data has a preferred language set, in which case, use it @@ -117,9 +118,44 @@ def process_form(mlist, doc, cgidata, lang): # Canonicalize the full name fullname = Utils.canonstr(fullname, lang) # Who was doing the subscribing? - remote = os.environ.get('REMOTE_HOST', - os.environ.get('REMOTE_ADDR', - 'unidentified origin')) + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) + # Are we checking the hidden data? + if mm_cfg.SUBSCRIBE_FORM_SECRET: + now = int(time.time()) + # Try to accept a range in case of load balancers, etc. (LP: #1447445) + if remote.find('.') >= 0: + # ipv4 - drop last octet + remote1 = remote.rsplit('.', 1)[0] + else: + # ipv6 - drop last 16 (could end with :: in which case we just + # drop one : resulting in an invalid format, but it's only + # for our hash so it doesn't matter. + remote1 = remote.rsplit(':', 1)[0] + try: + ftime, fhash = cgidata.getvalue('sub_form_token', '').split(':') + then = int(ftime) + except ValueError: + ftime = fhash = '' + then = 0 + token = Utils.sha_new(mm_cfg.SUBSCRIBE_FORM_SECRET + + ftime + + mlist.internal_name() + + remote1).hexdigest() + if ftime and now - then > mm_cfg.FORM_LIFETIME: + results.append(_('The form is too old. Please GET it again.')) + if ftime and now - then < mm_cfg.SUBSCRIBE_FORM_MIN_TIME: + results.append( + _('Please take a few seconds to fill out the form before submitting it.')) + if ftime and token != fhash: + results.append( + _("The hidden token didn't match. Did your IP change?")) + if not ftime: + results.append( + _('There was no hidden token in your submission or it was corrupted.')) + results.append(_('You must GET the form before submitting it.')) # Was an attempt made to subscribe the list to itself? if email == mlist.GetListEmail(): syslog('mischief', 'Attempt to self subscribe %s: %s', email, remote) diff --git a/Mailman/Commands/cmd_confirm.py b/Mailman/Commands/cmd_confirm.py index a3accf64..379d23c2 100644 --- a/Mailman/Commands/cmd_confirm.py +++ b/Mailman/Commands/cmd_confirm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -49,9 +49,9 @@ def process(res, args): days = int(mm_cfg.PENDING_REQUEST_LIFE / mm_cfg.days(1) + 0.5) res.results.append(_("""\ Invalid confirmation string. Note that confirmation strings expire -approximately %(days)s days after the initial subscription request. If your -confirmation has expired, please try to re-submit your original request or -message.""")) +approximately %(days)s days after the initial request. They also expire if +the request has already been handled in some way. If your confirmation has +expired, please try to re-submit your original request or message.""")) except Errors.MMNeedApproval: res.results.append(_("""\ Your request has been forwarded to the list moderator for approval.""")) diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 9aebaea2..0e376e1c 100644..100755 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1,6 +1,6 @@ # -*- python -*- -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -111,6 +111,34 @@ AUTHENTICATION_COOKIE_LIFETIME = 0 # Form lifetime is set against Cross Site Request Forgery. FORM_LIFETIME = hours(1) +# If the following is set to a non-empty string, this string in combination +# with the time, list name and the IP address of the requestor is used to +# create a hidden hash as part of the subscribe form on the listinfo page. +# This hash is checked upon form submission and the subscribe fails if it +# doesn't match. I.e. the form posted must be first retrieved from the +# listinfo CGI by the same IP that posts it. The subscribe also fails if +# the time the form was retrieved is more than the above FORM_LIFETIME or less +# than the below SUBSCRIBE_FORM_MIN_TIME before submission. +# Important: If you have any static subscribe forms on your web site, setting +# this option will break them. With this option set, subscribe forms must be +# dynamically generated to include the hidden data. See the code block +# beginning with "if mm_cfg.SUBSCRIBE_FORM_SECRET:" in Mailman/Cgi/listinfo.py +# for the details of the hidden data. +SUBSCRIBE_FORM_SECRET = None + +# If SUBSCRIBE_FORM_SECRET is not None, this is the minimum time the user must +# take after retrieving the form before submitting it. Set to 0 to skip this +# test. +SUBSCRIBE_FORM_MIN_TIME = seconds(5) + +# Installation wide ban list. This is a list of email addresses and regexp +# patterns (beginning with ^) which are not allowed to subscribe to any lists +# in the installation. This supplements the individual list's ban_list. +# For example, to ban xxx@aol.com and any @gmail.com address beginning with +# yyy, set +# GLOBAL_BAN_LIST = ['xxx@aol.com', '^yyy.*@gmail\.com'] +GLOBAL_BAN_LIST = [] + # Command that is used to convert text/html parts into plain text. This # should output results to standard output. %(filename)s will contain the # name of the temporary file that the program should operate on. @@ -118,9 +146,13 @@ HTML_TO_PLAIN_TEXT_COMMAND = '/usr/bin/lynx -dump %(filename)s' # A Python regular expression character class which defines the characters # allowed in list names. Lists cannot be created with names containing any -# character that doesn't match this class. +# character that doesn't match this class. Do not include '/' in this list. ACCEPTABLE_LISTNAME_CHARACTERS = '[-+_.=a-z0-9]' +# Shall the user's real names be displayed along with their email addresses +# in list rosters? Defaults to No to preserve prior behavior. +ROSTER_DISPLAY_REALNAME = No + ##### @@ -204,6 +236,20 @@ WEB_VLINK_COLOR = '' # If true, forces VLINK= WEB_HIGHLIGHT_COLOR = '#dddddd' # If true, alternating rows # in listinfo & admin display +# If you wish to include extra elements in the <HEAD> section of Mailman's +# web pages, e.g. style information or a link to a style sheet, you can set +# the following. For example, to include a css style sheet reference, you +# can put in mm_cfg.py +# WEB_HEAD_ADD = """<LINK REL=stylesheet +# TYPE="text/css" +# HREF="path or URL" +# >""" +# You can specify anything that is allowed in the <HEAD> section. The default +# is to not add anything. This only applies to internally generated pages. +# For pages built from templates you can create custom templates containing +# this information. +WEB_HEAD_ADD = None + # User entered data is escaped for redisplay in web responses to avoid Cross # Site Scripting (XSS) attacks. The normal escaping replaces the characters # <, >, & and " with the respective HTML entities <, >, & and @@ -227,6 +273,17 @@ BROKEN_BROWSER_REPLACEMENTS = {'\x8b': '‹', # single left angle quote '\xbe': '¾', # > plus high order bit '\xa2': '¢', # " plus high order bit } +# +# Shall the admindb held message summary display the grouping and sorting +# option radio buttons? Set this in mm_cfg.py to one of the following: +# SSENDER -> Default to grouped and sorted by sender. +# SSENDERTIME -> Default to grouped by sender and sorted by time. +# STIME -> Default to ungrouped and sorted by time. +DISPLAY_HELD_SUMMARY_SORT_BUTTONS = No +# +# Shall the default for the admin Mass Subscription function be Invite rather +# than Subscribe? Set to Yes in mm_cfg.py to make the default be Invite. +DEFAULT_SUBSCRIBE_OR_INVITE = No @@ -528,7 +585,10 @@ NNTP_REWRITE_DUPLICATE_HEADERS = [ # footer or scrubbing attachments or even reply-to munging can break these # signatures. It is generally felt that these signatures have value, even if # broken and even if the outgoing message is resigned. However, some sites -# may wish to remove these headers by setting this to Yes. +# may wish to remove these headers. Possible values and meanings are: +# No, 0, False -> do not remove headers. +# Yes, 1, True -> remove headers only if the list's from_is_list setting is 1. +# 2 -> always remove headers. REMOVE_DKIM_HEADERS = No # All `normal' messages which are delivered to the entire list membership go @@ -560,6 +620,7 @@ GLOBAL_PIPELINE = [ # (outgoing) path, finally leaving the message in the outgoing queue. 'AfterDelivery', 'Acknowledge', + 'WrapMessage', 'ToOutgoing', ] @@ -844,6 +905,12 @@ BAD_SHUNT_ARCHIVE_DIRECTORY = None # affects both message pickles and MailList config.pck files. SYNC_AFTER_WRITE = No +# This is the name used for the mailmanctl master lock file. In a clustered +# load sharing environment with a shared 'locks' directory, it is desirable +# to have separate locks for each host mailmanctl. This can be used to enable +# that. +MASTER_LOCK_FILE = 'master-qrunner' + ##### @@ -942,6 +1009,27 @@ USER_FRIENDLY_PASSWORDS = Yes MEMBER_PASSWORD_LENGTH = 8 ADMIN_PASSWORD_LENGTH = 10 +# The following headers are always removed from posts to anonymous lists as +# they can reveal the identity of the poster or at least the poster's domain. +# +# From:, Reply-To:, Sender:, Return-Path:, X-Originating-Email:, Received:, +# Message-ID: and X-Envelope-From:. +# +# In addition, Return-Receipt-To:, Disposition-Notification-To:, +# X-Confirm-Reading-To: and X-Pmrqc: headers are removed from all posts as +# they can be used to fish for list membership in addition to possibly +# revealing sender information. +# +# In addition to the above removals, all other headers except those matching +# regular expressions in the following setting are also removed. The default +# setting below keeps all non X- headers, those X- headers added by Mailman +# and any X-Spam- headers. +ANONYMOUS_LIST_KEEP_HEADERS = ['^(?!x-)', '^x-mailman-', + '^x-content-filtered-by:', '^x-topics:', + '^x-ack:', '^x-beenthere:', + '^x-list-administrivia:', '^x-spam-', + ] + ##### @@ -1002,7 +1090,56 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No # moderators? DEFAULT_FORWARD_AUTO_DISCARDS = Yes -# What shold happen to non-member posts which are do not match explicit +# Shall dmarc_moderation_action be applied to messages From: domains with +# a DMARC policy of quarantine as well as reject? This sets the default for +# the list setting that controls it. +DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION = Yes + +# Default action for posts whose From: address domain has a DMARC policy of +# reject or quarantine. See DEFAULT_FROM_IS_LIST below. Whatever is set as +# the default here precludes the list owner from setting a lower value. +# 0 = Accept +# 1 = Munge From +# 2 = Wrap Message +# 3 = Reject +# 4 = Discard +DEFAULT_DMARC_MODERATION_ACTION = 0 + +# Default for text to be added to a separate text/plain part preceding the +# message/rfc822 part containing the original message when +# dmarc_moderation_action is Wrap Message. +DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT = '' + +# Parameters for DMARC DNS lookups. If you are seeing 'DNSException: +# Unable to query DMARC policy ...' entries in your error log, you may need +# to adjust these. +# The time to wait for a response from a name server before timeout. +DMARC_RESOLVER_TIMEOUT = seconds(3) +# The total time to spend trying to get an answer to the question. +DMARC_RESOLVER_LIFETIME = seconds(5) + +# What domains should be considered equivalent when testing list membership +# for posting/moderation. +# If two poster addresses with the same local part but +# different domains are to be considered equivalents for list +# membership tests, the domains are put in the list's equivalent_domains. +# This provides a default value for new lists. +# The format is one or more groups of equivalent domains. Within a group, +# the domains are separated by commas and multiple groups are +# separated by semicolons. White space is ignored. +# For example: +# +# 'example.com,mail.example.com;mac.com,me.com,icloud.com' +# +# In this example, if user@example.com is a list member, +# a post from user@mail.example.com will be treated as if it is +# from user@example.com for list membership/moderation purposes, +# and likewise, if user@me.com is a list member, posts from +# user@mac.com or user@icloud.com will be treated as if from +# user@me.com. +DEFAULT_EQUIVALENT_DOMAINS = '' + +# What should happen to non-member posts which are do not match explicit # non-member actions? # 0 = Accept # 1 = Hold @@ -1027,6 +1164,7 @@ DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX = "-owner" # Exclude/include (sibling) lists for non-digest delivery. DEFAULT_REGULAR_EXCLUDE_LISTS = [] DEFAULT_REGULAR_INCLUDE_LISTS = [] +DEFAULT_REGULAR_EXCLUDE_IGNORE = True ALLOW_CROSS_DOMAIN_SIBLING = False # This variable controls whether monthly password reminders are sent. @@ -1038,6 +1176,16 @@ DEFAULT_SEND_WELCOME_MSG = Yes # Send goodbye messages to unsubscribed members? DEFAULT_SEND_GOODBYE_MSG = Yes +# The following is a three way setting. It sets the default for the list's +# from_is_list policy which is applied to all posts except those for which a +# dmarc_moderation_action other than accept applies. +# 0 -> Do not rewrite the From: or wrap the message. +# 1 -> Rewrite the From: header of posts replacing the posters address with +# that of the list. Also see REMOVE_DKIM_HEADERS above. +# 2 -> Do not modify the From: of the message, but wrap the message in an outer +# message From the list address. +DEFAULT_FROM_IS_LIST = 0 + # Wipe sender information, and make it look like the list-admin # address sends all messages DEFAULT_ANONYMOUS_LIST = No @@ -1078,6 +1226,10 @@ DEFAULT_SUBSCRIBE_POLICY = 1 # Does this site allow completely unchecked subscriptions? ALLOW_OPEN_SUBSCRIBE = No +# This is the default list of addresses and regular expressions (beginning +# with ^) that are exempt from approval if SUBSCRIBE_POLICY is 2 or 3. +DEFAULT_SUBSCRIBE_AUTO_APPROVAL = [] + # The default policy for unsubscriptions. 0 (unmoderated unsubscribes) is # highly recommended! # 0 - unmoderated unsubscribes @@ -1252,6 +1404,10 @@ DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL = days(7) # failed to match by the bounce detector? DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER = Yes +# Does the list owner get a copy of every recognized bounce that increments +# the score for a list member but doesn't result in a disable or probe? +DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT = No + # Notifications on bounce actions. The first specifies whether the list owner # should get a notification when a member is disabled due to bouncing, while # the second specifies whether the owner should get one when the member is @@ -1369,6 +1525,11 @@ UNSUBSCRIBE = 5 ACCEPT = 6 HOLD = 7 +# admindb summary sort button settings. All must evaluate to True. +SSENDER = 1 +SSENDERTIME = 2 +STIME = 3 + # Standard text field width TEXTFIELDWIDTH = 40 @@ -1488,6 +1649,7 @@ add_language('en', _('English (USA)'), 'us-ascii', 'ltr') add_language('es', _('Spanish (Spain)'), 'iso-8859-1', 'ltr') add_language('et', _('Estonian'), 'iso-8859-15', 'ltr') add_language('eu', _('Euskara'), 'iso-8859-15', 'ltr') # Basque +add_language('fa', _('Persian'), 'utf-8', 'rtl') add_language('fi', _('Finnish'), 'iso-8859-1', 'ltr') add_language('fr', _('French'), 'iso-8859-1', 'ltr') add_language('gl', _('Galician'), 'utf-8', 'ltr') @@ -1505,8 +1667,8 @@ add_language('no', _('Norwegian'), 'iso-8859-1', 'ltr') add_language('pl', _('Polish'), 'iso-8859-2', 'ltr') add_language('pt', _('Portuguese'), 'iso-8859-1', 'ltr') add_language('pt_BR', _('Portuguese (Brazil)'), 'iso-8859-1', 'ltr') -add_language('ro', _('Romanian'), 'iso-8859-2', 'ltr') -add_language('ru', _('Russian'), 'koi8-r', 'ltr') +add_language('ro', _('Romanian'), 'utf-8', 'ltr') +add_language('ru', _('Russian'), 'utf-8', 'ltr') add_language('sk', _('Slovak'), 'utf-8', 'ltr') add_language('sl', _('Slovenian'), 'iso-8859-2', 'ltr') add_language('sr', _('Serbian'), 'utf-8', 'ltr') diff --git a/Mailman/Deliverer.py b/Mailman/Deliverer.py index 0f8f26b8..dc867006 100644 --- a/Mailman/Deliverer.py +++ b/Mailman/Deliverer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -42,6 +42,8 @@ except NameError: class Deliverer: def SendSubscribeAck(self, name, password, digest, text=''): pluser = self.getMemberLanguage(name) + # Need to set this here to get the proper l10n of the Subject: + i18n.set_language(pluser) if self.welcome_msg: welcome = Utils.wrap(self.welcome_msg) + '\n' else: @@ -81,6 +83,7 @@ your membership administrative address, %(addr)s.''')) def SendUnsubscribeAck(self, addr, lang): realname = self.real_name + i18n.set_language(lang) msg = Message.UserNotification( self.GetMemberAdminEmail(addr), self.GetBouncesEmail(), _('You have been unsubscribed from the %(realname)s mailing list'), diff --git a/Mailman/Gui/Bounce.py b/Mailman/Gui/Bounce.py index 1dc837fc..e559dcc8 100644 --- a/Mailman/Gui/Bounce.py +++ b/Mailman/Gui/Bounce.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -142,6 +142,15 @@ class Bounce(GUIBase): <a href="?VARHELP=autoreply/autoresponse_admin_text">autoresponse message</a> for email to the -owner and -admin address.""")), + ('bounce_notify_owner_on_bounce_increment', mm_cfg.Toggle, + (_('No'), _('Yes')), 0, + _("""Should Mailman notify you, the list owner, when bounces + cause a member's bounce score to be incremented?"""), + _("""Setting this value to <em>Yes</em> will cause Mailman to + send a notice including a copy of the bounce message to the list + owners whenever a bounce increments a member's bounce score but + doesn't cause a disable or a probe to be sent.""")), + ('bounce_notify_owner_on_disable', mm_cfg.Toggle, (_('No'), _('Yes')), 0, _("""Should Mailman notify you, the list owner, when bounces diff --git a/Mailman/Gui/Digest.py b/Mailman/Gui/Digest.py index f7722019..77691aee 100644 --- a/Mailman/Gui/Digest.py +++ b/Mailman/Gui/Digest.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -56,8 +56,8 @@ class Digest(GUIBase): _('When receiving digests, which format is default?')), ('digest_size_threshhold', mm_cfg.Number, 3, 0, - _('How big in Kb should a digest be before it gets sent out?')), - # Should offer a 'set to 0' for no size threshhold. + _('How big in Kb should a digest be before it gets sent out?' + ' 0 implies no maximum size.')), ('digest_send_periodic', mm_cfg.Radio, (_('No'), _('Yes')), 1, _('Should a digest be dispatched daily when the size threshold ' diff --git a/Mailman/Gui/GUIBase.py b/Mailman/Gui/GUIBase.py index a365acaf..32d19929 100644 --- a/Mailman/Gui/GUIBase.py +++ b/Mailman/Gui/GUIBase.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -63,6 +63,7 @@ class GUIBase: if isinstance(val, ListType): return val addrs = [] + bad_addrs = [] for addr in [s.strip() for s in val.split(NL)]: # Discard empty lines if not addr: @@ -77,22 +78,25 @@ class GUIBase: try: re.compile(addr) except re.error: - raise ValueError + bad_addrs.append(addr) elif (wtype == mm_cfg.EmailListEx and addr.startswith('@') - and property.endswith('_these_nonmembers')): + and (property.endswith('_these_nonmembers') or + property == 'subscribe_auto_approval')): # XXX Needs to be reviewed for list@domain names. # don't reference your own list if addr[1:] == mlist.internal_name(): - raise ValueError + bad_addrs.append(addr) # check for existence of list? For now allow # reference to list before creating it. else: - raise + bad_addrs.append(addr) if property in ('regular_exclude_lists', 'regular_include_lists'): if addr.lower() == mlist.GetListEmail().lower(): - raise Errors.EmailAddressError + bad_addrs.append(addr) addrs.append(addr) + if bad_addrs: + raise Errors.EmailAddressError, ', '.join(bad_addrs) return addrs # This is a host name, i.e. verbatim if wtype == mm_cfg.Host: @@ -168,9 +172,9 @@ class GUIBase: except ValueError: doc.addError(_('Invalid value for variable: %(property)s')) # This is the parent of MMBadEmailError and MMHostileAddress - except Errors.EmailAddressError: + except Errors.EmailAddressError, error: doc.addError( - _('Bad email address for option %(property)s: %(val)s')) + _('Bad email address for option %(property)s: %(error)s')) else: # Set the attribute, which will normally delegate to the mlist self._setValue(mlist, property, val, doc) diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py index e9f8f9b5..980e5f2b 100644 --- a/Mailman/Gui/General.py +++ b/Mailman/Gui/General.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -154,6 +154,72 @@ class General(GUIBase): (listname %%05d) -> (listname 00123) """)), + ('from_is_list', mm_cfg.Radio, + (_('No'), _('Munge From'), _('Wrap Message')), 0, + _("""Replace the From: header address with the list's posting + address to mitigate issues stemming from the original From: + domain's DMARC or similar policies."""), + _("""Several protocols now in wide use attempt to ensure that use + of the domain in the author's address (ie, in the From: header + field) is authorized by that domain. These protocols may be + incompatible with common list features such as footers, causing + participating email services to bounce list traffic merely + because of the address in the From: field. <b>This has resulted + in members being unsubscribed despite being perfectly able to + receive mail.</b> + <p> + The following actions are applied to all list messages when + selected here. To apply these actions only to messages where the + domain in the From: header is determined to use such a protocol, + see the <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a> settings under Privacy options... + -> Sender filters. + <p>Settings:<p> + <dl> + <dt>No</dt> + <dd>Do nothing special. This is appropriate for anonymous lists. + It is appropriate for dedicated announcement lists, unless the + From: address of authorized posters might be in a domain with a + DMARC or similar policy. It is also appropriate if you choose to + use dmarc_moderation_action other than Accept for this list.</dd> + <dt>Munge From</dt> + <dd>This action replaces the poster's address in the From: header + with the list's posting address and adds the poster's address to + the addresses in the original Reply-To: header.</dd> + <dt>Wrap Message</dt> + <dd>Just wrap the message in an outer message with the From: + header containing the list's posting address and with the original + From: address added to the addresses in the original Reply-To: + header and with Content-Type: message/rfc822. This is effectively + a one message MIME format digest.</dd> + </dl> + <p>The transformations for anonymous_list are applied before + any of these actions. It is not useful to apply actions other + than No to an anonymous list, and if you do so, the result may + be surprising. + <p>The Reply-To: header munging actions below interact with these + actions as follows: + <p> first_strip_reply_to = Yes will remove all the incoming + Reply-To: addresses but will still add the poster's address to + Reply-To: for all three settings of reply_goes_to_list which + respectively will result in just the poster's address, the + poster's address and the list posting address or the poster's + address and the explicit reply_to_address in the outgoing + Reply-To: header. If first_strip_reply_to = No the poster's + address in the original From: header, if not already included in + the Reply-To:, will be added to any existing Reply-To: + address(es). + <p>These actions, whether selected here or via <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a>, do not apply to messages in digests + or archives or sent to usenet via the Mail<->News gateways. + <p>If <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a> applies to this message with an + action other than Accept, that action rather than this is + applied""")), + ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Hide the sender of a message, replacing it with the list address (Removes From, Sender and Reply-To fields)""")), @@ -189,11 +255,11 @@ class General(GUIBase): their own <tt>Reply-To:</tt> settings to convey their valid return address. Another is that modifying <tt>Reply-To:</tt> makes it much more difficult to send private replies. See <a - href="http://www.unicom.com/pw/reply-to-harmful.html">`Reply-To' + href="http://marc.merlins.org/netrants/reply-to-harmful.html">`Reply-To' Munging Considered Harmful</a> for a general discussion of this issue. See <a - href="http://www.metasystema.net/essays/reply-to.mhtml">Reply-To - Munging Considered Useful</a> for a dissenting opinion. + href="http://marc.merlins.org/netrants/reply-to-useful.html"> + Reply-To Munging Considered Useful</a> for a dissenting opinion. <p>Some mailing lists have restricted posting privileges, with a parallel list devoted to discussions. Examples are `patches' or @@ -217,11 +283,11 @@ class General(GUIBase): their own <tt>Reply-To:</tt> settings to convey their valid return address. Another is that modifying <tt>Reply-To:</tt> makes it much more difficult to send private replies. See <a - href="http://www.unicom.com/pw/reply-to-harmful.html">`Reply-To' + href="http://marc.merlins.org/netrants/reply-to-harmful.html">`Reply-To' Munging Considered Harmful</a> for a general discussion of this issue. See <a - href="http://www.metasystema.net/essays/reply-to.mhtml">Reply-To - Munging Considered Useful</a> for a dissenting opinion. + href="http://marc.merlins.org/netrants/reply-to-useful.html"> + Reply-To Munging Considered Useful</a> for a dissenting opinion. <p>Some mailing lists have restricted posting privileges, with a parallel list devoted to discussions. Examples are `patches' or diff --git a/Mailman/Gui/Membership.py b/Mailman/Gui/Membership.py index 7e1bf324..fdf5a7ca 100644 --- a/Mailman/Gui/Membership.py +++ b/Mailman/Gui/Membership.py @@ -30,5 +30,6 @@ class Membership: return [('list', _('Membership List')), ('add', _('Mass Subscription')), ('remove', _('Mass Removal')), + ('change', _('Address Change')), ] return None diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py index 9af4a2be..411f5d4f 100644..100755 --- a/Mailman/Gui/NonDigest.py +++ b/Mailman/Gui/NonDigest.py @@ -159,6 +159,17 @@ and footers: Note also that the site administrator may prohibit cross domain siblings.""")), + ('regular_exclude_ignore', mm_cfg.Toggle, (_('No'), _('Yes')), 0, + _("""Ignore regular_exlude_lists of which the poster is not a + member."""), + _("""If a post is addressed to this list and to one or more of + the exclude lists, regular members of those lists will not be + sent the post from this list, but if the poster is not a member + of an excluded list, the post may not be accepted by that list + which leaves the members of that list with no copy of the post. + Setting this to Yes ignores any of the exclude lists of which the + poster is not a member.""")), + ('regular_include_lists', mm_cfg.EmailList, (3, WIDTH), 0, _("""Other mailing lists on this site whose members are included in the regular (non-digest) delivery if those diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py index 75eff2b5..3af5d8ef 100644 --- a/Mailman/Gui/Privacy.py +++ b/Mailman/Gui/Privacy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -113,6 +113,18 @@ class Privacy(GUIBase): sub_cfentry, + ('subscribe_auto_approval', mm_cfg.EmailListEx, (10, WIDTH), 1, + _("""List of addresses (or regexps) whose subscriptions do not + require approval."""), + + (_("""When subscription requires approval, addresses in this list + are allowed to subscribe without administrator approval. Add + addresses one per line. You may begin a line with a ^ character + to designate a (case insensitive) regular expression match.""") + + ' ' + + _("""You may also use the @listname notation to designate the + members of another list in this installation."""))), + ('unsubscribe_policy', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Is the list moderator's approval required for unsubscription requests? (<em>No</em> is recommended)"""), @@ -158,6 +170,11 @@ class Privacy(GUIBase): ] adminurl = mlist.GetScriptURL('admin', absolute=1) + + if mlist.dmarc_quarantine_moderation_action: + quarantine = _('/Quarantine') + else: + quarantine = '' sender_rtn = [ _("""When a message is posted to the list, a series of moderation steps are taken to decide whether a moderator must @@ -235,6 +252,100 @@ class Privacy(GUIBase): >rejection notice</a> to be sent to moderated members who post to this list.""")), + ('dmarc_moderation_action', mm_cfg.Radio, + (_('Accept'), _('Munge From'), _('Wrap Message'), _('Reject'), + _('Discard')), 0, + _("""Action to take when anyone posts to the + list from a domain with a DMARC Reject%(quarantine)s Policy."""), + + _("""<ul><li><b>Munge From</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is_list Munge From</a> + transformation to these messages. + + <p><li><b>Wrap Message</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is_list Wrap + Message</a> transformation to these messages. + + <p><li><b>Reject</b> -- this automatically rejects the message by + sending a bounce notice to the post's author. The text of the + bounce notice can be <a + href="?VARHELP=privacy/sender/dmarc_moderation_notice" + >configured by you</a>. + + <p><li><b>Discard</b> -- this simply discards the message, with + no notice sent to the post's author. + </ul> + + <p>This setting takes precedence over the <a + href="?VARHELP=general/from_is_list"> from_is_list</a> setting + if the message is From: an affected domain and the setting is + other than Accept.""")), + + ('dmarc_quarantine_moderation_action', mm_cfg.Radio, + (_('No'), _('Yes')), 0, + _("""Shall the above dmarc_moderation_action apply to messages + From: domains with DMARC p=quarantine as well as p=reject"""), + + _("""<ul><li><b>No</b> -- this applies dmarc_moderation_action to + only those posts From: a domain with DMARC p=reject. This is + appropriate if you are concerned about bounced messages, but + want to apply dmarc_moderation_action to as few messages as + possible. + <p><li><b>Yes</b> -- this applies dmarc_moderation_action to + posts From: a domain with DMARC p=reject or p=quarantine. + </ul><p>If a message is From: a domain with DMARC p=quarantine + and dmarc_moderation_action is not applied (this set to No) + the message will likely not bounce, but will be delivered to + recipients' spam folders or other hard to find places.""")), + + ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1, + _("""Text to include in any + <a href="?VARHELP=privacy/sender/dmarc_moderation_action" + >rejection notice</a> to + be sent to anyone who posts to this list from a domain + with a DMARC Reject%(quarantine)s Policy.""")), + + ('dmarc_wrapped_message_text', mm_cfg.Text, (10, WIDTH), 1, + _("""If dmarc_moderation_action applies and is Wrap Message, + and this text is provided, the text will be placed in a + separate text/plain MIME part preceding the original message + part in the wrapped message."""), + + _("""A wrapped message will either be a multipart/mixed message + with up to four sub-parts; a text/plain part containing + msg_header, a text/plain part containing + dmarc_wrapped_message_text, a message/rfc822 part containing the + original message and a text/plain part containing msg_footer, or + a message/rfc822 message containing only the original message if + none of the other parts are applicable.""")), + + ('equivalent_domains', mm_cfg.Text, (10, WIDTH), 1, + _("""A 'two dimensional' list of email address domains which are + considered equivalent when checking if a post is from a list + member."""), + + _("""If two poster addresses with the same local part but + different domains are to be considered equivalents for list + membership tests, the domains are put here. The format is + one or more groups of equivalent domains. Within a group, + the domains are separated by commas and multiple groups are + separated by semicolons. White space is ignored. + <p>For example:<pre> + example.com,mail.example.com;mac.com,me.com,icloud.com + </pre> + <p>In this example, if user@example.com is a list member, + a post from user@mail.example.com will be treated as if it is + from user@example.com for list membership/moderation purposes, + and likewise, if user@me.com is a list member, posts from + user@mac.com or user@icloud.com will be treated as if from + user@me.com. + <p>Note that the poster's address is first tested for list + membership, and the equivalent domain addresses are only tested + if the poster's address is not that of a member. + <p>Also note that moderation of the equivalent domain address + will apply to the post, but other options such as 'ack' or + 'not metoo' will not.""")), + _('Non-member filters'), ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1, @@ -399,7 +510,7 @@ class Privacy(GUIBase): case, each rule is matched in turn, with processing stopped after the first match. - Note that headers are collected from all the attachments + Note that headers are collected from all the attachments (except for the mailman administrivia message) and matched against the regular expressions. With this feature, you can effectively sort out messages with dangerous file @@ -442,6 +553,11 @@ class Privacy(GUIBase): # an option. if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE: val += 1 + if (property == 'dmarc_moderation_action' and + val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION): + doc.addError(_("""dmarc_moderation_action must be >= the configured + default value.""")) + val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION setattr(mlist, property, val) # We need to handle the header_filter_rules widgets specially, but diff --git a/Mailman/Gui/Topics.py b/Mailman/Gui/Topics.py index 96f9b421..ec60dbda 100644 --- a/Mailman/Gui/Topics.py +++ b/Mailman/Gui/Topics.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -29,6 +29,8 @@ except NameError: True = 1 False = 0 +OR = '|' + class Topics(GUIBase): @@ -126,10 +128,10 @@ class Topics(GUIBase): # Make sure the pattern was a legal regular expression name = Utils.websafe(name) try: - # Tagger compiles in verbose mode so we do too. - re.compile(pattern, re.VERBOSE) + orpattern = OR.join(pattern.splitlines()) + re.compile(orpattern) except (re.error, TypeError): - safepattern = Utils.websafe(pattern) + safepattern = Utils.websafe(orpattern) doc.addError(_("""The topic pattern '%(safepattern)s' is not a legal regular expression. It will be discarded.""")) continue diff --git a/Mailman/HTMLFormatter.py b/Mailman/HTMLFormatter.py index dad51e74..901e7e58 100644 --- a/Mailman/HTMLFormatter.py +++ b/Mailman/HTMLFormatter.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -90,6 +90,9 @@ class HTMLFormatter: showing = Utils.ObscureEmail(person, for_text=1) else: showing = person + realname = Utils.uncanonstr(self.getMemberName(person), lang) + if realname and mm_cfg.ROSTER_DISPLAY_REALNAME: + showing += " (%s)" % Utils.websafe(realname) got = Link(url, showing) if self.getDeliveryStatus(person) <> MemberAdaptor.ENABLED: got = Italic('(', got, ')') diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py index 038034c7..549d8e79 100644 --- a/Mailman/Handlers/AvoidDuplicates.py +++ b/Mailman/Handlers/AvoidDuplicates.py @@ -24,6 +24,7 @@ warning header, or pass it through, depending on the user's preferences. from email.Utils import getaddresses, formataddr from Mailman import mm_cfg +from Mailman.Handlers.CookHeaders import change_header COMMASPACE = ', ' @@ -95,6 +96,10 @@ def process(mlist, msg, msgdata): # Set the new list of recipients msgdata['recips'] = newrecips # RFC 2822 specifies zero or one CC header - del msg['cc'] if ccaddrs: - msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]) + change_header('Cc', + COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]), + mlist, msg, msgdata) + else: + del msg['cc'] + diff --git a/Mailman/Handlers/CalcRecips.py b/Mailman/Handlers/CalcRecips.py index db10c432..069c88a8 100644..100755 --- a/Mailman/Handlers/CalcRecips.py +++ b/Mailman/Handlers/CalcRecips.py @@ -63,7 +63,8 @@ def process(mlist, msg, msgdata): missing = [] password = msg.get('urgent', missing) if password is not missing: - if mlist.Authenticate((mm_cfg.AuthListModerator, + if mlist.Authenticate((mm_cfg.AuthListPoster, + mm_cfg.AuthListModerator, mm_cfg.AuthListAdmin), password): recips = mlist.getMemberCPAddresses(mlist.getRegularMemberKeys() + @@ -179,6 +180,18 @@ def do_exclude(mlist, msg, msgdata, recips): syslog('error', 'Exclude list %s is not in the same domain.', listname) continue + if mlist.regular_exclude_ignore: + for sender in msg.get_senders(): + if slist.isMember(sender): + break + for sender in Utils.check_eq_domains(sender, + slist.equivalent_domains): + if slist.isMember(sender): + break + if slist.isMember(sender): + break + else: + continue srecips = set([slist.getMemberCPAddress(m) for m in slist.getRegularMemberKeys() if slist.getDeliveryStatus(m) == ENABLED]) diff --git a/Mailman/Handlers/Cleanse.py b/Mailman/Handlers/Cleanse.py index 725cb41b..c684cd19 100644 --- a/Mailman/Handlers/Cleanse.py +++ b/Mailman/Handlers/Cleanse.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -19,12 +19,34 @@ import re -from email.Utils import formataddr +from email.Utils import formataddr, getaddresses, parseaddr +from Mailman import mm_cfg from Mailman.Utils import unique_message_id from Mailman.Logging.Syslog import syslog from Mailman.Handlers.CookHeaders import uheader +cres = [] +for regexp in mm_cfg.ANONYMOUS_LIST_KEEP_HEADERS: + try: + if regexp.endswith(':'): + regexp = regexp[:-1] + '$' + cres.append(re.compile(regexp, re.IGNORECASE)) + except re.error, e: + syslog('error', + 'ANONYMOUS_LIST_KEEP_HEADERS: ignored bad regexp %s: %s', + regexp, e) + +def remove_nonkeepers(msg): + for hdr in msg.keys(): + keep = False + for cre in cres: + if cre.search(hdr): + keep = True + break + if not keep: + del msg[hdr] + def process(mlist, msg, msgdata): # Always remove this header from any outgoing messages. Be sure to do @@ -45,6 +67,7 @@ def process(mlist, msg, msgdata): del msg['from'] del msg['reply-to'] del msg['sender'] + del msg['organization'] del msg['return-path'] # Hotmail sets this one del msg['x-originating-email'] @@ -53,6 +76,10 @@ def process(mlist, msg, msgdata): # And so can the message-id so replace it. del msg['message-id'] msg['Message-ID'] = unique_message_id(mlist) + # And something sets this + del msg['x-envelope-from'] + # And now remove all but the keepers. + remove_nonkeepers(msg) i18ndesc = str(uheader(mlist, mlist.description, 'From')) msg['From'] = formataddr((i18ndesc, mlist.GetListEmail())) msg['Reply-To'] = mlist.GetListEmail() diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py index c4b06613..2cfb07af 100644 --- a/Mailman/Handlers/CleanseDKIM.py +++ b/Mailman/Handlers/CleanseDKIM.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006-2007 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -29,8 +29,21 @@ from Mailman import mm_cfg def process(mlist, msg, msgdata): - if mm_cfg.REMOVE_DKIM_HEADERS: - del msg['domainkey-signature'] - del msg['dkim-signature'] - del msg['authentication-results'] + if not (mm_cfg.REMOVE_DKIM_HEADERS or mlist.anonymous_list): + # We want to remove these headers from posts to anonymous lists. + # There can be interaction with the next test, but anonymous_list + # and Munge From are not compatible anyway, so don't worry. + return + if (mm_cfg.REMOVE_DKIM_HEADERS == 1 and not + # The following means 'Munge From' applies to this message. + # So this whole stanza means if RDH is 1 and we're not Munging, + # return and don't remove the headers. See Defaults.py. + (msgdata.get('from_is_list') == 1 or + (mlist.from_is_list == 1 and msgdata.get('from_is_list') != 2) + ) + ): + return + del msg['domainkey-signature'] + del msg['dkim-signature'] + del msg['authentication-results'] diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py index 904f5c3e..4e005dc5 100644..100755 --- a/Mailman/Handlers/CookHeaders.py +++ b/Mailman/Handlers/CookHeaders.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,7 +15,10 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -"""Cook a message's Subject header.""" +"""Cook a message's Subject header. +Also do other manipulations of From:, Reply-To: and Cc: depending on +list configuration. +""" from __future__ import nested_scopes import re @@ -31,7 +34,7 @@ from Mailman import Utils from Mailman.i18n import _ from Mailman.Logging.Syslog import syslog -CONTINUATION = ',\n\t' +CONTINUATION = ',\n ' COMMASPACE = ', ' MAXLINELEN = 78 @@ -49,7 +52,7 @@ def _isunicode(s): nonascii = re.compile('[^\s!-~]') -def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): +def uheader(mlist, s, header_name=None, continuation_ws=' ', maxlinelen=None): # Get the charset to encode the string in. Then search if there is any # non-ascii character is in the string. If there is and the charset is # us-ascii then we use iso-8859-1 instead. If the string is ascii only @@ -62,15 +65,35 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): else: # there is no nonascii so ... charset = 'us-ascii' - return Header(s, charset, maxlinelen, header_name, continuation_ws) + try: + return Header(s, charset, maxlinelen, header_name, continuation_ws) + except UnicodeError: + syslog('error', 'list: %s: can\'t decode "%s" as %s', + mlist.internal_name(), s, charset) + return Header('', charset, maxlinelen, header_name, continuation_ws) + +def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True): + if ((msgdata.get('from_is_list') == 2 or + (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and + not msgdata.get('_fasttrack') + ) or name.lower() in ('from', 'reply-to', 'cc'): + # The or name.lower() in ... above is because when we are munging + # the From:, we want to defer the resultant changes to From:, + # Reply-To:, and/or Cc: until after the message passes through + # ToDigest, ToArchive and ToUsenet. Thus, we put them in + # msgdata[add_header] here and apply them in WrapMessage. + msgdata.setdefault('add_header', {})[name] = value + elif repl or not msg.has_key(name): + if delete: + del msg[name] + msg[name] = value def process(mlist, msg, msgdata): # Set the "X-Ack: no" header if noack flag is set. if msgdata.get('noack'): - del msg['x-ack'] - msg['X-Ack'] = 'no' + change_header('X-Ack', 'no', mlist, msg, msgdata) # Because we're going to modify various important headers in the email # message, we want to save some of the information in the msgdata # dictionary for later. Specifically, the sender header will get waxed, @@ -87,7 +110,8 @@ def process(mlist, msg, msgdata): pass # Mark message so we know we've been here, but leave any existing # X-BeenThere's intact. - msg['X-BeenThere'] = mlist.GetListEmail() + change_header('X-BeenThere', mlist.GetListEmail(), + mlist, msg, msgdata, delete=False) # Add Precedence: and other useful headers. None of these are standard # and finding information on some of them are fairly difficult. Some are # just common practice, and we'll add more here as they become necessary. @@ -101,12 +125,40 @@ def process(mlist, msg, msgdata): # known exploits in a particular version of Mailman and we know a site is # using such an old version, they may be vulnerable. It's too easy to # edit the code to add a configuration variable to handle this. - if not msg.has_key('x-mailman-version'): - msg['X-Mailman-Version'] = mm_cfg.VERSION + change_header('X-Mailman-Version', mm_cfg.VERSION, + mlist, msg, msgdata, repl=False) # We set "Precedence: list" because this is the recommendation from the # sendmail docs, the most authoritative source of this header's semantics. - if not msg.has_key('precedence'): - msg['Precedence'] = 'list' + change_header('Precedence', 'list', + mlist, msg, msgdata, repl=False) + # Do we change the from so the list takes ownership of the email + if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack: + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if not realname: + if mlist.isMember(email): + realname = mlist.getMemberName(email) or email + else: + realname = email + # Remove domain from realname if it looks like an email address + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + change_header('From', + formataddr(('%s via %s' % (realname, mlist.real_name), + mlist.GetListEmail())), + mlist, msg, msgdata) + else: + # Use this as a flag + o_from = None # Reply-To: munging. Do not do this if the message is "fast tracked", # meaning it is internally crafted and delivered to a specific user. BAW: # Yuck, I really hate this feature but I've caved under the sheer pressure @@ -115,6 +167,23 @@ def process(mlist, msg, msgdata): # augment it. RFC 2822 allows max one Reply-To: header so collapse them # if we're adding a value, otherwise don't touch it. (Should we collapse # in all cases?) + # MAS: We need to do some things with the original From: if we've munged + # it for DMARC mitigation. We have goals for this process which are + # not completely compatible, so we do the best we can. Our goals are: + # 1) as long as the list is not anonymous, the original From: address + # should be obviously exposed, i.e. not just in a header that MUAs + # don't display. + # 2) the original From: address should not be in a comment or display + # name in the new From: because it is claimed that multiple domains + # in any fields in From: are indicative of spamminess. This means + # it should be in Reply-To: or Cc:. + # 3) the behavior of an MUA doing a 'reply' or 'reply all' should be + # consistent regardless of whether or not the From: is munged. + # Goal 3) implies sometimes the original From: should be in Reply-To: + # and sometimes in Cc:, and even so, this goal won't be achieved in + # all cases with all MUAs. In cases of conflict, the above ordering of + # goals is priority order. + if not fasttrack: # A convenience function, requires nested scopes. pair is (name, addr) new = [] @@ -132,22 +201,43 @@ def process(mlist, msg, msgdata): # the original Reply-To:'s to the list we're building up. In both # cases we'll zap the existing field because RFC 2822 says max one is # allowed. + o_rt = False if not mlist.first_strip_reply_to: orig = msg.get_all('reply-to', []) for pair in getaddresses(orig): + # There's an original Reply-To: and we're not removing it. add(pair) + o_rt = True + # We also need to put the old From: in Reply-To: in all cases where + # it is not going in Cc:. This is when reply_goes_to_list == 0 and + # either there was no original Reply-To: or we stripped it. + # However, if there was an original Reply-To:, unstripped, and it + # contained the original From: address we need to flag that it's + # there so we don't add the original From: to Cc: + if o_from and mlist.reply_goes_to_list == 0: + if o_rt: + if d.has_key(o_from[1].lower()): + # Original From: address is in original Reply-To:. + # Pretend we added it. + o_from = None + else: + add(o_from) + # Flag that we added it. + o_from = None # Set Reply-To: header to point back to this list. Add this last # because some folks think that some MUAs make it easier to delete # addresses from the right than from the left. if mlist.reply_goes_to_list == 1: i18ndesc = uheader(mlist, mlist.description, 'Reply-To') add((str(i18ndesc), mlist.GetListEmail())) - del msg['reply-to'] # Don't put Reply-To: back if there's nothing to add! if new: # Preserve order - msg['Reply-To'] = COMMASPACE.join( - [formataddr(pair) for pair in new]) + change_header('Reply-To', + COMMASPACE.join([formataddr(pair) for pair in new]), + mlist, msg, msgdata) + else: + del msg['reply-to'] # The To field normally contains the list posting address. However # when messages are fully personalized, that header will get # overwritten with the address of the recipient. We need to get the @@ -158,18 +248,38 @@ def process(mlist, msg, msgdata): # above code? # Also skip Cc if this is an anonymous list as list posting address # is already in From and Reply-To in this case. - if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \ - and not mlist.anonymous_list: + # We do add the Cc in cases where From: header munging is being done + # because even though the list address is in From:, the Reply-To: + # poster will override it. Brain dead MUAs may then address the list + # twice on a 'reply all', but reasonable MUAs should do the right + # thing. We also add the original From: to Cc: if it wasn't added + # to Reply-To: + add_list = (mlist.personalize == 2 and + mlist.reply_goes_to_list <> 1 and + not mlist.anonymous_list) + if add_list or o_from: # Watch out for existing Cc headers, merge, and remove dups. Note # that RFC 2822 says only zero or one Cc header is allowed. new = [] d = {} - for pair in getaddresses(msg.get_all('cc', [])): - add(pair) - i18ndesc = uheader(mlist, mlist.description, 'Cc') - add((str(i18ndesc), mlist.GetListEmail())) - del msg['Cc'] - msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) + # If we're adding the original From:, add it first. + if o_from: + add(o_from) + # AvoidDuplicates may have set a new Cc: in msgdata.add_header, + # so check that. + if (msgdata.has_key('add_header') and + msgdata['add_header'].has_key('Cc')): + for pair in getaddresses([msgdata['add_header']['Cc']]): + add(pair) + else: + for pair in getaddresses(msg.get_all('cc', [])): + add(pair) + if add_list: + i18ndesc = uheader(mlist, mlist.description, 'Cc') + add((str(i18ndesc), mlist.GetListEmail())) + change_header('Cc', + COMMASPACE.join([formataddr(pair) for pair in new]), + mlist, msg, msgdata) # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only # if the message is being crafted for a specific list (e.g. not for the # password reminders). @@ -191,8 +301,7 @@ def process(mlist, msg, msgdata): # without desc we need to ensure the MUST brackets listid_h = '<%s>' % listid # We always add a List-ID: header. - del msg['list-id'] - msg['List-Id'] = listid_h + change_header('List-Id', listid_h, mlist, msg, msgdata) # For internally crafted messages, we also add a (nonstandard), # "X-List-Administrivia: yes" header. For all others (i.e. those coming # from list posts), we add a bunch of other RFC 2369 headers. @@ -215,19 +324,16 @@ def process(mlist, msg, msgdata): # Add this header if we're archiving if mlist.archive: archiveurl = mlist.GetBaseArchiveURL() - if archiveurl.endswith('/'): - archiveurl = archiveurl[:-1] headers['List-Archive'] = '<%s>' % archiveurl # First we delete any pre-existing headers because the RFC permits only # one copy of each, and we want to be sure it's ours. for h, v in headers.items(): - del msg[h] # Wrap these lines if they are too long. 78 character width probably # shouldn't be hardcoded, but is at least text-MUA friendly. The # adding of 2 is for the colon-space separator. if len(h) + 2 + len(v) > 78: v = CONTINUATION.join(v.split(', ')) - msg[h] = v + change_header(h, v, mlist, msg, msgdata) @@ -244,7 +350,7 @@ def prefix_subject(mlist, msg, msgdata): lines = str(subject).splitlines() else: lines = subject.splitlines() - ws = '\t' + ws = ' ' if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': ws = lines[1][0] msgdata['origsubj'] = subject @@ -274,7 +380,13 @@ def prefix_subject(mlist, msg, msgdata): else: old_style = mm_cfg.OLD_STYLE_PREFIXING subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)\s*(\[\d+\])?\s*:\s*)+', subject, re.I) + # Previously the following re didn't have the first \s*. It would fail + # if the incoming Subject: was like '[prefix] Re: Re: Re:' because of the + # leading space after stripping the prefix. It is not known what MUA would + # create such a Subject:, but the issue was reported. + rematch = re.match( + '(\s*(RE|AW|SV|VS)\s*(\[\d+\])?\s*:\s*)+', + subject, re.I) if rematch: subject = subject[rematch.end():] recolon = 'Re:' @@ -304,8 +416,7 @@ def prefix_subject(mlist, msg, msgdata): h = u' '.join([prefix, subject]) h = h.encode('us-ascii') h = uheader(mlist, h, 'Subject', continuation_ws=ws) - del msg['subject'] - msg['Subject'] = h + change_header('Subject', h, mlist, msg, msgdata) ss = u' '.join([recolon, subject]) ss = ss.encode('us-ascii') ss = uheader(mlist, ss, 'Subject', continuation_ws=ws) @@ -314,6 +425,11 @@ def prefix_subject(mlist, msg, msgdata): except UnicodeError: pass # Get the header as a Header instance, with proper unicode conversion + # Because of rfc2047 encoding, spaces between encoded words can be + # insignificant, so we need to append a space to prefix but only when + # we have Re:. + if recolon: + prefix += ' ' if old_style: h = uheader(mlist, recolon, 'Subject', continuation_ws=ws) h.append(prefix) @@ -323,8 +439,7 @@ def prefix_subject(mlist, msg, msgdata): # TK: Subject is concatenated and unicode string. subject = subject.encode(cset, 'replace') h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h + change_header('Subject', h, mlist, msg, msgdata) ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) ss.append(subject, cset) msgdata['stripped_subject'] = ss diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py index 199c97ac..49ed1d7e 100644 --- a/Mailman/Handlers/Moderate.py +++ b/Mailman/Handlers/Moderate.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,6 +21,7 @@ import re from email.MIMEMessage import MIMEMessage from email.MIMEText import MIMEText +from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Utils @@ -47,12 +48,18 @@ class ModeratedMemberPost(Hold.ModeratedPost): def process(mlist, msg, msgdata): - if msgdata.get('approved') or msgdata.get('fromusenet'): + if msgdata.get('approved'): return - # First of all, is the poster a member or not? + # Is the poster a member or not? for sender in msg.get_senders(): if mlist.isMember(sender): break + for sender in Utils.check_eq_domains(sender, + mlist.equivalent_domains): + if mlist.isMember(sender): + break + if mlist.isMember(sender): + break else: sender = None if sender: @@ -90,22 +97,34 @@ def process(mlist, msg, msgdata): sender = msg.get_sender() # From here on out, we're dealing with non-members. listname = mlist.internal_name() - if matches_p(sender, mlist.accept_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.accept_these_nonmembers, + at_list='accept_these_nonmembers' + ): return - if matches_p(sender, mlist.hold_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.hold_these_nonmembers, + at_list='hold_these_nonmembers' + ): Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost) # No return - if matches_p(sender, mlist.reject_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.reject_these_nonmembers, + at_list='reject_these_nonmembers' + ): do_reject(mlist) # No return - if matches_p(sender, mlist.discard_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.discard_these_nonmembers, + at_list='discard_these_nonmembers' + ): do_discard(mlist, msg) # No return # Okay, so the sender wasn't specified explicitly by any of the non-member # moderation configuration variables. Handle by way of generic non-member # action. assert 0 <= mlist.generic_nonmember_action <= 4 - if mlist.generic_nonmember_action == 0: + if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'): # Accept return elif mlist.generic_nonmember_action == 1: @@ -117,43 +136,6 @@ def process(mlist, msg, msgdata): -def matches_p(sender, nonmembers, listname): - # First strip out all the regular expressions and listnames - plainaddrs = [addr for addr in nonmembers if not (addr.startswith('^') - or addr.startswith('@'))] - addrdict = Utils.List2Dict(plainaddrs, foldcase=1) - if addrdict.has_key(sender): - return 1 - # Now do the regular expression matches - for are in nonmembers: - if are.startswith('^'): - try: - cre = re.compile(are, re.IGNORECASE) - except re.error: - continue - if cre.search(sender): - return 1 - elif are.startswith('@'): - # XXX Needs to be reviewed for list@domain names. - try: - mname = are[1:].lower().strip() - if mname == listname: - # don't reference your own list - syslog('error', - '*_these_nonmembers in %s references own list', - listname) - else: - mother = MailList(mname, lock=0) - if mother.isMember(sender): - return 1 - except Errors.MMUnknownListError: - syslog('error', - '*_these_nonmembers in %s references non-existent list %s', - listname, mname) - return 0 - - - def do_reject(mlist): listowner = mlist.GetOwnerEmail() if mlist.nonmember_rejection_notice: @@ -161,9 +143,10 @@ def do_reject(mlist): Utils.wrap(_(mlist.nonmember_rejection_notice)) else: raise Errors.RejectMessage, Utils.wrap(_("""\ -You are not allowed to post to this mailing list, and your message has been -automatically rejected. If you think that your messages are being rejected in -error, contact the mailing list owner at %(listowner)s.""")) +Your message has been rejected, probably because you are not subscribed to the +mailing list and the list's policy is to prohibit non-members from posting to +it. If you think that your messages are being rejected in error, contact the +mailing list owner at %(listowner)s.""")) diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py index 8d26da03..d85cc6a6 100644 --- a/Mailman/Handlers/SpamDetect.py +++ b/Mailman/Handlers/SpamDetect.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,12 +27,14 @@ TBD: This needs to be made more configurable and robust. import re +from email.Errors import HeaderParseError from email.Header import decode_header +from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Errors from Mailman import i18n -from Mailman.Utils import GetCharSet +from Mailman import Utils from Mailman.Handlers.Hold import hold_for_approval try: @@ -68,17 +70,58 @@ def getDecodedHeaders(msg, cset='utf-8'): headers = '' for h, v in msg.items(): uvalue = u'' - v = decode_header(re.sub('\n\s', ' ', v)) + try: + v = decode_header(re.sub('\n\s', ' ', v)) + except HeaderParseError: + v = [(v, 'us-ascii')] for frag, cs in v: if not cs: cs = 'us-ascii' - uvalue += unicode(frag, cs, 'replace') + try: + uvalue += unicode(frag, cs, 'replace') + except LookupError: + # The encoding charset is unknown. At this point, frag + # has been QP or base64 decoded into a byte string whose + # charset we don't know how to handle. We will try to + # unicode it as iso-8859-1 which may result in a garbled + # mess, but we have to do something. + uvalue += unicode(frag, 'iso-8859-1', 'replace') headers += '%s: %s\n' % (h, uvalue.encode(cset, 'replace')) return headers def process(mlist, msg, msgdata): + # Before anything else, check DMARC if necessary. We do this as early + # as possible so reject/discard actions trump other holds/approvals and + # wrap/munge actions get flagged even for approved messages. + # But not for owner mail which should not be subject to DMARC reject or + # discard actions. + if not msgdata.get('toowner'): + msgdata['from_is_list'] = 0 + dn, addr = parseaddr(msg.get('from')) + if addr and mlist.dmarc_moderation_action > 0: + if Utils.IsDMARCProhibited(mlist, addr): + # Note that for dmarc_moderation_action, 0 = Accept, + # 1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard + if mlist.dmarc_moderation_action == 1: + msgdata['from_is_list'] = 1 + elif mlist.dmarc_moderation_action == 2: + msgdata['from_is_list'] = 2 + elif mlist.dmarc_moderation_action == 3: + # Reject + text = mlist.dmarc_moderation_notice + if text: + text = Utils.wrap(text) + else: + text = Utils.wrap(_( +"""You are not allowed to post to this mailing list From: a domain which +publishes a DMARC policy of reject or quarantine, and your message has been +automatically rejected. If you think that your messages are being rejected in +error, contact the mailing list owner at %(listowner)s.""")) + raise Errors.RejectMessage, text + elif mlist.dmarc_moderation_action == 4: + raise Errors.DiscardMessage if msgdata.get('approved'): return # First do site hard coded header spam checks @@ -94,7 +137,7 @@ def process(mlist, msg, msgdata): # extension may be a clue to possible virus/spam. headers = '' # Get the character set of the lists preferred language for headers - lcset = GetCharSet(mlist.preferred_language) + lcset = Utils.GetCharSet(mlist.preferred_language) for p in msg.walk(): headers += getDecodedHeaders(p, lcset) for patterns, action, empty in mlist.header_filter_rules: diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py index 38a8e465..ed9a7e71 100644 --- a/Mailman/Handlers/Tagger.py +++ b/Mailman/Handlers/Tagger.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,7 +27,9 @@ from email.Header import decode_header from Mailman import Utils from Mailman.Logging.Syslog import syslog +from Mailman.Handlers.CookHeaders import change_header +OR = '|' CRNL = '\r\n' EMPTYSTRING = '' NLTAB = '\n\t' @@ -62,15 +64,17 @@ def process(mlist, msg, msgdata): # added to the specific topics bucket. hits = {} for name, pattern, desc, emptyflag in mlist.topics: - cre = re.compile(pattern, re.IGNORECASE | re.VERBOSE) + pattern = OR.join(pattern.splitlines()) + cre = re.compile(pattern, re.IGNORECASE) for line in matchlines: if cre.search(line): hits[name] = 1 break if hits: msgdata['topichits'] = hits.keys() - msg['X-Topics'] = NLTAB.join(hits.keys()) - + change_header('X-Topics', NLTAB.join(hits.keys()), + mlist, msg, msgdata, delete=False) + def scanbody(msg, numlines=None): diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py index edbf40dc..2027a46c 100644 --- a/Mailman/Handlers/ToDigest.py +++ b/Mailman/Handlers/ToDigest.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -86,7 +86,8 @@ def process(mlist, msg, msgdata): # whether the size threshold has been reached. mboxfp.flush() size = os.path.getsize(mboxfile) - if size / 1024.0 >= mlist.digest_size_threshhold: + if (mlist.digest_size_threshhold > 0 and + size / 1024.0 >= mlist.digest_size_threshhold): # This is a bit of a kludge to get the mbox file moved to the digest # queue directory. try: diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py new file mode 100644 index 00000000..83f521fb --- /dev/null +++ b/Mailman/Handlers/WrapMessage.py @@ -0,0 +1,88 @@ +# Copyright (C) 2013-2015 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Wrap the message in an outer message/rfc822 part and transfer/add +some headers from the original. + +Also, in the case of Munge From, replace the From:, Reply-To: and Cc: in the +original message. +""" + +import copy + +from email.MIMEMessage import MIMEMessage +from email.MIMEText import MIMEText + +from Mailman import Utils + +# Headers from the original that we want to keep in the wrapper. +KEEPERS = ('to', + 'in-reply-to', + 'references', + 'x-mailman-approved-at', + ) + + + +def process(mlist, msg, msgdata): + # This is the negation of we're wrapping because dmarc_moderation_action + # is wrap this message or from_is_list applies and is wrap. + if not (msgdata.get('from_is_list') == 2 or + (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)): + # Now see if we need to add a From:, Reply-To: or Cc: without wrapping. + # See comments in CookHeaders.change_header for why we do this here. + a_h = msgdata.get('add_header') + if a_h: + if a_h.get('From'): + del msg['from'] + msg['From'] = a_h.get('From') + if a_h.get('Reply-To'): + del msg['reply-to'] + msg['Reply-To'] = a_h.get('Reply-To') + if a_h.get('Cc'): + del msg['cc'] + msg['Cc'] = a_h.get('Cc') + return + + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg.keys(): + if key.lower() not in KEEPERS: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = Utils.unique_message_id(mlist) + # Add the headers from CookHeaders. + for k, v in msgdata['add_header'].items(): + msg[k] = v + # Are we including dmarc_wrapped_message_text? I.e., do we have text and + # are we wrapping because of dmarc_moderation_action? + if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2: + part1 = MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text), + 'plain', + Utils.GetCharSet(mlist.preferred_language)) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + diff --git a/Mailman/ListAdmin.py b/Mailman/ListAdmin.py index 4a188057..877aa5c4 100755 --- a/Mailman/ListAdmin.py +++ b/Mailman/ListAdmin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -101,7 +101,7 @@ class ListAdmin: # should we be as paranoid as for the config.pck file? Should we # use pickle? tmpfile = self.__filename + '.tmp' - omask = os.umask(002) + omask = os.umask(007) try: fp = open(tmpfile, 'w') try: @@ -194,7 +194,7 @@ class ListAdmin: else: ext = 'txt' filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext) - omask = os.umask(002) + omask = os.umask(007) try: fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w') try: @@ -243,7 +243,11 @@ class ListAdmin: if e.errno <> errno.ENOENT: raise return LOST try: - msg = cPickle.load(fp) + if path.endswith('.pck'): + msg = cPickle.load(fp) + else: + assert path.endswith('.txt'), '%s not .pck or .txt' % path + msg = fp.read() finally: fp.close() # Save the plain text to a .msg file, not a .pck file @@ -252,8 +256,11 @@ class ListAdmin: outpath = head + '.msg' outfp = open(outpath, 'w') try: - g = Generator(outfp) - g.flatten(msg, 1) + if path.endswith('.pck'): + g = Generator(outfp) + g.flatten(msg, 1) + else: + outfp.write(msg) finally: outfp.close() # Now handle updates to the database @@ -285,7 +292,8 @@ class ListAdmin: # message directly here can lead to a huge delay in web # turnaround. Log the moderation and add a header. msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1) - syslog('vette', 'held message approved, message-id: %s', + syslog('vette', '%s: held message approved, message-id: %s', + self.internal_name(), msg.get('message-id', 'n/a')) # Stick the message back in the incoming queue for further # processing. @@ -390,6 +398,7 @@ class ListAdmin: self.internal_name(), addr) # Possibly notify the administrator in default list language if self.admin_immed_notify: + i18n.set_language(self.preferred_language) realname = self.real_name subject = _( 'New subscription request to list %(realname)s from %(addr)s') @@ -406,6 +415,8 @@ class ListAdmin: msg = Message.UserNotification(owneraddr, owneraddr, subject, text, self.preferred_language) msg.send(self, **{'tomoderators': 1}) + # Restore the user's preferred language. + i18n.set_language(lang) def __handlesubscription(self, record, value, comment): stime, addr, fullname, password, digest, lang = record diff --git a/Mailman/Logging/Logger.py b/Mailman/Logging/Logger.py index 617347d4..f3d30164 100644 --- a/Mailman/Logging/Logger.py +++ b/Mailman/Logging/Logger.py @@ -60,7 +60,7 @@ class Logger: return self.__fp else: try: - ou = os.umask(002) + ou = os.umask(007) try: try: f = codecs.open( diff --git a/Mailman/Logging/Syslog.py b/Mailman/Logging/Syslog.py index 531ab1d7..81e27214 100644 --- a/Mailman/Logging/Syslog.py +++ b/Mailman/Logging/Syslog.py @@ -62,6 +62,10 @@ class _Syslog: logf.write(msg + '\n') except UnicodeError: # Python 2.4 may fail to write 8bit (non-ascii) characters + # Also, if msg is unicode with non-ascii, quopri.encodestring() + # will throw UnicodeEncodeError, so avoid that. + if isinstance(msg, unicode): + msg = msg.encode('iso-8859-1', 'replace') logf.write(quopri.encodestring(msg) + '\n') # For the ultimate in convenience diff --git a/Mailman/MTA/Postfix.py b/Mailman/MTA/Postfix.py index 801ddc0f..3f5c9984 100644 --- a/Mailman/MTA/Postfix.py +++ b/Mailman/MTA/Postfix.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -116,6 +116,10 @@ def _addlist(mlist, fp): +def _isvirtual(mlist): + return (mlist and mlist.host_name.lower() in + [d.lower() for d in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS]) + def _addvirtual(mlist, fp): listname = mlist.internal_name() fieldsz = len(listname) + len('-unsubscribe') @@ -123,10 +127,13 @@ def _addvirtual(mlist, fp): # Set up the mailman-loop address loopaddr = Utils.get_site_email(mlist.host_name, extra='loop') loopdest = Utils.ParseEmail(loopaddr)[0] + # And the site list posting address. + siteaddr = Utils.get_site_email(mlist.host_name) + sitedest = Utils.ParseEmail(siteaddr)[0] if mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN: loopdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN # Seek to the end of the text file, but if it's empty write the standard - # disclaimer, and the loop catch address. + # disclaimer, and the loop catch address and site address. fp.seek(0, 2) if not fp.tell(): print >> fp, """\ @@ -141,7 +148,13 @@ def _addvirtual(mlist, fp): # LOOP ADDRESSES START %s\t%s # LOOP ADDRESSES END -""" % (loopaddr, loopdest) + +# We also add the site list address in each virtual domain as that address +# is exposed on admin and listinfo overviews. +# SITE ADDRESSES START +%s\t%s +# SITE ADDRESSES END +""" % (loopaddr, loopdest, siteaddr, sitedest) # The text file entries get a little extra info print >> fp, '# STANZA START:', listname print >> fp, '# CREATED:', time.ctime(time.time()) @@ -164,6 +177,8 @@ def _addvirtual(mlist, fp): def _check_for_virtual_loopaddr(mlist, filename): loopaddr = Utils.get_site_email(mlist.host_name, extra='loop') loopdest = Utils.ParseEmail(loopaddr)[0] + siteaddr = Utils.get_site_email(mlist.host_name) + sitedest = Utils.ParseEmail(siteaddr)[0] infp = open(filename) omask = os.umask(007) try: @@ -196,6 +211,32 @@ def _check_for_virtual_loopaddr(mlist, filename): else: # This isn't our loop address, so spit it out and continue outfp.write(line) + # Now do it all again for the site list address. It must follow the + # loop addresses. + while True: + line = infp.readline() + if not line: + break + outfp.write(line) + if line.startswith('# SITE ADDRESSES START'): + break + # Now see if our domain has already been written + while True: + line = infp.readline() + if not line: + break + if line.startswith('# SITE ADDRESSES END'): + # It hasn't + print >> outfp, '%s\t%s' % (siteaddr, sitedest) + outfp.write(line) + break + elif line.startswith(siteaddr): + # We just found it + outfp.write(line) + break + else: + # This isn't our loop address, so spit it out and continue + outfp.write(line) outfp.writelines(infp.readlines()) finally: infp.close() @@ -233,7 +274,7 @@ def create(mlist, cgi=False, nolock=False, quiet=False): # Do the aliases file, which need to be done in any case try: _do_create(mlist, ALIASFILE, _addlist) - if mlist and mlist.host_name in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS: + if _isvirtual(mlist): _do_create(mlist, VIRTFILE, _addvirtual) # bin/genaliases is the only one that calls create with nolock = True. # Use that to only update the maps at the end of genaliases. @@ -304,7 +345,7 @@ def remove(mlist, cgi=False): lock.lock() try: _do_remove(mlist, ALIASFILE, False) - if mlist.host_name in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS: + if _isvirtual(mlist): _do_remove(mlist, VIRTFILE, True) # Regenerate the alias and map files _update_maps() diff --git a/Mailman/MailList.py b/Mailman/MailList.py index 63a5b81e..0d47d812 100644..100755 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -74,6 +74,7 @@ from Mailman.Logging.Syslog import syslog _ = i18n._ EMPTYSTRING = '' +OR = '|' try: True, False @@ -339,6 +340,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.umbrella_member_suffix = \ mm_cfg.DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX self.regular_exclude_lists = mm_cfg.DEFAULT_REGULAR_EXCLUDE_LISTS + self.regular_exclude_ignore = mm_cfg.DEFAULT_REGULAR_EXCLUDE_IGNORE self.regular_include_lists = mm_cfg.DEFAULT_REGULAR_INCLUDE_LISTS self.send_reminders = mm_cfg.DEFAULT_SEND_REMINDERS self.send_welcome_msg = mm_cfg.DEFAULT_SEND_WELCOME_MSG @@ -346,6 +348,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.bounce_matching_headers = \ mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS self.header_filter_rules = [] + self.from_is_list = mm_cfg.DEFAULT_FROM_IS_LIST self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST internalname = self.internal_name() self.real_name = internalname[0].upper() + internalname[1:] @@ -354,6 +357,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.welcome_msg = '' self.goodbye_msg = '' self.subscribe_policy = mm_cfg.DEFAULT_SUBSCRIBE_POLICY + self.subscribe_auto_approval = mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL self.unsubscribe_policy = mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY self.private_roster = mm_cfg.DEFAULT_PRIVATE_ROSTER self.obscure_addresses = mm_cfg.DEFAULT_OBSCURE_ADDRESSES @@ -387,6 +391,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # 2==Discard self.member_moderation_action = 0 self.member_moderation_notice = '' + self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION + self.dmarc_quarantine_moderation_action = ( + mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION) + self.dmarc_moderation_notice = '' + self.dmarc_wrapped_message_text = ( + mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT) + self.equivalent_domains = ( + mm_cfg.DEFAULT_EQUIVALENT_DOMAINS) self.accept_these_nonmembers = [] self.hold_these_nonmembers = [] self.reject_these_nonmembers = [] @@ -767,10 +779,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, goodtopics = [] for name, pattern, desc, emptyflag in self.topics: try: - re.compile(pattern) + orpattern = OR.join(pattern.splitlines()) + re.compile(orpattern) except (re.error, TypeError): syslog('error', 'Bad topic pattern "%s" for list: %s', - pattern, self.internal_name()) + orpattern, self.internal_name()) else: goodtopics.append((name, pattern, desc, emptyflag)) self.topics = goodtopics @@ -819,6 +832,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, subj = self.GetConfirmJoinSubject(listname, cookie) del msg['subject'] msg['Subject'] = subj + del msg['auto-submitted'] + msg['Auto-Submitted'] = 'auto-generated' msg.send(self) def AddMember(self, userdesc, remote=None): @@ -868,8 +883,12 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Is the subscribing address banned from this list? pattern = self.GetBannedPattern(email) if pattern: - syslog('vette', '%s banned subscription: %s (matched: %s)', - realname, email, pattern) + if remote: + whence = ' from %s' % remote + else: + whence = '' + syslog('vette', '%s banned subscription: %s%s (matched: %s)', + realname, email, whence, pattern) raise Errors.MembershipIsBanned, pattern # Sanity check the digest flag if digest and not self.digestable: @@ -920,11 +939,22 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, del msg['subject'] msg['Subject'] = self.GetConfirmJoinSubject(realname, cookie) msg['Reply-To'] = self.GetRequestEmail(cookie) + # Is this confirmation a reply to an email subscribe from this + # address? + if remote.lower().endswith(email.lower()): + autosub = 'auto-replied' + else: + autosub = 'auto-generated' + del msg['auto-submitted'] + msg['Auto-Submitted'] = autosub msg.send(self) who = formataddr((name, email)) syslog('subscribe', '%s: pending %s %s', self.internal_name(), who, by) raise Errors.MMSubscribeNeedsConfirmation + elif self.HasAutoApprovedSender(email): + # no approval necessary: + self.ApprovedAddMember(userdesc) else: # Subscription approval is required. Add this entry to the admin # requests database. BAW: this should probably take a userdesc @@ -1035,7 +1065,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # And send an acknowledgement to the user... if userack: self.SendUnsubscribeAck(emailaddr, userlang) - # ...and to the administrator + # ...and to the administrator in the correct language. (LP: #1308655) + i18n.set_language(self.preferred_language) if admin_notif: realname = self.real_name subject = _('%(realname)s unsubscribe notification') @@ -1147,12 +1178,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # CP address of a member, then if the old address yields a different # CP address, we can simply remove the old address, otherwise we can # do nothing. + cpoldaddr = self.getMemberCPAddress(oldaddr) if self.isMember(newaddr) and (self.getMemberCPAddress(newaddr) == newaddr): - if self.getMemberCPAddress(oldaddr) <> newaddr: + if cpoldaddr <> newaddr: self.removeMember(oldaddr) else: self.changeMemberAddress(oldaddr, newaddr) + self.log_and_notify_admin(cpoldaddr, newaddr) # If globally is true, then we also include every list for which # oldaddr is a member. if not globally: @@ -1172,16 +1205,46 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, mlist.Lock() try: # Same logic as above, re newaddr is already a member + cpoldaddr = mlist.getMemberCPAddress(oldaddr) if mlist.isMember(newaddr) and ( mlist.getMemberCPAddress(newaddr) == newaddr): - if mlist.getMemberCPAddress(oldaddr) <> newaddr: + if cpoldaddr <> newaddr: mlist.removeMember(oldaddr) else: mlist.changeMemberAddress(oldaddr, newaddr) + mlist.log_and_notify_admin(cpoldaddr, newaddr) mlist.Save() finally: mlist.Unlock() + def log_and_notify_admin(self, oldaddr, newaddr): + """Log member address change and notify admin if requested.""" + syslog('subscribe', '%s: changed member address from %s to %s', + self.internal_name(), oldaddr, newaddr) + if self.admin_notify_mchanges: + lang = self.preferred_language + otrans = i18n.get_translation() + i18n.set_language(lang) + try: + realname = self.real_name + subject = _('%(realname)s address change notification') + finally: + i18n.set_translation(otrans) + name = self.getMemberName(newaddr) + if name is None: + name = '' + if isinstance(name, UnicodeType): + name = name.encode(Utils.GetCharSet(lang), 'replace') + text = Utils.maketext( + 'adminaddrchgack.txt', + {'name' : name, + 'oldaddr' : oldaddr, + 'newaddr' : newaddr, + 'listname': self.real_name, + }, mlist=self) + msg = Message.OwnerNotification(self, subject, text) + msg.send(self) + # # Confirmation processing @@ -1225,7 +1288,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # list administrators. self.SendHostileSubscriptionNotice(invitation, addr) raise Errors.HostileSubscriptionError - elif self.subscribe_policy in (2, 3): + elif self.subscribe_policy in (2, 3) and \ + not self.HasAutoApprovedSender(addr): self.HoldSubscription(addr, fullname, password, digest, lang) name = self.real_name raise Errors.MMNeedApproval, _( @@ -1341,6 +1405,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, del msg['subject'] msg['Subject'] = self.GetConfirmLeaveSubject(realname, cookie) msg['Reply-To'] = self.GetRequestEmail(cookie) + del msg['auto-submitted'] + msg['Auto-Submitted'] = 'auto-generated' msg.send(self) @@ -1502,26 +1568,85 @@ bad regexp in bounce_matching_header line: %s """Returns matched entry in ban_list if email matches. Otherwise returns None. """ - ban = False - for pattern in self.ban_list: + return (self.GetPattern(email, self.ban_list) or + self.GetPattern(email, mm_cfg.GLOBAL_BAN_LIST) + ) + + def HasAutoApprovedSender(self, sender): + """Returns True and logs if sender matches address or pattern + or is a member of a referenced list in subscribe_auto_approval. + Otherwise returns False. + """ + auto_approve = False + if self.GetPattern(sender, + self.subscribe_auto_approval, + at_list='subscribe_auto_approval' + ): + auto_approve = True + syslog('vette', '%s: auto approved subscribe from %s', + self.internal_name(), sender) + return auto_approve + + def GetPattern(self, email, pattern_list, at_list=None): + """Returns matched entry in pattern_list if email matches. + Otherwise returns None. The at_list argument, if "true", + says process the @listname syntax and provides the name of + the list attribute for log messages. + """ + matched = None + # First strip out all the regular expressions and listnames because + # documentation says we do non-regexp first (Why?). + plainaddrs = [x.strip() for x in pattern_list if x.strip() and not + (x.startswith('^') or x.startswith('@'))] + addrdict = Utils.List2Dict(plainaddrs, foldcase=1) + if addrdict.has_key(email.lower()): + return email + for pattern in pattern_list: if pattern.startswith('^'): # This is a regular expression match try: if re.search(pattern, email, re.IGNORECASE): - ban = True + matched = pattern break - except re.error: + except re.error, e: # BAW: we should probably remove this pattern - pass - else: - # Do the comparison case insensitively - if pattern.lower() == email.lower(): - ban = True + # The GUI won't add a bad regexp, but at least log it. + # The following kludge works because the ban_list stuff + # is the only caller with no at_list. + attr_name = at_list or 'ban_list' + syslog('error', + '%s in %s has bad regexp "%s": %s', + attr_name, + self.internal_name(), + pattern, + str(e) + ) + elif at_list and pattern.startswith('@'): + # XXX Needs to be reviewed for list@domain names. + # this refers to the members of another list in this + # installation. + mname = pattern[1:].lower().strip() + if mname == self.internal_name(): + # don't reference your own list + syslog('error', + '%s in %s references own list', + at_list, + self.internal_name()) + continue + try: + mother = MailList(mname, lock = False) + except Errors.MMUnknownListError: + syslog('error', + '%s in %s references non-existent list %s', + at_list, + self.internal_name(), + mname + ) + continue + if mother.isMember(email.lower()): + matched = pattern break - if ban: - return pattern - else: - return None + return matched diff --git a/Mailman/Mailbox.py b/Mailman/Mailbox.py index a7e8cced..a8fa4d0b 100644 --- a/Mailman/Mailbox.py +++ b/Mailman/Mailbox.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -110,3 +110,11 @@ class ArchiverMailbox(Mailbox): return self._scrubber(self._mlist, msg) else: return msg + + def skipping(self, flag): + """ This method allows the archiver to skip over messages without + scrubbing attachments into the attachments directory.""" + if flag: + self.factory = _safeparser + else: + self.factory = _archfactory(self) diff --git a/Mailman/Message.py b/Mailman/Message.py index 24c15a84..5d68e7ef 100644 --- a/Mailman/Message.py +++ b/Mailman/Message.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -284,8 +284,13 @@ class UserNotification(Message): # UserNotifications are typically for admin messages, and for messages # other than list explosions. Send these out as Precedence: bulk, but # don't override an existing Precedence: header. + # Also, if the message is To: the list-owner address, set Precedence: + # list. See note below in OwnerNotification. if not (self.has_key('precedence') or noprecedence): - self['Precedence'] = 'bulk' + if self.get('to') == mlist.GetOwnerEmail(): + self['Precedence'] = 'list' + else: + self['Precedence'] = 'bulk' self._enqueue(mlist, **_kws) def _enqueue(self, mlist, **_kws): @@ -318,6 +323,12 @@ class OwnerNotification(UserNotification): del self['to'] self['To'] = mlist.GetOwnerEmail() self._sender = sender + # User notifications are normally sent with Precedence: bulk. This + # is appropriate as they can be backscatter of rejected spam. + # Owner notifications are not backscatter and are perhaps more + # important than 'bulk' so give them Precedence: list by default. + # (LP: #1313146) + self['Precedence'] = 'list' def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index d219d6e9..651039d6 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -30,6 +30,7 @@ from Mailman import mm_cfg from Mailman import Utils from Mailman import LockFile from Mailman.Message import UserNotification +from Mailman.Bouncer import _BounceInfo from Mailman.Bouncers import BouncerAPI from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard @@ -151,6 +152,14 @@ class BounceMixin: try: op, addr, bmsg = mlist.pend_confirm(token) info = mlist.getBounceInfo(addr) + if not info: + # info was deleted before probe bounce was received. + # Just create a new info. + info = _BounceInfo(addr, + 0.0, + time.localtime()[:3], + mlist.bounce_you_are_disabled_warnings + ) mlist.disableBouncingMember(addr, info, bmsg) # Only save the list if we're unlocking it if not locked: @@ -244,6 +253,7 @@ class BounceRunner(Runner, BounceMixin): return # If that still didn't return us any useful addresses, then send it on # or discard it. + addrs = filter(None, addrs) if not addrs: syslog('bounce', '%s: bounce message w/no discernable addresses: %s', @@ -254,7 +264,8 @@ class BounceRunner(Runner, BounceMixin): # BAW: It's possible that there are None's in the list of addresses, # although I'm unsure how that could happen. Possibly ScanMessages() # can let None's sneak through. In any event, this will kill them. - addrs = filter(None, addrs) + # addrs = filter(None, addrs) + # MAS above filter moved up so we don't try to queue an empty list. self._queue_bounces(mlist.internal_name(), addrs, msg) _doperiodic = BounceMixin._doperiodic diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index b63b050c..c5cc3d94 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """-request robot command queue runner.""" @@ -132,22 +133,34 @@ class Results: __import__(modname) handler = sys.modules[modname] # ValueError can be raised if cmd has dots in it. - except (ImportError, ValueError): + # and KeyError if cmd is otherwise good but ends with a dot. + except (ImportError, ValueError, KeyError): # If we're on line zero, it was the Subject: header that didn't # contain a command. It's possible there's a Re: prefix (or # localized version thereof) on the Subject: line that's messing # things up. Pop the prefix off and try again... once. # + # At least one MUA (163.com web mail) has been observed that + # inserts 'Re:' with no following space, so try to account for + # that too. + # # If that still didn't work it isn't enough to stop processing. # BAW: should we include a message that the Subject: was ignored? - if not self.subjcmdretried and args: + # + # But first, be sure we're looking at the Subject: and not past + # it already. + if self.lineno != 0: + return BADCMD + if self.subjcmdretried < 1: + self.subjcmdretried += 1 + if re.search('^.*:.+', cmd): + cmd = re.sub('.*:', '', cmd).lower() + return self.do_command(cmd, args) + if self.subjcmdretried < 2 and args: self.subjcmdretried += 1 - cmd = args.pop(0) + cmd = args.pop(0).lower() return self.do_command(cmd, args) - if self.lineno <> 0: - return BADCMD - else: - return BADSUBJ + return BADSUBJ if handler.process(self, args): return STOP else: diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py index 71d93919..2c6c2815 100644 --- a/Mailman/Queue/IncomingRunner.py +++ b/Mailman/Queue/IncomingRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -157,14 +157,32 @@ class IncomingRunner(Runner): os._exit(1) except Errors.DiscardMessage: # Throw the message away; we need do nothing else with it. - syslog('vette', 'Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) + # We do need to push the current handler back in the pipeline + # just in case the syslog call throws an exception and the + # message is shunted. + pipeline.insert(0, handler) + syslog('vette', """Message discarded, msgid: %s' + list: %s, + handler: %s""", + msg.get('message-id', 'n/a'), + mlist.real_name, handler) return 0 except Errors.HoldMessage: # Let the approval process take it from here. The message no # longer needs to be queued. return 0 except Errors.RejectMessage, e: + # Log this. + # We do need to push the current handler back in the pipeline + # just in case the syslog call or BounceMessage throws an + # exception and the message is shunted. + pipeline.insert(0, handler) + syslog('vette', """Message rejected, msgid: %s + list: %s, + handler: %s, + reason: %s""", + msg.get('message-id', 'n/a'), + mlist.real_name, handler, e.notice()) mlist.BounceMessage(msg, msgdata, e) return 0 except: diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py index 44850063..449532fb 100644 --- a/Mailman/Queue/NewsRunner.py +++ b/Mailman/Queue/NewsRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2005 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -111,20 +111,12 @@ def prepare_message(mlist, msg, msgdata): del msg['subject'] msg['subject'] = stripped_subject # Add the appropriate Newsgroups: header - ngheader = msg['newsgroups'] - if ngheader is not None: - # See if the Newsgroups: header already contains our linked_newsgroup. - # If so, don't add it again. If not, append our linked_newsgroup to - # the end of the header list - ngroups = [s.strip() for s in ngheader.split(',')] - if mlist.linked_newsgroup not in ngroups: - ngroups.append(mlist.linked_newsgroup) - # Subtitute our new header for the old one. - del msg['newsgroups'] - msg['Newsgroups'] = COMMASPACE.join(ngroups) - else: - # Newsgroups: isn't in the message - msg['Newsgroups'] = mlist.linked_newsgroup + if msg['newsgroups'] is not None: + # This message is gated from our list to it's associated usnet group. + # If it has a Newsgroups: header mentioning other groups, it's not + # up to us to post it to those groups. + del msg['newsgroups'] + msg['Newsgroups'] = mlist.linked_newsgroup # Note: We need to be sure two messages aren't ever sent to the same list # in the same process, since message ids need to be unique. Further, if # messages are crossposted to two Usenet-gated mailing lists, they each diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py index 0f51b663..defaf02d 100644..100755 --- a/Mailman/Queue/OutgoingRunner.py +++ b/Mailman/Queue/OutgoingRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2007 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2012 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -89,6 +89,7 @@ class OutgoingRunner(Runner, BounceMixin): syslog('error', 'Cannot connect to SMTP server %s on port %s', mm_cfg.SMTPHOST, port) self.__logged = True + self._snooze(0) return True except Errors.SomeRecipientsFailed, e: # Handle local rejects of probe messages differently. diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index bd1cd357..a2c31263 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -184,8 +184,8 @@ class Switchboard: else: os.unlink(bakfile) except EnvironmentError, e: - syslog('error', 'Failed to unlink/preserve backup file: %s', - bakfile) + syslog('error', 'Failed to unlink/preserve backup file: %s\n%s', + bakfile, e) def files(self, extension='.pck'): times = {} diff --git a/Mailman/SecurityManager.py b/Mailman/SecurityManager.py index 4f6aa34a..7ca4e084 100644 --- a/Mailman/SecurityManager.py +++ b/Mailman/SecurityManager.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -319,8 +319,6 @@ class SecurityManager: for u in usernames]: ok = self.__checkone(c, authcontext, user) if ok: - # Refresh the cookie - print self.MakeCookie(authcontext, user) return True return False else: @@ -362,6 +360,8 @@ class SecurityManager: if mac <> received_mac: return False # Authenticated! + # Refresh the cookie + print self.MakeCookie(authcontext, user) return True diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 93e1fba1..f22e45b4 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -71,8 +71,16 @@ except NameError: True = 1 False = 0 +try: + import dns.resolver + from dns.exception import DNSException + dns_resolver = True +except ImportError: + dns_resolver = False + EMPTYSTRING = '' UEMPTYSTRING = u'' +CR = '\r' NL = '\n' DOT = '.' IDENTCHARS = ascii_letters + digits + '_' @@ -92,6 +100,12 @@ def list_exists(listname): # # The former two are for 2.1alpha3 and beyond, while the latter two are # for all earlier versions. + # + # But first ensure the list name doesn't contain a path traversal + # attack. + if len(re.sub(mm_cfg.ACCEPTABLE_LISTNAME_CHARACTERS, '', listname)) > 0: + syslog('mischief', 'Hostile listname: %s', listname) + return False basepath = Site.get_listpath(listname) for ext in ('.pck', '.pck.last', '.db', '.db.last'): dbfile = os.path.join(basepath, 'config' + ext) @@ -220,10 +234,18 @@ _valid_domain = re.compile('[-a-z0-9]', re.IGNORECASE) def ValidateEmail(s): """Verify that an email address isn't grossly evil.""" + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(s, list) or isinstance(s, tuple): + if len(s) == 0: + s = '' + else: + s = s[-1] # Pretty minimal, cheesy check. We could do better... if not s or s.count(' ') > 0: raise Errors.MMBadEmailError - if _badchars.search(s) or s[0] == '-': + if _badchars.search(s): raise Errors.MMHostileAddress, s user, domain_parts = ParseEmail(s) # This means local, unqualified addresses, are not allowed @@ -232,8 +254,9 @@ def ValidateEmail(s): if len(domain_parts) < 2: raise Errors.MMBadEmailError, s # domain parts may only contain ascii letters, digits and hyphen + # and must not begin with hyphen. for p in domain_parts: - if len(_valid_domain.sub('', p)) > 0: + if len(p) == 0 or p[0] == '-' or len(_valid_domain.sub('', p)) > 0: raise Errors.MMHostileAddress, s @@ -247,12 +270,24 @@ def GetPathPieces(envar='PATH_INFO'): if path: if CRNLpat.search(path): path = CRNLpat.split(path)[0] - syslog('error', 'Warning: Possible malformed path attack.') + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) + syslog('error', + 'Warning: Possible malformed path attack domain=%s remote=%s', + get_domain(), + remote) return [p for p in path.split('/') if p] return None +def GetRequestMethod(): + return os.environ.get('REQUEST_METHOD') + + + def ScriptURL(target, web_page_url=None, absolute=False): """target - scriptname only, nothing extra web_page_url - the list's configvar of the same name @@ -427,6 +462,14 @@ def check_global_password(response, siteadmin=True): _ampre = re.compile('&((?:#[0-9]+|[a-z]+);)', re.IGNORECASE) def websafe(s): + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(s, list) or isinstance(s, tuple): + if len(s) == 0: + s = '' + else: + s = s[-1] if mm_cfg.BROKEN_BROWSER_WORKAROUND: # Archiver can pass unicode here. Just skip them as the # archiver escapes non-ascii anyway. @@ -905,6 +948,61 @@ def oneline(s, cset): return EMPTYSTRING.join(s.splitlines()) +def strip_verbose_pattern(pattern): + # Remove white space and comments from a verbose pattern and return a + # non-verbose, equivalent pattern. Replace CR and NL in the result + # with '\\r' and '\\n' respectively to avoid multi-line results. + if not isinstance(pattern, str): + return pattern + newpattern = '' + i = 0 + inclass = False + skiptoeol = False + copynext = False + while i < len(pattern): + c = pattern[i] + if copynext: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + copynext = False + elif skiptoeol: + if c == NL: + skiptoeol = False + elif c == '#' and not inclass: + skiptoeol = True + elif c == '[' and not inclass: + inclass = True + newpattern += c + copynext = True + elif c == ']' and inclass: + inclass = False + newpattern += c + elif re.search('\s', c): + if inclass: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + elif c == '\\' and not inclass: + newpattern += c + copynext = True + else: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + i += 1 + return newpattern + + # Patterns and functions to flag possible XSS attacks in HTML. # This list is compiled from information at http://ha.ckers.org/xss.html, # http://www.quirksmode.org/js/events_compinfo.html, @@ -1057,3 +1155,127 @@ def suspiciousHTML(html): else: return False + +# This takes an email address, and returns True if DMARC policy is p=reject +# or possibly quarantine. +def IsDMARCProhibited(mlist, email): + if not dns_resolver: + # This is a problem; log it. + syslog('error', + 'DNS lookup for dmarc_moderation_action for list %s not available', + mlist.real_name) + return False + + email = email.lower() + at_sign = email.find('@') + if at_sign < 1: + return False + dmarc_domain = '_dmarc.' + email[at_sign+1:] + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = float(mm_cfg.DMARC_RESOLVER_TIMEOUT) + resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME) + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return False + except DNSException, e: + syslog('error', + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__class__) + return False + else: +# people are already being dumb, don't trust them to provide honest DNS +# where the answer section only contains what was asked for, nor to include +# CNAMEs before the values they point to. + full_record = "" + results_by_name = {} + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name.setdefault(txt_rec.name.to_text(), []).append( + "".join(txt_rec.items[0].strings)) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + syslog('error', + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = filter(lambda n: n.startswith('v=DMARC1;'), + results_by_name[name]) + if len(dmarcs) == 0: + return False + if len(dmarcs) > 1: + syslog('error', + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarc)) + for entry in dmarcs: + if re.search(r'\bp=reject\b', entry, re.IGNORECASE): + syslog('vette', + '%s: DMARC lookup for %s (%s) found p=reject in %s = %s', + mlist.real_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_quarantine_moderation_action and + re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)): + syslog('vette', + '%s: DMARC lookup for %s (%s) found p=quarantine in %s = %s', + mlist.real_name, email, dmarc_domain, name, entry) + return True + + return False + + +def check_eq_domains(email, domains_list): + """The arguments are an email address and a string representing a + list of lists in a form like 'a,b,c;1,2' representing [['a', 'b', + 'c'],['1', '2']]. The inner lists are domains which are + equivalent in some sense. The return is an empty list or a list + of email addresses equivalent to the first argument. + For example, given + + email = 'user@me.com' + domains_list = '''domain1, domain2; mac.com, me.com, icloud.com; + domaina, domainb + ''' + + check_eq_domains(email, domains_list) will return + ['user@mac.com', 'user@icloud.com'] + """ + if not domains_list: + return [] + try: + local, domain = email.rsplit('@', 1) + except ValueError: + return [] + domain = domain.lower() + domains_list = re.sub('\s', '', domains_list).lower() + domains = domains_list.split(';') + domains_list = [] + for d in domains: + domains_list.append(d.split(',')) + for domains in domains_list: + if domain in domains: + return [local + '@' + x for x in domains if x != domain] + return [] + diff --git a/Mailman/Version.py b/Mailman/Version.py index 54d11484..cdc8a043 100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -16,7 +16,7 @@ # USA. # Mailman version -VERSION = '2.1.14' +VERSION = '2.1.20' # And as a hex number in the manner of PY_VERSION_HEX ALPHA = 0xa @@ -28,7 +28,7 @@ FINAL = 0xf MAJOR_REV = 2 MINOR_REV = 1 -MICRO_REV = 14 +MICRO_REV = 20 REL_LEVEL = FINAL # at most 15 beta releases! REL_SERIAL = 0 @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) # config.pck schema version number -DATA_FILE_VERSION = 99 +DATA_FILE_VERSION = 108 # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 diff --git a/Mailman/htmlformat.py b/Mailman/htmlformat.py index 5d70ad28..419fa296 100644..100755 --- a/Mailman/htmlformat.py +++ b/Mailman/htmlformat.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -32,7 +32,7 @@ import types from Mailman import mm_cfg from Mailman import Utils -from Mailman.i18n import _ +from Mailman.i18n import _, get_translation from Mailman.CSRFcheck import csrf_token @@ -300,7 +300,7 @@ class Document(Container): def Format(self, indent=0, **kws): charset = 'us-ascii' - if self.language: + if self.language and Utils.IsLanguage(self.language): charset = Utils.GetCharSet(self.language) output = ['Content-Type: text/html; charset=%s\n' % charset] if not self.suppress_head: @@ -318,6 +318,8 @@ class Document(Container): 'content="text/html; charset=%s">' % charset) if self.title: output.append('%s<TITLE>%s</TITLE>' % (tab, self.title)) + if mm_cfg.WEB_HEAD_ADD: + output.append(mm_cfg.WEB_HEAD_ADD) output.append('%s</HEAD>' % tab) quals = [] # Default link colors @@ -441,6 +443,7 @@ class InputObj: self.kws = kws def Format(self, indent=0): + charset = get_translation().charset() or 'us-ascii' output = ['<INPUT name="%s" type="%s" value="%s"' % (self.name, self.type, self.value)] for item in self.kws.items(): @@ -448,7 +451,10 @@ class InputObj: if self.checked: output.append('CHECKED') output.append('>') - return SPACE.join(output) + ret = SPACE.join(output) + if self.type == 'TEXT' and isinstance(ret, unicode): + ret = ret.encode(charset, 'replace') + return ret class SubmitButton(InputObj): @@ -486,6 +492,7 @@ class TextArea: self.readonly = readonly def Format(self, indent=0): + charset = get_translation().charset() or 'us-ascii' output = '<TEXTAREA NAME=%s' % self.name if self.rows: output += ' ROWS=%s' % self.rows @@ -496,6 +503,8 @@ class TextArea: if self.readonly: output += ' READONLY' output += '>%s</TEXTAREA>' % self.text + if isinstance(output, unicode): + output = output.encode(charset, 'replace') return output class FileUpload(InputObj): @@ -540,7 +549,9 @@ class WidgetArray: self.button_names, self.values): ischecked = (self.ischecked(i)) - item = self.Widget(self.name, value, ischecked).Format() + name + item = ('<label>' + + self.Widget(self.name, value, ischecked).Format() + + name + '</label>') items.append(item) if not self.horizontal: t.AddRow(items) diff --git a/Mailman/versions.py b/Mailman/versions.py index 8de192cd..da33dc10 100644..100755 --- a/Mailman/versions.py +++ b/Mailman/versions.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -93,6 +93,81 @@ def UpdateOldVars(l, stored_state): if not hasattr(l, newname) and newdefault is not uniqueval: setattr(l, newname, newdefault) + def recode(mlist, f, t): + """If the character set for a list's preferred_language has changed, + attempt to recode old string values into the new character set. + + mlist is the list, f is the old charset and t is the new charset. + """ + for x in dir(mlist): + if x.startswith('_'): + continue + nv = doitem(getattr(mlist, x), f, t) + if nv: + setattr(mlist, x, nv) + + def doitem(v, f, t): + """Recursively process lists, tuples and dictionary values and + convert strings as needed. Return either the updated item or None + if no change.""" + changed = False + if isinstance(v, str): + return convert(v, f, t) + elif isinstance(v, list): + for i in range(len(v)): + nv = doitem(v[i], f, t) + if nv: + changed = True + v[i] = nv + if changed: + return v + else: + return None + elif isinstance(v, tuple): + nt = () + for i in range(len(v)): + nv = doitem(v[i], f, t) + if nv: + changed = True + nt += (nv,) + else: + nt += (v[i],) + if changed: + return nt + else: + return None + elif isinstance(v, dict): + for k, ov in v.items(): + nv = doitem(ov, f, t) + if nv: + changed = True + v[k] = nv + if changed: + return v + else: + return None + else: + return None + + def convert(s, f, t): + """This does the actual character set conversion of the string s + from charset f to charset t.""" + + try: + u = unicode(s, f) + is_f = True + except ValueError: + is_f = False + try: + unicode(s, t) + is_t = True + except ValueError: + is_t = False + if is_f and not is_t: + return u.encode(t, 'replace') + else: + return None + # Migrate to 2.1b3, baw 17-Aug-2001 if hasattr(l, 'dont_respond_to_post_requests'): oldval = getattr(l, 'dont_respond_to_post_requests') @@ -313,6 +388,31 @@ def UpdateOldVars(l, stored_state): pass else: l.digest_members[k] = 0 + # + # Convert pre 2.2 topics regexps which were compiled in verbose mode + # to a non-verbose equivalent. + # + if stored_state['data_version'] < 106 and stored_state.has_key('topics'): + l.topics = [] + for name, pattern, description, emptyflag in stored_state['topics']: + pattern = Utils.strip_verbose_pattern(pattern) + l.topics.append((name, pattern, description, emptyflag)) + # + # Romanian and Russian had their character sets changed in 2.1.19 + # to utf-8. If there are any strings in the old encoding, try to recode + # them. + # + if stored_state['data_version'] < 108: + if l.preferred_language == 'ro': + if Utils.GetCharSet('ro') == 'utf-8': + recode(l, 'iso-8859-2', 'utf-8') + if l.preferred_language == 'ru': + if Utils.GetCharSet('ru') == 'utf-8': + recode(l, 'koi8-r', 'utf-8') + # + # from_is_list was called author_is_list in 2.1.16rc2 (only). + PreferStored('author_is_list', 'from_is_list', + mm_cfg.DEFAULT_FROM_IS_LIST) @@ -349,6 +449,8 @@ def NewVars(l): add_only_if_missing('personalize', 0) add_only_if_missing('first_strip_reply_to', mm_cfg.DEFAULT_FIRST_STRIP_REPLY_TO) + add_only_if_missing('subscribe_auto_approval', + mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL) add_only_if_missing('unsubscribe_policy', mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY) add_only_if_missing('send_goodbye_msg', mm_cfg.DEFAULT_SEND_GOODBYE_MSG) @@ -368,6 +470,9 @@ def NewVars(l): 'bounce_unrecognized_goes_to_list_owner', mm_cfg.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER) add_only_if_missing( + 'bounce_notify_owner_on_bounce_increment', + mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT) + add_only_if_missing( 'bounce_notify_owner_on_disable', mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE) add_only_if_missing( @@ -385,6 +490,15 @@ def NewVars(l): # the current GUI description model. So, 0==Hold, 1==Reject, 2==Discard add_only_if_missing('member_moderation_action', 0) add_only_if_missing('member_moderation_notice', '') + add_only_if_missing('dmarc_moderation_action', + mm_cfg.DEFAULT_DMARC_MODERATION_ACTION) + add_only_if_missing('dmarc_quarantine_moderation_action', + mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION) + add_only_if_missing('dmarc_moderation_notice', '') + add_only_if_missing('dmarc_wrapped_message_text', + mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT) + add_only_if_missing('equivalent_domains', + mm_cfg.DEFAULT_EQUIVALENT_DOMAINS) add_only_if_missing('new_member_options', mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS) # Emergency moderation flag @@ -416,6 +530,8 @@ def NewVars(l): mm_cfg.DEFAULT_REGULAR_EXCLUDE_LISTS) add_only_if_missing('regular_include_lists', mm_cfg.DEFAULT_REGULAR_INCLUDE_LISTS) + add_only_if_missing('regular_exclude_ignore', + mm_cfg.DEFAULT_REGULAR_EXCLUDE_IGNORE) |