diff options
Diffstat (limited to 'Mailman')
64 files changed, 2820 insertions, 484 deletions
diff --git a/Mailman/Archiver/HyperArch.py b/Mailman/Archiver/HyperArch.py index 33a77f0b..0c0e3356 100644 --- a/Mailman/Archiver/HyperArch.py +++ b/Mailman/Archiver/HyperArch.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -226,9 +226,9 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None): Utils.GetCharSet(lang), 'replace') text = sdict.interpolate(utemplate) - except (TypeError, ValueError): + except (TypeError, ValueError), e: # The template is really screwed up - pass + syslog('error', 'broken template: %s\n%s', filepath, e) # Make sure the text is in the given character set, or html-ify any bogus # characters. return Utils.uncanonstr(text, lang) @@ -271,6 +271,8 @@ class Article(pipermail.Article): if result: i = result.end(0) self.subject = self.subject[i:] + if self.subject == '': + self.subject = _('No subject') else: i = -1 # Useful to keep around @@ -471,6 +473,7 @@ class Article(pipermail.Article): d["email_html"] = self.quote(self.email) d["title"] = self.quote(self.subject) d["subject_html"] = self.quote(self.subject) + d["message_id"] = self.quote(self._message_id) # TK: These two _url variables are used to compose a response # from the archive web page. So, ... d["subject_url"] = url_quote('Re: ' + self.subject) @@ -506,7 +509,7 @@ class Article(pipermail.Article): subject = self._get_subject_enc(self.prev) prev = ('<LINK REL="Previous" HREF="%s">' % (url_quote(self.prev.filename))) - prev_wsubj = ('<LI>' + _('Previous message:') + + prev_wsubj = ('<LI>' + _('Previous message (by thread):') + ' <A HREF="%s">%s\n</A></li>' % (url_quote(self.prev.filename), self.quote(subject))) @@ -528,7 +531,7 @@ class Article(pipermail.Article): subject = self._get_subject_enc(self.next) next = ('<LINK REL="Next" HREF="%s">' % (url_quote(self.next.filename))) - next_wsubj = ('<LI>' + _('Next message:') + + next_wsubj = ('<LI>' + _('Next message (by thread):') + ' <A HREF="%s">%s\n</A></li>' % (url_quote(self.next.filename), self.quote(subject))) diff --git a/Mailman/Archiver/pipermail.py b/Mailman/Archiver/pipermail.py index 939602ba..c03d43b3 100644 --- a/Mailman/Archiver/pipermail.py +++ b/Mailman/Archiver/pipermail.py @@ -16,6 +16,7 @@ __version__ = '0.09 (Mailman edition)' VERSION = __version__ CACHESIZE = 100 # Number of slots in the cache +from Mailman import mm_cfg from Mailman import Errors from Mailman.Mailbox import ArchiverMailbox from Mailman.Logging.Syslog import syslog @@ -230,21 +231,30 @@ class Article: self.body = s.readlines() def _set_date(self, message): - def floatdate(header): - missing = [] - datestr = message.get(header, missing) - if datestr is missing: + def floatdate(datestr): + if not datestr: return None date = parsedate_tz(datestr) try: - return mktime_tz(date) + date = mktime_tz(date) + if (date < 0 or + date - time.time() > + mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW + ): + return None + return date except (TypeError, ValueError, OverflowError): return None - date = floatdate('date') + date = floatdate(message.get('date')) + if date is None: + date = floatdate(message.get('x-list-received-date')) + if date is None: + date = floatdate(re.sub(r'^.*;\s*', '', + message.get('received', ''), flags=re.S)) if date is None: - date = floatdate('x-list-received-date') + date = floatdate(re.sub(r'From \s*\S+\s+', '', + message.get_unixfrom() or '' )) if date is None: - # What's left to try? date = self._last_article_time + 1 self._last_article_time = date self.date = '%011i' % date @@ -552,6 +562,8 @@ class T: if start is None: start = 0 counter = 0 + if start: + mbox.skipping(True) while counter < start: try: m = mbox.next() @@ -560,6 +572,8 @@ class T: if m is None: return counter += 1 + if start: + mbox.skipping(False) while 1: try: pos = input.tell() diff --git a/Mailman/Bouncer.py b/Mailman/Bouncer.py index 573b46e4..2a1cb539 100644 --- a/Mailman/Bouncer.py +++ b/Mailman/Bouncer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -89,6 +89,8 @@ class Bouncer: mm_cfg.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL self.bounce_unrecognized_goes_to_list_owner = \ mm_cfg.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER + self.bounce_notify_owner_on_bounce_increment = \ + mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT self.bounce_notify_owner_on_disable = \ mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE self.bounce_notify_owner_on_removal = \ @@ -135,6 +137,7 @@ class Bouncer: siblist.Unlock() return info = self.getBounceInfo(member) + first_today = True if day is None: # Use today's date day = time.localtime()[:3] @@ -155,6 +158,7 @@ class Bouncer: return elif info.date == day: # We've already scored any bounces for this day, so ignore it. + first_today = False syslog('bounce', '%s: %s already scored a bounce for date %s', self.internal_name(), member, time.strftime('%d-%b-%Y', day + (0,0,0,0,1,0))) @@ -189,6 +193,9 @@ class Bouncer: info.reset(0, info.date, info.noticesleft) else: self.disableBouncingMember(member, info, msg) + elif self.bounce_notify_owner_on_bounce_increment and first_today: + self.__sendAdminBounceNotice(member, msg, + did=_('bounce score incremented')) # We've set/changed bounce info above. We now need to tell the # MemberAdaptor to set/update it. We do it here in case the # MemberAdaptor stores bounce info externally to the list object to @@ -218,24 +225,27 @@ class Bouncer: if self.bounce_notify_owner_on_disable: self.__sendAdminBounceNotice(member, msg) - def __sendAdminBounceNotice(self, member, msg): + def __sendAdminBounceNotice(self, member, msg, did=_('disabled')): # BAW: This is a bit kludgey, but we're not providing as much # information in the new admin bounce notices as we used to (some of # it was of dubious value). However, we'll provide empty, strange, or # meaningless strings for the unused %()s fields so that the language # translators don't have to provide new templates. + siteowner = Utils.get_site_email(self.host_name) text = Utils.maketext( 'bounce.txt', {'listname' : self.real_name, 'addr' : member, 'negative' : '', - 'did' : _('disabled'), + 'did' : did, 'but' : '', 'reenable' : '', - 'owneraddr': self.GetOwnerEmail(), + 'owneraddr': siteowner, }, mlist=self) subject = _('Bounce action notification') - umsg = Message.OwnerNotification(self, subject, tomoderators=0) + umsg = Message.UserNotification(self.GetOwnerEmail(), + siteowner, subject, + lang=self.preferred_language) # BAW: Be sure you set the type before trying to attach, or you'll get # a MultipartConversionError. umsg.set_type('multipart/mixed') diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py index 0607ce86..e84d2255 100644 --- a/Mailman/Bouncers/SimpleMatch.py +++ b/Mailman/Bouncers/SimpleMatch.py @@ -42,7 +42,7 @@ PATTERNS = [ # sz-sb.de, corridor.com, nfg.nl (_c('the following addresses had'), _c('transcript of session follows'), - _c(r'<(?P<fulladdr>[^>]*)>|\(expanded from: <?(?P<addr>[^>)]*)>?\)')), + _c(r'^ *(\(expanded from: )?<?(?P<addr>[^\s@]+@[^\s@>]+?)>?\)?\s*$')), # robanal.demon.co.uk (_c('this message was created automatically by mail delivery software'), _c('original message follows'), @@ -184,6 +184,14 @@ PATTERNS = [ _c( 'Your message to (?P<addr>[^\s@]+@[^\s@]+) was automatically rejected' )), + # mail.ru + (_c('A message that you sent was rejected'), + _c('This is a copy of your message'), + _c('\s(?P<addr>[^\s@]+@[^\s@]+)')), + # MailEnable + (_c('Message could not be delivered to some recipients.'), + _c('Message headers follow'), + _c('Recipient: \[SMTP:(?P<addr>[^\s@]+@[^\s@]+)\]')), # Next one goes here... ] diff --git a/Mailman/Bouncers/SimpleWarning.py b/Mailman/Bouncers/SimpleWarning.py index ab8d6aa2..4f5958ea 100644 --- a/Mailman/Bouncers/SimpleWarning.py +++ b/Mailman/Bouncers/SimpleWarning.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -57,6 +57,10 @@ patterns = [ (_c('We will continue to try to deliver'), _c('.+'), _c('(?P<addr>.+)')), + # kundenserver.de + (_c('not yet been delivered'), + _c('No action is required on your part'), + _c(r'\s*<?(?P<addr>\S+@[^>\s]+)>?\s*')), # Next one goes here... ] diff --git a/Mailman/Bouncers/Yahoo.py b/Mailman/Bouncers/Yahoo.py index b3edf4fa..47fedce2 100644 --- a/Mailman/Bouncers/Yahoo.py +++ b/Mailman/Bouncers/Yahoo.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Yahoo! has its own weird format for bounces.""" @@ -20,9 +21,15 @@ import re import email from email.Utils import parseaddr -tcre = re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE) +tcre = (re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE), + re.compile(r'Sorry, we were unable to deliver your message to ' + r'the following address(\(es\))?\.', + re.IGNORECASE), + ) acre = re.compile(r'<(?P<addr>[^>]*)>:') -ecre = re.compile(r'--- Original message follows') +ecre = (re.compile(r'--- Original message follows'), + re.compile(r'--- Below this line is a copy of the message'), + ) @@ -36,18 +43,26 @@ def process(msg): # simple state machine # 0 == nothing seen # 1 == tag line seen + # 2 == end line seen state = 0 for line in email.Iterators.body_line_iterator(msg): line = line.strip() - if state == 0 and tcre.match(line): - state = 1 + if state == 0: + for cre in tcre: + if cre.match(line): + state = 1 + break elif state == 1: mo = acre.match(line) if mo: addrs.append(mo.group('addr')) continue - mo = ecre.match(line) - if mo: - # we're at the end of the error response - break + for cre in ecre: + mo = cre.match(line) + if mo: + # we're at the end of the error response + state = 2 + break + elif state == 2: + break return addrs diff --git a/Mailman/CSRFcheck.py b/Mailman/CSRFcheck.py index a3b6885a..d531ffc2 100644 --- a/Mailman/CSRFcheck.py +++ b/Mailman/CSRFcheck.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 by the Free Software Foundation, Inc. +# Copyright (C) 2011-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -55,8 +55,9 @@ def csrf_check(mlist, token): try: issued, keymac = marshal.loads(binascii.unhexlify(token)) key, received_mac = keymac.split(':', 1) - klist, key = key.split('+', 1) - assert klist == mlist.internal_name() + if not key.startswith(mlist.internal_name() + '+'): + return False + key = key[len(mlist.internal_name()) + 1:] if '+' in key: key, user = key.split('+', 1) else: diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py index b5c19544..41875533 100644 --- a/Mailman/Cgi/admin.py +++ b/Mailman/Cgi/admin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -32,6 +32,7 @@ from email.Utils import unquote, parseaddr, formataddr from Mailman import mm_cfg from Mailman import Utils +from Mailman import Message from Mailman import MailList from Mailman import Errors from Mailman import MemberAdaptor @@ -77,14 +78,26 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' admin_overview(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'admin.py access for non-existent list: %s', - listname) + syslog('error', 'admin: No such list "%s": %s\n', + listname, e) return # Now that we know what list has been requested, all subsequent admin # pages are shown in that list's preferred language. i18n.set_language(mlist.preferred_language) # If the user is not authenticated, we're done. cgidata = cgi.FieldStorage(keep_blank_values=1) + try: + cgidata.getvalue('csrf_token', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc = Document() + doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return # CSRF check safe_params = ['VARHELP', 'adminpw', 'admlogin', @@ -259,7 +272,11 @@ def admin_overview(msg=''): listnames.sort() for name in listnames: - mlist = MailList.MailList(name, lock=0) + try: + mlist = MailList.MailList(name, lock=0) + except Errors.MMUnknownListError: + # The list could have been deleted by another process. + continue if mlist.advertised: if mm_cfg.VIRTUAL_HOST_OVERVIEW and ( mlist.web_page_url.find('/%s/' % hostname) == -1 and @@ -523,7 +540,7 @@ def show_results(mlist, doc, category, subcat, cgidata): if category == 'members': # Figure out which subcategory we should display subcat = Utils.GetPathPieces()[-1] - if subcat not in ('list', 'add', 'remove'): + if subcat not in ('list', 'add', 'remove', 'change'): subcat = 'list' # Add member category specific tables form.AddItem(membership_options(mlist, subcat, cgidata, doc, form)) @@ -877,6 +894,13 @@ def membership_options(mlist, subcat, cgidata, doc, form): container.AddItem(header) mass_remove(mlist, container) return container + if subcat == 'change': + header.AddRow([Center(Header(2, _('Address Change')))]) + header.AddCellInfo(header.GetCurrentRowIndex(), 0, colspan=2, + bgcolor=mm_cfg.WEB_HEADER_COLOR) + container.AddItem(header) + address_change(mlist, container) + return container # Otherwise... header.AddRow([Center(Header(2, _('Membership List')))]) header.AddCellInfo(header.GetCurrentRowIndex(), 0, colspan=2, @@ -903,6 +927,15 @@ def membership_options(mlist, subcat, cgidata, doc, form): all.sort(lambda x, y: cmp(x.lower(), y.lower())) # See if the query has a regular expression regexp = cgidata.getvalue('findmember', '').strip() + try: + regexp = regexp.decode(Utils.GetCharSet(mlist.preferred_language)) + except UnicodeDecodeError: + # This is probably a non-ascii character and an English language + # (ascii) list. Even if we didn't throw the UnicodeDecodeError, + # the input may have contained mnemonic or numeric HTML entites mixed + # with other characters. Trying to grok the real meaning out of that + # is complex and error prone, so we don't try. + pass if regexp: try: cre = re.compile(regexp, re.IGNORECASE) @@ -977,6 +1010,9 @@ def membership_options(mlist, subcat, cgidata, doc, form): if regexp: findfrag = '&findmember=' + urllib.quote(regexp) url = adminurl + '/members?letter=' + letter + findfrag + if isinstance(url, unicode): + url = url.encode(Utils.GetCharSet(mlist.preferred_language), + errors='ignore') if letter == bucket: show = Bold('[%s]' % letter.upper()).Format() else: @@ -1152,7 +1188,12 @@ def membership_options(mlist, subcat, cgidata, doc, form): continue start = chunkmembers[i*chunksz] end = chunkmembers[min((i+1)*chunksz, last)-1] - link = Link(url + 'chunk=%d' % i, _('from %(start)s to %(end)s')) + thisurl = url + 'chunk=%d' % i + findfrag + if isinstance(thisurl, unicode): + thisurl = thisurl.encode( + Utils.GetCharSet(mlist.preferred_language), + errors='ignore') + link = Link(thisurl, _('from %(start)s to %(end)s')) buttons.append(link) buttons = UnorderedList(*buttons) container.AddItem(footer + buttons.Format() + '<p>') @@ -1168,12 +1209,13 @@ def mass_subscribe(mlist, container): Label(_('Subscribe these users now or invite them?')), RadioButtonArray('subscribe_or_invite', (_('Subscribe'), _('Invite')), - 0, values=(0, 1)) + mm_cfg.DEFAULT_SUBSCRIBE_OR_INVITE, + values=(0, 1)) ]) table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) table.AddRow([ - Label(_('Send welcome messages to new subscribees?')), + Label(_('Send welcome messages to new subscribers?')), RadioButtonArray('send_welcome_msg_to_this_batch', (_('No'), _('Yes')), mlist.send_welcome_msg, @@ -1242,6 +1284,38 @@ def mass_remove(mlist, container): +def address_change(mlist, container): + # ADDRESS CHANGE + GREY = mm_cfg.WEB_ADMINITEM_COLOR + table = Table(width='90%') + table.AddRow([Italic(_("""To change a list member's address, enter the + member's current and new addresses below. Use the check boxes to send + notice of the change to the old and/or new address(es)."""))]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, colspan=3) + table.AddRow([ + Label(_("Member's current address")), + TextBox(name='change_from'), + CheckBox('notice_old', 'yes', 0).Format() + + ' ' + + _('Send notice') + ]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 2, bgcolor=GREY) + table.AddRow([ + Label(_('Address to change to')), + TextBox(name='change_to'), + CheckBox('notice_new', 'yes', 0).Format() + + ' ' + + _('Send notice') + ]) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 1, bgcolor=GREY) + table.AddCellInfo(table.GetCurrentRowIndex(), 2, bgcolor=GREY) + container.AddItem(Center(table)) + + + def password_inputs(mlist): adminurl = mlist.GetScriptURL('admin', absolute=1) table = Table(cellspacing=3, cellpadding=4) @@ -1464,6 +1538,74 @@ def change_options(mlist, category, subcat, cgidata, doc): color='#ff0000', size='+2')).Format())) doc.AddItem(UnorderedList(*unsubscribe_errors)) doc.AddItem('<p>') + # Address Changes + if cgidata.has_key('change_from'): + change_from = cgidata.getvalue('change_from', '') + change_to = cgidata.getvalue('change_to', '') + schange_from = Utils.websafe(change_from) + schange_to = Utils.websafe(change_to) + success = False + msg = None + if not (change_from and change_to): + msg = _('You must provide both current and new addresses.') + elif change_from == change_to: + msg = _('Current and new addresses must be different.') + elif mlist.isMember(change_to): + # ApprovedChangeMemberAddress will just delete the old address + # and we don't want that here. + msg = _('%(schange_to)s is already a list member.') + else: + try: + Utils.ValidateEmail(change_to) + except (Errors.MMBadEmailError, Errors.MMHostileAddress): + msg = _('%(schange_to)s is not a valid email address.') + if msg: + doc.AddItem(Header(3, msg)) + doc.AddItem('<p>') + return + try: + mlist.ApprovedChangeMemberAddress(change_from, change_to, False) + except Errors.NotAMemberError: + msg = _('%(schange_from)s is not a member') + except Errors.MMAlreadyAMember: + msg = _('%(schange_to)s is already a member') + except Errors.MembershipIsBanned, pat: + spat = Utils.websafe(str(pat)) + msg = _('%(schange_to)s matches banned pattern %(spat)s') + else: + msg = _('Address %(schange_from)s changed to %(schange_to)s') + success = True + doc.AddItem(Header(3, msg)) + lang = mlist.getMemberLanguage(change_to) + otrans = i18n.get_translation() + i18n.set_language(lang) + list_name = mlist.getListAddress() + text = Utils.wrap(_("""The member address %(change_from)s on the +%(list_name)s list has been changed to %(change_to)s. +""")) + subject = _('%(list_name)s address change notice.') + i18n.set_translation(otrans) + if success and cgidata.getvalue('notice_old', '') == 'yes': + # Send notice to old address. + msg = Message.UserNotification(change_from, + mlist.GetOwnerEmail(), + text=text, + subject=subject, + lang=lang + ) + msg.send(mlist) + doc.AddItem(Header(3, _('Notification sent to %(schange_from)s.'))) + if success and cgidata.getvalue('notice_new', '') == 'yes': + # Send notice to new address. + msg = Message.UserNotification(change_to, + mlist.GetOwnerEmail(), + text=text, + subject=subject, + lang=lang + ) + msg.send(mlist) + doc.AddItem(Header(3, _('Notification sent to %(schange_to)s.'))) + doc.AddItem('<p>') # See if this was a moderation bit operation if cgidata.has_key('allmodbit_btn'): val = safeint('allmodbit_val') diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py index d1873321..3c9f4002 100644 --- a/Mailman/Cgi/admindb.py +++ b/Mailman/Cgi/admindb.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -39,6 +39,7 @@ from Mailman.ListAdmin import readMessage from Mailman.Cgi import Auth from Mailman.htmlformat import * from Mailman.Logging.Syslog import syslog +from Mailman.CSRFcheck import csrf_check EMPTYSTRING = '' NL = '\n' @@ -50,16 +51,41 @@ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) EXCERPT_HEIGHT = 10 EXCERPT_WIDTH = 76 +SSENDER = mm_cfg.SSENDER +SSENDERTIME = mm_cfg.SSENDERTIME +STIME = mm_cfg.STIME +if mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS in (SSENDERTIME, STIME): + ssort = mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS +else: + ssort = SSENDER + +AUTH_CONTEXTS = (mm_cfg.AuthListAdmin, mm_cfg.AuthSiteAdmin, + mm_cfg.AuthListModerator) -def helds_by_sender(mlist): +def helds_by_skey(mlist, ssort=SSENDER): heldmsgs = mlist.GetHeldMessageIds() - bysender = {} + byskey = {} for id in heldmsgs: + ptime = mlist.GetRecord(id)[0] sender = mlist.GetRecord(id)[1] - bysender.setdefault(sender, []).append(id) - return bysender + if ssort in (SSENDER, SSENDERTIME): + skey = (0, sender) + else: + skey = (ptime, sender) + byskey.setdefault(skey, []).append((ptime, id)) + # Sort groups by time + for k, v in byskey.items(): + if len(v) > 1: + v.sort() + byskey[k] = v + if ssort == SSENDERTIME: + # Rekey with time + newkey = (v[0][0], k[1]) + del byskey[k] + byskey[newkey] = v + return byskey def hacky_radio_buttons(btnname, labels, values, defaults, spacing=3): @@ -76,6 +102,7 @@ def hacky_radio_buttons(btnname, labels, values, defaults, spacing=3): def main(): + global ssort # Figure out which list is being requested parts = Utils.GetPathPieces() if not parts: @@ -91,7 +118,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' handle_no_list(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'admindb: No such list "%s": %s\n', listname, e) return # Now that we know which list to use, set the system's language to it. @@ -99,6 +126,30 @@ def main(): # Make sure the user is authorized to see this page. cgidata = cgi.FieldStorage(keep_blank_values=1) + try: + cgidata.getvalue('adminpw', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc = Document() + doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + + # CSRF check + safe_params = ['adminpw', 'admlogin', 'msgid', 'sender', 'details'] + params = cgidata.keys() + if set(params) - set(safe_params): + csrf_checked = csrf_check(mlist, cgidata.getvalue('csrf_token')) + else: + csrf_checked = True + # if password is present, void cookie to force password authentication. + if cgidata.getvalue('adminpw'): + os.environ['HTTP_COOKIE'] = '' + csrf_checked = True if not mlist.WebAuthenticate((mm_cfg.AuthListAdmin, mm_cfg.AuthListModerator, @@ -177,7 +228,11 @@ def main(): elif not details: # This is a form submission doc.SetTitle(_('%(realname)s Administrative Database Results')) - process_form(mlist, doc, cgidata) + if csrf_checked: + process_form(mlist, doc, cgidata) + else: + doc.addError( + _('The form lifetime has expired. (request forgery check)')) # Now print the results and we're done. Short circuit for when there # are no pending requests, but be sure to save the results! admindburl = mlist.GetScriptURL('admindb', absolute=1) @@ -190,15 +245,16 @@ def main(): doc.AddItem(Link(admindburl, _('Click here to reload this page.'))) # Put 'Logout' link before the footer + doc.AddItem('\n<div align="right"><font size="+2">') doc.AddItem(Link('%s/logout' % admindburl, - '<div align="right"><font size="+2"><b>%s</b></font></div>' % - _('Logout'))) + '<b>%s</b>' % _('Logout'))) + doc.AddItem('</font></div>\n') doc.AddItem(mlist.GetMailmanFooter()) print doc.Format() mlist.Save() return - form = Form(admindburl) + form = Form(admindburl, mlist=mlist, contexts=AUTH_CONTEXTS) # Add the instructions template if details == 'instructions': doc.AddItem(Header( @@ -213,9 +269,11 @@ def main(): nomessages = not mlist.GetHeldMessageIds() if not (details or sender or msgid or nomessages): form.AddItem(Center( + '<label>' + CheckBox('discardalldefersp', 0).Format() + ' ' + - _('Discard all messages marked <em>Defer</em>') + _('Discard all messages marked <em>Defer</em>') + + '</label>' )) # Add a link back to the overview, if we're not viewing the overview! adminurl = mlist.GetScriptURL('admin', absolute=1) @@ -253,7 +311,7 @@ def main(): raw=1, mlist=mlist)) num = show_pending_subs(mlist, form) num += show_pending_unsubs(mlist, form) - num += show_helds_overview(mlist, form) + num += show_helds_overview(mlist, form, ssort) addform = num > 0 # Finish up the document, adding buttons to the form if addform: @@ -261,15 +319,18 @@ def main(): form.AddItem('<hr>') if not (details or sender or msgid or nomessages): form.AddItem(Center( + '<label>' + CheckBox('discardalldefersp', 0).Format() + ' ' + - _('Discard all messages marked <em>Defer</em>') + _('Discard all messages marked <em>Defer</em>') + + '</label>' )) form.AddItem(Center(SubmitButton('submit', _('Submit All Data')))) # Put 'Logout' link before the footer + doc.AddItem('\n<div align="right"><font size="+2">') doc.AddItem(Link('%s/logout' % admindburl, - '<div align="right"><font size="+2"><b>%s</b></font></div>' % - _('Logout'))) + '<b>%s</b>' % _('Logout'))) + doc.AddItem('</font></div>\n') doc.AddItem(mlist.GetMailmanFooter()) print doc.Format() # Commit all changes @@ -314,10 +375,10 @@ def show_pending_subs(mlist, form): for id in pendingsubs: addr = mlist.GetRecord(id)[1] byaddrs.setdefault(addr, []).append(id) - addrs = byaddrs.keys() + addrs = byaddrs.items() addrs.sort() num = 0 - for addr, ids in byaddrs.items(): + for addr, ids in addrs: # Eliminate duplicates for id in ids[1:]: mlist.HandleRequest(id, mm_cfg.DISCARD) @@ -334,8 +395,10 @@ def show_pending_subs(mlist, form): mm_cfg.DISCARD), checked=0).Format() if addr not in mlist.ban_list: - radio += '<br>' + CheckBox('ban-%d' % id, 1).Format() + \ - ' ' + _('Permanently ban from this list') + radio += ('<br>' + '<label>' + + CheckBox('ban-%d' % id, 1).Format() + + ' ' + _('Permanently ban from this list') + + '</label>') # While the address may be a unicode, it must be ascii paddr = addr.encode('us-ascii', 'replace') table.AddRow(['%s<br><em>%s</em>' % (paddr, Utils.websafe(fullname)), @@ -365,10 +428,10 @@ def show_pending_unsubs(mlist, form): for id in pendingunsubs: addr = mlist.GetRecord(id) byaddrs.setdefault(addr, []).append(id) - addrs = byaddrs.keys() + addrs = byaddrs.items() addrs.sort() num = 0 - for addr, ids in byaddrs.items(): + for addr, ids in addrs: # Eliminate duplicates for id in ids[1:]: mlist.HandleRequest(id, mm_cfg.DISCARD) @@ -402,20 +465,29 @@ def show_pending_unsubs(mlist, form): -def show_helds_overview(mlist, form): - # Sort the held messages by sender - bysender = helds_by_sender(mlist) - if not bysender: +def show_helds_overview(mlist, form, ssort=SSENDER): + # Sort the held messages. + byskey = helds_by_skey(mlist, ssort) + if not byskey: return 0 form.AddItem('<hr>') form.AddItem(Center(Header(2, _('Held Messages')))) + # Add the sort sequence choices if wanted + if mm_cfg.DISPLAY_HELD_SUMMARY_SORT_BUTTONS: + form.AddItem(Center(_('Show this list grouped/sorted by'))) + form.AddItem(Center(hacky_radio_buttons( + 'summary_sort', + (_('sender/sender'), _('sender/time'), _('ungrouped/time')), + (SSENDER, SSENDERTIME, STIME), + (ssort == SSENDER, ssort == SSENDERTIME, ssort == STIME)))) # Add the by-sender overview tables admindburl = mlist.GetScriptURL('admindb', absolute=1) table = Table(border=0) form.AddItem(table) - senders = bysender.keys() - senders.sort() - for sender in senders: + skeys = byskey.keys() + skeys.sort() + for skey in skeys: + sender = skey[1] qsender = quote_plus(sender) esender = Utils.websafe(sender) senderurl = admindburl + '?sender=' + qsender @@ -434,15 +506,19 @@ def show_helds_overview(mlist, form): left.AddRow([btns]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ + '<label>' + CheckBox('senderpreserve-' + qsender, 1).Format() + ' ' + - _('Preserve messages for the site administrator') + _('Preserve messages for the site administrator') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ + '<label>' + CheckBox('senderforward-' + qsender, 1).Format() + ' ' + - _('Forward messages (individually) to:') + _('Forward messages (individually) to:') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ @@ -458,9 +534,11 @@ def show_helds_overview(mlist, form): if mlist.isMember(sender): if mlist.getMemberOption(sender, mm_cfg.Moderate): left.AddRow([ + '<label>' + CheckBox('senderclearmodp-' + qsender, 1).Format() + ' ' + - _("Clear this member's <em>moderate</em> flag") + _("Clear this member's <em>moderate</em> flag") + + '</label>' ]) else: left.AddRow( @@ -471,9 +549,11 @@ def show_helds_overview(mlist, form): mlist.reject_these_nonmembers + mlist.discard_these_nonmembers): left.AddRow([ + '<label>' + CheckBox('senderfilterp-' + qsender, 1).Format() + ' ' + - _('Add <b>%(esender)s</b> to one of these sender filters:') + _('Add <b>%(esender)s</b> to one of these sender filters:') + + '</label>' ]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) btns = hacky_radio_buttons( @@ -485,10 +565,11 @@ def show_helds_overview(mlist, form): left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) if sender not in mlist.ban_list: left.AddRow([ + '<label>' + CheckBox('senderbanp-' + qsender, 1).Format() + ' ' + _("""Ban <b>%(esender)s</b> from ever subscribing to this - mailing list""")]) + mailing list""") + '</label>']) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) right = Table(border=0) right.AddRow([ @@ -499,7 +580,7 @@ def show_helds_overview(mlist, form): right.AddCellInfo(right.GetCurrentRowIndex(), 0, colspan=2) right.AddRow([' ', ' ']) counter = 1 - for id in bysender[sender]: + for ptime, id in byskey[skey]: info = mlist.GetRecord(id) ptime, sender, subject, reason, filename, msgdata = info # BAW: This is really the size of the message pickle, which should @@ -540,13 +621,14 @@ def show_helds_overview(mlist, form): def show_sender_requests(mlist, form, sender): - bysender = helds_by_sender(mlist) - if not bysender: + byskey = helds_by_skey(mlist, SSENDER) + if not byskey: return - sender_ids = bysender.get(sender) + sender_ids = byskey.get((0, sender)) if sender_ids is None: # BAW: should we print an error message? return + sender_ids = [x[1] for x in sender_ids] total = len(sender_ids) count = 1 for id in sender_ids: @@ -623,13 +705,11 @@ def show_post_requests(mlist, id, info, total, count, form): for line in email.Iterators.body_line_iterator(msg, decode=True): lines.append(line) chars += len(line) - if chars > limit > 0: + if chars >= limit > 0: break - # Negative values mean display the entire message, regardless of size - if limit > 0: - body = EMPTYSTRING.join(lines)[:mm_cfg.ADMINDB_PAGE_TEXT_LIMIT] - else: - body = EMPTYSTRING.join(lines) + # We may have gone over the limit on the last line, but keep the full line + # anyway to avoid losing part of a multibyte character. + body = EMPTYSTRING.join(lines) # Get message charset and try encode in list charset # We get it from the first text part. # We need to replace invalid characters here or we can throw an uncaught @@ -644,7 +724,7 @@ def show_post_requests(mlist, id, info, total, count, form): lcset = Utils.GetCharSet(mlist.preferred_language) if mcset <> lcset: try: - body = unicode(body, mcset).encode(lcset, 'replace') + body = unicode(body, mcset, 'replace').encode(lcset, 'replace') except (LookupError, UnicodeError, ValueError): pass hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()]) @@ -677,12 +757,16 @@ def show_post_requests(mlist, id, info, total, count, form): t.AddRow([Bold(_('Action:')), buttons]) t.AddCellInfo(row+3, col-1, align='right') t.AddRow([' ', + '<label>' + CheckBox('preserve-%d' % id, 'on', 0).Format() + - ' ' + _('Preserve message for site administrator') + ' ' + _('Preserve message for site administrator') + + '</label>' ]) t.AddRow([' ', + '<label>' + CheckBox('forward-%d' % id, 'on', 0).Format() + ' ' + _('Additionally, forward this message to: ') + + '</label>' + TextBox('forward-addr-%d' % id, size=47, value=mlist.GetOwnerEmail()).Format() ]) @@ -709,7 +793,9 @@ def show_post_requests(mlist, id, info, total, count, form): def process_form(mlist, doc, cgidata): + global ssort senderactions = {} + badaddrs = [] # Sender-centric actions for k in cgidata.keys(): for prefix in ('senderaction-', 'senderpreserve-', 'senderforward-', @@ -729,6 +815,8 @@ def process_form(mlist, doc, cgidata): discardalldefersp = cgidata.getvalue('discardalldefersp', 0) except ValueError: discardalldefersp = 0 + # Get the summary sequence + ssort = int(cgidata.getvalue('summary_sort', SSENDER)) for sender in senderactions.keys(): actions = senderactions[sender] # Handle what to do about all this sender's held messages @@ -743,8 +831,8 @@ def process_form(mlist, doc, cgidata): preserve = actions.get('senderpreserve', 0) forward = actions.get('senderforward', 0) forwardaddr = actions.get('senderforwardto', '') - bysender = helds_by_sender(mlist) - for id in bysender.get(sender, []): + byskey = helds_by_skey(mlist, SSENDER) + for ptime, id in byskey.get((0, sender), []): if id not in senderactions[sender]['message_ids']: # It arrived after the page was displayed. Skip it. continue @@ -762,20 +850,27 @@ def process_form(mlist, doc, cgidata): # Now see if this sender should be added to one of the nonmember # sender filters. if actions.get('senderfilterp', 0): + # Check for an invalid sender address. try: - which = int(actions.get('senderfilter')) - except ValueError: - # Bogus form - which = 'ignore' - if which == mm_cfg.ACCEPT: - mlist.accept_these_nonmembers.append(sender) - elif which == mm_cfg.HOLD: - mlist.hold_these_nonmembers.append(sender) - elif which == mm_cfg.REJECT: - mlist.reject_these_nonmembers.append(sender) - elif which == mm_cfg.DISCARD: - mlist.discard_these_nonmembers.append(sender) - # Otherwise, it's a bogus form, so ignore it + Utils.ValidateEmail(sender) + except Errors.EmailAddressError: + # Don't check for dups. Report it once for each checked box. + badaddrs.append(sender) + else: + try: + which = int(actions.get('senderfilter')) + except ValueError: + # Bogus form + which = 'ignore' + if which == mm_cfg.ACCEPT: + mlist.accept_these_nonmembers.append(sender) + elif which == mm_cfg.HOLD: + mlist.hold_these_nonmembers.append(sender) + elif which == mm_cfg.REJECT: + mlist.reject_these_nonmembers.append(sender) + elif which == mm_cfg.DISCARD: + mlist.discard_these_nonmembers.append(sender) + # Otherwise, it's a bogus form, so ignore it # And now see if we're to clear the member's moderation flag. if actions.get('senderclearmodp', 0): try: @@ -785,8 +880,15 @@ def process_form(mlist, doc, cgidata): pass # And should this address be banned? if actions.get('senderbanp', 0): - if sender not in mlist.ban_list: - mlist.ban_list.append(sender) + # Check for an invalid sender address. + try: + Utils.ValidateEmail(sender) + except Errors.EmailAddressError: + # Don't check for dups. Report it once for each checked box. + badaddrs.append(sender) + else: + if sender not in mlist.ban_list: + mlist.ban_list.append(sender) # Now, do message specific actions banaddrs = [] erroraddrs = [] @@ -836,6 +938,8 @@ def process_form(mlist, doc, cgidata): if cgidata.getvalue(bankey): sender = mlist.GetRecord(request_id)[1] if sender not in mlist.ban_list: + # We don't need to validate the sender. An invalid address + # can't get here. mlist.ban_list.append(sender) # Handle the request id try: @@ -854,7 +958,14 @@ def process_form(mlist, doc, cgidata): doc.AddItem(Header(2, _('Database Updated...'))) if erroraddrs: for addr in erroraddrs: + addr = Utils.websafe(addr) doc.AddItem(`addr` + _(' is already a member') + '<br>') if banaddrs: for addr, patt in banaddrs: + addr = Utils.websafe(addr) doc.AddItem(_('%(addr)s is banned (matched: %(patt)s)') + '<br>') + if badaddrs: + for addr in badaddrs: + addr = Utils.websafe(addr) + doc.AddItem(`addr` + ': ' + _('Bad/Invalid email address') + + '<br>') diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py index 607f1784..fec69dd2 100644 --- a/Mailman/Cgi/confirm.py +++ b/Mailman/Cgi/confirm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -64,7 +64,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'confirm: No such list "%s": %s', listname, e) return # Set the language for the list @@ -73,7 +73,17 @@ def main(): # Get the form data to see if this is a second-step confirmation cgidata = cgi.FieldStorage(keep_blank_values=1) - cookie = cgidata.getvalue('cookie') + try: + cookie = cgidata.getvalue('cookie') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + if cookie == '': ask_for_cookie(mlist, doc, _('Confirmation string was empty.')) return @@ -99,8 +109,9 @@ def main(): %(safecookie)s. <p>Note that confirmation strings expire approximately - %(days)s days after the initial subscription request. If your - confirmation has expired, please try to re-submit your subscription. + %(days)s days after the initial request. They also expire if the + request has already been handled in some way. If your confirmation + has expired, please try to re-submit your request. Otherwise, <a href="%(confirmurl)s">re-enter</a> your confirmation string.''') @@ -258,7 +269,8 @@ def subscription_prompt(mlist, doc, cookie, userdesc): <p>Or hit <em>Cancel my subscription request</em> if you no longer want to subscribe to this list.""") + '<p><hr>' - if mlist.subscribe_policy in (2, 3): + if (mlist.subscribe_policy in (2, 3) and + not getattr(userdesc, 'invitation', False)): # Confirmation is required result = _("""Your confirmation is required in order to continue with the subscription request to the mailing list <em>%(listname)s</em>. diff --git a/Mailman/Cgi/create.py b/Mailman/Cgi/create.py index cac5a2e7..3c2a7dc4 100644 --- a/Mailman/Cgi/create.py +++ b/Mailman/Cgi/create.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -43,6 +43,17 @@ def main(): doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) cgidata = cgi.FieldStorage() + try: + cgidata.getvalue('doit', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + parts = Utils.GetPathPieces() if parts: # Bad URL specification @@ -250,9 +261,10 @@ def process_request(doc, cgidata): 'requestaddr' : mlist.GetRequestEmail(), 'siteowner' : siteowner, }, mlist=mlist) - msg = Message.OwnerNotification(mlist, + msg = Message.UserNotification( + owner, siteowner, _('Your new mailing list: %(listname)s'), - text=text, tomoderators=0) + text, mlist.preferred_language) msg.send(mlist) # Success! diff --git a/Mailman/Cgi/edithtml.py b/Mailman/Cgi/edithtml.py index ee1ccd04..0628f30b 100644 --- a/Mailman/Cgi/edithtml.py +++ b/Mailman/Cgi/edithtml.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -30,9 +30,12 @@ from Mailman import Errors from Mailman.Cgi import Auth from Mailman.Logging.Syslog import syslog from Mailman import i18n +from Mailman.CSRFcheck import csrf_check _ = i18n._ +AUTH_CONTEXTS = (mm_cfg.AuthListAdmin, mm_cfg.AuthSiteAdmin) + def main(): @@ -47,6 +50,18 @@ def main(): ('options.html', _('User specific options page')), ('subscribeack.txt', _('Welcome email text file')), ('masthead.txt', _('Digest masthead')), + ('postheld.txt', _('User notice of held post')), + ('approve.txt', _('User notice of held subscription')), + ('refuse.txt', _('Notice of post refused by moderator')), + ('invite.txt', _('Invitation to join list')), + ('verify.txt', _('Request to confirm subscription')), + ('unsub.txt', _('Request to confirm unsubscription')), + ('nomoretoday.txt', _('User notice of autoresponse limit')), + ('postack.txt', _('User post acknowledgement')), + ('disabled.txt', _('Subscription disabled by bounce warning')), + ('admlogin.html', _('Admin/moderator login page')), + ('private.html', _('Private archive login page')), + ('userpass.txt', _('On demand password reminder')), ) _ = i18n._ @@ -72,7 +87,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'edithtml: No such list "%s": %s', listname, e) return # Now that we have a valid list, set the language to its default @@ -81,6 +96,28 @@ def main(): # Must be authenticated to get any farther cgidata = cgi.FieldStorage() + try: + cgidata.getvalue('adminpw', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + + # CSRF check + safe_params = ['VARHELP', 'adminpw', 'admlogin'] + params = cgidata.keys() + if set(params) - set(safe_params): + csrf_checked = csrf_check(mlist, cgidata.getvalue('csrf_token')) + else: + csrf_checked = True + # if password is present, void cookie to force password authentication. + if cgidata.getvalue('adminpw'): + os.environ['HTTP_COOKIE'] = '' + csrf_checked = True # Editing the html for a list is limited to the list admin and site admin. if not mlist.WebAuthenticate((mm_cfg.AuthListAdmin, @@ -126,7 +163,11 @@ def main(): try: if cgidata.keys(): - ChangeHTML(mlist, cgidata, template_name, doc) + if csrf_checked: + ChangeHTML(mlist, cgidata, template_name, doc) + else: + doc.addError( + _('The form lifetime has expired. (request forgery check)')) FormatHTML(mlist, doc, template_name, template_info) finally: doc.AddItem(mlist.GetMailmanFooter()) @@ -145,7 +186,8 @@ def FormatHTML(mlist, doc, template_name, template_info): doc.AddItem(FontSize("+1", link)) doc.AddItem('<p>') doc.AddItem('<hr>') - form = Form(mlist.GetScriptURL('edithtml') + '/' + template_name) + form = Form(mlist.GetScriptURL('edithtml') + '/' + template_name, + mlist=mlist, contexts=AUTH_CONTEXTS) text = Utils.maketext(template_name, raw=1, mlist=mlist) # MAS: Don't websafe twice. TextArea does it. form.AddItem(TextArea('html_code', text, rows=40, cols=75)) diff --git a/Mailman/Cgi/listinfo.py b/Mailman/Cgi/listinfo.py index 5fbaaaf3..340f0fc1 100644 --- a/Mailman/Cgi/listinfo.py +++ b/Mailman/Cgi/listinfo.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -53,12 +53,24 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' listinfo_overview(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'No such list "%s": %s', listname, e) + syslog('error', 'listinfo: No such list "%s": %s', listname, e) return # See if the user want to see this page in other language cgidata = cgi.FieldStorage() - language = cgidata.getvalue('language') + try: + language = cgidata.getvalue('language') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc = Document() + doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + if not Utils.IsLanguage(language): language = mlist.preferred_language i18n.set_language(language) @@ -88,7 +100,11 @@ def listinfo_overview(msg=''): listnames.sort() for name in listnames: - mlist = MailList.MailList(name, lock=0) + try: + mlist = MailList.MailList(name, lock=0) + except Errors.MMUnknownListError: + # The list could have been deleted by another process. + continue if mlist.advertised: if mm_cfg.VIRTUAL_HOST_OVERVIEW and ( mlist.web_page_url.find('/%s/' % hostname) == -1 and @@ -187,14 +203,25 @@ def list_listinfo(mlist, lang): 'subscribe') if mm_cfg.SUBSCRIBE_FORM_SECRET: now = str(int(time.time())) + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'w.x.y.z'))) + # Try to accept a range in case of load balancers, etc. (LP: #1447445) + if remote.find('.') >= 0: + # ipv4 - drop last octet + remote = remote.rsplit('.', 1)[0] + else: + # ipv6 - drop last 16 (could end with :: in which case we just + # drop one : resulting in an invalid format, but it's only + # for our hash so it doesn't matter. + remote = remote.rsplit(':', 1)[0] replacements['<mm-subscribe-form-start>'] += ( '<input type="hidden" name="sub_form_token" value="%s:%s">\n' % (now, Utils.sha_new(mm_cfg.SUBSCRIBE_FORM_SECRET + now + mlist.internal_name() + - os.environ.get('REMOTE_HOST', - os.environ.get('REMOTE_ADDR', - 'w.x.y.z')) + remote ).hexdigest() ) ) diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py index 9a2389a9..faf732da 100644 --- a/Mailman/Cgi/options.py +++ b/Mailman/Cgi/options.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,6 +17,7 @@ """Produce and handle the member options.""" +import re import sys import os import cgi @@ -32,9 +33,14 @@ from Mailman import MemberAdaptor from Mailman import i18n from Mailman.htmlformat import * from Mailman.Logging.Syslog import syslog +from Mailman.CSRFcheck import csrf_check +OR = '|' SLASH = '/' SETLANGUAGE = -1 +DIGRE = re.compile( + '<!--Start-Digests-Delete-->.*<!--End-Digests-Delete-->', + re.DOTALL) # Set up i18n _ = i18n._ @@ -46,12 +52,26 @@ except NameError: True = 1 False = 0 +AUTH_CONTEXTS = (mm_cfg.AuthListAdmin, mm_cfg.AuthSiteAdmin, + mm_cfg.AuthListModerator, mm_cfg.AuthUser) def main(): doc = Document() doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + method = Utils.GetRequestMethod() + if method.lower() not in ('get', 'post'): + title = _('CGI script error') + doc.SetTitle(title) + doc.AddItem(Header(2, title)) + doc.addError(_('Invalid request method: %(method)s')) + doc.AddItem('<hr>') + doc.AddItem(MailmanLogo()) + print 'Status: 405 Method Not Allowed' + print doc.Format() + return + parts = Utils.GetPathPieces() lenparts = parts and len(parts) if not parts or lenparts < 1: @@ -81,17 +101,40 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'options: No such list "%s": %s\n', listname, e) return # The total contents of the user's response cgidata = cgi.FieldStorage(keep_blank_values=1) + # CSRF check + safe_params = ['displang-button', 'language', 'email', 'password', 'login', + 'login-unsub', 'login-remind', 'VARHELP', 'UserOptions'] + params = cgidata.keys() + if set(params) - set(safe_params): + csrf_checked = csrf_check(mlist, cgidata.getvalue('csrf_token')) + else: + csrf_checked = True + # if password is present, void cookie to force password authentication. + if cgidata.getvalue('password'): + os.environ['HTTP_COOKIE'] = '' + csrf_checked = True + # Set the language for the page. If we're coming from the listinfo cgi, # we might have a 'language' key in the cgi data. That was an explicit # preference to view the page in, so we should honor that here. If that's # not available, use the list's default language. - language = cgidata.getvalue('language') + try: + language = cgidata.getvalue('language') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + if not Utils.IsLanguage(language): language = mlist.preferred_language i18n.set_language(language) @@ -112,6 +155,14 @@ def main(): return else: user = Utils.LCDomain(Utils.UnobscureEmail(SLASH.join(parts[1:]))) + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(user, list) or isinstance(user, tuple): + if len(user) == 0: + user = '' + else: + user = user[-1] # Avoid cross-site scripting attacks safeuser = Utils.websafe(user) @@ -164,6 +215,9 @@ def main(): return # Are we processing an unsubscription request from the login screen? + msgc = _('If you are a list member, a confirmation email has been sent.') + msga = _("""If you are a list member, your unsubscription request has been + forwarded to the list administrator for approval.""") if cgidata.has_key('login-unsub'): # Because they can't supply a password for unsubscribing, we'll need # to do the confirmation dance. @@ -175,14 +229,14 @@ def main(): # be held. Otherwise, send a confirmation. if mlist.unsubscribe_policy: mlist.HoldUnsubscription(user) - doc.addError(_("""Your unsubscription request has been - forwarded to the list administrator for approval."""), - tag='') + doc.addError(msga, tag='') else: - ip = os.environ.get('REMOTE_ADDR') + ip = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) mlist.ConfirmUnsubscription(user, userlang, remote=ip) - doc.addError(_('The confirmation email has been sent.'), - tag='') + doc.addError(msgc, tag='') mlist.Save() finally: mlist.Unlock() @@ -195,19 +249,21 @@ def main(): syslog('mischief', 'Unsub attempt of non-member w/ private rosters: %s', user) - doc.addError(_('The confirmation email has been sent.'), - tag='') + if mlist.unsubscribe_policy: + doc.addError(msga, tag='') + else: + doc.addError(msgc, tag='') loginpage(mlist, doc, user, language) print doc.Format() return # Are we processing a password reminder from the login screen? + msg = _("""If you are a list member, + your password has been emailed to you.""") if cgidata.has_key('login-remind'): if mlist.isMember(user): mlist.MailUserPassword(user) - doc.addError( - _('A reminder of your password has been emailed to you.'), - tag='') + doc.addError(msg, tag='') else: # Not a member if mlist.private_roster == 0: @@ -217,9 +273,7 @@ def main(): syslog('mischief', 'Reminder attempt of non-member w/ private rosters: %s', user) - doc.addError( - _('A reminder of your password has been emailed to you.'), - tag='') + doc.addError(msg, tag='') loginpage(mlist, doc, user, language) print doc.Format() return @@ -251,9 +305,13 @@ def main(): # So as not to allow membership leakage, prompt for the email # address and the password here. if mlist.private_roster <> 0: + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) syslog('mischief', - 'Login failure with private rosters: %s', - user) + 'Login failure with private rosters: %s from %s', + user, remote) user = None # give an HTTP 401 for authentication failure print 'Status: 401 Unauthorized' @@ -265,6 +323,23 @@ def main(): # options. The first set of checks does not require the list to be # locked. + # However, if a form is submitted for a user who has been asynchronously + # unsubscribed, uncaught NotAMemberError exceptions can be thrown. + + if not mlist.isMember(user): + loginpage(mlist, doc, user, language) + print doc.Format() + return + + # Before going further, get the result of CSRF check and do nothing + # if it has failed. + if csrf_checked == False: + doc.addError( + _('The form lifetime has expired. (request forgery check)')) + options_page(mlist, doc, user, cpuser, userlang) + print doc.Format() + return + if cgidata.has_key('logout'): print mlist.ZapCookie(mm_cfg.AuthUser, user) loginpage(mlist, doc, user, language) @@ -506,6 +581,13 @@ address. Upon confirmation, any other mailing list containing the address user, 'via the member options page', userack=1) except Errors.MMNeedApproval: needapproval = True + except Errors.NotAMemberError: + # MAS This except should really be in the outer try so we + # don't save the list redundantly, but except and finally in + # the same try requires Python >= 2.5. + # Setting a switch and making the Save() conditional doesn't + # seem worth it as the Save() won't change anything. + pass mlist.Save() finally: mlist.Unlock() @@ -775,7 +857,8 @@ def options_page(mlist, doc, user, cpuser, userlang, message=''): mlist.FormatButton('othersubs', _('List my other subscriptions'))) replacements['<mm-form-start>'] = ( - mlist.FormatFormStart('options', user)) + mlist.FormatFormStart('options', user, mlist=mlist, + contexts=AUTH_CONTEXTS, user=user)) replacements['<mm-user>'] = user replacements['<mm-presentable-user>'] = presentable_user replacements['<mm-email-my-pw>'] = mlist.FormatButton( @@ -846,8 +929,10 @@ You are subscribed to this list with the case-preserved address else: replacements['<mm-case-preserved-user>'] = '' - doc.AddItem(mlist.ParseTags('options.html', replacements, userlang)) - + page_text = mlist.ParseTags('options.html', replacements, userlang) + if not (mlist.digestable or mlist.getMemberOption(user, mm_cfg.Digests)): + page_text = DIGRE.sub('', page_text) + doc.AddItem(page_text) def loginpage(mlist, doc, user, lang): @@ -1049,7 +1134,8 @@ def topic_details(mlist, doc, user, cpuser, userlang, varhelp): table.AddRow([Bold(Label(_('Name:'))), Utils.websafe(name)]) table.AddRow([Bold(Label(_('Pattern (as regexp):'))), - '<pre>' + Utils.websafe(pattern) + '</pre>']) + '<pre>' + Utils.websafe(OR.join(pattern.splitlines())) + + '</pre>']) table.AddRow([Bold(Label(_('Description:'))), Utils.websafe(description)]) # Make colors look nice diff --git a/Mailman/Cgi/private.py b/Mailman/Cgi/private.py index 6eb40943..0f7597a2 100755 --- a/Mailman/Cgi/private.py +++ b/Mailman/Cgi/private.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -111,14 +111,23 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'private: No such list "%s": %s\n', listname, e) return i18n.set_language(mlist.preferred_language) doc.set_language(mlist.preferred_language) cgidata = cgi.FieldStorage() - username = cgidata.getvalue('username', '') + try: + username = cgidata.getvalue('username', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return password = cgidata.getvalue('password', '') is_auth = 0 diff --git a/Mailman/Cgi/rmlist.py b/Mailman/Cgi/rmlist.py index 8988dc42..3149700d 100644 --- a/Mailman/Cgi/rmlist.py +++ b/Mailman/Cgi/rmlist.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -41,6 +41,17 @@ def main(): doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) cgidata = cgi.FieldStorage() + try: + cgidata.getvalue('password', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + parts = Utils.GetPathPieces() if not parts: @@ -62,7 +73,7 @@ def main(): # Avoid cross-site scripting attacks safelistname = Utils.websafe(listname) title = _('No such list <em>%(safelistname)s</em>') - doc.SetTitle(title) + doc.SetTitle(_('No such list %(safelistname)s')) doc.AddItem( Header(3, Bold(FontAttr(title, color='#ff0000', size='+2')))) @@ -71,7 +82,7 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'rmlist: No such list "%s": %s\n', listname, e) return # Now that we have a valid mailing list, set the language @@ -188,7 +199,7 @@ def process_request(doc, cgidata, mlist): def request_deletion(doc, mlist, errmsg=None): realname = mlist.real_name title = _('Permanently remove mailing list <em>%(realname)s</em>') - doc.SetTitle(title) + doc.SetTitle(_('Permanently remove mailing list %(realname)s')) table = Table(border=0, width='100%') table.AddRow([Center(Bold(FontAttr(title, size='+1')))]) diff --git a/Mailman/Cgi/roster.py b/Mailman/Cgi/roster.py index 6260c973..cb6847af 100644 --- a/Mailman/Cgi/roster.py +++ b/Mailman/Cgi/roster.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -57,13 +57,25 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' error_page(_('No such list <em>%(safelistname)s</em>')) - syslog('error', 'roster: no such list "%s": %s', listname, e) + syslog('error', 'roster: No such list "%s": %s', listname, e) return cgidata = cgi.FieldStorage() # messages in form should go in selected language (if any...) - lang = cgidata.getvalue('language') + try: + lang = cgidata.getvalue('language') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc = Document() + doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return + if not Utils.IsLanguage(lang): lang = mlist.preferred_language i18n.set_language(lang) @@ -129,8 +141,8 @@ def error_page(errmsg): print doc.Format() -def error_page_doc(doc, errmsg, *args): +def error_page_doc(doc, errmsg): # Produce a simple error-message page on stdout and exit. doc.SetTitle(_("Error")) doc.AddItem(Header(2, _("Error"))) - doc.AddItem(Bold(errmsg % args)) + doc.AddItem(Bold(errmsg)) diff --git a/Mailman/Cgi/subscribe.py b/Mailman/Cgi/subscribe.py index d6b1517d..b2f8925e 100755 --- a/Mailman/Cgi/subscribe.py +++ b/Mailman/Cgi/subscribe.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -64,13 +64,22 @@ def main(): # Send this with a 404 status. print 'Status: 404 Not Found' print doc.Format() - syslog('error', 'No such list "%s": %s\n', listname, e) + syslog('error', 'subscribe: No such list "%s": %s\n', listname, e) return # See if the form data has a preferred language set, in which case, use it # for the results. If not, use the list's preferred language. cgidata = cgi.FieldStorage() - language = cgidata.getvalue('language') + try: + language = cgidata.getvalue('language', '') + except TypeError: + # Someone crafted a POST with a bad Content-Type:. + doc.AddItem(Header(2, _("Error"))) + doc.AddItem(Bold(_('Invalid options to CGI script.'))) + # Send this with a 400 status. + print 'Status: 400 Bad Request' + print doc.Format() + return if not Utils.IsLanguage(language): language = mlist.preferred_language i18n.set_language(language) @@ -118,29 +127,43 @@ def process_form(mlist, doc, cgidata, lang): # Canonicalize the full name fullname = Utils.canonstr(fullname, lang) # Who was doing the subscribing? - remote = os.environ.get('REMOTE_HOST', - os.environ.get('REMOTE_ADDR', - 'unidentified origin')) + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) # Are we checking the hidden data? if mm_cfg.SUBSCRIBE_FORM_SECRET: now = int(time.time()) + # Try to accept a range in case of load balancers, etc. (LP: #1447445) + if remote.find('.') >= 0: + # ipv4 - drop last octet + remote1 = remote.rsplit('.', 1)[0] + else: + # ipv6 - drop last 16 (could end with :: in which case we just + # drop one : resulting in an invalid format, but it's only + # for our hash so it doesn't matter. + remote1 = remote.rsplit(':', 1)[0] try: ftime, fhash = cgidata.getvalue('sub_form_token', '').split(':') then = int(ftime) except ValueError: ftime = fhash = '' - then = now + then = 0 token = Utils.sha_new(mm_cfg.SUBSCRIBE_FORM_SECRET + ftime + mlist.internal_name() + - remote).hexdigest() - if now - then > mm_cfg.FORM_LIFETIME: + remote1).hexdigest() + if ftime and now - then > mm_cfg.FORM_LIFETIME: results.append(_('The form is too old. Please GET it again.')) - if now - then < mm_cfg.SUBSCRIBE_FORM_MIN_TIME: + if ftime and now - then < mm_cfg.SUBSCRIBE_FORM_MIN_TIME: + results.append( + _('Please take a few seconds to fill out the form before submitting it.')) + if ftime and token != fhash: results.append( - _('Please take a few seconds to fill out the form before submitting it.') - ) - if token != fhash: + _("The hidden token didn't match. Did your IP change?")) + if not ftime: + results.append( + _('There was no hidden token in your submission or it was corrupted.')) results.append(_('You must GET the form before submitting it.')) # Was an attempt made to subscribe the list to itself? if email == mlist.GetListEmail(): @@ -162,7 +185,7 @@ def process_form(mlist, doc, cgidata, lang): if digestflag: try: digest = int(digestflag) - except ValueError: + except (TypeError, ValueError): digest = 0 else: digest = mlist.digest_is_default diff --git a/Mailman/Commands/cmd_confirm.py b/Mailman/Commands/cmd_confirm.py index a3accf64..379d23c2 100644 --- a/Mailman/Commands/cmd_confirm.py +++ b/Mailman/Commands/cmd_confirm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -49,9 +49,9 @@ def process(res, args): days = int(mm_cfg.PENDING_REQUEST_LIFE / mm_cfg.days(1) + 0.5) res.results.append(_("""\ Invalid confirmation string. Note that confirmation strings expire -approximately %(days)s days after the initial subscription request. If your -confirmation has expired, please try to re-submit your original request or -message.""")) +approximately %(days)s days after the initial request. They also expire if +the request has already been handled in some way. If your confirmation has +expired, please try to re-submit your original request or message.""")) except Errors.MMNeedApproval: res.results.append(_("""\ Your request has been forwarded to the list moderator for approval.""")) diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index be1ac735..e6738192 100755 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1,6 +1,6 @@ # -*- python -*- -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -131,6 +131,14 @@ SUBSCRIBE_FORM_SECRET = None # test. SUBSCRIBE_FORM_MIN_TIME = seconds(5) +# Installation wide ban list. This is a list of email addresses and regexp +# patterns (beginning with ^) which are not allowed to subscribe to any lists +# in the installation. This supplements the individual list's ban_list. +# For example, to ban xxx@aol.com and any @gmail.com address beginning with +# yyy, set +# GLOBAL_BAN_LIST = ['xxx@aol.com', '^yyy.*@gmail\.com$'] +GLOBAL_BAN_LIST = [] + # Command that is used to convert text/html parts into plain text. This # should output results to standard output. %(filename)s will contain the # name of the temporary file that the program should operate on. @@ -138,9 +146,20 @@ HTML_TO_PLAIN_TEXT_COMMAND = '/usr/bin/lynx -dump %(filename)s' # A Python regular expression character class which defines the characters # allowed in list names. Lists cannot be created with names containing any -# character that doesn't match this class. +# character that doesn't match this class. Do not include '/' in this list. ACCEPTABLE_LISTNAME_CHARACTERS = '[-+_.=a-z0-9]' +# Shall the user's real names be displayed along with their email addresses +# in list rosters? Defaults to No to preserve prior behavior. +ROSTER_DISPLAY_REALNAME = No + +# Beginning in Mailman 2.1.21, localized help and some other output from +# Mailman's bin/ commands is converted to the character set of the user's +# workstation (LC_CTYPE) if different from the character set of the language. +# This is not well tested over a wide range of locales, so if it causes +# problems, it can be disabled by setting the following to Yes. +DISABLE_COMMAND_LOCALE_CSET = No + ##### @@ -166,7 +185,7 @@ VIRTUAL_HOST_OVERVIEW = On # omitted it defaults to urlhost with the first name stripped off, e.g. # # add_virtualhost('www.dom.ain') -# VIRTUAL_HOST['www.dom.ain'] +# VIRTUAL_HOSTS['www.dom.ain'] # ==> 'dom.ain' # def add_virtualhost(urlhost, emailhost=None): @@ -199,6 +218,13 @@ add_virtualhost(DEFAULT_URL_HOST, DEFAULT_EMAIL_HOST) # -owners address, unless the message is explicitly approved. KNOWN_SPAMMERS = [] +# The header_filter_rules in Privacy options... -> Spam filters are matched as +# normalized unicodes against normalized unicode headers. This setting +# determines the normalization form. It is one of 'NFC', 'NFD', 'NFKC' or +# 'NFKD'. See +# https://docs.python.org/2/library/unicodedata.html#unicodedata.normalize +NORMALIZE_FORM = 'NFKC' + ##### @@ -224,6 +250,20 @@ WEB_VLINK_COLOR = '' # If true, forces VLINK= WEB_HIGHLIGHT_COLOR = '#dddddd' # If true, alternating rows # in listinfo & admin display +# If you wish to include extra elements in the <HEAD> section of Mailman's +# web pages, e.g. style information or a link to a style sheet, you can set +# the following. For example, to include a css style sheet reference, you +# can put in mm_cfg.py +# WEB_HEAD_ADD = """<LINK REL=stylesheet +# TYPE="text/css" +# HREF="path or URL" +# >""" +# You can specify anything that is allowed in the <HEAD> section. The default +# is to not add anything. This only applies to internally generated pages. +# For pages built from templates you can create custom templates containing +# this information. +WEB_HEAD_ADD = None + # User entered data is escaped for redisplay in web responses to avoid Cross # Site Scripting (XSS) attacks. The normal escaping replaces the characters # <, >, & and " with the respective HTML entities <, >, & and @@ -247,6 +287,17 @@ BROKEN_BROWSER_REPLACEMENTS = {'\x8b': '‹', # single left angle quote '\xbe': '¾', # > plus high order bit '\xa2': '¢', # " plus high order bit } +# +# Shall the admindb held message summary display the grouping and sorting +# option radio buttons? Set this in mm_cfg.py to one of the following: +# SSENDER -> Default to grouped and sorted by sender. +# SSENDERTIME -> Default to grouped by sender and sorted by time. +# STIME -> Default to ungrouped and sorted by time. +DISPLAY_HELD_SUMMARY_SORT_BUTTONS = No +# +# Shall the default for the admin Mass Subscription function be Invite rather +# than Subscribe? Set to Yes in mm_cfg.py to make the default be Invite. +DEFAULT_SUBSCRIBE_OR_INVITE = No @@ -425,6 +476,12 @@ PUBLIC_MBOX = No #DELIVERY_MODULE = 'Sendmail' DELIVERY_MODULE = 'SMTPDirect' +# Sometimes there are 'low level' smtplib failures that are difficult to +# debug. To enable very verbose debugging info from smtplib to Mailman's +# error log, set the following to 1. This will only work if +# DELIVERY_MODULE = 'SMTPDirect' and Python is >= 2.4. +SMTPLIB_DEBUG_LEVEL = 0 + # MTA should name a module in Mailman/MTA which provides the MTA specific # functionality for creating and removing lists. Some MTAs like Exim can be # configured to automatically recognize new lists, in which case the MTA @@ -512,6 +569,22 @@ SMTPPORT = 0 # default from smtplib # when DELIVERY_MODULE is 'Sendmail'. SENDMAIL_CMD = '/usr/lib/sendmail' +# SMTP authentication for DELIVERY_MODULE = 'SMTPDirect'. To enable SASL +# authentication for SMTPDirect, set SMTP_AUTH = Yes and provide appropriate +# settings for SMTP_USER and SMTP_PASSWD. +SMTP_AUTH = No +SMTP_USER = '' +SMTP_PASSWD = '' + +# If using SASL authentication (SMTP_AUTH = Yes), set the following to Yes +# to also use TLS. This has no effect if SMTP_AUTH = No. +SMTP_USE_TLS = No + +# When using TLS the following should be set to the hostname that should be +# used in order to identify Mailman to the SMTP server. By default, it +# uses DEFAULT_URL_HOST. Normally, you should not change this. +SMTP_HELO_HOST = DEFAULT_URL_HOST + # Set these variables if you need to authenticate to your NNTP server for # Usenet posting or reading. If no authentication is necessary, specify None # for both variables. @@ -548,9 +621,25 @@ NNTP_REWRITE_DUPLICATE_HEADERS = [ # footer or scrubbing attachments or even reply-to munging can break these # signatures. It is generally felt that these signatures have value, even if # broken and even if the outgoing message is resigned. However, some sites -# may wish to remove these headers by setting this to Yes. +# may wish to remove these headers. Possible values and meanings are: +# No, 0, False -> do not remove headers. +# Yes, 1, True -> remove headers only if we are munging the from header due +# to from_is_list or dmarc_moderation_action. +# 2 -> always remove headers. +# 3 -> always remove, rename and preserve original DKIM headers. REMOVE_DKIM_HEADERS = No +# If the following is set to a non-empty string, that string is the name of a +# header that will be added to personalized and VERPed deliveries with value +# equal to the base64 encoding of the recipient's email address. This is +# intended to enable identification of the recipient otherwise redacted from +# "spam report" feedback loop messages. For example, if +# RCPT_BASE64_HEADER_NAME = 'X-Mailman-R-Data' +# a header like +# X-Mailman-R-Data: dXNlckBleGFtcGxlLmNvbQo= +# will be added to messages sent to user@@example.com. +RCPT_BASE64_HEADER_NAME = '' + # All `normal' messages which are delivered to the entire list membership go # through this pipeline of handler modules. Lists themselves can override the # global pipeline by defining a `pipeline' attribute. @@ -580,6 +669,7 @@ GLOBAL_PIPELINE = [ # (outgoing) path, finally leaving the message in the outgoing queue. 'AfterDelivery', 'Acknowledge', + 'WrapMessage', 'ToOutgoing', ] @@ -587,7 +677,6 @@ GLOBAL_PIPELINE = [ OWNER_PIPELINE = [ 'SpamDetect', 'Replybot', - 'CleanseDKIM', 'OwnerRecips', 'ToOutgoing', ] @@ -968,6 +1057,27 @@ USER_FRIENDLY_PASSWORDS = Yes MEMBER_PASSWORD_LENGTH = 8 ADMIN_PASSWORD_LENGTH = 10 +# The following headers are always removed from posts to anonymous lists as +# they can reveal the identity of the poster or at least the poster's domain. +# +# From:, Reply-To:, Sender:, Return-Path:, X-Originating-Email:, Received:, +# Message-ID: and X-Envelope-From:. +# +# In addition, Return-Receipt-To:, Disposition-Notification-To:, +# X-Confirm-Reading-To: and X-Pmrqc: headers are removed from all posts as +# they can be used to fish for list membership in addition to possibly +# revealing sender information. +# +# In addition to the above removals, all other headers except those matching +# regular expressions in the following setting are also removed. The default +# setting below keeps all non X- headers, those X- headers added by Mailman +# and any X-Spam- headers. +ANONYMOUS_LIST_KEEP_HEADERS = ['^(?!x-)', '^x-mailman-', + '^x-content-filtered-by:', '^x-topics:', + '^x-ack:', '^x-beenthere:', + '^x-list-administrivia:', '^x-spam-', + ] + ##### @@ -988,7 +1098,7 @@ DEFAULT_MAX_MESSAGE_SIZE = 40 # KB DEFAULT_SUBJECT_PREFIX = "[%(real_name)s] " # DEFAULT_SUBJECT_PREFIX = "[%(real_name)s %%d]" # for numbering DEFAULT_MSG_HEADER = "" -DEFAULT_MSG_FOOTER = """_______________________________________________ +DEFAULT_MSG_FOOTER = """-- %(real_name)s mailing list %(real_name)s@%(host_name)s %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s @@ -1028,7 +1138,94 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No # moderators? DEFAULT_FORWARD_AUTO_DISCARDS = Yes -# What shold happen to non-member posts which are do not match explicit +# Shall dmarc_moderation_action be applied to messages From: domains with +# a DMARC policy of quarantine as well as reject? This sets the default for +# the list setting that controls it. +DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION = Yes + +# Default action for posts whose From: address domain has a DMARC policy of +# reject or quarantine. See DEFAULT_FROM_IS_LIST below. Whatever is set as +# the default here precludes the list owner from setting a lower value. +# 0 = Accept +# 1 = Munge From +# 2 = Wrap Message +# 3 = Reject +# 4 = Discard +DEFAULT_DMARC_MODERATION_ACTION = 0 + +# Domain owners can publish DMARC p=none policy in order to request that +# reports of DMARC failures be sent but special action not be taken on +# messages From: their domain that fail DMARC. This can result in over +# estimation of the number of messages that would be quarantined or rejected +# with a stronger DMARC policy if such a policy would result in message +# modification because dmarc_moderation_action is 1 or 2. Thus, there is +# a list setting to apply dmarc_moderaction_action of 1 or 2 to messages +# From: domains with DMARC p=none. Setting this to Yes is only effective if +# dmarc_quarantine_moderaction_action is also Yes. The following is the +# default for this setting for new lists. +DEFAULT_DMARC_NONE_MODERATION_ACTION = No + +# Default for text to be added to a separate text/plain part preceding the +# message/rfc822 part containing the original message when +# dmarc_moderation_action is Wrap Message. +DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT = '' + +# Parameters for DMARC DNS lookups. If you are seeing 'DNSException: +# Unable to query DMARC policy ...' entries in your error log, you may need +# to adjust these. +# The time to wait for a response from a name server before timeout. +DMARC_RESOLVER_TIMEOUT = seconds(3) +# The total time to spend trying to get an answer to the question. +DMARC_RESOLVER_LIFETIME = seconds(5) + +# A URL from which to retrieve the data for the algorithm that computes +# Organizational Domains for DMARC policy lookup purposes. This can be +# anything handled by the Python urllib2.urlopen function. See +# https://publicsuffix.org/list/ for info. +DMARC_ORGANIZATIONAL_DOMAIN_DATA_URL = \ +'https://publicsuffix.org/list/public_suffix_list.dat' + +# Should the list server auto-moderate members who post too frequently +# This is intended to stop people who join a list and then use a bot to +# send many spam messages in a short interval. These are default settings +# for new lists. See the web admin Privacy options -> Sender filters page +# and the Details for member_verbosity_threshold and member_verbosity_interval +# links for more information. +# DEFAULT_MEMBER_VERBOSITY_INTERVAL = number of seconds to track posts +# DEFAULT_MEMBER_VERBOSITY_THRESHOLD = number of allowed posts per interval +# (0 to disable). +DEFAULT_MEMBER_VERBOSITY_INTERVAL = 300 +DEFAULT_MEMBER_VERBOSITY_THRESHOLD = 0 + +# This controls how often to clean old post time entries from the dictionary +# used to implement the member verbosity feature. This is a compromise between +# using resources for cleaning and allowing the dictionary to grow very large. +# The setting is the number of passes through the code before the dictionary +# is cleaned. +VERBOSE_CLEAN_LIMIT = 1000 + +# What domains should be considered equivalent when testing list membership +# for posting/moderation. +# If two poster addresses with the same local part but +# different domains are to be considered equivalents for list +# membership tests, the domains are put in the list's equivalent_domains. +# This provides a default value for new lists. +# The format is one or more groups of equivalent domains. Within a group, +# the domains are separated by commas and multiple groups are +# separated by semicolons. White space is ignored. +# For example: +# +# 'example.com,mail.example.com;mac.com,me.com,icloud.com' +# +# In this example, if user@example.com is a list member, +# a post from user@mail.example.com will be treated as if it is +# from user@example.com for list membership/moderation purposes, +# and likewise, if user@me.com is a list member, posts from +# user@mac.com or user@icloud.com will be treated as if from +# user@me.com. +DEFAULT_EQUIVALENT_DOMAINS = '' + +# What should happen to non-member posts which are do not match explicit # non-member actions? # 0 = Accept # 1 = Hold @@ -1065,6 +1262,16 @@ DEFAULT_SEND_WELCOME_MSG = Yes # Send goodbye messages to unsubscribed members? DEFAULT_SEND_GOODBYE_MSG = Yes +# The following is a three way setting. It sets the default for the list's +# from_is_list policy which is applied to all posts except those for which a +# dmarc_moderation_action other than accept applies. +# 0 -> Do not rewrite the From: or wrap the message. +# 1 -> Rewrite the From: header of posts replacing the posters address with +# that of the list. Also see REMOVE_DKIM_HEADERS above. +# 2 -> Do not modify the From: of the message, but wrap the message in an outer +# message From the list address. +DEFAULT_FROM_IS_LIST = 0 + # Wipe sender information, and make it look like the list-admin # address sends all messages DEFAULT_ANONYMOUS_LIST = No @@ -1105,6 +1312,10 @@ DEFAULT_SUBSCRIBE_POLICY = 1 # Does this site allow completely unchecked subscriptions? ALLOW_OPEN_SUBSCRIBE = No +# This is the default list of addresses and regular expressions (beginning +# with ^) that are exempt from approval if SUBSCRIBE_POLICY is 2 or 3. +DEFAULT_SUBSCRIBE_AUTO_APPROVAL = [] + # The default policy for unsubscriptions. 0 (unmoderated unsubscribes) is # highly recommended! # 0 - unmoderated unsubscribes @@ -1149,9 +1360,11 @@ DEFAULT_FILTER_MIME_TYPES = [] # DEFAULT_PASS_MIME_TYPES is a list of MIME types to be passed through. # Format is the same as DEFAULT_FILTER_MIME_TYPES -DEFAULT_PASS_MIME_TYPES = ['multipart/mixed', - 'multipart/alternative', - 'text/plain'] +DEFAULT_PASS_MIME_TYPES = ['multipart', + 'message/rfc822', + 'application/pgp-signature', + 'text/plain', + ] # DEFAULT_FILTER_FILENAME_EXTENSIONS is a list of filename extensions to be # removed. It is useful because many viruses fake their content-type as @@ -1195,7 +1408,10 @@ DEFAULT_NONDIGESTABLE = Yes # Will list be available in digested form? DEFAULT_DIGESTABLE = Yes DEFAULT_DIGEST_HEADER = "" -DEFAULT_DIGEST_FOOTER = DEFAULT_MSG_FOOTER +DEFAULT_DIGEST_FOOTER = """%(real_name)s mailing list +%(real_name)s@%(host_name)s +%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s +""" DEFAULT_DIGEST_IS_DEFAULT = No DEFAULT_MIME_IS_DEFAULT_DIGEST = No @@ -1279,6 +1495,10 @@ DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL = days(7) # failed to match by the bounce detector? DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER = Yes +# Does the list owner get a copy of every recognized bounce that increments +# the score for a list member but doesn't result in a disable or probe? +DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT = No + # Notifications on bounce actions. The first specifies whether the list owner # should get a notification when a member is disabled due to bouncing, while # the second specifies whether the owner should get one when the member is @@ -1396,6 +1616,11 @@ UNSUBSCRIBE = 5 ACCEPT = 6 HOLD = 7 +# admindb summary sort button settings. All must evaluate to True. +SSENDER = 1 +SSENDERTIME = 2 +STIME = 3 + # Standard text field width TEXTFIELDWIDTH = 40 @@ -1515,6 +1740,7 @@ add_language('en', _('English (USA)'), 'us-ascii', 'ltr') add_language('es', _('Spanish (Spain)'), 'iso-8859-1', 'ltr') add_language('et', _('Estonian'), 'iso-8859-15', 'ltr') add_language('eu', _('Euskara'), 'iso-8859-15', 'ltr') # Basque +add_language('fa', _('Persian'), 'utf-8', 'rtl') add_language('fi', _('Finnish'), 'iso-8859-1', 'ltr') add_language('fr', _('French'), 'iso-8859-1', 'ltr') add_language('gl', _('Galician'), 'utf-8', 'ltr') @@ -1532,8 +1758,8 @@ add_language('no', _('Norwegian'), 'iso-8859-1', 'ltr') add_language('pl', _('Polish'), 'iso-8859-2', 'ltr') add_language('pt', _('Portuguese'), 'iso-8859-1', 'ltr') add_language('pt_BR', _('Portuguese (Brazil)'), 'iso-8859-1', 'ltr') -add_language('ro', _('Romanian'), 'iso-8859-2', 'ltr') -add_language('ru', _('Russian'), 'koi8-r', 'ltr') +add_language('ro', _('Romanian'), 'utf-8', 'ltr') +add_language('ru', _('Russian'), 'utf-8', 'ltr') add_language('sk', _('Slovak'), 'utf-8', 'ltr') add_language('sl', _('Slovenian'), 'iso-8859-2', 'ltr') add_language('sr', _('Serbian'), 'utf-8', 'ltr') diff --git a/Mailman/Deliverer.py b/Mailman/Deliverer.py index 0f8f26b8..dc867006 100644 --- a/Mailman/Deliverer.py +++ b/Mailman/Deliverer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -42,6 +42,8 @@ except NameError: class Deliverer: def SendSubscribeAck(self, name, password, digest, text=''): pluser = self.getMemberLanguage(name) + # Need to set this here to get the proper l10n of the Subject: + i18n.set_language(pluser) if self.welcome_msg: welcome = Utils.wrap(self.welcome_msg) + '\n' else: @@ -81,6 +83,7 @@ your membership administrative address, %(addr)s.''')) def SendUnsubscribeAck(self, addr, lang): realname = self.real_name + i18n.set_language(lang) msg = Message.UserNotification( self.GetMemberAdminEmail(addr), self.GetBouncesEmail(), _('You have been unsubscribed from the %(realname)s mailing list'), diff --git a/Mailman/Gui/Bounce.py b/Mailman/Gui/Bounce.py index 1dc837fc..e559dcc8 100644 --- a/Mailman/Gui/Bounce.py +++ b/Mailman/Gui/Bounce.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -142,6 +142,15 @@ class Bounce(GUIBase): <a href="?VARHELP=autoreply/autoresponse_admin_text">autoresponse message</a> for email to the -owner and -admin address.""")), + ('bounce_notify_owner_on_bounce_increment', mm_cfg.Toggle, + (_('No'), _('Yes')), 0, + _("""Should Mailman notify you, the list owner, when bounces + cause a member's bounce score to be incremented?"""), + _("""Setting this value to <em>Yes</em> will cause Mailman to + send a notice including a copy of the bounce message to the list + owners whenever a bounce increments a member's bounce score but + doesn't cause a disable or a probe to be sent.""")), + ('bounce_notify_owner_on_disable', mm_cfg.Toggle, (_('No'), _('Yes')), 0, _("""Should Mailman notify you, the list owner, when bounces diff --git a/Mailman/Gui/Digest.py b/Mailman/Gui/Digest.py index f7722019..77691aee 100644 --- a/Mailman/Gui/Digest.py +++ b/Mailman/Gui/Digest.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -56,8 +56,8 @@ class Digest(GUIBase): _('When receiving digests, which format is default?')), ('digest_size_threshhold', mm_cfg.Number, 3, 0, - _('How big in Kb should a digest be before it gets sent out?')), - # Should offer a 'set to 0' for no size threshhold. + _('How big in Kb should a digest be before it gets sent out?' + ' 0 implies no maximum size.')), ('digest_send_periodic', mm_cfg.Radio, (_('No'), _('Yes')), 1, _('Should a digest be dispatched daily when the size threshold ' diff --git a/Mailman/Gui/GUIBase.py b/Mailman/Gui/GUIBase.py index a365acaf..32d19929 100644 --- a/Mailman/Gui/GUIBase.py +++ b/Mailman/Gui/GUIBase.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -63,6 +63,7 @@ class GUIBase: if isinstance(val, ListType): return val addrs = [] + bad_addrs = [] for addr in [s.strip() for s in val.split(NL)]: # Discard empty lines if not addr: @@ -77,22 +78,25 @@ class GUIBase: try: re.compile(addr) except re.error: - raise ValueError + bad_addrs.append(addr) elif (wtype == mm_cfg.EmailListEx and addr.startswith('@') - and property.endswith('_these_nonmembers')): + and (property.endswith('_these_nonmembers') or + property == 'subscribe_auto_approval')): # XXX Needs to be reviewed for list@domain names. # don't reference your own list if addr[1:] == mlist.internal_name(): - raise ValueError + bad_addrs.append(addr) # check for existence of list? For now allow # reference to list before creating it. else: - raise + bad_addrs.append(addr) if property in ('regular_exclude_lists', 'regular_include_lists'): if addr.lower() == mlist.GetListEmail().lower(): - raise Errors.EmailAddressError + bad_addrs.append(addr) addrs.append(addr) + if bad_addrs: + raise Errors.EmailAddressError, ', '.join(bad_addrs) return addrs # This is a host name, i.e. verbatim if wtype == mm_cfg.Host: @@ -168,9 +172,9 @@ class GUIBase: except ValueError: doc.addError(_('Invalid value for variable: %(property)s')) # This is the parent of MMBadEmailError and MMHostileAddress - except Errors.EmailAddressError: + except Errors.EmailAddressError, error: doc.addError( - _('Bad email address for option %(property)s: %(val)s')) + _('Bad email address for option %(property)s: %(error)s')) else: # Set the attribute, which will normally delegate to the mlist self._setValue(mlist, property, val, doc) diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py index e9f8f9b5..980e5f2b 100644 --- a/Mailman/Gui/General.py +++ b/Mailman/Gui/General.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -154,6 +154,72 @@ class General(GUIBase): (listname %%05d) -> (listname 00123) """)), + ('from_is_list', mm_cfg.Radio, + (_('No'), _('Munge From'), _('Wrap Message')), 0, + _("""Replace the From: header address with the list's posting + address to mitigate issues stemming from the original From: + domain's DMARC or similar policies."""), + _("""Several protocols now in wide use attempt to ensure that use + of the domain in the author's address (ie, in the From: header + field) is authorized by that domain. These protocols may be + incompatible with common list features such as footers, causing + participating email services to bounce list traffic merely + because of the address in the From: field. <b>This has resulted + in members being unsubscribed despite being perfectly able to + receive mail.</b> + <p> + The following actions are applied to all list messages when + selected here. To apply these actions only to messages where the + domain in the From: header is determined to use such a protocol, + see the <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a> settings under Privacy options... + -> Sender filters. + <p>Settings:<p> + <dl> + <dt>No</dt> + <dd>Do nothing special. This is appropriate for anonymous lists. + It is appropriate for dedicated announcement lists, unless the + From: address of authorized posters might be in a domain with a + DMARC or similar policy. It is also appropriate if you choose to + use dmarc_moderation_action other than Accept for this list.</dd> + <dt>Munge From</dt> + <dd>This action replaces the poster's address in the From: header + with the list's posting address and adds the poster's address to + the addresses in the original Reply-To: header.</dd> + <dt>Wrap Message</dt> + <dd>Just wrap the message in an outer message with the From: + header containing the list's posting address and with the original + From: address added to the addresses in the original Reply-To: + header and with Content-Type: message/rfc822. This is effectively + a one message MIME format digest.</dd> + </dl> + <p>The transformations for anonymous_list are applied before + any of these actions. It is not useful to apply actions other + than No to an anonymous list, and if you do so, the result may + be surprising. + <p>The Reply-To: header munging actions below interact with these + actions as follows: + <p> first_strip_reply_to = Yes will remove all the incoming + Reply-To: addresses but will still add the poster's address to + Reply-To: for all three settings of reply_goes_to_list which + respectively will result in just the poster's address, the + poster's address and the list posting address or the poster's + address and the explicit reply_to_address in the outgoing + Reply-To: header. If first_strip_reply_to = No the poster's + address in the original From: header, if not already included in + the Reply-To:, will be added to any existing Reply-To: + address(es). + <p>These actions, whether selected here or via <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a>, do not apply to messages in digests + or archives or sent to usenet via the Mail<->News gateways. + <p>If <a + href="?VARHELP=privacy/sender/dmarc_moderation_action"> + dmarc_moderation_action</a> applies to this message with an + action other than Accept, that action rather than this is + applied""")), + ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Hide the sender of a message, replacing it with the list address (Removes From, Sender and Reply-To fields)""")), @@ -189,11 +255,11 @@ class General(GUIBase): their own <tt>Reply-To:</tt> settings to convey their valid return address. Another is that modifying <tt>Reply-To:</tt> makes it much more difficult to send private replies. See <a - href="http://www.unicom.com/pw/reply-to-harmful.html">`Reply-To' + href="http://marc.merlins.org/netrants/reply-to-harmful.html">`Reply-To' Munging Considered Harmful</a> for a general discussion of this issue. See <a - href="http://www.metasystema.net/essays/reply-to.mhtml">Reply-To - Munging Considered Useful</a> for a dissenting opinion. + href="http://marc.merlins.org/netrants/reply-to-useful.html"> + Reply-To Munging Considered Useful</a> for a dissenting opinion. <p>Some mailing lists have restricted posting privileges, with a parallel list devoted to discussions. Examples are `patches' or @@ -217,11 +283,11 @@ class General(GUIBase): their own <tt>Reply-To:</tt> settings to convey their valid return address. Another is that modifying <tt>Reply-To:</tt> makes it much more difficult to send private replies. See <a - href="http://www.unicom.com/pw/reply-to-harmful.html">`Reply-To' + href="http://marc.merlins.org/netrants/reply-to-harmful.html">`Reply-To' Munging Considered Harmful</a> for a general discussion of this issue. See <a - href="http://www.metasystema.net/essays/reply-to.mhtml">Reply-To - Munging Considered Useful</a> for a dissenting opinion. + href="http://marc.merlins.org/netrants/reply-to-useful.html"> + Reply-To Munging Considered Useful</a> for a dissenting opinion. <p>Some mailing lists have restricted posting privileges, with a parallel list devoted to discussions. Examples are `patches' or diff --git a/Mailman/Gui/Membership.py b/Mailman/Gui/Membership.py index 7e1bf324..fdf5a7ca 100644 --- a/Mailman/Gui/Membership.py +++ b/Mailman/Gui/Membership.py @@ -30,5 +30,6 @@ class Membership: return [('list', _('Membership List')), ('add', _('Mass Subscription')), ('remove', _('Mass Removal')), + ('change', _('Address Change')), ] return None diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py index 411f5d4f..e4d7d526 100755 --- a/Mailman/Gui/NonDigest.py +++ b/Mailman/Gui/NonDigest.py @@ -160,7 +160,7 @@ and footers: siblings.""")), ('regular_exclude_ignore', mm_cfg.Toggle, (_('No'), _('Yes')), 0, - _("""Ignore regular_exlude_lists of which the poster is not a + _("""Ignore regular_exclude_lists of which the poster is not a member."""), _("""If a post is addressed to this list and to one or more of the exclude lists, regular members of those lists will not be diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py index 75eff2b5..2a9cca26 100644 --- a/Mailman/Gui/Privacy.py +++ b/Mailman/Gui/Privacy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,6 +17,7 @@ """MailList mixin class managing the privacy options.""" +import os import re from Mailman import mm_cfg @@ -61,7 +62,7 @@ class Privacy(GUIBase): _('Confirm and approve')), 0, _('What steps are required for subscription?<br>'), - _('''None - no verification steps (<em>Not + _("""None - no verification steps (<em>Not Recommended </em>)<br> Confirm (*) - email confirmation step required <br> Require approval - require list administrator @@ -75,7 +76,7 @@ class Privacy(GUIBase): This prevents mischievous (or malicious) people from creating subscriptions for others without - their consent.''')) + their consent.""")) else: sub_cfentry = ('subscribe_policy', mm_cfg.Radio, # choices @@ -84,7 +85,7 @@ class Privacy(GUIBase): _('Confirm and approve')), 1, _('What steps are required for subscription?<br>'), - _('''Confirm (*) - email confirmation required <br> + _("""Confirm (*) - email confirmation required <br> Require approval - require list administrator approval for subscriptions <br> Confirm and approve - both confirm and approve @@ -94,7 +95,7 @@ class Privacy(GUIBase): subscription request number that they must reply to in order to subscribe.<br> This prevents mischievous (or malicious) people from creating - subscriptions for others without their consent.''')) + subscriptions for others without their consent.""")) # some helpful values admin = mlist.GetScriptURL('admin') @@ -108,11 +109,23 @@ class Privacy(GUIBase): _('Subscribing'), ('advertised', mm_cfg.Radio, (_('No'), _('Yes')), 0, - _('''Advertise this list when people ask what lists are on this - machine?''')), + _("""Advertise this list when people ask what lists are on this + machine?""")), sub_cfentry, + ('subscribe_auto_approval', mm_cfg.EmailListEx, (10, WIDTH), 1, + _("""List of addresses (or regexps) whose subscriptions do not + require approval."""), + + (_("""When subscription requires approval, addresses in this list + are allowed to subscribe without administrator approval. Add + addresses one per line. You may begin a line with a ^ character + to designate a (case insensitive) regular expression match.""") + + ' ' + + _("""You may also use the @listname notation to designate the + members of another list in this installation."""))), + ('unsubscribe_policy', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Is the list moderator's approval required for unsubscription requests? (<em>No</em> is recommended)"""), @@ -143,8 +156,8 @@ class Privacy(GUIBase): (_('Anyone'), _('List members'), _('List admin only')), 0, _('Who can view subscription list?'), - _('''When set, the list of subscribers is protected by member or - admin password authentication.''')), + _("""When set, the list of subscribers is protected by member or + admin password authentication.""")), ('obscure_addresses', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Show member addresses so they're not directly recognizable @@ -158,6 +171,11 @@ class Privacy(GUIBase): ] adminurl = mlist.GetScriptURL('admin', absolute=1) + + if mlist.dmarc_quarantine_moderation_action: + quarantine = _('/Quarantine') + else: + quarantine = '' sender_rtn = [ _("""When a message is posted to the list, a series of moderation steps are taken to decide whether a moderator must @@ -212,6 +230,37 @@ class Privacy(GUIBase): <a href="%(adminurl)s/members">membership management screens</a>.""")), + ('member_verbosity_threshold', mm_cfg.Number, 5, 0, + _("""Ceiling on acceptable number of member posts, per interval, + before automatic moderation."""), + + _("""If a member posts this many times, within a period of time + the member is automatically moderated. Use 0 to disable. See + <a href="?VARHELP=privacy/sender/member_verbosity_interval" + >member_verbosity_interval</a> for details on the time period. + + <p>This is intended to stop people who join a list or lists and + then use a bot to send many spam messages in a short interval. + + <p>Be careful when using this setting. If it is set too low, + this can be triggered by a single post cross-posted to + multiple lists or by a single post to an umbrella list.""")), + + ('member_verbosity_interval', mm_cfg.Number, 5, 0, + _("""Number of seconds to remember posts to this list to determine + member_verbosity_threshold for automatic moderation of a + member."""), + + _("""If a member's total posts to all lists in this installation + with member_verbosity_threshold enabled reaches this list's + member_verbosity_threshold, the member is automatically + moderated on this list. + + <p>Posts which are counted towards this list's + member_verbosity_threshold are all posts to any list with + member_verbosity_threshold enabled that arrived within that + list's member_verbosity_interval.""")), + ('member_moderation_action', mm_cfg.Radio, (_('Hold'), _('Reject'), _('Discard')), 0, _("""Action to take when a moderated member posts to the @@ -235,6 +284,119 @@ class Privacy(GUIBase): >rejection notice</a> to be sent to moderated members who post to this list.""")), + ('dmarc_moderation_action', mm_cfg.Radio, + (_('Accept'), _('Munge From'), _('Wrap Message'), _('Reject'), + _('Discard')), 0, + _("""Action to take when anyone posts to the + list from a domain with a DMARC Reject%(quarantine)s Policy."""), + + _("""<ul><li><b>Munge From</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is_list Munge From</a> + transformation to these messages. + + <p><li><b>Wrap Message</b> -- applies the <a + href="?VARHELP=general/from_is_list">from_is_list Wrap + Message</a> transformation to these messages. + + <p><li><b>Reject</b> -- this automatically rejects the message by + sending a bounce notice to the post's author. The text of the + bounce notice can be <a + href="?VARHELP=privacy/sender/dmarc_moderation_notice" + >configured by you</a>. + + <p><li><b>Discard</b> -- this simply discards the message, with + no notice sent to the post's author. + </ul> + + <p>This setting takes precedence over the <a + href="?VARHELP=general/from_is_list"> from_is_list</a> setting + if the message is From: an affected domain and the setting is + other than Accept.""")), + + ('dmarc_quarantine_moderation_action', mm_cfg.Radio, + (_('No'), _('Yes')), 0, + _("""Shall the above dmarc_moderation_action apply to messages + From: domains with DMARC p=quarantine as well as p=reject"""), + + _("""<ul><li><b>No</b> -- this applies dmarc_moderation_action to + only those posts From: a domain with DMARC p=reject. This is + appropriate if you are concerned about bounced messages, but + want to apply dmarc_moderation_action to as few messages as + possible. + <p><li><b>Yes</b> -- this applies dmarc_moderation_action to + posts From: a domain with DMARC p=reject or p=quarantine. + </ul><p>If a message is From: a domain with DMARC p=quarantine + and dmarc_moderation_action is not applied (this set to No) + the message will likely not bounce, but will be delivered to + recipients' spam folders or other hard to find places.""")), + + ('dmarc_none_moderation_action', mm_cfg.Radio, + (_('No'), _('Yes')), 0, + _("""Shall the above dmarc_moderation_action apply to messages + From: domains with DMARC p=none as well as p=quarantine and + p=reject"""), + + _("""<ul><li><b>No</b> -- this applies dmarc_moderation_action to + only those posts From: a domain with DMARC p=reject and + possibly p=quarantine depending on the setting of + dmarc_quarantine_moderation_action. + <p><li><b>Yes</b> -- this applies dmarc_moderation_action to + posts From: a domain with DMARC p=none if + dmarc_moderation_action is Munge From or Wrap Message and + dmarc_quarantine_moderation_action is Yes. + <p>The intent of this setting is to eliminate failure reports + to the owner of a domain that publishes DMARC p=none by applying + the message transformations that would be applied if the + domain's DMARC policy were stronger.""")), + + ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1, + _("""Text to include in any + <a href="?VARHELP=privacy/sender/dmarc_moderation_action" + >rejection notice</a> to + be sent to anyone who posts to this list from a domain + with a DMARC Reject%(quarantine)s Policy.""")), + + ('dmarc_wrapped_message_text', mm_cfg.Text, (10, WIDTH), 1, + _("""If dmarc_moderation_action applies and is Wrap Message, + and this text is provided, the text will be placed in a + separate text/plain MIME part preceding the original message + part in the wrapped message."""), + + _("""A wrapped message will either be a multipart/mixed message + with up to four sub-parts; a text/plain part containing + msg_header, a text/plain part containing + dmarc_wrapped_message_text, a message/rfc822 part containing the + original message and a text/plain part containing msg_footer, or + a message/rfc822 message containing only the original message if + none of the other parts are applicable.""")), + + ('equivalent_domains', mm_cfg.Text, (10, WIDTH), 1, + _("""A 'two dimensional' list of email address domains which are + considered equivalent when checking if a post is from a list + member."""), + + _("""If two poster addresses with the same local part but + different domains are to be considered equivalents for list + membership tests, the domains are put here. The format is + one or more groups of equivalent domains. Within a group, + the domains are separated by commas and multiple groups are + separated by semicolons. White space is ignored. + <p>For example:<pre> + example.com,mail.example.com;mac.com,me.com,icloud.com + </pre> + <p>In this example, if user@example.com is a list member, + a post from user@mail.example.com will be treated as if it is + from user@example.com for list membership/moderation purposes, + and likewise, if user@me.com is a list member, posts from + user@mac.com or user@icloud.com will be treated as if from + user@me.com. + <p>Note that the poster's address is first tested for list + membership, and the equivalent domain addresses are only tested + if the poster's address is not that of a member. + <p>Also note that moderation of the equivalent domain address + will apply to the post, but other options such as 'ack' or + 'not metoo' will not.""")), + _('Non-member filters'), ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1, @@ -373,8 +535,8 @@ class Privacy(GUIBase): ('max_num_recipients', mm_cfg.Number, 5, 0, _('Ceiling on acceptable number of recipients for a posting.'), - _('''If a posting has this number, or more, of recipients, it is - held for admin approval. Use 0 for no ceiling.''')), + _("""If a posting has this number, or more, of recipients, it is + held for admin approval. Use 0 for no ceiling.""")), ] spam_rtn = [ @@ -399,7 +561,7 @@ class Privacy(GUIBase): case, each rule is matched in turn, with processing stopped after the first match. - Note that headers are collected from all the attachments + Note that headers are collected from all the attachments (except for the mailman administrivia message) and matched against the regular expressions. With this feature, you can effectively sort out messages with dangerous file @@ -442,6 +604,11 @@ class Privacy(GUIBase): # an option. if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE: val += 1 + if (property == 'dmarc_moderation_action' and + val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION): + doc.addError(_("""dmarc_moderation_action must be >= the configured + default value.""")) + val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION setattr(mlist, property, val) # We need to handle the header_filter_rules widgets specially, but @@ -492,9 +659,20 @@ class Privacy(GUIBase): doc.addError(_("""Header filter rules require a pattern. Incomplete filter rules will be ignored.""")) continue - # Make sure the pattern was a legal regular expression + # Make sure the pattern was a legal regular expression. + # Convert it to unicode if necessary. + mo = re.match('.*charset=([-_a-z0-9]+)', + os.environ.get('CONTENT_TYPE', ''), + re.IGNORECASE + ) + if mo: + cset = mo.group(1) + else: + cset = Utils.GetCharSet(mlist.preferred_language) try: - re.compile(pattern) + upattern = Utils.xml_to_unicode(pattern, cset) + re.compile(upattern) + pattern = upattern except (re.error, TypeError): safepattern = Utils.websafe(pattern) doc.addError(_("""The header filter rule pattern diff --git a/Mailman/Gui/Topics.py b/Mailman/Gui/Topics.py index 96f9b421..ec60dbda 100644 --- a/Mailman/Gui/Topics.py +++ b/Mailman/Gui/Topics.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -29,6 +29,8 @@ except NameError: True = 1 False = 0 +OR = '|' + class Topics(GUIBase): @@ -126,10 +128,10 @@ class Topics(GUIBase): # Make sure the pattern was a legal regular expression name = Utils.websafe(name) try: - # Tagger compiles in verbose mode so we do too. - re.compile(pattern, re.VERBOSE) + orpattern = OR.join(pattern.splitlines()) + re.compile(orpattern) except (re.error, TypeError): - safepattern = Utils.websafe(pattern) + safepattern = Utils.websafe(orpattern) doc.addError(_("""The topic pattern '%(safepattern)s' is not a legal regular expression. It will be discarded.""")) continue diff --git a/Mailman/HTMLFormatter.py b/Mailman/HTMLFormatter.py index dad51e74..2a3e08db 100644 --- a/Mailman/HTMLFormatter.py +++ b/Mailman/HTMLFormatter.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -28,6 +28,8 @@ from Mailman.htmlformat import * from Mailman.i18n import _ +from Mailman.CSRFcheck import csrf_token + EMPTYSTRING = '' BR = '<br>' @@ -38,8 +40,7 @@ COMMASPACE = ', ' class HTMLFormatter: def GetMailmanFooter(self): - ownertext = COMMASPACE.join([Utils.ObscureEmail(a, 1) - for a in self.owner]) + ownertext = Utils.ObscureEmail(self.GetOwnerEmail(), 1) # Remove the .Format() when htmlformat conversion is done. realname = self.real_name hostname = self.host_name @@ -90,6 +91,9 @@ class HTMLFormatter: showing = Utils.ObscureEmail(person, for_text=1) else: showing = person + realname = Utils.uncanonstr(self.getMemberName(person), lang) + if realname and mm_cfg.ROSTER_DISPLAY_REALNAME: + showing += " (%s)" % Utils.websafe(realname) got = Link(url, showing) if self.getDeliveryStatus(person) <> MemberAdaptor.ENABLED: got = Italic('(', got, ')') @@ -314,12 +318,17 @@ class HTMLFormatter: container.AddItem("</center>") return container - def FormatFormStart(self, name, extra=''): + def FormatFormStart(self, name, extra='', + mlist=None, contexts=None, user=None): base_url = self.GetScriptURL(name) if extra: full_url = "%s/%s" % (base_url, extra) else: full_url = base_url + if mlist: + return ("""<form method="POST" action="%s"> +<input type="hidden" name="csrf_token" value="%s">""" + % (full_url, csrf_token(mlist, contexts, user))) return ('<FORM Method=POST ACTION="%s">' % full_url) def FormatArchiveAnchor(self): diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py index 038034c7..549d8e79 100644 --- a/Mailman/Handlers/AvoidDuplicates.py +++ b/Mailman/Handlers/AvoidDuplicates.py @@ -24,6 +24,7 @@ warning header, or pass it through, depending on the user's preferences. from email.Utils import getaddresses, formataddr from Mailman import mm_cfg +from Mailman.Handlers.CookHeaders import change_header COMMASPACE = ', ' @@ -95,6 +96,10 @@ def process(mlist, msg, msgdata): # Set the new list of recipients msgdata['recips'] = newrecips # RFC 2822 specifies zero or one CC header - del msg['cc'] if ccaddrs: - msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]) + change_header('Cc', + COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]), + mlist, msg, msgdata) + else: + del msg['cc'] + diff --git a/Mailman/Handlers/CalcRecips.py b/Mailman/Handlers/CalcRecips.py index 39fe0671..069c88a8 100755 --- a/Mailman/Handlers/CalcRecips.py +++ b/Mailman/Handlers/CalcRecips.py @@ -63,7 +63,8 @@ def process(mlist, msg, msgdata): missing = [] password = msg.get('urgent', missing) if password is not missing: - if mlist.Authenticate((mm_cfg.AuthListModerator, + if mlist.Authenticate((mm_cfg.AuthListPoster, + mm_cfg.AuthListModerator, mm_cfg.AuthListAdmin), password): recips = mlist.getMemberCPAddresses(mlist.getRegularMemberKeys() + @@ -183,6 +184,12 @@ def do_exclude(mlist, msg, msgdata, recips): for sender in msg.get_senders(): if slist.isMember(sender): break + for sender in Utils.check_eq_domains(sender, + slist.equivalent_domains): + if slist.isMember(sender): + break + if slist.isMember(sender): + break else: continue srecips = set([slist.getMemberCPAddress(m) diff --git a/Mailman/Handlers/Cleanse.py b/Mailman/Handlers/Cleanse.py index 725cb41b..5270bb5a 100644 --- a/Mailman/Handlers/Cleanse.py +++ b/Mailman/Handlers/Cleanse.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2010 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -19,12 +19,34 @@ import re -from email.Utils import formataddr +from email.Utils import formataddr, getaddresses, parseaddr +from Mailman import mm_cfg from Mailman.Utils import unique_message_id from Mailman.Logging.Syslog import syslog from Mailman.Handlers.CookHeaders import uheader +cres = [] +for regexp in mm_cfg.ANONYMOUS_LIST_KEEP_HEADERS: + try: + if regexp.endswith(':'): + regexp = regexp[:-1] + '$' + cres.append(re.compile(regexp, re.IGNORECASE)) + except re.error, e: + syslog('error', + 'ANONYMOUS_LIST_KEEP_HEADERS: ignored bad regexp %s: %s', + regexp, e) + +def remove_nonkeepers(msg): + for hdr in msg.keys(): + keep = False + for cre in cres: + if cre.search(hdr): + keep = True + break + if not keep: + del msg[hdr] + def process(mlist, msg, msgdata): # Always remove this header from any outgoing messages. Be sure to do @@ -38,6 +60,9 @@ def process(mlist, msg, msgdata): del msg['x-approve'] # Also remove this header since it can contain a password del msg['urgent'] + # If we're anonymizing, we need to save the sender here, and we may as + # well do it for all. + msgdata['original_sender'] = msg.get_sender() # We remove other headers from anonymous lists if mlist.anonymous_list: syslog('post', 'post to %s from %s anonymized', @@ -45,6 +70,7 @@ def process(mlist, msg, msgdata): del msg['from'] del msg['reply-to'] del msg['sender'] + del msg['organization'] del msg['return-path'] # Hotmail sets this one del msg['x-originating-email'] @@ -53,6 +79,10 @@ def process(mlist, msg, msgdata): # And so can the message-id so replace it. del msg['message-id'] msg['Message-ID'] = unique_message_id(mlist) + # And something sets this + del msg['x-envelope-from'] + # And now remove all but the keepers. + remove_nonkeepers(msg) i18ndesc = str(uheader(mlist, mlist.description, 'From')) msg['From'] = formataddr((i18ndesc, mlist.GetListEmail())) msg['Reply-To'] = mlist.GetListEmail() diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py index c4b06613..3e70313b 100644 --- a/Mailman/Handlers/CleanseDKIM.py +++ b/Mailman/Handlers/CleanseDKIM.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006-2007 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -29,8 +29,28 @@ from Mailman import mm_cfg def process(mlist, msg, msgdata): - if mm_cfg.REMOVE_DKIM_HEADERS: - del msg['domainkey-signature'] - del msg['dkim-signature'] - del msg['authentication-results'] + if not (mm_cfg.REMOVE_DKIM_HEADERS or mlist.anonymous_list): + # We want to remove these headers from posts to anonymous lists. + # There can be interaction with the next test, but anonymous_list + # and Munge From are not compatible anyway, so don't worry. + return + if (mm_cfg.REMOVE_DKIM_HEADERS == 1 and not + # The following means 'Munge From' applies to this message. + # So this whole stanza means if RDH is 1 and we're not Munging, + # return and don't remove the headers. See Defaults.py. + (msgdata.get('from_is_list') == 1 or + (mlist.from_is_list == 1 and msgdata.get('from_is_list') != 2) + ) + ): + return + if (mm_cfg.REMOVE_DKIM_HEADERS == 3): + for value in msg.get_all('domainkey-signature', []): + msg['X-Mailman-Original-DomainKey-Signature'] = value + for value in msg.get_all('dkim-signature', []): + msg['X-Mailman-Original-DKIM-Signature'] = value + for value in msg.get_all('authentication-results', []): + msg['X-Mailman-Original-Authentication-Results'] = value + del msg['domainkey-signature'] + del msg['dkim-signature'] + del msg['authentication-results'] diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py index a2096172..3e2806f0 100755 --- a/Mailman/Handlers/CookHeaders.py +++ b/Mailman/Handlers/CookHeaders.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,7 +15,10 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -"""Cook a message's Subject header.""" +"""Cook a message's Subject header. +Also do other manipulations of From:, Reply-To: and Cc: depending on +list configuration. +""" from __future__ import nested_scopes import re @@ -26,12 +29,13 @@ from email.Header import Header, decode_header, make_header from email.Utils import parseaddr, formataddr, getaddresses from email.Errors import HeaderParseError +from Mailman import i18n from Mailman import mm_cfg from Mailman import Utils from Mailman.i18n import _ from Mailman.Logging.Syslog import syslog -CONTINUATION = ',\n\t' +CONTINUATION = ',\n ' COMMASPACE = ', ' MAXLINELEN = 78 @@ -49,7 +53,7 @@ def _isunicode(s): nonascii = re.compile('[^\s!-~]') -def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): +def uheader(mlist, s, header_name=None, continuation_ws=' ', maxlinelen=None): # Get the charset to encode the string in. Then search if there is any # non-ascii character is in the string. If there is and the charset is # us-ascii then we use iso-8859-1 instead. If the string is ascii only @@ -62,20 +66,42 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): else: # there is no nonascii so ... charset = 'us-ascii' - return Header(s, charset, maxlinelen, header_name, continuation_ws) + try: + return Header(s, charset, maxlinelen, header_name, continuation_ws) + except UnicodeError: + syslog('error', 'list: %s: can\'t decode "%s" as %s', + mlist.internal_name(), s, charset) + return Header('', charset, maxlinelen, header_name, continuation_ws) + +def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True): + if ((msgdata.get('from_is_list') == 2 or + (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and + not msgdata.get('_fasttrack') + ) or name.lower() in ('from', 'reply-to', 'cc'): + # The or name.lower() in ... above is because when we are munging + # the From:, we want to defer the resultant changes to From:, + # Reply-To:, and/or Cc: until after the message passes through + # ToDigest, ToArchive and ToUsenet. Thus, we put them in + # msgdata[add_header] here and apply them in WrapMessage. + msgdata.setdefault('add_header', {})[name] = value + elif repl or not msg.has_key(name): + if delete: + del msg[name] + msg[name] = value def process(mlist, msg, msgdata): # Set the "X-Ack: no" header if noack flag is set. if msgdata.get('noack'): - del msg['x-ack'] - msg['X-Ack'] = 'no' + change_header('X-Ack', 'no', mlist, msg, msgdata) # Because we're going to modify various important headers in the email # message, we want to save some of the information in the msgdata # dictionary for later. Specifically, the sender header will get waxed, # but we need it for the Acknowledge module later. - msgdata['original_sender'] = msg.get_sender() + # We may have already saved it; if so, don't clobber it here. + if 'original_sender' not in msgdata: + msgdata['original_sender'] = msg.get_sender() # VirginRunner sets _fasttrack for internally crafted messages. fasttrack = msgdata.get('_fasttrack') if not msgdata.get('isdigest') and not fasttrack: @@ -87,7 +113,8 @@ def process(mlist, msg, msgdata): pass # Mark message so we know we've been here, but leave any existing # X-BeenThere's intact. - msg['X-BeenThere'] = mlist.GetListEmail() + change_header('X-BeenThere', mlist.GetListEmail(), + mlist, msg, msgdata, delete=False) # Add Precedence: and other useful headers. None of these are standard # and finding information on some of them are fairly difficult. Some are # just common practice, and we'll add more here as they become necessary. @@ -101,12 +128,68 @@ def process(mlist, msg, msgdata): # known exploits in a particular version of Mailman and we know a site is # using such an old version, they may be vulnerable. It's too easy to # edit the code to add a configuration variable to handle this. - if not msg.has_key('x-mailman-version'): - msg['X-Mailman-Version'] = mm_cfg.VERSION + change_header('X-Mailman-Version', mm_cfg.VERSION, + mlist, msg, msgdata, repl=False) # We set "Precedence: list" because this is the recommendation from the # sendmail docs, the most authoritative source of this header's semantics. - if not msg.has_key('precedence'): - msg['Precedence'] = 'list' + change_header('Precedence', 'list', + mlist, msg, msgdata, repl=False) + # Do we change the from so the list takes ownership of the email + if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack: + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if not realname: + if mlist.isMember(email): + realname = mlist.getMemberName(email) or email + else: + realname = email + # Remove domain from realname if it looks like an email address + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + # Make a display name and RFC 2047 encode it if necessary. This is + # difficult and kludgy. If the realname came from From: it should be + # ascii or RFC 2047 encoded. If it came from the list, it should be + # in the charset of the list's preferred language or possibly unicode. + # if it's from the email address, it should be ascii. In any case, + # make it a unicode. + if isinstance(realname, unicode): + urn = realname + else: + rn, cs = ch_oneline(realname) + urn = unicode(rn, cs, errors='replace') + # likewise, the list's real_name which should be ascii, but use the + # charset of the list's preferred_language which should be a superset. + lcs = Utils.GetCharSet(mlist.preferred_language) + ulrn = unicode(mlist.real_name, lcs, errors='replace') + # get translated 'via' with dummy replacements + realname = '%(realname)s' + lrn = '%(lrn)s' + # We want the i18n context to be the list's preferred_language. It + # could be the poster's. + otrans = i18n.get_translation() + i18n.set_language(mlist.preferred_language) + via = _('%(realname)s via %(lrn)s') + i18n.set_translation(otrans) + uvia = unicode(via, lcs, errors='replace') + # Replace the dummy replacements. + uvia = re.sub(u'%\(lrn\)s', ulrn, re.sub(u'%\(realname\)s', urn, uvia)) + # And get an RFC 2047 encoded header string. + dn = str(Header(uvia, lcs)) + change_header('From', + formataddr((dn, mlist.GetListEmail())), + mlist, msg, msgdata) + else: + # Use this as a flag + o_from = None # Reply-To: munging. Do not do this if the message is "fast tracked", # meaning it is internally crafted and delivered to a specific user. BAW: # Yuck, I really hate this feature but I've caved under the sheer pressure @@ -115,6 +198,23 @@ def process(mlist, msg, msgdata): # augment it. RFC 2822 allows max one Reply-To: header so collapse them # if we're adding a value, otherwise don't touch it. (Should we collapse # in all cases?) + # MAS: We need to do some things with the original From: if we've munged + # it for DMARC mitigation. We have goals for this process which are + # not completely compatible, so we do the best we can. Our goals are: + # 1) as long as the list is not anonymous, the original From: address + # should be obviously exposed, i.e. not just in a header that MUAs + # don't display. + # 2) the original From: address should not be in a comment or display + # name in the new From: because it is claimed that multiple domains + # in any fields in From: are indicative of spamminess. This means + # it should be in Reply-To: or Cc:. + # 3) the behavior of an MUA doing a 'reply' or 'reply all' should be + # consistent regardless of whether or not the From: is munged. + # Goal 3) implies sometimes the original From: should be in Reply-To: + # and sometimes in Cc:, and even so, this goal won't be achieved in + # all cases with all MUAs. In cases of conflict, the above ordering of + # goals is priority order. + if not fasttrack: # A convenience function, requires nested scopes. pair is (name, addr) new = [] @@ -132,22 +232,43 @@ def process(mlist, msg, msgdata): # the original Reply-To:'s to the list we're building up. In both # cases we'll zap the existing field because RFC 2822 says max one is # allowed. + o_rt = False if not mlist.first_strip_reply_to: orig = msg.get_all('reply-to', []) for pair in getaddresses(orig): + # There's an original Reply-To: and we're not removing it. add(pair) + o_rt = True + # We also need to put the old From: in Reply-To: in all cases where + # it is not going in Cc:. This is when reply_goes_to_list == 0 and + # either there was no original Reply-To: or we stripped it. + # However, if there was an original Reply-To:, unstripped, and it + # contained the original From: address we need to flag that it's + # there so we don't add the original From: to Cc: + if o_from and mlist.reply_goes_to_list == 0: + if o_rt: + if d.has_key(o_from[1].lower()): + # Original From: address is in original Reply-To:. + # Pretend we added it. + o_from = None + else: + add(o_from) + # Flag that we added it. + o_from = None # Set Reply-To: header to point back to this list. Add this last # because some folks think that some MUAs make it easier to delete # addresses from the right than from the left. if mlist.reply_goes_to_list == 1: i18ndesc = uheader(mlist, mlist.description, 'Reply-To') add((str(i18ndesc), mlist.GetListEmail())) - del msg['reply-to'] # Don't put Reply-To: back if there's nothing to add! if new: # Preserve order - msg['Reply-To'] = COMMASPACE.join( - [formataddr(pair) for pair in new]) + change_header('Reply-To', + COMMASPACE.join([formataddr(pair) for pair in new]), + mlist, msg, msgdata) + else: + del msg['reply-to'] # The To field normally contains the list posting address. However # when messages are fully personalized, that header will get # overwritten with the address of the recipient. We need to get the @@ -158,18 +279,38 @@ def process(mlist, msg, msgdata): # above code? # Also skip Cc if this is an anonymous list as list posting address # is already in From and Reply-To in this case. - if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \ - and not mlist.anonymous_list: + # We do add the Cc in cases where From: header munging is being done + # because even though the list address is in From:, the Reply-To: + # poster will override it. Brain dead MUAs may then address the list + # twice on a 'reply all', but reasonable MUAs should do the right + # thing. We also add the original From: to Cc: if it wasn't added + # to Reply-To: + add_list = (mlist.personalize == 2 and + mlist.reply_goes_to_list <> 1 and + not mlist.anonymous_list) + if add_list or o_from: # Watch out for existing Cc headers, merge, and remove dups. Note # that RFC 2822 says only zero or one Cc header is allowed. new = [] d = {} - for pair in getaddresses(msg.get_all('cc', [])): - add(pair) - i18ndesc = uheader(mlist, mlist.description, 'Cc') - add((str(i18ndesc), mlist.GetListEmail())) - del msg['Cc'] - msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) + # If we're adding the original From:, add it first. + if o_from: + add(o_from) + # AvoidDuplicates may have set a new Cc: in msgdata.add_header, + # so check that. + if (msgdata.has_key('add_header') and + msgdata['add_header'].has_key('Cc')): + for pair in getaddresses([msgdata['add_header']['Cc']]): + add(pair) + else: + for pair in getaddresses(msg.get_all('cc', [])): + add(pair) + if add_list: + i18ndesc = uheader(mlist, mlist.description, 'Cc') + add((str(i18ndesc), mlist.GetListEmail())) + change_header('Cc', + COMMASPACE.join([formataddr(pair) for pair in new]), + mlist, msg, msgdata) # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only # if the message is being crafted for a specific list (e.g. not for the # password reminders). @@ -191,8 +332,7 @@ def process(mlist, msg, msgdata): # without desc we need to ensure the MUST brackets listid_h = '<%s>' % listid # We always add a List-ID: header. - del msg['list-id'] - msg['List-Id'] = listid_h + change_header('List-Id', listid_h, mlist, msg, msgdata) # For internally crafted messages, we also add a (nonstandard), # "X-List-Administrivia: yes" header. For all others (i.e. those coming # from list posts), we add a bunch of other RFC 2369 headers. @@ -219,13 +359,12 @@ def process(mlist, msg, msgdata): # First we delete any pre-existing headers because the RFC permits only # one copy of each, and we want to be sure it's ours. for h, v in headers.items(): - del msg[h] # Wrap these lines if they are too long. 78 character width probably # shouldn't be hardcoded, but is at least text-MUA friendly. The # adding of 2 is for the colon-space separator. if len(h) + 2 + len(v) > 78: v = CONTINUATION.join(v.split(', ')) - msg[h] = v + change_header(h, v, mlist, msg, msgdata) @@ -242,7 +381,7 @@ def prefix_subject(mlist, msg, msgdata): lines = str(subject).splitlines() else: lines = subject.splitlines() - ws = '\t' + ws = ' ' if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': ws = lines[1][0] msgdata['origsubj'] = subject @@ -272,16 +411,29 @@ def prefix_subject(mlist, msg, msgdata): else: old_style = mm_cfg.OLD_STYLE_PREFIXING subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)\s*(\[\d+\])?\s*:\s*)+', subject, re.I) + # Previously the following re didn't have the first \s*. It would fail + # if the incoming Subject: was like '[prefix] Re: Re: Re:' because of the + # leading space after stripping the prefix. It is not known what MUA would + # create such a Subject:, but the issue was reported. + rematch = re.match( + '(\s*(RE|AW|SV|VS)\s*(\[\d+\])?\s*:\s*)+', + subject, re.I) if rematch: subject = subject[rematch.end():] recolon = 'Re:' else: recolon = '' + # Strip leading and trailing whitespace from subject. + subject = subject.strip() # At this point, subject may become null if someone post mail with - # subject: [subject prefix] - if subject.strip() == '': + # Subject: [subject prefix] + if subject == '': + # We want the i18n context to be the list's preferred_language. It + # could be the poster's. + otrans = i18n.get_translation() + i18n.set_language(mlist.preferred_language) subject = _('(no subject)') + i18n.set_translation(otrans) cset = Utils.GetCharSet(mlist.preferred_language) subject = unicode(subject, cset) # and substitute %d in prefix with post_id @@ -302,8 +454,7 @@ def prefix_subject(mlist, msg, msgdata): h = u' '.join([prefix, subject]) h = h.encode('us-ascii') h = uheader(mlist, h, 'Subject', continuation_ws=ws) - del msg['subject'] - msg['Subject'] = h + change_header('Subject', h, mlist, msg, msgdata) ss = u' '.join([recolon, subject]) ss = ss.encode('us-ascii') ss = uheader(mlist, ss, 'Subject', continuation_ws=ws) @@ -312,6 +463,11 @@ def prefix_subject(mlist, msg, msgdata): except UnicodeError: pass # Get the header as a Header instance, with proper unicode conversion + # Because of rfc2047 encoding, spaces between encoded words can be + # insignificant, so we need to append spaces to our encoded stuff. + prefix += ' ' + if recolon: + recolon += ' ' if old_style: h = uheader(mlist, recolon, 'Subject', continuation_ws=ws) h.append(prefix) @@ -321,8 +477,7 @@ def prefix_subject(mlist, msg, msgdata): # TK: Subject is concatenated and unicode string. subject = subject.encode(cset, 'replace') h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h + change_header('Subject', h, mlist, msg, msgdata) ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) ss.append(subject, cset) msgdata['stripped_subject'] = ss diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py index 69e86d5b..d1c8c5b4 100644 --- a/Mailman/Handlers/Decorate.py +++ b/Mailman/Handlers/Decorate.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -210,7 +210,11 @@ def process(mlist, msg, msgdata): def decorate(mlist, template, what, extradict=None): # `what' is just a descriptive phrase used in the log message - # + + # If template is only whitespace, ignore it. + if len(re.sub('\s', '', template)) == 0: + return '' + # BAW: We've found too many situations where Python can be fooled into # interpolating too much revealing data into a format string. For # example, a footer of "% silly %(real_name)s" would give a header @@ -240,4 +244,7 @@ def decorate(mlist, template, what, extradict=None): except (ValueError, TypeError), e: syslog('error', 'Exception while calculating %s:\n%s', what, e) text = template + # Ensure text ends with new-line + if not text.endswith('\n'): + text += '\n' return text diff --git a/Mailman/Handlers/Hold.py b/Mailman/Handlers/Hold.py index d0d22690..2faebae1 100644 --- a/Mailman/Handlers/Hold.py +++ b/Mailman/Handlers/Hold.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -220,7 +220,12 @@ def hold_for_approval(mlist, msg, msgdata, exc): # We need to send both the reason and the rejection notice through the # translator again, because of the games we play above reason = Utils.wrap(exc.reason_notice()) - msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist)) + if isinstance(exc, NonMemberPost) and mlist.nonmember_rejection_notice: + msgdata['rejection_notice'] = Utils.wrap( + mlist.nonmember_rejection_notice.replace( + '%(listowner)s', owneraddr)) + else: + msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist)) id = mlist.HoldMessage(msg, reason, msgdata) # Now we need to craft and send a message to the list admin so they can # deal with the held message. @@ -264,7 +269,8 @@ def hold_for_approval(mlist, msg, msgdata, exc): d['subject'] = usersubject # craft the admin notification message and deliver it subject = _('%(listname)s post from %(sender)s requires approval') - nmsg = Message.OwnerNotification(mlist, subject, tomoderators=1) + nmsg = Message.UserNotification(owneraddr, owneraddr, subject, + lang=lang) nmsg.set_type('multipart/mixed') text = MIMEText( Utils.maketext('postauth.txt', d, raw=1, mlist=mlist), diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py index ab7483ba..691a6e85 100644 --- a/Mailman/Handlers/MimeDel.py +++ b/Mailman/Handlers/MimeDel.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -210,6 +210,11 @@ def recast_multipart(msg): # If we're left with a multipart message with only one sub-part, recast # the message to just the sub-part, but not if the part is message/rfc822 # because we don't want to lose the headers. + # Also, if this is a multipart/signed part, stop now as the original part + # may have had a multipart sub-part with only one sub-sub-part, the sig + # may still be valid and going further may break it. (LP: #1551075) + if msg.get_content_type() == 'multipart/signed': + return if msg.is_multipart(): if (len(msg.get_payload()) == 1 and msg.get_content_type() <> 'message/rfc822'): diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py index f9e79cbe..49ed1d7e 100644 --- a/Mailman/Handlers/Moderate.py +++ b/Mailman/Handlers/Moderate.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -21,6 +21,7 @@ import re from email.MIMEMessage import MIMEMessage from email.MIMEText import MIMEText +from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Utils @@ -47,12 +48,18 @@ class ModeratedMemberPost(Hold.ModeratedPost): def process(mlist, msg, msgdata): - if msgdata.get('approved') or msgdata.get('fromusenet'): + if msgdata.get('approved'): return - # First of all, is the poster a member or not? + # Is the poster a member or not? for sender in msg.get_senders(): if mlist.isMember(sender): break + for sender in Utils.check_eq_domains(sender, + mlist.equivalent_domains): + if mlist.isMember(sender): + break + if mlist.isMember(sender): + break else: sender = None if sender: @@ -90,22 +97,34 @@ def process(mlist, msg, msgdata): sender = msg.get_sender() # From here on out, we're dealing with non-members. listname = mlist.internal_name() - if matches_p(sender, mlist.accept_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.accept_these_nonmembers, + at_list='accept_these_nonmembers' + ): return - if matches_p(sender, mlist.hold_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.hold_these_nonmembers, + at_list='hold_these_nonmembers' + ): Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost) # No return - if matches_p(sender, mlist.reject_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.reject_these_nonmembers, + at_list='reject_these_nonmembers' + ): do_reject(mlist) # No return - if matches_p(sender, mlist.discard_these_nonmembers, listname): + if mlist.GetPattern(sender, + mlist.discard_these_nonmembers, + at_list='discard_these_nonmembers' + ): do_discard(mlist, msg) # No return # Okay, so the sender wasn't specified explicitly by any of the non-member # moderation configuration variables. Handle by way of generic non-member # action. assert 0 <= mlist.generic_nonmember_action <= 4 - if mlist.generic_nonmember_action == 0: + if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'): # Accept return elif mlist.generic_nonmember_action == 1: @@ -117,43 +136,6 @@ def process(mlist, msg, msgdata): -def matches_p(sender, nonmembers, listname): - # First strip out all the regular expressions and listnames - plainaddrs = [addr for addr in nonmembers if not (addr.startswith('^') - or addr.startswith('@'))] - addrdict = Utils.List2Dict(plainaddrs, foldcase=1) - if addrdict.has_key(sender): - return 1 - # Now do the regular expression matches - for are in nonmembers: - if are.startswith('^'): - try: - cre = re.compile(are, re.IGNORECASE) - except re.error: - continue - if cre.search(sender): - return 1 - elif are.startswith('@'): - # XXX Needs to be reviewed for list@domain names. - try: - mname = are[1:].lower().strip() - if mname == listname: - # don't reference your own list - syslog('error', - '*_these_nonmembers in %s references own list', - listname) - else: - mother = MailList(mname, lock=0) - if mother.isMember(sender): - return 1 - except Errors.MMUnknownListError: - syslog('error', - '*_these_nonmembers in %s references non-existent list %s', - listname, mname) - return 0 - - - def do_reject(mlist): listowner = mlist.GetOwnerEmail() if mlist.nonmember_rejection_notice: @@ -161,9 +143,10 @@ def do_reject(mlist): Utils.wrap(_(mlist.nonmember_rejection_notice)) else: raise Errors.RejectMessage, Utils.wrap(_("""\ -You are not allowed to post to this mailing list, and your message has been -automatically rejected. If you think that your messages are being rejected in -error, contact the mailing list owner at %(listowner)s.""")) +Your message has been rejected, probably because you are not subscribed to the +mailing list and the list's policy is to prohibit non-members from posting to +it. If you think that your messages are being rejected in error, contact the +mailing list owner at %(listowner)s.""")) @@ -174,9 +157,10 @@ def do_discard(mlist, msg): lang = mlist.preferred_language varhelp = '%s/?VARHELP=privacy/sender/discard_these_nonmembers' % \ mlist.GetScriptURL('admin', absolute=1) - nmsg = Message.OwnerNotification(mlist, + nmsg = Message.UserNotification(mlist.GetOwnerEmail(), + mlist.GetBouncesEmail(), _('Auto-discard notification'), - lang=lang, tomoderators=0) + lang=lang) nmsg.set_type('multipart/mixed') text = MIMEText(Utils.wrap(_( 'The attached message has been automatically discarded.')), diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py index 1d11d19a..ca6aebdd 100644 --- a/Mailman/Handlers/SMTPDirect.py +++ b/Mailman/Handlers/SMTPDirect.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -30,6 +30,7 @@ import copy import time import socket import smtplib +from base64 import b64encode from types import UnicodeType from Mailman import mm_cfg @@ -61,7 +62,38 @@ class Connection: def __connect(self): self.__conn = smtplib.SMTP() + self.__conn.set_debuglevel(mm_cfg.SMTPLIB_DEBUG_LEVEL) self.__conn.connect(mm_cfg.SMTPHOST, mm_cfg.SMTPPORT) + if mm_cfg.SMTP_AUTH: + if mm_cfg.SMTP_USE_TLS: + try: + self.__conn.starttls() + except SMTPException, e: + syslog('smtp-failure', 'SMTP TLS error: %s', e) + self.quit() + raise + try: + self.__conn.ehlo(mm_cfg.SMTP_HELO_HOST) + except SMTPException, e: + syslog('smtp-failure', 'SMTP EHLO error: %s', e) + self.quit() + raise + try: + self.__conn.login(mm_cfg.SMTP_USER, mm_cfg.SMTP_PASSWD) + except smtplib.SMTPHeloError, e: + syslog('smtp-failure', 'SMTP HELO error: %s', e) + self.quit() + raise + except smtplib.SMTPAuthenticationError, e: + syslog('smtp-failure', 'SMTP AUTH error: %s', e) + self.quit() + raise + except smtplib.SMTPException, e: + syslog('smtp-failure', + 'SMTP - no suitable authentication method found: %s', e) + self.quit() + raise + self.__numsessions = mm_cfg.SMTP_MAX_SESSIONS_PER_CONNECTION def sendmail(self, envsender, recips, msgtext): @@ -340,6 +372,10 @@ def verpdeliver(mlist, msg, msgdata, envsender, failures, conn): del msgcopy['x-mailman-copy'] if msgdata.get('add-dup-header', {}).has_key(recip): msgcopy['X-Mailman-Copy'] = 'yes' + # If desired, add the RCPT_BASE64_HEADER_NAME header + if len(mm_cfg.RCPT_BASE64_HEADER_NAME) > 0: + del msgcopy[mm_cfg.RCPT_BASE64_HEADER_NAME] + msgcopy[mm_cfg.RCPT_BASE64_HEADER_NAME] = b64encode(recip) # For the final delivery stage, we can just bulk deliver to a party of # one. ;) bulkdeliver(mlist, msgcopy, msgdata, envsender, failures, conn) diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py index 8d26da03..aaddff5f 100644 --- a/Mailman/Handlers/SpamDetect.py +++ b/Mailman/Handlers/SpamDetect.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,13 +27,17 @@ TBD: This needs to be made more configurable and robust. import re +from unicodedata import normalize +from email.Errors import HeaderParseError from email.Header import decode_header +from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Errors from Mailman import i18n -from Mailman.Utils import GetCharSet +from Mailman import Utils from Mailman.Handlers.Hold import hold_for_approval +from Mailman.Logging.Syslog import syslog try: True, False @@ -61,24 +65,81 @@ _ = i18n._ def getDecodedHeaders(msg, cset='utf-8'): - """Returns a string containing all the headers of msg, unfolded and - RFC 2047 decoded and encoded in cset. + """Returns a unicode containing all the headers of msg, unfolded and + RFC 2047 decoded, normalized and separated by new lines. """ - headers = '' + headers = u'' for h, v in msg.items(): uvalue = u'' - v = decode_header(re.sub('\n\s', ' ', v)) + try: + v = decode_header(re.sub('\n\s', ' ', v)) + except HeaderParseError: + v = [(v, 'us-ascii')] for frag, cs in v: if not cs: cs = 'us-ascii' - uvalue += unicode(frag, cs, 'replace') - headers += '%s: %s\n' % (h, uvalue.encode(cset, 'replace')) + try: + uvalue += unicode(frag, cs, 'replace') + except LookupError: + # The encoding charset is unknown. At this point, frag + # has been QP or base64 decoded into a byte string whose + # charset we don't know how to handle. We will try to + # unicode it as iso-8859-1 which may result in a garbled + # mess, but we have to do something. + uvalue += unicode(frag, 'iso-8859-1', 'replace') + uhdr = h.decode('us-ascii', 'replace') + headers += u'%s: %s\n' % (h, normalize(mm_cfg.NORMALIZE_FORM, uvalue)) return headers def process(mlist, msg, msgdata): + # Before anything else, check DMARC if necessary. We do this as early + # as possible so reject/discard actions trump other holds/approvals and + # wrap/munge actions get flagged even for approved messages. + # But not for owner mail which should not be subject to DMARC reject or + # discard actions. + if not msgdata.get('toowner'): + msgdata['from_is_list'] = 0 + dn, addr = parseaddr(msg.get('from')) + if addr and mlist.dmarc_moderation_action > 0: + if Utils.IsDMARCProhibited(mlist, addr): + # Note that for dmarc_moderation_action, 0 = Accept, + # 1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard + if mlist.dmarc_moderation_action == 1: + msgdata['from_is_list'] = 1 + elif mlist.dmarc_moderation_action == 2: + msgdata['from_is_list'] = 2 + elif mlist.dmarc_moderation_action == 3: + # Reject + text = mlist.dmarc_moderation_notice + if text: + text = Utils.wrap(text) + else: + text = Utils.wrap(_( +"""You are not allowed to post to this mailing list From: a domain which +publishes a DMARC policy of reject or quarantine, and your message has been +automatically rejected. If you think that your messages are being rejected in +error, contact the mailing list owner at %(listowner)s.""")) + raise Errors.RejectMessage, text + elif mlist.dmarc_moderation_action == 4: + raise Errors.DiscardMessage + + # Get member address if any. + for sender in msg.get_senders(): + if mlist.isMember(sender): + break + else: + sender = msg.get_sender() + if (mlist.member_verbosity_threshold > 0 and + Utils.IsVerboseMember(mlist, sender) + ): + mlist.setMemberOption(sender, mm_cfg.Moderate, 1) + syslog('vette', + '%s: Automatically Moderated %s for verbose postings.', + mlist.real_name, sender) + if msgdata.get('approved'): return # First do site hard coded header spam checks @@ -92,9 +153,9 @@ def process(mlist, msg, msgdata): # Now do header_filter_rules # TK: Collect headers in sub-parts because attachment filename # extension may be a clue to possible virus/spam. - headers = '' + headers = u'' # Get the character set of the lists preferred language for headers - lcset = GetCharSet(mlist.preferred_language) + lcset = Utils.GetCharSet(mlist.preferred_language) for p in msg.walk(): headers += getDecodedHeaders(p, lcset) for patterns, action, empty in mlist.header_filter_rules: @@ -106,7 +167,17 @@ def process(mlist, msg, msgdata): # ignore 'empty' patterns if not pattern.strip(): continue - if re.search(pattern, headers, re.IGNORECASE|re.MULTILINE): + pattern = Utils.xml_to_unicode(pattern, lcset) + pattern = normalize(mm_cfg.NORMALIZE_FORM, pattern) + try: + mo = re.search(pattern, + headers, + re.IGNORECASE|re.MULTILINE|re.UNICODE) + except (re.error, TypeError): + syslog('error', + 'ignoring header_filter_rules invalid pattern: %s', + pattern) + if mo: if action == mm_cfg.DISCARD: raise Errors.DiscardMessage if action == mm_cfg.REJECT: diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py index 38a8e465..ed9a7e71 100644 --- a/Mailman/Handlers/Tagger.py +++ b/Mailman/Handlers/Tagger.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,7 +27,9 @@ from email.Header import decode_header from Mailman import Utils from Mailman.Logging.Syslog import syslog +from Mailman.Handlers.CookHeaders import change_header +OR = '|' CRNL = '\r\n' EMPTYSTRING = '' NLTAB = '\n\t' @@ -62,15 +64,17 @@ def process(mlist, msg, msgdata): # added to the specific topics bucket. hits = {} for name, pattern, desc, emptyflag in mlist.topics: - cre = re.compile(pattern, re.IGNORECASE | re.VERBOSE) + pattern = OR.join(pattern.splitlines()) + cre = re.compile(pattern, re.IGNORECASE) for line in matchlines: if cre.search(line): hits[name] = 1 break if hits: msgdata['topichits'] = hits.keys() - msg['X-Topics'] = NLTAB.join(hits.keys()) - + change_header('X-Topics', NLTAB.join(hits.keys()), + mlist, msg, msgdata, delete=False) + def scanbody(msg, numlines=None): diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py index edbf40dc..046cbaba 100644 --- a/Mailman/Handlers/ToDigest.py +++ b/Mailman/Handlers/ToDigest.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -68,6 +68,17 @@ except NameError: +def to_cset_out(text, lcset): + # Convert text from unicode or lcset to output cset. + ocset = Charset(lcset).get_output_charset() or lcset + if isinstance(text, unicode): + return text.encode(ocset, errors='replace') + else: + return text.decode(lcset, errors='replace').encode(ocset, + errors='replace') + + + def process(mlist, msg, msgdata): # Short circuit non-digestable lists. if not mlist.digestable or msgdata.get('isdigest'): @@ -86,7 +97,8 @@ def process(mlist, msg, msgdata): # whether the size threshold has been reached. mboxfp.flush() size = os.path.getsize(mboxfile) - if size / 1024.0 >= mlist.digest_size_threshhold: + if (mlist.digest_size_threshhold > 0 and + size / 1024.0 >= mlist.digest_size_threshhold): # This is a bit of a kludge to get the mbox file moved to the digest # queue directory. try: @@ -203,8 +215,8 @@ def send_i18n_digests(mlist, mboxfp): # RFC 1153 print >> plainmsg, mastheadtxt print >> plainmsg - # Now add the optional digest header - if mlist.digest_header: + # Now add the optional digest header but only if more than whitespace. + if re.sub('\s', '', mlist.digest_header): headertxt = decorate(mlist, mlist.digest_header, _('digest header')) # MIME header = MIMEText(headertxt, _charset=lcset) @@ -298,7 +310,7 @@ def send_i18n_digests(mlist, mboxfp): if msgcount == 0: # Why did we even get here? return - toctext = toc.getvalue() + toctext = to_cset_out(toc.getvalue(), lcset) # MIME tocpart = MIMEText(toctext, _charset=lcset) tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)") @@ -353,8 +365,8 @@ def send_i18n_digests(mlist, mboxfp): print >> plainmsg, payload if not payload.endswith('\n'): print >> plainmsg - # Now add the footer - if mlist.digest_footer: + # Now add the footer but only if more than whitespace. + if re.sub('\s', '', mlist.digest_footer): footertxt = decorate(mlist, mlist.digest_footer, _('digest footer')) # MIME footer = MIMEText(footertxt, _charset=lcset) @@ -411,7 +423,7 @@ def send_i18n_digests(mlist, mboxfp): listname=mlist.internal_name(), isdigest=True) # RFC 1153 - rfc1153msg.set_payload(plainmsg.getvalue(), lcset) + rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset) virginq.enqueue(rfc1153msg, recips=plainrecips, listname=mlist.internal_name(), diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py new file mode 100644 index 00000000..2bb540d6 --- /dev/null +++ b/Mailman/Handlers/WrapMessage.py @@ -0,0 +1,89 @@ +# Copyright (C) 2013-2016 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Wrap the message in an outer message/rfc822 part and transfer/add +some headers from the original. + +Also, in the case of Munge From, replace the From:, Reply-To: and Cc: in the +original message. +""" + +import copy + +from email.MIMEMessage import MIMEMessage +from email.MIMEText import MIMEText + +from Mailman import Utils + +# Headers from the original that we want to keep in the wrapper. +KEEPERS = ('to', + 'in-reply-to', + 'references', + 'x-mailman-approved-at', + 'date', + ) + + + +def process(mlist, msg, msgdata): + # This is the negation of we're wrapping because dmarc_moderation_action + # is wrap this message or from_is_list applies and is wrap. + if not (msgdata.get('from_is_list') == 2 or + (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)): + # Now see if we need to add a From:, Reply-To: or Cc: without wrapping. + # See comments in CookHeaders.change_header for why we do this here. + a_h = msgdata.get('add_header') + if a_h: + if a_h.get('From'): + del msg['from'] + msg['From'] = a_h.get('From') + if a_h.get('Reply-To'): + del msg['reply-to'] + msg['Reply-To'] = a_h.get('Reply-To') + if a_h.get('Cc'): + del msg['cc'] + msg['Cc'] = a_h.get('Cc') + return + + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg.keys(): + if key.lower() not in KEEPERS: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = Utils.unique_message_id(mlist) + # Add the headers from CookHeaders. + for k, v in msgdata['add_header'].items(): + msg[k] = v + # Are we including dmarc_wrapped_message_text? I.e., do we have text and + # are we wrapping because of dmarc_moderation_action? + if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2: + part1 = MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text), + 'plain', + Utils.GetCharSet(mlist.preferred_language)) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + diff --git a/Mailman/ListAdmin.py b/Mailman/ListAdmin.py index 13502352..877aa5c4 100755 --- a/Mailman/ListAdmin.py +++ b/Mailman/ListAdmin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -101,7 +101,7 @@ class ListAdmin: # should we be as paranoid as for the config.pck file? Should we # use pickle? tmpfile = self.__filename + '.tmp' - omask = os.umask(002) + omask = os.umask(007) try: fp = open(tmpfile, 'w') try: @@ -194,7 +194,7 @@ class ListAdmin: else: ext = 'txt' filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext) - omask = os.umask(002) + omask = os.umask(007) try: fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w') try: @@ -243,7 +243,11 @@ class ListAdmin: if e.errno <> errno.ENOENT: raise return LOST try: - msg = cPickle.load(fp) + if path.endswith('.pck'): + msg = cPickle.load(fp) + else: + assert path.endswith('.txt'), '%s not .pck or .txt' % path + msg = fp.read() finally: fp.close() # Save the plain text to a .msg file, not a .pck file @@ -252,8 +256,11 @@ class ListAdmin: outpath = head + '.msg' outfp = open(outpath, 'w') try: - g = Generator(outfp) - g.flatten(msg, 1) + if path.endswith('.pck'): + g = Generator(outfp) + g.flatten(msg, 1) + else: + outfp.write(msg) finally: outfp.close() # Now handle updates to the database @@ -285,7 +292,8 @@ class ListAdmin: # message directly here can lead to a huge delay in web # turnaround. Log the moderation and add a header. msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1) - syslog('vette', 'held message approved, message-id: %s', + syslog('vette', '%s: held message approved, message-id: %s', + self.internal_name(), msg.get('message-id', 'n/a')) # Stick the message back in the incoming queue for further # processing. @@ -403,8 +411,10 @@ class ListAdmin: }, mlist=self) # This message should appear to come from the <list>-owner so as # to avoid any useless bounce processing. - msg = Message.OwnerNotification(self, subject, text=text, - tomoderators=1) + owneraddr = self.GetOwnerEmail() + msg = Message.UserNotification(owneraddr, owneraddr, subject, text, + self.preferred_language) + msg.send(self, **{'tomoderators': 1}) # Restore the user's preferred language. i18n.set_language(lang) @@ -458,8 +468,10 @@ class ListAdmin: }, mlist=self) # This message should appear to come from the <list>-owner so as # to avoid any useless bounce processing. - msg = Message.OwnerNotification(self, subject, text=text, - tomoderators=1) + owneraddr = self.GetOwnerEmail() + msg = Message.UserNotification(owneraddr, owneraddr, subject, text, + self.preferred_language) + msg.send(self, **{'tomoderators': 1}) def __handleunsubscription(self, record, value, comment): addr = record diff --git a/Mailman/Logging/Logger.py b/Mailman/Logging/Logger.py index 617347d4..f3d30164 100644 --- a/Mailman/Logging/Logger.py +++ b/Mailman/Logging/Logger.py @@ -60,7 +60,7 @@ class Logger: return self.__fp else: try: - ou = os.umask(002) + ou = os.umask(007) try: try: f = codecs.open( diff --git a/Mailman/MTA/Manual.py b/Mailman/MTA/Manual.py index 46adfae2..14158263 100644 --- a/Mailman/MTA/Manual.py +++ b/Mailman/MTA/Manual.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2005 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,7 +25,7 @@ from Mailman import mm_cfg from Mailman import Message from Mailman import Utils from Mailman.Queue.sbcache import get_switchboard -from Mailman.i18n import _ +from Mailman.i18n import _, C_ from Mailman.MTA.Utils import makealiases try: @@ -74,12 +74,12 @@ Here are the entries for the /etc/aliases file: outfp = sfp else: if not quiet: - print _("""\ + print C_("""\ To finish creating your mailing list, you must edit your /etc/aliases (or equivalent) file by adding the following lines, and possibly running the `newaliases' program: """) - print _("""\ + print C_("""\ ## %(listname)s mailing list""") outfp = sys.stdout # Common path @@ -94,9 +94,10 @@ equivalent) file by adding the following lines, and possibly running the # this request. siteowner = Utils.get_site_email(extra='owner') # Should this be sent in the site list's preferred language? - msg = Message.OwnerNotification(mlist, + msg = Message.UserNotification( + siteowner, siteowner, _('Mailing list creation request for list %(listname)s'), - sfp.getvalue()) + sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE) msg.send(mlist) @@ -119,7 +120,7 @@ Here are the entries in the /etc/aliases file that should be removed: """) outfp = sfp else: - print _(""" + print C_(""" To finish removing your mailing list, you must edit your /etc/aliases (or equivalent) file by removing the following lines, and possibly running the `newaliases' program: @@ -134,9 +135,12 @@ equivalent) file by removing the following lines, and possibly running the if not cgi: print >> outfp return - msg = Message.OwnerNotification(mlist, + siteowner = Utils.get_site_email(extra='owner') + # Should this be sent in the site list's preferred language? + msg = Message.UserNotification( + siteowner, siteowner, _('Mailing list removal request for list %(listname)s'), - sfp.getvalue()) + sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE) msg['Date'] = email.Utils.formatdate(localtime=1) outq = get_switchboard(mm_cfg.OUTQUEUE_DIR) outq.enqueue(msg, recips=[siteowner], nodecorate=1) diff --git a/Mailman/MTA/Postfix.py b/Mailman/MTA/Postfix.py index 801ddc0f..aed36bc4 100644 --- a/Mailman/MTA/Postfix.py +++ b/Mailman/MTA/Postfix.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,7 +27,8 @@ from stat import * from Mailman import mm_cfg from Mailman import Utils from Mailman import LockFile -from Mailman.i18n import _ +from Mailman.i18n import C_ +from Mailman.MailList import MailList from Mailman.MTA.Utils import makealiases from Mailman.Logging.Syslog import syslog @@ -116,6 +117,10 @@ def _addlist(mlist, fp): +def _isvirtual(mlist): + return (mlist and mlist.host_name.lower() in + [d.lower() for d in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS]) + def _addvirtual(mlist, fp): listname = mlist.internal_name() fieldsz = len(listname) + len('-unsubscribe') @@ -123,10 +128,26 @@ def _addvirtual(mlist, fp): # Set up the mailman-loop address loopaddr = Utils.get_site_email(mlist.host_name, extra='loop') loopdest = Utils.ParseEmail(loopaddr)[0] + # And the site list posting address. + siteaddr = Utils.get_site_email(mlist.host_name) + sitedest = Utils.ParseEmail(siteaddr)[0] + # And the site list -owner address. + siteowneraddr = Utils.get_site_email(mlist.host_name, extra='owner') + siteownerdest = Utils.ParseEmail(siteowneraddr)[0] if mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN: loopdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + sitedest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + siteownerdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + # If the site list's host_name is a virtual domain, adding the list and + # owner addresses to the SITE ADDRESSES will duplicate the entries in the + # stanza for the list. Postfix doesn't like dups so we try to comment them + # here, but only for the actual site list domain. + if (MailList(mm_cfg.MAILMAN_SITE_LIST, lock=False).host_name.lower() == + hostname.lower()): + siteaddr = '#' + siteaddr + siteowneraddr = '#' + siteowneraddr # Seek to the end of the text file, but if it's empty write the standard - # disclaimer, and the loop catch address. + # disclaimer, and the loop catch address and site address. fp.seek(0, 2) if not fp.tell(): print >> fp, """\ @@ -141,7 +162,16 @@ def _addvirtual(mlist, fp): # LOOP ADDRESSES START %s\t%s # LOOP ADDRESSES END -""" % (loopaddr, loopdest) + +# We also add the site list address in each virtual domain as that address +# is exposed on admin and listinfo overviews, and we add the site list-owner +# address as it is exposed in the list created email notice. + +# SITE ADDRESSES START +%s\t%s +%s\t%s +# SITE ADDRESSES END +""" % (loopaddr, loopdest, siteaddr, sitedest, siteowneraddr, siteownerdest) # The text file entries get a little extra info print >> fp, '# STANZA START:', listname print >> fp, '# CREATED:', time.ctime(time.time()) @@ -164,6 +194,22 @@ def _addvirtual(mlist, fp): def _check_for_virtual_loopaddr(mlist, filename): loopaddr = Utils.get_site_email(mlist.host_name, extra='loop') loopdest = Utils.ParseEmail(loopaddr)[0] + siteaddr = Utils.get_site_email(mlist.host_name) + sitedest = Utils.ParseEmail(siteaddr)[0] + siteowneraddr = Utils.get_site_email(mlist.host_name, extra='owner') + siteownerdest = Utils.ParseEmail(siteowneraddr)[0] + if mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN: + loopdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + sitedest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + siteownerdest += '@' + mm_cfg.VIRTUAL_MAILMAN_LOCAL_DOMAIN + # If the site list's host_name is a virtual domain, adding the list and + # owner addresses to the SITE ADDRESSES will duplicate the entries in the + # stanza for the list. Postfix doesn't like dups so we try to comment them + # here, but only for the actual site list domain. + if (MailList(mm_cfg.MAILMAN_SITE_LIST, lock=False).host_name.lower() == + mlist.host_name.lower()): + siteaddr = '#' + siteaddr + siteowneraddr = '#' + siteowneraddr infp = open(filename) omask = os.umask(007) try: @@ -196,6 +242,33 @@ def _check_for_virtual_loopaddr(mlist, filename): else: # This isn't our loop address, so spit it out and continue outfp.write(line) + # Now do it all again for the site list address. It must follow the + # loop addresses. + while True: + line = infp.readline() + if not line: + break + outfp.write(line) + if line.startswith('# SITE ADDRESSES START'): + break + # Now see if our domain has already been written + while True: + line = infp.readline() + if not line: + break + if line.startswith('# SITE ADDRESSES END'): + # It hasn't + print >> outfp, '%s\t%s' % (siteaddr, sitedest) + print >> outfp, '%s\t%s' % (siteowneraddr, siteownerdest) + outfp.write(line) + break + elif line.startswith(siteaddr) or line.startswith('#' + siteaddr): + # We just found it + outfp.write(line) + break + else: + # This isn't our loop address, so spit it out and continue + outfp.write(line) outfp.writelines(infp.readlines()) finally: infp.close() @@ -233,7 +306,7 @@ def create(mlist, cgi=False, nolock=False, quiet=False): # Do the aliases file, which need to be done in any case try: _do_create(mlist, ALIASFILE, _addlist) - if mlist and mlist.host_name in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS: + if _isvirtual(mlist): _do_create(mlist, VIRTFILE, _addvirtual) # bin/genaliases is the only one that calls create with nolock = True. # Use that to only update the maps at the end of genaliases. @@ -304,7 +377,7 @@ def remove(mlist, cgi=False): lock.lock() try: _do_remove(mlist, ALIASFILE, False) - if mlist.host_name in mm_cfg.POSTFIX_STYLE_VIRTUAL_DOMAINS: + if _isvirtual(mlist): _do_remove(mlist, VIRTFILE, True) # Regenerate the alias and map files _update_maps() @@ -317,7 +390,7 @@ def checkperms(state): targetmode = S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP for file in ALIASFILE, VIRTFILE: if state.VERBOSE: - print _('checking permissions on %(file)s') + print C_('checking permissions on %(file)s') stat = None try: stat = os.stat(file) @@ -327,9 +400,9 @@ def checkperms(state): if stat and (stat[ST_MODE] & targetmode) <> targetmode: state.ERRORS += 1 octmode = oct(stat[ST_MODE]) - print _('%(file)s permissions must be 066x (got %(octmode)s)'), + print C_('%(file)s permissions must be 066x (got %(octmode)s)'), if state.FIX: - print _('(fixing)') + print C_('(fixing)') os.chmod(file, stat[ST_MODE] | targetmode) else: print @@ -345,7 +418,7 @@ def checkperms(state): raise continue if state.VERBOSE: - print _('checking ownership of %(dbfile)s') + print C_('checking ownership of %(dbfile)s') user = mm_cfg.MAILMAN_USER ownerok = stat[ST_UID] == pwd.getpwnam(user)[2] if not ownerok: @@ -353,10 +426,11 @@ def checkperms(state): owner = pwd.getpwuid(stat[ST_UID])[0] except KeyError: owner = 'uid %d' % stat[ST_UID] - print _('%(dbfile)s owned by %(owner)s (must be owned by %(user)s'), + print C_( + '%(dbfile)s owned by %(owner)s (must be owned by %(user)s'), state.ERRORS += 1 if state.FIX: - print _('(fixing)') + print C_('(fixing)') uid = pwd.getpwnam(user)[2] gid = grp.getgrnam(mm_cfg.MAILMAN_GROUP)[2] os.chown(dbfile, uid, gid) @@ -365,9 +439,9 @@ def checkperms(state): if stat and (stat[ST_MODE] & targetmode) <> targetmode: state.ERRORS += 1 octmode = oct(stat[ST_MODE]) - print _('%(dbfile)s permissions must be 066x (got %(octmode)s)'), + print C_('%(dbfile)s permissions must be 066x (got %(octmode)s)'), if state.FIX: - print _('(fixing)') + print C_('(fixing)') os.chmod(dbfile, stat[ST_MODE] | targetmode) else: print diff --git a/Mailman/MailList.py b/Mailman/MailList.py index 2d653acb..d1dc17a4 100755 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -74,6 +74,7 @@ from Mailman.Logging.Syslog import syslog _ = i18n._ EMPTYSTRING = '' +OR = '|' try: True, False @@ -347,6 +348,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.bounce_matching_headers = \ mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS self.header_filter_rules = [] + self.from_is_list = mm_cfg.DEFAULT_FROM_IS_LIST self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST internalname = self.internal_name() self.real_name = internalname[0].upper() + internalname[1:] @@ -355,6 +357,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.welcome_msg = '' self.goodbye_msg = '' self.subscribe_policy = mm_cfg.DEFAULT_SUBSCRIBE_POLICY + self.subscribe_auto_approval = mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL self.unsubscribe_policy = mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY self.private_roster = mm_cfg.DEFAULT_PRIVATE_ROSTER self.obscure_addresses = mm_cfg.DEFAULT_OBSCURE_ADDRESSES @@ -383,11 +386,25 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, mm_cfg.DEFAULT_DEFAULT_MEMBER_MODERATION # Emergency moderation bit self.emergency = 0 + self.member_verbosity_threshold = ( + mm_cfg.DEFAULT_MEMBER_VERBOSITY_THRESHOLD) + self.member_verbosity_interval = ( + mm_cfg.DEFAULT_MEMBER_VERBOSITY_INTERVAL) # This really ought to default to mm_cfg.HOLD, but that doesn't work # with the current GUI description model. So, 0==Hold, 1==Reject, # 2==Discard self.member_moderation_action = 0 self.member_moderation_notice = '' + self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION + self.dmarc_quarantine_moderation_action = ( + mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION) + self.dmarc_none_moderation_action = ( + mm_cfg.DEFAULT_DMARC_NONE_MODERATION_ACTION) + self.dmarc_moderation_notice = '' + self.dmarc_wrapped_message_text = ( + mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT) + self.equivalent_domains = ( + mm_cfg.DEFAULT_EQUIVALENT_DOMAINS) self.accept_these_nonmembers = [] self.hold_these_nonmembers = [] self.reject_these_nonmembers = [] @@ -617,6 +634,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if e.errno <> errno.ENOENT: raise # The file doesn't exist yet return None, e + now = int(time.time()) try: try: dict = loadfunc(fp) @@ -628,8 +646,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, finally: fp.close() # Update the timestamp. We use current time here rather than mtime - # so the test above might succeed the next time. - self.__timestamp = int(time.time()) + # so the test above might succeed the next time. And we get the time + # before unpickling in case it takes more than a second. (LP: #266464) + self.__timestamp = now return dict, None def Load(self, check_version=True): @@ -768,10 +787,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, goodtopics = [] for name, pattern, desc, emptyflag in self.topics: try: - re.compile(pattern) + orpattern = OR.join(pattern.splitlines()) + re.compile(orpattern) except (re.error, TypeError): syslog('error', 'Bad topic pattern "%s" for list: %s', - pattern, self.internal_name()) + orpattern, self.internal_name()) else: goodtopics.append((name, pattern, desc, emptyflag)) self.topics = goodtopics @@ -791,6 +811,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # check for banned address pattern = self.GetBannedPattern(invitee) if pattern: + syslog('vette', '%s banned invitation: %s (matched: %s)', + self.real_name, invitee, pattern) raise Errors.MembershipIsBanned, pattern # Hack alert! Squirrel away a flag that only invitations have, so # that we can do something slightly different when an invitation @@ -871,8 +893,12 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Is the subscribing address banned from this list? pattern = self.GetBannedPattern(email) if pattern: - syslog('vette', '%s banned subscription: %s (matched: %s)', - realname, email, pattern) + if remote: + whence = ' from %s' % remote + else: + whence = '' + syslog('vette', '%s banned subscription: %s%s (matched: %s)', + realname, email, whence, pattern) raise Errors.MembershipIsBanned, pattern # Sanity check the digest flag if digest and not self.digestable: @@ -936,6 +962,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, syslog('subscribe', '%s: pending %s %s', self.internal_name(), who, by) raise Errors.MMSubscribeNeedsConfirmation + elif self.HasAutoApprovedSender(email): + # no approval necessary: + self.ApprovedAddMember(userdesc) else: # Subscription approval is required. Add this entry to the admin # requests database. BAW: this should probably take a userdesc @@ -986,6 +1015,12 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # and confirmations. pattern = self.GetBannedPattern(email) if pattern: + if whence: + source = ' from %s' % whence + else: + source = '' + syslog('vette', '%s banned subscription: %s%s (matched: %s)', + self.real_name, email, source, pattern) raise Errors.MembershipIsBanned, pattern # Do the actual addition self.addNewMember(email, realname=name, digest=digest, @@ -1046,7 +1081,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # And send an acknowledgement to the user... if userack: self.SendUnsubscribeAck(emailaddr, userlang) - # ...and to the administrator + # ...and to the administrator in the correct language. (LP: #1308655) + i18n.set_language(self.preferred_language) if admin_notif: realname = self.real_name subject = _('%(realname)s unsubscribe notification') @@ -1149,6 +1185,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # exception. pattern = self.GetBannedPattern(newaddr) if pattern: + syslog('vette', + '%s banned address change: %s -> %s (matched: %s)', + self.real_name, oldaddr, newaddr, pattern) raise Errors.MembershipIsBanned, pattern # It's possible they were a member of this list, but choose to change # their membership globally. In that case, we simply remove the old @@ -1158,12 +1197,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # CP address of a member, then if the old address yields a different # CP address, we can simply remove the old address, otherwise we can # do nothing. + cpoldaddr = self.getMemberCPAddress(oldaddr) if self.isMember(newaddr) and (self.getMemberCPAddress(newaddr) == newaddr): - if self.getMemberCPAddress(oldaddr) <> newaddr: + if cpoldaddr <> newaddr: self.removeMember(oldaddr) else: self.changeMemberAddress(oldaddr, newaddr) + self.log_and_notify_admin(cpoldaddr, newaddr) # If globally is true, then we also include every list for which # oldaddr is a member. if not globally: @@ -1183,16 +1224,46 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, mlist.Lock() try: # Same logic as above, re newaddr is already a member + cpoldaddr = mlist.getMemberCPAddress(oldaddr) if mlist.isMember(newaddr) and ( mlist.getMemberCPAddress(newaddr) == newaddr): - if mlist.getMemberCPAddress(oldaddr) <> newaddr: + if cpoldaddr <> newaddr: mlist.removeMember(oldaddr) else: mlist.changeMemberAddress(oldaddr, newaddr) + mlist.log_and_notify_admin(cpoldaddr, newaddr) mlist.Save() finally: mlist.Unlock() + def log_and_notify_admin(self, oldaddr, newaddr): + """Log member address change and notify admin if requested.""" + syslog('subscribe', '%s: changed member address from %s to %s', + self.internal_name(), oldaddr, newaddr) + if self.admin_notify_mchanges: + lang = self.preferred_language + otrans = i18n.get_translation() + i18n.set_language(lang) + try: + realname = self.real_name + subject = _('%(realname)s address change notification') + finally: + i18n.set_translation(otrans) + name = self.getMemberName(newaddr) + if name is None: + name = '' + if isinstance(name, UnicodeType): + name = name.encode(Utils.GetCharSet(lang), 'replace') + text = Utils.maketext( + 'adminaddrchgack.txt', + {'name' : name, + 'oldaddr' : oldaddr, + 'newaddr' : newaddr, + 'listname': self.real_name, + }, mlist=self) + msg = Message.OwnerNotification(self, subject, text) + msg.send(self) + # # Confirmation processing @@ -1236,7 +1307,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # list administrators. self.SendHostileSubscriptionNotice(invitation, addr) raise Errors.HostileSubscriptionError - elif self.subscribe_policy in (2, 3): + elif self.subscribe_policy in (2, 3) and \ + not self.HasAutoApprovedSender(addr): self.HoldSubscription(addr, fullname, password, digest, lang) name = self.real_name raise Errors.MMNeedApproval, _( @@ -1515,26 +1587,85 @@ bad regexp in bounce_matching_header line: %s """Returns matched entry in ban_list if email matches. Otherwise returns None. """ - ban = False - for pattern in self.ban_list: + return (self.GetPattern(email, self.ban_list) or + self.GetPattern(email, mm_cfg.GLOBAL_BAN_LIST) + ) + + def HasAutoApprovedSender(self, sender): + """Returns True and logs if sender matches address or pattern + or is a member of a referenced list in subscribe_auto_approval. + Otherwise returns False. + """ + auto_approve = False + if self.GetPattern(sender, + self.subscribe_auto_approval, + at_list='subscribe_auto_approval' + ): + auto_approve = True + syslog('vette', '%s: auto approved subscribe from %s', + self.internal_name(), sender) + return auto_approve + + def GetPattern(self, email, pattern_list, at_list=None): + """Returns matched entry in pattern_list if email matches. + Otherwise returns None. The at_list argument, if "true", + says process the @listname syntax and provides the name of + the list attribute for log messages. + """ + matched = None + # First strip out all the regular expressions and listnames because + # documentation says we do non-regexp first (Why?). + plainaddrs = [x.strip() for x in pattern_list if x.strip() and not + (x.startswith('^') or x.startswith('@'))] + addrdict = Utils.List2Dict(plainaddrs, foldcase=1) + if addrdict.has_key(email.lower()): + return email + for pattern in pattern_list: if pattern.startswith('^'): # This is a regular expression match try: if re.search(pattern, email, re.IGNORECASE): - ban = True + matched = pattern break - except re.error: + except re.error, e: # BAW: we should probably remove this pattern - pass - else: - # Do the comparison case insensitively - if pattern.lower() == email.lower(): - ban = True + # The GUI won't add a bad regexp, but at least log it. + # The following kludge works because the ban_list stuff + # is the only caller with no at_list. + attr_name = at_list or 'ban_list' + syslog('error', + '%s in %s has bad regexp "%s": %s', + attr_name, + self.internal_name(), + pattern, + str(e) + ) + elif at_list and pattern.startswith('@'): + # XXX Needs to be reviewed for list@domain names. + # this refers to the members of another list in this + # installation. + mname = pattern[1:].lower().strip() + if mname == self.internal_name(): + # don't reference your own list + syslog('error', + '%s in %s references own list', + at_list, + self.internal_name()) + continue + try: + mother = MailList(mname, lock = False) + except Errors.MMUnknownListError: + syslog('error', + '%s in %s references non-existent list %s', + at_list, + self.internal_name(), + mname + ) + continue + if mother.isMember(email.lower()): + matched = pattern break - if ban: - return pattern - else: - return None + return matched diff --git a/Mailman/Mailbox.py b/Mailman/Mailbox.py index a7e8cced..a8fa4d0b 100644 --- a/Mailman/Mailbox.py +++ b/Mailman/Mailbox.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -110,3 +110,11 @@ class ArchiverMailbox(Mailbox): return self._scrubber(self._mlist, msg) else: return msg + + def skipping(self, flag): + """ This method allows the archiver to skip over messages without + scrubbing attachments into the attachments directory.""" + if flag: + self.factory = _safeparser + else: + self.factory = _archfactory(self) diff --git a/Mailman/Message.py b/Mailman/Message.py index 0edc3c31..5d68e7ef 100644 --- a/Mailman/Message.py +++ b/Mailman/Message.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2014 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -284,8 +284,13 @@ class UserNotification(Message): # UserNotifications are typically for admin messages, and for messages # other than list explosions. Send these out as Precedence: bulk, but # don't override an existing Precedence: header. + # Also, if the message is To: the list-owner address, set Precedence: + # list. See note below in OwnerNotification. if not (self.has_key('precedence') or noprecedence): - self['Precedence'] = 'bulk' + if self.get('to') == mlist.GetOwnerEmail(): + self['Precedence'] = 'list' + else: + self['Precedence'] = 'bulk' self._enqueue(mlist, **_kws) def _enqueue(self, mlist, **_kws): @@ -311,14 +316,19 @@ class OwnerNotification(UserNotification): recips.extend(mlist.moderator) # We have to set the owner to the site's -bounces address, otherwise # we'll get a mail loop if an owner's address bounces. - sender = mlist.getListAddress('bounces') + sender = Utils.get_site_email(mlist.host_name, 'bounces') lang = mlist.preferred_language UserNotification.__init__(self, recips, sender, subject, text, lang) # Hack the To header to look like it's going to the -owner address del self['to'] self['To'] = mlist.GetOwnerEmail() self._sender = sender - self.envsender = sender + # User notifications are normally sent with Precedence: bulk. This + # is appropriate as they can be backscatter of rejected spam. + # Owner notifications are not backscatter and are perhaps more + # important than 'bulk' so give them Precedence: list by default. + # (LP: #1313146) + self['Precedence'] = 'list' def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop diff --git a/Mailman/Queue/ArchRunner.py b/Mailman/Queue/ArchRunner.py index 62714537..17056e29 100644 --- a/Mailman/Queue/ArchRunner.py +++ b/Mailman/Queue/ArchRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -19,6 +19,7 @@ import time from email.Utils import parsedate_tz, mktime_tz, formatdate +from Mailman import i18n from Mailman import mm_cfg from Mailman import LockFile from Mailman.Queue.Runner import Runner @@ -70,6 +71,9 @@ class ArchRunner(Runner): # oh well, try again later return 1 try: + # Archiving should be done in the list's preferred language, not + # the sender's language. + i18n.set_language(mlist.preferred_language) mlist.ArchiveMail(msg) mlist.Save() finally: diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index d219d6e9..2d14f284 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -29,7 +29,9 @@ from email.Utils import parseaddr from Mailman import mm_cfg from Mailman import Utils from Mailman import LockFile +from Mailman.Errors import NotAMemberError from Mailman.Message import UserNotification +from Mailman.Bouncer import _BounceInfo from Mailman.Bouncers import BouncerAPI from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard @@ -150,11 +152,26 @@ class BounceMixin: mlist.Lock() try: op, addr, bmsg = mlist.pend_confirm(token) - info = mlist.getBounceInfo(addr) - mlist.disableBouncingMember(addr, info, bmsg) - # Only save the list if we're unlocking it - if not locked: - mlist.Save() + # For Python 2.4 compatibility we need an inner try because + # try: ... except: ... finally: requires Python 2.5+ + try: + info = mlist.getBounceInfo(addr) + if not info: + # info was deleted before probe bounce was received. + # Just create a new info. + info = _BounceInfo(addr, + 0.0, + time.localtime()[:3], + mlist.bounce_you_are_disabled_warnings + ) + mlist.disableBouncingMember(addr, info, bmsg) + # Only save the list if we're unlocking it + if not locked: + mlist.Save() + except NotAMemberError: + # Member was removed before probe bounce returned. + # Just ignore it. + pass finally: if not locked: mlist.Unlock() @@ -244,6 +261,7 @@ class BounceRunner(Runner, BounceMixin): return # If that still didn't return us any useful addresses, then send it on # or discard it. + addrs = filter(None, addrs) if not addrs: syslog('bounce', '%s: bounce message w/no discernable addresses: %s', @@ -254,7 +272,8 @@ class BounceRunner(Runner, BounceMixin): # BAW: It's possible that there are None's in the list of addresses, # although I'm unsure how that could happen. Possibly ScanMessages() # can let None's sneak through. In any event, this will kill them. - addrs = filter(None, addrs) + # addrs = filter(None, addrs) + # MAS above filter moved up so we don't try to queue an empty list. self._queue_bounces(mlist.internal_name(), addrs, msg) _doperiodic = BounceMixin._doperiodic diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index b63b050c..a9f6f000 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """-request robot command queue runner.""" @@ -132,22 +133,35 @@ class Results: __import__(modname) handler = sys.modules[modname] # ValueError can be raised if cmd has dots in it. - except (ImportError, ValueError): + # and KeyError if cmd is otherwise good but ends with a dot. + # and TypeError if cmd has a null byte. + except (ImportError, ValueError, KeyError, TypeError): # If we're on line zero, it was the Subject: header that didn't # contain a command. It's possible there's a Re: prefix (or # localized version thereof) on the Subject: line that's messing # things up. Pop the prefix off and try again... once. # + # At least one MUA (163.com web mail) has been observed that + # inserts 'Re:' with no following space, so try to account for + # that too. + # # If that still didn't work it isn't enough to stop processing. # BAW: should we include a message that the Subject: was ignored? - if not self.subjcmdretried and args: + # + # But first, be sure we're looking at the Subject: and not past + # it already. + if self.lineno != 0: + return BADCMD + if self.subjcmdretried < 1: + self.subjcmdretried += 1 + if re.search('^.*:.+', cmd): + cmd = re.sub('.*:', '', cmd).lower() + return self.do_command(cmd, args) + if self.subjcmdretried < 2 and args: self.subjcmdretried += 1 - cmd = args.pop(0) + cmd = args.pop(0).lower() return self.do_command(cmd, args) - if self.lineno <> 0: - return BADCMD - else: - return BADSUBJ + return BADSUBJ if handler.process(self, args): return STOP else: diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py index bcc51209..2c6c2815 100644 --- a/Mailman/Queue/IncomingRunner.py +++ b/Mailman/Queue/IncomingRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -157,8 +157,15 @@ class IncomingRunner(Runner): os._exit(1) except Errors.DiscardMessage: # Throw the message away; we need do nothing else with it. - syslog('vette', 'Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) + # We do need to push the current handler back in the pipeline + # just in case the syslog call throws an exception and the + # message is shunted. + pipeline.insert(0, handler) + syslog('vette', """Message discarded, msgid: %s' + list: %s, + handler: %s""", + msg.get('message-id', 'n/a'), + mlist.real_name, handler) return 0 except Errors.HoldMessage: # Let the approval process take it from here. The message no @@ -166,6 +173,10 @@ class IncomingRunner(Runner): return 0 except Errors.RejectMessage, e: # Log this. + # We do need to push the current handler back in the pipeline + # just in case the syslog call or BounceMessage throws an + # exception and the message is shunted. + pipeline.insert(0, handler) syslog('vette', """Message rejected, msgid: %s list: %s, handler: %s, diff --git a/Mailman/Queue/MaildirRunner.py b/Mailman/Queue/MaildirRunner.py index 5959dcd6..d9fe02cb 100644 --- a/Mailman/Queue/MaildirRunner.py +++ b/Mailman/Queue/MaildirRunner.py @@ -172,6 +172,7 @@ class MaildirRunner(Runner): elif subq == 'owner': msgdata.update({ 'toowner': 1, + 'envsender': Utils.get_site_email(extra='bounces'), 'pipeline': mm_cfg.OWNER_PIPELINE, }) queue = get_switchboard(mm_cfg.INQUEUE_DIR) diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py index 44850063..fe693f28 100644 --- a/Mailman/Queue/NewsRunner.py +++ b/Mailman/Queue/NewsRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2005 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """NNTP queue runner.""" @@ -109,22 +110,18 @@ def prepare_message(mlist, msg, msgdata): or msgdata.get('origsubj') if not mlist.news_prefix_subject_too and stripped_subject is not None: del msg['subject'] - msg['subject'] = stripped_subject + msg['Subject'] = stripped_subject + # Make sure we have a non-blank subject. + if not msg.get('subject', ''): + del msg['subject'] + msg['Subject'] = '(no subject)' # Add the appropriate Newsgroups: header - ngheader = msg['newsgroups'] - if ngheader is not None: - # See if the Newsgroups: header already contains our linked_newsgroup. - # If so, don't add it again. If not, append our linked_newsgroup to - # the end of the header list - ngroups = [s.strip() for s in ngheader.split(',')] - if mlist.linked_newsgroup not in ngroups: - ngroups.append(mlist.linked_newsgroup) - # Subtitute our new header for the old one. - del msg['newsgroups'] - msg['Newsgroups'] = COMMASPACE.join(ngroups) - else: - # Newsgroups: isn't in the message - msg['Newsgroups'] = mlist.linked_newsgroup + if msg['newsgroups'] is not None: + # This message is gated from our list to it's associated usnet group. + # If it has a Newsgroups: header mentioning other groups, it's not + # up to us to post it to those groups. + del msg['newsgroups'] + msg['Newsgroups'] = mlist.linked_newsgroup # Note: We need to be sure two messages aren't ever sent to the same list # in the same process, since message ids need to be unique. Further, if # messages are crossposted to two Usenet-gated mailing lists, they each @@ -133,6 +130,9 @@ def prepare_message(mlist, msg, msgdata): # isn't ours with one of ours, so we need to parse it to be sure we're not # looping. # + # We also add the original Message-ID: to References: to try to help with + # threading issues and create another header for documentation. + # # Our Message-ID format is <mailman.secs.pid.listname@hostname> msgid = msg['message-id'] hackmsgid = True @@ -145,6 +145,18 @@ def prepare_message(mlist, msg, msgdata): if hackmsgid: del msg['message-id'] msg['Message-ID'] = Utils.unique_message_id(mlist) + if msgid: + msg['X-Mailman-Original-Message-ID'] = msgid + refs = msg['references'] + del msg['references'] + if not refs: + refs = msg.get('in-reply-to', '') + else: + msg['X-Mailman-Original-References'] = refs + if refs: + msg['References'] = '\n '.join([refs, msgid]) + else: + msg['References'] = msgid # Lines: is useful if msg['Lines'] is None: # BAW: is there a better way? diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index bd1cd357..a2c31263 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -184,8 +184,8 @@ class Switchboard: else: os.unlink(bakfile) except EnvironmentError, e: - syslog('error', 'Failed to unlink/preserve backup file: %s', - bakfile) + syslog('error', 'Failed to unlink/preserve backup file: %s\n%s', + bakfile, e) def files(self, extension='.pck'): times = {} diff --git a/Mailman/SecurityManager.py b/Mailman/SecurityManager.py index 4f6aa34a..7ca4e084 100644 --- a/Mailman/SecurityManager.py +++ b/Mailman/SecurityManager.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2013 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -319,8 +319,6 @@ class SecurityManager: for u in usernames]: ok = self.__checkone(c, authcontext, user) if ok: - # Refresh the cookie - print self.MakeCookie(authcontext, user) return True return False else: @@ -362,6 +360,8 @@ class SecurityManager: if mac <> received_mac: return False # Authenticated! + # Refresh the cookie + print self.MakeCookie(authcontext, user) return True diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 93e1fba1..75481563 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2017 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -34,6 +34,7 @@ import time import errno import base64 import random +import urllib2 import urlparse import htmlentitydefs import email.Header @@ -71,8 +72,16 @@ except NameError: True = 1 False = 0 +try: + import dns.resolver + from dns.exception import DNSException + dns_resolver = True +except ImportError: + dns_resolver = False + EMPTYSTRING = '' UEMPTYSTRING = u'' +CR = '\r' NL = '\n' DOT = '.' IDENTCHARS = ascii_letters + digits + '_' @@ -92,6 +101,12 @@ def list_exists(listname): # # The former two are for 2.1alpha3 and beyond, while the latter two are # for all earlier versions. + # + # But first ensure the list name doesn't contain a path traversal + # attack. + if len(re.sub(mm_cfg.ACCEPTABLE_LISTNAME_CHARACTERS, '', listname)) > 0: + syslog('mischief', 'Hostile listname: %s', listname) + return False basepath = Site.get_listpath(listname) for ext in ('.pck', '.pck.last', '.db', '.db.last'): dbfile = os.path.join(basepath, 'config' + ext) @@ -220,10 +235,18 @@ _valid_domain = re.compile('[-a-z0-9]', re.IGNORECASE) def ValidateEmail(s): """Verify that an email address isn't grossly evil.""" + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(s, list) or isinstance(s, tuple): + if len(s) == 0: + s = '' + else: + s = s[-1] # Pretty minimal, cheesy check. We could do better... if not s or s.count(' ') > 0: raise Errors.MMBadEmailError - if _badchars.search(s) or s[0] == '-': + if _badchars.search(s): raise Errors.MMHostileAddress, s user, domain_parts = ParseEmail(s) # This means local, unqualified addresses, are not allowed @@ -232,8 +255,9 @@ def ValidateEmail(s): if len(domain_parts) < 2: raise Errors.MMBadEmailError, s # domain parts may only contain ascii letters, digits and hyphen + # and must not begin with hyphen. for p in domain_parts: - if len(_valid_domain.sub('', p)) > 0: + if len(p) == 0 or p[0] == '-' or len(_valid_domain.sub('', p)) > 0: raise Errors.MMHostileAddress, s @@ -247,12 +271,24 @@ def GetPathPieces(envar='PATH_INFO'): if path: if CRNLpat.search(path): path = CRNLpat.split(path)[0] - syslog('error', 'Warning: Possible malformed path attack.') + remote = os.environ.get('HTTP_FORWARDED_FOR', + os.environ.get('HTTP_X_FORWARDED_FOR', + os.environ.get('REMOTE_ADDR', + 'unidentified origin'))) + syslog('error', + 'Warning: Possible malformed path attack domain=%s remote=%s', + get_domain(), + remote) return [p for p in path.split('/') if p] return None +def GetRequestMethod(): + return os.environ.get('REQUEST_METHOD') + + + def ScriptURL(target, web_page_url=None, absolute=False): """target - scriptname only, nothing extra web_page_url - the list's configvar of the same name @@ -427,6 +463,14 @@ def check_global_password(response, siteadmin=True): _ampre = re.compile('&((?:#[0-9]+|[a-z]+);)', re.IGNORECASE) def websafe(s): + # If a user submits a form or URL with post data or query fragments + # with multiple occurrences of the same variable, we can get a list + # here. Be as careful as possible. + if isinstance(s, list) or isinstance(s, tuple): + if len(s) == 0: + s = '' + else: + s = s[-1] if mm_cfg.BROKEN_BROWSER_WORKAROUND: # Archiver can pass unicode here. Just skip them as the # archiver escapes non-ascii anyway. @@ -715,7 +759,7 @@ def get_domain(): if port and host.endswith(':' + port): host = host[:-len(port)-1] if mm_cfg.VIRTUAL_HOST_OVERVIEW and host: - return host.lower() + return websafe(host.lower()) else: # See the note in Defaults.py concerning DEFAULT_URL # vs. DEFAULT_URL_HOST. @@ -905,6 +949,61 @@ def oneline(s, cset): return EMPTYSTRING.join(s.splitlines()) +def strip_verbose_pattern(pattern): + # Remove white space and comments from a verbose pattern and return a + # non-verbose, equivalent pattern. Replace CR and NL in the result + # with '\\r' and '\\n' respectively to avoid multi-line results. + if not isinstance(pattern, str): + return pattern + newpattern = '' + i = 0 + inclass = False + skiptoeol = False + copynext = False + while i < len(pattern): + c = pattern[i] + if copynext: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + copynext = False + elif skiptoeol: + if c == NL: + skiptoeol = False + elif c == '#' and not inclass: + skiptoeol = True + elif c == '[' and not inclass: + inclass = True + newpattern += c + copynext = True + elif c == ']' and inclass: + inclass = False + newpattern += c + elif re.search('\s', c): + if inclass: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + elif c == '\\' and not inclass: + newpattern += c + copynext = True + else: + if c == NL: + newpattern += '\\n' + elif c == CR: + newpattern += '\\r' + else: + newpattern += c + i += 1 + return newpattern + + # Patterns and functions to flag possible XSS attacks in HTML. # This list is compiled from information at http://ha.ckers.org/xss.html, # http://www.quirksmode.org/js/events_compinfo.html, @@ -1057,3 +1156,307 @@ def suspiciousHTML(html): else: return False + +# The next functions read data from +# https://publicsuffix.org/list/public_suffix_list.dat and implement the +# algorithm at https://publicsuffix.org/list/ to find the "Organizational +# Domain corresponding to a From: domain. + +s_dict = {} + +def get_suffixes(url): + """This loads and parses the data from the url argument into s_dict for + use by get_org_dom.""" + global s_dict + if s_dict: + return + if not url: + return + try: + d = urllib2.urlopen(url) + except urllib2.URLError, e: + syslog('error', + 'Unable to retrieve data from %s: %s', + url, e) + return + for line in d.readlines(): + if not line.strip() or line.startswith(' ') or line.startswith('//'): + continue + line = re.sub(' .*', '', line.strip()) + if not line: + continue + parts = line.lower().split('.') + if parts[0].startswith('!'): + exc = True + parts = [parts[0][1:]] + parts[1:] + else: + exc = False + parts.reverse() + k = '.'.join(parts) + s_dict[k] = exc + +def _get_dom(d, l): + """A helper to get a domain name consisting of the first l+1 labels + in d.""" + dom = d[:min(l+1, len(d))] + dom.reverse() + return '.'.join(dom) + +def get_org_dom(domain): + """Given a domain name, this returns the corresponding Organizational + Domain which may be the same as the input.""" + global s_dict + if not s_dict: + get_suffixes(mm_cfg.DMARC_ORGANIZATIONAL_DOMAIN_DATA_URL) + hits = [] + d = domain.lower().split('.') + d.reverse() + for k in s_dict.keys(): + ks = k.split('.') + if len(d) >= len(ks): + for i in range(len(ks)-1): + if d[i] != ks[i] and ks[i] != '*': + break + else: + if d[len(ks)-1] == ks[-1] or ks[-1] == '*': + hits.append(k) + if not hits: + return _get_dom(d, 1) + l = 0 + for k in hits: + if s_dict[k]: + # It's an exception + return _get_dom(d, len(k.split('.'))-1) + if len(k.split('.')) > l: + l = len(k.split('.')) + return _get_dom(d, l) + + +# This takes an email address, and returns True if DMARC policy is p=reject +# or possibly quarantine. +def IsDMARCProhibited(mlist, email): + if not dns_resolver: + # This is a problem; log it. + syslog('error', + 'DNS lookup for dmarc_moderation_action for list %s not available', + mlist.real_name) + return False + + email = email.lower() + # Scan from the right in case quoted local part has an '@'. + at_sign = email.rfind('@') + if at_sign < 1: + return False + f_dom = email[at_sign+1:] + x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom) + if x != 'continue': + return x + o_dom = get_org_dom(f_dom) + if o_dom != f_dom: + x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True) + if x != 'continue': + return x + return False + +def _DMARCProhibited(mlist, email, dmarc_domain, org=False): + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = float(mm_cfg.DMARC_RESOLVER_TIMEOUT) + resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME) + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return 'continue' + except DNSException, e: + syslog('error', + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__doc__) + return 'continue' + else: + # Be as robust as possible in parsing the result. + results_by_name = {} + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name.setdefault(txt_rec.name.to_text(), []).append( + "".join(txt_rec.items[0].strings)) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + syslog('error', + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = filter(lambda n: n.startswith('v=DMARC1;'), + results_by_name[name]) + if len(dmarcs) == 0: + return 'continue' + if len(dmarcs) > 1: + syslog('error', + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarcs)) + for entry in dmarcs: + mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE) + if org and mo: + policy = mo.group(1).lower() + else: + mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE) + if mo: + policy = mo.group(1).lower() + else: + continue + if policy == 'reject': + syslog('vette', + '%s: DMARC lookup for %s (%s) found p=reject in %s = %s', + mlist.real_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_quarantine_moderation_action and + policy == 'quarantine'): + syslog('vette', + '%s: DMARC lookup for %s (%s) found p=quarantine in %s = %s', + mlist.real_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_none_moderation_action and + mlist.dmarc_quarantine_moderation_action and + mlist.dmarc_moderation_action in (1, 2) and + policy == 'none'): + syslog('vette', + '%s: DMARC lookup for %s (%s) found p=none in %s = %s', + mlist.real_name, email, dmarc_domain, name, entry) + return True + + return False + + +# Check a known list in order to auto-moderate verbose members +# dictionary to remember recent posts. +recentMemberPostings = {} +# counter of times through +clean_count = 0 +def IsVerboseMember(mlist, email): + """For lists that request it, we keep track of recent posts by address. +A message from an address to a list, if the list requests it, is remembered +for a specified time whether or not the address is a list member, and if the +address is a member and the member is over the threshold for the list, that +fact is returned.""" + + global clean_count + + if mlist.member_verbosity_threshold == 0: + return False + + email = email.lower() + + now = time.time() + recentMemberPostings.setdefault(email,[]).append(now + + float(mlist.member_verbosity_interval) + ) + x = range(len(recentMemberPostings[email])) + x.reverse() + for i in x: + if recentMemberPostings[email][i] < now: + del recentMemberPostings[email][i] + + clean_count += 1 + if clean_count >= mm_cfg.VERBOSE_CLEAN_LIMIT: + clean_count = 0 + for addr in recentMemberPostings.keys(): + x = range(len(recentMemberPostings[addr])) + x.reverse() + for i in x: + if recentMemberPostings[addr][i] < now: + del recentMemberPostings[addr][i] + if not recentMemberPostings[addr]: + del recentMemberPostings[addr] + if not mlist.isMember(email): + return False + return (len(recentMemberPostings.get(email, [])) > + mlist.member_verbosity_threshold + ) + + +def check_eq_domains(email, domains_list): + """The arguments are an email address and a string representing a + list of lists in a form like 'a,b,c;1,2' representing [['a', 'b', + 'c'],['1', '2']]. The inner lists are domains which are + equivalent in some sense. The return is an empty list or a list + of email addresses equivalent to the first argument. + For example, given + + email = 'user@me.com' + domains_list = '''domain1, domain2; mac.com, me.com, icloud.com; + domaina, domainb + ''' + + check_eq_domains(email, domains_list) will return + ['user@mac.com', 'user@icloud.com'] + """ + if not domains_list: + return [] + try: + local, domain = email.rsplit('@', 1) + except ValueError: + return [] + domain = domain.lower() + domains_list = re.sub('\s', '', domains_list).lower() + domains = domains_list.split(';') + domains_list = [] + for d in domains: + domains_list.append(d.split(',')) + for domains in domains_list: + if domain in domains: + return [local + '@' + x for x in domains if x != domain] + return [] + + +def _invert_xml(mo): + # This is used with re.sub below to convert XML char refs and textual \u + # escapes to unicodes. + try: + if mo.group(1)[:1] == '#': + return unichr(int(mo.group(1)[1:])) + elif mo.group(1)[:1].lower() == 'u': + return unichr(int(mo.group(1)[1:], 16)) + else: + return(u'\ufffd') + except ValueError: + # Value is out of range. Return the unicode replace character. + return(u'\ufffd') + + +def xml_to_unicode(s, cset): + """This converts a string s, encoded in cset to a unicode with translation + of XML character references and textual \uxxxx escapes. It is more or less + the inverse of unicode.decode(cset, errors='xmlcharrefreplace'). It is + similar to canonstr above except for replacing invalid refs with the + unicode replace character and recognizing \u escapes. + """ + if isinstance(s, str): + us = s.decode(cset, 'replace') + us = re.sub(u'&(#[0-9]+);', _invert_xml, us) + us = re.sub(u'(?i)\\\\(u[a-f0-9]{4})', _invert_xml, us) + return us + else: + return s + diff --git a/Mailman/Version.py b/Mailman/Version.py index 4cd2d551..22fa59be 100755..100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -16,7 +16,7 @@ # USA. # Mailman version -VERSION = '2.1.15' +VERSION = '2.1.23' # And as a hex number in the manner of PY_VERSION_HEX ALPHA = 0xa @@ -28,7 +28,7 @@ FINAL = 0xf MAJOR_REV = 2 MINOR_REV = 1 -MICRO_REV = 15 +MICRO_REV = 23 REL_LEVEL = FINAL # at most 15 beta releases! REL_SERIAL = 0 @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) # config.pck schema version number -DATA_FILE_VERSION = 100 +DATA_FILE_VERSION = 110 # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 diff --git a/Mailman/htmlformat.py b/Mailman/htmlformat.py index 2387096e..f144c069 100755 --- a/Mailman/htmlformat.py +++ b/Mailman/htmlformat.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -32,7 +32,7 @@ import types from Mailman import mm_cfg from Mailman import Utils -from Mailman.i18n import _ +from Mailman.i18n import _, get_translation from Mailman.CSRFcheck import csrf_token @@ -318,6 +318,8 @@ class Document(Container): 'content="text/html; charset=%s">' % charset) if self.title: output.append('%s<TITLE>%s</TITLE>' % (tab, self.title)) + if mm_cfg.WEB_HEAD_ADD: + output.append(mm_cfg.WEB_HEAD_ADD) output.append('%s</HEAD>' % tab) quals = [] # Default link colors @@ -405,13 +407,14 @@ class Center(StdContainer): class Form(Container): def __init__(self, action='', method='POST', encoding=None, - mlist=None, contexts=None, *items): + mlist=None, contexts=None, user=None, *items): apply(Container.__init__, (self,) + items) self.action = action self.method = method self.encoding = encoding self.mlist = mlist self.contexts = contexts + self.user = user def set_action(self, action): self.action = action @@ -426,7 +429,7 @@ class Form(Container): if self.mlist: output = output + \ '<input type="hidden" name="csrf_token" value="%s">\n' \ - % csrf_token(self.mlist, self.contexts) + % csrf_token(self.mlist, self.contexts, self.user) output = output + Container.Format(self, indent+2) output = '%s\n%s</FORM>\n' % (output, spaces) return output @@ -441,6 +444,7 @@ class InputObj: self.kws = kws def Format(self, indent=0): + charset = get_translation().charset() or 'us-ascii' output = ['<INPUT name="%s" type="%s" value="%s"' % (self.name, self.type, self.value)] for item in self.kws.items(): @@ -448,7 +452,10 @@ class InputObj: if self.checked: output.append('CHECKED') output.append('>') - return SPACE.join(output) + ret = SPACE.join(output) + if self.type == 'TEXT' and isinstance(ret, unicode): + ret = ret.encode(charset, 'xmlcharrefreplace') + return ret class SubmitButton(InputObj): @@ -486,6 +493,7 @@ class TextArea: self.readonly = readonly def Format(self, indent=0): + charset = get_translation().charset() or 'us-ascii' output = '<TEXTAREA NAME=%s' % self.name if self.rows: output += ' ROWS=%s' % self.rows @@ -496,6 +504,8 @@ class TextArea: if self.readonly: output += ' READONLY' output += '>%s</TEXTAREA>' % self.text + if isinstance(output, unicode): + output = output.encode(charset, 'xmlcharrefreplace') return output class FileUpload(InputObj): @@ -540,7 +550,9 @@ class WidgetArray: self.button_names, self.values): ischecked = (self.ischecked(i)) - item = self.Widget(self.name, value, ischecked).Format() + name + item = ('<label>' + + self.Widget(self.name, value, ischecked).Format() + + name + '</label>') items.append(item) if not self.horizontal: t.AddRow(items) diff --git a/Mailman/i18n.py b/Mailman/i18n.py index 5f926b77..605d4e76 100644 --- a/Mailman/i18n.py +++ b/Mailman/i18n.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2010 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,6 +17,7 @@ import sys import time +import locale import gettext from types import StringType, UnicodeType @@ -25,6 +26,18 @@ from Mailman.SafeDict import SafeDict _translation = None + +def _get_ctype_charset(): + old = locale.setlocale(locale.LC_CTYPE, '') + charset = locale.nl_langinfo(locale.CODESET) + locale.setlocale(locale.LC_CTYPE, old) + return charset + +if not mm_cfg.DISABLE_COMMAND_LOCALE_CSET: + _ctype_charset = _get_ctype_charset() +else: + _ctype_charset = None + def set_language(language=None): @@ -54,7 +67,7 @@ if _translation is None: -def _(s): +def _(s, frame=1): if s == '': return s assert s @@ -70,7 +83,7 @@ def _(s): # original string is 1) locals dictionary, 2) globals dictionary. # # First, get the frame of the caller - frame = sys._getframe(1) + frame = sys._getframe(frame) # A `safe' dictionary is used so we won't get an exception if there's a # missing key in the dictionary. dict = SafeDict(frame.f_globals.copy()) @@ -95,6 +108,23 @@ def _(s): +def tolocale(s): + global _ctype_charset + if isinstance(s, UnicodeType) or _ctype_charset is None: + return s + source = _translation.charset () + if not source: + return s + return unicode(s, source, 'replace').encode(_ctype_charset, 'replace') + +if mm_cfg.DISABLE_COMMAND_LOCALE_CSET: + C_ = _ +else: + def C_(s): + return tolocale(_(s, 2)) + + + def ctime(date): # Don't make these module globals since we have to do runtime translation # of the strings anyway. diff --git a/Mailman/versions.py b/Mailman/versions.py index 84943efc..b6c4231f 100755 --- a/Mailman/versions.py +++ b/Mailman/versions.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -93,6 +93,81 @@ def UpdateOldVars(l, stored_state): if not hasattr(l, newname) and newdefault is not uniqueval: setattr(l, newname, newdefault) + def recode(mlist, f, t): + """If the character set for a list's preferred_language has changed, + attempt to recode old string values into the new character set. + + mlist is the list, f is the old charset and t is the new charset. + """ + for x in dir(mlist): + if x.startswith('_'): + continue + nv = doitem(getattr(mlist, x), f, t) + if nv: + setattr(mlist, x, nv) + + def doitem(v, f, t): + """Recursively process lists, tuples and dictionary values and + convert strings as needed. Return either the updated item or None + if no change.""" + changed = False + if isinstance(v, str): + return convert(v, f, t) + elif isinstance(v, list): + for i in range(len(v)): + nv = doitem(v[i], f, t) + if nv: + changed = True + v[i] = nv + if changed: + return v + else: + return None + elif isinstance(v, tuple): + nt = () + for i in range(len(v)): + nv = doitem(v[i], f, t) + if nv: + changed = True + nt += (nv,) + else: + nt += (v[i],) + if changed: + return nt + else: + return None + elif isinstance(v, dict): + for k, ov in v.items(): + nv = doitem(ov, f, t) + if nv: + changed = True + v[k] = nv + if changed: + return v + else: + return None + else: + return None + + def convert(s, f, t): + """This does the actual character set conversion of the string s + from charset f to charset t.""" + + try: + u = unicode(s, f) + is_f = True + except ValueError: + is_f = False + try: + unicode(s, t) + is_t = True + except ValueError: + is_t = False + if is_f and not is_t: + return u.encode(t, 'replace') + else: + return None + # Migrate to 2.1b3, baw 17-Aug-2001 if hasattr(l, 'dont_respond_to_post_requests'): oldval = getattr(l, 'dont_respond_to_post_requests') @@ -313,6 +388,31 @@ def UpdateOldVars(l, stored_state): pass else: l.digest_members[k] = 0 + # + # Convert pre 2.2 topics regexps which were compiled in verbose mode + # to a non-verbose equivalent. + # + if stored_state['data_version'] < 106 and stored_state.has_key('topics'): + l.topics = [] + for name, pattern, description, emptyflag in stored_state['topics']: + pattern = Utils.strip_verbose_pattern(pattern) + l.topics.append((name, pattern, description, emptyflag)) + # + # Romanian and Russian had their character sets changed in 2.1.19 + # to utf-8. If there are any strings in the old encoding, try to recode + # them. + # + if stored_state['data_version'] < 108: + if l.preferred_language == 'ro': + if Utils.GetCharSet('ro') == 'utf-8': + recode(l, 'iso-8859-2', 'utf-8') + if l.preferred_language == 'ru': + if Utils.GetCharSet('ru') == 'utf-8': + recode(l, 'koi8-r', 'utf-8') + # + # from_is_list was called author_is_list in 2.1.16rc2 (only). + PreferStored('author_is_list', 'from_is_list', + mm_cfg.DEFAULT_FROM_IS_LIST) @@ -349,6 +449,8 @@ def NewVars(l): add_only_if_missing('personalize', 0) add_only_if_missing('first_strip_reply_to', mm_cfg.DEFAULT_FIRST_STRIP_REPLY_TO) + add_only_if_missing('subscribe_auto_approval', + mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL) add_only_if_missing('unsubscribe_policy', mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY) add_only_if_missing('send_goodbye_msg', mm_cfg.DEFAULT_SEND_GOODBYE_MSG) @@ -368,6 +470,9 @@ def NewVars(l): 'bounce_unrecognized_goes_to_list_owner', mm_cfg.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER) add_only_if_missing( + 'bounce_notify_owner_on_bounce_increment', + mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_BOUNCE_INCREMENT) + add_only_if_missing( 'bounce_notify_owner_on_disable', mm_cfg.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE) add_only_if_missing( @@ -385,6 +490,21 @@ def NewVars(l): # the current GUI description model. So, 0==Hold, 1==Reject, 2==Discard add_only_if_missing('member_moderation_action', 0) add_only_if_missing('member_moderation_notice', '') + add_only_if_missing('dmarc_moderation_action', + mm_cfg.DEFAULT_DMARC_MODERATION_ACTION) + add_only_if_missing('dmarc_quarantine_moderation_action', + mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION) + add_only_if_missing('dmarc_none_moderation_action', + mm_cfg.DEFAULT_DMARC_NONE_MODERATION_ACTION) + add_only_if_missing('dmarc_moderation_notice', '') + add_only_if_missing('dmarc_wrapped_message_text', + mm_cfg.DEFAULT_DMARC_WRAPPED_MESSAGE_TEXT) + add_only_if_missing('member_verbosity_threshold', + mm_cfg.DEFAULT_MEMBER_VERBOSITY_THRESHOLD) + add_only_if_missing('member_verbosity_interval', + mm_cfg.DEFAULT_MEMBER_VERBOSITY_INTERVAL) + add_only_if_missing('equivalent_domains', + mm_cfg.DEFAULT_EQUIVALENT_DOMAINS) add_only_if_missing('new_member_options', mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS) # Emergency moderation flag |