# Copyright (C) 1998-2018 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Cook a message's Subject header.
Also do other manipulations of From:, Reply-To: and Cc: depending on
list configuration.
"""
from __future__ import nested_scopes
import re
from types import UnicodeType
from email.Charset import Charset
from email.Header import Header, decode_header, make_header
from email.Utils import parseaddr, formataddr, getaddresses
from email.Errors import HeaderParseError
from Mailman import i18n
from Mailman import mm_cfg
from Mailman import Utils
from Mailman.i18n import _
from Mailman.Logging.Syslog import syslog
CONTINUATION = ',\n '
COMMASPACE = ', '
MAXLINELEN = 78
# True/False
try:
True, False
except NameError:
True = 1
False = 0
def _isunicode(s):
return isinstance(s, UnicodeType)
nonascii = re.compile('[^\s!-~]')
def uheader(mlist, s, header_name=None, continuation_ws=' ', maxlinelen=None):
# Get the charset to encode the string in. Then search if there is any
# non-ascii character is in the string. If there is and the charset is
# us-ascii then we use iso-8859-1 instead. If the string is ascii only
# we use 'us-ascii' if another charset is specified.
charset = Utils.GetCharSet(mlist.preferred_language)
if nonascii.search(s):
# use list charset but ...
if charset == 'us-ascii':
charset = 'iso-8859-1'
else:
# there is no nonascii so ...
charset = 'us-ascii'
try:
return Header(s, charset, maxlinelen, header_name, continuation_ws)
except UnicodeError:
syslog('error', 'list: %s: can\'t decode "%s" as %s',
mlist.internal_name(), s, charset)
return Header('', charset, maxlinelen, header_name, continuation_ws)
def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True):
if ((msgdata.get('from_is_list') == 2 or
(msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and
not msgdata.get('_fasttrack')
) or name.lower() in ('from', 'reply-to', 'cc'):
# The or name.lower() in ... above is because when we are munging
# the From:, we want to defer the resultant changes to From:,
# Reply-To:, and/or Cc: until after the message passes through
# ToDigest, ToArchive and ToUsenet. Thus, we put them in
# msgdata[add_header] here and apply them in WrapMessage.
msgdata.setdefault('add_header', {})[name] = value
elif repl or not msg.has_key(name):
if delete:
del msg[name]
msg[name] = value
def process(mlist, msg, msgdata):
# Set the "X-Ack: no" header if noack flag is set.
if msgdata.get('noack'):
change_header('X-Ack', 'no', mlist, msg, msgdata)
# Because we're going to modify various important headers in the email
# message, we want to save some of the information in the msgdata
# dictionary for later. Specifically, the sender header will get waxed,
# but we need it for the Acknowledge module later.
# We may have already saved it; if so, don't clobber it here.
if 'original_sender' not in msgdata:
msgdata['original_sender'] = msg.get_sender()
# VirginRunner sets _fasttrack for internally crafted messages.
fasttrack = msgdata.get('_fasttrack')
if not msgdata.get('isdigest') and not fasttrack:
try:
prefix_subject(mlist, msg, msgdata)
except (UnicodeError, ValueError):
# TK: Sometimes subject header is not MIME encoded for 8bit
# simply abort prefixing.
pass
# Mark message so we know we've been here, but leave any existing
# X-BeenThere's intact.
change_header('X-BeenThere', mlist.GetListEmail(),
mlist, msg, msgdata, delete=False)
# Add Precedence: and other useful headers. None of these are standard
# and finding information on some of them are fairly difficult. Some are
# just common practice, and we'll add more here as they become necessary.
# Good places to look are:
#
# http://www.dsv.su.se/~jpalme/ietf/jp-ietf-home.html
# http://www.faqs.org/rfcs/rfc2076.html
#
# None of these headers are added if they already exist. BAW: some
# consider the advertising of this a security breach. I.e. if there are
# known exploits in a particular version of Mailman and we know a site is
# using such an old version, they may be vulnerable. It's too easy to
# edit the code to add a configuration variable to handle this.
change_header('X-Mailman-Version', mm_cfg.VERSION,
mlist, msg, msgdata, repl=False)
# We set "Precedence: list" because this is the recommendation from the
# sendmail docs, the most authoritative source of this header's semantics.
change_header('Precedence', 'list',
mlist, msg, msgdata, repl=False)
# Do we change the from so the list takes ownership of the email
if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack:
# Be as robust as possible here.
faddrs = getaddresses(msg.get_all('from', []))
# Strip the nulls and bad emails.
faddrs = [x for x in faddrs if x[1].find('@') > 0]
if len(faddrs) == 1:
realname, email = o_from = faddrs[0]
else:
# No From: or multiple addresses. Just punt and take
# the get_sender result.
realname = ''
email = msgdata['original_sender']
o_from = (realname, email)
if not realname:
if mlist.isMember(email):
realname = mlist.getMemberName(email) or email
else:
realname = email
# Remove domain from realname if it looks like an email address
realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
# Make a display name and RFC 2047 encode it if necessary. This is
# difficult and kludgy. If the realname came from From: it should be
# ascii or RFC 2047 encoded. If it came from the list, it should be
# in the charset of the list's preferred language or possibly unicode.
# if it's from the email address, it should be ascii. In any case,
# make it a unicode.
if isinstance(realname, unicode):
urn = realname
else:
rn, cs = ch_oneline(realname)
urn = unicode(rn, cs, errors='replace')
# likewise, the list's real_name which should be ascii, but use the
# charset of the list's preferred_language which should be a superset.
lcs = Utils.GetCharSet(mlist.preferred_language)
ulrn = unicode(mlist.real_name, lcs, errors='replace')
# get translated 'via' with dummy replacements
realname = '%(realname)s'
lrn = '%(lrn)s'
# We want the i18n context to be the list's preferred_language. It
# could be the poster's.
otrans = i18n.get_translation()
i18n.set_language(mlist.preferred_language)
via = _('%(realname)s via %(lrn)s')
i18n.set_translation(otrans)
uvia = unicode(via, lcs, errors='replace')
# Replace the dummy replacements.
uvia = re.sub(u'%\(lrn\)s', ulrn, re.sub(u'%\(realname\)s', urn, uvia))
# And get an RFC 2047 encoded header string.
dn = str(Header(uvia, lcs))
change_header('From',
formataddr((dn, mlist.GetListEmail())),
mlist, msg, msgdata)
else:
# Use this as a flag
o_from = None
# Reply-To: munging. Do not do this if the message is "fast tracked",
# meaning it is internally crafted and delivered to a specific user. BAW:
# Yuck, I really hate this feature but I've caved under the sheer pressure
# of the (very vocal) folks want it. OTOH, RFC 2822 allows Reply-To: to
# be a list of addresses, so instead of replacing the original, simply
# augment it. RFC 2822 allows max one Reply-To: header so collapse them
# if we're adding a value, otherwise don't touch it. (Should we collapse
# in all cases?)
# MAS: We need to do some things with the original From: if we've munged
# it for DMARC mitigation. We have goals for this process which are
# not completely compatible, so we do the best we can. Our goals are:
# 1) as long as the list is not anonymous, the original From: address
# should be obviously exposed, i.e. not just in a header that MUAs
# don't display.
# 2) the original From: address should not be in a comment or display
# name in the new From: because it is claimed that multiple domains
# in any fields in From: are indicative of spamminess. This means
# it should be in Reply-To: or Cc:.
# 3) the behavior of an MUA doing a 'reply' or 'reply all' should be
# consistent regardless of whether or not the From: is munged.
# Goal 3) implies sometimes the original From: should be in Reply-To:
# and sometimes in Cc:, and even so, this goal won't be achieved in
# all cases with all MUAs. In cases of conflict, the above ordering of
# goals is priority order.
if not fasttrack:
# A convenience function, requires nested scopes. pair is (name, addr)
new = []
d = {}
def add(pair):
lcaddr = pair[1].lower()
if d.has_key(lcaddr):
return
d[lcaddr] = pair
new.append(pair)
# List admin wants an explicit Reply-To: added
if mlist.reply_goes_to_list == 2:
add(parseaddr(mlist.reply_to_address))
# If we're not first stripping existing Reply-To: then we need to add
# the original Reply-To:'s to the list we're building up. In both
# cases we'll zap the existing field because RFC 2822 says max one is
# allowed.
o_rt = False
if not mlist.first_strip_reply_to:
orig = msg.get_all('reply-to', [])
for pair in getaddresses(orig):
# There's an original Reply-To: and we're not removing it.
add(pair)
o_rt = True
# We also need to put the old From: in Reply-To: in all cases where
# it is not going in Cc:. This is when reply_goes_to_list == 0 and
# either there was no original Reply-To: or we stripped it.
# However, if there was an original Reply-To:, unstripped, and it
# contained the original From: address we need to flag that it's
# there so we don't add the original From: to Cc:
if o_from and mlist.reply_goes_to_list == 0:
if o_rt:
if d.has_key(o_from[1].lower()):
# Original From: address is in original Reply-To:.
# Pretend we added it.
o_from = None
else:
add(o_from)
# Flag that we added it.
o_from = None
# Set Reply-To: header to point back to this list. Add this last
# because some folks think that some MUAs make it easier to delete
# addresses from the right than from the left.
if mlist.reply_goes_to_list == 1:
i18ndesc = uheader(mlist, mlist.description, 'Reply-To')
add((str(i18ndesc), mlist.GetListEmail()))
# Don't put Reply-To: back if there's nothing to add!
if new:
# Preserve order
change_header('Reply-To',
COMMASPACE.join([formataddr(pair) for pair in new]),
mlist, msg, msgdata)
else:
del msg['reply-to']
# The To field normally contains the list posting address. However
# when messages are fully personalized, that header will get
# overwritten with the address of the recipient. We need to get the
# posting address in one of the recipient headers or they won't be
# able to reply back to the list. It's possible the posting address
# was munged into the Reply-To header, but if not, we'll add it to a
# Cc header. BAW: should we force it into a Reply-To header in the
# above code?
# Also skip Cc if this is an anonymous list as list posting address
# is already in From and Reply-To in this case.
# We do add the Cc in cases where From: header munging is being done
# because even though the list address is in From:, the Reply-To:
# poster will override it. Brain dead MUAs may then address the list
# twice on a 'reply all', but reasonable MUAs should do the right
# thing. We also add the original From: to Cc: if it wasn't added
# to Reply-To:
add_list = (mlist.personalize == 2 and
mlist.reply_goes_to_list <> 1 and
not mlist.anonymous_list)
if add_list or o_from:
# Watch out for existing Cc headers, merge, and remove dups. Note
# that RFC 2822 says only zero or one Cc header is allowed.
new = []
d = {}
# If we're adding the original From:, add it first.
if o_from:
add(o_from)
# AvoidDuplicates may have set a new Cc: in msgdata.add_header,
# so check that.
if (msgdata.has_key('add_header') and
msgdata['add_header'].has_key('Cc')):
for pair in getaddresses([msgdata['add_header']['Cc']]):
add(pair)
else:
for pair in getaddresses(msg.get_all('cc', [])):
add(pair)
if add_list:
i18ndesc = uheader(mlist, mlist.description, 'Cc')
add((str(i18ndesc), mlist.GetListEmail()))
change_header('Cc',
COMMASPACE.join([formataddr(pair) for pair in new]),
mlist, msg, msgdata)
# Add list-specific headers as defined in RFC 2369 and RFC 2919, but only
# if the message is being crafted for a specific list (e.g. not for the
# password reminders).
#
# BAW: Some people really hate the List-* headers. It seems that the free
# version of Eudora (possibly on for some platforms) does not hide these
# headers by default, pissing off their users. Too bad. Fix the MUAs.
if msgdata.get('_nolist') or not mlist.include_rfc2369_headers:
return
# This will act like an email address for purposes of formataddr()
listid = '%s.%s' % (mlist.internal_name(), mlist.host_name)
cset = Utils.GetCharSet(mlist.preferred_language)
if mlist.description:
# Don't wrap the header since here we just want to get it properly RFC
# 2047 encoded.
i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998)
listid_h = formataddr((str(i18ndesc), listid))
# With some charsets (utf-8?) and some invalid chars, str(18ndesc) can
# be empty.
if str(i18ndesc):
listid_h = formataddr((str(i18ndesc), listid))
else:
listid_h = '<%s>' % listid
else:
# without desc we need to ensure the MUST brackets
listid_h = '<%s>' % listid
# We always add a List-ID: header.
change_header('List-Id', listid_h, mlist, msg, msgdata)
# For internally crafted messages, we also add a (nonstandard),
# "X-List-Administrivia: yes" header. For all others (i.e. those coming
# from list posts), we add a bunch of other RFC 2369 headers.
requestaddr = mlist.GetRequestEmail()
subfieldfmt = '<%s>, <mailto:%s?subject=%ssubscribe>'
listinfo = mlist.GetScriptURL('listinfo', absolute=1)
useropts = mlist.GetScriptURL('options', absolute=1)
headers = {}
if msgdata.get('reduced_list_headers'):
headers['X-List-Administrivia'] = 'yes'
else:
headers.update({
'List-Help' : '<mailto:%s?subject=help>' % requestaddr,
'List-Unsubscribe': subfieldfmt % (useropts, requestaddr, 'un'),
'List-Subscribe' : subfieldfmt % (listinfo, requestaddr, ''),
})
# List-Post: is controlled by a separate attribute
if mlist.include_list_post_header:
headers['List-Post'] = '<mailto:%s>' % mlist.GetListEmail()
# Add this header if we're archiving
if mlist.archive:
archiveurl = mlist.GetBaseArchiveURL()
headers['List-Archive'] = '<%s>' % archiveurl
# First we delete any pre-existing headers because the RFC permits only
# one copy of each, and we want to be sure it's ours.
for h, v in headers.items():
# Wrap these lines if they are too long. 78 character width probably
# shouldn't be hardcoded, but is at least text-MUA friendly. The
# adding of 2 is for the colon-space separator.
if len(h) + 2 + len(v) > 78:
v = CONTINUATION.join(v.split(', '))
change_header(h, v, mlist, msg, msgdata)
def prefix_subject(mlist, msg, msgdata):
# Add the subject prefix unless the message is a digest or is being fast
# tracked (e.g. internally crafted, delivered to a single user such as the
# list admin).
prefix = mlist.subject_prefix.strip()
if not prefix:
return
subject = msg.get('subject', '')
# Try to figure out what the continuation_ws is for the header
if isinstance(subject, Header):
lines = str(subject).splitlines()
else:
lines = subject.splitlines()
ws = ' '
if len(lines) > 1 and lines[1] and lines[1][0] in ' \t':
ws = lines[1][0]
msgdata['origsubj'] = subject
# The subject may be multilingual but we take the first charset as major
# one and try to decode. If it is decodable, returned subject is in one
# line and cset is properly set. If fail, subject is mime-encoded and
# cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one
# line function).
subject, cset = ch_oneline(subject)
# TK: Python interpreter has evolved to be strict on ascii charset code
# range. It is safe to use unicode string when manupilating header
# contents with re module. It would be best to return unicode in
# ch_oneline() but here is temporary solution.
subject = unicode(subject, cset)
# If the subject_prefix contains '%d', it is replaced with the
# mailing list sequential number. Sequential number format allows
# '%d' or '%05d' like pattern.
prefix_pattern = re.escape(prefix)
# unescape '%' :-<
prefix_pattern = '%'.join(prefix_pattern.split(r'\%'))
p = re.compile('%\d*d')
if p.search(prefix, 1):
# prefix have number, so we should search prefix w/number in subject.
# Also, force new style.
prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern)
old_style = False
else:
old_style = mm_cfg.OLD_STYLE_PREFIXING
subject = re.sub(prefix_pattern, '', subject)
# Previously the following re didn't have the first \s*. It would fail
# if the incoming Subject: was like '[prefix] Re: Re: Re:' because of the
# leading space after stripping the prefix. It is not known what MUA would
# create such a Subject:, but the issue was reported.
rematch = re.match(
'(\s*(RE|AW|SV|VS)\s*(\[\d+\])?\s*:\s*)+',
subject, re.I)
if rematch:
subject = subject[rematch.end():]
recolon = 'Re:'
else:
recolon = ''
# Strip leading and trailing whitespace from subject.
subject = subject.strip()
# At this point, subject may become null if someone post mail with
# Subject: [subject prefix]
if subject == '':
# We want the i18n context to be the list's preferred_language. It
# could be the poster's.
otrans = i18n.get_translation()
i18n.set_language(mlist.preferred_language)
subject = _('(no subject)')
i18n.set_translation(otrans)
cset = Utils.GetCharSet(mlist.preferred_language)
subject = unicode(subject, cset)
# and substitute %d in prefix with post_id
try:
prefix = prefix % mlist.post_id
except TypeError:
pass
# If charset is 'us-ascii', try to concatnate as string because there
# is some weirdness in Header module (TK)
if cset == 'us-ascii':
try:
if old_style:
h = u' '.join([recolon, prefix, subject])
else:
if recolon:
h = u' '.join([prefix, recolon, subject])
else:
h = u' '.join([prefix, subject])
h = h.encode('us-ascii')
h = uheader(mlist, h, 'Subject', continuation_ws=ws)
change_header('Subject', h, mlist, msg, msgdata)
ss = u' '.join([recolon, subject])
ss = ss.encode('us-ascii')
ss = uheader(mlist, ss, 'Subject', continuation_ws=ws)
msgdata['stripped_subject'] = ss
return
except UnicodeError:
pass
# Get the header as a Header instance, with proper unicode conversion
# Because of rfc2047 encoding, spaces between encoded words can be
# insignificant, so we need to append spaces to our encoded stuff.
prefix += ' '
if recolon:
recolon += ' '
if old_style:
h = uheader(mlist, recolon, 'Subject', continuation_ws=ws)
h.append(prefix)
else:
h = uheader(mlist, prefix, 'Subject', continuation_ws=ws)
h.append(recolon)
# TK: Subject is concatenated and unicode string.
subject = subject.encode(cset, 'replace')
h.append(subject, cset)
change_header('Subject', h, mlist, msg, msgdata)
ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws)
ss.append(subject, cset)
msgdata['stripped_subject'] = ss
def ch_oneline(headerstr):
# Decode header string in one line and convert into single charset
# copied and modified from ToDigest.py and Utils.py
# return (string, cset) tuple as check for failure
try:
d = decode_header(headerstr)
# at this point, we should rstrip() every string because some
# MUA deliberately add trailing spaces when composing return
# message.
d = [(s.rstrip(), c) for (s,c) in d]
cset = 'us-ascii'
for x in d:
# search for no-None charset
if x[1]:
cset = x[1]
break
h = make_header(d)
ustr = h.__unicode__()
oneline = u''.join(ustr.splitlines())
return oneline.encode(cset, 'replace'), cset
except (LookupError, UnicodeError, ValueError, HeaderParseError):
# possibly charset problem. return with undecoded string in one line.
return ''.join(headerstr.splitlines()), 'us-ascii'
