path: root/Mailman/ListAdmin.py

# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

"""Mixin class for MailList which handles administrative requests.

Two types of admin requests are currently supported: adding members to a
closed or semi-closed list, and moderated posts.

Pending subscriptions which are requiring a user's confirmation are handled
elsewhere.
"""

import os
import time
import errno
import cPickle
import marshal
from cStringIO import StringIO

import email
from email.MIMEMessage import MIMEMessage
from email.Generator import Generator
from email.Utils import getaddresses

from Mailman import mm_cfg
from Mailman import Utils
from Mailman import Message
from Mailman import Errors
from Mailman.UserDesc import UserDesc
from Mailman.Queue.sbcache import get_switchboard
from Mailman.Logging.Syslog import syslog
from Mailman import i18n

_ = i18n._
D_ = i18n.D_

# Request types requiring admin approval
IGN = 0
HELDMSG = 1
SUBSCRIPTION = 2
UNSUBSCRIPTION = 3

# Return status from __handlepost()
DEFER = 0
REMOVE = 1
LOST = 2

DASH = '-'
NL = '\n'

try:
    True, False
except NameError:
    True = 1
    False = 0



class ListAdmin:
    def InitVars(self):
        # non-configurable data
        self.next_request_id = 1

    def InitTempVars(self):
        self.__db = None
        self.__filename = os.path.join(self.fullpath(), 'request.pck')

    def __opendb(self):
        if self.__db is None:
            assert self.Locked()
            try:
                fp = open(self.__filename)
                try:
                    self.__db = cPickle.load(fp)
                finally:
                    fp.close()
            except IOError, e:
                if e.errno <> errno.ENOENT: raise
                self.__db = {}
                # put version number in new database
                self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION

    def __closedb(self):
        if self.__db is not None:
            assert self.Locked()
            # Save the version number
            self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION
            # Now save a temp file and do the tmpfile->real file dance.  BAW:
            # should we be as paranoid as for the config.pck file?  Should we
            # use pickle?
            tmpfile = self.__filename + '.tmp'
            omask = os.umask(007)
            try:
                fp = open(tmpfile, 'w')
                try:
                    cPickle.dump(self.__db, fp, 1)
                    fp.flush()
                    os.fsync(fp.fileno())
                finally:
                    fp.close()
            finally:
                os.umask(omask)
            self.__db = None
            # Do the dance
            os.rename(tmpfile, self.__filename)

    def __nextid(self):
        assert self.Locked()
        while True:
            next = self.next_request_id
            self.next_request_id += 1
            if not self.__db.has_key(next):
                break
        return next

    def SaveRequestsDb(self):
        self.__closedb()

    def NumRequestsPending(self):
        self.__opendb()
        # Subtract one for the version pseudo-entry
        return len(self.__db) - 1

    def __getmsgids(self, rtype):
        self.__opendb()
        ids = [k for k, (op, data) in self.__db.items() if op == rtype]
        ids.sort()
        return ids

    def GetHeldMessageIds(self):
        return self.__getmsgids(HELDMSG)

    def GetSubscriptionIds(self):
        return self.__getmsgids(SUBSCRIPTION)

    def GetUnsubscriptionIds(self):
        return self.__getmsgids(UNSUBSCRIPTION)

    def GetRecord(self, id):
        self.__opendb()
        type, data = self.__db[id]
        return data

    def GetRecordType(self, id):
        self.__opendb()
        type, data = self.__db[id]
        return type

    def HandleRequest(self, id, value, comment=None, preserve=None,
                      forward=None, addr=None):
        self.__opendb()
        rtype, data = self.__db[id]
        if rtype == HELDMSG:
            status = self.__handlepost(data, value, comment, preserve,
                                       forward, addr)
        elif rtype == UNSUBSCRIPTION:
            status = self.__handleunsubscription(data, value, comment)
        else:
            assert rtype == SUBSCRIPTION
            status = self.__handlesubscription(data, value, comment)
        if status <> DEFER:
            # BAW: Held message ids are linked to Pending cookies, allowing
            # the user to cancel their post before the moderator has approved
            # it.  We should probably remove the cookie associated with this
            # id, but we have no way currently of correlating them. :(
            del self.__db[id]

    def HoldMessage(self, msg, reason, msgdata={}):
        # Make a copy of msgdata so that subsequent changes won't corrupt the
        # request database.  TBD: remove the `filebase' key since this will
        # not be relevant when the message is resurrected.
        msgdata = msgdata.copy()
        # assure that the database is open for writing
        self.__opendb()
        # get the next unique id
        id = self.__nextid()
        # get the message sender
        sender = msg.get_sender()
        # calculate the file name for the message text and write it to disk
        if mm_cfg.HOLD_MESSAGES_AS_PICKLES:
            ext = 'pck'
        else:
            ext = 'txt'
        filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext)
        omask = os.umask(007)
        try:
            fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w')
            try:
                if mm_cfg.HOLD_MESSAGES_AS_PICKLES:
                    cPickle.dump(msg, fp, 1)
                else:
                    g = Generator(fp)
                    g.flatten(msg, 1)
                fp.flush()
                os.fsync(fp.fileno())
            finally:
                fp.close()
        finally:
            os.umask(omask)
        # save the information to the request database.  for held message
        # entries, each record in the database will be of the following
        # format:
        #
        # the time the message was received
        # the sender of the message
        # the message's subject
        # a string description of the problem
        # name of the file in $PREFIX/data containing the msg text
        # an additional dictionary of message metadata
        #
        msgsubject = msg.get('subject', _('(no subject)'))
        if not sender:
            sender = _('<missing>')
        data = time.time(), sender, msgsubject, reason, filename, msgdata
        self.__db[id] = (HELDMSG, data)
        return id

    def __handlepost(self, record, value, comment, preserve, forward, addr):
        # For backwards compatibility with pre 2.0beta3
        ptime, sender, subject, reason, filename, msgdata = record
        path = os.path.join(mm_cfg.DATA_DIR, filename)
        # Handle message preservation
        if preserve:
            parts = os.path.split(path)[1].split(DASH)
            parts[0] = 'spam'
            spamfile = DASH.join(parts)
            # Preserve the message as plain text, not as a pickle
            try:
                fp = open(path)
            except IOError, e:
                if e.errno <> errno.ENOENT: raise
                return LOST
            try:
                if path.endswith('.pck'):
                    msg = cPickle.load(fp)
                else:
                    assert path.endswith('.txt'), '%s not .pck or .txt' % path
                    msg = fp.read()
            finally:
                fp.close()
            # Save the plain text to a .msg file, not a .pck file
            outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile)
            head, ext = os.path.splitext(outpath)
            outpath = head + '.msg'
            outfp = open(outpath, 'w')
            try:
                if path.endswith('.pck'):
                    g = Generator(outfp)
                    g.flatten(msg, 1)
                else:
                    outfp.write(msg)
            finally:
                outfp.close()
        # Now handle updates to the database
        rejection = None
        fp = None
        msg = None
        status = REMOVE
        if value == mm_cfg.DEFER:
            # Defer
            status = DEFER
        elif value == mm_cfg.APPROVE:
            # Approved.
            try:
                msg = readMessage(path)
            except IOError, e:
                if e.errno <> errno.ENOENT: raise
                return LOST
            msg = readMessage(path)
            msgdata['approved'] = 1
            # adminapproved is used by the Emergency handler
            msgdata['adminapproved'] = 1
            # Calculate a new filebase for the approved message, otherwise
            # delivery errors will cause duplicates.
            try:
                del msgdata['filebase']
            except KeyError:
                pass
            # Queue the file for delivery by qrunner.  Trying to deliver the
            # message directly here can lead to a huge delay in web
            # turnaround.  Log the moderation and add a header.
            msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
            syslog('vette', '%s: held message approved, message-id: %s',
                   self.internal_name(),
                   msg.get('message-id', 'n/a'))
            # Stick the message back in the incoming queue for further
            # processing.
            inq = get_switchboard(mm_cfg.INQUEUE_DIR)
            inq.enqueue(msg, _metadata=msgdata)
        elif value == mm_cfg.REJECT:
            # Rejected
            rejection = 'Refused'
            lang = self.getMemberLanguage(sender)
            subject = Utils.oneline(subject, Utils.GetCharSet(lang))
            self.__refuse(_('Posting of your message titled "%(subject)s"'),
                          sender, comment or _('[No reason given]'),
                          lang=lang)
        else:
            assert value == mm_cfg.DISCARD
            # Discarded
            rejection = 'Discarded'
        # Forward the message
        if forward and addr:
            # If we've approved the message, we need to be sure to craft a
            # completely unique second message for the forwarding operation,
            # since we don't want to share any state or information with the
            # normal delivery.
            try:
                copy = readMessage(path)
            except IOError, e:
                if e.errno <> errno.ENOENT: raise
                raise Errors.LostHeldMessage(path)
            # It's possible the addr is a comma separated list of addresses.
            addrs = getaddresses([addr])
            if len(addrs) == 1:
                realname, addr = addrs[0]
                # If the address getting the forwarded message is a member of
                # the list, we want the headers of the outer message to be
                # encoded in their language.  Otherwise it'll be the preferred
                # language of the mailing list.
                lang = self.getMemberLanguage(addr)
            else:
                # Throw away the realnames
                addr = [a for realname, a in addrs]
                # Which member language do we attempt to use?  We could use
                # the first match or the first address, but in the face of
                # ambiguity, let's just use the list's preferred language
                lang = self.preferred_language
            otrans = i18n.get_translation()
            i18n.set_language(lang)
            try:
                fmsg = Message.UserNotification(
                    addr, self.GetBouncesEmail(),
                    _('Forward of moderated message'),
                    lang=lang)
            finally:
                i18n.set_translation(otrans)
            fmsg.set_type('message/rfc822')
            fmsg.attach(copy)
            fmsg.send(self)
        # Log the rejection
        if rejection:
            note = '''%(listname)s: %(rejection)s posting:
\tFrom: %(sender)s
\tSubject: %(subject)s''' % {
                'listname' : self.internal_name(),
                'rejection': rejection,
                'sender'   : str(sender).replace('%', '%%'),
                'subject'  : str(subject).replace('%', '%%'),
                }
            if comment:
                note += '\n\tReason: ' + comment.replace('%', '%%')
            syslog('vette', note)
        # Always unlink the file containing the message text.  It's not
        # necessary anymore, regardless of the disposition of the message.
        if status <> DEFER:
            try:
                os.unlink(path)
            except OSError, e:
                if e.errno <> errno.ENOENT: raise
                # We lost the message text file.  Clean up our housekeeping
                # and inform of this status.
                return LOST
        return status

    def HoldSubscription(self, addr, fullname, password, digest, lang):
        # Assure that the database is open for writing
        self.__opendb()
        # Get the next unique id
        id = self.__nextid()
        # Save the information to the request database. for held subscription
        # entries, each record in the database will be one of the following
        # format:
        #
        # the time the subscription request was received
        # the subscriber's address
        # the subscriber's selected password (TBD: is this safe???)
        # the digest flag
        # the user's preferred language
        data = time.time(), addr, fullname, password, digest, lang
        self.__db[id] = (SUBSCRIPTION, data)
        #
        # TBD: this really shouldn't go here but I'm not sure where else is
        # appropriate.
        syslog('vette', '%s: held subscription request from %s',
               self.internal_name(), addr)
        # Possibly notify the administrator in default list language
        if self.admin_immed_notify:
            i18n.set_language(self.preferred_language)
            realname = self.real_name
            subject = _(
                'New subscription request to list %(realname)s from %(addr)s')
            text = Utils.maketext(
                'subauth.txt',
                {'username'   : addr,
                 'listname'   : self.internal_name(),
                 'hostname'   : self.host_name,
                 'admindb_url': self.GetScriptURL('admindb', absolute=1),
                 }, mlist=self)
            # This message should appear to come from the <list>-owner so as
            # to avoid any useless bounce processing.
            owneraddr = self.GetOwnerEmail()
            msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
                                           self.preferred_language)
            msg.send(self, **{'tomoderators': 1})
            # Restore the user's preferred language.
            i18n.set_language(lang)

    def __handlesubscription(self, record, value, comment):
        stime, addr, fullname, password, digest, lang = record
        if value == mm_cfg.DEFER:
            return DEFER
        elif value == mm_cfg.DISCARD:
            syslog('vette', '%s: discarded subscription request from %s',
                   self.internal_name(), addr)
        elif value == mm_cfg.REJECT:
            self.__refuse(_('Subscription request'), addr,
                          comment or _('[No reason given]'),
                          lang=lang)
            syslog('vette', """%s: rejected subscription request from %s
\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]')
        else:
            # subscribe
            assert value == mm_cfg.SUBSCRIBE
            try:
                userdesc = UserDesc(addr, fullname, password, digest, lang)
                self.ApprovedAddMember(userdesc,
                                       whence=D_('via admin approval'))
            except Errors.MMAlreadyAMember:
                # User has already been subscribed, after sending the request
                pass
            # TBD: disgusting hack: ApprovedAddMember() can end up closing
            # the request database.
            self.__opendb()
        return REMOVE

    def HoldUnsubscription(self, addr):
        # Assure the database is open for writing
        self.__opendb()
        # Get the next unique id
        id = self.__nextid()
        # All we need to do is save the unsubscribing address
        self.__db[id] = (UNSUBSCRIPTION, addr)
        syslog('vette', '%s: held unsubscription request from %s',
               self.internal_name(), addr)
        # Possibly notify the administrator of the hold
        if self.admin_immed_notify:
            realname = self.real_name
            subject = _(
                'New unsubscription request from %(realname)s by %(addr)s')
            text = Utils.maketext(
                'unsubauth.txt',
                {'username'   : addr,
                 'listname'   : self.internal_name(),
                 'hostname'   : self.host_name,
                 'admindb_url': self.GetScriptURL('admindb', absolute=1),
                 }, mlist=self)
            # This message should appear to come from the <list>-owner so as
            # to avoid any useless bounce processing.
            owneraddr = self.GetOwnerEmail()
            msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
                                           self.preferred_language)
            msg.send(self, **{'tomoderators': 1})

    def __handleunsubscription(self, record, value, comment):
        addr = record
        if value == mm_cfg.DEFER:
            return DEFER
        elif value == mm_cfg.DISCARD:
            syslog('vette', '%s: discarded unsubscription request from %s',
                   self.internal_name(), addr)
        elif value == mm_cfg.REJECT:
            self.__refuse(_('Unsubscription request'), addr, comment)
            syslog('vette', """%s: rejected unsubscription request from %s
\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]')
        else:
            assert value == mm_cfg.UNSUBSCRIBE
            try:
                self.ApprovedDeleteMember(addr)
            except Errors.NotAMemberError:
                # User has already been unsubscribed
                pass
        return REMOVE

    def __refuse(self, request, recip, comment, origmsg=None, lang=None):
        # As this message is going to the requestor, try to set the language
        # to his/her language choice, if they are a member.  Otherwise use the
        # list's preferred language.
        realname = self.real_name
        if lang is None:
            lang = self.getMemberLanguage(recip)
        text = Utils.maketext(
            'refuse.txt',
            {'listname' : realname,
             'request'  : request,
             'reason'   : comment,
             'adminaddr': self.GetOwnerEmail(),
            }, lang=lang, mlist=self)
        otrans = i18n.get_translation()
        i18n.set_language(lang)
        try:
            # add in original message, but not wrap/filled
            if origmsg:
                text = NL.join(
                    [text,
                     '---------- ' + _('Original Message') + ' ----------',
                     str(origmsg)
                     ])
            subject = _('Request to mailing list %(realname)s rejected')
        finally:
            i18n.set_translation(otrans)
        msg = Message.UserNotification(recip, self.GetOwnerEmail(),
                                       subject, text, lang)
        msg.send(self)

    def _UpdateRecords(self):
        # Subscription records have changed since MM2.0.x.  In that family,
        # the records were of length 4, containing the request time, the
        # address, the password, and the digest flag.  In MM2.1a2, they grew
        # an additional language parameter at the end.  In MM2.1a4, they grew
        # a fullname slot after the address.  This semi-public method is used
        # by the update script to coerce all subscription records to the
        # latest MM2.1 format.
        #
        # Held message records have historically either 5 or 6 items too.
        # These always include the requests time, the sender, subject, default
        # rejection reason, and message text.  When of length 6, it also
        # includes the message metadata dictionary on the end of the tuple.
        #
        # In Mailman 2.1.5 we converted these files to pickles.
        filename = os.path.join(self.fullpath(), 'request.db')
        try:
            fp = open(filename)
            try:
                self.__db = marshal.load(fp)
            finally:
                fp.close()
            os.unlink(filename)
        except IOError, e:
            if e.errno <> errno.ENOENT: raise
            filename = os.path.join(self.fullpath(), 'request.pck')
            try:
                fp = open(filename)
                try:
                    self.__db = cPickle.load(fp)
                finally:
                    fp.close()
            except IOError, e:
                if e.errno <> errno.ENOENT: raise
                self.__db = {}
        for id, x in self.__db.items():
            # A bug in versions 2.1.1 through 2.1.11 could have resulted in
            # just info being stored instead of (op, info)
            if len(x) == 2:
                op, info = x
            elif len(x) == 6:
                # This is the buggy info. Check for digest flag.
                if x[4] in (0, 1):
                    op = SUBSCRIPTION
                else:
                    op = HELDMSG
                self.__db[id] = op, x
                continue
            else:
                assert False, 'Unknown record format in %s' % self.__filename
            if op == SUBSCRIPTION:
                if len(info) == 4:
                    # pre-2.1a2 compatibility
                    when, addr, passwd, digest = info
                    fullname = ''
                    lang = self.preferred_language
                elif len(info) == 5:
                    # pre-2.1a4 compatibility
                    when, addr, passwd, digest, lang = info
                    fullname = ''
                else:
                    assert len(info) == 6, 'Unknown subscription record layout'
                    continue
                # Here's the new layout
                self.__db[id] = op, (when, addr, fullname, passwd,
                                     digest, lang)
            elif op == HELDMSG:
                if len(info) == 5:
                    when, sender, subject, reason, text = info
                    msgdata = {}
                else:
                    assert len(info) == 6, 'Unknown held msg record layout'
                    continue
                # Here's the new layout
                self.__db[id] = op, (when, sender, subject, reason,
                                     text, msgdata)
        # All done
        self.__closedb()



def readMessage(path):
    # For backwards compatibility, we must be able to read either a flat text
    # file or a pickle.
    ext = os.path.splitext(path)[1]
    fp = open(path)
    try:
        if ext == '.txt':
            msg = email.message_from_file(fp, Message.Message)
        else:
            assert ext == '.pck'
            msg = cPickle.load(fp)
    finally:
        fp.close()
    return msg