diff options
| -rwxr-xr-x | bin/update | 256 |
1 files changed, 227 insertions, 29 deletions
@@ -1,6 +1,6 @@ #! @PYTHON@ # -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -40,12 +40,17 @@ import time import errno import getopt import shutil +import cPickle import marshal import paths +import email + from Mailman import mm_cfg from Mailman import Utils from Mailman import MailList +from Mailman import Message +from Mailman import Pending from Mailman.LockFile import TimeOutError from Mailman.i18n import _ from Mailman.Queue.Switchboard import Switchboard @@ -406,17 +411,223 @@ def update_qfiles(): # Be sure the qfiles/in directory exists (we don't really need the # switchboard object, but it's convenient for creating the directory). sb = Switchboard(mm_cfg.INQUEUE_DIR) - for file in os.listdir(mm_cfg.QUEUE_DIR): - # Updating means just mokving the .db and .msg files to qfiles/in where + for filename in os.listdir(mm_cfg.QUEUE_DIR): + # Updating means just moving the .db and .msg files to qfiles/in where # it should be dequeued, converted, and processed normally. - if file.endswith('.msg'): - oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, file) - newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file) + if filename.endswith('.msg'): + oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename) + newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) os.rename(oldmsgfile, newmsgfile) - elif file.endswith('.db'): - olddbfile = os.path.join(mm_cfg.QUEUE_DIR, file) - newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file) + elif filename.endswith('.db'): + olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename) + newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) os.rename(olddbfile, newdbfile) + # Now update for the Mailman 2.1.5 qfile format. For every filebase in + # the qfiles/* directories that has both a .pck and a .db file, pull the + # data out and re-queue them. + for dirname in os.listdir(mm_cfg.QUEUE_DIR): + dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname) + sb = Switchboard(dirpath) + for filename in os.listdir(dirpath): + filepath = os.path.join(dirpath, filename) + filebase, ext = os.path.splitext(filepath) + # Handle the .db metadata files as part of the handling of the + # .pck or .msg message files. + if ext not in ('.pck', '.msg'): + continue + msg, data = dequeue(filebase) + if msg is not None and data is not None: + sb.enqueue(msg, data) + + + +# Implementations taken from the pre-2.1.5 Switchboard +def ext_read(filename): + fp = open(filename) + d = marshal.load(fp) + # Update from version 2 files + if d.get('version', 0) == 2: + del d['filebase'] + # Do the reverse conversion (repr -> float) + for attr in ['received_time']: + try: + sval = d[attr] + except KeyError: + pass + else: + # Do a safe eval by setting up a restricted execution + # environment. This may not be strictly necessary since we + # know they are floats, but it can't hurt. + d[attr] = eval(sval, {'__builtins__': {}}) + fp.close() + return d + + +def dequeue(filebase): + # Calculate the .db and .msg filenames from the given filebase. + msgfile = os.path.join(filebase + '.msg') + pckfile = os.path.join(filebase + '.pck') + dbfile = os.path.join(filebase + '.db') + # Now we are going to read the message and metadata for the given + # filebase. We want to read things in this order: first, the metadata + # file to find out whether the message is stored as a pickle or as + # plain text. Second, the actual message file. However, we want to + # first unlink the message file and then the .db file, because the + # qrunner only cues off of the .db file + msg = None + try: + data = ext_read(dbfile) + os.unlink(dbfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + data = {} + # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata + # was renamed to `rejection_notice', since dashes in the keys are not + # supported in METAFMT_ASCII. + if data.has_key('rejection-notice'): + data['rejection_notice'] = data['rejection-notice'] + del data['rejection-notice'] + msgfp = None + try: + try: + msgfp = open(pckfile) + msg = cPickle.load(msgfp) + os.unlink(pckfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + msgfp = None + try: + msgfp = open(msgfile) + msg = email.message_from_file(msgfp, Message.Message) + os.unlink(msgfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + except email.Errors.MessageParseError, e: + # This message was unparsable, most likely because its + # MIME encapsulation was broken. For now, there's not + # much we can do about it. + print _('message is unparsable: %(filebase)s') + msgfp.close() + msgfp = None + if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES: + # Cheapo way to ensure the directory exists w/ the + # proper permissions. + sb = Switchboard(mm_cfg.BADQUEUE_DIR) + os.rename(msgfile, os.path.join( + mm_cfg.BADQUEUE_DIR, filebase + '.txt')) + else: + os.unlink(msgfile) + msg = data = None + finally: + if msgfp: + msgfp.close() + return msg, data + + + +def update_pending(): + file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db') + file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck') + db = None + # Try to load the Mailman 2.0 file + try: + fp = open(file20) + except IOError, e: + if e.errno <> errno.ENOENT: raise + else: + print _('Updating Mailman 2.0 pending_subscriptions.db database') + db = marshal.load(fp) + # Convert to the pre-Mailman 2.1.5 format + db = Pending._update(db) + if db is None: + # Try to load the Mailman 2.1.x where x < 5, file + try: + fp = open(file214) + except IOError, e: + if e.errno <> errno.ENOENT: raise + else: + print _('Updating Mailman 2.1.4 pending.pck database') + db = cPickle.load(fp) + # Now upgrade the database to the 2.1.5 format. Each list now has its own + # pending.pck file, but only the RE_ENABLE operation actually recorded the + # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and + # CHANGE_OF_ADDRESS operations, we know the address of the person making + # the request so we can repend this request just for the lists the person + # is a member of. For the HELD_MESSAGE operation, we can check the list's + # requests.pck file for correlation. Evictions will take care of any + # misdirected pendings. + reenables_by_list = {} + addrops_by_address = {} + holds_by_id = {} + subs_by_address = {} + for key, val in db.items(): + if key in ('evictions', 'version'): + continue + try: + op = val[0] + data = val[1:] + except (IndexError, ValueError): + print _('Ignoring bad pended data: %(key)s: %(val)s') + continue + if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS): + # data[0] is the address being unsubscribed + addrops_by_address.setdefault(data[0], []).append((key, val)) + elif op == Pending.SUBSCRIPTION: + # data[0] is a UserDesc object + addr = data[0].address + subs_by_address.setdefault(addr, []).append((key, val)) + elif op == Pending.RE_ENABLE: + # data[0] is the mailing list's internal name + reenables_by_list.setdefault(data[0], []).append((key, val)) + elif op == Pending.HELD_MESSAGE: + # data[0] is the hold id. There better only be one entry per id + id = data[0] + assert not holds_by_id.has_key(id) + holds_by_id[id] = (key, val) + # Now we have to lock every list and re-pend all the appropriate + # requests. Note that this will reset all the expiration dates, but that + # should be fine. + for listname in Utils.list_names(): + mlist = MailList.MailList(listname) + # This is not the most efficient way to do this because it loads and + # saves the pending.pck file each time. :( + try: + for cookie, data in reenables_by_list.get(listname, []): + mlist.pend_repend(cookie, data) + for id, (cookie, data) in holds_by_id.items(): + try: + rec = mlist.GetRecord(id) + except KeyError: + # Not for this list + pass + else: + mlist.pend_repend(cookie, data) + del holds_by_id[id] + for addr, recs in subs_by_address.items(): + # We shouldn't have a subscription confirmation if the address + # is already a member of the mailing list. + if mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + for addr, recs in addrops_by_address.items(): + # We shouldn't have unsubscriptions or change of address + # requests for addresses which aren't members of the list. + if not mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + mlist.Save() + finally: + mlist.Unlock() + try: + os.unlink(file20) + except OSError, e: + if e.errno <> errno.ENOENT: raise + try: + os.unlink(file214) + except OSError, e: + if e.errno <> errno.ENOENT: raise @@ -479,31 +690,18 @@ If your archives are big, this could take a minute or two...""") mlist.Unlock() os.unlink(wmfile) print _('- usenet watermarks updated and gate_watermarks removed') - # - # In Mailman 2.1, the pending database format and file name has changed. - # - oldpendingfile = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db') - try: - fp = open(oldpendingfile) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - print _('Updating old pending_subscriptions.db database') - from Mailman import Pending - db = marshal.load(fp) - Pending._update(db) - fp.close() - os.unlink(oldpendingfile) - # + # In Mailman 2.1, the pending database format and file name changed, but + # in Mailman 2.1.5 it changed again. This should update all existing + # files to the 2.1.5 format. + update_pending() # In Mailman 2.1, the qfiles directory has a different structure and a - # different content. - # + # different content. Also, in Mailman 2.1.5 we collapsed the message + # files from separate .msg (pickled Message objects) and .db (marshalled + # dictionaries) to a shared .pck file containing two pickles. update_qfiles() - # # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. # There's no good way of figuring this out for releases prior to 2.0beta2 # :( - # if lastversion == NOTFRESH: print _(""" |