update_qfiles(), ext_read(), dequeue(): More gross stuff to upgrade the

separate .db and .msg qfiles to the single .pck file.

update_pending(): Updates the global data/pending.pck file to the per-list
.pck files as best as possible.

main(): update the qfiles and pending database.

1 files changed, 227 insertions, 29 deletions

diff --git a/bin/update b/bin/update
index 97c35c45..47affa25 100755
--- a/bin/update
+++ b/bin/update
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2004 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -40,12 +40,17 @@ import time
 import errno
 import getopt
 import shutil
+import cPickle
 import marshal
 
 import paths
+import email
+
 from Mailman import mm_cfg
 from Mailman import Utils
 from Mailman import MailList
+from Mailman import Message
+from Mailman import Pending
 from Mailman.LockFile import TimeOutError
 from Mailman.i18n import _
 from Mailman.Queue.Switchboard import Switchboard
@@ -406,17 +411,223 @@ def update_qfiles():
     # Be sure the qfiles/in directory exists (we don't really need the
     # switchboard object, but it's convenient for creating the directory).
     sb = Switchboard(mm_cfg.INQUEUE_DIR)
-    for file in os.listdir(mm_cfg.QUEUE_DIR):
-        # Updating means just mokving the .db and .msg files to qfiles/in where
+    for filename in os.listdir(mm_cfg.QUEUE_DIR):
+        # Updating means just moving the .db and .msg files to qfiles/in where
         # it should be dequeued, converted, and processed normally.
-        if file.endswith('.msg'):
-            oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, file)
-            newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+        if filename.endswith('.msg'):
+            oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+            newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
             os.rename(oldmsgfile, newmsgfile)
-        elif file.endswith('.db'):
-            olddbfile = os.path.join(mm_cfg.QUEUE_DIR, file)
-            newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+        elif filename.endswith('.db'):
+            olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+            newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
             os.rename(olddbfile, newdbfile)
+    # Now update for the Mailman 2.1.5 qfile format.  For every filebase in
+    # the qfiles/* directories that has both a .pck and a .db file, pull the
+    # data out and re-queue them.
+    for dirname in os.listdir(mm_cfg.QUEUE_DIR):
+        dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname)
+        sb = Switchboard(dirpath)
+        for filename in os.listdir(dirpath):
+            filepath = os.path.join(dirpath, filename)
+            filebase, ext = os.path.splitext(filepath)
+            # Handle the .db metadata files as part of the handling of the
+            # .pck or .msg message files.
+            if ext not in ('.pck', '.msg'):
+                continue
+            msg, data = dequeue(filebase)
+            if msg is not None and data is not None:
+                sb.enqueue(msg, data)
+
+
+
+# Implementations taken from the pre-2.1.5 Switchboard
+def ext_read(filename):
+    fp = open(filename)
+    d = marshal.load(fp)
+    # Update from version 2 files
+    if d.get('version', 0) == 2:
+        del d['filebase']
+    # Do the reverse conversion (repr -> float)
+    for attr in ['received_time']:
+        try:
+            sval = d[attr]
+        except KeyError:
+            pass
+        else:
+            # Do a safe eval by setting up a restricted execution
+            # environment.  This may not be strictly necessary since we
+            # know they are floats, but it can't hurt.
+            d[attr] = eval(sval, {'__builtins__': {}})
+    fp.close()
+    return d
+
+
+def dequeue(filebase):
+    # Calculate the .db and .msg filenames from the given filebase.
+    msgfile = os.path.join(filebase + '.msg')
+    pckfile = os.path.join(filebase + '.pck')
+    dbfile = os.path.join(filebase + '.db')
+    # Now we are going to read the message and metadata for the given
+    # filebase.  We want to read things in this order: first, the metadata
+    # file to find out whether the message is stored as a pickle or as
+    # plain text.  Second, the actual message file.  However, we want to
+    # first unlink the message file and then the .db file, because the
+    # qrunner only cues off of the .db file
+    msg = None
+    try:
+        data = ext_read(dbfile)
+        os.unlink(dbfile)
+    except EnvironmentError, e:
+        if e.errno <> errno.ENOENT: raise
+        data = {}
+    # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata
+    # was renamed to `rejection_notice', since dashes in the keys are not
+    # supported in METAFMT_ASCII.
+    if data.has_key('rejection-notice'):
+        data['rejection_notice'] = data['rejection-notice']
+        del data['rejection-notice']
+    msgfp = None
+    try:
+        try:
+            msgfp = open(pckfile)
+            msg = cPickle.load(msgfp)
+            os.unlink(pckfile)
+        except EnvironmentError, e:
+            if e.errno <> errno.ENOENT: raise
+            msgfp = None
+            try:
+                msgfp = open(msgfile)
+                msg = email.message_from_file(msgfp, Message.Message)
+                os.unlink(msgfile)
+            except EnvironmentError, e:
+                if e.errno <> errno.ENOENT: raise
+            except email.Errors.MessageParseError, e:
+                # This message was unparsable, most likely because its
+                # MIME encapsulation was broken.  For now, there's not
+                # much we can do about it.
+                print _('message is unparsable: %(filebase)s')
+                msgfp.close()
+                msgfp = None
+                if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES:
+                    # Cheapo way to ensure the directory exists w/ the
+                    # proper permissions.
+                    sb = Switchboard(mm_cfg.BADQUEUE_DIR)
+                    os.rename(msgfile, os.path.join(
+                        mm_cfg.BADQUEUE_DIR, filebase + '.txt'))
+                else:
+                    os.unlink(msgfile)
+                msg = data = None
+    finally:
+        if msgfp:
+            msgfp.close()
+    return msg, data
+
+
+
+def update_pending():
+    file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
+    file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck')
+    db = None
+    # Try to load the Mailman 2.0 file
+    try:
+        fp = open(file20)
+    except IOError, e:
+        if e.errno <> errno.ENOENT: raise
+    else:
+        print _('Updating Mailman 2.0 pending_subscriptions.db database')
+        db = marshal.load(fp)
+        # Convert to the pre-Mailman 2.1.5 format
+        db = Pending._update(db)
+    if db is None:
+        # Try to load the Mailman 2.1.x where x < 5, file
+        try:
+            fp = open(file214)
+        except IOError, e:
+            if e.errno <> errno.ENOENT: raise
+        else:
+            print _('Updating Mailman 2.1.4 pending.pck database')
+            db = cPickle.load(fp)
+    # Now upgrade the database to the 2.1.5 format.  Each list now has its own
+    # pending.pck file, but only the RE_ENABLE operation actually recorded the
+    # listname in the request.  For the SUBSCRIPTION, UNSUBSCRIPTION, and
+    # CHANGE_OF_ADDRESS operations, we know the address of the person making
+    # the request so we can repend this request just for the lists the person
+    # is a member of.  For the HELD_MESSAGE operation, we can check the list's
+    # requests.pck file for correlation.  Evictions will take care of any
+    # misdirected pendings.
+    reenables_by_list = {}
+    addrops_by_address = {}
+    holds_by_id = {}
+    subs_by_address = {}
+    for key, val in db.items():
+        if key in ('evictions', 'version'):
+            continue
+        try:
+            op = val[0]
+            data = val[1:]
+        except (IndexError, ValueError):
+            print _('Ignoring bad pended data: %(key)s: %(val)s')
+            continue
+        if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS):
+            # data[0] is the address being unsubscribed
+            addrops_by_address.setdefault(data[0], []).append((key, val))
+        elif op == Pending.SUBSCRIPTION:
+            # data[0] is a UserDesc object
+            addr = data[0].address
+            subs_by_address.setdefault(addr, []).append((key, val))
+        elif op == Pending.RE_ENABLE:
+            # data[0] is the mailing list's internal name
+            reenables_by_list.setdefault(data[0], []).append((key, val))
+        elif op == Pending.HELD_MESSAGE:
+            # data[0] is the hold id.  There better only be one entry per id
+            id = data[0]
+            assert not holds_by_id.has_key(id)
+            holds_by_id[id] = (key, val)
+    # Now we have to lock every list and re-pend all the appropriate
+    # requests.  Note that this will reset all the expiration dates, but that
+    # should be fine.
+    for listname in Utils.list_names():
+        mlist = MailList.MailList(listname)
+        # This is not the most efficient way to do this because it loads and
+        # saves the pending.pck file each time. :(
+        try:
+            for cookie, data in reenables_by_list.get(listname, []):
+                mlist.pend_repend(cookie, data)
+            for id, (cookie, data) in holds_by_id.items():
+                try:
+                    rec = mlist.GetRecord(id)
+                except KeyError:
+                    # Not for this list
+                    pass
+                else:
+                    mlist.pend_repend(cookie, data)
+                    del holds_by_id[id]
+            for addr, recs in subs_by_address.items():
+                # We shouldn't have a subscription confirmation if the address
+                # is already a member of the mailing list.
+                if mlist.isMember(addr):
+                    continue
+                for cookie, data in recs:
+                    mlist.pend_repend(cookie, data)
+            for addr, recs in addrops_by_address.items():
+                # We shouldn't have unsubscriptions or change of address
+                # requests for addresses which aren't members of the list.
+                if not mlist.isMember(addr):
+                    continue
+                for cookie, data in recs:
+                    mlist.pend_repend(cookie, data)
+            mlist.Save()
+        finally:
+            mlist.Unlock()
+    try:
+        os.unlink(file20)
+    except OSError, e:
+        if e.errno <> errno.ENOENT: raise
+    try:
+        os.unlink(file214)
+    except OSError, e:
+        if e.errno <> errno.ENOENT: raise
 
 
 
@@ -479,31 +690,18 @@ If your archives are big, this could take a minute or two...""")
                 mlist.Unlock()
         os.unlink(wmfile)
         print _('- usenet watermarks updated and gate_watermarks removed')
-    #
-    # In Mailman 2.1, the pending database format and file name has changed.
-    #
-    oldpendingfile = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
-    try:
-        fp = open(oldpendingfile)
-    except IOError, e:
-        if e.errno <> errno.ENOENT: raise
-    else:
-        print _('Updating old pending_subscriptions.db database')
-        from Mailman import Pending
-        db = marshal.load(fp)
-        Pending._update(db)
-        fp.close()
-        os.unlink(oldpendingfile)
-    #
+    # In Mailman 2.1, the pending database format and file name changed, but
+    # in Mailman 2.1.5 it changed again.  This should update all existing
+    # files to the 2.1.5 format.
+    update_pending()
     # In Mailman 2.1, the qfiles directory has a different structure and a
-    # different content.
-    #
+    # different content.  Also, in Mailman 2.1.5 we collapsed the message
+    # files from separate .msg (pickled Message objects) and .db (marshalled
+    # dictionaries) to a shared .pck file containing two pickles.
     update_qfiles()
-    #
     # This warning was necessary for the upgrade from 1.0b9 to 1.0b10.
     # There's no good way of figuring this out for releases prior to 2.0beta2
     # :(
-    #
     if lastversion == NOTFRESH:
         print _("""