1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#! @PYTHON@
#
# Copyright (C) 1998-2014 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
"""Re-generate the Pipermail gzip'd archive flat files.
This script should be run nightly from cron. When run from the command line,
the following usage is understood:
Usage: %(program)s [-v] [-h] [listnames]
Where:
--verbose
-v
print each file as it's being gzip'd
--help
-h
print this message and exit
listnames
Optionally, only compress the .txt files for the named lists. Without
this, all archivable lists are processed.
"""
import sys
import os
import time
from stat import *
import getopt
try:
import gzip
except ImportError:
gzip = None
import paths
# mm_cfg must be imported before the other modules, due to the side-effect of
# it hacking sys.paths to include site-packages. Without this, running this
# script from cron with python -S will fail.
from Mailman import mm_cfg
from Mailman import Utils
from Mailman import MailList
from Mailman import i18n
program = sys.argv[0]
VERBOSE = 0
_ = i18n._
i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE)
def usage(code, msg=''):
if code:
fd = sys.stderr
else:
fd = sys.stdout
print >> fd, _(__doc__) % globals()
if msg:
print >> fd, msg
sys.exit(code)
def compress(txtfile):
if VERBOSE:
print "gzip'ing:", txtfile
infp = open(txtfile)
outfp = gzip.open(txtfile+'.gz', 'wb', 6)
outfp.write(infp.read())
outfp.close()
infp.close()
def main():
global VERBOSE
try:
opts, args = getopt.getopt(sys.argv[1:], 'vh', ['verbose', 'help'])
except getopt.error, msg:
usage(1, msg)
# defaults
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
VERBOSE = 1
# limit to the specified lists?
if args:
listnames = args
else:
listnames = Utils.list_names()
# process all the specified lists
for name in listnames:
mlist = MailList.MailList(name, lock=0)
if not mlist.archive:
continue
dir = mlist.archive_dir()
try:
allfiles = os.listdir(dir)
except os.error:
# has the list received any messages? if not, last_post_time will
# be zero, so it's not really a bogus archive dir.
if mlist.last_post_time > 0:
print 'List', name, 'has a bogus archive_directory:', dir
continue
if VERBOSE:
print 'Processing list:', name
files = []
for f in allfiles:
if f[-4:] <> '.txt':
continue
# stat both the .txt and .txt.gz files and append them only if
# the former is newer than the latter.
txtfile = os.path.join(dir, f)
gzpfile = txtfile + '.gz'
txt_mtime = os.stat(txtfile)[ST_MTIME]
try:
gzp_mtime = os.stat(gzpfile)[ST_MTIME]
except os.error:
gzp_mtime = -1
if txt_mtime > gzp_mtime:
files.append(txtfile)
for f in files:
compress(f)
if __name__ == '__main__' and \
gzip is not None and \
mm_cfg.ARCHIVE_TO_MBOX in (1, 2) and \
not mm_cfg.GZIP_ARCHIVE_TXT_FILES:
# we're only going to run the nightly archiver if messages are archived to
# the mbox, and the gzip file is not created on demand (i.e. for every
# individual post). This is the normal mode of operation. Also, be sure
# we can actually import the gzip module!
omask = os.umask(002)
try:
main()
finally:
os.umask(omask)
|
