1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
|
#! @PYTHON@
# Copyright (C) 2001-2013 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Primary start-up and shutdown script for Mailman's qrunner daemon.
This script starts, stops, and restarts the main Mailman queue runners, making
sure that the various long-running qrunners are still alive and kicking. It
does this by forking and exec'ing the qrunners and waiting on their pids.
When it detects a subprocess has exited, it may restart it.
The qrunners respond to SIGINT, SIGTERM, and SIGHUP. SIGINT and SIGTERM both
cause the qrunners to exit cleanly, but the master will only restart qrunners
that have exited due to a SIGINT. SIGHUP causes the master and the qrunners
to close their log files, and reopen then upon the next printed message.
The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply
passes on to the qrunners (note that the master will close and reopen its own
log files on receipt of a SIGHUP). The master also leaves its own process id
in the file data/master-qrunner.pid but you normally don't need to use this
pid directly. The `start', `stop', `restart', and `reopen' commands handle
everything for you.
Usage: %(PROGRAM)s [options] [ start | stop | restart | reopen ]
Options:
-n/--no-restart
Don't restart the qrunners when they exit because of an error or a
SIGINT. They are never restarted if they exit in response to a
SIGTERM. Use this only for debugging. Only useful if the `start'
command is given.
-u/--run-as-user
Normally, this script will refuse to run if the user id and group id
are not set to the `mailman' user and group (as defined when you
configured Mailman). If run as root, this script will change to this
user and group before the check is made.
This can be inconvenient for testing and debugging purposes, so the -u
flag means that the step that sets and checks the uid/gid is skipped,
and the program is run as the current user and group. This flag is
not recommended for normal production environments.
Note though, that if you run with -u and are not in the mailman group,
you may have permission problems, such as begin unable to delete a
list's archives through the web. Tough luck!
-s/--stale-lock-cleanup
If mailmanctl finds an existing master lock, it will normally exit
with an error message. With this option, mailmanctl will perform an
extra level of checking. If a process matching the host/pid described
in the lock file is running, mailmanctl will still exit, but if no
matching process is found, mailmanctl will remove the apparently stale
lock and make another attempt to claim the master lock.
-q/--quiet
Don't print status messages. Error messages are still printed to
standard error.
-h/--help
Print this message and exit.
Commands:
start - Start the master daemon and all qrunners. Prints a message and
exits if the master daemon is already running.
stop - Stops the master daemon and all qrunners. After stopping, no
more messages will be processed.
restart - Restarts the qrunners, but not the master process. Use this
whenever you upgrade or update Mailman so that the qrunners will
use the newly installed code.
reopen - This will close all log files, causing them to be re-opened the
next time a message is written to them
"""
import sys
import os
import time
import getopt
import signal
import errno
import pwd
import grp
import socket
import paths
from Mailman import mm_cfg
from Mailman import Utils
from Mailman import LockFile
from Mailman import Errors
from Mailman.MailList import MailList
from Mailman.i18n import _
from Mailman.Logging.Syslog import syslog
from Mailman.Logging.Utils import LogStdErr
PROGRAM = sys.argv[0]
COMMASPACE = ', '
DOT = '.'
# Locking contantsa
LOCKFILE = os.path.join(mm_cfg.LOCK_DIR, mm_cfg.MASTER_LOCK_FILE)
# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME
# needn't be (much) longer than SNOOZE. We pad it 6 hours just to be safe.
LOCK_LIFETIME = mm_cfg.days(1) + mm_cfg.hours(6)
SNOOZE = mm_cfg.days(1)
MAX_RESTARTS = 10
LogStdErr('error', 'mailmanctl', manual_reprime=0)
def usage(code, msg=''):
if code:
fd = sys.stderr
else:
fd = sys.stdout
print >> fd, _(__doc__)
if msg:
print >> fd, msg
sys.exit(code)
def kill_watcher(sig):
try:
fp = open(mm_cfg.PIDFILE)
pidstr = fp.read()
fp.close()
pid = int(pidstr.strip())
except (IOError, ValueError), e:
# For i18n convenience
pidfile = mm_cfg.PIDFILE
print >> sys.stderr, _('PID unreadable in: %(pidfile)s')
print >> sys.stderr, e
print >> sys.stderr, _('Is qrunner even running?')
return
try:
os.kill(pid, sig)
except OSError, e:
if e.errno <> errno.ESRCH: raise
print >> sys.stderr, _('No child with pid: %(pid)s')
print >> sys.stderr, e
print >> sys.stderr, _('Stale pid file removed.')
os.unlink(mm_cfg.PIDFILE)
def get_lock_data():
# Return the hostname, pid, and tempfile
fp = open(LOCKFILE)
filename = os.path.split(fp.read().strip())[1]
fp.close()
parts = filename.split('.')
hostname = DOT.join(parts[1:-1])
pid = int(parts[-1])
return hostname, int(pid), filename
def qrunner_state():
# 1 if proc exists on host (but is it qrunner? ;)
# 0 if host matches but no proc
# hostname if hostname doesn't match
hostname, pid, tempfile = get_lock_data()
if hostname <> socket.gethostname():
return hostname
# Find out if the process exists by calling kill with a signal 0.
try:
os.kill(pid, 0)
except OSError, e:
if e.errno <> errno.ESRCH: raise
return 0
return 1
def acquire_lock_1(force):
# Be sure we can acquire the master qrunner lock. If not, it means some
# other master qrunner daemon is already going.
lock = LockFile.LockFile(LOCKFILE, LOCK_LIFETIME)
try:
lock.lock(0.1)
return lock
except LockFile.TimeOutError:
# If we're not forcing or the lock can't be determined to be stale.
if not force or qrunner_state():
raise
# Force removal of lock first
lock._disown()
hostname, pid, tempfile = get_lock_data()
os.unlink(LOCKFILE)
os.unlink(os.path.join(mm_cfg.LOCK_DIR, tempfile))
return acquire_lock_1(force=0)
def acquire_lock(force):
try:
lock = acquire_lock_1(force)
return lock
except LockFile.TimeOutError:
status = qrunner_state()
if status == 1:
# host matches and proc exists
print >> sys.stderr, _("""\
The master qrunner lock could not be acquired because it appears as if another
master qrunner is already running.
""")
elif status == 0:
# host matches but no proc
print >> sys.stderr, _("""\
The master qrunner lock could not be acquired. It appears as though there is
a stale master qrunner lock. Try re-running mailmanctl with the -s flag.
""")
else:
# host doesn't even match
print >> sys.stderr, _("""\
The master qrunner lock could not be acquired, because it appears as if some
process on some other host may have acquired it. We can't test for stale
locks across host boundaries, so you'll have to do this manually. Or, if you
know the lock is stale, re-run mailmanctl with the -s flag.
Lock file: %(LOCKFILE)s
Lock host: %(status)s
Exiting.""")
def start_runner(qrname, slice, count):
pid = os.fork()
if pid:
# parent
return pid
# child
#
# Craft the command line arguments for the exec() call.
rswitch = '--runner=%s:%d:%d' % (qrname, slice, count)
exe = os.path.join(mm_cfg.BIN_DIR, 'qrunner')
# mm_cfg.PYTHON, which is the absolute path to the Python interpreter,
# must be given as argv[0] due to Python's library search algorithm.
os.execl(mm_cfg.PYTHON, mm_cfg.PYTHON, exe, rswitch, '-s')
# Should never get here
raise RuntimeError, 'os.execl() failed'
def start_all_runners():
kids = {}
for qrname, count in mm_cfg.QRUNNERS:
for slice in range(count):
# queue runner name, slice, numslices, restart count
info = (qrname, slice, count, 0)
pid = start_runner(qrname, slice, count)
kids[pid] = info
return kids
def check_for_site_list():
sitelistname = mm_cfg.MAILMAN_SITE_LIST
try:
sitelist = MailList(sitelistname, lock=0)
except Errors.MMUnknownListError:
print >> sys.stderr, _('Site list is missing: %(sitelistname)s')
syslog('error', 'Site list is missing: %s', mm_cfg.MAILMAN_SITE_LIST)
sys.exit(1)
def check_privs():
# If we're running as root (uid == 0), coerce the uid and gid to that
# which Mailman was configured for, and refuse to run if we didn't coerce
# the uid/gid.
gid = grp.getgrnam(mm_cfg.MAILMAN_GROUP)[2]
uid = pwd.getpwnam(mm_cfg.MAILMAN_USER)[2]
myuid = os.geteuid()
if myuid == 0:
# Set the process's supplimental groups.
groups = [x[2] for x in grp.getgrall() if mm_cfg.MAILMAN_USER in x[3]]
groups.append(gid)
try:
os.setgroups(groups)
except AttributeError:
# Python 2.1 doesn't have setgroups
syslog('error', 'Warning: unable to setgroups(%s)' % groups)
os.setgid(gid)
os.setuid(uid)
elif myuid <> uid:
name = mm_cfg.MAILMAN_USER
usage(1, _(
'Run this program as root or as the %(name)s user, or use -u.'))
def main():
global quiet
try:
opts, args = getopt.getopt(sys.argv[1:], 'hnusq',
['help', 'no-start', 'run-as-user',
'stale-lock-cleanup', 'quiet'])
except getopt.error, msg:
usage(1, msg)
restart = 1
checkprivs = 1
force = 0
quiet = 0
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-n', '--no-restart'):
restart = 0
elif opt in ('-u', '--run-as-user'):
checkprivs = 0
elif opt in ('-s', '--stale-lock-cleanup'):
force = 1
elif opt in ('-q', '--quiet'):
quiet = 1
if len(args) < 1:
usage(1, _('No command given.'))
elif len(args) > 1:
command = COMMASPACE.join(args)
usage(1, _('Bad command: %(command)s'))
if checkprivs:
check_privs()
else:
print _('Warning! You may encounter permission problems.')
# Handle the commands
command = args[0].lower()
if command == 'stop':
# Sent the master qrunner process a SIGINT, which is equivalent to
# giving cron/qrunner a ctrl-c or KeyboardInterrupt. This will
# effectively shut everything down.
if not quiet:
print _("Shutting down Mailman's master qrunner")
kill_watcher(signal.SIGTERM)
elif command == 'restart':
# Sent the master qrunner process a SIGHUP. This will cause the
# master qrunner to kill and restart all the worker qrunners, and to
# close and re-open its log files.
if not quiet:
print _("Restarting Mailman's master qrunner")
kill_watcher(signal.SIGINT)
elif command == 'reopen':
if not quiet:
print _('Re-opening all log files')
kill_watcher(signal.SIGHUP)
elif command == 'start':
# First, complain loudly if there's no site list.
check_for_site_list()
# Here's the scoop on the processes we're about to create. We'll need
# one for each qrunner, and one for a master child process watcher /
# lock refresher process.
#
# The child watcher process simply waits on the pids of the children
# qrunners. Unless explicitly disabled by a mailmanctl switch (or the
# children are killed with SIGTERM instead of SIGINT), the watcher
# will automatically restart any child process that exits. This
# allows us to be more robust, and also to implement restart by simply
# SIGINT'ing the qrunner children, and letting the watcher restart
# them.
#
# Under normal operation, we have a child per queue. This lets us get
# the most out of the available resources, since a qrunner with no
# files in its queue directory is pretty cheap, but having a separate
# runner process per queue allows for a very responsive system. Some
# people want a more traditional (i.e. MM2.0.x) cron-invoked qrunner.
# No problem, but using mailmanctl isn't the answer. So while
# mailmanctl hard codes some things, others, such as the number of
# qrunners per queue, is configurable in mm_cfg.py.
#
# First, acquire the master mailmanctl lock
lock = acquire_lock(force)
if not lock:
return
# Daemon process startup according to Stevens, Advanced Programming in
# the UNIX Environment, Chapter 13.
pid = os.fork()
if pid:
# parent
if not quiet:
print _("Starting Mailman's master qrunner.")
# Give up the lock "ownership". This just means the foreground
# process won't close/unlock the lock when it finalizes this lock
# instance. We'll let the mater watcher subproc own the lock.
lock._transfer_to(pid)
return
# child
lock._take_possession()
# First, save our pid in a file for "mailmanctl stop" rendezvous. We
# want the perms on the .pid file to be rw-rw----
omask = os.umask(6)
try:
fp = open(mm_cfg.PIDFILE, 'w')
print >> fp, os.getpid()
fp.close()
finally:
os.umask(omask)
# Create a new session and become the session leader, but since we
# won't be opening any terminal devices, don't do the ultra-paranoid
# suggestion of doing a second fork after the setsid() call.
os.setsid()
# Be sure to close any open std{in,out,err}
devnull = os.open('/dev/null', 0)
os.dup2(devnull, 0)
os.dup2(devnull, 1)
os.dup2(devnull, 2)
# Instead of cd'ing to root, cd to the Mailman installation home
os.chdir(mm_cfg.PREFIX)
# Set our file mode creation umask
os.umask(007)
# I don't think we have any unneeded file descriptors.
#
# Now start all the qrunners. This returns a dictionary where the
# keys are qrunner pids and the values are tuples of the following
# form: (qrname, slice, count). This does its own fork and exec, and
# sets up its own signal handlers.
kids = start_all_runners()
# Set up a SIGALRM handler to refresh the lock once per day. The lock
# lifetime is 1day+6hours so this should be plenty.
def sigalrm_handler(signum, frame, lock=lock):
lock.refresh()
signal.alarm(mm_cfg.days(1))
signal.signal(signal.SIGALRM, sigalrm_handler)
signal.alarm(mm_cfg.days(1))
# Set up a SIGHUP handler so that if we get one, we'll pass it along
# to all the qrunner children. This will tell them to close and
# reopen their log files
def sighup_handler(signum, frame, kids=kids):
# Closing our syslog will cause it to be re-opened at the next log
# print output.
syslog.close()
for pid in kids.keys():
os.kill(pid, signal.SIGHUP)
# And just to tweak things...
syslog('qrunner',
'Master watcher caught SIGHUP. Re-opening log files.')
signal.signal(signal.SIGHUP, sighup_handler)
# We also need to install a SIGTERM handler because that's what init
# will kill this process with when changing run levels.
def sigterm_handler(signum, frame, kids=kids):
for pid in kids.keys():
try:
os.kill(pid, signal.SIGTERM)
except OSError, e:
if e.errno <> errno.ESRCH: raise
syslog('qrunner', 'Master watcher caught SIGTERM. Exiting.')
signal.signal(signal.SIGTERM, sigterm_handler)
# Finally, we need a SIGINT handler which will cause the sub-qrunners
# to exit, but the master will restart SIGINT'd sub-processes unless
# the -n flag was given.
def sigint_handler(signum, frame, kids=kids):
for pid in kids.keys():
os.kill(pid, signal.SIGINT)
syslog('qrunner', 'Master watcher caught SIGINT. Restarting.')
signal.signal(signal.SIGINT, sigint_handler)
# Now we're ready to simply do our wait/restart loop. This is the
# master qrunner watcher.
try:
while 1:
try:
pid, status = os.wait()
except OSError, e:
# No children? We're done
if e.errno == errno.ECHILD:
break
# If the system call got interrupted, just restart it.
elif e.errno <> errno.EINTR:
raise
continue
killsig = exitstatus = None
if os.WIFSIGNALED(status):
killsig = os.WTERMSIG(status)
if os.WIFEXITED(status):
exitstatus = os.WEXITSTATUS(status)
# We'll restart the process unless we were given the
# "no-restart" switch, or if the process was SIGTERM'd or
# exitted with a SIGTERM exit status. This lets us better
# handle runaway restarts (say, if the subproc had a syntax
# error!)
restarting = ''
if restart:
if (exitstatus == None and killsig <> signal.SIGTERM) or \
(killsig == None and exitstatus <> signal.SIGTERM):
# Then
restarting = '[restarting]'
qrname, slice, count, restarts = kids[pid]
del kids[pid]
syslog('qrunner', """\
Master qrunner detected subprocess exit
(pid: %d, sig: %s, sts: %s, class: %s, slice: %d/%d) %s""",
pid, killsig, exitstatus, qrname,
slice+1, count, restarting)
# See if we've reached the maximum number of allowable restarts
if exitstatus <> signal.SIGINT:
restarts += 1
if restarts > MAX_RESTARTS:
syslog('qrunner', """\
Qrunner %s reached maximum restart limit of %d, not restarting.""",
qrname, MAX_RESTARTS)
restarting = ''
# Now perhaps restart the process unless it exited with a
# SIGTERM or we aren't restarting.
if restarting:
newpid = start_runner(qrname, slice, count)
kids[newpid] = (qrname, slice, count, restarts)
finally:
# Should we leave the main loop for any reason, we want to be sure
# all of our children are exited cleanly. Send SIGTERMs to all
# the child processes and wait for them all to exit.
for pid in kids.keys():
try:
os.kill(pid, signal.SIGTERM)
except OSError, e:
if e.errno == errno.ESRCH:
# The child has already exited
syslog('qrunner', 'ESRCH on pid: %d', pid)
del kids[pid]
# Wait for all the children to go away
while 1:
try:
pid, status = os.wait()
except OSError, e:
if e.errno == errno.ECHILD:
break
elif e.errno <> errno.EINTR:
raise
continue
# Finally, give up the lock
lock.unlock(unconditionally=1)
os._exit(0)
if __name__ == '__main__':
main()
|