django-orchestra/orchestra/contrib/lists/backends.py

328 lines
12 KiB
Python
Raw Normal View History

2014-07-25 15:17:50 +00:00
import textwrap
from django.utils.translation import gettext_lazy as _
2014-05-08 16:59:35 +00:00
2015-04-05 18:02:36 +00:00
from orchestra.contrib.orchestration import ServiceController, replace
2015-04-05 10:46:24 +00:00
from orchestra.contrib.resources import ServiceMonitor
2014-05-08 16:59:35 +00:00
2014-07-11 14:48:46 +00:00
from . import settings
2014-10-09 17:04:12 +00:00
from .models import List
2014-07-11 14:48:46 +00:00
2014-05-08 16:59:35 +00:00
2016-03-08 10:16:49 +00:00
class MailmanVirtualDomainController(ServiceController):
2015-05-05 19:42:55 +00:00
"""
Only syncs virtualdomains used on mailman addresses
"""
verbose_name = _("Mailman virtdomain-only")
model = 'lists.List'
doc_settings = (settings,
('LISTS_VIRTUAL_ALIAS_DOMAINS_PATH',)
)
2015-07-08 10:21:19 +00:00
def is_hosted_domain(self, domain):
2015-05-05 19:42:55 +00:00
""" whether or not domain MX points to this server """
return domain.has_default_mx()
def include_virtual_alias_domain(self, context):
domain = context['address_domain']
2015-07-08 10:21:19 +00:00
if domain and self.is_hosted_domain(domain):
2015-05-05 19:42:55 +00:00
self.append(textwrap.dedent("""
# Add virtual domain %(address_domain)s
2015-05-05 19:42:55 +00:00
[[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || {
echo '%(address_domain)s' >> %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
2015-07-08 10:21:19 +00:00
}""") % context
2015-05-05 19:42:55 +00:00
)
def is_last_domain(self, domain):
return not List.objects.filter(address_domain=domain).exists()
def exclude_virtual_alias_domain(self, context):
domain = context['address_domain']
if domain and self.is_last_domain(domain):
self.append(textwrap.dedent("""
# Remove %(address_domain)s from virtual domains
sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s\
""") % context
)
2015-05-05 19:42:55 +00:00
def save(self, mail_list):
context = self.get_context(mail_list)
self.include_virtual_alias_domain(context)
2015-05-05 19:42:55 +00:00
def delete(self, mail_list):
context = self.get_context(mail_list)
self.exclude_virtual_alias_domain(context)
2015-05-05 19:42:55 +00:00
def commit(self):
context = self.get_context_files()
2016-03-08 10:16:49 +00:00
super(MailmanVirtualDomainController, self).commit()
2015-05-05 19:42:55 +00:00
def get_context_files(self):
return {
'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH,
}
def get_context(self, mail_list):
context = self.get_context_files()
context.update({
'address_domain': mail_list.address_domain,
})
return replace(context, "'", '"')
2016-03-08 10:16:49 +00:00
class MailmanController(MailmanVirtualDomainController):
2015-04-23 19:46:23 +00:00
"""
2015-04-24 11:39:20 +00:00
Mailman 2 backend based on <tt>newlist</tt>, it handles custom domains.
2016-03-08 10:16:49 +00:00
Includes <tt>MailmanVirtualDomainController</tt>
2015-04-23 19:46:23 +00:00
"""
2014-05-08 16:59:35 +00:00
verbose_name = "Mailman"
2015-05-12 12:38:40 +00:00
address_suffixes = [
2014-10-15 12:47:28 +00:00
'',
'-admin',
'-bounces',
'-confirm',
'-join',
'-leave',
'-owner',
'-request',
'-subscribe',
'-unsubscribe'
]
2015-04-24 11:39:20 +00:00
doc_settings = (settings, (
'LISTS_VIRTUAL_ALIAS_PATH',
'LISTS_VIRTUAL_ALIAS_DOMAINS_PATH',
'LISTS_DEFAULT_DOMAIN',
'LISTS_MAILMAN_ROOT_DIR'
))
2014-10-09 17:04:12 +00:00
def get_virtual_aliases(self, context):
2015-03-18 21:51:12 +00:00
aliases = ['# %(banner)s' % context]
2015-05-12 12:38:40 +00:00
for suffix in self.address_suffixes:
context['suffix'] = suffix
# Because mailman doesn't properly handle lists aliases we need virtual aliases
if context['address_name'] != context['name'] or context['address_domain'] != settings.LISTS_DEFAULT_DOMAIN:
aliases.append("%(address_name)s%(suffix)s@%(domain)s\t%(name)s%(suffix)s@grups.pangea.org" % context)
2014-10-09 17:04:12 +00:00
return '\n'.join(aliases)
2014-10-09 17:04:12 +00:00
def save(self, mail_list):
context = self.get_context(mail_list)
2014-10-15 12:47:28 +00:00
# Create list
cmd = "/opt/mailman/venv/bin/python /usr/local/admin/orchestra_mailman3/save.py %(name)s %(admin)s %(address_name)s@%(domain)s" % context
if not mail_list.active:
cmd += ' --inactive'
self.append(cmd)
# Custom domain
if mail_list.address:
context.update({
'aliases': self.get_virtual_aliases(context),
'num_entries': 2 if context['address_name'] != context['name'] else 1,
})
self.append(textwrap.dedent("""\
# Create list alias for custom domain
aliases='%(aliases)s'
if ! grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then
echo "${aliases}" >> %(virtual_alias)s
UPDATED_VIRTUAL_ALIAS=1
else
if grep -E '(%(address_name)s|%(name)s)@(%(address_domain)s|grups.pangea.org)' %(virtual_alias)s > /dev/null ; then
sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\
-e '/# .*%(name)s$/d' %(virtual_alias)s
echo "${aliases}" >> %(virtual_alias)s
UPDATED_VIRTUAL_ALIAS=1
fi
fi """) % context
)
else:
self.append(textwrap.dedent("""\
# Cleanup possible ex-custom domain
if grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then
#sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s
sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\
-e '/# .*%(name)s$/d' %(virtual_alias)s
fi""") % context
)
2014-10-09 17:04:12 +00:00
def delete(self, mail_list):
2014-10-15 12:47:28 +00:00
context = self.get_context(mail_list)
# Custom domain delete
self.append(textwrap.dedent("""\
# Cleanup possible ex-custom domain
if grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then
sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\
-e '/# .*%(name)s$/d' %(virtual_alias)s
fi""") % context
)
# Delete list
cmd = "/opt/mailman/venv/bin/python /usr/local/admin/orchestra_mailman3/delete.py %(name)s" % context
self.append(cmd)
2014-10-09 17:04:12 +00:00
def commit(self):
pass
2014-10-09 17:04:12 +00:00
def get_context_files(self):
return {
'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH,
2014-10-15 12:47:28 +00:00
'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH,
2014-10-09 17:04:12 +00:00
}
2014-10-27 14:31:04 +00:00
def get_banner(self, mail_list):
2016-03-08 10:16:49 +00:00
banner = super(MailmanController, self).get_banner()
2014-10-27 14:31:04 +00:00
return '%s %s' % (banner, mail_list.name)
2014-10-09 17:04:12 +00:00
def get_context(self, mail_list):
context = self.get_context_files()
context.update({
2014-10-27 15:15:22 +00:00
'banner': self.get_banner(mail_list),
2014-10-09 17:04:12 +00:00
'name': mail_list.name,
'domain': mail_list.address_domain or settings.LISTS_DEFAULT_DOMAIN,
2014-10-23 21:25:44 +00:00
'address_name': mail_list.get_address_name(),
2014-10-09 17:04:12 +00:00
'address_domain': mail_list.address_domain,
2015-05-12 12:38:40 +00:00
'suffixes_regex': '\|'.join(self.address_suffixes),
2014-10-09 17:04:12 +00:00
'admin': mail_list.admin_email,
2015-04-16 13:15:21 +00:00
'mailman_root': settings.LISTS_MAILMAN_ROOT_DIR,
2014-10-09 17:04:12 +00:00
})
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2014-07-09 16:17:43 +00:00
class MailmanTraffic(ServiceMonitor):
2015-04-23 19:46:23 +00:00
"""
2015-04-24 11:39:20 +00:00
Parses mailman log file looking for email size and multiples it by <tt>list_members</tt> count.
2015-04-23 19:46:23 +00:00
"""
model = 'lists.List'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Mailman traffic")
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
2015-04-24 11:39:20 +00:00
doc_settings = (settings,
('LISTS_MAILMAN_POST_LOG_PATH',)
)
def prepare(self):
postlog = settings.LISTS_MAILMAN_POST_LOG_PATH
context = {
'postlogs': str((postlog, postlog+'.1')),
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
}
self.append(textwrap.dedent("""\
import re
import subprocess
import sys
from datetime import datetime
from dateutil import tz
2015-04-23 19:46:23 +00:00
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
2015-04-23 19:46:23 +00:00
postlogs = {postlogs}
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
lists = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
2016-07-15 08:41:38 +00:00
mailman_addr = re.compile(r'.*-(admin|bounces|confirm|join|leave|owner|request|subscribe|unsubscribe)@.*|mailman@.*')
2015-04-23 19:46:23 +00:00
def prepare(object_id, list_name, ini_date):
global lists
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
lists[list_name] = [ini_date, object_id, 0]
2015-04-23 19:46:23 +00:00
def monitor(lists, end_date, months, postlogs):
for postlog in postlogs:
try:
with open(postlog, 'r') as postlog:
for line in postlog.readlines():
2016-05-26 08:15:34 +00:00
line = line.split()
if len(line) < 11:
continue
month, day, time, year, __, __, __, list_name, __, addr, size = line[:11]
try:
list = lists[list_name]
except KeyError:
continue
else:
# discard mailman messages because of inconsistent POST logging
if mailman_addr.match(addr):
continue
date = year + months[month] + day + time.replace(':', '')
if list[0] < int(date) < end_date:
size = size[5:-1]
try:
list[2] += int(size)
except ValueError:
# anonymized post
pass
except IOError as e:
sys.stderr.write(str(e)+'\\n')
2015-04-02 16:14:55 +00:00
for list_name, opts in lists.items():
__, object_id, size = opts
if size:
cmd = ' '.join(('list_members', list_name, '| wc -l'))
ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
subscribers = ps.communicate()[0].strip()
size *= int(subscribers)
2016-07-15 08:41:38 +00:00
sys.stderr.write("%s %s*%s traffic*subscribers\\n" % (object_id, size, subscribers))
print object_id, size
""").format(**context)
2015-03-02 10:37:25 +00:00
)
2014-07-11 14:48:46 +00:00
def monitor(self, user):
context = self.get_context(user)
self.append("prepare(%(object_id)s, '%(list_name)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(lists, end_date, months, postlogs)')
2014-07-11 14:48:46 +00:00
def get_context(self, mail_list):
2015-04-05 18:02:36 +00:00
context = {
2014-07-11 14:48:46 +00:00
'list_name': mail_list.name,
'object_id': mail_list.pk,
2015-03-02 10:37:25 +00:00
'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
2014-07-11 14:48:46 +00:00
}
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2014-10-27 13:29:02 +00:00
class MailmanSubscribers(ServiceMonitor):
2015-04-23 19:46:23 +00:00
"""
2015-04-24 11:39:20 +00:00
Monitors number of list subscribers via <tt>list_members</tt>
2015-04-23 19:46:23 +00:00
"""
model = 'lists.List'
verbose_name = _("Mailman subscribers")
delete_old_equal_values = True
def monitor(self, mail_list):
context = self.get_context(mail_list)
self.append('echo %(object_id)i $(list_members %(list_name)s | wc -l)' % context)
def get_context(self, mail_list):
2015-04-05 18:02:36 +00:00
context = {
'list_name': mail_list.name,
'object_id': mail_list.pk,
}
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')