django-orchestra-test/orchestra/contrib/mailboxes/backends.py

575 lines
23 KiB
Python

import logging
import os
import re
import textwrap
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from . import settings
from .models import Address
logger = logging.getLogger(__name__)
class SieveFilteringMixin(object):
def generate_filter(self, mailbox, context):
name, content = mailbox.get_filtering()
for box in re.findall(r'fileinto\s+"([^"]+)"', content):
# create mailboxes if fileinfo is provided witout ':create' option
context['box'] = box
self.append(textwrap.dedent("""
# Create %(box)s mailbox
mkdir -p %(maildir)s/.%(box)s
chown %(user)s:%(group)s %(maildir)s/.%(box)s
if [[ ! $(grep '%(box)s' %(maildir)s/subscriptions) ]]; then
echo '%(box)s' >> %(maildir)s/subscriptions
fi
""") % context
)
context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context
context['filtering_cpath'] = re.sub(r'\.sieve$', '.svbin', context['filtering_path'])
if content:
context['filtering'] = ('# %(banner)s\n' + content) % context
self.append(textwrap.dedent("""\
# Create and compile orchestra sieve filtering
mkdir -p $(dirname '%(filtering_path)s')
cat << 'EOF' > %(filtering_path)s
%(filtering)s
EOF
sievec %(filtering_path)s
chown %(user)s:%(group)s %(filtering_path)s
chown %(user)s:%(group)s %(filtering_cpath)s
""") % context
)
else:
self.append("echo '' > %(filtering_path)s" % context)
class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
"""
Assumes that all system users on this servers all mail accounts.
If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes.
Supports quota allocation via <tt>resources.disk.allocated</tt>.
"""
SHELL = '/dev/null'
verbose_name = _("UNIX maildir user")
model = 'mailboxes.Mailbox'
doc_settings = (settings,
('MAILBOXES_USE_ACCOUNT_AS_GROUP',)
)
def save(self, mailbox):
context = self.get_context(mailbox)
self.append(textwrap.dedent("""
# Update/create %(user)s user state
if [[ $( id %(user)s ) ]]; then
old_password=$(getent shadow %(user)s | cut -d':' -f2)
usermod %(user)s \\
--shell %(initial_shell)s \\
--password '%(password)s'
if [[ "$old_password" != '%(password)s' ]]; then
# Postfix SASL caches passwords
RESTART_POSTFIX=1
fi
else
useradd %(user)s \\
--home %(home)s \\
--password '%(password)s'
fi
mkdir -p %(home)s
chmod 751 %(home)s
chown %(user)s:%(group)s %(home)s""") % context
)
if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'):
self.set_quota(mailbox, context)
self.generate_filter(mailbox, context)
def set_quota(self, mailbox, context):
allocated = mailbox.resources.disk.allocated
scale = mailbox.resources.disk.resource.get_scale()
context['quota'] = allocated * scale
#unit_to_bytes(mailbox.resources.disk.unit)
self.append(textwrap.dedent("""
# Set Maildir quota for %(user)s
mkdir -p %(maildir)s
chown %(user)s:%(group)s %(maildir)s
if [[ ! -f %(maildir)s/maildirsize ]]; then
echo "%(quota)iS" > %(maildir)s/maildirsize
chown %(user)s:%(group)s %(maildir)s/maildirsize
else
sed -i '1s/.*/%(quota)iS/' %(maildir)s/maildirsize
fi""") % context
)
def delete(self, mailbox):
context = self.get_context(mailbox)
self.append('mv %(home)s %(home)s.deleted || exit_code=$?' % context)
self.append(textwrap.dedent("""
nohup bash -c '{ sleep 2 && killall -u %(user)s -s KILL; }' &> /dev/null &
killall -u %(user)s || true
# Fucking postfix SASL caches credentials
userdel %(user)s || true && RESTART_POSTFIX=1
groupdel %(user)s || true""") % context
)
def commit(self):
self.append('[[ $RESTART_POSTFIX -eq 1 ]] && service postfix restart')
super(UNIXUserMaildirBackend, self).commit()
def get_context(self, mailbox):
account_as_group = settings.MAILBOXES_USE_ACCOUNT_AS_GROUP
context = {
'user': mailbox.name,
'group': mailbox.account.username if account_as_group else mailbox.name,
'name': mailbox.name,
'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
'home': mailbox.get_home(),
'maildir': os.path.join(mailbox.get_home(), 'Maildir'),
'initial_shell': self.SHELL,
'banner': self.get_banner(),
}
return context
class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceController):
"""
WARNING: This backends is not fully implemented
"""
DEFAULT_GROUP = 'postfix'
verbose_name = _("Dovecot-Postfix virtualuser")
model = 'mailboxes.Mailbox'
# TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data
def set_user(self, context):
self.append(textwrap.dedent("""
if [[ $( grep '^%(user)s:' %(passwd_path)s ) ]]; then
sed -i 's#^%(user)s:.*#%(passwd)s#' %(passwd_path)s
else
echo '%(passwd)s' >> %(passwd_path)s
fi""") % context
)
self.append("mkdir -p %(home)s" % context)
self.append("chown %(uid)s:%(gid)s %(home)s" % context)
def set_mailbox(self, context):
self.append(textwrap.dedent("""
if [[ ! $(grep '^%(user)s@%(mailbox_domain)s\s' %(virtual_mailbox_maps)s) ]]; then
echo "%(user)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s
UPDATED_VIRTUAL_MAILBOX_MAPS=1
fi""") % context
)
def save(self, mailbox):
context = self.get_context(mailbox)
self.set_user(context)
self.set_mailbox(context)
self.generate_filter(mailbox, context)
def delete(self, mailbox):
context = self.get_context(mailbox)
self.append(textwrap.dedent("""
nohup bash -c 'sleep 2 && killall -u %(uid)s -s KILL' &> /dev/null &
killall -u %(uid)s || true
sed -i '/^%(user)s:.*/d' %(passwd_path)s
sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s
UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context
)
if context['deleted_home']:
self.append("mv %(home)s %(deleted_home)s || exit_code=$?" % context)
else:
self.append("rm -fr %(home)s" % context)
def get_extra_fields(self, mailbox, context):
context['quota'] = self.get_quota(mailbox)
return 'userdb_mail=maildir:~/Maildir {quota}'.format(**context)
def get_quota(self, mailbox):
try:
quota = mailbox.resources.disk.allocated
except (AttributeError, ObjectDoesNotExist):
return ''
unit = mailbox.resources.disk.unit[0].upper()
return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit)
def commit(self):
context = {
'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH
}
self.append(textwrap.dedent("""
[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && {
postmap %(virtual_mailbox_maps)s
}""") % context
)
def get_context(self, mailbox):
context = {
'name': mailbox.name,
'user': mailbox.name,
'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
'uid': 10000 + mailbox.pk,
'gid': 10000 + mailbox.pk,
'group': self.DEFAULT_GROUP,
'quota': self.get_quota(mailbox),
'passwd_path': settings.MAILBOXES_PASSWD_PATH,
'home': mailbox.get_home(),
'banner': self.get_banner(),
'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH,
'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
}
context['extra_fields'] = self.get_extra_fields(mailbox, context)
context.update({
'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context),
'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context,
})
return context
class PostfixAddressVirtualDomainBackend(ServiceController):
"""
Secondary SMTP server without mailboxes in it, only syncs virtual domains.
"""
verbose_name = _("Postfix address virtdomain-only")
model = 'mailboxes.Address'
related_models = (
('mailboxes.Mailbox', 'addresses'),
)
doc_settings = (settings,
('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH')
)
def is_local_domain(self, domain):
""" whether or not domain MX points to this server """
return domain.has_default_mx()
def include_virtual_alias_domain(self, context):
domain = context['domain']
if domain.name != context['local_domain'] and self.is_local_domain(domain):
self.append(textwrap.dedent("""
# %(domain)s is a virtual domain belonging to this server
if [[ ! $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]]; then
echo '%(domain)s' >> %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
fi""") % context
)
def is_last_domain(self, domain):
return not Address.objects.filter(domain=domain).exists()
def exclude_virtual_alias_domain(self, context):
domain = context['domain']
if self.is_last_domain(domain):
self.append(textwrap.dedent("""
# Delete %(domain)s virtual domain
if [[ $(grep '^%(domain)s\s*$' %(virtual_alias_domains)s) ]]; then
sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
fi""") % context
)
def save(self, address):
context = self.get_context(address)
self.include_virtual_alias_domain(context)
return context
def delete(self, address):
context = self.get_context(address)
self.exclude_virtual_alias_domain(context)
return context
def commit(self):
context = self.get_context_files()
self.append(textwrap.dedent("""
[[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && {
service postfix reload
}
exit $exit_code
""") % context
)
def get_context_files(self):
return {
'virtual_alias_domains': settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH,
'virtual_alias_maps': settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH
}
def get_context(self, address):
context = self.get_context_files()
context.update({
'domain': address.domain,
'email': address.email,
'local_domain': settings.MAILBOXES_LOCAL_DOMAIN,
})
return context
class PostfixAddressBackend(PostfixAddressVirtualDomainBackend):
"""
Addresses based on Postfix virtual alias domains, includes <tt>PostfixAddressVirtualDomainBackend</tt>.
"""
verbose_name = _("Postfix address")
doc_settings = (settings, (
'MAILBOXES_LOCAL_DOMAIN',
'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH',
'MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH'
))
def update_virtual_alias_maps(self, address, context):
destination = address.destination
if destination:
context['destination'] = destination
self.append(textwrap.dedent("""
# Set virtual alias entry for %(email)s
LINE='%(email)s\t%(destination)s'
if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
# Add new line
echo "${LINE}" >> %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
else
# Update existing line, if needed
if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then
sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
fi
fi""") % context)
else:
logger.warning("Address %i is empty" % address.pk)
self.exclude_virtual_alias_maps(context)
# Virtual mailbox stuff
# destination = []
# for mailbox in address.get_mailboxes():
# context['mailbox'] = mailbox
# destination.append("%(mailbox)s@%(local_domain)s" % context)
# for forward in address.forward:
# if '@' in forward:
# destination.append(forward)
def exclude_virtual_alias_maps(self, context):
self.append(textwrap.dedent("""
# Remove %(email)s virtual alias entry
if [[ $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
fi""") % context
)
def save(self, address):
context = super(PostfixAddressBackend, self).save(address)
self.update_virtual_alias_maps(address, context)
def delete(self, address):
context = super(PostfixAddressBackend, self).save(address)
self.exclude_virtual_alias_maps(context)
def commit(self):
context = self.get_context_files()
self.append(textwrap.dedent("""
# Apply changes if needed
[[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && {
service postfix reload
}
[[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && {
postmap %(virtual_alias_maps)s
}
exit $exit_code
""") % context
)
class AutoresponseBackend(ServiceController):
"""
WARNING: not implemented
"""
verbose_name = _("Mail autoresponse")
model = 'mailboxes.Autoresponse'
class DovecotMaildirDisk(ServiceMonitor):
"""
Maildir disk usage based on Dovecot maildirsize file
http://wiki2.dovecot.org/Quota/Maildir
"""
model = 'mailboxes.Mailbox'
resource = ServiceMonitor.DISK
verbose_name = _("Dovecot Maildir size")
doc_settings = (settings,
('MAILBOXES_MAILDIRSIZE_PATH',)
)
def prepare(self):
super(DovecotMaildirDisk, self).prepare()
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z")
self.append(textwrap.dedent("""\
function monitor () {
awk 'BEGIN { size = 0 } NR > 1 { size += $1 } END { print size }' $1 || echo 0
}"""))
def monitor(self, mailbox):
context = self.get_context(mailbox)
self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context)
def get_context(self, mailbox):
context = {
'home': mailbox.get_home(),
'object_id': mailbox.pk
}
context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context
return context
class PostfixMailscannerTraffic(ServiceMonitor):
"""
A high-performance log parser.
Reads the mail.log file only once, for all users.
"""
model = 'mailboxes.Mailbox'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Postfix-Mailscanner traffic")
script_executable = '/usr/bin/python'
doc_settings = (settings,
('MAILBOXES_MAIL_LOG_PATH',)
)
def prepare(self):
mail_log = settings.MAILBOXES_MAIL_LOG_PATH
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'mail_logs': str((mail_log, mail_log+'.1')),
}
self.append(textwrap.dedent("""\
import re
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
# Converts orchestra's UTC dates to local timezone
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
maillogs = {mail_logs}
end_datetime = to_local_timezone('{current_date}')
end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
months = dict((m, '%02d' % n) for n, m in enumerate(months, 1))
def inside_period(month, day, time, ini_date):
global months
global end_datetime
# Mar 9 17:13:22
month = months[month]
year = end_datetime.year
if month == '12' and end_datetime.month == 1:
year = year+1
if len(day) == 1:
day = '0' + day
date = str(year) + month + day
date += time.replace(':', '')
return ini_date < int(date) < end_date
users = {{}}
delivers = {{}}
reverse = {{}}
def prepare(object_id, mailbox, ini_date):
global users
global delivers
global reverse
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[mailbox] = (ini_date, object_id)
delivers[mailbox] = set()
reverse[mailbox] = set()
def monitor(users, delivers, reverse, maillogs):
targets = {{}}
counter = {{}}
user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
for maillog in maillogs:
try:
with open(maillog, 'r') as maillog:
for line in maillog.readlines():
# Only search for Authenticated sendings
if '(Authenticated sender: ' in line:
username = user_regex.search(line).groups()[0]
try:
sender = users[username]
except KeyError:
continue
else:
month, day, time, __, proc, id = line.split()[:6]
if inside_period(month, day, time, sender[0]):
# Add new email
delivers[id[:-1]] = username
# Look for a MailScanner requeue ID
elif ' Requeue: ' in line:
id, __, req_id = line.split()[6:9]
id = id.split('.')[0]
try:
username = delivers[id]
except KeyError:
pass
else:
targets[req_id] = (username, 0)
reverse[username].add(req_id)
# Look for the mail size and count the number of recipients of each email
else:
try:
month, day, time, __, proc, req_id, __, msize = line.split()[:8]
except ValueError:
# not interested in this line
continue
if proc.startswith('postfix/'):
req_id = req_id[:-1]
if msize.startswith('size='):
try:
target = targets[req_id]
except KeyError:
pass
else:
targets[req_id] = (target[0], int(msize[5:-1]))
elif proc.startswith('postfix/smtp'):
try:
target = targets[req_id]
except KeyError:
pass
else:
if inside_period(month, day, time, users[target[0]][0]):
try:
counter[req_id] += 1
except KeyError:
counter[req_id] = 1
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
size = 0
for req_id in reverse[username]:
size += targets[req_id][1] * counter.get(req_id, 0)
print opts[1], size
""").format(**context)
)
def commit(self):
self.append('monitor(users, delivers, reverse, maillogs)')
def monitor(self, mailbox):
context = self.get_context(mailbox)
self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context)
def get_context(self, mailbox):
context = {
'mailbox': mailbox.name,
'object_id': mailbox.pk,
'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
return context