django-orchestra-test/orchestra/contrib/mailboxes/backends.py

import logging
import textwrap

from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext_lazy as _

from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.resources import ServiceMonitor
#from orchestra.utils.humanize import unit_to_bytes

from . import settings
from .models import Address

# TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
# TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
# TODO mount the filesystem with "nosuid" option


logger = logging.getLogger(__name__)


class UNIXUserMaildirBackend(ServiceController):
    """
    Assumes that all system users on this servers all mail accounts.
    If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes
    """
    verbose_name = _("UNIX maildir user")
    model = 'mailboxes.Mailbox'
    
    def save(self, mailbox):
        context = self.get_context(mailbox)
        self.append(textwrap.dedent("""
            if [[ $( id %(user)s ) ]]; then
               usermod  %(user)s --password '%(password)s' --shell %(initial_shell)s
            else
               useradd %(user)s --home %(home)s --password '%(password)s'
            fi
            mkdir -p %(home)s
            chmod 751 %(home)s
            chown %(user)s:%(group)s %(home)s""") % context
        )
        if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'):
            self.set_quota(mailbox, context)
    
    def set_quota(self, mailbox, context):
        context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale()
        #unit_to_bytes(mailbox.resources.disk.unit)
        self.append(textwrap.dedent("""
            mkdir -p %(home)s/Maildir
            chown %(user)s:%(group)s %(home)s/Maildir
            if [[ ! -f %(home)s/Maildir/maildirsize ]]; then
                echo "%(quota)iS" > %(home)s/Maildir/maildirsize
                chown %(user)s:%(group)s %(home)s/Maildir/maildirsize
            else
                sed -i '1s/.*/%(quota)iS/' %(home)s/Maildir/maildirsize
            fi""") % context
        )
    
    def delete(self, mailbox):
        context = self.get_context(mailbox)
        self.append('mv %(home)s %(home)s.deleted || exit_code=1' % context)
        self.append(textwrap.dedent("""
            { sleep 2 && killall -u %(user)s -s KILL; } &
            killall -u %(user)s || true
            userdel %(user)s || true
            groupdel %(user)s || true""") % context
        )
    
    def get_context(self, mailbox):
        context = {
            'user': mailbox.name,
            'group': mailbox.name,
            'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
            'home': mailbox.get_home(),
            'initial_shell': '/dev/null',
        }
        return replace(context, "'", '"')


class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
    """
    WARNING: This backends is not fully implemented
    """
    verbose_name = _("Dovecot-Postfix virtualuser")
    model = 'mailboxes.Mailbox'
    # TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data
    
    DEFAULT_GROUP = 'postfix'
    
    def set_user(self, context):
        self.append(textwrap.dedent("""
            if [[ $( grep "^%(user)s:" %(passwd_path)s ) ]]; then
               sed -i 's#^%(user)s:.*#%(passwd)s#' %(passwd_path)s
            else
               echo '%(passwd)s' >> %(passwd_path)s
            fi""") % context
        )
        self.append("mkdir -p %(home)s" % context)
        self.append("chown %(uid)s:%(gid)s %(home)s" % context)
    
    def set_mailbox(self, context):
        self.append(textwrap.dedent("""
            if [[ ! $(grep "^%(user)s@%(mailbox_domain)s\s" %(virtual_mailbox_maps)s) ]]; then
                echo "%(user)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s
                UPDATED_VIRTUAL_MAILBOX_MAPS=1
            fi""") % context
        )
    
    def generate_filter(self, mailbox, context):
        self.append("doveadm mailbox create -u %(user)s Spam" % context)
        context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context
        filtering = mailbox.get_filtering()
        if filtering:
            context['filtering'] = '# %(banner)s\n' + filtering
            self.append("mkdir -p $(dirname '%(filtering_path)s')" % context)
            self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
        else:
            self.append("rm -f %(filtering_path)s" % context)
    
    def save(self, mailbox):
        context = self.get_context(mailbox)
        self.set_user(context)
        self.set_mailbox(context)
        self.generate_filter(mailbox, context)
    
    def delete(self, mailbox):
        context = self.get_context(mailbox)
        self.append(textwrap.dedent("""\
            { sleep 2 && killall -u %(uid)s -s KILL; } &
            killall -u %(uid)s || true
            sed -i '/^%(user)s:.*/d' %(passwd_path)s
            sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s
            UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context
        )
        if context['deleted_home']:
            self.append("mv %(home)s %(deleted_home)s || exit_code=1" % context)
        else:
            self.append("rm -fr %(home)s" % context)
    
    def get_extra_fields(self, mailbox, context):
        context['quota'] = self.get_quota(mailbox)
        return 'userdb_mail=maildir:~/Maildir {quota}'.format(**context)
    
    def get_quota(self, mailbox):
        try:
            quota = mailbox.resources.disk.allocated
        except (AttributeError, ObjectDoesNotExist):
            return ''
        unit = mailbox.resources.disk.unit[0].upper()
        return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit)
    
    def commit(self):
        context = {
            'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH
        }
        self.append(textwrap.dedent("""\
            [[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && {
                postmap %(virtual_mailbox_maps)s
            }""") % context
        )
    
    def get_context(self, mailbox):
        context = {
            'name': mailbox.name,
            'user': mailbox.name,
            'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
            'uid': 10000 + mailbox.pk,
            'gid': 10000 + mailbox.pk,
            'group': self.DEFAULT_GROUP,
            'quota': self.get_quota(mailbox),
            'passwd_path': settings.MAILBOXES_PASSWD_PATH,
            'home': mailbox.get_home(),
            'banner': self.get_banner(),
            'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH,
            'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
        }
        context['extra_fields'] = self.get_extra_fields(mailbox, context)
        context.update({
            'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context),
            'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context,
        })
        return replace(context, "'", '"')


class PostfixAddressBackend(ServiceController):
    """
    Addresses based on Postfix virtual alias domains.
    <tt>MAILBOXES_LOCAL_DOMAIN = '%s'
    MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH = '%s'
    MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH = '%s'</tt>
    """
    format_docstring = (
        settings.MAILBOXES_LOCAL_DOMAIN,
        settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH,
        settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH,
    )
    verbose_name = _("Postfix address")
    model = 'mailboxes.Address'
    related_models = (
        ('mailboxes.Mailbox', 'addresses'),
    )
    
    def include_virtual_alias_domain(self, context):
        if context['domain'] != context['local_domain']:
            self.append(textwrap.dedent("""
                [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || {
                    echo '%(domain)s' >> %(virtual_alias_domains)s
                    UPDATED_VIRTUAL_ALIAS_DOMAINS=1
                }""") % context
            )
    
    def exclude_virtual_alias_domain(self, context):
        domain = context['domain']
        if not Address.objects.filter(domain=domain).exists():
            self.append("sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s" % context)
    
    def update_virtual_alias_maps(self, address, context):
        # Virtual mailbox stuff
#        destination = []
#        for mailbox in address.get_mailboxes():
#            context['mailbox'] = mailbox
#            destination.append("%(mailbox)s@%(local_domain)s" % context)
#        for forward in address.forward:
#            if '@' in forward:
#                destination.append(forward)
        destination = address.destination
        if destination:
            context['destination'] = destination
            self.append(textwrap.dedent("""
                LINE='%(email)s\t%(destination)s'
                if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
                   echo "${LINE}" >> %(virtual_alias_maps)s
                   UPDATED_VIRTUAL_ALIAS_MAPS=1
                else
                   if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then
                       sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s
                       UPDATED_VIRTUAL_ALIAS_MAPS=1
                   fi
                fi""") % context)
        else:
            logger.warning("Address %i is empty" % address.pk)
            self.append("sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s" % context)
            self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1')
    
    def exclude_virtual_alias_maps(self, context):
        self.append(textwrap.dedent("""
            if [[ $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
               sed -i '/^%(email)s\s.*$/d' %(virtual_alias_maps)s
               UPDATED_VIRTUAL_ALIAS_MAPS=1
            fi""") % context)
    
    def save(self, address):
        context = self.get_context(address)
        self.include_virtual_alias_domain(context)
        self.update_virtual_alias_maps(address, context)
    
    def delete(self, address):
        context = self.get_context(address)
        self.exclude_virtual_alias_domain(context)
        self.exclude_virtual_alias_maps(context)
    
    def commit(self):
        context = self.get_context_files()
        self.append(textwrap.dedent("""
            [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; }
            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload; }
            """) % context
        )
        self.append('exit 0')
    
    def get_context_files(self):
        return {
            'virtual_alias_domains': settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH,
            'virtual_alias_maps': settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH
        }
    
    def get_context(self, address):
        context = self.get_context_files()
        context.update({
            'domain': address.domain,
            'email': address.email,
            'local_domain': settings.MAILBOXES_LOCAL_DOMAIN,
        })
        return replace(context, "'", '"')


class AutoresponseBackend(ServiceController):
    """
    WARNING: not implemented
    """
    verbose_name = _("Mail autoresponse")
    model = 'mailboxes.Autoresponse'


class DovecotMaildirDisk(ServiceMonitor):
    """
    Maildir disk usage based on Dovecot maildirsize file
    http://wiki2.dovecot.org/Quota/Maildir
    
    MAILBOXES_MAILDIRSIZE_PATH = '%s'
    """
    format_docstring = (
        settings.MAILBOXES_MAILDIRSIZE_PATH,
    )
    model = 'mailboxes.Mailbox'
    resource = ServiceMonitor.DISK
    verbose_name = _("Dovecot Maildir size")
    
    def prepare(self):
        super(DovecotMaildirDisk, self).prepare()
        current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z")
        self.append(textwrap.dedent("""\
            function monitor () {
                awk 'BEGIN { size = 0 } NR > 1 { size += $1 } END { print size }' $1 || echo 0
            }"""))
    
    def monitor(self, mailbox):
        context = self.get_context(mailbox)
        self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context)
    
    def get_context(self, mailbox):
        context = {
            'home': mailbox.get_home(),
            'object_id': mailbox.pk
        }
        context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context
        return replace(context, "'", '"')


class PostfixMailscannerTraffic(ServiceMonitor):
    """
    A high-performance log parser.
    Reads the mail.log file only once, for all users.
    MAILBOXES_MAIL_LOG_PATH = '%s'
    """
    format_docstring = (
        settings.MAILBOXES_MAIL_LOG_PATH,
    )
    model = 'mailboxes.Mailbox'
    resource = ServiceMonitor.TRAFFIC
    verbose_name = _("Postfix-Mailscanner traffic")
    script_executable = '/usr/bin/python'
    
    def prepare(self):
        mail_log = settings.MAILBOXES_MAIL_LOG_PATH
        context = {
            'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
            'mail_logs': str((mail_log, mail_log+'.1')),
        }
        self.append(textwrap.dedent("""\
            import re
            import sys
            from datetime import datetime
            from dateutil import tz
            
            def to_local_timezone(date, tzlocal=tz.tzlocal()):
                # Converts orchestra's UTC dates to local timezone
                date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
                date = date.replace(tzinfo=tz.tzutc())
                date = date.astimezone(tzlocal)
                return date
            
            maillogs = {mail_logs}
            end_datetime = to_local_timezone('{current_date}')
            end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
            months = {{
                "Jan": "01",
                "Feb": "02",
                "Mar": "03",
                "Apr": "04",
                "May": "05",
                "Jun": "06",
                "Jul": "07",
                "Aug": "08",
                "Sep": "09",
                "Oct": "10",
                "Nov": "11",
                "Dec": "12",
            }}
            
            def inside_period(month, day, time, ini_date):
                global months
                global end_datetime
                # Mar  9 17:13:22
                month = months[month]
                year = end_datetime.year
                if month == '12' and end_datetime.month == 1:
                    year = year+1
                if len(day) == 1:
                    day = '0' + day
                date = str(year) + month + day
                date += time.replace(':', '')
                return ini_date < int(date) < end_date
            
            users = {{}}
            delivers = {{}}
            reverse = {{}}
            
            def prepare(object_id, mailbox, ini_date):
                global users
                global delivers
                global reverse
                ini_date = to_local_timezone(ini_date)
                ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
                users[mailbox] = (ini_date, object_id)
                delivers[mailbox] = set()
                reverse[mailbox] = set()
            
            def monitor(users, delivers, reverse, maillogs):
                targets = {{}}
                counter = {{}}
                user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
                for maillog in maillogs:
                    try:
                        with open(maillog, 'r') as maillog:
                            for line in maillog.readlines():
                                # Only search for Authenticated sendings
                                if '(Authenticated sender: ' in line:
                                    username = user_regex.search(line).groups()[0]
                                    try:
                                        sender = users[username]
                                    except KeyError:
                                        continue
                                    else:
                                        month, day, time, __, proc, id = line.split()[:6]
                                        if inside_period(month, day, time, sender[0]):
                                            # Add new email
                                            delivers[id[:-1]] = username
                                # Look for a MailScanner requeue ID
                                elif ' Requeue: ' in line:
                                    id, __, req_id = line.split()[6:9]
                                    id = id.split('.')[0]
                                    try:
                                        username = delivers[id]
                                    except KeyError:
                                        pass
                                    else:
                                        targets[req_id] = (username, 0)
                                        reverse[username].add(req_id)
                                # Look for the mail size and count the number of recipients of each email
                                else:
                                    try:
                                        month, day, time, __, proc, req_id, __, msize = line.split()[:8]
                                    except ValueError:
                                        # not interested in this line
                                        continue
                                    if proc.startswith('postfix/'):
                                        req_id = req_id[:-1]
                                        if msize.startswith('size='):
                                            try:
                                                target = targets[req_id]
                                            except KeyError:
                                                pass
                                            else:
                                                targets[req_id] = (target[0], int(msize[5:-1]))
                                        elif proc.startswith('postfix/smtp'):
                                            try:
                                                target = targets[req_id]
                                            except KeyError:
                                                pass
                                            else:
                                                if inside_period(month, day, time, users[target[0]][0]):
                                                    try:
                                                        counter[req_id] += 1
                                                    except KeyError:
                                                        counter[req_id] = 1
                    except IOError as e:
                        sys.stderr.write(e)
                    
                for username, opts in users.iteritems():
                    size = 0
                    for req_id in reverse[username]:
                        size += targets[req_id][1] * counter.get(req_id, 0)
                    print opts[1], size
            """).format(**context)
        )
    
    def commit(self):
        self.append('monitor(users, delivers, reverse, maillogs)')
    
    def monitor(self, mailbox):
        context = self.get_context(mailbox)
        self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context)
    
    def get_context(self, mailbox):
        context = {
            'mailbox': mailbox.name,
            'object_id': mailbox.pk,
            'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
        }
        return replace(context, "'", '"')