From 831347fb0335762c5e4d87acd739b5756d957e72 Mon Sep 17 00:00:00 2001 From: Marc Date: Thu, 9 Oct 2014 17:04:12 +0000 Subject: [PATCH] Initial lists tests --- TODO.md | 4 + orchestra/admin/forms.py | 1 - orchestra/admin/utils.py | 33 ++-- orchestra/apps/databases/admin.py | 2 +- orchestra/apps/databases/backends.py | 18 +- orchestra/apps/databases/forms.py | 26 +-- orchestra/apps/databases/models.py | 11 +- orchestra/apps/databases/serializers.py | 6 +- .../databases/tests/functional_tests/tests.py | 13 +- orchestra/apps/lists/backends.py | 81 +++++++++ orchestra/apps/lists/models.py | 14 +- orchestra/apps/lists/serializers.py | 25 ++- orchestra/apps/lists/settings.py | 13 +- orchestra/apps/lists/tests/__init__.py | 0 .../lists/tests/functional_tests/__init__.py | 0 .../lists/tests/functional_tests/tests.py | 158 ++++++++++++++++++ orchestra/apps/mails/admin.py | 29 ++-- orchestra/apps/mails/backends.py | 131 +++++++++------ orchestra/apps/mails/forms.py | 18 +- orchestra/apps/mails/models.py | 56 ++++++- orchestra/apps/mails/serializers.py | 12 +- orchestra/apps/mails/settings.py | 49 ++++-- .../mails/tests/functional_tests/tests.py | 62 +++++-- orchestra/apps/mails/validators.py | 36 ++-- orchestra/apps/orchestration/admin.py | 14 +- orchestra/apps/orchestration/manager.py | 6 +- orchestra/apps/orchestration/middlewares.py | 7 +- orchestra/apps/orchestration/models.py | 12 +- orchestra/apps/resources/admin.py | 6 +- orchestra/apps/resources/models.py | 7 +- orchestra/apps/services/models.py | 5 +- orchestra/bin/orchestra-admin | 6 +- scripts/services/postfix.md | 5 +- 33 files changed, 650 insertions(+), 216 deletions(-) create mode 100644 orchestra/apps/lists/tests/__init__.py create mode 100644 orchestra/apps/lists/tests/functional_tests/__init__.py create mode 100644 orchestra/apps/lists/tests/functional_tests/tests.py diff --git a/TODO.md b/TODO.md index 178b3ecf..7df42033 100644 --- a/TODO.md +++ b/TODO.md @@ -166,3 +166,7 @@ APPS app? * disable account triggers save on cascade to execute backends save(update_field=[]) + + +* validate database user names +* multiple domains creation; line separated domains diff --git a/orchestra/admin/forms.py b/orchestra/admin/forms.py index 8b9dab9c..ef00dfa3 100644 --- a/orchestra/admin/forms.py +++ b/orchestra/admin/forms.py @@ -115,7 +115,6 @@ class AdminPasswordChangeForm(forms.Form): for ix, rel in enumerate(self.related): password = self.cleaned_data['password1_%s' % ix] if password: - print password set_password = getattr(rel, 'set_password') set_password(password) if commit: diff --git a/orchestra/admin/utils.py b/orchestra/admin/utils.py index f67be137..2988b654 100644 --- a/orchestra/admin/utils.py +++ b/orchestra/admin/utils.py @@ -1,4 +1,5 @@ import datetime +import inspect from functools import wraps from django.conf import settings @@ -31,30 +32,26 @@ def get_modeladmin(model, import_module=True): return get_modeladmin(model, import_module=False) -def insertattr(model, name, value, weight=0): +def insertattr(model, name, value): """ Inserts attribute to a modeladmin """ - modeladmin_class = model + modeladmin = None if models.Model in model.__mro__: - modeladmin_class = type(get_modeladmin(model)) + modeladmin = get_modeladmin(model) + modeladmin_class = type(modeladmin) + elif not inspect.isclass(model): + modeladmin = model + modeladmin_class = type(modeladmin) + else: + modeladmin_class = model # Avoid inlines defined on parent class be shared between subclasses # Seems that if we use tuples they are lost in some conditions like changing # the tuple in modeladmin.__init__ if not getattr(modeladmin_class, name): setattr(modeladmin_class, name, []) - - inserted_attrs = getattr(modeladmin_class, '__inserted_attrs__', {}) - if not name in inserted_attrs: - weights = {} - if hasattr(modeladmin_class, 'weights') and name in modeladmin_class.weights: - weights = modeladmin_class.weights.get(name) - inserted_attrs[name] = [ - (attr, weights.get(attr, 0)) for attr in getattr(modeladmin_class, name) - ] - - inserted_attrs[name].append((value, weight)) - inserted_attrs[name].sort(key=lambda a: a[1]) - setattr(modeladmin_class, name, [ attr[0] for attr in inserted_attrs[name] ]) - setattr(modeladmin_class, '__inserted_attrs__', inserted_attrs) + setattr(modeladmin_class, name, list(getattr(modeladmin_class, name))+[value]) + if modeladmin: + # make sure class and object share the same attribute, to avoid wierd bugs + setattr(modeladmin, name, getattr(modeladmin_class, name)) def wrap_admin_view(modeladmin, view): @@ -84,7 +81,7 @@ def action_to_view(action, modeladmin): response = action(modeladmin, request, queryset) if not response: opts = modeladmin.model._meta - url = 'admin:%s_%s_change' % (opts.app_label, opts.module_name) + url = 'admin:%s_%s_change' % (opts.app_label, opts.model_name) return redirect(url, object_id) return response return action_view diff --git a/orchestra/apps/databases/admin.py b/orchestra/apps/databases/admin.py index 293612ae..1b11e0cf 100644 --- a/orchestra/apps/databases/admin.py +++ b/orchestra/apps/databases/admin.py @@ -89,7 +89,7 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): if not change: user = form.cleaned_data['user'] if not user: - user = DatabaseUser.objects.create( + user = DatabaseUser( username=form.cleaned_data['username'], type=obj.type, account_id = obj.account.pk, diff --git a/orchestra/apps/databases/backends.py b/orchestra/apps/databases/backends.py index 1985e965..ee3fe7a7 100644 --- a/orchestra/apps/databases/backends.py +++ b/orchestra/apps/databases/backends.py @@ -41,28 +41,28 @@ class MySQLUserBackend(ServiceController): verbose_name = "MySQL user" model = 'databases.DatabaseUser' - def save(self, database): - if database.type == database.MYSQL: - context = self.get_context(database) + def save(self, user): + if user.type == user.MYSQL: + context = self.get_context(user) self.append( - "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context + "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context ) self.append( "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" " " WHERE User=\"%(username)s\";'" % context ) - def delete(self, database): - if database.type == database.MYSQL: + def delete(self, user): + if user.type == user.MYSQL: context = self.get_context(database) self.append( "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context ) - def get_context(self, database): + def get_context(self, user): return { - 'username': database.username, - 'password': database.password, + 'username': user.username, + 'password': user.password, 'host': settings.DATABASES_DEFAULT_HOST, } diff --git a/orchestra/apps/databases/forms.py b/orchestra/apps/databases/forms.py index a9ff2d38..a048782a 100644 --- a/orchestra/apps/databases/forms.py +++ b/orchestra/apps/databases/forms.py @@ -30,9 +30,9 @@ class DatabaseUserCreationForm(forms.ModelForm): def save(self, commit=True): user = super(DatabaseUserCreationForm, self).save(commit=False) - user.set_password(self.cleaned_data["password1"]) - if commit: - user.save() +# user.set_password(self.cleaned_data["password1"]) +# if commit: +# user.save() return user @@ -89,16 +89,16 @@ class DatabaseCreationForm(DatabaseUserCreationForm): def save(self, commit=True): db = super(DatabaseUserCreationForm, self).save(commit=False) - user = self.cleaned_data['user'] - if commit: - if not user: - user = DatabaseUser( - username=self.cleaned_data['username'], - type=self.cleaned_data['type'], - ) - user.set_password(self.cleaned_data["password1"]) - user.save() - role, __ = Role.objects.get_or_create(database=db, user=user) +# if commit: +# user = self.cleaned_data['user'] +# if not user: +# user = DatabaseUser( +# username=self.cleaned_data['username'], +# type=self.cleaned_data['type'], +# ) +# user.set_password(self.cleaned_data["password1"]) +# user.save() +# role, __ = Role.objects.get_or_create(database=db, user=user) return db diff --git a/orchestra/apps/databases/models.py b/orchestra/apps/databases/models.py index 33081a14..f6bfda61 100644 --- a/orchestra/apps/databases/models.py +++ b/orchestra/apps/databases/models.py @@ -13,7 +13,7 @@ class Database(models.Model): MYSQL = 'mysql' POSTGRESQL = 'postgresql' - name = models.CharField(_("name"), max_length=128, + name = models.CharField(_("name"), max_length=64, # MySQL limit validators=[validators.validate_name]) users = models.ManyToManyField('databases.DatabaseUser', verbose_name=_("users"), @@ -53,9 +53,7 @@ class Role(models.Model): msg = _("Database and user type doesn't match") raise validators.ValidationError(msg) roles = self.database.roles.values('id') - print roles if not roles or (len(roles) == 1 and roles[0].id == self.id): - print 'seld' self.is_owner = True @@ -63,9 +61,9 @@ class DatabaseUser(models.Model): MYSQL = 'mysql' POSTGRESQL = 'postgresql' - username = models.CharField(_("username"), max_length=128, + username = models.CharField(_("username"), max_length=16, # MySQL usernames 16 char long validators=[validators.validate_name]) - password = models.CharField(_("password"), max_length=128) + password = models.CharField(_("password"), max_length=256) type = models.CharField(_("type"), max_length=32, choices=settings.DATABASES_TYPE_CHOICES, default=settings.DATABASES_DEFAULT_TYPE) @@ -87,8 +85,7 @@ class DatabaseUser(models.Model): # MySQL stores sha1(sha1(password).binary).hex binary = hashlib.sha1(password).digest() hexdigest = hashlib.sha1(binary).hexdigest() - password = '*%s' % hexdigest.upper() - self.password = password + self.password = '*%s' % hexdigest.upper() else: raise TypeError("Database type '%s' not supported" % self.type) diff --git a/orchestra/apps/databases/serializers.py b/orchestra/apps/databases/serializers.py index 740aa357..51b51c7f 100644 --- a/orchestra/apps/databases/serializers.py +++ b/orchestra/apps/databases/serializers.py @@ -8,7 +8,7 @@ from orchestra.core.validators import validate_password from .models import Database, DatabaseUser, Role -class UserSerializer(serializers.HyperlinkedModelSerializer): +class UserRoleSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Role fields = ('user', 'is_owner',) @@ -21,11 +21,11 @@ class RoleSerializer(serializers.HyperlinkedModelSerializer): class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): - users = UserSerializer(source='roles', many=True) + roles = UserRoleSerializer(many=True) class Meta: model = Database - fields = ('url', 'name', 'type', 'users') + fields = ('url', 'name', 'type', 'roles') class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): diff --git a/orchestra/apps/databases/tests/functional_tests/tests.py b/orchestra/apps/databases/tests/functional_tests/tests.py index 4885e4f7..d202c3ba 100644 --- a/orchestra/apps/databases/tests/functional_tests/tests.py +++ b/orchestra/apps/databases/tests/functional_tests/tests.py @@ -1,5 +1,6 @@ import MySQLdb import os +import time from functools import partial from django.conf import settings as djsettings @@ -52,7 +53,7 @@ class DatabaseTestMixin(object): def test_add(self): dbname = '%s_database' % random_ascii(5) - username = '%s_dbuser' % random_ascii(10) + username = '%s_dbuser' % random_ascii(5) password = '@!?%spppP001' % random_ascii(5) self.add(dbname, username, password) self.validate_create_table(dbname, username, password) @@ -61,6 +62,10 @@ class DatabaseTestMixin(object): class MySQLBackendMixin(object): db_type = 'mysql' + def setUp(self): + super(MySQLBackendMixin, self).setUp() + settings.DATABASES_DEFAULT_HOST = '10.228.207.207' + def add_route(self): server = Server.objects.create(name=self.MASTER_SERVER) backend = backends.MySQLBackend.get_name() @@ -73,14 +78,13 @@ class MySQLBackendMixin(object): def validate_create_table(self, name, username, password): db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name) cur = db.cursor() - cur.execute('CREATE TABLE test;') + cur.execute('CREATE TABLE test ( id INT ) ;') def validate_delete(self, name, username, password): self.asseRaises(MySQLdb.ConnectionError, self.validate_create_table, name, username, password) - class RESTDatabaseMixin(DatabaseTestMixin): def setUp(self): super(RESTDatabaseMixin, self).setUp() @@ -89,7 +93,8 @@ class RESTDatabaseMixin(DatabaseTestMixin): @save_response_on_error def add(self, dbname, username, password): user = self.rest.databaseusers.create(username=username, password=password) - self.rest.databases.create(name=dbname, user=user, type=self.db_type) + # TODO fucking nested objects + self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type) class AdminDatabaseMixin(DatabaseTestMixin): diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index d23b686a..055288e8 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -6,11 +6,92 @@ from orchestra.apps.orchestration import ServiceController from orchestra.apps.resources import ServiceMonitor from . import settings +from .models import List class MailmanBackend(ServiceController): verbose_name = "Mailman" model = 'lists.List' + + def include_virtual_alias_domain(self, context): + if context['address_domain']: + self.append(textwrap.dedent(""" + [[ $(grep "^\s*%(address_domain)s\s*$" %(virtual_alias_domains)s) ]] || { + echo "%(address_domain)s" >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""" % context + )) + + def exclude_virtual_alias_domain(self, context): + address_domain = context['address_domain'] + if not List.objects.filter(address_domain=address_domain).exists(): + self.append('sed -i "/^%(address_domain)s\s*/d" %(virtual_alias_domains)s' % context) + + def get_virtual_aliases(self, context): + aliases = [] + addresses = [ + '', + '-admin', + '-bounces', + '-confirm', + '-join', + '-leave', + '-owner', + '-request', + '-subscribe', + '-unsubscribe' + ] + for address in addresses: + context['address'] = address + aliases.append("%(address_name)s%(address)s@%(domain)s\t%(name)s%(address)s" % context) + return '\n'.join(aliases) + + def save(self, mail_list): + if not getattr(mail_list, 'password', None): + # TODO + # Create only support for now + return + context = self.get_context(mail_list) + self.append("newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s'" % context) + if mail_list.address: + context['aliases'] = self.get_virtual_aliases(context) + self.append( + "if [[ ! $(grep '^\s*%(name)s\s' %(virtual_alias)s) ]]; then\n" + " echo '# %(banner)s\n%(aliases)s\n' >> %(virtual_alias)s\n" + " UPDATED_VIRTUAL_ALIAS=1\n" + "fi" % context + ) + self.include_virtual_alias_domain(context) + + def delete(self, mail_list): + pass + + def commit(self): + context = self.get_context_files() + self.append(textwrap.dedent(""" + [[ $UPDATED_VIRTUAL_ALIAS == 1 ]] && { postmap %(virtual_alias)s; } + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; } + """ % context + )) + + def get_context_files(self): + return { + 'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH, + 'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH, + } + + def get_context(self, mail_list): + context = self.get_context_files() + context.update({ + 'banner': self.get_banner(), + 'name': mail_list.name, + 'password': mail_list.password, + 'domain': mail_list.address_domain or settings.LISTS_DEFAULT_DOMAIN, + 'address_name': mail_list.address_name, + 'address_domain': mail_list.address_domain, + 'admin': mail_list.admin_email, + }) + return context class MailmanTraffic(ServiceMonitor): diff --git a/orchestra/apps/lists/models.py b/orchestra/apps/lists/models.py index 410e10fd..d4b9c75c 100644 --- a/orchestra/apps/lists/models.py +++ b/orchestra/apps/lists/models.py @@ -7,9 +7,11 @@ from orchestra.core.validators import validate_name from . import settings +# TODO address and domain, perhaps allow only domain? + class List(models.Model): - name = models.CharField(_("name"), max_length=128, unique=True, - validators=[validate_name]) + name = models.CharField(_("name"), max_length=128, unique=True, validators=[validate_name], + help_text=_("Default list address <name>@%s") % settings.LISTS_DEFAULT_DOMAIN) address_name = models.CharField(_("address name"), max_length=128, validators=[validate_name], blank=True) address_domain = models.ForeignKey(settings.LISTS_DOMAIN_MODEL, @@ -23,7 +25,13 @@ class List(models.Model): unique_together = ('address_name', 'address_domain') def __unicode__(self): - return "%s@%s" % (self.address_name, self.address_domain) + return self.name + + @property + def address(self): + if self.address_name and self.address_domain: + return "%s@%s" % (self.address_name, self.address_domain) + return '' def get_username(self): return self.name diff --git a/orchestra/apps/lists/serializers.py b/orchestra/apps/lists/serializers.py index 21b36a8b..cdebcccd 100644 --- a/orchestra/apps/lists/serializers.py +++ b/orchestra/apps/lists/serializers.py @@ -1,11 +1,34 @@ +from django.forms import widgets +from django.utils.translation import ugettext, ugettext_lazy as _ from rest_framework import serializers from orchestra.apps.accounts.serializers import AccountSerializerMixin +from orchestra.core.validators import validate_password from .models import List +# TODO create PasswordSerializerMixin + class ListSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + password = serializers.CharField(max_length=128, label=_('Password'), + validators=[validate_password], write_only=True, required=False, + widget=widgets.PasswordInput) + class Meta: model = List - fields = ('url', 'name', 'address_name', 'address_domain') + fields = ('url', 'name', 'address_name', 'address_domain', 'admin_email') + + def validate_password(self, attrs, source): + """ POST only password """ + if self.object: + if 'password' in attrs: + raise serializers.ValidationError(_("Can not set password")) + elif 'password' not in attrs: + raise serializers.ValidationError(_("Password required")) + return attrs + + def save_object(self, obj, **kwargs): + if not obj.pk: + obj.set_password(self.init_data.get('password', '')) + super(ListSerializer, self).save_object(obj, **kwargs) diff --git a/orchestra/apps/lists/settings.py b/orchestra/apps/lists/settings.py index a3faf55b..6b2de33f 100644 --- a/orchestra/apps/lists/settings.py +++ b/orchestra/apps/lists/settings.py @@ -1,11 +1,20 @@ from django.conf import settings -# Data access LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain') -LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan') + +LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'lists.orchestra.lan') + LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH', '/var/log/mailman/post') + + +LISTS_VIRTUAL_ALIAS_PATH = getattr(settings, 'LISTS_VIRTUAL_ALIAS_PATH', + '/etc/postfix/mailman_virtual_aliases') + + +MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH', + '/etc/postfix/mailman_virtual_domains') diff --git a/orchestra/apps/lists/tests/__init__.py b/orchestra/apps/lists/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/lists/tests/functional_tests/__init__.py b/orchestra/apps/lists/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/lists/tests/functional_tests/tests.py b/orchestra/apps/lists/tests/functional_tests/tests.py new file mode 100644 index 00000000..ce08b33f --- /dev/null +++ b/orchestra/apps/lists/tests/functional_tests/tests.py @@ -0,0 +1,158 @@ +import email.utils +import os +import smtplib +import time +import textwrap +from email.mime.text import MIMEText + +from django.conf import settings as djsettings +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import CommandError +from django.core.urlresolvers import reverse +from selenium.webdriver.support.select import Select + +from orchestra.apps.accounts.models import Account +from orchestra.apps.domains.models import Domain +from orchestra.apps.orchestration.models import Server, Route +from orchestra.apps.resources.models import Resource +from orchestra.utils.system import run, sshrun +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error + +from ... import backends, settings +from ...models import List + + +class ListMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.apps.orchestration', + 'orchestra.apps.domains', + 'orchestra.apps.lists', + ) + + def setUp(self): + super(ListMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def validate_add(self, name, address=None): + sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False) + if not address: + address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN) + subscribe_address = "{}-subscribe@{}".format(*address.split('@')) + self.subscribe(subscribe_address) + time.sleep(2) + sshrun(self.MASTER_SERVER, + 'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"' % address, display=False) + + def subscribe(self, subscribe_address): + msg = MIMEText('') + msg['To'] = subscribe_address + msg['From'] = 'root@%s' % self.MASTER_SERVER + msg['Subject'] = 'subscribe' + server = smtplib.SMTP(self.MASTER_SERVER, 25) + try: + server.ehlo() + server.starttls() + server.ehlo() + server.sendmail(msg['From'], msg['To'], msg.as_string()) + finally: + server.quit() + + def add_route(self): + server = Server.objects.create(name=self.MASTER_SERVER) + backend = backends.MailmanBackend.get_name() + Route.objects.create(backend=backend, match=True, host=server) + + def atest_add(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + self.add(name, password, admin_email) + self.validate_add(name) +# self.addCleanup(self.delete, username) + + def test_add_with_address(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + print password + admin_email = 'root@test3.orchestra.lan' + address_name = '%s_name' % random_ascii(10) + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain) + self.validate_add(name, address="%s@%s" % (address_name, address_domain)) + + +class RESTListMixin(ListMixin): + def setUp(self): + super(RESTListMixin, self).setUp() + self.rest_login() + + @save_response_on_error + def add(self, name, password, admin_email, address_name=None, address_domain=None): + extra = {} + if address_name: + extra.update({ + 'address_name': address_name, + 'address_domain': self.rest.domains.retrieve(name=address_domain.name).get().url, + }) + self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra) + + @save_response_on_error + def delete(self, username): + list = self.rest.lists.retrieve(name=username).get() + list.delete() + + +class AdminListMixin(ListMixin): + def setUp(self): + super(AdminListMixin, self).setUp() + self.admin_login() + + @snapshot_on_error + def add(self, name, password, admin_email): + url = self.live_server_url + reverse('admin:mails_List_add') + self.selenium.get(url) + + account_input = self.selenium.find_element_by_id('id_account') + account_select = Select(account_input) + account_select.select_by_value(str(self.account.pk)) + + name_field = self.selenium.find_element_by_id('id_name') + name_field.send_keys(username) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + if quota is not None: + quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated' + quota_field = self.selenium.find_element_by_id(quota_id) + quota_field.clear() + quota_field.send_keys(quota) + + if filtering is not None: + filtering_input = self.selenium.find_element_by_id('id_filtering') + filtering_select = Select(filtering_input) + filtering_select.select_by_value("CUSTOM") + filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0') + filtering_inline.click() + time.sleep(0.5) + filtering_field = self.selenium.find_element_by_id('id_custom_filtering') + filtering_field.send_keys(filtering) + + name_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + +class RESTListTest(RESTListMixin, BaseLiveServerTestCase): + pass + + +#class AdminListTest(AdminListMixin, BaseLiveServerTestCase): +# pass + + + diff --git a/orchestra/apps/mails/admin.py b/orchestra/apps/mails/admin.py index 7f008a9c..52321c9f 100644 --- a/orchestra/apps/mails/admin.py +++ b/orchestra/apps/mails/admin.py @@ -9,10 +9,9 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin from orchestra.admin.utils import admin_link, change_url from orchestra.apps.accounts.admin import SelectAccountAdminMixin, AccountAdminMixin -from orchestra.forms import UserCreationForm, UserChangeForm from .filters import HasMailboxListFilter, HasForwardListFilter, HasAddressListFilter -from .forms import MailboxCreationForm, AddressForm +from .forms import MailboxCreationForm, MailboxChangeForm, AddressForm from .models import Mailbox, Address, Autoresponse @@ -28,36 +27,34 @@ class AutoresponseInline(admin.StackedInline): class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin): list_display = ( - 'name', 'account_link', 'uses_custom_filtering', 'display_addresses' + 'name', 'account_link', 'filtering', 'display_addresses' ) - list_filter = (HasAddressListFilter,) + list_filter = (HasAddressListFilter, 'filtering') add_fieldsets = ( (None, { - 'fields': ('account', 'name', 'password1', 'password2'), + 'fields': ('account', 'name', 'password1', 'password2', 'filtering'), }), - (_("Filtering"), { + (_("Custom filtering"), { 'classes': ('collapse',), 'fields': ('custom_filtering',), }), ) fieldsets = ( (None, { - 'classes': ('wide',), - 'fields': ('name', 'password', 'is_active', 'account_link'), + 'fields': ('name', 'password', 'is_active', 'account_link', 'filtering'), }), - (_("Filtering"), { + (_("Custom filtering"), { 'classes': ('collapse',), 'fields': ('custom_filtering',), }), (_("Addresses"), { - 'classes': ('wide',), 'fields': ('addresses_field',) }), ) readonly_fields = ('account_link', 'display_addresses', 'addresses_field') change_readonly_fields = ('name',) add_form = MailboxCreationForm - form = UserChangeForm + form = MailboxChangeForm def display_addresses(self, mailbox): addresses = [] @@ -68,16 +65,10 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm display_addresses.short_description = _("Addresses") display_addresses.allow_tags = True - def uses_custom_filtering(self, mailbox): - return bool(mailbox.custom_filtering) - uses_custom_filtering.short_description = _("Custom filter") - uses_custom_filtering.boolean = True - uses_custom_filtering.admin_order_field = 'custom_filtering' - def get_fieldsets(self, request, obj=None): """ not collapsed filtering when exists """ fieldsets = super(MailboxAdmin, self).get_fieldsets(request, obj=obj) - if obj and obj.custom_filtering: + if obj and obj.filtering == obj.CUSTOM: fieldsets = copy.deepcopy(fieldsets) fieldsets[1][1]['classes'] = fieldsets[0][1]['fields'] + ('open',) return fieldsets @@ -97,7 +88,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm name = '%s@%s' % (name, domain) value += '
  • %s
    • ' % (url, name) value = '' % value - return mark_safe('
    %s
    ' % value) + return mark_safe('
    %s
    ' % value) addresses_field.short_description = _("Addresses") addresses_field.allow_tags = True diff --git a/orchestra/apps/mails/backends.py b/orchestra/apps/mails/backends.py index 021968b9..d2ed5e26 100644 --- a/orchestra/apps/mails/backends.py +++ b/orchestra/apps/mails/backends.py @@ -1,3 +1,4 @@ +import logging import textwrap import os @@ -13,11 +14,12 @@ from .models import Address # TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall # TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix -# TODO Set first/last_valid_uid/gid settings to contain only the range actually used by mail processes -# TODO Insert "/./" inside the returned home directory, eg.: home=/home/./user to chroot into /home, or home=/home/user/./ to chroot into /home/user. # TODO mount the filesystem with "nosuid" option +logger = logging.getLogger(__name__) + + class PasswdVirtualUserBackend(ServiceController): verbose_name = _("Mail virtual user (passwd-file)") model = 'mails.Mailbox' @@ -36,22 +38,27 @@ class PasswdVirtualUserBackend(ServiceController): self.append("mkdir -p %(home)s" % context) self.append("chown %(uid)s.%(gid)s %(home)s" % context) + def set_mailbox(self, context): + self.append(textwrap.dedent(""" + if [[ ! $(grep "^%(username)s@%(mailbox_domain)s\s" %(virtual_mailbox_maps)s) ]]; then + echo "%(username)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s + UPDATED_VIRTUAL_MAILBOX_MAPS=1 + fi""" % context)) + def generate_filter(self, mailbox, context): - now = timezone.now().strftime("%B %d, %Y, %H:%M") - context['filtering'] = ( - "# Sieve Filter\n" - "# Generated by Orchestra %s\n\n" % now - ) - if mailbox.custom_filtering: - context['filtering'] += mailbox.custom_filtering + self.append("doveadm mailbox create -u %(username)s Spam" % context) # TODO override webmail filters??? + context['filtering_path'] = os.path.join(context['home'], '.dovecot.sieve') + filtering = mailbox.get_filtering() + if filtering: + context['filtering'] = '# %(banner)s\n' + filtering + self.append("echo '%(filtering)s' > %(filtering_path)s" % context) else: - context['filtering'] += settings.MAILS_DEFAUL_FILTERING - context['filter_path'] = os.path.join(context['home'], '.orchestra.sieve') - self.append("echo '%(filtering)s' > %(filter_path)s" % context) + self.append("rm -f %(filtering_path)s" % context) def save(self, mailbox): context = self.get_context(mailbox) self.set_user(context) + self.set_mailbox(context) self.generate_filter(mailbox, context) def delete(self, mailbox): @@ -59,6 +66,8 @@ class PasswdVirtualUserBackend(ServiceController): self.append("{ sleep 2 && killall -u %(uid)s -s KILL; } &" % context) self.append("killall -u %(uid)s || true" % context) self.append("sed -i '/^%(username)s:.*/d' %(passwd_path)s" % context) + self.append("sed -i '/^%(username)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s" % context) + self.append("UPDATED_VIRTUAL_MAILBOX_MAPS=1") # TODO delete context['deleted'] = context['home'].rstrip('/') + '.deleted' self.append("mv %(home)s %(deleted)s" % context) @@ -75,6 +84,15 @@ class PasswdVirtualUserBackend(ServiceController): unit = mailbox.resources.disk.unit[0].upper() return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit) + def commit(self): + context = { + 'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH + } + self.append( + "[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && { postmap %(virtual_mailbox_maps)s; }" + % context + ) + def get_context(self, mailbox): context = { 'name': mailbox.name, @@ -86,9 +104,12 @@ class PasswdVirtualUserBackend(ServiceController): 'quota': self.get_quota(mailbox), 'passwd_path': settings.MAILS_PASSWD_PATH, 'home': mailbox.get_home(), + 'banner': self.get_banner(), + 'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH, + 'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, } context['extra_fields'] = self.get_extra_fields(mailbox, context) - context['passwd'] = '{username}:{password}:{uid}:{gid}:,,,:{home}:{extra_fields}'.format(**context) + context['passwd'] = '{username}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context) return context @@ -96,62 +117,76 @@ class PostfixAddressBackend(ServiceController): verbose_name = _("Postfix address") model = 'mails.Address' - def include_virtdomain(self, context): - self.append( - '[[ $(grep "^\s*%(domain)s\s*$" %(virtdomains)s) ]]' - ' || { echo "%(domain)s" >> %(virtdomains)s; UPDATED_VIRTDOMAINS=1; }' % context - ) - - def exclude_virtdomain(self, context): - domain = context['domain'] - if not Address.objects.filter(domain=domain).exists(): - self.append('sed -i "s/^%(domain)s//" %(virtdomains)s' % context) - - def update_virtusertable(self, context): + def include_virtual_alias_domain(self, context): self.append(textwrap.dedent(""" - LINE="%(email)s\t%(destination)s" - if [[ ! $(grep "^%(email)s\s" %(virtusertable)s) ]]; then - echo "${LINE}" >> %(virtusertable)s - UPDATED_VIRTUSERTABLE=1 - else - if [[ ! $(grep "^${LINE}$" %(virtusertable)s) ]]; then - sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtusertable)s - UPDATED_VIRTUSERTABLE=1 - fi - fi""" % context + [[ $(grep "^\s*%(domain)s\s*$" %(virtual_alias_domains)s) ]] || { + echo "%(domain)s" >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""" % context )) - def exclude_virtusertable(self, context): + def exclude_virtual_alias_domain(self, context): + domain = context['domain'] + if not Address.objects.filter(domain=domain).exists(): + self.append('sed -i "/^%(domain)s\s*/d" %(virtual_alias_domains)s' % context) + + def update_virtual_alias_maps(self, address, context): + destination = [] + for mailbox in address.get_mailboxes(): + context['mailbox'] = mailbox + destination.append("%(mailbox)s@%(mailbox_domain)s" % context) + for forward in address.forward: + if '@' in forward: + destination.append(forward) + if destination: + context['destination'] = ' '.join(destination) + self.append(textwrap.dedent(""" + LINE="%(email)s\t%(destination)s" + if [[ ! $(grep "^%(email)s\s" %(virtual_alias_maps)s) ]]; then + echo "${LINE}" >> %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + else + if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then + sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + fi + fi""" % context + )) + else: + logger.warning("Address %i is empty" % address.pk) + self.append('sed -i "/^%(email)s\s/d" %(virtual_alias_maps)s') + self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1') + + def exclude_virtual_alias_maps(self, context): self.append(textwrap.dedent(""" if [[ $(grep "^%(email)s\s") ]]; then - sed -i "s/^%(email)s\s.*$//" %(virtusertable)s - UPDATED=1 + sed -i "/^%(email)s\s.*$/d" %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 fi""" )) def save(self, address): context = self.get_context(address) - self.include_virtdomain(context) - self.update_virtusertable(context) + self.include_virtual_alias_domain(context) + self.update_virtual_alias_maps(address, context) def delete(self, address): context = self.get_context(address) - self.exclude_virtdomain(context) - self.exclude_virtusertable(context) + self.exclude_virtual_alias_domain(context) + self.exclude_virtual_alias_maps(context) def commit(self): context = self.get_context_files() self.append(textwrap.dedent(""" - [[ $UPDATED_VIRTUSERTABLE == 1 ]] && { postmap %(virtusertable)s; } - # TODO not sure if always needed - [[ $UPDATED_VIRTDOMAINS == 1 ]] && { /etc/init.d/postfix reload; } + [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; } + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; } """ % context )) def get_context_files(self): return { - 'virtdomains': settings.MAILS_VIRTDOMAINS_PATH, - 'virtusertable': settings.MAILS_VIRTUSERTABLE_PATH, + 'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH, + 'virtual_alias_maps': settings.MAILS_VIRTUAL_ALIAS_MAPS_PATH } def get_context(self, address): @@ -159,7 +194,7 @@ class PostfixAddressBackend(ServiceController): context.update({ 'domain': address.domain, 'email': address.email, - 'destination': address.destination, + 'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, }) return context diff --git a/orchestra/apps/mails/forms.py b/orchestra/apps/mails/forms.py index 3a500c03..f05bb145 100644 --- a/orchestra/apps/mails/forms.py +++ b/orchestra/apps/mails/forms.py @@ -1,9 +1,23 @@ from django import forms +from django.utils.translation import ugettext_lazy as _ -from orchestra.forms import UserCreationForm +from orchestra.forms import UserCreationForm, UserChangeForm -class MailboxCreationForm(UserCreationForm): +class CleanCustomFilteringMixin(object): + def clean_custom_filtering(self): + filtering = self.cleaned_data['filtering'] + custom_filtering = self.cleaned_data['custom_filtering'] + if filtering == self._meta.model.CUSTOM and not custom_filtering: + raise forms.ValidationError(_("You didn't provide any custom filtering")) + return custom_filtering + + +class MailboxChangeForm(CleanCustomFilteringMixin, UserChangeForm): + pass + + +class MailboxCreationForm(CleanCustomFilteringMixin, UserCreationForm): def clean_name(self): # Since model.clean() will check this, this is redundant, # but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth diff --git a/orchestra/apps/mails/models.py b/orchestra/apps/mails/models.py index 1f8e4c25..b6c445d1 100644 --- a/orchestra/apps/mails/models.py +++ b/orchestra/apps/mails/models.py @@ -11,6 +11,8 @@ from . import validators, settings # TODO rename app to mailboxes class Mailbox(models.Model): + CUSTOM = 'CUSTOM' + name = models.CharField(_("name"), max_length=64, unique=True, help_text=_("Required. 30 characters or fewer. Letters, digits and " "@/./+/-/_ only."), @@ -19,6 +21,9 @@ class Mailbox(models.Model): password = models.CharField(_("password"), max_length=128) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), related_name='mailboxes') + filtering = models.CharField(max_length=16, + choices=[(k, v[0]) for k,v in settings.MAILS_MAILBOX_FILTERINGS.iteritems()], + default=settings.MAILS_MAILBOX_DEFAULT_FILTERING) custom_filtering = models.TextField(_("filtering"), blank=True, validators=[validators.validate_sieve], help_text=_("Arbitrary email filtering in sieve language. " @@ -51,6 +56,28 @@ class Mailbox(models.Model): } home = settings.MAILS_HOME % context return home.rstrip('/') + + def clean(self): + if self.custom_filtering and self.filtering != self.CUSTOM: + self.custom_filtering = '' + + def get_filtering(self): + __, filtering = settings.MAILS_MAILBOX_FILTERINGS[self.filtering] + if isinstance(filtering, basestring): + return filtering + return filtering(self) + + def delete(self, *args, **kwargs): + super(Mailbox, self).delete(*args, **kwargs) + # Cleanup related addresses + for address in Address.objects.filter(forward__regex=r'.*(^|\s)+%s($|\s)+.*' % self.name): + forward = address.forward.split() + forward.remove(self.name) + address.forward = ' '.join(forward) + if not address.destination: + address.delete() + else: + address.save() class Address(models.Model): @@ -63,7 +90,8 @@ class Address(models.Model): verbose_name=_("mailboxes"), related_name='addresses', blank=True) forward = models.CharField(_("forward"), max_length=256, blank=True, - validators=[validators.validate_forward], help_text=_("Space separated email addresses")) + validators=[validators.validate_forward], + help_text=_("Space separated email addresses or mailboxes")) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), related_name='addresses') @@ -78,12 +106,26 @@ class Address(models.Model): def email(self): return "%s@%s" % (self.name, self.domain) - @property - def destination(self): - destinations = list(self.mailboxes.values_list('name', flat=True)) - if self.forward: - destinations.append(self.forward) - return ' '.join(destinations) +# @property +# def destination(self): +# destinations = list(self.mailboxes.values_list('name', flat=True)) +# if self.forward: +# destinations.append(self.forward) +# return ' '.join(destinations) + + def get_forward_mailboxes(self): + for forward in self.forward.split(): + if '@' not in forward: + try: + yield Mailbox.objects.get(name=forward) + except Mailbox.DoesNotExist: + pass + + def get_mailboxes(self): + for mailbox in self.mailboxes.all(): + yield mailbox + for mailbox in self.get_forward_mailboxes(): + yield mailbox class Autoresponse(models.Model): diff --git a/orchestra/apps/mails/serializers.py b/orchestra/apps/mails/serializers.py index d48121e6..30fcaa27 100644 --- a/orchestra/apps/mails/serializers.py +++ b/orchestra/apps/mails/serializers.py @@ -1,15 +1,23 @@ +from django.forms import widgets +from django.utils.translation import ugettext, ugettext_lazy as _ from rest_framework import serializers from orchestra.apps.accounts.serializers import AccountSerializerMixin +from orchestra.core.validators import validate_password from .models import Mailbox, Address class MailboxSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + password = serializers.CharField(max_length=128, label=_('Password'), + validators=[validate_password], write_only=True, required=False, + widget=widgets.PasswordInput) + class Meta: model = Mailbox - # TODO 'use_custom_filtering', - fields = ('url', 'name', 'password', 'custom_filtering', 'addresses', 'is_active') + fields = ( + 'url', 'name', 'password', 'filtering', 'custom_filtering', 'addresses', 'is_active' + ) def validate_password(self, attrs, source): """ POST only password """ diff --git a/orchestra/apps/mails/settings.py b/orchestra/apps/mails/settings.py index 73b30774..9281f5be 100644 --- a/orchestra/apps/mails/settings.py +++ b/orchestra/apps/mails/settings.py @@ -1,4 +1,7 @@ +import textwrap + from django.conf import settings +from django.utils.translation import ugettext_lazy as _ MAILS_DOMAIN_MODEL = getattr(settings, 'MAILS_DOMAIN_MODEL', 'domains.Domain') @@ -14,23 +17,43 @@ MAILS_SIEVETEST_BIN_PATH = getattr(settings, 'MAILS_SIEVETEST_BIN_PATH', '%(orchestra_root)s/bin/sieve-test') -MAILS_VIRTUSERTABLE_PATH = getattr(settings, 'MAILS_VIRTUSERTABLE_PATH', - '/etc/postfix/virtusertable') +MAILS_VIRTUAL_MAILBOX_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_MAPS_PATH', + '/etc/postfix/virtual_mailboxes') + + +MAILS_VIRTUAL_ALIAS_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_MAPS_PATH', + '/etc/postfix/virtual_aliases') -MAILS_VIRTDOMAINS_PATH = getattr(settings, 'MAILS_VIRTDOMAINS_PATH', - '/etc/postfix/virtdomains') +MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH', + '/etc/postfix/virtual_domains') +MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN', + 'orchestra.lan') + MAILS_PASSWD_PATH = getattr(settings, 'MAILS_PASSWD_PATH', - '/etc/dovecot/virtual_users') + '/etc/dovecot/passwd') -MAILS_DEFAUL_FILTERING = getattr(settings, 'MAILS_DEFAULT_FILTERING', - 'require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];\n' - '\n' - 'if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {\n' - ' fileinto "Junk";\n' - ' discard;\n' - '}' -) + +MAILS_MAILBOX_FILTERINGS = getattr(settings, 'MAILS_MAILBOX_FILTERINGS', { + # value: (verbose_name, filter) + 'DISABLE': (_("Disable"), ''), + 'REJECT': (_("Reject spam"), textwrap.dedent(""" + require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; + if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" { + discard; + stop; + }""")), + 'REDIRECT': (_("Archive spam"), textwrap.dedent(""" + require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; + if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" { + fileinto "Spam"; + stop; + }""")), + 'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering), +}) + + +MAILS_MAILBOX_DEFAULT_FILTERING = getattr(settings, 'MAILS_MAILBOX_DEFAULT_FILTERING', 'REDIRECT') diff --git a/orchestra/apps/mails/tests/functional_tests/tests.py b/orchestra/apps/mails/tests/functional_tests/tests.py index 2d519d4e..0b415b83 100644 --- a/orchestra/apps/mails/tests/functional_tests/tests.py +++ b/orchestra/apps/mails/tests/functional_tests/tests.py @@ -4,8 +4,10 @@ import os import poplib import smtplib import time +import textwrap from email.mime.text import MIMEText +from django.apps import apps from django.conf import settings as djsettings from django.contrib.contenttypes.models import ContentType from django.core.management.base import CommandError @@ -33,8 +35,6 @@ class MailboxMixin(object): def setUp(self): super(MailboxMixin, self).setUp() self.add_route() - # TODO fix this - from django.apps import apps # clean resource relation from other tests apps.get_app_config('resources').reload_relations() djsettings.DEBUG = True @@ -92,7 +92,7 @@ class MailboxMixin(object): def send_email(self, to, token): msg = MIMEText(token) msg['To'] = to - msg['From'] = 'orchestra@test.orchestra.lan' + msg['From'] = 'orchestra@%s' % self.MASTER_SERVER msg['Subject'] = 'test' server = smtplib.SMTP(self.MASTER_SERVER, 25) try: @@ -176,7 +176,7 @@ class MailboxMixin(object): password = '@!?%spppP001' % random_ascii(5) self.add(username, password) self.validate_mailbox(username) - self.addCleanup(self.delete, username) +# self.addCleanup(self.delete, username) imap = self.login_imap(username, password) self.disable(username) self.assertRaises(imap.error, self.login_imap, username, password) @@ -211,6 +211,27 @@ class MailboxMixin(object): self.delete_address(username) self.send_email("%s@%s" % (name, domain), token) self.validate_email(username, token) + + def test_custom_filtering(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + folder = random_ascii(5) + filtering = textwrap.dedent(""" + require "fileinto"; + if true { + fileinto "%s"; + stop; + }""" % folder) + self.add(username, password, filtering=filtering) + self.addCleanup(self.delete, username) + imap = self.login_imap(username, password) + imap.create(folder) + self.validate_mailbox(username) + token = random_ascii(100) + self.send_email("%s@%s" % (username, settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN), token) + home = Mailbox.objects.get(name=username).get_home() + sshrun(self.MASTER_SERVER, + "grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder), display=False) class RESTMailboxMixin(MailboxMixin): @@ -219,17 +240,22 @@ class RESTMailboxMixin(MailboxMixin): self.rest_login() @save_response_on_error - def add(self, username, password, quota=None): + def add(self, username, password, quota=None, filtering=None): extra = {} if quota: - extra = { + extra.update({ "resources": [ { "name": "disk", "allocated": quota }, ] - } + }) + if filtering: + extra.update({ + 'filtering': 'CUSTOM', + 'custom_filtering': filtering, + }) self.rest.mailboxes.create(name=username, password=password, **extra) @save_response_on_error @@ -270,7 +296,7 @@ class AdminMailboxMixin(MailboxMixin): self.admin_login() @snapshot_on_error - def add(self, username, password, quota=None): + def add(self, username, password, quota=None, filtering=None): url = self.live_server_url + reverse('admin:mails_mailbox_add') self.selenium.get(url) @@ -285,17 +311,23 @@ class AdminMailboxMixin(MailboxMixin): password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) + if quota is not None: - from orchestra.admin.utils import get_modeladmin - m = get_modeladmin(Mailbox) - print 't', type(m).inlines - print 'm', m.inlines - self.take_screenshot() - quota_field = self.selenium.find_element_by_id( - 'id_resources-resourcedata-content_type-object_id-0-allocated') + quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated' + quota_field = self.selenium.find_element_by_id(quota_id) quota_field.clear() quota_field.send_keys(quota) + if filtering is not None: + filtering_input = self.selenium.find_element_by_id('id_filtering') + filtering_select = Select(filtering_input) + filtering_select.select_by_value("CUSTOM") + filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0') + filtering_inline.click() + time.sleep(0.5) + filtering_field = self.selenium.find_element_by_id('id_custom_filtering') + filtering_field.send_keys(filtering) + name_field.submit() self.assertNotEqual(url, self.selenium.current_url) diff --git a/orchestra/apps/mails/validators.py b/orchestra/apps/mails/validators.py index eab400fa..03e0c1da 100644 --- a/orchestra/apps/mails/validators.py +++ b/orchestra/apps/mails/validators.py @@ -2,6 +2,7 @@ import hashlib import os import re +from django.core.management.base import CommandError from django.core.validators import ValidationError, EmailValidator from django.utils.translation import ugettext_lazy as _ @@ -22,38 +23,33 @@ def validate_emailname(value): raise ValidationError(msg) -#def validate_destination(value): -# """ space separated mailboxes or emails """ -# for destination in value.split(): -# msg = _("'%s' is not an existent mailbox" % destination) -# if '@' in destination: -# if not destination[-1].isalpha(): -# raise ValidationError(msg) -# EmailValidator(destination) -# else: -# from .models import Mailbox -# if not Mailbox.objects.filter(user__username=destination).exists(): -# raise ValidationError(msg) -# validate_emailname(destination) - - def validate_forward(value): """ space separated mailboxes or emails """ + from .models import Mailbox for destination in value.split(): - EmailValidator(destination) + msg = _("'%s' is not an existent mailbox" % destination) + if '@' in destination: + if not destination[-1].isalpha(): + raise ValidationError(msg) + EmailValidator(destination) + else: + if not Mailbox.objects.filter(user__username=destination).exists(): + raise ValidationError(msg) + validate_emailname(destination) def validate_sieve(value): sieve_name = '%s.sieve' % hashlib.md5(value).hexdigest() - path = os.path.join(settings.EMAILS_SIEVETEST_PATH, sieve_name) + path = os.path.join(settings.MAILS_SIEVETEST_PATH, sieve_name) with open(path, 'wb') as f: f.write(value) context = { 'orchestra_root': paths.get_orchestra_root() } - sievetest = settings.EMAILS_SIEVETEST_BIN_PATH % context - test = run(' '.join([sievetest, path, '/dev/null']), display=False) - if test.return_code: + sievetest = settings.MAILS_SIEVETEST_BIN_PATH % context + try: + test = run(' '.join([sievetest, path, '/dev/null']), display=False) + except CommandError: errors = [] for line in test.stderr.splitlines(): error = re.match(r'^.*(line\s+[0-9]+:.*)', line) diff --git a/orchestra/apps/orchestration/admin.py b/orchestra/apps/orchestration/admin.py index fdbbffb6..ae932b0d 100644 --- a/orchestra/apps/orchestration/admin.py +++ b/orchestra/apps/orchestration/admin.py @@ -46,8 +46,8 @@ class RouteAdmin(admin.ModelAdmin): class BackendOperationInline(admin.TabularInline): model = BackendOperation - fields = ('action', 'instance_link') - readonly_fields = ('action', 'instance_link') + fields = ('action', 'content_object_link') + readonly_fields = ('action', 'content_object_link') extra = 0 can_delete = False @@ -56,22 +56,22 @@ class BackendOperationInline(admin.TabularInline): 'all': ('orchestra/css/hide-inline-id.css',) } - def instance_link(self, operation): + def content_object_link(self, operation): try: - return admin_link('instance')(self, operation) + return admin_link('content_object')(self, operation) except: return _("deleted {0} {1}").format( escape(operation.content_type), escape(operation.object_id) ) - instance_link.allow_tags = True - instance_link.short_description = _("Instance") + content_object_link.allow_tags = True + content_object_link.short_description = _("Content_object") def has_add_permission(self, *args, **kwargs): return False def get_queryset(self, request): queryset = super(BackendOperationInline, self).get_queryset(request) - return queryset.prefetch_related('instance') + return queryset.prefetch_related('content_object') def display_mono(field): diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index f2373302..6af93bd0 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -14,9 +14,12 @@ logger = logging.getLogger(__name__) def as_task(execute): def wrapper(*args, **kwargs): - with db.transaction.commit_manually(): + db.transaction.set_autocommit(False) + try: log = execute(*args, **kwargs) + finally: db.transaction.commit() + db.transaction.set_autocommit(True) if log.state != log.SUCCESS: send_report(execute, args, log) return log @@ -25,7 +28,6 @@ def as_task(execute): def close_connection(execute): """ Threads have their own connection pool, closing it when finishing """ - # TODO rewrite as context manager def wrapper(*args, **kwargs): log = execute(*args, **kwargs) db.connection.close() diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index f642b6a1..a368bc19 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -84,7 +84,12 @@ class OperationsMiddleware(object): if not execute: continue instance = copy.copy(instance) - pending_operations.add(Operation.create(backend, instance, action)) + operation = Operation.create(backend, instance, action) + if action != Operation.DELETE: + # usually we expect to be using last object state, + # except when we are deleting it + pending_operations.discard(operation) + pending_operations.add(operation) def process_request(self, request): """ Store request on a thread local variable """ diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index 826b795f..8ef143a9 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -102,19 +102,23 @@ class BackendOperation(models.Model): content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() - instance = generic.GenericForeignKey('content_type', 'object_id') + content_object = generic.GenericForeignKey('content_type', 'object_id') class Meta: verbose_name = _("Operation") verbose_name_plural = _("Operations") + def __init__(self, *args, **kwargs): + self.instance = kwargs.pop('instance', None) + super(BackendOperation, self).__init__(*args, **kwargs) + def __unicode__(self): - return '%s.%s(%s)' % (self.backend, self.action, self.instance) + return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.content_object) def __hash__(self): """ set() """ backend = getattr(self, 'backend', self.backend) - return hash(backend) + hash(self.instance) + hash(self.action) + return hash(backend) + hash(self.instance or self.content_object) + hash(self.action) def __eq__(self, operation): """ set() """ @@ -122,7 +126,7 @@ class BackendOperation(models.Model): @classmethod def create(cls, backend, instance, action): - op = cls(backend=backend.get_name(), instance=instance, action=action) + op = cls(backend=backend.get_name(), instance=instance, content_object=instance, action=action) op.backend = backend return op diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 8d5cb35d..4fff58d6 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -7,6 +7,7 @@ from orchestra.admin import ExtendedModelAdmin from orchestra.admin.filters import UsedContentTypeFilter from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date from orchestra.core import services +from orchestra.utils import database_ready from .forms import ResourceForm from .models import Resource, ResourceData, MonitorData @@ -135,7 +136,6 @@ def resource_inline_factory(resources): return ResourceInline -from orchestra.utils import database_ready def insert_resource_inlines(): # Clean previous state for related in Resource._related: @@ -144,14 +144,12 @@ def insert_resource_inlines(): for inline in getattr(modeladmin_class, 'inlines', []): if inline.__name__ == 'ResourceInline': modeladmin_class.inlines.remove(inline) - modeladmin.inlines = modeladmin_class.inlines for ct, resources in Resource.objects.group_by('content_type').iteritems(): inline = resource_inline_factory(resources) model = ct.model_class() - modeladmin = get_modeladmin(model) insertattr(model, 'inlines', inline) - modeladmin.inlines = type(modeladmin).inlines + if database_ready(): insert_resource_inlines() diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index 9e09bf42..19fac85a 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -101,10 +101,9 @@ class Resource(models.Model): elif task.crontab != self.crontab: task.crontab = self.crontab task.save(update_fields=['crontab']) - if created: - # This only work on tests because of multiprocessing used on real deployments - print 'saved' - apps.get_app_config('resources').reload_relations() + # This only work on tests (multiprocessing used on real deployments) + apps.get_app_config('resources').reload_relations() + # TODO touch wsgi.py for code reloading? def delete(self, *args, **kwargs): super(Resource, self).delete(*args, **kwargs) diff --git a/orchestra/apps/services/models.py b/orchestra/apps/services/models.py index 53c45fd8..9e3c5182 100644 --- a/orchestra/apps/services/models.py +++ b/orchestra/apps/services/models.py @@ -230,10 +230,11 @@ class Service(models.Model): def get_services(cls, instance): cache = caches.get_request_cache() ct = ContentType.objects.get_for_model(instance) - services = cache.get(ct) + key = 'services.Service-%i' % ct.pk + services = cache.get(key) if services is None: services = cls.objects.filter(content_type=ct, is_active=True) - cache.set(ct, services) + cache.set(key, services) return services # FIXME some times caching is nasty, do we really have to? make get_plugin more efficient? diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index c6dd6a20..dc818034 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -130,8 +130,7 @@ function install_requirements () { libxml2-dev \ libxslt1-dev \ wkhtmltopdf \ - xvfb \ - python-mysqldb" + xvfb" PIP="django==1.7 \ django-celery-email==1.0.4 \ @@ -159,7 +158,8 @@ function install_requirements () { if $testing; then APT="${APT} \ iceweasel \ - dnsutils" + dnsutils \ + python-mysqldb" PIP="${PIP} \ selenium \ xvfbwrapper \ diff --git a/scripts/services/postfix.md b/scripts/services/postfix.md index 9e7f7873..add04c55 100644 --- a/scripts/services/postfix.md +++ b/scripts/services/postfix.md @@ -6,7 +6,7 @@ -apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve +apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve dovecot-managesieved echo 'mail_location = maildir:~/Maildir mail_plugins = quota @@ -42,3 +42,6 @@ echo 'mailbox_transport = lmtp:unix:private/dovecot-lmtp' >> /etc/postfix/main.c /etc/init.d/dovecot restart /etc/init.d/postfix restart +# TODO check postfix and dovecot configs + +# TODO crontab that deletes message +30 days on spam folders