From f9154a837448ec956f2cf9b01aacb32cf8fab3a6 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Tue, 28 Apr 2015 15:23:57 +0000 Subject: [PATCH] Improved orders filtering and added mailbox filtering --- TODO.md | 3 ++ orchestra/contrib/issues/admin.py | 6 ++-- orchestra/contrib/issues/models.py | 2 +- orchestra/contrib/issues/serializers.py | 18 ++++++---- orchestra/contrib/issues/settings.py | 8 +++-- orchestra/contrib/mailboxes/backends.py | 36 +++++++++++-------- orchestra/contrib/mailboxes/models.py | 8 ++--- orchestra/contrib/mailboxes/settings.py | 8 +++-- orchestra/contrib/mailboxes/validators.py | 11 +++--- orchestra/contrib/orchestration/backends.py | 3 ++ orchestra/contrib/orchestration/tasks.py | 2 +- orchestra/contrib/orders/admin.py | 18 +++++++++- orchestra/contrib/orders/filters.py | 38 +++++++++++++++++---- orchestra/contrib/webapps/types/php.py | 23 ++++++++----- orchestra/utils/options.py | 2 +- 15 files changed, 128 insertions(+), 58 deletions(-) diff --git a/TODO.md b/TODO.md index 9344216f..8696a41d 100644 --- a/TODO.md +++ b/TODO.md @@ -293,3 +293,6 @@ https://code.djangoproject.com/ticket/24576 # insert settings on dashboard dynamically # convert all complex settings to string +# @ something database names +# password validation cracklib on change password form=????? +# reset setting buton diff --git a/orchestra/contrib/issues/admin.py b/orchestra/contrib/issues/admin.py index 4a4f097b..d837366d 100644 --- a/orchestra/contrib/issues/admin.py +++ b/orchestra/contrib/issues/admin.py @@ -190,7 +190,7 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin): display_creator = admin_link('creator') display_queue = admin_link('queue') display_owner = admin_link('owner') - updated = admin_date('updated') + updated = admin_date('updated_at') display_state = admin_colored('state', colors=STATE_COLORS, bold=False) display_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False) @@ -270,8 +270,8 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin): ticket.mark_as_read_by(request.user) context = {'title': "Issue #%i - %s" % (ticket.id, ticket.subject)} context.update(extra_context or {}) - return super(TicketAdmin, self).change_view( - request, object_id, form_url, extra_context=context) + return super(TicketAdmin, self).change_view(request, object_id, form_url=form_url, + extra_context=context) def changelist_view(self, request, extra_context=None): # Hook user for bold_subject diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py index 40e27b88..645521b0 100644 --- a/orchestra/contrib/issues/models.py +++ b/orchestra/contrib/issues/models.py @@ -86,7 +86,7 @@ class Ticket(models.Model): emails.append(self.creator.email) if self.owner: emails.append(self.owner.email) - for contact in self.creator.account.contacts.all(): + for contact in self.creator.contacts.all(): if self.queue and set(contact.email_usage).union(set(self.queue.notify)): emails.append(contact.email) for message in self.messages.distinct('author'): diff --git a/orchestra/contrib/issues/serializers.py b/orchestra/contrib/issues/serializers.py index 3a2062a9..1518c504 100644 --- a/orchestra/contrib/issues/serializers.py +++ b/orchestra/contrib/issues/serializers.py @@ -19,14 +19,18 @@ class MessageSerializer(serializers.HyperlinkedModelSerializer): def get_identity(self, data): return data.get('id') - def save_object(self, obj, **kwargs): - obj.author = self.context['request'].user - super(MessageSerializer, self).save_object(obj, **kwargs) + def create(self, validated_data): + validated_data['account'] = self.account + return super(AccountSerializerMixin, self).create(validated_data) + + def create(self, validated_data): + validated_data['author'] = self.context['request'].user + super(MessageSerializer, self).create(validated_data) class TicketSerializer(serializers.HyperlinkedModelSerializer): """ Validates if this zone generates a correct zone file """ - messages = MessageSerializer(required=False, many=True) + messages = MessageSerializer(required=False, many=True, read_only=True) is_read = serializers.SerializerMethodField() class Meta: @@ -40,6 +44,6 @@ class TicketSerializer(serializers.HyperlinkedModelSerializer): def get_is_read(self, obj): return obj.is_read_by(self.context['request'].user) - def save_object(self, obj, **kwargs): - obj.creator = self.context['request'].user - super(TicketSerializer, self).save_object(obj, **kwargs) + def create(self, validated_data): + validated_data['creator'] = self.context['request'].user + return super(TicketSerializer, self).create(validated_data) diff --git a/orchestra/contrib/issues/settings.py b/orchestra/contrib/issues/settings.py index 2b92f2a7..9346f8cc 100644 --- a/orchestra/contrib/issues/settings.py +++ b/orchestra/contrib/issues/settings.py @@ -1,8 +1,12 @@ -from orchestra.settings import Setting +from django.core.validators import validate_email + +from orchestra.settings import Setting, ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS', - () + (ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL,), + validators=[lambda emails: [validate_email(e) for e in emails]], + help_text="Includes ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL by default", ) diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index d4ea4986..bf39d265 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -19,7 +19,21 @@ from .models import Address logger = logging.getLogger(__name__) -class UNIXUserMaildirBackend(ServiceController): +class FilteringMixin(object): + def generate_filter(self, mailbox, context): + name, content = mailbox.get_filtering() + if name == 'REDIRECT': + self.append("doveadm mailbox create -u %(user)s Spam" % context) + context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context + if content: + context['filtering'] = ('# %(banner)s\n' + filtering) % context + self.append("mkdir -p $(dirname '%(filtering_path)s')" % context) + self.append("echo '%(filtering)s' > %(filtering_path)s" % context) + else: + self.append("echo '' > %(filtering_path)s" % context) + + +class UNIXUserMaildirBackend(FilteringMixin, ServiceController): """ Assumes that all system users on this servers all mail accounts. If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes @@ -41,6 +55,7 @@ class UNIXUserMaildirBackend(ServiceController): ) if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'): self.set_quota(mailbox, context) + self.generate_filter(mailbox, context) def set_quota(self, mailbox, context): context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale() @@ -70,23 +85,25 @@ class UNIXUserMaildirBackend(ServiceController): context = { 'user': mailbox.name, 'group': mailbox.name, + 'name': mailbox.name, 'password': mailbox.password if mailbox.active else '*%s' % mailbox.password, 'home': mailbox.get_home(), 'initial_shell': '/dev/null', + 'banner': self.get_banner(), } return replace(context, "'", '"') -class DovecotPostfixPasswdVirtualUserBackend(ServiceController): +class DovecotPostfixPasswdVirtualUserBackend(FilteringMixin, ServiceController): """ WARNING: This backends is not fully implemented """ + DEFAULT_GROUP = 'postfix' + verbose_name = _("Dovecot-Postfix virtualuser") model = 'mailboxes.Mailbox' # TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data - DEFAULT_GROUP = 'postfix' - def set_user(self, context): self.append(textwrap.dedent(""" if [[ $( grep "^%(user)s:" %(passwd_path)s ) ]]; then @@ -106,17 +123,6 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController): fi""") % context ) - def generate_filter(self, mailbox, context): - self.append("doveadm mailbox create -u %(user)s Spam" % context) - context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context - filtering = mailbox.get_filtering() - if filtering: - context['filtering'] = '# %(banner)s\n' + filtering - self.append("mkdir -p $(dirname '%(filtering_path)s')" % context) - self.append("echo '%(filtering)s' > %(filtering_path)s" % context) - else: - self.append("rm -f %(filtering_path)s" % context) - def save(self, mailbox): context = self.get_context(mailbox) self.set_user(context) diff --git a/orchestra/contrib/mailboxes/models.py b/orchestra/contrib/mailboxes/models.py index 90293eda..d70a6227 100644 --- a/orchestra/contrib/mailboxes/models.py +++ b/orchestra/contrib/mailboxes/models.py @@ -59,10 +59,10 @@ class Mailbox(models.Model): self.custom_filtering = '' def get_filtering(self): - __, filtering = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering] - if isinstance(filtering, str): - return filtering - return filtering(self) + name, content = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering] + if callable(content): + return content(self) + return (name, content) def delete(self, *args, **kwargs): super(Mailbox, self).delete(*args, **kwargs) diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py index bc9a4392..a315719c 100644 --- a/orchestra/contrib/mailboxes/settings.py +++ b/orchestra/contrib/mailboxes/settings.py @@ -7,8 +7,8 @@ from orchestra.core.validators import validate_name from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting -_names = ('name', 'username') -_backend_names = _names + ('group', 'home') +_names = ('name', 'username',) +_backend_names = _names + ('user', 'group', 'home') MAILBOXES_DOMAIN_MODEL = Setting('MAILBOXES_DOMAIN_MODEL', 'domains.Domain', @@ -24,7 +24,9 @@ MAILBOXES_HOME = Setting('MAILBOXES_HOME', MAILBOXES_SIEVE_PATH = Setting('MAILBOXES_SIEVE_PATH', - os.path.join(MAILBOXES_HOME, 'Maildir/sieve/orchestra.sieve') + os.path.join(MAILBOXES_HOME, 'Maildir/sieve/orchestra.sieve'), + help_text="Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_backend_names)], ) diff --git a/orchestra/contrib/mailboxes/validators.py b/orchestra/contrib/mailboxes/validators.py index 6f63a1cf..88dfce7d 100644 --- a/orchestra/contrib/mailboxes/validators.py +++ b/orchestra/contrib/mailboxes/validators.py @@ -50,19 +50,18 @@ def validate_forward(value): def validate_sieve(value): - sieve_name = '%s.sieve' % hashlib.md5(value).hexdigest() + sieve_name = '%s.sieve' % hashlib.md5(value.encode('utf8')).hexdigest() path = os.path.join(settings.MAILBOXES_SIEVETEST_PATH, sieve_name) - with open(path, 'wb') as f: + with open(path, 'w') as f: f.write(value) context = { 'orchestra_root': paths.get_orchestra_dir() } sievetest = settings.MAILBOXES_SIEVETEST_BIN_PATH % context - try: - test = run(' '.join([sievetest, path, '/dev/null']), display=False) - except CommandError: + test = run(' '.join([sievetest, path, '/dev/null']), silent=True) + if test.return_code: errors = [] - for line in test.stderr.splitlines(): + for line in test.stderr.decode('utf8').splitlines(): error = re.match(r'^.*(line\s+[0-9]+:.*)', line) if error: errors += error.groups() diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py index 13d9de6f..02bb96f7 100644 --- a/orchestra/contrib/orchestration/backends.py +++ b/orchestra/contrib/orchestration/backends.py @@ -170,6 +170,9 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): else: self.cmd_section[-1][1].append(cmd) + def get_context(self, obj): + return {} + def prepare(self): """ hook for executing something at the beging diff --git a/orchestra/contrib/orchestration/tasks.py b/orchestra/contrib/orchestration/tasks.py index 92b9bd3f..6483182f 100644 --- a/orchestra/contrib/orchestration/tasks.py +++ b/orchestra/contrib/orchestration/tasks.py @@ -8,7 +8,7 @@ from .models import BackendLog @periodic_task(run_every=crontab(hour=7, minute=30, day_of_week=1)) -def backend_logs_cleanup(run_every=run_every): +def backend_logs_cleanup(): days = settings.ORCHESTRATION_BACKEND_CLEANUP_DAYS epoch = timezone.now()-timedelta(days=days) BackendLog.objects.filter(created_at__lt=epoch).delete() diff --git a/orchestra/contrib/orders/admin.py b/orchestra/contrib/orders/admin.py index 07667e97..593c07e0 100644 --- a/orchestra/contrib/orders/admin.py +++ b/orchestra/contrib/orders/admin.py @@ -51,7 +51,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): 'id', 'service_link', 'account_link', 'content_object_link', 'display_registered_on', 'display_billed_until', 'display_cancelled_on', 'display_metric' ) - list_filter = (ActiveOrderListFilter, BilledOrderListFilter, IgnoreOrderListFilter, 'service',) + list_filter = (ActiveOrderListFilter, IgnoreOrderListFilter, 'service', BilledOrderListFilter) default_changelist_filters = ( ('ignore', '0'), ) @@ -93,6 +93,22 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): return metric.value display_metric.short_description = _("Metric") +# def get_changelist(self, request, **kwargs): +# ChangeList = super(OrderAdmin, self).get_changelist(request, **kwargs) +# class OrderFilterChangeList(ChangeList): +# def get_filters(self, request): +# filters = super(OrderFilterChangeList, self).get_filters(request) +# tail = [] +# filters_copy = [] +# for list_filter in filters[0]: +# if getattr(list_filter, 'apply_last', False): +# tail.append(list_filter) +# else: +# filters_copy.append(list_filter) +# filters = ((filters_copy+tail),) + filters[1:] +# return filters +# return OrderFilterChangeList + class MetricStorageAdmin(admin.ModelAdmin): list_display = ('order', 'value', 'created_on', 'updated_on') diff --git a/orchestra/contrib/orders/filters.py b/orchestra/contrib/orders/filters.py index de81f5db..c9f7070c 100644 --- a/orchestra/contrib/orders/filters.py +++ b/orchestra/contrib/orders/filters.py @@ -1,9 +1,13 @@ +from datetime import timedelta, datetime + from django.contrib.admin import SimpleListFilter -from django.db.models import Q +from django.db.models import Q, Prefetch, F from django.utils import timezone from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from .models import MetricStorage, Order + class ActiveOrderListFilter(SimpleListFilter): """ Filter tickets by created_by according to request.user """ @@ -27,7 +31,8 @@ class ActiveOrderListFilter(SimpleListFilter): class BilledOrderListFilter(SimpleListFilter): """ Filter tickets by created_by according to request.user """ title = _("billed") - parameter_name = 'pending' + parameter_name = 'billed' +# apply_last = True def lookups(self, request, model_admin): return ( @@ -37,12 +42,33 @@ class BilledOrderListFilter(SimpleListFilter): def queryset(self, request, queryset): if self.value() == 'yes': - return queryset.filter(billed_until__isnull=False, - billed_until__gte=timezone.now()) + return queryset.filter(billed_until__isnull=False, billed_until__gte=timezone.now()) elif self.value() == 'no': + mindelta = timedelta(days=2) # TODO + metric_pks = [] + prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics', + queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'), + created_on__lte=(F('updated_on')-mindelta)) + ) + prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric', + queryset=MetricStorage.objects.filter(order__billed_on__isnull=False, + created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on')) + ) + metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True) + for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric): + if len(order.billed_metric) != 1: + raise ValueError("Data inconsistency.") + billed_metric = order.billed_metric[0].value + for metric in order.valid_metrics: + if metric.created_on <= order.billed_on: + raise ValueError("This value should already be filtered on the prefetch query.") + if metric.value > billed_metric: + metric_pks.append(order.pk) + break return queryset.filter( - Q(billed_until__isnull=True) | - Q(billed_until__lt=timezone.now()) + Q(pk__in=metric_pks) | Q( + Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now()) + ) ) return queryset diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py index e2bbb065..0a093250 100644 --- a/orchestra/contrib/webapps/types/php.py +++ b/orchestra/contrib/webapps/types/php.py @@ -80,29 +80,36 @@ class PHPApp(AppType): for webapp in webapps: if webapp.type_instance.get_php_version() == php_version: options += list(webapp.options.all()) - php_options = [option.name for option in self.get_php_options()] - enabled_functions = set() - for opt in options: - if opt.name in php_options: - if opt.name == 'enable_functions': - enabled_functions = enabled_functions.union(set(opt.value.split(','))) - else: - init_vars[opt.name] = opt.value + init_vars = OrderedDict((opt.name, opt.value) for opt in options) + # Enabled functions + enabled_functions = init_vars.pop('enabled_functions', None) if enabled_functions: + enabled_functions = set(enabled_functions.split(',')) disabled_functions = [] for function in self.PHP_DISABLED_FUNCTIONS: if function not in enabled_functions: disabled_functions.append(function) init_vars['disable_functions'] = ','.join(disabled_functions) + # process timeout timeout = self.instance.options.filter(name='timeout').first() if timeout: # Give a little slack here timeout = str(int(timeout.value)-2) init_vars['max_execution_time'] = timeout + # Custom error log if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars: context = self.get_directive_context() error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context) init_vars['error_log'] = error_log_path + # auto update max_post_size + if 'upload_max_filesize' in init_vars: + upload_max_filesize = init_vars['upload_max_filesize'] + post_max_size = init_vars.get('post_max_size', '0') + upload_max_filesize_value = eval(upload_max_filesize.replace('M', '*1024')) + post_max_size_value = eval(post_max_size.replace('M', '*1024')) + init_vars['post_max_size'] = post_max_size + if upload_max_filesize_value > post_max_size_value: + init_vars['post_max_size'] = upload_max_filesize return init_vars def get_directive_context(self): diff --git a/orchestra/utils/options.py b/orchestra/utils/options.py index 4e3609a5..65da0fcb 100644 --- a/orchestra/utils/options.py +++ b/orchestra/utils/options.py @@ -22,7 +22,7 @@ def send_email_template(template, context, to, email_from=None, html=None, attac if not 'site' in context: from orchestra import settings - url = urlparse.urlparse(settings.ORCHESTRA_SITE_URL) + url = urlparse(settings.ORCHESTRA_SITE_URL) context['site'] = { 'name': settings.ORCHESTRA_SITE_NAME, 'scheme': url.scheme,