From f6e79fd1611efdbe442961725c88536a7c538cfa Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Mon, 15 Feb 2016 13:08:49 +0000 Subject: [PATCH] Added support for retrying backend executions --- TODO.md | 5 -- orchestra/contrib/mailboxes/forms.py | 4 +- orchestra/contrib/orchestration/__init__.py | 12 ++++ orchestra/contrib/orchestration/actions.py | 63 +++++++++++++++++++ orchestra/contrib/orchestration/admin.py | 7 ++- orchestra/contrib/orchestration/methods.py | 7 ++- orchestra/contrib/orchestration/models.py | 2 +- .../admin/orchestration/backends/retry.html | 9 +++ orchestra/contrib/systemusers/actions.py | 2 +- orchestra/contrib/webapps/types/php.py | 9 ++- 10 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 orchestra/contrib/orchestration/actions.py create mode 100644 orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html diff --git a/TODO.md b/TODO.md index b783c23c..700d5139 100644 --- a/TODO.md +++ b/TODO.md @@ -457,18 +457,13 @@ mkhomedir_helper or create ssh homes with bash.rc and such * setuppostgres use porject_name for db name and user instead of orchestra - - # POSTFIX web traffic monitor '": uid=" from=<%(user)s>' - # Mv .deleted make sure it works with nested destinations -# Re-run backends (save regenerate, delete run same script) warning on confirmation page: DELETED objects will be deleted on the server if you have recreated them. # Automatically re-run backends until success? only timedout executions? - ### Quick start 0. Install orchestra following any of these methods: 1. [PIP-only, Fast deployment setup (demo)](README.md#fast-deployment-setup) diff --git a/orchestra/contrib/mailboxes/forms.py b/orchestra/contrib/mailboxes/forms.py index ae2dd213..1675af03 100644 --- a/orchestra/contrib/mailboxes/forms.py +++ b/orchestra/contrib/mailboxes/forms.py @@ -49,7 +49,7 @@ class MailboxForm(forms.ModelForm): name = self.cleaned_data['name'] max_length = settings.MAILBOXES_NAME_MAX_LENGTH if len(name) > max_length: - raise ValidationError("Name length should be less than %i" % max_length) + raise ValidationError("Name length should be less than %i." % max_length) return name @@ -61,7 +61,7 @@ class MailboxCreationForm(UserCreationForm, MailboxForm): def clean_name(self): # Since model.clean() will check this, this is redundant, # but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth - name = self.cleaned_data["name"] + name = super().clean_name() try: self._meta.model._default_manager.get(name=name) except self._meta.model.DoesNotExist: diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py index 76880092..b5aded20 100644 --- a/orchestra/contrib/orchestration/__init__.py +++ b/orchestra/contrib/orchestration/__init__.py @@ -1,6 +1,8 @@ import collections import copy +from orchestra.utils.python import AttrDict + from .backends import ServiceBackend, ServiceController, replace @@ -77,3 +79,13 @@ class Operation(): instance=self.instance, action=self.action, ) + + @classmethod + def load(cls, operation, log=None): + routes = None + if log: + routes = { + (operation.backend, operation.action): AttrDict(host=log.server) + } + return cls(operation.backend_class, operation.instance, operation.action, routes=routes) + diff --git a/orchestra/contrib/orchestration/actions.py b/orchestra/contrib/orchestration/actions.py new file mode 100644 index 00000000..cb94b7ce --- /dev/null +++ b/orchestra/contrib/orchestration/actions.py @@ -0,0 +1,63 @@ +from django.contrib import messages +from django.contrib.admin import helpers +from django.shortcuts import render +from django.utils.safestring import mark_safe +from django.utils.translation import ungettext, ugettext_lazy as _ + +from orchestra.admin.utils import get_object_from_url, change_url +from orchestra.contrib.orchestration.helpers import message_user + +from . import Operation +from .models import BackendOperation + + +def retry_backend(modeladmin, request, queryset): + if request.POST.get('post') == 'generic_confirmation': + operations = [] + for log in queryset.prefetch_related('operations__instance'): + for operation in log.operations.all(): + if operation.instance: + op = Operation.load(operation) + operations.append(op) + if not operations: + messages.warning(request, _("No backend operation has been executed.")) + else: + logs = Operation.execute(operations) + message_user(request, logs) + Operation.execute(operations) + return + opts = modeladmin.model._meta + display_objects = [] + deleted_objects = [] + related_operations = queryset.values_list('operations__id', flat=True).distinct() + related_operations = BackendOperation.objects.filter(pk__in=related_operations) + for op in related_operations.select_related('log__server').prefetch_related('instance'): + if not op.instance: + deleted_objects.append(op) + else: + context = { + 'backend': op.log.backend, + 'action': op.action, + 'instance': op.instance, + 'instance_url': change_url(op.instance), + 'server': op.log.server, + 'server_url': change_url(op.log.server), + } + display_objects.append(mark_safe( + '%(backend)s.%(action)s(%(instance)s) @ %(server)s' % context + )) + context = { + 'title': _("Are you sure to execute the following backends?"), + 'action_name': _('Retry backend'), + 'action_value': 'retry_backend', + 'display_objects': display_objects, + 'deleted_objects': deleted_objects, + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/orchestration/backends/retry.html', context) +retry_backend.short_description = _("Retry") +retry_backend.url_name = 'retry' diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index 000dadf9..80ff6213 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -3,11 +3,12 @@ from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ -from orchestra.admin import ExtendedModelAdmin +from orchestra.admin import ExtendedModelAdmin, ChangeViewActionsMixin from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono, display_code from orchestra.plugins.admin import display_plugin_field from . import settings, helpers +from .actions import retry_backend from .backends import ServiceBackend from .forms import RouteForm from .models import Server, Route, BackendLog, BackendOperation @@ -128,7 +129,7 @@ class BackendOperationInline(admin.TabularInline): return queryset.prefetch_related('instance') -class BackendLogAdmin(admin.ModelAdmin): +class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin): list_display = ( 'id', 'backend', 'server_link', 'display_state', 'exit_code', 'display_created', 'execution_time', @@ -144,6 +145,8 @@ class BackendLogAdmin(admin.ModelAdmin): 'execution_time' ) readonly_fields = fields + actions = (retry_backend,) + change_view_actions = actions server_link = admin_link('server') display_created = admin_date('created_at', short_description=_("Created")) diff --git a/orchestra/contrib/orchestration/methods.py b/orchestra/contrib/orchestration/methods.py index 8596b03c..fde43b4b 100644 --- a/orchestra/contrib/orchestration/methods.py +++ b/orchestra/contrib/orchestration/methods.py @@ -149,9 +149,14 @@ def SSH(*args, **kwargs): def Python(backend, log, server, cmds, async=False): script = '' + functions = set() + for cmd in cmds: + if cmd.func not in functions: + functions.add(cmd.func) + script += textwrap.dedent(''.join(inspect.getsourcelines(cmd.func)[0])) + script += '\n' for cmd in cmds: script += '# %s %s\n' % (cmd.func.__name__, cmd.args) - script += textwrap.dedent(''.join(inspect.getsourcelines(cmd.func)[0])) log.state = log.STARTED log.script = '\n'.join((log.script, script)) log.save(update_fields=('script', 'state', 'updated_at')) diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index bcd45464..c99cbd52 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -150,7 +150,7 @@ class BackendOperation(models.Model): verbose_name_plural = _("Operations") def __str__(self): - return '%s.%s(%s)' % (self.backend, self.action, self.instance) + return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.instance_repr) @cached_property def backend_class(self): diff --git a/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html b/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html new file mode 100644 index 00000000..65f2147b --- /dev/null +++ b/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html @@ -0,0 +1,9 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} + + +{% block form %} + {% if deleted_objects %} +

The following operations refere to deleted objects and will not be executed

+ + {% endif %} +{% endblock %} diff --git a/orchestra/contrib/systemusers/actions.py b/orchestra/contrib/systemusers/actions.py index 24194ae1..f896e797 100644 --- a/orchestra/contrib/systemusers/actions.py +++ b/orchestra/contrib/systemusers/actions.py @@ -53,7 +53,7 @@ def set_permission(modeladmin, request, queryset): msg = _("%(action)s %(perms)s permission to %(to)s") % context modeladmin.log_change(request, user, msg) if not operations: - messages.error(request, "No backend operation has been executed.") + messages.error(request, _("No backend operation has been executed.")) else: logs = Operation.execute(operations) helpers.message_user(request, logs) diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py index fdbc279b..3ab2c4df 100644 --- a/orchestra/contrib/webapps/types/php.py +++ b/orchestra/contrib/webapps/types/php.py @@ -7,6 +7,7 @@ from rest_framework import serializers from orchestra.plugins.forms import PluginDataForm from orchestra.utils.functional import cached +from orchestra.utils.python import OrderedSet from .. import settings, utils from ..options import AppOption @@ -89,12 +90,13 @@ class PHPApp(AppType): init_vars[name] = value # Disable functions if self.PHP_DISABLED_FUNCTIONS: - enable_functions = init_vars.pop('enable_functions', '') - disable_functions = set(init_vars.pop('disable_functions', '').split(',')) + enable_functions = init_vars.pop('enable_functions', None) + enable_functions = OrderedSet(enable_functions.split(',') if enable_functions else ()) + disable_functions = init_vars.pop('disable_functions', None) + disable_functions = OrderedSet(disable_functions.split(',') if disable_functions else ()) if disable_functions or enable_functions or self.is_fpm: # FPM: Defining 'disable_functions' or 'disable_classes' will not overwrite previously # defined php.ini values, but will append the new value - enable_functions = set(enable_functions.split(',')) for function in self.PHP_DISABLED_FUNCTIONS: if function not in enable_functions: disable_functions.add(function) @@ -119,6 +121,7 @@ class PHPApp(AppType): init_vars['post_max_size'] = post_max_size if upload_max_filesize_value > post_max_size_value: init_vars['post_max_size'] = upload_max_filesize + print(init_vars) return init_vars def get_directive_context(self):