From 81f5ef5686c1d1a38c3b5a0f4005a6c509939c15 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Fri, 12 Jun 2015 12:17:05 +0000 Subject: [PATCH] Added support for multiple php fpm webapps --- TODO.md | 3 ++ orchestra/contrib/domains/backends.py | 7 ++- orchestra/contrib/orchestration/helpers.py | 20 +++++++- orchestra/contrib/orchestration/manager.py | 4 +- orchestra/contrib/webapps/admin.py | 4 +- orchestra/contrib/webapps/backends/php.py | 52 +++++++++++++++------ orchestra/contrib/webapps/filters.py | 14 ++++++ orchestra/contrib/webapps/options.py | 6 +++ orchestra/contrib/webapps/settings.py | 7 ++- orchestra/contrib/webapps/types/php.py | 11 ++--- orchestra/management/commands/setupnginx.py | 2 +- 11 files changed, 100 insertions(+), 30 deletions(-) diff --git a/TODO.md b/TODO.md index 0178dca8..f840e6f0 100644 --- a/TODO.md +++ b/TODO.md @@ -427,3 +427,6 @@ serailzer self.instance on create. # IF modsecurity... and Merge websites locations +# backend email error log with links to instances +# PHP backend multiple FPM directories support + diff --git a/orchestra/contrib/domains/backends.py b/orchestra/contrib/domains/backends.py index e5d23466..f6f7789d 100644 --- a/orchestra/contrib/domains/backends.py +++ b/orchestra/contrib/domains/backends.py @@ -111,7 +111,7 @@ class Bind9MasterDomainBackend(ServiceController): from orchestra.contrib.orchestration.manager import router operation = Operation(backend, domain, Operation.SAVE) servers = [] - for routes in router.get_routes(operation): + for route in router.get_routes(operation): servers.append(route.host.get_ip()) return servers @@ -125,6 +125,7 @@ class Bind9MasterDomainBackend(ServiceController): ips = [] masters_ips = self.get_masters_ips(domain) records = domain.get_records() + # Slaves from NS for record in records.by_type(Record.NS): hostname = record.value.rstrip('.') # First try with a DNS query, a more reliable source @@ -141,6 +142,10 @@ class Bind9MasterDomainBackend(ServiceController): addr = records.by_type(Record.A)[0].value if addr not in masters_ips: ips.append(addr) + # Slaves from internal networks + if not settings.DOMAINS_MASTERS: + for server in self.get_servers(domain, Bind9SlaveDomainBackend): + ips.append(server) return OrderedSet(sorted(ips)) def get_context(self, domain): diff --git a/orchestra/contrib/orchestration/helpers.py b/orchestra/contrib/orchestration/helpers.py index e93748f1..8370c086 100644 --- a/orchestra/contrib/orchestration/helpers.py +++ b/orchestra/contrib/orchestration/helpers.py @@ -2,11 +2,14 @@ import textwrap from django.contrib import messages from django.core.mail import mail_admins -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse, NoReverseMatch from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import ungettext, ugettext_lazy as _ +from orchestra import settings as orchestra_settings +from orchestra.admin.utils import change_url + def get_backends_help_text(backends): help_texts = {} @@ -44,17 +47,29 @@ def get_backends_help_text(backends): return help_texts +def get_instance_url(operation): + try: + url = change_url(operation.instance) + except NoReverseMatch: + return _("Deleted {0}").format(operation.instance_repr or '-'.join( + (escape(operation.content_type), escape(operation.object_id)))) + return orchestra_settings.ORCHESTRA_SITE_URL + url + + def send_report(method, args, log): server = args[0] backend = method.__self__.__class__.__name__ subject = '[Orchestra] %s execution %s on %s' % (backend, log.state, server) separator = "\n%s\n\n" % ('~ '*40,) + print(log.operations.all()) + operations = '\n'.join([' '.join((op.action, get_instance_url(op))) for op in log.operations.all()]) message = separator.join([ "[EXIT CODE] %s" % log.exit_code, "[STDERR]\n%s" % log.stderr, "[STDOUT]\n%s" % log.stdout, "[SCRIPT]\n%s" % log.script, "[TRACEBACK]\n%s" % log.traceback, + "[OPERATIONS]\n%s" % operations, ]) html_message = '\n\n'.join([ '

Exit code %s

' % log.exit_code, @@ -66,6 +81,8 @@ def send_report(method, args, log): '
%s
' % escape(log.script), '

Traceback

' '
%s
' % escape(log.traceback), + '

Operations

' + '
%s
' % escape(operations), ]) mail_admins(subject, message, html_message=html_message) @@ -78,6 +95,7 @@ def get_backend_url(ids): return url + '?id__in=%s' % ','.join(map(str, ids)) return '' + def message_user(request, logs): total, successes, async = 0, 0, 0 ids = [] diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index cf66e212..613e03ca 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -27,8 +27,6 @@ def keep_log(execute, log, operations): log = kwargs['log'] try: log = execute(*args, **kwargs) - if not log.is_success: - send_report(execute, args, log) except Exception as e: trace = traceback.format_exc() log.state = log.EXCEPTION @@ -44,6 +42,8 @@ def keep_log(execute, log, operations): for operation in operations: logger.info("Executed %s" % str(operation)) operation.store(log) + if not log.is_success: + send_report(execute, args, log) stdout = log.stdout.strip() stdout and logger.debug('STDOUT %s', stdout) stderr = log.stderr.strip() diff --git a/orchestra/contrib/webapps/admin.py b/orchestra/contrib/webapps/admin.py index 7484790e..4fa6c1b9 100644 --- a/orchestra/contrib/webapps/admin.py +++ b/orchestra/contrib/webapps/admin.py @@ -10,7 +10,7 @@ from orchestra.contrib.accounts.admin import AccountAdminMixin from orchestra.forms.widgets import DynamicHelpTextSelect from orchestra.plugins.admin import SelectPluginAdminMixin -from .filters import HasWebsiteListFilter +from .filters import HasWebsiteListFilter, PHPVersionListFilter from .models import WebApp, WebAppOption from .options import AppOption from .types import AppType @@ -49,7 +49,7 @@ class WebAppOptionInline(admin.TabularInline): class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link') - list_filter = ('type', HasWebsiteListFilter) + list_filter = ('type', HasWebsiteListFilter, PHPVersionListFilter) inlines = [WebAppOptionInline] readonly_fields = ('account_link', ) change_readonly_fields = ('name', 'type', 'display_websites') diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py index b8586d59..3f6b49a5 100644 --- a/orchestra/contrib/webapps/backends/php.py +++ b/orchestra/contrib/webapps/backends/php.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration import ServiceController from . import WebAppServiceMixin -from .. import settings +from .. import settings, utils class PHPBackend(WebAppServiceMixin, ServiceController): @@ -36,10 +36,13 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.create_webapp_dir(context) if webapp.type_instance.is_fpm: self.save_fpm(webapp, context) - self.delete_fcgid(webapp, context) elif webapp.type_instance.is_fcgid: self.save_fcgid(webapp, context) - self.delete_fpm(webapp, context) + else: + raise TypeError("Unknown PHP execution type") + # Clean php fcgid/fpm apps in order to effectively support change of php-version + self.delete_fcgid(webapp, context, preserve=True) + self.delete_fpm(webapp, context, preserve=True) self.set_under_construction(context) def save_fpm(self, webapp, context): @@ -103,16 +106,39 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.delete_fcgid(webapp, context) self.delete_webapp_dir(context) - def delete_fpm(self, webapp, context): - # Better not delete a pool used by other apps - if not self.MERGE: - self.append("rm -f %(fpm_path)s" % context) + def has_sibilings(self, webapp, context): + return type(webapp).objects.filter( + account=webapp.account_id, + data__contains='"php_version":"%s"' % context['php_version'], + ).exclude(id=webapp.pk).exists() - def delete_fcgid(self, webapp, context): - # Better not delete a wrapper used by other apps - if not self.MERGE: - self.append("rm -f %(wrapper_path)s" % context) - self.append("rm -f %(cmd_options_path)s" % context) + def delete_fpm(self, webapp, context, preserve=False): + """ delete all pools in order to efectively support changing php-fpm version """ + context_copy = dict(context) + for php_version, verbose in settings.WEBAPPS_PHP_VERSIONS: + if preserve and php_version == context['php_version']: + continue + php_version_number = utils.extract_version_number(php_version) + context_copy['php_version_number'] = php_version_number + if not self.MERGE or not self.has_sibilings(webapp, context_copy): + context_copy['fpm_path'] = settings.WEBAPPS_PHPFPM_POOL_PATH % context_copy + self.append("rm -f %(fpm_path)s" % context_copy) + + def delete_fcgid(self, webapp, context, preserve=False): + """ delete all pools in order to efectively support changing php-fcgid version """ + context_copy = dict(context) + for php_version, verbose in settings.WEBAPPS_PHP_VERSIONS: + if preserve and php_version == context['php_version']: + continue + php_version_number = utils.extract_version_number(php_version) + context_copy['php_version_number'] = php_version_number + if not self.MERGE or not self.has_sibilings(webapp, context_copy): + context_copy.update({ + 'wrapper_path': settings.WEBAPPS_FCGID_WRAPPER_PATH % context_copy, + 'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context_copy, + }) + self.append("rm -f %(wrapper_path)s" % context_copy) + self.append("rm -f %(cmd_options_path)s" % context_copy) def prepare(self): super(PHPBackend, self).prepare() @@ -237,7 +263,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): return ' \\\n '.join(cmd_options) def update_fcgid_context(self, webapp, context): - wrapper_path = webapp.type_instance.FCGID_WRAPPER_PATH % context + wrapper_path = settings.WEBAPPS_FCGID_WRAPPER_PATH % context context.update({ 'wrapper': self.get_fcgid_wrapper(webapp, context), 'wrapper_path': wrapper_path, diff --git a/orchestra/contrib/webapps/filters.py b/orchestra/contrib/webapps/filters.py index d0b328d9..025f200f 100644 --- a/orchestra/contrib/webapps/filters.py +++ b/orchestra/contrib/webapps/filters.py @@ -1,6 +1,8 @@ from django.contrib.admin import SimpleListFilter from django.utils.translation import ugettext_lazy as _ +from . import settings + class HasWebsiteListFilter(SimpleListFilter): title = _("Has website") @@ -20,3 +22,15 @@ class HasWebsiteListFilter(SimpleListFilter): return queryset +class PHPVersionListFilter(SimpleListFilter): + title = _("PHP version") + parameter_name = 'php_version' + + def lookups(self, request, model_admin): + return settings.WEBAPPS_PHP_VERSIONS + + def queryset(self, request, queryset): + value = self.value() + if value: + return queryset.filter(data__contains='"php_version":"%s"' % value) + return queryset diff --git a/orchestra/contrib/webapps/options.py b/orchestra/contrib/webapps/options.py index 85165d96..c132d372 100644 --- a/orchestra/contrib/webapps/options.py +++ b/orchestra/contrib/webapps/options.py @@ -159,6 +159,12 @@ class PHPExtension(PHPAppOption): regex = r'^[^ ]+$' +class PHPIncludePath(PHPAppOption): + name = 'include_path' + verbose_name = _("Include path") + regex = r'^[^ ]+$' + + class PHPMagicQuotesGPC(PHPAppOption): name = 'magic_quotes_gpc' verbose_name = _("Magic quotes GPC") diff --git a/orchestra/contrib/webapps/settings.py b/orchestra/contrib/webapps/settings.py index 707866da..5f673f9b 100644 --- a/orchestra/contrib/webapps/settings.py +++ b/orchestra/contrib/webapps/settings.py @@ -30,7 +30,7 @@ WEBAPPS_FPM_DEFAULT_MAX_CHILDREN = Setting('WEBAPPS_FPM_DEFAULT_MAX_CHILDREN', WEBAPPS_PHPFPM_POOL_PATH = Setting('WEBAPPS_PHPFPM_POOL_PATH', - '/etc/php5/fpm/pool.d/%(user)s-%(app_name)s.conf', + '/etc/php%(php_version_number)s/fpm/pool.d/%(user)s-%(app_name)s.conf', help_text="Available fromat names: %s" % ', '.join(_php_names), validators=[Setting.string_format_validator(_php_names)], ) @@ -84,6 +84,8 @@ WEBAPPS_TYPES = Setting('WEBAPPS_TYPES', ( WEBAPPS_PHP_VERSIONS = Setting('WEBAPPS_PHP_VERSIONS', ( + ('5.6-fpm', 'PHP 5.6 FPM'), + ('5.6-cgi', 'PHP 5.6 FCGID'), ('5.4-fpm', 'PHP 5.4 FPM'), ('5.4-cgi', 'PHP 5.4 FCGID'), ('5.3-cgi', 'PHP 5.3 FCGID'), @@ -96,7 +98,7 @@ WEBAPPS_PHP_VERSIONS = Setting('WEBAPPS_PHP_VERSIONS', ( WEBAPPS_DEFAULT_PHP_VERSION = Setting('WEBAPPS_DEFAULT_PHP_VERSION', - '5.4-cgi', + '5.6-fpm', choices=WEBAPPS_PHP_VERSIONS ) @@ -223,6 +225,7 @@ WEBAPPS_ENABLED_OPTIONS = Setting('WEBAPPS_ENABLED_OPTIONS', ( 'orchestra.contrib.webapps.options.PHPDefaultSocketTimeout', 'orchestra.contrib.webapps.options.PHPDisplayErrors', 'orchestra.contrib.webapps.options.PHPExtension', + 'orchestra.contrib.webapps.options.PHPIncludePath', 'orchestra.contrib.webapps.options.PHPMagicQuotesGPC', 'orchestra.contrib.webapps.options.PHPMagicQuotesRuntime', 'orchestra.contrib.webapps.options.PHPMaginQuotesSybase', diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py index b22340c5..c86a672b 100644 --- a/orchestra/contrib/webapps/types/php.py +++ b/orchestra/contrib/webapps/types/php.py @@ -1,5 +1,4 @@ import os -import re from collections import OrderedDict from django import forms @@ -9,7 +8,7 @@ from rest_framework import serializers from orchestra.plugins.forms import PluginDataForm from orchestra.utils.functional import cached -from .. import settings +from .. import settings, utils from ..options import AppOption from . import AppType @@ -146,9 +145,5 @@ class PHPApp(AppType): def get_php_version_number(self): php_version = self.get_php_version() - number = re.findall(r'[0-9]+\.?[0-9]?', php_version) - if not number: - raise ValueError("No version number matches for '%s'" % php_version) - if len(number) > 1: - raise ValueError("Multiple version number matches for '%s'" % php_version) - return number[0] + return utils.extract_version_number(php_version) + diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index eceed05b..fd910614 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -152,7 +152,7 @@ class Command(BaseCommand): 'project_dir': paths.get_project_dir(), 'site_dir': paths.get_site_dir(), 'static_root': settings.STATIC_ROOT, - 'static_url': (settings.STATIC_URL or '/static').rstrip('/') + 'static_url': (settings.STATIC_URL or '/static').rstrip('/'), 'user': user, 'group': options.get('group') or user, 'home': expanduser("~%s" % options.get('user')),