Improvements on databases, webapps and websites

TODO.md

@@ -138,3 +138,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 * DN: Transaction atomicity and backend failure
 
 * SaaS Icons
+
+* offer to create mailbox on account creation
+
+* init.d celery scripts
+    -# Required-Start:    $network $local_fs $remote_fs postgresql celeryd
+    -# Required-Stop:     $network $local_fs $remote_fs postgresql celeryd
+
+
+* POST only fields (account, username, name) etc

orchestra/apps/accounts/models.py

@@ -72,7 +72,8 @@ class Account(auth.AbstractBaseUser):
     
     def disable(self):
         self.is_active = False
-#        self.save(update_fields=['is_active'])
+        self.save(update_fields=['is_active'])
+        # Trigger save() on related objects that depend on this account
         for rel in self._meta.get_all_related_objects():
             if not rel.model in services:
                 continue

orchestra/apps/databases/admin.py

@@ -4,68 +4,24 @@ from django.contrib.auth.admin import UserAdmin
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.admin import ExtendedModelAdmin
+from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
 from orchestra.admin.utils import admin_link
 from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin
 
-from .forms import (DatabaseUserChangeForm, DatabaseUserCreationForm,
+from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm
-        DatabaseCreationForm)
+from .models import Database, DatabaseUser
-from .models import Database, Role, DatabaseUser
-
-
-class UserInline(admin.TabularInline):
-    model = Role
-    verbose_name_plural = _("Users")
-    readonly_fields = ('user_link',)
-    extra = 0
-    
-    user_link = admin_link('user')
-    
-    def formfield_for_dbfield(self, db_field, **kwargs):
-        """ Make value input widget bigger """
-        if db_field.name == 'user':
-            users = db_field.rel.to.objects.filter(type=self.parent_object.type)
-            kwargs['queryset'] = users.filter(account=self.account)
-        return super(UserInline, self).formfield_for_dbfield(db_field, **kwargs)
-
-
-class PermissionInline(AccountAdminMixin, admin.TabularInline):
-    model = Role
-    verbose_name_plural = _("Permissions")
-    readonly_fields = ('database_link',)
-    extra = 0
-    filter_by_account_fields = ['database']
-    
-    database_link = admin_link('database', popup=True)
-    
-    def formfield_for_dbfield(self, db_field, **kwargs):
-        """ Make value input widget bigger """
-        formfield = super(PermissionInline, self).formfield_for_dbfield(db_field, **kwargs)
-        if db_field.name == 'database':
-            # Hack widget render in order to append ?type='db_type' to the add url
-            db_type = self.parent_object.type
-            old_render = formfield.widget.render
-            def render(*args, **kwargs):
-                output = old_render(*args, **kwargs)
-                output = output.replace('/add/?', '/add/?type=%s&' % db_type)
-                return mark_safe(output)
-            formfield.widget.render = render
-            formfield.queryset = formfield.queryset.filter(type=db_type)
-        return formfield
 
 
 class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
     list_display = ('name', 'type', 'account_link')
     list_filter = ('type',)
     search_fields = ['name']
-    inlines = [UserInline]
-    add_inlines = []
     change_readonly_fields = ('name', 'type')
     extra = 1
     fieldsets = (
         (None, {
             'classes': ('extrapretty',),
-            'fields': ('account_link', 'name', 'type'),
+            'fields': ('account_link', 'name', 'type', 'users'),
         }),
     )
     add_fieldsets = (
@@ -92,22 +48,20 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
                 user = DatabaseUser(
                     username=form.cleaned_data['username'],
                     type=obj.type,
-                    account_id = obj.account.pk,
+                    account_id=obj.account.pk,
                 )
                 user.set_password(form.cleaned_data["password1"])
             user.save()
-            Role.objects.create(database=obj, user=user, is_owner=True)
+            obj.users.add(user)
 
 
-class DatabaseUserAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
+class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin):
     list_display = ('username', 'type', 'account_link')
     list_filter = ('type',)
     search_fields = ['username']
     form = DatabaseUserChangeForm
     add_form = DatabaseUserCreationForm
     change_readonly_fields = ('username', 'type')
-    inlines = [PermissionInline]
-    add_inlines = []
     fieldsets = (
         (None, {
             'classes': ('extrapretty',),

orchestra/apps/databases/backends.py

@@ -1,3 +1,5 @@
+import textwrap
+
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.apps.orchestration import ServiceController
@@ -14,12 +16,12 @@ class MySQLBackend(ServiceController):
         if database.type == database.MYSQL:
             context = self.get_context(database)
             self.append(
-                "mysql -e 'CREATE DATABASE `%(database)s`;'" % context
+                "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
-            )
-            self.append(
-                "mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
-                "  TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
             )
+            self.append(textwrap.dedent("""\
+                mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(owner)s"@"%(host)s" WITH GRANT OPTION;' \
+                """ % context
+            ))
     
     def delete(self, database):
         if database.type == database.MYSQL:
@@ -44,20 +46,25 @@ class MySQLUserBackend(ServiceController):
     def save(self, user):
         if user.type == user.MYSQL:
             context = self.get_context(user)
-            self.append(
+            self.append(textwrap.dedent("""\
-                "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context
+                mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \
-            )
+                """ % context
-            self.append(
+            ))
-                "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
+            self.append(textwrap.dedent("""\
-                "  WHERE User=\"%(username)s\";'" % context
+                mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \
-            )
+                """ % context
+            ))
     
     def delete(self, user):
         if user.type == user.MYSQL:
             context = self.get_context(database)
-            self.append(
+            self.append(textwrap.dedent("""\
-                "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
+                mysql -e 'DROP USER "%(username)s"@"%(host)s";' \
-            )
+                """ % context
+            ))
+    
+    def commit(self):
+        self.append("mysql -e 'FLUSH PRIVILEGES;'")
     
     def get_context(self, user):
         return {
@@ -78,27 +85,28 @@ class MysqlDisk(ServiceMonitor):
     
     def exceeded(self, db):
         context = self.get_context(db)
-        self.append("mysql -e '"
+        self.append(textwrap.dedent("""\
-            "UPDATE db SET Insert_priv=\"N\", Create_priv=\"N\""
+            mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";' \
-            "   WHERE Db=\"%(db_name)s\";'" % context
+            """ % context
-        )
+        ))
     
     def recovery(self, db):
         context = self.get_context(db)
-        self.append("mysql -e '"
+        self.append(textwrap.dedent("""\
-            "UPDATE db SET Insert_priv=\"Y\", Create_priv=\"Y\""
+            mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";' \
-            "   WHERE Db=\"%(db_name)s\";'" % context
+            """ % context
-        )
+        ))
     
     def monitor(self, db):
         context = self.get_context(db)
-        self.append(
+        self.append(textwrap.dedent("""\
-            "echo %(db_id)s $(mysql -B -e '"
+            echo %(db_id)s $(mysql -B -e '"
-            "   SELECT sum( data_length + index_length ) \"Size\"\n"
+               SELECT sum( data_length + index_length ) "Size"
-            "      FROM information_schema.TABLES\n"
+                  FROM information_schema.TABLES
-            "      WHERE table_schema=\"gisp\"\n"
+                  WHERE table_schema = "gisp"
-            "      GROUP BY table_schema;' | tail -n 1)" % context
+                  GROUP BY table_schema;' | tail -n 1) \
-        )
+            """ % context
+        ))
     
     def get_context(self, db):
         return {

orchestra/apps/databases/forms.py

@@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.core.validators import validate_password
 
-from .models import DatabaseUser, Database, Role
+from .models import DatabaseUser, Database
 
 
 class DatabaseUserCreationForm(forms.ModelForm):
@@ -28,13 +28,6 @@ class DatabaseUserCreationForm(forms.ModelForm):
             raise forms.ValidationError(msg)
         return password2
 
-    def save(self, commit=True):
-        user = super(DatabaseUserCreationForm, self).save(commit=False)
-#        user.set_password(self.cleaned_data["password1"])
-#        if commit:
-#            user.save()
-        return user
-
 
 class DatabaseCreationForm(DatabaseUserCreationForm):
     username = forms.RegexField(label=_("Username"), max_length=30,
@@ -87,20 +80,6 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
                 raise forms.ValidationError(msg)
         return cleaned_data
 
-    def save(self, commit=True):
-        db = super(DatabaseUserCreationForm, self).save(commit=False)
-#        if commit:
-#            user = self.cleaned_data['user']
-#            if not user:
-#                user = DatabaseUser(
-#                    username=self.cleaned_data['username'],
-#                    type=self.cleaned_data['type'],
-#                )
-#                user.set_password(self.cleaned_data["password1"])
-#            user.save()
-#            role, __ = Role.objects.get_or_create(database=db, user=user)
-        return db
-
 
 class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField):
     class ReadOnlyPasswordHashWidget(forms.Widget):

orchestra/apps/databases/models.py

@@ -16,8 +16,8 @@ class Database(models.Model):
     name = models.CharField(_("name"), max_length=64, # MySQL limit
             validators=[validators.validate_name])
     users = models.ManyToManyField('databases.DatabaseUser',
-            verbose_name=_("users"),
+            verbose_name=_("users"),related_name='databases')
-           through='databases.Role', related_name='users')
+#           through='databases.Role', 
     type = models.CharField(_("type"), max_length=32,
             choices=settings.DATABASES_TYPE_CHOICES,
             default=settings.DATABASES_DEFAULT_TYPE)
@@ -32,29 +32,38 @@ class Database(models.Model):
     
     @property
     def owner(self):
-        return self.roles.get(is_owner=True).user
+        """ database owner is the first user related to it """
+        # Accessing intermediary model to get which is the first user
+        users = Database.users.through.objects.filter(database_id=self.id)
+        return users.order_by('-id').first().databaseuser
 
 
-class Role(models.Model):
+Database.users.through._meta.unique_together = (('database', 'databaseuser'),)
-    database = models.ForeignKey(Database, verbose_name=_("database"),
-            related_name='roles')
-    user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
-            related_name='roles')
-    is_owner = models.BooleanField(_("owner"), default=False)
 
-    class Meta:
+#class Role(models.Model):
-        unique_together = ('database', 'user')
+#    database = models.ForeignKey(Database, verbose_name=_("database"),
-    
+#            related_name='roles')
-    def __unicode__(self):
+#    user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
-        return "%s@%s" % (self.user, self.database)
+#            related_name='roles')
-    
+##    is_owner = models.BooleanField(_("owner"), default=False)
-    def clean(self):
+#    
-        if self.user.type != self.database.type:
+#    class Meta:
-            msg = _("Database and user type doesn't match")
+#        unique_together = ('database', 'user')
-            raise validators.ValidationError(msg)
+#    
-        roles = self.database.roles.values('id')
+#    def __unicode__(self):
-        if not roles or (len(roles) == 1 and roles[0].id == self.id):
+#        return "%s@%s" % (self.user, self.database)
-            self.is_owner = True
+#    
+#    @property
+#    def is_owner(self):
+#        return datatase.owner == self
+#    
+#    def clean(self):
+#        if self.user.type != self.database.type:
+#            msg = _("Database and user type doesn't match")
+#            raise validators.ValidationError(msg)
+#        roles = self.database.roles.values('id')
+#        if not roles or (len(roles) == 1 and roles[0].id == self.id):
+#            self.is_owner = True
 
 
 class DatabaseUser(models.Model):

orchestra/apps/databases/serializers.py

@@ -5,36 +5,47 @@ from rest_framework import serializers
 from orchestra.apps.accounts.serializers import AccountSerializerMixin
 from orchestra.core.validators import validate_password
 
-from .models import Database, DatabaseUser, Role
+from .models import Database, DatabaseUser
 
 
-class UserRoleSerializer(serializers.HyperlinkedModelSerializer):
+class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
-        model = Role
+        model = DatabaseUser
-        fields = ('user', 'is_owner',)
+        fields = ('url', 'username')
     
-
+    def from_native(self, data, files=None):
-class RoleSerializer(serializers.HyperlinkedModelSerializer):
+        return DatabaseUser.objects.get(username=data['username'])
-    class Meta:
-        model = Role
-        fields = ('database', 'is_owner',)
 
 
 class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
-    roles = UserRoleSerializer(many=True)
+    users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
     
     class Meta:
         model = Database
-        fields = ('url', 'name', 'type', 'roles')
+        fields = ('url', 'name', 'type', 'users')
+
+
+class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Database
+        fields = ('url', 'name',)
+    
+    def from_native(self, data, files=None):
+        return Database.objects.get(name=data['name'])
 
 
 class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
     password = serializers.CharField(max_length=128, label=_('Password'),
             validators=[validate_password], write_only=True,
             widget=widgets.PasswordInput)
-    roles = RoleSerializer(many=True, read_only=True)
+    databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
     
     class Meta:
         model = DatabaseUser
-        fields = ('url', 'username', 'password', 'type', 'roles')
+        fields = ('url', 'username', 'password', 'type', 'databases')
-        write_only_fields = ('username',)
+    
+    def save_object(self, obj, **kwargs):
+        # FIXME this method will be called when saving nested serializers :(
+        if not obj.pk:
+            obj.set_password(obj.password)
+        super(DatabaseUserSerializer, self).save_object(obj, **kwargs)

orchestra/apps/databases/tests/functional_tests/tests.py

@@ -1,5 +1,6 @@
 import MySQLdb
 import os
+import socket
 import time
 from functools import partial
 
@@ -58,13 +59,28 @@ class DatabaseTestMixin(object):
         self.add(dbname, username, password)
         self.validate_create_table(dbname, username, password)
     
+    def test_change_password(self):
+        dbname = '%s_database' % random_ascii(5)
+        username = '%s_dbuser' % random_ascii(5)
+        password = '@!?%spppP001' % random_ascii(5)
+        self.add(dbname, username, password)
+        self.validate_create_table(dbname, username, password)
+        new_password = '@!?%spppP001' % random_ascii(5)
+        self.change_password(username, new_password)
+        self.validate_login_error(dbname, username, password)
+        self.validate_create_table(dbname, username, new_password)
+
 
 class MySQLBackendMixin(object):
     db_type = 'mysql'
     
     def setUp(self):
         super(MySQLBackendMixin, self).setUp()
-        settings.DATABASES_DEFAULT_HOST = '10.228.207.207'
+        # Get local ip address used to reach self.MASTER_SERVER
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect((self.MASTER_SERVER, 22))
+        settings.DATABASES_DEFAULT_HOST = s.getsockname()[0]
+        s.close()
     
     def add_route(self):
         server = Server.objects.create(name=self.MASTER_SERVER)
@@ -78,7 +94,11 @@ class MySQLBackendMixin(object):
     def validate_create_table(self, name, username, password):
         db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
         cur = db.cursor()
-        cur.execute('CREATE TABLE test ( id INT ) ;')
+        cur.execute('CREATE TABLE %s ( id INT ) ;' % random_ascii(20))
+    
+    def validate_login_error(self, dbname, username, password):
+        self.assertRaises(MySQLdb.OperationalError,
+            self.validate_create_table, dbname, username, password)
     
     def validate_delete(self, name, username, password):
         self.asseRaises(MySQLdb.ConnectionError,
@@ -92,9 +112,16 @@ class RESTDatabaseMixin(DatabaseTestMixin):
     
     @save_response_on_error
     def add(self, dbname, username, password):
-        user = self.rest.databaseusers.create(username=username, password=password)
+        user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type)
-        # TODO fucking nested objects
+        users = [{
-        self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type)
+            'username': user.username
+        }]
+        self.rest.databases.create(name=dbname, users=users, type=self.db_type)
+    
+    @save_response_on_error
+    def change_password(self, username, password):
+        user = self.rest.databaseusers.retrieve(username=username).get()
+        user.set_password(password)
 
 
 class AdminDatabaseMixin(DatabaseTestMixin):
@@ -135,6 +162,11 @@ class AdminDatabaseMixin(DatabaseTestMixin):
         user = DatabaseUser.objects.get(username=username)
         self.admin_delete(user)
     
+    @snapshot_on_error
+    def change_password(self, username, password):
+        user = DatabaseUser.objects.get(username=username)
+        self.admin_change_password(user, password)
+
 
 class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase):
     pass

orchestra/apps/orchestration/methods.py

@@ -37,7 +37,8 @@ def BashSSH(backend, log, server, cmds):
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         addr = server.get_address()
         try:
-            ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH)
+            # TODO timeout
+            ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10)
         except socket.error:
             logger.error('%s timed out on %s' % (backend, server))
             log.state = BackendLog.TIMEOUT

orchestra/apps/webapps/backends/__init__.py

@@ -1,6 +1,7 @@
 import pkgutil
 import textwrap
 
+
 class WebAppServiceMixin(object):
     model = 'webapps.WebApp'
     
@@ -16,6 +17,30 @@ class WebAppServiceMixin(object):
             done
         """ % context))
     
+    def get_php_init_vars(self, webapp, per_account=False):
+        """
+        process php options for inclusion on php.ini
+        per_account=True merges all (account, webapp.type) options
+        """
+        init_vars = []
+        options = webapp.options.all()
+        if per_account:
+            options = webapp.account.webapps.filter(webapp_type=webapp.type)
+        for opt in options:
+            name = opt.name.replace('PHP-', '')
+            value = "%s" % opt.value
+            init_vars.append((name, value))
+        enabled_functions = []
+        for value in options.filter(name='enabled_functions').values_list('value', flat=True):
+            enabled_functions += enabled_functions.get().value.split(',')
+        if enabled_functions:
+            disabled_functions = []
+            for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
+                if function not in enabled_functions:
+                    disabled_functions.append(function)
+            init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
+        return init_vars
+    
     def delete_webapp_dir(self, context):
         self.append("rm -fr %(app_path)s" % context)
     
@@ -30,5 +55,7 @@ class WebAppServiceMixin(object):
         }
 
 
+
 for __, module_name, __ in pkgutil.walk_packages(__path__):
+    # sorry for the exec(), but Import module function fails :(
     exec('from . import %s' % module_name)

orchestra/apps/webapps/backends/phpfcgid.py

@@ -10,6 +10,7 @@ from .. import settings
 
 
 class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
+    """ Per-webapp fcgid application """
     verbose_name = _("PHP-Fcgid")
     
     def save(self, webapp):
@@ -27,6 +28,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
     
     def delete(self, webapp):
         context = self.get_context(webapp)
+        self.append("rm '%(wrapper_path)s'" % context)
         self.delete_webapp_dir(context)
     
     def commit(self):
@@ -35,7 +37,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
     
     def get_context(self, webapp):
         context = super(PHPFcgidBackend, self).get_context(webapp)
-        init_vars = webapp.get_php_init_vars()
+        init_vars = self.get_php_init_vars(webapp)
         if init_vars:
             init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ]
             init_vars = ', -d '.join(init_vars)

orchestra/apps/webapps/backends/phpfpm.py

@@ -1,4 +1,5 @@
 import os
+import textwrap
 
 from django.template import Template, Context
 from django.utils.translation import ugettext_lazy as _
@@ -10,49 +11,57 @@ from .. import settings
 
 
 class PHPFPMBackend(WebAppServiceMixin, ServiceController):
+    """ Per-webapp php application """
     verbose_name = _("PHP-FPM")
     
     def save(self, webapp):
         context = self.get_context(webapp)
         self.create_webapp_dir(context)
-        self.append(
+        self.append(textwrap.dedent("""\
-            "{ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - ; } ||"
+            {
-            "  { echo -e '%(fpm_config)s' > %(fpm_path)s; UPDATEDFPM=1; }" % context
+                echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s -
-        )
+            } || {
+                echo -e '%(fpm_config)s' > %(fpm_path)s
+                UPDATEDFPM=1
+            }""" % context))
     
     def delete(self, webapp):
         context = self.get_context(webapp)
+        self.append("rm '%(fpm_config)s'" % context)
         self.delete_webapp_dir(context)
     
     def commit(self):
         super(PHPFPMBackend, self).commit()
-        self.append('[[ $UPDATEDFPM == 1 ]] && service php5-fpm reload')
+        self.append(textwrap.dedent("""
+            [[ $UPDATEDFPM == 1 ]] && {
+                service php5-fpm start
+                service php5-fpm reload
+            }"""))
     
     def get_context(self, webapp):
         context = super(PHPFPMBackend, self).get_context(webapp)
         context.update({
-            'init_vars': webapp.get_php_init_vars(),
+            'init_vars': self.get_php_init_vars(webapp),
             'fpm_port': webapp.get_fpm_port(),
         })
         context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context
-        fpm_config = Template(
+        fpm_config = Template(textwrap.dedent("""\
-            ";; {{ banner }}\n"
+            ;; {{ banner }}
-            "[{{ user }}]\n"
+            [{{ user }}]
-            "user = {{ user }}\n"
+            user = {{ user }}
-            "group = {{ group }}\n\n"
+            group = {{ group }}
-            "listen = {{ fpm_listen | safe }}\n"
+            
-            "listen.owner = {{ user }}\n"
+            listen = {{ fpm_listen | safe }}
-            "listen.group = {{ group }}\n"
+            listen.owner = {{ user }}
-            "pm = ondemand\n"
+            listen.group = {{ group }}
-            "pm.max_children = 4\n"
+            pm = ondemand
-            "{% for name,value in init_vars.iteritems %}"
+            pm.max_children = 4
-            "php_admin_value[{{ name | safe }}] = {{ value | safe }}\n"
+            {% for name,value in init_vars.iteritems %}
-            "{% endfor %}"
+            php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}"""
-        ) 
+        ))
-        fpm_file = '%(user)s.conf' % context
         context.update({
             'fpm_config': fpm_config.render(Context(context)),
-            'fpm_path': os.path.join(settings.WEBAPPS_PHPFPM_POOL_PATH, fpm_file),
+            'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context,
         })
         return context
 

orchestra/apps/webapps/models.py

@@ -39,23 +39,6 @@ class WebApp(models.Model):
     def get_options(self):
         return { opt.name: opt.value for opt in self.options.all() }
     
-    def get_php_init_vars(self):
-        init_vars = []
-        options = WebAppOption.objects.filter(webapp__type=self.type)
-        for opt in options.filter(webapp__account=self.account):
-            name = opt.name.replace('PHP-', '')
-            value = "%s" % opt.value
-            init_vars.append((name, value))
-        enabled_functions = self.options.filter(name='enabled_functions')
-        if enabled_functions:
-            enabled_functions = enabled_functions.get().value.split(',')
-            disabled_functions = []
-            for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
-                if function not in enabled_functions:
-                    disabled_functions.append(function)
-            init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
-        return init_vars
-    
     def get_fpm_port(self):
         return settings.WEBAPPS_FPM_START_PORT + self.account.pk
     

orchestra/apps/webapps/settings.py

@@ -9,10 +9,16 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN',
 #    '/var/run/%(user)s-%(app_name)s.sock')
     '127.0.0.1:%(fpm_port)s')
 
+
 WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000)
 
+
+WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
+    '/etc/php5/fpm/pool.d/%(app_name)s.conf')
+
+
 WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH',
-    '/home/httpd/fcgid/%(user)s/%(type)s-wrapper')
+    '/home/httpd/fcgid/%(app_name)s-wrapper')
 
 
 WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', {
@@ -166,10 +172,26 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', {
 
 
 WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [
-    'exec',  'passthru', 'shell_exec', 'system', 'proc_open', 'popen', 'curl_exec',
+    'exec',
-    'curl_multi_exec', 'show_source', 'pcntl_exec', 'proc_close',
+    'passthru',
-    'proc_get_status', 'proc_nice', 'proc_terminate', 'ini_alter', 'virtual',
+    'shell_exec',
-    'openlog', 'escapeshellcmd', 'escapeshellarg', 'dl'
+    'system',
+    'proc_open',
+    'popen',
+    'curl_exec',
+    'curl_multi_exec',
+    'show_source',
+    'pcntl_exec',
+    'proc_close',
+    'proc_get_status',
+    'proc_nice',
+    'proc_terminate',
+    'ini_alter',
+    'virtual',
+    'openlog',
+    'escapeshellcmd',
+    'escapeshellarg',
+    'dl'
 ])
 
 
@@ -181,9 +203,6 @@ WEBAPPS_WORDPRESSMU_ADMIN_PASSWORD = getattr(settings, 'WEBAPPS_WORDPRESSMU_ADMI
     'secret')
 
 
-
-
-
 WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH',
     '/home/httpd/htdocs/wikifarm/template.tar.gz')
 
@@ -195,6 +214,3 @@ WEBAPPS_DOKUWIKIMU_FARM_PATH = getattr(settings, 'WEBAPPS_DOKUWIKIMU_FARM_PATH',
 WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH',
     '/home/httpd/htdocs/drupal-mu/sites/%(app_name)s')
 
-
-WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
-    '/etc/php5/fpm/pool.d')

orchestra/apps/webapps/tests/functional_tests/tests.py

@@ -36,39 +36,14 @@ class WebAppMixin(object):
         djsettings.DEBUG = True
     
     def add_route(self):
-#        backends = [
+        server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER)
-#            # TODO MU apps on SaaS?
+        backend = SystemUserBackend.get_name()
-#            backends.awstats.AwstatsBackend,
+        Route.objects.get_or_create(backend=backend, match=True, host=server)
-#            backends.dokuwikimu.DokuWikiMuBackend,
+        backend = self.backend.get_name()
-#            backends.drupalmu.DrupalMuBackend,
+        match = 'webapp.type == "%s"' % self.type_value
-#            backends.phpfcgid.PHPFcgidBackend,
+        Route.objects.create(backend=backend, match=match, host=server)
-#            backends.phpfpm.PHPFPMBackend,
-#            backends.static.StaticBackend,
-#            backends.wordpressmu.WordpressMuBackend,
-#        ]
-        server = Server.objects.create(name=self.MASTER_SERVER)
-        for backend in [SystemUserBackend, self.backend]:
-            backend = backend.get_name()
-            Route.objects.create(backend=backend, match=True, host=server)
     
-    def test_add(self):
+    def upload_webapp(self, name):
-        name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
-        self.add_webapp(name)
-        self.validate_add_webapp(name)
-#        self.addCleanup(self.delete, username)
-
-
-class StaticWebAppMixin(object):
-    backend = backends.static.StaticBackend
-    type_value = 'static'
-    token = random_ascii(100)
-    page = (
-        'index.html',
-        '<html>Hello World! %s </html>\n' % token,
-        '<html>Hello World! %s </html>\n' % token,
-    )
-    
-    def validate_add_webapp(self, name):
         try:
             ftp = ftplib.FTP(self.MASTER_SERVER)
             ftp.login(user=self.account.username, passwd=self.account_password)
@@ -81,6 +56,23 @@ class StaticWebAppMixin(object):
         finally:
             ftp.close()
     
+    def test_add(self):
+        name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+        self.add_webapp(name)
+        self.addCleanup(self.delete_webapp, name)
+        self.upload_webapp(name)
+
+
+class StaticWebAppMixin(object):
+    backend = backends.static.StaticBackend
+    type_value = 'static'
+    token = random_ascii(100)
+    page = (
+        'index.html',
+        '<html>Hello World! %s </html>\n' % token,
+        '<html>Hello World! %s </html>\n' % token,
+    )
+
 
 class PHPFcidWebAppMixin(StaticWebAppMixin):
     backend = backends.phpfcgid.PHPFcgidBackend
@@ -111,12 +103,11 @@ class RESTWebAppMixin(object):
     
     @save_response_on_error
     def add_webapp(self, name, options=[]):
-        self.rest.webapps.create(name=name, type=self.type_value)
+        self.rest.webapps.create(name=name, type=self.type_value, options=options)
     
     @save_response_on_error
     def delete_webapp(self, name):
-        list = self.rest.lists.retrieve(name=name).get()
+        self.rest.webapps.retrieve(name=name).delete()
-        list.delete()
 
 
 class AdminWebAppMixin(WebAppMixin):
@@ -125,54 +116,25 @@ class AdminWebAppMixin(WebAppMixin):
         self.admin_login()
         # create main user
         self.save_systemuser()
-    # TODO save_account()
+    
+    @snapshot_on_error
+    def save_systemuser(self):
+        url = ''
     
     @snapshot_on_error
     def add(self, name, password, admin_email):
-        url = self.live_server_url + reverse('admin:mails_List_add')
-        self.selenium.get(url)
-        
-        account_input = self.selenium.find_element_by_id('id_account')
-        account_select = Select(account_input)
-        account_select.select_by_value(str(self.account.pk))
-        
-        name_field = self.selenium.find_element_by_id('id_name')
-        name_field.send_keys(username)
-        
-        password_field = self.selenium.find_element_by_id('id_password1')
-        password_field.send_keys(password)
-        password_field = self.selenium.find_element_by_id('id_password2')
-        password_field.send_keys(password)
-        
-        if quota is not None:
-            quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
-            quota_field = self.selenium.find_element_by_id(quota_id)
-            quota_field.clear()
-            quota_field.send_keys(quota)
-        
-        if filtering is not None:
-            filtering_input = self.selenium.find_element_by_id('id_filtering')
-            filtering_select = Select(filtering_input)
-            filtering_select.select_by_value("CUSTOM")
-            filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
-            filtering_inline.click()
-            time.sleep(0.5)
-            filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
-            filtering_field.send_keys(filtering)
-        
-        name_field.submit()
-        self.assertNotEqual(url, self.selenium.current_url)
-
-
-class RESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
         pass
 
 
-class RESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
     pass
 
 
-class RESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+    pass
+
+
+class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
     pass
 
 

orchestra/apps/websites/backends/apache.py

@@ -112,11 +112,11 @@ class Apache2Backend(ServiceController):
                 directives += "SecRuleRemoveById %d" % rule
         
         for modsecurity in site.options.filter(name='sec_rule_off'):
-            directives += (
+            directives += textwrap.dedent("""\
-                "<LocationMatch %s>\n"
+                <LocationMatch %s>
-                "    SecRuleEngine Off\n"
+                    SecRuleEngine Off
-                "</LocationMatch>\n" % modsecurity.value
+                </LocationMatch>
-            )
+                """ % modsecurity.value)
         return directives
     
     def get_protections(self, site):

orchestra/apps/websites/models.py

@@ -39,7 +39,9 @@ class Website(models.Model):
     
     @cached
     def get_options(self):
-        return { opt.name: opt.value for opt in self.options.all() }
+        return {
+            opt.name: opt.value for opt in self.options.all()
+        }
     
     @property
     def protocol(self):
@@ -81,12 +83,15 @@ class Content(models.Model):
     class Meta:
         unique_together = ('website', 'path')
     
+    def __unicode__(self):
+        try:
+            return self.website.name + self.path
+        except Website.DoesNotExist:
+            return self.path
+    
     def clean(self):
         if not self.path.startswith('/'):
             self.path = '/' + self.path
 
-    def __unicode__(self):
-        return self.website.name + self.path
-
 
 services.register(Website)

orchestra/apps/websites/serializers.py

@@ -11,6 +11,9 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
         model = Content
         fields = ('webapp', 'path')
     
+    def get_identity(self, data):
+        return '%s-%s' % (data.get('website'), data.get('path'))
+
 
 class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
     contents = ContentSerializer(required=False, many=True, allow_add_remove=True,

orchestra/apps/websites/tests/functional_tests/tests.py

@@ -38,9 +38,9 @@ class WebsiteMixin(WebAppMixin):
         super(WebsiteMixin, self).add_route()
         server = Server.objects.get()
         backend = backends.apache.Apache2Backend.get_name()
-        Route.objects.create(backend=backend, match=True, host=server)
+        Route.objects.get_or_create(backend=backend, match=True, host=server)
         backend = Bind9MasterDomainBackend.get_name()
-        Route.objects.create(backend=backend, match=True, host=server)
+        Route.objects.get_or_create(backend=backend, match=True, host=server)
     
     def validate_add_website(self, name, domain):
         url = 'http://%s/%s' % (domain.name, self.page[0])
@@ -54,9 +54,11 @@ class WebsiteMixin(WebAppMixin):
         self.save_domain(domain)
         webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
         self.add_webapp(webapp)
-        self.validate_add_webapp(webapp)
+        self.addCleanup(self.delete_webapp, webapp)
+        self.upload_webapp(webapp)
         website = '%s_website' % random_ascii(10)
         self.add_website(website, domain, webapp)
+        self.addCleanup(self.delete_website, website)
         self.validate_add_website(website, domain)
 
 
@@ -65,21 +67,82 @@ class RESTWebsiteMixin(RESTWebAppMixin):
     def save_domain(self, domain):
         self.rest.domains.retrieve().get().save()
     
-    def add_website(self, name, domain, webapp):
+    @save_response_on_error
-        domain = self.rest.domains.retrieve().get()
+    def add_website(self, name, domain, webapp, path='/'):
-        webapp = self.rest.webapps.retrieve().get()
+        domain = self.rest.domains.retrieve(name=domain).get()
-        self.rest.websites.create(name=name, domains=[domain.url], contents=[{'webapp': webapp.url}])
+        webapp = self.rest.webapps.retrieve(name=webapp).get()
+        contents = [{
+            'webapp': webapp.url,
+            'path': path
+        }]
+        self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
+    
+    @save_response_on_error
+    def delete_website(self, name):
+        print 'hola'
+        pass
+        self.rest.websites.retrieve(name=name).delete()
+#        self.rest.websites.retrieve(name=name).delete()
+    
+    @save_response_on_error
+    def add_content(self, website, webapp, path):
+        website = self.rest.websites.retrieve(name=website).get()
+        webapp = self.rest.webapps.retrieve(name=webapp).get()
+        website.contents.append({
+            'webapp': webapp.url,
+            'path': path,
+        })
+        website.save()
 
 
-class RESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+    def test_mix_webapps(self):
+        domain_name = '%sdomain.lan' % random_ascii(10)
+        domain = Domain.objects.create(name=domain_name, account=self.account)
+        domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR)
+        self.save_domain(domain)
+        webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+        self.add_webapp(webapp)
+        self.addCleanup(self.delete_webapp, webapp)
+        self.upload_webapp(webapp)
+        website = '%s_website' % random_ascii(10)
+        self.add_website(website, domain, webapp)
+        self.addCleanup(self.delete_website, website)
+        self.validate_add_website(website, domain)
+        
+        self.type_value = PHPFcidWebAppMixin.type_value
+        self.backend = PHPFcidWebAppMixin.backend
+        self.page = PHPFcidWebAppMixin.page
+        self.add_route()
+        webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+        self.add_webapp(webapp)
+        self.addCleanup(self.delete_webapp, webapp)
+        self.upload_webapp(webapp)
+        path = '/%s' % webapp
+        self.add_content(website, webapp, path)
+        url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
+        self.assertEqual(self.page[2], requests.get(url).content)
+        
+        self.type_value = PHPFPMWebAppMixin.type_value
+        self.backend = PHPFPMWebAppMixin.backend
+        self.page = PHPFPMWebAppMixin.page
+        self.add_route()
+        webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+        self.add_webapp(webapp)
+        self.addCleanup(self.delete_webapp, webapp)
+        self.upload_webapp(webapp)
+        path = '/%s' % webapp
+        
+        self.add_content(website, webapp, path)
+        url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
+        self.assertEqual(self.page[2], requests.get(url).content)
+
+
+class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
     pass
 
 
-class RESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
-    pass
-
-
-class RESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
     pass
 
 #class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase):

orchestra/templates/admin/base_site.html

@@ -27,6 +27,7 @@
 {% if not is_popup %}
 {% admin_tools_render_menu %}
 {% endif %}
+</div>
 {% endif %}
 {% endblock %}
 

scripts/services/apache_full_stack.md

@@ -84,6 +84,12 @@ The goal of this setup is having a high-performance state-of-the-art deployment
 </Directory>
 
 
+
+# TODO pool per website or pool per user? memory consumption 
+events.mechanism = epoll
+# TODO multiple master processes, opcache is held in master, and reload/restart affects all pools
+# http://mattiasgeniar.be/2014/04/09/a-better-way-to-run-php-fpm/
+
 TODO CHRoot
     https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/
 
@@ -92,10 +98,10 @@ TODO CHRoot
     [vhost]
     istemplate = 1
     listen.mode = 0660
+    pm = ondemand
     pm.max_children = 5
-    pm.start_servers = 1
+    pm.process_idle_timeout = 10s
-    pm.min_spare_servers = 1
+    pm.max_requests = 200
-    pm.max_spare_servers = 2
     ' > /etc/php5/fpm/conf.d/vhost-template.conf
     ```