Refactoring rest api nested serialization

TODO.md

@@ -141,7 +141,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
     -# Required-Stop:     $network $local_fs $remote_fs postgresql celeryd
 
 
-* POST only fields (account, username, name) etc http://inka-labs.com/blog/2013/04/18/post-only-fields-django-rest-framework/
 * for list virtual_domains cleaning up we need to know the old domain name when a list changes its address domain, but this is not possible with the current design.
 * regenerate virtual_domains every time (configure a separate file for orchestra on postfix)
 * update_fields=[] doesn't trigger post save!

orchestra/api/serializers.py

@@ -10,53 +10,24 @@ class SetPasswordSerializer(serializers.Serializer):
             widget=widgets.PasswordInput, validators=[validate_password])
 
 
-
+class HyperlinkedModelSerializerOptions(serializers.HyperlinkedModelSerializerOptions):
-from rest_framework.serializers import (HyperlinkedModelSerializerOptions,
-                                        HyperlinkedModelSerializer)
- 
- 
-class tHyperlinkedModelSerializerOptions(serializers.HyperlinkedModelSerializerOptions):
-    """ Options for PostHyperlinkedModelSerializer """
-    
     def __init__(self, meta):
         super(HyperlinkedModelSerializerOptions, self).__init__(meta)
         self.postonly_fields = getattr(meta, 'postonly_fields', ())
 
 
-class HyperlinkedModelSerializer(HyperlinkedModelSerializer):
+class HyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer):
+    """ support for postonly_fields, fields whose value can only be set on post """
     _options_class = HyperlinkedModelSerializerOptions
     
-    def to_native(self, obj):
-        """ Serialize objects -> primitives. """
-        ret = self._dict_class()
-        ret.fields = {}
-        
-        for field_name, field in self.fields.items():
-            # Ignore all postonly_fields fron serialization
-            if field_name in self.opts.postonly_fields:
-                continue
-            field.initialize(parent=self, field_name=field_name)
-            key = self.get_field_key(field_name)
-            value = field.field_to_native(obj, field_name)
-            ret[key] = value
-            ret.fields[key] = field
-        return ret
-    
     def restore_object(self, attrs, instance=None):
-        model_attrs, post_attrs = {}, {}
+        """ removes postonly_fields from attrs when not posting """
+        model_attrs = dict(**attrs)
+        if instance is not None:
             for attr, value in attrs.iteritems():
                 if attr in self.opts.postonly_fields:
-                post_attrs[attr] = value
+                    model_attrs.pop(attr)
-            else:
+        return super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance)
-                model_attrs[attr] = value
-        obj = super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance)
-        # Method to process ignored postonly_fields
-        self.process_postonly_fields(obj, post_attrs)
-        return obj
- 
-    def process_postonly_fields(self, obj, post_attrs):
-        """ Placeholder method for processing data sent in POST. """
-        pass
 
 
 class MultiSelectField(serializers.ChoiceField):

orchestra/apps/accounts/serializers.py

@@ -12,6 +12,13 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
 
 
 class AccountSerializerMixin(object):
+    def __init__(self, *args, **kwargs):
+        super(AccountSerializerMixin, self).__init__(*args, **kwargs)
+        self.account = None
+        request = self.context.get('request')
+        if request:
+            self.account = request.user
+    
     def save_object(self, obj, **kwargs):
-        obj.account = self.context['request'].user
+        obj.account = self.account
         super(AccountSerializerMixin, self).save_object(obj, **kwargs)

orchestra/apps/databases/backends.py

@@ -19,6 +19,8 @@ class MySQLBackend(ServiceController):
         self.append(
             "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
         )
+        # clean previous privileges
+        self.append("""mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'""" % context)
         for user in database.users.all():
             context.update({
                 'username': user.username,

orchestra/apps/databases/serializers.py

@@ -1,5 +1,6 @@
 from django.forms import widgets
 from django.utils.translation import ugettext, ugettext_lazy as _
+from django.shortcuts import get_object_or_404
 from rest_framework import serializers
 
 from orchestra.api.serializers import HyperlinkedModelSerializer
@@ -9,32 +10,39 @@ from orchestra.core.validators import validate_password
 from .models import Database, DatabaseUser
 
 
-class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
+class RelatedDatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
     class Meta:
         model = DatabaseUser
         fields = ('url', 'username')
     
     def from_native(self, data, files=None):
-        return DatabaseUser.objects.get(username=data['username'])
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, username=data['username'])
 
 
 class DatabaseSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
     users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
-    # TODO clean user.type = db.type
     
     class Meta:
         model = Database
         fields = ('url', 'name', 'type', 'users')
         postonly_fields = ('name', 'type')
     
+    def validate(self, attrs):
+        for user in attrs['users']:
+            if user.type != attrs['type']:
+                raise serializers.ValidationError("User type must be" % attrs['type'])
+        return attrs
 
-class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
+
+class RelatedDatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Database
         fields = ('url', 'name',)
     
     def from_native(self, data, files=None):
-        return Database.objects.get(name=data['name'])
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
 
 
 class DatabaseUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
@@ -42,13 +50,18 @@ class DatabaseUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer)
             validators=[validate_password], write_only=True,
             widget=widgets.PasswordInput)
     databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
-    # TODO clean user.type = db.type
         
     class Meta:
         model = DatabaseUser
         fields = ('url', 'username', 'password', 'type', 'databases')
         postonly_fields = ('username', 'type')
     
+    def validate(self, attrs):
+        for database in attrs.get('databases', []):
+            if database.type != attrs['type']:
+                raise serializers.ValidationError("Database type must be" % attrs['type'])
+        return attrs
+    
     def save_object(self, obj, **kwargs):
         # FIXME this method will be called when saving nested serializers :(
         if not obj.pk:

orchestra/apps/databases/tests/functional_tests/tests.py

@@ -115,10 +115,28 @@ class DatabaseTestMixin(object):
         self.add_user(username2, password2)
         self.add_user_to_db(username2, dbname)
         self.delete_user(username)
+        self.validate_delete_user(username, password)
         self.validate_login_error(dbname, username, password)
         self.validate_create_table(dbname, username2, password2)
         self.delete_user(username2)
         self.validate_login_error(dbname, username2, password2)
+        self.validate_delete_user(username2, password2)
+    
+    def test_swap_user(self):
+        dbname = '%s_database' % random_ascii(5)
+        username = '%s_dbuser' % random_ascii(5)
+        password = '@!?%spppP001' % random_ascii(5)
+        self.add(dbname, username, password)
+        self.addCleanup(self.delete, dbname)
+        self.addCleanup(self.delete_user, username)
+        self.validate_create_table(dbname, username, password)
+        username2 = '%s_dbuser' % random_ascii(5)
+        password2 = '@!?%spppP001' % random_ascii(5)
+        self.add_user(username2, password2)
+        self.addCleanup(self.delete_user, username2)
+        self.swap_user(username, username2, dbname)
+        self.validate_login_error(dbname, username, password)
+        self.validate_create_table(dbname, username2, password2)
 
 
 class MySQLBackendMixin(object):
@@ -151,10 +169,10 @@ class MySQLBackendMixin(object):
             self.validate_create_table, dbname, username, password
         )
     
-    def validate_delete(self, name, username, password):
+    def validate_delete(self, dbname, username, password):
-        self.assertRaises(MySQLdb.OperationalError,
+        self.validate_login_error(dbname, username, password)
-                self.validate_create_table, name, username, password
+        self.assertRaises(CommandError,
-        )
+            sshrun, self.MASTER_SERVER, 'mysql %s' % dbname, display=False)
     
     def validate_delete_user(self, name, username):
         context = {
@@ -166,8 +184,6 @@ class MySQLBackendMixin(object):
         self.assertEqual('', sshrun(self.MASTER_SERVER,
             """mysql mysql -e 'SELECT * FROM user WHERE user="%(username)s";'""" % context, display=False).stdout)
 
-    # TODO remove used from database
-
 
 class RESTDatabaseMixin(DatabaseTestMixin):
     def setUp(self):
@@ -206,6 +222,14 @@ class RESTDatabaseMixin(DatabaseTestMixin):
     def delete_user(self, username):
         self.rest.databaseusers.retrieve(username=username).delete()
     
+    @save_response_on_error
+    def swap_user(self, username, username2, dbname):
+        user = self.rest.databaseusers.retrieve(username=username2).get()
+        db = self.rest.databases.retrieve(name=dbname).get()
+        db.users = db.users.exclude(username=username)
+        db.users.append(user)
+        db.save()
+
 
 class AdminDatabaseMixin(DatabaseTestMixin):
     def setUp(self):
@@ -280,6 +304,24 @@ class AdminDatabaseMixin(DatabaseTestMixin):
         save.submit()
         self.assertNotEqual(url, self.selenium.current_url)
     
+    @snapshot_on_error
+    def swap_user(self, username, username2, dbname):
+        database = Database.objects.get(name=dbname, type=self.db_type)
+        url = self.live_server_url + change_url(database)
+        self.selenium.get(url)
+        
+        user = DatabaseUser.objects.get(username=username, type=self.db_type)
+        users_input = self.selenium.find_element_by_id('id_users')
+        users_select = Select(users_input)
+        users_select.deselect_by_value(str(user.pk))
+        
+        user = DatabaseUser.objects.get(username=username2, type=self.db_type)
+        users_select.select_by_value(str(user.pk))
+        
+        save = self.selenium.find_element_by_name('_save')
+        save.submit()
+        self.assertNotEqual(url, self.selenium.current_url)
+    
     @snapshot_on_error
     def delete_user(self, username):
         user = DatabaseUser.objects.get(username=username)

orchestra/apps/lists/serializers.py

@@ -1,5 +1,6 @@
 from django.forms import widgets
 from django.utils.translation import ugettext, ugettext_lazy as _
+from django.shortcuts import get_object_or_404
 from rest_framework import serializers
 
 from orchestra.api.serializers import HyperlinkedModelSerializer
@@ -9,10 +10,21 @@ from orchestra.core.validators import validate_password
 from .models import List
 
 
+class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = List.address_domain.field.rel.to
+        fields = ('url', 'name')
+    
+    def from_native(self, data, files=None):
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
+
+
 class ListSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
     password = serializers.CharField(max_length=128, label=_('Password'),
             validators=[validate_password], write_only=True, required=False,
             widget=widgets.PasswordInput)
+    address_domain = RelatedDomainSerializer(required=False)
     
     class Meta:
         model = List
@@ -28,6 +40,17 @@ class ListSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
             raise serializers.ValidationError(_("Password required"))
         return attrs
     
+    def validate(self, attrs):
+        address_domain = attrs.get('address_domain')
+        address_name = attrs.get('address_name', )
+        if self.object:
+            address_domain = address_domain or self.object.address_domain
+            address_name = address_name or self.object.address_name
+        if bool(address_domain) != bool(address_name):
+            raise serializers.ValidationError(
+                _("address_name and address_domain should go in tandem"))
+        return attrs
+    
     def save_object(self, obj, **kwargs):
         if not obj.pk:
             obj.set_password(self.init_data.get('password', ''))

orchestra/apps/lists/tests/functional_tests/tests.py

@@ -142,7 +142,7 @@ class ListMixin(object):
         domain_name = '%sdomain.lan' % random_ascii(10)
         address_domain = Domain.objects.create(name=domain_name, account=self.account)
         self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain)
-#        self.addCleanup(self.delete, name)
+        self.addCleanup(self.delete, name)
         # Mailman doesn't support changing the address, only the domain
         address_name = '%s_name' % random_ascii(10)
         self.update_address_name(name, address_name)
@@ -174,7 +174,7 @@ class RESTListMixin(ListMixin):
         if address_name:
             extra.update({
                 'address_name': address_name,
-                'address_domain': self.rest.domains.retrieve(name=address_domain.name).get().url,
+                'address_domain': self.rest.domains.retrieve(name=address_domain.name).get(),
             })
         self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra)
     
@@ -191,7 +191,7 @@ class RESTListMixin(ListMixin):
     def update_domain(self, name, domain_name):
         mail_list = self.rest.lists.retrieve(name=name).get()
         domain = self.rest.domains.retrieve(name=domain_name).get()
-        mail_list.update(address_domain=domain.url)
+        mail_list.update(address_domain=domain)
     
     @save_response_on_error
     def update_address_name(self, name, address_name):

orchestra/apps/mails/serializers.py

@@ -1,4 +1,5 @@
 from django.forms import widgets
+from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext, ugettext_lazy as _
 from rest_framework import serializers
 
@@ -37,21 +38,34 @@ class MailboxSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
         super(MailboxSerializer, self).save_object(obj, **kwargs)
 
 
+class RelatedMailboxSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Mailbox
+        fields = ('url', 'name')
+    
+    def from_native(self, data, files=None):
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
+
+
+class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Address.domain.field.rel.to
+        fields = ('url', 'name')
+    
+    def from_native(self, data, files=None):
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
+
+
 class AddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    domain = RelatedDomainSerializer()
+    mailboxes = RelatedMailboxSerializer(many=True, allow_add_remove=True, required=False)
+    
     class Meta:
         model = Address
         fields = ('url', 'name', 'domain', 'mailboxes', 'forward')
     
-    def get_fields(self, *args, **kwargs):
-        fields = super(AddressSerializer, self).get_fields(*args, **kwargs)
-        account = self.context['view'].request.user.pk
-        mailboxes = fields['mailboxes'].queryset
-        fields['mailboxes'].queryset = mailboxes.filter(account=account)
-        # TODO do it on permissions or in self.filter_by_account_field ?
-        domain = fields['domain'].queryset
-        fields['domain'].queryset = domain.filter(account=account)
-        return fields
-    
     def validate(self, attrs):
         if not attrs['mailboxes'] and not attrs['forward']:
             raise serializers.ValidationError("mailboxes or forward addresses should be provided")

orchestra/apps/saas/backends/__init__.py

@@ -0,0 +1,17 @@
+import pkgutil
+import textwrap
+
+
+class SaaSServiceMixin(object):
+    model = 'saas.SaaS'
+    # TODO Match definition support on backends (mysql) and saas
+    
+    def get_context(self, webapp):
+        # TODO
+        return {
+        }
+
+
+for __, module_name, __ in pkgutil.walk_packages(__path__):
+    # sorry for the exec(), but Import module function fails :(
+    exec('from . import %s' % module_name)

orchestra/apps/saas/backends/dokuwikimu.py

@@ -4,11 +4,11 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.apps.orchestration import ServiceController
 
-from . import WebAppServiceMixin
+from . import SaaSServiceMixin
 from .. import settings
 
 
-class DokuWikiMuBackend(WebAppServiceMixin, ServiceController):
+class DokuWikiMuBackend(SaaSServiceMixin, ServiceController):
     verbose_name = _("DokuWiki multisite")
     
     def save(self, webapp):

orchestra/apps/saas/backends/drupalmu.py

@@ -4,11 +4,11 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.apps.orchestration import ServiceController
 
-from . import WebAppServiceMixin
+from . import SaaSServiceMixin
 from .. import settings
 
 
-class DrupalMuBackend(WebAppServiceMixin, ServiceController):
+class DrupalMuBackend(SaaSServiceMixin, ServiceController):
     verbose_name = _("Drupal multisite")
     
     def save(self, webapp):

orchestra/apps/saas/backends/wordpressmu.py

@@ -5,11 +5,11 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.apps.orchestration import ServiceController
 
-from . import WebAppServiceMixin
+from . import SaaSServiceMixin
 from .. import settings
 
 
-class WordpressMuBackend(WebAppServiceMixin, ServiceController):
+class WordpressMuBackend(SaaSServiceMixin, ServiceController):
     verbose_name = _("Wordpress multisite")
     
     @property

orchestra/apps/systemusers/serializers.py

@@ -1,5 +1,6 @@
 from django.contrib.auth import get_user_model
 from django.forms import widgets
+from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext, ugettext_lazy as _
 from rest_framework import serializers
 
@@ -10,13 +11,14 @@ from orchestra.core.validators import validate_password
 from .models import SystemUser
 
 
-class GroupSerializer(serializers.ModelSerializer):
+class GroupSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
     class Meta:
         model = SystemUser
-        fields = ('username',)
+        fields = ('url', 'username',)
     
     def from_native(self, data, files=None):
-        return SystemUser.objects.get(username=data['username'])
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, username=data['username'])
 
 
 class SystemUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
@@ -41,7 +43,14 @@ class SystemUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
             raise serializers.ValidationError(_("Password required"))
         return attrs
     
-    # TODO validate gruops != self
+    def validate_groups(self, attrs, source):
+        groups = attrs.get(source)
+        if groups:
+            for group in groups:
+                if group.username == attrs['username']:
+                    raise serializers.ValidationError(
+                        _("Do not make the user member of its group"))
+        return attrs
     
     def save_object(self, obj, **kwargs):
         # FIXME this method will be called when saving nested serializers :(

orchestra/apps/websites/serializers.py

@@ -1,3 +1,4 @@
+from django.shortcuts import get_object_or_404
 from rest_framework import serializers
 
 from orchestra.api.fields import OptionField
@@ -7,7 +8,29 @@ from orchestra.apps.accounts.serializers import AccountSerializerMixin
 from .models import Website, Content
 
 
+class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Website.domains.field.rel.to
+        fields = ('url', 'name')
+    
+    def from_native(self, data, files=None):
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
+
+
+class RelatedWebAppSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Content.webapp.field.rel.to
+        fields = ('url', 'name', 'type')
+    
+    def from_native(self, data, files=None):
+        queryset = self.opts.model.objects.filter(account=self.account)
+        return get_object_or_404(queryset, name=data['name'])
+
+
 class ContentSerializer(serializers.HyperlinkedModelSerializer):
+    webapp = RelatedWebAppSerializer()
+    
     class Meta:
         model = Content
         fields = ('webapp', 'path')
@@ -17,6 +40,7 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
 
 
 class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
+    domains = RelatedDomainSerializer(many=True, allow_add_remove=True, required=False)
     contents = ContentSerializer(required=False, many=True, allow_add_remove=True,
             source='content_set')
     options = OptionField(required=False)

orchestra/apps/websites/tests/functional_tests/tests.py

@@ -72,10 +72,10 @@ class RESTWebsiteMixin(RESTWebAppMixin):
         domain = self.rest.domains.retrieve(name=domain).get()
         webapp = self.rest.webapps.retrieve(name=webapp).get()
         contents = [{
-            'webapp': webapp.url,
+            'webapp': webapp,
             'path': path
         }]
-        self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
+        self.rest.websites.create(name=name, domains=[domain], contents=contents)
     
     @save_response_on_error
     def delete_website(self, name):
@@ -86,7 +86,7 @@ class RESTWebsiteMixin(RESTWebAppMixin):
         website = self.rest.websites.retrieve(name=website).get()
         webapp = self.rest.webapps.retrieve(name=webapp).get()
         website.contents.append({
-            'webapp': webapp.url,
+            'webapp': webapp,
             'path': path,
         })
         website.save()

orchestra/utils/html.py

@@ -3,6 +3,9 @@ from orchestra.utils.system import run
 
 def html_to_pdf(html):
     """ converts HTL to PDF using wkhtmltopdf """
-    return run('xvfb-run -a -s "-screen 0 640x4800x16" '
+    return run(
+        'PATH=$PATH:/usr/local/bin/\n'
+        'xvfb-run -a -s "-screen 0 640x4800x16" '
             'wkhtmltopdf --footer-center "Page [page] of [topage]" --footer-font-size 9 - -',
-        stdin=html.encode('utf-8'), display=False)
+        stdin=html.encode('utf-8'), display=False
+    )

scripts/container/deploy.sh

@@ -101,7 +101,7 @@ cat <<- EOF | python $MANAGE shell
 from orchestra.apps.accounts.models import Account
 if not Account.objects.filter(username="$USER").exists():
     print 'Creating orchestra superuser'
-    __ = Account.objects.create_superuser("$USER", "'$USER@localhost'", "$PASSWORD")
+    __ = Account.objects.create_superuser("$USER", "$USER@localhost", "$PASSWORD")
 
 EOF