From e80f9216018dcc55d00eef10e8ecc14d9e1d5df5 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 11 Mar 2015 20:01:08 +0000 Subject: [PATCH] Implemented batch domain creation0 --- TODO.md | 44 ++++--------- orchestra/apps/domains/admin.py | 40 +++++++++--- orchestra/apps/domains/backends.py | 4 +- orchestra/apps/domains/forms.py | 96 +++++++++++++---------------- orchestra/apps/webapps/types/php.py | 17 +++-- 5 files changed, 103 insertions(+), 98 deletions(-) diff --git a/TODO.md b/TODO.md index ce13247c..25804c9a 100644 --- a/TODO.md +++ b/TODO.md @@ -11,21 +11,16 @@ * add `BackendLog` retry action * webmail identities and addresses -* use Code: https://github.com/django/django/blob/master/django/forms/forms.py#L415 for domain.refresh_serial() * Permissions .filter_queryset() * env vars instead of multiple settings files: https://devcenter.heroku.com/articles/config-vars ? * Log changes from rest api (serialized objects) -* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0) - -* Settings dictionary like DRF2 in order to better override large settings like WEBSITES_APPLICATIONS.etc * backend logs with hal logo * set_password orchestration method? -* make account_link to autoreplace account on change view. * LAST version of this shit http://wkhtmltopdf.org/downloads.html @@ -48,22 +43,20 @@ * Maildir billing tests/ webdisk billing tests (avg metric) -* move icons to apps, and use appconfig to cleanup config stuff -* when using modeladmin to store shit like self.account, make sure to have a cleanslate in each request? +* when using modeladmin to store shit like self.account, make sure to have a cleanslate in each request? no, better reuse the last one -* jabber with mailbox accounts (dovecto mail notification) +* jabber with mailbox accounts (dovecot mail notification) * rename accounts register to "account", and reated api and admin references -* take a look icons from ajenti ;) * Disable services is_active should be computed on the fly in order to distinguish account.is_active from service.is_active when reactivation. * Perhaps it is time to create a ServiceModel ? * prevent deletion of main user by the user itself -* AccountAdminMixin auto adds 'account__name' on searchfields and handle account_link on fieldsets +* AccountAdminMixin auto adds 'account__name' on searchfields * Separate panel from server passwords? Store passwords on panel? set_password special backend operation? @@ -75,16 +68,15 @@ * delete main user -> delete account or prevent delete main user -* Ansible orchestration *method* (methods.py) -* multiple domains creation; line separated domains -* Move MU webapps to SaaS? -* offer to create mailbox on account creation +* multiple domains creation; line separated domains + + * init.d celery scripts -# Required-Start: $network $local_fs $remote_fs postgresql celeryd -# Required-Stop: $network $local_fs $remote_fs postgresql celeryd -* for list virtual_domains cleaning up we need to know the old domain name when a list changes its address domain, but this is not possible with the current design. + * regenerate virtual_domains every time (configure a separate file for orchestra on postfix) * update_fields=[] doesn't trigger post save! @@ -97,9 +89,10 @@ * proforma without billing contact? +* print open invoices as proforma? + * env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python manage.py test orchestra.apps.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest -* Pangea modifications: domain registered/non-registered list_display and field with register link: inconsistent, what happen to related objects with a domain that is converted to register-only? * ForeignKey.swappable * Field.editable @@ -111,16 +104,12 @@ * multiple files monitoring -* Split plans into a separate app (plans and rates / services ) interface ? - * sync() ServiceController method that synchronizes orchestra and servers (delete or import) * consider removing mailbox support on forward (user@pangea.org instead) * Databases.User add reverse M2M databases widget (like mailbox.addresses) -* Root owned logs on user's home ? yes - * reconsider binding webapps to systemusers (pangea multiple users wordpress-ftp, moodle-pangea, etc) * Secondary user home in /home/secondaryuser and simlink to /home/main/webapps/app so it can have private storage? * Grant permissions to systemusers, the problem of creating a related permission model is out of sync with the server-side. evaluate tradeoff @@ -155,7 +144,6 @@ * Create an admin service_view with icons (like SaaS app) -* Fix ftp traffic * Resource graph for each related object @@ -179,22 +167,15 @@ Multi-tenant WebApps * SaaS - Those apps that can't use custom domain * WebApp - Those apps that can use custom domain -* Howto upgrade webapp PHP version? SetHandler php54-cgi ? or create a new app * prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org -* fcgid kill instead of apache reload? - -* username maximum as group user in UNIX - * forms autocomplete="off", doesn't work in chrome ln -s /proc/self/fd /dev/fd -* http-https/https-only/http-only - POST INSTALL ------------ @@ -212,9 +193,10 @@ Php binaries should have this format: /usr/bin/php5.2-cgi * and other IfModule on backend SecRule -* webalizer backend on webapps and check webapps.websites.all() * monitor in batches doesnt work!!! -* mv: cannot move `/home/marcay/webapps/webalizer/' to a subdirectory of itself, `/home/marcay/webapps/webalizer/.deleted' -* Create utility for dealing with web paths '//', leading and ending '/' +* Orchestra global search box on the header, based https://github.com/django/django/blob/master/django/contrib/admin/options.py#L866 and iterating over all registered services and inspectin its admin.search_fields + + +* contain error on plugin missing key (plugin dissabled) diff --git a/orchestra/apps/domains/admin.py b/orchestra/apps/domains/admin.py index fb518a13..f91104d4 100644 --- a/orchestra/apps/domains/admin.py +++ b/orchestra/apps/domains/admin.py @@ -10,7 +10,7 @@ from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.utils import apps from .actions import view_zone -from .forms import RecordInlineFormSet, CreateDomainAdminForm +from .forms import RecordInlineFormSet, BatchDomainCreationAdminForm from .filters import TopDomainListFilter from .models import Domain, Record @@ -68,7 +68,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): list_filter = [TopDomainListFilter] change_readonly_fields = ('name',) search_fields = ['name',] - add_form = CreateDomainAdminForm + add_form = BatchDomainCreationAdminForm change_view_actions = [view_zone] def structured_name(self, domain): @@ -115,11 +115,37 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): qs = qs.prefetch_related('websites') return qs -# def save_related(self, request, form, formsets, change): -# super(DomainAdmin, self).save_related(request, form, formsets, change) -# if form.cleaned_data['migrate_subdomains']: -# domain = form.instance -# domain.subdomains.update(account_id=domain.account_id) + def save_model(self, request, obj, form, change): + """ batch domain creation support """ + super(DomainAdmin, self).save_model(request, obj, form, change) + self.extra_domains = [] + if not change: + for name in form.extra_names: + domain = Domain.objects.create(name=name, account_id=obj.account_id) + self.extra_domains.append(domain) + + def save_formset(self, request, form, formset, change): + """ + Given an inline formset save it to the database. + """ + formset.save() + + def save_related(self, request, form, formsets, change): + """ batch domain creation support """ + super(DomainAdmin, self).save_related(request, form, formsets, change) + if not change: + # Clone records to extra_domains, if any + for formset in formsets: + if formset.model is Record: + for domain in self.extra_domains: + # Reset pk value of the record instances to force creation of new ones + for record_form in formset.forms: + record = record_form.instance + if record.pk: + record.pk = None + formset.instance = domain + form.instance = domain + self.save_formset(request, form, formset, change=change) admin.site.register(Domain, DomainAdmin) diff --git a/orchestra/apps/domains/backends.py b/orchestra/apps/domains/backends.py index 2e653ed3..09186ede 100644 --- a/orchestra/apps/domains/backends.py +++ b/orchestra/apps/domains/backends.py @@ -33,9 +33,9 @@ class Bind9MasterDomainBackend(ServiceController): self.append(textwrap.dedent("""\ echo -e '%(zone)s' > %(zone_path)s.tmp diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1 + # Because bind reload will not display any fucking error + named-checkzone -k fail -n fail %(name)s %(zone_path)s.tmp mv %(zone_path)s.tmp %(zone_path)s - # Because bind realod will not display any fucking error - named-checkzone -k fail -n fail %(name)s %(zone_path)s """) % context ) self.update_conf(context) diff --git a/orchestra/apps/domains/forms.py b/orchestra/apps/domains/forms.py index 072a211a..e53f1682 100644 --- a/orchestra/apps/domains/forms.py +++ b/orchestra/apps/domains/forms.py @@ -7,67 +7,55 @@ from .helpers import domain_for_validation from .models import Domain -class CreateDomainAdminForm(forms.ModelForm): -# migrate_subdomains = forms.BooleanField(label=_("Migrate subdomains"), required=False, -# initial=False, help_text=_("Propagate the account owner change to subdomains.")) +class BatchDomainCreationAdminForm(forms.ModelForm): + name = forms.CharField(label=_("Names"), widget=forms.Textarea(attrs={'rows': 5, 'cols': 50}), + help_text=_("Domain per line. All domains will share the same attributes.")) + + def clean_name(self): + self.extra_names = [] + target = None + for name in self.cleaned_data['name'].strip().splitlines(): + name = name.strip() + if not name: + continue + if target is None: + target = name + else: + domain = Domain(name=name) + try: + domain.full_clean(exclude=['top']) + except ValidationError as e: + raise ValidationError(e.error_dict['name']) + self.extra_names.append(name) + return target def clean(self): """ inherit related top domain account, when exists """ - cleaned_data = super(CreateDomainAdminForm, self).clean() + cleaned_data = super(BatchDomainCreationAdminForm, self).clean() if not cleaned_data['account']: - domain = Domain(name=cleaned_data['name']) - top = domain.get_top() - if not top: - # Fake an account to make django validation happy - account_model = self.fields['account']._queryset.model - cleaned_data['account'] = account_model() - raise ValidationError({ - 'account': _("An account should be provided for top domain names."), - }) - cleaned_data['account'] = top.account + account = None + for name in [cleaned_data['name']] + self.extra_names: + domain = Domain(name=name) + top = domain.get_top() + if not top: + # Fake an account to make django validation happy + account_model = self.fields['account']._queryset.model + cleaned_data['account'] = account_model() + raise ValidationError({ + 'account': _("An account should be provided for top domain names."), + }) + elif account and top.account != account: + # Fake an account to make django validation happy + account_model = self.fields['account']._queryset.model + cleaned_data['account'] = account_model() + raise ValidationError({ + 'account': _("Provided domain names belong to different accounts."), + }) + account = top.account + cleaned_data['account'] = account return cleaned_data -#class BatchDomainCreationAdminForm(DomainAdminForm): -# # TODO -# name = forms.CharField(widget=forms.Textarea, label=_("Names"), -# help_text=_("Domain per line. All domains will share the same attributes.")) -# -# def clean_name(self): -# self.names = [] -# target = None -# for name in self.cleaned_data['name'].splitlines(): -# name = name.strip() -# if target is None: -# target = name -# else: -# domain = Domain(name=name) -# try: -# domain.full_clean(exclude=['top']) -# except ValidationError as e: -# raise ValidationError(e.error_dict['name']) -# self.names.append(name) -# return target -# -# def save_model(self, request, obj, form, change): -# # TODO thsi is modeladmin -# """ batch domain creation support """ -# super(DomainAdmin, self).save_model(request, obj, form, change) -# if not change: -# for name in form.names: -# domain = Domain.objects.create(name=name, account_id=obj.account_id) -# -# def save_related(self, request, form, formsets, change): -# # TODO thsi is modeladmin -# """ batch domain creation support """ -# super(DomainAdmin, self).save_related(request, form, formsets, change) -# if not change: -# for name in form.names: -# for formset in formsets: -# formset.instance = form.instance -# self.save_formset(request, form, formset, change=change) - - class RecordInlineFormSet(forms.models.BaseInlineFormSet): # TODO def clean(self): diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index e324abc4..b8098544 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -57,16 +57,23 @@ class PHPAppType(AppType): return init_vars +help_message = _("Version of PHP used to execute this webapp.
" + "Changing the PHP version may result in application malfunction, " + "make sure that everything continue to work as expected.") + + class PHPFPMAppForm(PluginDataForm): php_version = forms.ChoiceField(label=_("PHP version"), choices=settings.WEBAPPS_PHP_FPM_VERSIONS, - initial=settings.WEBAPPS_PHP_FPM_DEFAULT_VERSION) + initial=settings.WEBAPPS_PHP_FPM_DEFAULT_VERSION, + help_text=help_message) class PHPFPMAppSerializer(serializers.Serializer): php_version = serializers.ChoiceField(label=_("PHP version"), choices=settings.WEBAPPS_PHP_FPM_VERSIONS, - default=settings.WEBAPPS_PHP_FPM_DEFAULT_VERSION) + default=settings.WEBAPPS_PHP_FPM_DEFAULT_VERSION, + help_text=help_message) class PHPFPMApp(PHPAppType): @@ -91,13 +98,15 @@ class PHPFPMApp(PHPAppType): class PHPFCGIDAppForm(PluginDataForm): php_version = forms.ChoiceField(label=_("PHP version"), choices=settings.WEBAPPS_PHP_FCGID_VERSIONS, - initial=settings.WEBAPPS_PHP_FCGID_DEFAULT_VERSION) + initial=settings.WEBAPPS_PHP_FCGID_DEFAULT_VERSION, + help_text=help_message) class PHPFCGIDAppSerializer(serializers.Serializer): php_version = serializers.ChoiceField(label=_("PHP version"), choices=settings.WEBAPPS_PHP_FCGID_VERSIONS, - default=settings.WEBAPPS_PHP_FCGID_DEFAULT_VERSION) + default=settings.WEBAPPS_PHP_FCGID_DEFAULT_VERSION, + help_text=help_message) class PHPFCGIDApp(PHPAppType):