diff --git a/orchestra/admin/__init__.py b/orchestra/admin/__init__.py index d59c7fab..508aee46 100644 --- a/orchestra/admin/__init__.py +++ b/orchestra/admin/__init__.py @@ -3,7 +3,7 @@ from collections import OrderedDict from functools import update_wrapper from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.shortcuts import render, redirect from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ @@ -56,7 +56,7 @@ def search(request): if service.search: models.add(service.model) model_name_map[service.model._meta.model_name] = service.model - + # Account direct access if search_term.endswith('!'): from ..contrib.accounts.models import Account diff --git a/orchestra/admin/dashboard.py b/orchestra/admin/dashboard.py index 5cded62c..9237470d 100644 --- a/orchestra/admin/dashboard.py +++ b/orchestra/admin/dashboard.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from fluent_dashboard import dashboard, appsettings from fluent_dashboard.modules import CmsAppIconList @@ -11,7 +11,7 @@ class AppDefaultIconList(CmsAppIconList): def __init__(self, *args, **kwargs): self.icons = kwargs.pop('icons') super(AppDefaultIconList, self).__init__(*args, **kwargs) - + def get_icon_for_model(self, app_name, model_name, default=None): icon = self.icons.get('.'.join((app_name, model_name))) return super(AppDefaultIconList, self).get_icon_for_model(app_name, model_name, default=icon) @@ -19,7 +19,7 @@ class AppDefaultIconList(CmsAppIconList): class OrchestraIndexDashboard(dashboard.FluentIndexDashboard): """ Gets application modules from services, accounts and administration registries """ - + def __init__(self, **kwargs): super(dashboard.FluentIndexDashboard, self).__init__(**kwargs) self.children.append(self.get_personal_module()) @@ -27,7 +27,7 @@ class OrchestraIndexDashboard(dashboard.FluentIndexDashboard): recent_actions = self.get_recent_actions_module() recent_actions.enabled = True self.children.append(recent_actions) - + def process_registered_view(self, module, view_name, options): app_name, name = view_name.split('_')[:-1] module.icons['.'.join((app_name, name))] = options.get('icon') @@ -47,7 +47,7 @@ class OrchestraIndexDashboard(dashboard.FluentIndexDashboard): 'title': options.get('verbose_name_plural'), 'url': add_url, }) - + def get_application_modules(self): modules = [] # Honor settings override, hacky. I Know diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py index 6f5db81e..1c16f761 100644 --- a/orchestra/admin/menu.py +++ b/orchestra/admin/menu.py @@ -1,7 +1,7 @@ from copy import deepcopy from admin_tools.menu import items, Menu -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.text import capfirst from django.utils.translation import ugettext_lazy as _ @@ -16,7 +16,7 @@ def api_link(context): opts = context['cl'].opts else: return reverse('api-root') - if 'object_id' in context: + if 'object_id' in context: object_id = context['object_id'] try: return reverse('%s-detail' % opts.model_name, args=[object_id]) @@ -42,7 +42,7 @@ def process_registry(register): item = items.MenuItem(name, url) item.options = options return item - + childrens = {} for model, options in register.get().items(): if options.get('menu', True): @@ -68,7 +68,7 @@ def process_registry(register): class OrchestraMenu(Menu): template = 'admin/orchestra/menu.html' - + def init_with_context(self, context): self.children = [ # items.MenuItem( diff --git a/orchestra/admin/utils.py b/orchestra/admin/utils.py index 9c485d2a..ccf22b49 100644 --- a/orchestra/admin/utils.py +++ b/orchestra/admin/utils.py @@ -6,7 +6,7 @@ from functools import wraps from django.conf import settings from django.contrib import admin from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.db import models from django.shortcuts import redirect from django.utils import timezone diff --git a/orchestra/api/helpers.py b/orchestra/api/helpers.py index 6bc8f749..67594455 100644 --- a/orchestra/api/helpers.py +++ b/orchestra/api/helpers.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import NoReverseMatch +from django.urls import NoReverseMatch from rest_framework.reverse import reverse diff --git a/orchestra/api/root.py b/orchestra/api/root.py index 12551571..51fa23c1 100644 --- a/orchestra/api/root.py +++ b/orchestra/api/root.py @@ -11,7 +11,7 @@ class APIRoot(views.APIView): 'ORCHESTRA_SITE_NAME', 'ORCHESTRA_SITE_VERBOSE_NAME' ) - + def get(self, request, format=None): root_url = reverse('api-root', request=request, format=format) token_url = reverse('api-token-auth', request=request, format=format) @@ -23,7 +23,7 @@ class APIRoot(views.APIView): 'accountancy': {}, 'services': {}, } - if not request.user.is_anonymous(): + if not request.user.is_anonymous: list_name = '{basename}-list' detail_name = '{basename}-detail' for prefix, viewset, basename in self.router.registry: @@ -60,7 +60,7 @@ class APIRoot(views.APIView): for name in self.names }) return Response(body, headers=headers) - + def options(self, request): metadata = super(APIRoot, self).options(request) metadata.data['settings'] = { diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index 82dba1b9..a5b6e1de 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -21,22 +21,22 @@ function help () { function print_help () { cat <<- EOF - + ${bold}NAME${normal} ${bold}orchestra-admin${normal} - Orchetsra administration script - + ${bold}OPTIONS${normal} ${bold}install_requirements${normal} Installs Orchestra requirements using apt-get and pip - + ${bold}startproject${normal} Creates a new Django-orchestra instance - + ${bold}help${normal} Displays this help text or related help page as argument for example: ${bold}orchestra-admin help startproject${normal} - + EOF } @@ -73,17 +73,17 @@ export -f get_orchestra_dir function print_install_requirements_help () { cat <<- EOF - + ${bold}NAME${normal} ${bold}orchetsra-admin install_requirements${normal} - Installs all Orchestra requirements using apt-get and pip - + ${bold}OPTIONS${normal} ${bold}-t, --testing${normal} Install Orchestra normal requirements plus those needed for running functional tests - + ${bold}-h, --help${normal} Displays this help text - + EOF } @@ -92,7 +92,7 @@ function install_requirements () { opts=$(getopt -o h,t -l help,testing -- "$@") || exit 1 set -- $opts testing=false - + while [ $# -gt 0 ]; do case $1 in -h|--help) print_deploy_help; exit 0 ;; @@ -105,17 +105,17 @@ function install_requirements () { done unset OPTIND unset opt - + check_root || true ORCHESTRA_PATH=$(get_orchestra_dir) || true - + # Make sure locales are in place before installing postgres if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen run locale-gen update-locale LANG=en_US.UTF-8 fi - + # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev APT="bind9utils \ ca-certificates \ @@ -136,10 +136,10 @@ function install_requirements () { iceweasel \ dnsutils" fi - + run apt-get update run apt-get install -y $APT - + # Install ca certificates before executing pip install if [[ ! -e /usr/local/share/ca-certificates/cacert.org ]]; then mkdir -p /usr/local/share/ca-certificates/cacert.org @@ -148,7 +148,7 @@ function install_requirements () { http://www.cacert.org/certs/class3.crt update-ca-certificates fi - + # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies PIP="$(wget http://git.io/orchestra-requirements.txt -O - | tr '\n' ' ') \ cracklib \ @@ -157,7 +157,7 @@ function install_requirements () { PIP="${PIP} \ selenium \ xvfbwrapper \ - freezegun \ + freezegun==0.3.14 \ coverage \ flake8 \ django-debug-toolbar==1.3.0 \ @@ -166,9 +166,9 @@ function install_requirements () { pyinotify \ PyMySQL" fi - + run pip3 install $PIP - + # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support) wkhtmltox_version=$(dpkg --list | grep wkhtmltox | awk {'print $3'}) minor=$(echo -e "$wkhtmltox_version\n0.12.2.1" | sort -V | head -n 1) @@ -183,30 +183,30 @@ export -f install_requirements print_startproject_help () { cat <<- EOF - + ${bold}NAME${normal} ${bold}orchestra-admin startproject${normal} - Create a new Django-Orchestra instance - + ${bold}SYNOPSIS${normal} Options: [ -h ] - + ${bold}OPTIONS${normal} ${bold}-h, --help${normal} This help message - + ${bold}EXAMPLES${normal} orchestra-admin startproject controlpanel - + EOF } function startproject () { local PROJECT_NAME="$2"; shift - + opts=$(getopt -o h -l help -- "$@") || exit 1 set -- $opts - + set -- $opts while [ $# -gt 0 ]; do case $1 in @@ -217,10 +217,10 @@ function startproject () { esac shift done - + unset OPTIND unset opt - + [ $(whoami) == 'root' ] && { echo -e "\nYou don't want to run this as root\n" >&2; exit 1; } ORCHESTRA_PATH=$(get_orchestra_dir) || { echo "Error getting orchestra dir"; exit 1; } if [[ ! -e $PROJECT_NAME/manage.py ]]; then diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat index 09d13fa2..b11eda09 100755 --- a/orchestra/bin/orchestra-beat +++ b/orchestra/bin/orchestra-beat @@ -27,7 +27,7 @@ class crontab_parser(object): _range = r'(\w+?)-(\w+)' _steps = r'/(\w+)?' _star = r'\*' - + def __init__(self, max_=60, min_=0): self.max_ = max_ self.min_ = min_ @@ -45,14 +45,14 @@ class crontab_parser(object): raise self.ParseException('empty part') acc |= set(self._parse_part(part)) return acc - + def _parse_part(self, part): for regex, handler in self.pats: m = regex.match(part) if m: return handler(m.groups()) return self._expand_range((part, )) - + def _expand_range(self, toks): fr = self._expand_number(toks[0]) if len(toks) > 1: @@ -62,19 +62,19 @@ class crontab_parser(object): list(range(self.min_, to + 1))) return list(range(fr, to + 1)) return [fr] - + def _range_steps(self, toks): if len(toks) != 3 or not toks[2]: raise self.ParseException('empty filter') return self._expand_range(toks[:2])[::int(toks[2])] - + def _star_steps(self, toks): if not toks or not toks[0]: raise self.ParseException('empty filter') return self._expand_star()[::int(toks[0])] def _expand_star(self, *args): return list(range(self.min_, self.max_ + self.min_)) - + def _expand_number(self, s): if isinstance(s, str) and s[0] == '-': raise self.ParseException('negative numbers not supported') @@ -99,7 +99,7 @@ class Setting(object): def __init__(self, manage): self.manage = manage self.settings_file = self.get_settings_file(manage) - + def get_settings(self): """ get db settings from settings.py file without importing """ settings = {'__file__': self.settings_file} @@ -111,7 +111,7 @@ class Setting(object): content += line exec(content, settings) return settings - + def get_settings_file(self, manage): with open(manage, 'r') as handler: regex = re.compile(r'"DJANGO_SETTINGS_MODULE"\s*,\s*"([^"]+)"') @@ -128,7 +128,7 @@ class Setting(object): class DB(object): def __init__(self, settings): self.settings = settings['DATABASES']['default'] - + def connect(self): if self.settings['ENGINE'] == 'django.db.backends.sqlite3': import sqlite3 @@ -138,7 +138,7 @@ class DB(object): self.conn = psycopg2.connect("dbname='{NAME}' user='{USER}' host='{HOST}' password='{PASSWORD}'".format(**self.settings)) else: raise ValueError("%s engine not supported." % self.settings['ENGINE']) - + def query(self, query): cur = self.conn.cursor() try: @@ -147,7 +147,7 @@ class DB(object): finally: cur.close() return result - + def close(self): self.conn.close() @@ -161,7 +161,7 @@ def fire_pending_tasks(manage, db): "WHERE p.crontab_id = c.id AND p.enabled = {}" ).format(enabled) return db.query(query) - + def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now return ( @@ -171,14 +171,14 @@ def fire_pending_tasks(manage, db): n_day_of_month in crontab_parser(31, 1).parse(day_of_month) and n_month_of_year in crontab_parser(12, 1).parse(month_of_year) ) - + now = datetime.utcnow() now = tuple(map(int, now.strftime("%M %H %w %d %m").split())) for minute, hour, day_of_week, day_of_month, month_of_year, task_id in get_tasks(db): if is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): command = 'python3 -W ignore::DeprecationWarning {manage} runtask {task_id}'.format( manage=manage, task_id=task_id) - proc = run(command, async=True) + proc = run(command, run_async=True) yield proc @@ -187,7 +187,7 @@ def fire_pending_messages(settings, db): MAILER_DEFERE_SECONDS = settings.get('MAILER_DEFERE_SECONDS', (300, 600, 60*60, 60*60*24)) now = datetime.utcnow() query_or = [] - + for num, seconds in enumerate(MAILER_DEFERE_SECONDS): delta = timedelta(seconds=seconds) epoch = now-delta @@ -198,10 +198,10 @@ def fire_pending_messages(settings, db): WHERE (mailer_message.state = 'QUEUED' OR (mailer_message.state = 'DEFERRED' AND (%s))) LIMIT 1""" % ' OR '.join(query_or) return bool(db.query(query)) - + if has_pending_messages(settings, db): command = 'python3 -W ignore::DeprecationWarning {manage} sendpendingmessages'.format(manage=manage) - proc = run(command, async=True) + proc = run(command, run_async=True) yield proc diff --git a/orchestra/conf/project_template/project_name/settings.py b/orchestra/conf/project_template/project_name/settings.py index 571f8762..209d91a4 100644 --- a/orchestra/conf/project_template/project_name/settings.py +++ b/orchestra/conf/project_template/project_name/settings.py @@ -228,7 +228,7 @@ REST_FRAMEWORK = { 'rest_framework.authentication.TokenAuthentication', ), 'DEFAULT_FILTER_BACKENDS': ( - ('rest_framework.filters.DjangoFilterBackend',) + ('django_filters.rest_framework.DjangoFilterBackend',) ), } diff --git a/orchestra/conf/ribaguifi_template/project_name/settings.py b/orchestra/conf/ribaguifi_template/project_name/settings.py index ee1a9512..2fa397a5 100644 --- a/orchestra/conf/ribaguifi_template/project_name/settings.py +++ b/orchestra/conf/ribaguifi_template/project_name/settings.py @@ -228,7 +228,7 @@ REST_FRAMEWORK = { 'rest_framework.authentication.TokenAuthentication', ), 'DEFAULT_FILTER_BACKENDS': ( - ('rest_framework.filters.DjangoFilterBackend',) + ('django_filters.rest_framework.DjangoFilterBackend',) ), } diff --git a/orchestra/contrib/accounts/actions.py b/orchestra/contrib/accounts/actions.py index f7f7ba91..da58022c 100644 --- a/orchestra/contrib/accounts/actions.py +++ b/orchestra/contrib/accounts/actions.py @@ -4,7 +4,7 @@ from django.contrib import messages from django.contrib.admin import helpers from django.contrib.admin.utils import NestedObjects, quote from django.contrib.auth import get_permission_codename -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.db import router from django.shortcuts import redirect, render from django.template.response import TemplateResponse @@ -53,14 +53,14 @@ def service_report(modeladmin, request, queryset): fields.append((model, name)) fields = sorted(fields, key=lambda f: f[0]._meta.verbose_name_plural.lower()) fields = [field for model, field in fields] - + for account in queryset.prefetch_related(*fields): items = [] for field in fields: related_manager = getattr(account, field) items.append((related_manager.model._meta, related_manager.all())) accounts.append((account, items)) - + context = { 'accounts': accounts, 'date': timezone.now().today() @@ -71,21 +71,21 @@ def service_report(modeladmin, request, queryset): def delete_related_services(modeladmin, request, queryset): opts = modeladmin.model._meta app_label = opts.app_label - + using = router.db_for_write(modeladmin.model) collector = NestedObjects(using=using) collector.collect(queryset) registered_services = services.get() related_services = [] to_delete = [] - + admin_site = modeladmin.admin_site - + def format(obj, account=False): has_admin = obj.__class__ in admin_site._registry opts = obj._meta no_edit_link = '%s: %s' % (capfirst(opts.verbose_name), force_text(obj)) - + if has_admin: try: admin_url = reverse( @@ -95,7 +95,7 @@ def delete_related_services(modeladmin, request, queryset): except NoReverseMatch: # Change url doesn't exist -- don't display link to edit return no_edit_link - + # Display a link to the admin page. context = (capfirst(opts.verbose_name), admin_url, obj) if account: @@ -106,7 +106,7 @@ def delete_related_services(modeladmin, request, queryset): # Don't display link to edit, because it either has no # admin or is edited inline. return no_edit_link - + def format_nested(objs, result): if isinstance(objs, list): current = [] @@ -115,7 +115,7 @@ def delete_related_services(modeladmin, request, queryset): result.append(current) else: result.append(format(objs)) - + for nested in collector.nested(): if isinstance(nested, list): # Is lists of objects @@ -141,7 +141,7 @@ def delete_related_services(modeladmin, request, queryset): # Prevent the deletion of the main system user, which will delete the account main_systemuser = nested.main_systemuser related_services.append(format(nested, account=True)) - + # The user has already confirmed the deletion. # Do the deletion and return a None to display the change list view again. if request.POST.get('post'): @@ -165,17 +165,17 @@ def delete_related_services(modeladmin, request, queryset): modeladmin.message_user(request, msg, messages.SUCCESS) # Return None to display the change list page again. return None - + if len(queryset) == 1: objects_name = force_text(opts.verbose_name) else: objects_name = force_text(opts.verbose_name_plural) - + model_count = {} for model, objs in collector.model_objs.items(): count = 0 # discount main systemuser - if model is modeladmin.model.main_systemuser.field.rel.to: + if model is modeladmin.model.main_systemuser.field.model: count = len(objs) - 1 # Discount account elif model is not modeladmin.model and model in registered_services: @@ -220,10 +220,10 @@ def disable_selected(modeladmin, request, queryset, disable=True): n) ) return None - + user = request.user admin_site = modeladmin.admin_site - + def format(obj): has_admin = obj.__class__ in admin_site._registry opts = obj._meta @@ -238,7 +238,7 @@ def disable_selected(modeladmin, request, queryset, disable=True): except NoReverseMatch: # Change url doesn't exist -- don't display link to edit return no_edit_link - + p = '%s.%s' % (opts.app_label, get_permission_codename('delete', opts)) if not user.has_perm(p): perms_needed.add(opts.verbose_name) @@ -249,19 +249,19 @@ def disable_selected(modeladmin, request, queryset, disable=True): # Don't display link to edit, because it either has no # admin or is edited inline. return no_edit_link - + display = [] for account in queryset: current = [] for related in account.get_services_to_disable(): current.append(format(related)) display.append([format(account), current]) - + if len(queryset) == 1: objects_name = force_text(opts.verbose_name) else: objects_name = force_text(opts.verbose_name_plural) - + context = dict( admin_site.each_context(request), action_name='disable_selected' if disable else 'enable_selected', diff --git a/orchestra/contrib/accounts/admin.py b/orchestra/contrib/accounts/admin.py index b2e7973b..2008fb6b 100644 --- a/orchestra/contrib/accounts/admin.py +++ b/orchestra/contrib/accounts/admin.py @@ -8,7 +8,7 @@ from django.conf.urls import url from django.contrib import admin, messages from django.contrib.admin.utils import unquote from django.contrib.auth import admin as auth -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponseRedirect from django.templatetags.static import static from django.utils.safestring import mark_safe @@ -71,15 +71,15 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) ) change_view_actions = (disable_selected, service_report, enable_selected) ordering = () - + main_systemuser_link = admin_link('main_systemuser') - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'comments': kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4}) return super(AccountAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): if not add: if request.method == 'GET' and not obj.is_active: @@ -96,7 +96,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) }) return super(AccountAdmin, self).render_change_form( request, context, add, change, form_url, obj) - + def get_fieldsets(self, request, obj=None): fieldsets = super(AccountAdmin, self).get_fieldsets(request, obj) if not obj: @@ -106,7 +106,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) fieldsets = list(fieldsets) fieldsets.insert(1, (_("Related services"), {'fields': fields})) return fieldsets - + def save_model(self, request, obj, form, change): if not change: form.save_model(obj) @@ -133,7 +133,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) if msg: messages.warning(request, mark_safe(msg % context)) super(AccountAdmin, self).save_model(request, obj, form, change) - + def get_change_view_actions(self, obj=None): views = super().get_change_view_actions(obj=obj) if obj is not None: @@ -141,7 +141,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) return [view for view in views if view.url_name != 'enable'] return [view for view in views if view.url_name != 'disable'] return views - + def get_actions(self, request): actions = super().get_actions(request) if 'delete_selected' in actions: @@ -157,7 +157,7 @@ class AccountListAdmin(AccountAdmin): list_display = ('select_account', 'username', 'type', 'username') actions = None change_list_template = 'admin/accounts/account/select_account_list.html' - + def select_account(self, instance): # TODO get query string from request.META['QUERY_STRING'] to preserve filters context = { @@ -169,7 +169,7 @@ class AccountListAdmin(AccountAdmin): select_account.short_description = _("account") select_account.allow_tags = True select_account.admin_order_field = 'username' - + def changelist_view(self, request, extra_context=None): app_label = request.META['PATH_INFO'].split('/')[-5] model = request.META['PATH_INFO'].split('/')[-4] @@ -206,7 +206,7 @@ class AccountAdminMixin(object): change_form_template = 'admin/accounts/account/change_form.html' account = None list_select_related = ('account',) - + def display_active(self, instance): if not instance.is_active: return 'False' % static('admin/img/icon-no.svg') @@ -217,14 +217,14 @@ class AccountAdminMixin(object): display_active.short_description = _("active") display_active.allow_tags = True display_active.admin_order_field = 'is_active' - + def account_link(self, instance): account = instance.account if instance.pk else self.account return admin_link()(account) account_link.short_description = _("account") account_link.allow_tags = True account_link.admin_order_field = 'account__username' - + def get_form(self, request, obj=None, **kwargs): """ Warns user when object's account is disabled """ form = super(AccountAdminMixin, self).get_form(request, obj, **kwargs) @@ -247,7 +247,7 @@ class AccountAdminMixin(object): # Not available in POST form.initial_account = self.get_changeform_initial_data(request).get('account') return form - + def get_fields(self, request, obj=None): """ remove account or account_link depending on the case """ fields = super(AccountAdminMixin, self).get_fields(request, obj) @@ -263,13 +263,13 @@ class AccountAdminMixin(object): except ValueError: pass return fields - + def get_readonly_fields(self, request, obj=None): """ provide account for filter_by_account_fields """ if obj: self.account = obj.account return super(AccountAdminMixin, self).get_readonly_fields(request, obj) - + def formfield_for_dbfield(self, db_field, **kwargs): """ Filter by account """ formfield = super(AccountAdminMixin, self).formfield_for_dbfield(db_field, **kwargs) @@ -277,14 +277,14 @@ class AccountAdminMixin(object): if self.account: # Hack widget render in order to append ?account=id to the add url old_render = formfield.widget.render - + def render(*args, **kwargs): output = old_render(*args, **kwargs) output = output.replace('/add/"', '/add/?account=%s"' % self.account.pk) with_qargs = r'/add/?\1&account=%s"' % self.account.pk output = re.sub(r'/add/\?([^".]*)"', with_qargs, output) return mark_safe(output) - + formfield.widget.render = render # Filter related object by account formfield.queryset = formfield.queryset.filter(account=self.account) @@ -302,21 +302,21 @@ class AccountAdminMixin(object): formfield.initial = 1 formfield.queryset = formfield.queryset.order_by('username') return formfield - + def get_formset(self, request, obj=None, **kwargs): """ provides form.account for convinience """ formset = super(AccountAdminMixin, self).get_formset(request, obj, **kwargs) formset.form.account = self.account formset.account = self.account return formset - + def get_account_from_preserve_filters(self, request): preserved_filters = self.get_preserved_filters(request) preserved_filters = dict(parse_qsl(preserved_filters)) cl_filters = preserved_filters.get('_changelist_filters') if cl_filters: return dict(parse_qsl(cl_filters)).get('account') - + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): account_id = self.get_account_from_preserve_filters(request) if not object_id: @@ -331,7 +331,7 @@ class AccountAdminMixin(object): context.update(extra_context or {}) return super(AccountAdminMixin, self).changeform_view( request, object_id, form_url=form_url, extra_context=context) - + def changelist_view(self, request, extra_context=None): account_id = request.GET.get('account') context = {} @@ -367,7 +367,7 @@ class SelectAccountAdminMixin(AccountAdminMixin): account = Account.objects.get(pk=request.GET['account']) [setattr(inline, 'account', account) for inline in inlines] return inlines - + def get_urls(self): """ Hooks select account url """ urls = super(AccountAdminMixin, self).get_urls() @@ -381,7 +381,7 @@ class SelectAccountAdminMixin(AccountAdminMixin): name='%s_%s_select_account' % info), ] return select_urls + urls - + def add_view(self, request, form_url='', extra_context=None): """ Redirects to select account view if required """ if request.user.is_superuser: @@ -406,7 +406,7 @@ class SelectAccountAdminMixin(AccountAdminMixin): return super(AccountAdminMixin, self).add_view( request, form_url=form_url, extra_context=context) return HttpResponseRedirect('./select-account/?%s' % request.META['QUERY_STRING']) - + def save_model(self, request, obj, form, change): """ Given a model instance save it to the database. diff --git a/orchestra/contrib/accounts/forms.py b/orchestra/contrib/accounts/forms.py index 4a7dcc6b..c3e308b1 100644 --- a/orchestra/contrib/accounts/forms.py +++ b/orchestra/contrib/accounts/forms.py @@ -34,7 +34,7 @@ def create_account_creation_form(): fields[field_name] = forms.BooleanField( initial=True, required=False, label=label, help_text=help_text) create_related.append((model, key, kwargs, help_text)) - + def clean(self, create_related=create_related): """ unique usernames between accounts and system users """ cleaned_data = UserCreationForm.clean(self) @@ -47,7 +47,7 @@ def create_account_creation_form(): # Previous validation error return errors = {} - systemuser_model = Account.main_systemuser.field.rel.to + systemuser_model = Account.main_systemuser.field.model if systemuser_model.objects.filter(username=account.username).exists(): errors['username'] = _("A system user with this name already exists.") for model, key, related_kwargs, __ in create_related: @@ -62,11 +62,11 @@ def create_account_creation_form(): params={'type': verbose_name}) if errors: raise ValidationError(errors) - + def save_model(self, account): enable_systemuser=self.cleaned_data['enable_systemuser'] account.save(active_systemuser=enable_systemuser) - + def save_related(self, account): for model, key, related_kwargs, __ in settings.ACCOUNTS_CREATE_RELATED: model = apps.get_model(model) @@ -76,14 +76,14 @@ def create_account_creation_form(): key: eval(value, {'account': account}) for key, value in related_kwargs.items() } model.objects.create(account=account, **kwargs) - + fields.update({ 'create_related_fields': list(fields.keys()), 'clean': clean, 'save_model': save_model, 'save_related': save_related, }) - + return type('AccountCreationForm', (UserCreationForm,), fields) diff --git a/orchestra/contrib/accounts/migrations/0001_initial.py b/orchestra/contrib/accounts/migrations/0001_initial.py index e6ff1344..a855bc5f 100644 --- a/orchestra/contrib/accounts/migrations/0001_initial.py +++ b/orchestra/contrib/accounts/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations import django.core.validators +import django.db.models.deletion import django.utils.timezone import django.contrib.auth.models @@ -32,7 +33,7 @@ class Migration(migrations.Migration): ('is_superuser', models.BooleanField(help_text='Designates that this user has all permissions without explicitly assigning them.', default=False, verbose_name='superuser status')), ('is_active', models.BooleanField(help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', default=True, verbose_name='active')), ('date_joined', models.DateTimeField(verbose_name='date joined', default=django.utils.timezone.now)), - ('main_systemuser', models.ForeignKey(to='systemusers.SystemUser', editable=False, null=True, related_name='accounts_main')), + ('main_systemuser', models.ForeignKey(to='systemusers.SystemUser', editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='accounts_main')), ], options={ 'abstract': False, diff --git a/orchestra/contrib/accounts/migrations/0001_squashed_0004_auto_20210422_1108.py b/orchestra/contrib/accounts/migrations/0001_squashed_0004_auto_20210422_1108.py new file mode 100644 index 00000000..69810092 --- /dev/null +++ b/orchestra/contrib/accounts/migrations/0001_squashed_0004_auto_20210422_1108.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:08 +from __future__ import unicode_literals + +import django.contrib.auth.models +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import orchestra.contrib.accounts.models + + +class Migration(migrations.Migration): + + replaces = [('accounts', '0001_initial'), ('accounts', '0002_auto_20170528_2005'), ('accounts', '0003_auto_20210330_1049'), ('accounts', '0004_auto_20210422_1108')] + + initial = True + + dependencies = [ + ('systemusers', '0001_initial'), + ('auth', '0006_require_contenttypes_0002'), + ] + + operations = [ + migrations.CreateModel( + name='Account', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('username', models.CharField(help_text='Required. 64 characters or fewer. Letters, digits and ./-/_ only.', max_length=32, unique=True, validators=[django.core.validators.RegexValidator('^[\\w.-]+$', 'Enter a valid username.', 'invalid')], verbose_name='username')), + ('short_name', models.CharField(blank=True, max_length=64, verbose_name='short name')), + ('full_name', models.CharField(max_length=256, verbose_name='full name')), + ('email', models.EmailField(help_text='Used for password recovery', max_length=254, verbose_name='email address')), + ('type', models.CharField(choices=[('INDIVIDUAL', 'Individual'), ('ASSOCIATION', 'Association'), ('CUSTOMER', 'Customer'), ('COMPANY', 'Company'), ('PUBLICBODY', 'Public body'), ('STAFF', 'Staff'), ('FRIEND', 'Friend')], default='INDIVIDUAL', max_length=32, verbose_name='type')), + ('language', models.CharField(choices=[('EN', 'English')], default='EN', max_length=2, verbose_name='language')), + ('comments', models.TextField(blank=True, max_length=256, verbose_name='comments')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('main_systemuser', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='accounts_main', to='systemusers.SystemUser')), + ], + options={ + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + migrations.AlterModelManagers( + name='account', + managers=[ + ('objects', orchestra.contrib.accounts.models.AccountManager()), + ], + ), + migrations.AlterField( + model_name='account', + name='language', + field=models.CharField(choices=[('CA', 'Catalan'), ('ES', 'Spanish'), ('EN', 'English')], default='CA', max_length=2, verbose_name='language'), + ), + migrations.AlterField( + model_name='account', + name='type', + field=models.CharField(choices=[('INDIVIDUAL', 'Individual'), ('ASSOCIATION', 'Association'), ('CUSTOMER', 'Customer'), ('STAFF', 'Staff'), ('FRIEND', 'Friend')], default='INDIVIDUAL', max_length=32, verbose_name='type'), + ), + migrations.AlterField( + model_name='account', + name='username', + field=models.CharField(help_text='Required. 32 characters or fewer. Letters, digits and ./-/_ only.', max_length=32, unique=True, validators=[django.core.validators.RegexValidator('^[\\w.-]+$', 'Enter a valid username.', 'invalid')], verbose_name='username'), + ), + migrations.AlterField( + model_name='account', + name='language', + field=models.CharField(choices=[('EN', 'English')], default='EN', max_length=2, verbose_name='language'), + ), + migrations.AlterField( + model_name='account', + name='type', + field=models.CharField(choices=[('INDIVIDUAL', 'Individual'), ('ASSOCIATION', 'Association'), ('CUSTOMER', 'Customer'), ('COMPANY', 'Company'), ('PUBLICBODY', 'Public body'), ('STAFF', 'Staff'), ('FRIEND', 'Friend')], default='INDIVIDUAL', max_length=32, verbose_name='type'), + ), + migrations.AlterField( + model_name='account', + name='main_systemuser', + field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='accounts_main', to='systemusers.SystemUser'), + ), + ] diff --git a/orchestra/contrib/accounts/migrations/0003_auto_20210330_1049.py b/orchestra/contrib/accounts/migrations/0003_auto_20210330_1049.py new file mode 100644 index 00000000..ae8dc55a --- /dev/null +++ b/orchestra/contrib/accounts/migrations/0003_auto_20210330_1049.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0002_auto_20170528_2005'), + ] + + operations = [ + migrations.AlterField( + model_name='account', + name='language', + field=models.CharField(choices=[('EN', 'English')], default='EN', max_length=2, verbose_name='language'), + ), + migrations.AlterField( + model_name='account', + name='type', + field=models.CharField(choices=[('INDIVIDUAL', 'Individual'), ('ASSOCIATION', 'Association'), ('CUSTOMER', 'Customer'), ('COMPANY', 'Company'), ('PUBLICBODY', 'Public body'), ('STAFF', 'Staff'), ('FRIEND', 'Friend')], default='INDIVIDUAL', max_length=32, verbose_name='type'), + ), + ] diff --git a/orchestra/contrib/accounts/models.py b/orchestra/contrib/accounts/models.py index 52d77525..94cd29c6 100644 --- a/orchestra/contrib/accounts/models.py +++ b/orchestra/contrib/accounts/models.py @@ -29,7 +29,7 @@ class Account(auth.AbstractBaseUser): validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username."), 'invalid') ]) main_systemuser = models.ForeignKey(settings.ACCOUNTS_SYSTEMUSER_MODEL, null=True, - related_name='accounts_main', editable=False) + related_name='accounts_main', editable=False, on_delete=models.SET_NULL) short_name = models.CharField(_("short name"), max_length=64, blank=True) full_name = models.CharField(_("full name"), max_length=256) email = models.EmailField(_('email address'), help_text=_("Used for password recovery")) @@ -46,23 +46,28 @@ class Account(auth.AbstractBaseUser): help_text=_("Designates whether this account should be treated as active. " "Unselect this instead of deleting accounts.")) date_joined = models.DateTimeField(_("date joined"), default=timezone.now) - + objects = AccountManager() - + USERNAME_FIELD = 'username' REQUIRED_FIELDS = ['email'] - + + def __init__(self, *args, **kwargs): + # ignore `is_staff` kwarg because is handled with `is_superuser` + kwargs.pop('is_staff', None) + super().__init__(*args, **kwargs) + def __str__(self): return self.name - + @property def name(self): return self.username - + @property def is_staff(self): return self.is_superuser - + def save(self, active_systemuser=False, *args, **kwargs): created = not self.pk if not created: @@ -75,21 +80,21 @@ class Account(auth.AbstractBaseUser): self.save(update_fields=('main_systemuser',)) elif was_active != self.is_active: self.notify_related() - + def clean(self): self.short_name = self.short_name.strip() self.full_name = self.full_name.strip() - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) self.notify_related() - + def enable(self): self.is_active = True self.save(update_fields=('is_active',)) self.notify_related() - + def get_services_to_disable(self): related_fields = [ f for f in self._meta.get_fields() @@ -101,20 +106,20 @@ class Account(auth.AbstractBaseUser): if source in core.services and hasattr(source, 'active'): for obj in getattr(self, rel.get_accessor_name()).all(): yield obj - + def notify_related(self): """ Trigger save() on related objects that depend on this account """ for obj in self.get_services_to_disable(): signals.pre_save.send(sender=type(obj), instance=obj) signals.post_save.send(sender=type(obj), instance=obj) # OperationsMiddleware.collect(Operation.SAVE, instance=obj, update_fields=()) - + def get_contacts_emails(self, usages=None): contacts = self.contacts.all() if usages is not None: contactes = contacts.filter(email_usages=usages) return contacts.values_list('email', flat=True) - + def send_email(self, template, context, email_from=None, usages=None, attachments=[], html=None): contacts = self.contacts.filter(email_usages=usages) email_to = self.get_contacts_emails(usages) @@ -126,14 +131,14 @@ class Account(auth.AbstractBaseUser): with translation.override(self.language): send_email_template(template, extra_context, email_to, email_from=email_from, html=html, attachments=attachments) - + def get_full_name(self): return self.full_name or self.short_name or self.username - + def get_short_name(self): """ Returns the short name for the user """ return self.short_name or self.username or self.full_name - + def has_perm(self, perm, obj=None): """ Returns True if the user has the specified permission. This method @@ -160,7 +165,7 @@ class Account(auth.AbstractBaseUser): elif obj and getattr(obj, 'account', None) == self: return True - + def has_perms(self, perm_list, obj=None): """ Returns True if the user has each of the specified permissions. If @@ -171,7 +176,7 @@ class Account(auth.AbstractBaseUser): if not self.has_perm(perm, obj): return False return True - + def has_module_perms(self, app_label): """ Returns True if the user has any permissions in the given app label. diff --git a/orchestra/contrib/bills/actions.py b/orchestra/contrib/bills/actions.py index dc36938b..37723b42 100644 --- a/orchestra/contrib/bills/actions.py +++ b/orchestra/contrib/bills/actions.py @@ -5,7 +5,7 @@ from datetime import date from django.contrib import messages from django.contrib.admin import helpers from django.core.exceptions import ValidationError -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import transaction from django.forms.models import modelformset_factory from django.http import HttpResponse, HttpResponseRedirect @@ -179,7 +179,7 @@ def undo_billing(modeladmin, request, queryset): group[line.order].append(line) except KeyError: group[line.order] = [line] - + # Validate for order, lines in group.items(): prev = None @@ -211,7 +211,7 @@ def undo_billing(modeladmin, request, queryset): messages.error(request, "Order does not have lines!.") order.billed_until = billed_until order.billed_on = billed_on - + # Commit changes norders, nlines = 0, 0 for order, lines in group.items(): @@ -221,7 +221,7 @@ def undo_billing(modeladmin, request, queryset): # TODO update order history undo billing order.save(update_fields=('billed_until', 'billed_on')) norders += 1 - + messages.success(request, _("%(norders)s orders and %(nlines)s lines undoed.") % { 'nlines': nlines, 'norders': norders diff --git a/orchestra/contrib/bills/admin.py b/orchestra/contrib/bills/admin.py index befd2235..560819c8 100644 --- a/orchestra/contrib/bills/admin.py +++ b/orchestra/contrib/bills/admin.py @@ -2,7 +2,7 @@ from django import forms from django.conf.urls import url from django.contrib import admin, messages from django.contrib.admin.utils import unquote -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.db.models import F, Sum, Prefetch from django.db.models.functions import Coalesce @@ -39,18 +39,18 @@ PAYMENT_STATE_COLORS = { class BillSublineInline(admin.TabularInline): model = BillSubline fields = ('description', 'total', 'type') - + def get_readonly_fields(self, request, obj=None): fields = super().get_readonly_fields(request, obj) if obj and not obj.bill.is_open: return self.get_fields(request) return fields - + def get_max_num(self, request, obj=None): if obj and not obj.bill.is_open: return 0 return super().get_max_num(request, obj) - + def has_delete_permission(self, request, obj=None): if obj and not obj.bill.is_open: return False @@ -64,9 +64,9 @@ class BillLineInline(admin.TabularInline): 'subtotal', 'display_total', ) readonly_fields = ('display_total', 'order_link') - + order_link = admin_link('order', display='pk') - + def display_total(self, line): if line.pk: total = line.compute_total() @@ -79,7 +79,7 @@ class BillLineInline(admin.TabularInline): return '%s' % (url, total) display_total.short_description = _("Total") display_total.allow_tags = True - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'description': @@ -87,7 +87,7 @@ class BillLineInline(admin.TabularInline): elif db_field.name not in ('start_on', 'end_on'): kwargs['widget'] = forms.TextInput(attrs={'size':'6'}) return super().formfield_for_dbfield(db_field, **kwargs) - + def get_queryset(self, request): qs = super().get_queryset(request) return qs.prefetch_related('sublines').select_related('order') @@ -96,14 +96,14 @@ class BillLineInline(admin.TabularInline): class ClosedBillLineInline(BillLineInline): # TODO reimplement as nested inlines when upstream # https://code.djangoproject.com/ticket/9025 - + fields = ( 'display_description', 'order_link', 'start_on', 'end_on', 'rate', 'quantity', 'tax', 'display_subtotal', 'display_total' ) readonly_fields = fields can_delete = False - + def display_description(self, line): descriptions = [line.description] for subline in line.sublines.all(): @@ -111,7 +111,7 @@ class ClosedBillLineInline(BillLineInline): return '
'.join(descriptions) display_description.short_description = _("Description") display_description.allow_tags = True - + def display_subtotal(self, line): subtotals = [' ' + str(line.subtotal)] for subline in line.sublines.all(): @@ -119,13 +119,13 @@ class ClosedBillLineInline(BillLineInline): return '
'.join(subtotals) display_subtotal.short_description = _("Subtotal") display_subtotal.allow_tags = True - + def display_total(self, line): if line.pk: return line.compute_total() display_total.short_description = _("Total") display_total.allow_tags = True - + def has_add_permission(self, request): return False @@ -158,28 +158,28 @@ class BillLineAdmin(admin.ModelAdmin): list_select_related = ('bill', 'bill__account') search_fields = ('description', 'bill__number') inlines = (BillSublineInline,) - + account_link = admin_link('bill__account') bill_link = admin_link('bill') order_link = admin_link('order') amended_line_link = admin_link('amended_line') - + def display_is_open(self, instance): return instance.bill.is_open display_is_open.short_description = _("Is open") display_is_open.boolean = True - + def display_sublinetotal(self, instance): total = instance.subline_total return total if total is not None else '---' display_sublinetotal.short_description = _("Sublines") display_sublinetotal.admin_order_field = 'subline_total' - + def display_total(self, instance): return round(instance.computed_total or 0, 2) display_total.short_description = _("Total") display_total.admin_order_field = 'computed_total' - + def get_readonly_fields(self, request, obj=None): fields = super().get_readonly_fields(request, obj) if obj and not obj.bill.is_open: @@ -188,7 +188,7 @@ class BillLineAdmin(admin.ModelAdmin): 'subtotal', 'order_billed_on', 'order_billed_until' ] return fields - + def get_queryset(self, request): qs = super().get_queryset(request) qs = qs.annotate( @@ -196,7 +196,7 @@ class BillLineAdmin(admin.ModelAdmin): computed_total=(F('subtotal') + Sum(Coalesce('sublines__total', 0))) * (1+F('tax')/100), ) return qs - + def has_delete_permission(self, request, obj=None): if obj and not obj.bill.is_open: return False @@ -209,7 +209,7 @@ class BillLineManagerAdmin(BillLineAdmin): if self.bill_ids: return qset.filter(bill_id__in=self.bill_ids) return qset - + def changelist_view(self, request, extra_context=None): GET_copy = request.GET.copy() bill_ids = GET_copy.pop('ids', None) @@ -304,9 +304,9 @@ class AmendInline(BillAdminMixin, admin.TabularInline): verbose_name_plural = _("Amends") can_delete = False extra = 0 - + self_link = admin_link('__str__') - + def has_add_permission(self, *args, **kwargs): return False @@ -354,12 +354,12 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): 'closed_on_display', 'updated_on_display', 'display_total_with_subtotals', ) date_hierarchy = 'closed_on' - + created_on_display = admin_date('created_on', short_description=_("Created")) closed_on_display = admin_date('closed_on', short_description=_("Closed")) updated_on_display = admin_date('updated_on', short_description=_("Updated")) amend_of_link = admin_link('amend_of') - + # def amend_links(self, bill): # links = [] # for amend in bill.amends.all(): @@ -368,19 +368,19 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): # return '
'.join(links) # amend_links.short_description = _("Amends") # amend_links.allow_tags = True - + def num_lines(self, bill): return bill.lines__count num_lines.admin_order_field = 'lines__count' num_lines.short_description = _("lines") - + def display_total(self, bill): currency = settings.BILLS_CURRENCY.lower() return '%s &%s;' % (bill.compute_total(), currency) display_total.allow_tags = True display_total.short_description = _("total") display_total.admin_order_field = 'approx_total' - + def type_link(self, bill): bill_type = bill.type.lower() url = reverse('admin:bills_%s_changelist' % bill_type) @@ -388,7 +388,7 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): type_link.allow_tags = True type_link.short_description = _("type") type_link.admin_order_field = 'type' - + def get_urls(self): """ Hook bill lines management URLs on bill admin """ urls = super().get_urls() @@ -399,13 +399,13 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): name='bills_bill_manage_lines'), ] return extra_urls + urls - + def get_readonly_fields(self, request, obj=None): fields = super().get_readonly_fields(request, obj) if obj and not obj.is_open: fields += self.add_fields return fields - + def get_fieldsets(self, request, obj=None): fieldsets = super().get_fieldsets(request, obj) if obj: @@ -418,7 +418,7 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): if obj.is_open: fieldsets = fieldsets[0:-1] return fieldsets - + def get_change_view_actions(self, obj=None): actions = super().get_change_view_actions(obj) exclude = [] @@ -428,7 +428,7 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): if obj.type not in obj.AMEND_MAP: exclude += ['amend_bills'] return [action for action in actions if action.__name__ not in exclude] - + def get_inline_instances(self, request, obj=None): cls = type(self) if obj and not obj.is_open: @@ -439,7 +439,7 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): else: cls.inlines = [BillLineInline] return super().get_inline_instances(request, obj) - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'comments': @@ -450,7 +450,7 @@ class BillAdmin(BillAdminMixin, ExtendedModelAdmin): if db_field.name == 'amend_of': formfield.queryset = formfield.queryset.filter(is_open=False) return formfield - + def change_view(self, request, object_id, **kwargs): # TODO raise404, here and everywhere bill = self.get_object(request, unquote(object_id)) @@ -471,7 +471,7 @@ admin.site.register(BillLine, BillLineAdmin) class BillContactInline(admin.StackedInline): model = BillContact fields = ('name', 'address', ('city', 'zipcode'), 'country', 'vat') - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'name': diff --git a/orchestra/contrib/bills/filters.py b/orchestra/contrib/bills/filters.py index de45ca37..a4beb32f 100644 --- a/orchestra/contrib/bills/filters.py +++ b/orchestra/contrib/bills/filters.py @@ -1,5 +1,5 @@ from django.contrib.admin import SimpleListFilter -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import Q from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ @@ -11,11 +11,11 @@ class BillTypeListFilter(SimpleListFilter): """ Filter tickets by created_by according to request.user """ title = 'Type' parameter_name = '' - + def __init__(self, request, *args, **kwargs): super(BillTypeListFilter, self).__init__(request, *args, **kwargs) self.request = request - + def lookups(self, request, model_admin): return ( ('bill', _("All")), @@ -25,13 +25,13 @@ class BillTypeListFilter(SimpleListFilter): ('amendmentfee', _("Amendment fee")), ('amendmentinvoice', _("Amendment invoice")), ) - + def queryset(self, request, queryset): return queryset - + def value(self): return self.request.path.split('/')[-2] - + def choices(self, cl): query = self.request.GET.urlencode() for lookup, title in self.lookup_choices: @@ -45,7 +45,7 @@ class BillTypeListFilter(SimpleListFilter): class TotalListFilter(SimpleListFilter): title = _("total") parameter_name = 'total' - + def lookups(self, request, model_admin): return ( ('gt', mark_safe("total > 0")), @@ -53,7 +53,7 @@ class TotalListFilter(SimpleListFilter): ('eq', "total = 0"), ('ne', mark_safe("total ≠ 0")), ) - + def queryset(self, request, queryset): if self.value() == 'gt': return queryset.filter(approx_total__gt=0) @@ -70,13 +70,13 @@ class HasBillContactListFilter(SimpleListFilter): """ Filter Nodes by group according to request.user """ title = _("has bill contact") parameter_name = 'bill' - + def lookups(self, request, model_admin): return ( ('True', _("Yes")), ('False', _("No")), ) - + def queryset(self, request, queryset): if self.value() == 'True': return queryset.filter(billcontact__isnull=False) @@ -87,7 +87,7 @@ class HasBillContactListFilter(SimpleListFilter): class PaymentStateListFilter(SimpleListFilter): title = _("payment state") parameter_name = 'payment_state' - + def lookups(self, request, model_admin): return ( ('OPEN', _("Open")), @@ -95,7 +95,7 @@ class PaymentStateListFilter(SimpleListFilter): ('PENDING', _("Pending")), ('BAD_DEBT', _("Bad debt")), ) - + def queryset(self, request, queryset): # FIXME use queryset.computed_total instead of approx_total, bills.admin.BillAdmin.get_queryset Transaction = queryset.model.transactions.field.remote_field.related_model @@ -137,7 +137,7 @@ class PaymentStateListFilter(SimpleListFilter): class AmendedListFilter(SimpleListFilter): title = _("amended") parameter_name = 'amended' - + def lookups(self, request, model_admin): return ( ('3', _("Closed amends")), @@ -145,7 +145,7 @@ class AmendedListFilter(SimpleListFilter): ('1', _("Any amends")), ('0', _("No amends")), ) - + def queryset(self, request, queryset): if self.value() is None: return queryset diff --git a/orchestra/contrib/bills/helpers.py b/orchestra/contrib/bills/helpers.py index 99fc3319..e5986b5c 100644 --- a/orchestra/contrib/bills/helpers.py +++ b/orchestra/contrib/bills/helpers.py @@ -1,5 +1,5 @@ from django.contrib import messages -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.encoding import force_text from django.utils.html import format_html from django.utils.safestring import mark_safe @@ -21,7 +21,7 @@ def validate_contact(request, bill, error=True): message = msg.format(relation=_("Related"), account=account, url=url) send(request, mark_safe(message)) valid = False - main = type(bill).account.field.rel.to.objects.get_main() + main = type(bill).account.field.model.objects.get_main() if not hasattr(main, 'billcontact'): account = force_text(main) url = reverse('admin:accounts_account_change', args=(main.id,)) diff --git a/orchestra/contrib/bills/migrations/0001_initial.py b/orchestra/contrib/bills/migrations/0001_initial.py index a572887e..ba1d9dcb 100644 --- a/orchestra/contrib/bills/migrations/0001_initial.py +++ b/orchestra/contrib/bills/migrations/0001_initial.py @@ -30,7 +30,7 @@ class Migration(migrations.Migration): ('total', models.DecimalField(default=0, decimal_places=2, max_digits=12)), ('comments', models.TextField(verbose_name='comments', blank=True)), ('html', models.TextField(verbose_name='HTML', blank=True)), - ('account', models.ForeignKey(related_name='bill', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bill', to=settings.AUTH_USER_MODEL, verbose_name='account')), ], options={ 'get_latest_by': 'id', @@ -46,7 +46,7 @@ class Migration(migrations.Migration): ('zipcode', models.CharField(verbose_name='zip code', validators=[django.core.validators.RegexValidator('^[0-9A-Z]{3,10}$', 'Enter a valid zipcode.')], max_length=10)), ('country', models.CharField(default='ES', verbose_name='country', choices=[('PR', 'Puerto Rico'), ('BV', 'Bouvet Island'), ('BT', 'Bhutan'), ('MY', 'Malaysia'), ('AQ', 'Antarctica'), ('MT', 'Malta'), ('BE', 'Belgium'), ('SM', 'San Marino'), ('AZ', 'Azerbaijan'), ('CA', 'Canada'), ('HR', 'Croatia'), ('GH', 'Ghana'), ('MZ', 'Mozambique'), ('PA', 'Panama'), ('GR', 'Greece'), ('AE', 'United Arab Emirates'), ('CK', 'Cook Islands'), ('SK', 'Slovakia'), ('PN', 'Pitcairn'), ('ZA', 'South Africa'), ('AU', 'Australia'), ('BF', 'Burkina Faso'), ('FI', 'Finland'), ('MC', 'Monaco'), ('RE', 'Réunion'), ('TV', 'Tuvalu'), ('HN', 'Honduras'), ('IL', 'Israel'), ('SV', 'El Salvador'), ('VN', 'Viet Nam'), ('MV', 'Maldives'), ('BA', 'Bosnia and Herzegovina'), ('UA', 'Ukraine'), ('BW', 'Botswana'), ('UZ', 'Uzbekistan'), ('ID', 'Indonesia'), ('LY', 'Libya'), ('MM', 'Myanmar'), ('TZ', 'Tanzania, United Republic of'), ('GL', 'Greenland'), ('LV', 'Latvia'), ('DZ', 'Algeria'), ('AO', 'Angola'), ('GE', 'Georgia'), ('SO', 'Somalia'), ('CX', 'Christmas Island'), ('NP', 'Nepal'), ('AI', 'Anguilla'), ('GP', 'Guadeloupe'), ('UY', 'Uruguay'), ('LC', 'Saint Lucia'), ('EH', 'Western Sahara'), ('IO', 'British Indian Ocean Territory'), ('TH', 'Thailand'), ('AR', 'Argentina'), ('PY', 'Paraguay'), ('AW', 'Aruba'), ('IE', 'Ireland'), ('CF', 'Central African Republic'), ('TW', 'Taiwan (Province of China)'), ('KZ', 'Kazakhstan'), ('TJ', 'Tajikistan'), ('RW', 'Rwanda'), ('SR', 'Suriname'), ('AT', 'Austria'), ('GN', 'Guinea'), ('SS', 'South Sudan'), ('IT', 'Italy'), ('BO', 'Bolivia (Plurinational State of)'), ('GD', 'Grenada'), ('KE', 'Kenya'), ('GS', 'South Georgia and the South Sandwich Islands'), ('TG', 'Togo'), ('CY', 'Cyprus'), ('TT', 'Trinidad and Tobago'), ('CM', 'Cameroon'), ('QA', 'Qatar'), ('GM', 'Gambia'), ('WS', 'Samoa'), ('DJ', 'Djibouti'), ('PL', 'Poland'), ('CV', 'Cabo Verde'), ('PE', 'Peru'), ('TN', 'Tunisia'), ('HT', 'Haiti'), ('AF', 'Afghanistan'), ('YT', 'Mayotte'), ('NR', 'Nauru'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('IR', 'Iran (Islamic Republic of)'), ('EE', 'Estonia'), ('ER', 'Eritrea'), ('RU', 'Russian Federation'), ('LB', 'Lebanon'), ('CU', 'Cuba'), ('CZ', 'Czech Republic'), ('AX', 'Ã…land Islands'), ('CD', 'Congo (the Democratic Republic of the)'), ('HK', 'Hong Kong'), ('PS', 'Palestine, State of'), ('FK', 'Falkland Islands [Malvinas]'), ('MR', 'Mauritania'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('LK', 'Sri Lanka'), ('NA', 'Namibia'), ('SI', 'Slovenia'), ('BY', 'Belarus'), ('MX', 'Mexico'), ('ZW', 'Zimbabwe'), ('CI', "Côte d'Ivoire"), ('GU', 'Guam'), ('PW', 'Palau'), ('SC', 'Seychelles'), ('GT', 'Guatemala'), ('CL', 'Chile'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('BH', 'Bahrain'), ('SB', 'Solomon Islands'), ('KM', 'Comoros'), ('MF', 'Saint Martin (French part)'), ('FO', 'Faroe Islands'), ('BD', 'Bangladesh'), ('AM', 'Armenia'), ('BJ', 'Benin'), ('SA', 'Saudi Arabia'), ('NU', 'Niue'), ('VI', 'Virgin Islands (U.S.)'), ('CN', 'China'), ('EC', 'Ecuador'), ('CC', 'Cocos (Keeling) Islands'), ('BS', 'Bahamas'), ('JM', 'Jamaica'), ('RO', 'Romania'), ('KI', 'Kiribati'), ('GY', 'Guyana'), ('TL', 'Timor-Leste'), ('AS', 'American Samoa'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('BN', 'Brunei Darussalam'), ('ET', 'Ethiopia'), ('FJ', 'Fiji'), ('BG', 'Bulgaria'), ('VG', 'Virgin Islands (British)'), ('AD', 'Andorra'), ('KN', 'Saint Kitts and Nevis'), ('MA', 'Morocco'), ('MU', 'Mauritius'), ('DK', 'Denmark'), ('TO', 'Tonga'), ('CO', 'Colombia'), ('GA', 'Gabon'), ('MK', 'Macedonia (the former Yugoslav Republic of)'), ('MD', 'Moldova (the Republic of)'), ('VA', 'Holy See'), ('KY', 'Cayman Islands'), ('LU', 'Luxembourg'), ('AL', 'Albania'), ('LI', 'Liechtenstein'), ('KP', "Korea (the Democratic People's Republic of)"), ('BZ', 'Belize'), ('ME', 'Montenegro'), ('NC', 'New Caledonia'), ('VC', 'Saint Vincent and the Grenadines'), ('UM', 'United States Minor Outlying Islands'), ('IQ', 'Iraq'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('KW', 'Kuwait'), ('MS', 'Montserrat'), ('SD', 'Sudan'), ('JP', 'Japan'), ('DE', 'Germany'), ('SN', 'Senegal'), ('PK', 'Pakistan'), ('MO', 'Macao'), ('RS', 'Serbia'), ('KR', 'Korea (the Republic of)'), ('BL', 'Saint Barthélemy'), ('MP', 'Northern Mariana Islands'), ('AG', 'Antigua and Barbuda'), ('FM', 'Micronesia (Federated States of)'), ('HM', 'Heard Island and McDonald Islands'), ('BR', 'Brazil'), ('PF', 'French Polynesia'), ('MQ', 'Martinique'), ('VU', 'Vanuatu'), ('LT', 'Lithuania'), ('ES', 'Spain'), ('ML', 'Mali'), ('NE', 'Niger'), ('EG', 'Egypt'), ('WF', 'Wallis and Futuna'), ('ZM', 'Zambia'), ('US', 'United States of America'), ('DO', 'Dominican Republic'), ('NO', 'Norway'), ('UG', 'Uganda'), ('GQ', 'Equatorial Guinea'), ('GW', 'Guinea-Bissau'), ('JE', 'Jersey'), ('HU', 'Hungary'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('JO', 'Jordan'), ('ST', 'Sao Tome and Principe'), ('SJ', 'Svalbard and Jan Mayen'), ('MN', 'Mongolia'), ('BB', 'Barbados'), ('CH', 'Switzerland'), ('KG', 'Kyrgyzstan'), ('PG', 'Papua New Guinea'), ('NG', 'Nigeria'), ('GF', 'French Guiana'), ('CR', 'Costa Rica'), ('LA', "Lao People's Democratic Republic"), ('CG', 'Congo'), ('NI', 'Nicaragua'), ('TK', 'Tokelau'), ('SE', 'Sweden'), ('NF', 'Norfolk Island'), ('PT', 'Portugal'), ('FR', 'France'), ('SZ', 'Swaziland'), ('YE', 'Yemen'), ('NL', 'Netherlands'), ('GI', 'Gibraltar'), ('TR', 'Turkey'), ('OM', 'Oman'), ('TF', 'French Southern Territories'), ('PM', 'Saint Pierre and Miquelon'), ('TC', 'Turks and Caicos Islands'), ('SL', 'Sierra Leone'), ('SX', 'Sint Maarten (Dutch part)'), ('DM', 'Dominica'), ('KH', 'Cambodia'), ('SG', 'Singapore'), ('BM', 'Bermuda'), ('CW', 'Curaçao'), ('GG', 'Guernsey'), ('TD', 'Chad'), ('IN', 'India'), ('BI', 'Burundi'), ('TM', 'Turkmenistan'), ('IS', 'Iceland'), ('MW', 'Malawi'), ('SY', 'Syrian Arab Republic'), ('NZ', 'New Zealand'), ('IM', 'Isle of Man'), ('PH', 'Philippines')], max_length=20)), ('vat', models.CharField(verbose_name='VAT number', max_length=64)), - ('account', models.OneToOneField(related_name='billcontact', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ('account', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='billcontact', to=settings.AUTH_USER_MODEL, verbose_name='account')), ], ), migrations.CreateModel( @@ -64,8 +64,8 @@ class Migration(migrations.Migration): ('order_billed_on', models.DateField(verbose_name='order billed', null=True, blank=True)), ('order_billed_until', models.DateField(verbose_name='order billed until', null=True, blank=True)), ('created_on', models.DateField(auto_now_add=True, verbose_name='created')), - ('amended_line', models.ForeignKey(blank=True, to='bills.BillLine', verbose_name='amended line', null=True, related_name='amendment_lines')), - ('bill', models.ForeignKey(related_name='lines', to='bills.Bill', verbose_name='bill')), + ('amended_line', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='bills.BillLine', verbose_name='amended line', null=True, related_name='amendment_lines')), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='lines', to='bills.Bill', verbose_name='bill')), ('order', models.ForeignKey(blank=True, to='orders.Order', null=True, on_delete=django.db.models.deletion.SET_NULL, help_text='Informative link back to the order')), ], ), @@ -76,7 +76,7 @@ class Migration(migrations.Migration): ('description', models.CharField(verbose_name='description', max_length=256)), ('total', models.DecimalField(decimal_places=2, max_digits=12)), ('type', models.CharField(default='OTHER', verbose_name='type', choices=[('VOLUME', 'Volume'), ('COMPENSATION', 'Compensation'), ('OTHER', 'Other')], max_length=16)), - ('line', models.ForeignKey(related_name='sublines', to='bills.BillLine', verbose_name='bill line')), + ('line', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sublines', to='bills.BillLine', verbose_name='bill line')), ], ), migrations.CreateModel( diff --git a/orchestra/contrib/bills/migrations/0001_squashed_0017_auto_20210422_1108.py b/orchestra/contrib/bills/migrations/0001_squashed_0017_auto_20210422_1108.py new file mode 100644 index 00000000..7070870d --- /dev/null +++ b/orchestra/contrib/bills/migrations/0001_squashed_0017_auto_20210422_1108.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:08 +from __future__ import unicode_literals + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('bills', '0001_initial'), ('bills', '0002_auto_20150429_1417'), ('bills', '0003_auto_20150612_0944'), ('bills', '0004_auto_20150618_1311'), ('bills', '0005_auto_20150623_1031'), ('bills', '0006_auto_20150709_1016'), ('bills', '0007_auto_20170528_2011'), ('bills', '0008_auto_20170625_1813'), ('bills', '0009_auto_20170625_1840'), ('bills', '0010_auto_20170625_1840'), ('bills', '0011_auto_20170625_1840'), ('bills', '0012_auto_20170625_1841'), ('bills', '0013_auto_20190805_1134'), ('bills', '0014_auto_20200204_1217'), ('bills', '0015_auto_20200204_1218'), ('bills', '0016_auto_20210330_1049'), ('bills', '0017_auto_20210422_1108')] + + initial = True + + dependencies = [ + ('orders', '__first__'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Bill', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.CharField(blank=True, max_length=16, unique=True, verbose_name='number')), + ('type', models.CharField(choices=[('INVOICE', 'Invoice'), ('AMENDMENTINVOICE', 'Amendment invoice'), ('FEE', 'Fee'), ('AMENDMENTFEE', 'Amendment Fee'), ('PROFORMA', 'Pro forma')], max_length=16, verbose_name='type')), + ('created_on', models.DateField(auto_now_add=True, verbose_name='created on')), + ('closed_on', models.DateField(blank=True, null=True, verbose_name='closed on')), + ('is_open', models.BooleanField(default=True, verbose_name='open')), + ('is_sent', models.BooleanField(default=False, verbose_name='sent')), + ('due_on', models.DateField(blank=True, null=True, verbose_name='due on')), + ('updated_on', models.DateField(auto_now=True, verbose_name='updated on')), + ('total', models.DecimalField(decimal_places=2, default=0, max_digits=12)), + ('comments', models.TextField(blank=True, verbose_name='comments')), + ('html', models.TextField(blank=True, verbose_name='HTML')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bill', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + options={ + 'get_latest_by': 'id', + }, + ), + migrations.CreateModel( + name='BillContact', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, help_text='Account full name will be used when left blank.', max_length=256, verbose_name='name')), + ('address', models.TextField(verbose_name='address')), + ('city', models.CharField(default='Barcelona', max_length=128, verbose_name='city')), + ('zipcode', models.CharField(max_length=10, validators=[django.core.validators.RegexValidator('^[0-9A-Z]{3,10}$', 'Enter a valid zipcode.')], verbose_name='zip code')), + ('country', models.CharField(choices=[('AF', 'Afghanistan'), ('AX', 'Ã…land Islands'), ('AL', 'Albania'), ('DZ', 'Algeria'), ('AS', 'American Samoa'), ('AD', 'Andorra'), ('AO', 'Angola'), ('AI', 'Anguilla'), ('AQ', 'Antarctica'), ('AG', 'Antigua and Barbuda'), ('AR', 'Argentina'), ('AM', 'Armenia'), ('AW', 'Aruba'), ('AU', 'Australia'), ('AT', 'Austria'), ('AZ', 'Azerbaijan'), ('BS', 'Bahamas'), ('BH', 'Bahrain'), ('BD', 'Bangladesh'), ('BB', 'Barbados'), ('BY', 'Belarus'), ('BE', 'Belgium'), ('BZ', 'Belize'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BT', 'Bhutan'), ('BO', 'Bolivia (Plurinational State of)'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('BA', 'Bosnia and Herzegovina'), ('BW', 'Botswana'), ('BV', 'Bouvet Island'), ('BR', 'Brazil'), ('IO', 'British Indian Ocean Territory'), ('BN', 'Brunei Darussalam'), ('BG', 'Bulgaria'), ('BF', 'Burkina Faso'), ('BI', 'Burundi'), ('CV', 'Cabo Verde'), ('KH', 'Cambodia'), ('CM', 'Cameroon'), ('CA', 'Canada'), ('KY', 'Cayman Islands'), ('CF', 'Central African Republic'), ('TD', 'Chad'), ('CL', 'Chile'), ('CN', 'China'), ('CX', 'Christmas Island'), ('CC', 'Cocos (Keeling) Islands'), ('CO', 'Colombia'), ('KM', 'Comoros'), ('CG', 'Congo'), ('CD', 'Congo (the Democratic Republic of the)'), ('CK', 'Cook Islands'), ('CR', 'Costa Rica'), ('CI', "Côte d'Ivoire"), ('HR', 'Croatia'), ('CU', 'Cuba'), ('CW', 'Curaçao'), ('CY', 'Cyprus'), ('CZ', 'Czechia'), ('DK', 'Denmark'), ('DJ', 'Djibouti'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('EC', 'Ecuador'), ('EG', 'Egypt'), ('SV', 'El Salvador'), ('GQ', 'Equatorial Guinea'), ('ER', 'Eritrea'), ('EE', 'Estonia'), ('SZ', 'Eswatini'), ('ET', 'Ethiopia'), ('FK', 'Falkland Islands (Malvinas)'), ('FO', 'Faroe Islands'), ('FJ', 'Fiji'), ('FI', 'Finland'), ('FR', 'France'), ('GF', 'French Guiana'), ('PF', 'French Polynesia'), ('TF', 'French Southern Territories'), ('GA', 'Gabon'), ('GM', 'Gambia'), ('GE', 'Georgia'), ('DE', 'Germany'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GR', 'Greece'), ('GL', 'Greenland'), ('GD', 'Grenada'), ('GP', 'Guadeloupe'), ('GU', 'Guam'), ('GT', 'Guatemala'), ('GG', 'Guernsey'), ('GN', 'Guinea'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HT', 'Haiti'), ('HM', 'Heard Island and McDonald Islands'), ('VA', 'Holy See'), ('HN', 'Honduras'), ('HK', 'Hong Kong'), ('HU', 'Hungary'), ('IS', 'Iceland'), ('IN', 'India'), ('ID', 'Indonesia'), ('IR', 'Iran (Islamic Republic of)'), ('IQ', 'Iraq'), ('IE', 'Ireland'), ('IM', 'Isle of Man'), ('IL', 'Israel'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JP', 'Japan'), ('JE', 'Jersey'), ('JO', 'Jordan'), ('KZ', 'Kazakhstan'), ('KE', 'Kenya'), ('KI', 'Kiribati'), ('KP', "Korea (the Democratic People's Republic of)"), ('KR', 'Korea (the Republic of)'), ('KW', 'Kuwait'), ('KG', 'Kyrgyzstan'), ('LA', "Lao People's Democratic Republic"), ('LV', 'Latvia'), ('LB', 'Lebanon'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('LY', 'Libya'), ('LI', 'Liechtenstein'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('MO', 'Macao'), ('MG', 'Madagascar'), ('MW', 'Malawi'), ('MY', 'Malaysia'), ('MV', 'Maldives'), ('ML', 'Mali'), ('MT', 'Malta'), ('MH', 'Marshall Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MU', 'Mauritius'), ('YT', 'Mayotte'), ('MX', 'Mexico'), ('FM', 'Micronesia (Federated States of)'), ('MD', 'Moldova (the Republic of)'), ('MC', 'Monaco'), ('MN', 'Mongolia'), ('ME', 'Montenegro'), ('MS', 'Montserrat'), ('MA', 'Morocco'), ('MZ', 'Mozambique'), ('MM', 'Myanmar'), ('NA', 'Namibia'), ('NR', 'Nauru'), ('NP', 'Nepal'), ('NL', 'Netherlands'), ('NC', 'New Caledonia'), ('NZ', 'New Zealand'), ('NI', 'Nicaragua'), ('NE', 'Niger'), ('NG', 'Nigeria'), ('NU', 'Niue'), ('NF', 'Norfolk Island'), ('MK', 'North Macedonia'), ('MP', 'Northern Mariana Islands'), ('NO', 'Norway'), ('OM', 'Oman'), ('PK', 'Pakistan'), ('PW', 'Palau'), ('PS', 'Palestine, State of'), ('PA', 'Panama'), ('PG', 'Papua New Guinea'), ('PY', 'Paraguay'), ('PE', 'Peru'), ('PH', 'Philippines'), ('PN', 'Pitcairn'), ('PL', 'Poland'), ('PT', 'Portugal'), ('PR', 'Puerto Rico'), ('QA', 'Qatar'), ('RE', 'Réunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('BL', 'Saint Barthélemy'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('KN', 'Saint Kitts and Nevis'), ('LC', 'Saint Lucia'), ('MF', 'Saint Martin (French part)'), ('PM', 'Saint Pierre and Miquelon'), ('VC', 'Saint Vincent and the Grenadines'), ('WS', 'Samoa'), ('SM', 'San Marino'), ('ST', 'Sao Tome and Principe'), ('SA', 'Saudi Arabia'), ('SN', 'Senegal'), ('RS', 'Serbia'), ('SC', 'Seychelles'), ('SL', 'Sierra Leone'), ('SG', 'Singapore'), ('SX', 'Sint Maarten (Dutch part)'), ('SK', 'Slovakia'), ('SI', 'Slovenia'), ('SB', 'Solomon Islands'), ('SO', 'Somalia'), ('ZA', 'South Africa'), ('GS', 'South Georgia and the South Sandwich Islands'), ('SS', 'South Sudan'), ('ES', 'Spain'), ('LK', 'Sri Lanka'), ('SD', 'Sudan'), ('SR', 'Suriname'), ('SJ', 'Svalbard and Jan Mayen'), ('SE', 'Sweden'), ('CH', 'Switzerland'), ('SY', 'Syrian Arab Republic'), ('TW', 'Taiwan (Province of China)'), ('TJ', 'Tajikistan'), ('TZ', 'Tanzania, the United Republic of'), ('TH', 'Thailand'), ('TL', 'Timor-Leste'), ('TG', 'Togo'), ('TK', 'Tokelau'), ('TO', 'Tonga'), ('TT', 'Trinidad and Tobago'), ('TN', 'Tunisia'), ('TR', 'Turkey'), ('TM', 'Turkmenistan'), ('TC', 'Turks and Caicos Islands'), ('TV', 'Tuvalu'), ('UG', 'Uganda'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VU', 'Vanuatu'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('VN', 'Viet Nam'), ('VG', 'Virgin Islands (British)'), ('VI', 'Virgin Islands (U.S.)'), ('WF', 'Wallis and Futuna'), ('EH', 'Western Sahara'), ('YE', 'Yemen'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe')], default='ES', max_length=20, verbose_name='country')), + ('vat', models.CharField(max_length=64, verbose_name='VAT number')), + ('account', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='billcontact', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + ), + migrations.CreateModel( + name='BillLine', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.CharField(max_length=256, verbose_name='description')), + ('rate', models.DecimalField(blank=True, decimal_places=2, max_digits=12, null=True, verbose_name='rate')), + ('quantity', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='quantity')), + ('verbose_quantity', models.CharField(max_length=16, verbose_name='Verbose quantity')), + ('subtotal', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='subtotal')), + ('tax', models.DecimalField(decimal_places=2, max_digits=4, verbose_name='tax')), + ('start_on', models.DateField(verbose_name='start')), + ('end_on', models.DateField(null=True, verbose_name='end')), + ('order_billed_on', models.DateField(blank=True, null=True, verbose_name='order billed')), + ('order_billed_until', models.DateField(blank=True, null=True, verbose_name='order billed until')), + ('created_on', models.DateField(auto_now_add=True, verbose_name='created')), + ('amended_line', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='amendment_lines', to='bills.BillLine', verbose_name='amended line')), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='lines', to='bills.Bill', verbose_name='bill')), + ('order', models.ForeignKey(blank=True, help_text='Informative link back to the order', null=True, on_delete=django.db.models.deletion.SET_NULL, to='orders.Order')), + ], + ), + migrations.CreateModel( + name='BillSubline', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.CharField(max_length=256, verbose_name='description')), + ('total', models.DecimalField(decimal_places=2, max_digits=12)), + ('type', models.CharField(choices=[('VOLUME', 'Volume'), ('COMPENSATION', 'Compensation'), ('OTHER', 'Other')], default='OTHER', max_length=16, verbose_name='type')), + ('line', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sublines', to='bills.BillLine', verbose_name='bill line')), + ], + ), + migrations.CreateModel( + name='AmendmentFee', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.CreateModel( + name='AmendmentInvoice', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.CreateModel( + name='Fee', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.CreateModel( + name='Invoice', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.CreateModel( + name='ProForma', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.RemoveField( + model_name='bill', + name='total', + ), + migrations.AlterField( + model_name='billline', + name='quantity', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=12, null=True, verbose_name='quantity'), + ), + migrations.AddField( + model_name='bill', + name='amend_of', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='amends', to='bills.Bill', verbose_name='amend of'), + ), + migrations.AlterField( + model_name='bill', + name='closed_on', + field=models.DateField(blank=True, db_index=True, null=True, verbose_name='closed on'), + ), + migrations.AlterField( + model_name='billline', + name='end_on', + field=models.DateField(blank=True, null=True, verbose_name='end'), + ), + migrations.AlterModelOptions( + name='billline', + options={'get_latest_by': 'id'}, + ), + migrations.AlterField( + model_name='billline', + name='order', + field=models.ForeignKey(blank=True, help_text='Informative link back to the order', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='lines', to='orders.Order'), + ), + migrations.AlterField( + model_name='billline', + name='verbose_quantity', + field=models.CharField(blank=True, max_length=16, verbose_name='Verbose quantity'), + ), + migrations.CreateModel( + name='AbonoInvoice', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('bills.bill',), + ), + migrations.AlterField( + model_name='bill', + name='type', + field=models.CharField(choices=[('INVOICE', 'Invoice'), ('AMENDMENTINVOICE', 'Amendment invoice'), ('FEE', 'Fee'), ('AMENDMENTFEE', 'Amendment Fee'), ('ABONOINVOICE', 'Abono Invoice'), ('PROFORMA', 'Pro forma')], max_length=16, verbose_name='type'), + ), + migrations.AlterField( + model_name='bill', + name='amend_of', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='amends', to='bills.Bill', verbose_name='amend of'), + ), + ] diff --git a/orchestra/contrib/bills/migrations/0005_auto_20150623_1031.py b/orchestra/contrib/bills/migrations/0005_auto_20150623_1031.py index d00b968e..92aba3b9 100644 --- a/orchestra/contrib/bills/migrations/0005_auto_20150623_1031.py +++ b/orchestra/contrib/bills/migrations/0005_auto_20150623_1031.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import django.db.models.deletion from django.db import models, migrations @@ -14,7 +15,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='bill', name='amend_of', - field=models.ForeignKey(to='bills.Bill', blank=True, related_name='amends', verbose_name='amend of', null=True), + field=models.ForeignKey(to='bills.Bill', blank=True, on_delete=django.db.models.deletion.CASCADE, related_name='amends', verbose_name='amend of', null=True), ), migrations.AlterField( model_name='billcontact', diff --git a/orchestra/contrib/bills/migrations/0016_auto_20210330_1049.py b/orchestra/contrib/bills/migrations/0016_auto_20210330_1049.py new file mode 100644 index 00000000..7df6b7a1 --- /dev/null +++ b/orchestra/contrib/bills/migrations/0016_auto_20210330_1049.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bills', '0015_auto_20200204_1218'), + ] + + operations = [ + migrations.AlterField( + model_name='billcontact', + name='country', + field=models.CharField(choices=[('AF', 'Afghanistan'), ('AX', 'Ã…land Islands'), ('AL', 'Albania'), ('DZ', 'Algeria'), ('AS', 'American Samoa'), ('AD', 'Andorra'), ('AO', 'Angola'), ('AI', 'Anguilla'), ('AQ', 'Antarctica'), ('AG', 'Antigua and Barbuda'), ('AR', 'Argentina'), ('AM', 'Armenia'), ('AW', 'Aruba'), ('AU', 'Australia'), ('AT', 'Austria'), ('AZ', 'Azerbaijan'), ('BS', 'Bahamas'), ('BH', 'Bahrain'), ('BD', 'Bangladesh'), ('BB', 'Barbados'), ('BY', 'Belarus'), ('BE', 'Belgium'), ('BZ', 'Belize'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BT', 'Bhutan'), ('BO', 'Bolivia (Plurinational State of)'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('BA', 'Bosnia and Herzegovina'), ('BW', 'Botswana'), ('BV', 'Bouvet Island'), ('BR', 'Brazil'), ('IO', 'British Indian Ocean Territory'), ('BN', 'Brunei Darussalam'), ('BG', 'Bulgaria'), ('BF', 'Burkina Faso'), ('BI', 'Burundi'), ('CV', 'Cabo Verde'), ('KH', 'Cambodia'), ('CM', 'Cameroon'), ('CA', 'Canada'), ('KY', 'Cayman Islands'), ('CF', 'Central African Republic'), ('TD', 'Chad'), ('CL', 'Chile'), ('CN', 'China'), ('CX', 'Christmas Island'), ('CC', 'Cocos (Keeling) Islands'), ('CO', 'Colombia'), ('KM', 'Comoros'), ('CG', 'Congo'), ('CD', 'Congo (the Democratic Republic of the)'), ('CK', 'Cook Islands'), ('CR', 'Costa Rica'), ('CI', "Côte d'Ivoire"), ('HR', 'Croatia'), ('CU', 'Cuba'), ('CW', 'Curaçao'), ('CY', 'Cyprus'), ('CZ', 'Czechia'), ('DK', 'Denmark'), ('DJ', 'Djibouti'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('EC', 'Ecuador'), ('EG', 'Egypt'), ('SV', 'El Salvador'), ('GQ', 'Equatorial Guinea'), ('ER', 'Eritrea'), ('EE', 'Estonia'), ('SZ', 'Eswatini'), ('ET', 'Ethiopia'), ('FK', 'Falkland Islands (Malvinas)'), ('FO', 'Faroe Islands'), ('FJ', 'Fiji'), ('FI', 'Finland'), ('FR', 'France'), ('GF', 'French Guiana'), ('PF', 'French Polynesia'), ('TF', 'French Southern Territories'), ('GA', 'Gabon'), ('GM', 'Gambia'), ('GE', 'Georgia'), ('DE', 'Germany'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GR', 'Greece'), ('GL', 'Greenland'), ('GD', 'Grenada'), ('GP', 'Guadeloupe'), ('GU', 'Guam'), ('GT', 'Guatemala'), ('GG', 'Guernsey'), ('GN', 'Guinea'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HT', 'Haiti'), ('HM', 'Heard Island and McDonald Islands'), ('VA', 'Holy See'), ('HN', 'Honduras'), ('HK', 'Hong Kong'), ('HU', 'Hungary'), ('IS', 'Iceland'), ('IN', 'India'), ('ID', 'Indonesia'), ('IR', 'Iran (Islamic Republic of)'), ('IQ', 'Iraq'), ('IE', 'Ireland'), ('IM', 'Isle of Man'), ('IL', 'Israel'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JP', 'Japan'), ('JE', 'Jersey'), ('JO', 'Jordan'), ('KZ', 'Kazakhstan'), ('KE', 'Kenya'), ('KI', 'Kiribati'), ('KP', "Korea (the Democratic People's Republic of)"), ('KR', 'Korea (the Republic of)'), ('KW', 'Kuwait'), ('KG', 'Kyrgyzstan'), ('LA', "Lao People's Democratic Republic"), ('LV', 'Latvia'), ('LB', 'Lebanon'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('LY', 'Libya'), ('LI', 'Liechtenstein'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('MO', 'Macao'), ('MG', 'Madagascar'), ('MW', 'Malawi'), ('MY', 'Malaysia'), ('MV', 'Maldives'), ('ML', 'Mali'), ('MT', 'Malta'), ('MH', 'Marshall Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MU', 'Mauritius'), ('YT', 'Mayotte'), ('MX', 'Mexico'), ('FM', 'Micronesia (Federated States of)'), ('MD', 'Moldova (the Republic of)'), ('MC', 'Monaco'), ('MN', 'Mongolia'), ('ME', 'Montenegro'), ('MS', 'Montserrat'), ('MA', 'Morocco'), ('MZ', 'Mozambique'), ('MM', 'Myanmar'), ('NA', 'Namibia'), ('NR', 'Nauru'), ('NP', 'Nepal'), ('NL', 'Netherlands'), ('NC', 'New Caledonia'), ('NZ', 'New Zealand'), ('NI', 'Nicaragua'), ('NE', 'Niger'), ('NG', 'Nigeria'), ('NU', 'Niue'), ('NF', 'Norfolk Island'), ('MK', 'North Macedonia'), ('MP', 'Northern Mariana Islands'), ('NO', 'Norway'), ('OM', 'Oman'), ('PK', 'Pakistan'), ('PW', 'Palau'), ('PS', 'Palestine, State of'), ('PA', 'Panama'), ('PG', 'Papua New Guinea'), ('PY', 'Paraguay'), ('PE', 'Peru'), ('PH', 'Philippines'), ('PN', 'Pitcairn'), ('PL', 'Poland'), ('PT', 'Portugal'), ('PR', 'Puerto Rico'), ('QA', 'Qatar'), ('RE', 'Réunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('BL', 'Saint Barthélemy'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('KN', 'Saint Kitts and Nevis'), ('LC', 'Saint Lucia'), ('MF', 'Saint Martin (French part)'), ('PM', 'Saint Pierre and Miquelon'), ('VC', 'Saint Vincent and the Grenadines'), ('WS', 'Samoa'), ('SM', 'San Marino'), ('ST', 'Sao Tome and Principe'), ('SA', 'Saudi Arabia'), ('SN', 'Senegal'), ('RS', 'Serbia'), ('SC', 'Seychelles'), ('SL', 'Sierra Leone'), ('SG', 'Singapore'), ('SX', 'Sint Maarten (Dutch part)'), ('SK', 'Slovakia'), ('SI', 'Slovenia'), ('SB', 'Solomon Islands'), ('SO', 'Somalia'), ('ZA', 'South Africa'), ('GS', 'South Georgia and the South Sandwich Islands'), ('SS', 'South Sudan'), ('ES', 'Spain'), ('LK', 'Sri Lanka'), ('SD', 'Sudan'), ('SR', 'Suriname'), ('SJ', 'Svalbard and Jan Mayen'), ('SE', 'Sweden'), ('CH', 'Switzerland'), ('SY', 'Syrian Arab Republic'), ('TW', 'Taiwan (Province of China)'), ('TJ', 'Tajikistan'), ('TZ', 'Tanzania, the United Republic of'), ('TH', 'Thailand'), ('TL', 'Timor-Leste'), ('TG', 'Togo'), ('TK', 'Tokelau'), ('TO', 'Tonga'), ('TT', 'Trinidad and Tobago'), ('TN', 'Tunisia'), ('TR', 'Turkey'), ('TM', 'Turkmenistan'), ('TC', 'Turks and Caicos Islands'), ('TV', 'Tuvalu'), ('UG', 'Uganda'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VU', 'Vanuatu'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('VN', 'Viet Nam'), ('VG', 'Virgin Islands (British)'), ('VI', 'Virgin Islands (U.S.)'), ('WF', 'Wallis and Futuna'), ('EH', 'Western Sahara'), ('YE', 'Yemen'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe')], default='ES', max_length=20, verbose_name='country'), + ), + ] diff --git a/orchestra/contrib/bills/models.py b/orchestra/contrib/bills/models.py index eb90776f..b39661a9 100644 --- a/orchestra/contrib/bills/models.py +++ b/orchestra/contrib/bills/models.py @@ -1,7 +1,7 @@ import datetime from dateutil.relativedelta import relativedelta -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.validators import ValidationError, RegexValidator from django.db import models from django.db.models import F, Sum @@ -24,7 +24,7 @@ from . import settings class BillContact(models.Model): account = models.OneToOneField('accounts.Account', verbose_name=_("account"), - related_name='billcontact') + related_name='billcontact', on_delete=models.CASCADE) name = models.CharField(_("name"), max_length=256, blank=True, help_text=_("Account full name will be used when left blank.")) address = models.TextField(_("address")) @@ -36,13 +36,13 @@ class BillContact(models.Model): choices=settings.BILLS_CONTACT_COUNTRIES, default=settings.BILLS_CONTACT_DEFAULT_COUNTRY) vat = models.CharField(_("VAT number"), max_length=64) - + def __str__(self): return self.name - + def get_name(self): return self.name or self.account.get_full_name() - + def clean(self): self.vat = self.vat.strip() self.city = self.city.strip() @@ -99,12 +99,12 @@ class Bill(models.Model): INVOICE: AMENDMENTINVOICE, FEE: AMENDMENTFEE, } - + number = models.CharField(_("number"), max_length=16, unique=True, blank=True) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='%(class)s') + related_name='%(class)s', on_delete=models.CASCADE) amend_of = models.ForeignKey('self', null=True, blank=True, verbose_name=_("amend of"), - related_name='amends') + related_name='amends', on_delete=models.SET_NULL) type = models.CharField(_("type"), max_length=16, choices=TYPES) created_on = models.DateField(_("created on"), auto_now_add=True) closed_on = models.DateField(_("closed on"), blank=True, null=True, db_index=True) @@ -115,37 +115,37 @@ class Bill(models.Model): # total = models.DecimalField(max_digits=12, decimal_places=2, null=True) comments = models.TextField(_("comments"), blank=True) html = models.TextField(_("HTML"), blank=True) - + objects = BillManager() - + class Meta: get_latest_by = 'id' - + def __str__(self): return self.number - + @classmethod def get_class_type(cls): if cls is models.DEFERRED: cls = cls.__base__ return cls.__name__.upper() - + @cached_property def total(self): return self.compute_total() - + @cached_property def seller(self): return Account.objects.get_main().billcontact - + @cached_property def buyer(self): return self.account.billcontact - + @property def has_multiple_pages(self): return self.type != self.FEE - + @cached_property def payment_state(self): if self.is_open or self.get_type() == self.PROFORMA: @@ -192,7 +192,7 @@ class Bill(models.Model): elif executed: return self.EXECUTED return self.BAD_DEBT - + def clean(self): if self.amend_of_id: errors = {} @@ -206,27 +206,27 @@ class Bill(models.Model): errors['amend_of'] = _("Related invoice is an amendment.") if errors: raise ValidationError(errors) - + def get_payment_state_display(self): value = self.payment_state return force_text(dict(self.PAYMENT_STATES).get(value, value)) - + def get_current_transaction(self): return self.transactions.exclude_rejected().first() - + def get_type(self): return self.type or self.get_class_type() - + @property def is_amend(self): return self.type in self.AMEND_MAP.values() - + def get_amend_type(self): amend_type = self.AMEND_MAP.get(self.type) if amend_type is None: raise TypeError("%s has no associated amend type." % self.type) return amend_type - + def get_number(self): cls = type(self) if cls is models.DEFERRED: @@ -250,16 +250,16 @@ class Bill(models.Model): zeros = (number_length - len(str(number))) * '0' number = zeros + str(number) return '{prefix}{year}{number}'.format(prefix=prefix, year=year, number=number) - + def get_due_date(self, payment=None): now = timezone.now() if payment: return now + payment.get_due_delta() return now + relativedelta(months=1) - + def get_absolute_url(self): return reverse('admin:bills_bill_view', args=(self.pk,)) - + def close(self, payment=False): if not self.is_open: raise TypeError("Bill not in Open state.") @@ -278,10 +278,10 @@ class Bill(models.Model): self.html = self.render(payment=payment) self.save() return transaction - + def get_billing_contact_emails(self): return self.account.get_contacts_emails(usages=(Contact.BILLING,)) - + def send(self): pdf = self.as_pdf() self.account.send_email( @@ -298,7 +298,7 @@ class Bill(models.Model): ) self.is_sent = True self.save(update_fields=['is_sent']) - + def render(self, payment=False, language=None): with translation.override(language or self.account.language): if payment is False: @@ -325,22 +325,22 @@ class Bill(models.Model): html = bill_template.render(context) html = html.replace('-pageskip-', '') return html - + def as_pdf(self): html = self.html or self.render() return html_to_pdf(html, pagination=self.has_multiple_pages) - + def updated(self): self.updated_on = timezone.now() self.save(update_fields=('updated_on',)) - + def save(self, *args, **kwargs): if not self.type: self.type = self.get_type() if not self.number: self.number = self.get_number() super(Bill, self).save(*args, **kwargs) - + @cached def compute_subtotals(self): subtotals = {} @@ -354,21 +354,21 @@ class Bill(models.Model): for tax, subtotal in subtotals.items(): result[tax] = [subtotal, round(tax/100*subtotal, 2)] return result - + @cached def compute_base(self): bases = self.lines.annotate( bases=F('subtotal') + Sum(Coalesce('sublines__total', 0)) ) return round(bases.aggregate(Sum('bases'))['bases__sum'] or 0, 2) - + @cached def compute_tax(self): taxes = self.lines.annotate( taxes=(F('subtotal') + Coalesce(Sum('sublines__total'), 0)) * (F('tax')/100) ) return round(taxes.aggregate(Sum('taxes'))['taxes__sum'] or 0, 2) - + @cached def compute_total(self): if 'lines' in getattr(self, '_prefetched_objects_cache', ()): @@ -416,7 +416,7 @@ class ProForma(Bill): class BillLine(models.Model): """ Base model for bill item representation """ - bill = models.ForeignKey(Bill, verbose_name=_("bill"), related_name='lines') + bill = models.ForeignKey(Bill, verbose_name=_("bill"), related_name='lines', on_delete=models.CASCADE) description = models.CharField(_("description"), max_length=256) rate = models.DecimalField(_("rate"), blank=True, null=True, max_digits=12, decimal_places=2) quantity = models.DecimalField(_("quantity"), blank=True, null=True, max_digits=12, @@ -434,24 +434,24 @@ class BillLine(models.Model): created_on = models.DateField(_("created"), auto_now_add=True) # Amendment amended_line = models.ForeignKey('self', verbose_name=_("amended line"), - related_name='amendment_lines', null=True, blank=True) - + related_name='amendment_lines', null=True, blank=True, on_delete=models.CASCADE) + class Meta: get_latest_by = 'id' - + def __str__(self): return "#%i" % self.pk if self.pk else self.description - + def get_verbose_quantity(self): return self.verbose_quantity or self.quantity - + def clean(self): if not self.verbose_quantity: quantity = str(self.quantity) # Strip trailing zeros if quantity.endswith('0'): self.verbose_quantity = quantity.strip('0').strip('.') - + def get_verbose_period(self): from django.template.defaultfilters import date date_format = "N 'y" @@ -467,7 +467,7 @@ class BillLine(models.Model): if ini == end: return ini return "{ini} / {end}".format(ini=ini, end=end) - + @cached def compute_total(self): total = self.subtotal or 0 @@ -478,7 +478,7 @@ class BillLine(models.Model): else: total += self.sublines.aggregate(sub_total=Sum('total'))['sub_total'] or 0 return round(total, 2) - + def get_absolute_url(self): return change_url(self) @@ -493,12 +493,12 @@ class BillSubline(models.Model): (COMPENSATION, _("Compensation")), (OTHER, _("Other")), ) - + # TODO: order info for undoing - line = models.ForeignKey(BillLine, verbose_name=_("bill line"), related_name='sublines') + line = models.ForeignKey(BillLine, verbose_name=_("bill line"), related_name='sublines', on_delete=models.CASCADE) description = models.CharField(_("description"), max_length=256) total = models.DecimalField(max_digits=12, decimal_places=2) type = models.CharField(_("type"), max_length=16, choices=TYPES, default=OTHER) - + def __str__(self): return "%s %i" % (self.description, self.total) diff --git a/orchestra/contrib/contacts/migrations/0001_initial.py b/orchestra/contrib/contacts/migrations/0001_initial.py index 5b66d4cf..a3e5ae07 100644 --- a/orchestra/contrib/contacts/migrations/0001_initial.py +++ b/orchestra/contrib/contacts/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations import django.core.validators +import django.db.models.deletion import orchestra.contrib.contacts.validators import orchestra.models.fields from django.conf import settings @@ -29,7 +30,7 @@ class Migration(migrations.Migration): ('city', models.CharField(max_length=128, verbose_name='city', blank=True)), ('zipcode', models.CharField(max_length=10, blank=True, validators=[django.core.validators.RegexValidator('^[0-9,A-Z]{3,10}$', 'Enter a valid zipcode.', 'invalid')], verbose_name='zip code')), ('country', models.CharField(choices=[('VC', 'Saint Vincent and the Grenadines'), ('TM', 'Turkmenistan'), ('CL', 'Chile'), ('BN', 'Brunei Darussalam'), ('IS', 'Iceland'), ('AM', 'Armenia'), ('FI', 'Finland'), ('TK', 'Tokelau'), ('AF', 'Afghanistan'), ('IE', 'Ireland'), ('CW', 'Curaçao'), ('PY', 'Paraguay'), ('WF', 'Wallis and Futuna'), ('PK', 'Pakistan'), ('JP', 'Japan'), ('AO', 'Angola'), ('FM', 'Micronesia (Federated States of)'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('SG', 'Singapore'), ('BL', 'Saint Barthélemy'), ('MK', 'Macedonia (the former Yugoslav Republic of)'), ('MY', 'Malaysia'), ('IM', 'Isle of Man'), ('GW', 'Guinea-Bissau'), ('IQ', 'Iraq'), ('GR', 'Greece'), ('VA', 'Holy See'), ('RW', 'Rwanda'), ('GD', 'Grenada'), ('TZ', 'Tanzania, United Republic of'), ('DZ', 'Algeria'), ('BF', 'Burkina Faso'), ('HU', 'Hungary'), ('TT', 'Trinidad and Tobago'), ('LU', 'Luxembourg'), ('BA', 'Bosnia and Herzegovina'), ('ET', 'Ethiopia'), ('TG', 'Togo'), ('RU', 'Russian Federation'), ('EG', 'Egypt'), ('RO', 'Romania'), ('SR', 'Suriname'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('JM', 'Jamaica'), ('HK', 'Hong Kong'), ('BH', 'Bahrain'), ('KM', 'Comoros'), ('HN', 'Honduras'), ('TD', 'Chad'), ('RS', 'Serbia'), ('PH', 'Philippines'), ('PE', 'Peru'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('KW', 'Kuwait'), ('GE', 'Georgia'), ('NA', 'Namibia'), ('CZ', 'Czech Republic'), ('CY', 'Cyprus'), ('LA', "Lao People's Democratic Republic"), ('BZ', 'Belize'), ('MX', 'Mexico'), ('MZ', 'Mozambique'), ('FR', 'France'), ('KG', 'Kyrgyzstan'), ('PW', 'Palau'), ('MG', 'Madagascar'), ('AU', 'Australia'), ('AI', 'Anguilla'), ('UZ', 'Uzbekistan'), ('NL', 'Netherlands'), ('VI', 'Virgin Islands (U.S.)'), ('LT', 'Lithuania'), ('WS', 'Samoa'), ('PA', 'Panama'), ('CO', 'Colombia'), ('AL', 'Albania'), ('PN', 'Pitcairn'), ('SC', 'Seychelles'), ('CH', 'Switzerland'), ('DO', 'Dominican Republic'), ('AW', 'Aruba'), ('GH', 'Ghana'), ('MM', 'Myanmar'), ('ML', 'Mali'), ('PS', 'Palestine, State of'), ('UY', 'Uruguay'), ('MN', 'Mongolia'), ('NE', 'Niger'), ('FK', 'Falkland Islands [Malvinas]'), ('US', 'United States of America'), ('BD', 'Bangladesh'), ('SB', 'Solomon Islands'), ('ZA', 'South Africa'), ('SJ', 'Svalbard and Jan Mayen'), ('IT', 'Italy'), ('HT', 'Haiti'), ('BW', 'Botswana'), ('MA', 'Morocco'), ('GP', 'Guadeloupe'), ('NI', 'Nicaragua'), ('CU', 'Cuba'), ('GL', 'Greenland'), ('MQ', 'Martinique'), ('NZ', 'New Zealand'), ('DE', 'Germany'), ('GY', 'Guyana'), ('YT', 'Mayotte'), ('MR', 'Mauritania'), ('IR', 'Iran (Islamic Republic of)'), ('SL', 'Sierra Leone'), ('MD', 'Moldova (the Republic of)'), ('SM', 'San Marino'), ('SS', 'South Sudan'), ('DM', 'Dominica'), ('NG', 'Nigeria'), ('UM', 'United States Minor Outlying Islands'), ('BI', 'Burundi'), ('GU', 'Guam'), ('GQ', 'Equatorial Guinea'), ('UG', 'Uganda'), ('VU', 'Vanuatu'), ('GT', 'Guatemala'), ('TR', 'Turkey'), ('BO', 'Bolivia (Plurinational State of)'), ('MP', 'Northern Mariana Islands'), ('AQ', 'Antarctica'), ('BS', 'Bahamas'), ('SK', 'Slovakia'), ('BY', 'Belarus'), ('AR', 'Argentina'), ('QA', 'Qatar'), ('LV', 'Latvia'), ('EH', 'Western Sahara'), ('LK', 'Sri Lanka'), ('LB', 'Lebanon'), ('JE', 'Jersey'), ('DJ', 'Djibouti'), ('LC', 'Saint Lucia'), ('CF', 'Central African Republic'), ('KE', 'Kenya'), ('NF', 'Norfolk Island'), ('FO', 'Faroe Islands'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('TF', 'French Southern Territories'), ('TC', 'Turks and Caicos Islands'), ('GM', 'Gambia'), ('PF', 'French Polynesia'), ('DK', 'Denmark'), ('GI', 'Gibraltar'), ('SN', 'Senegal'), ('ER', 'Eritrea'), ('CR', 'Costa Rica'), ('AZ', 'Azerbaijan'), ('BR', 'Brazil'), ('SE', 'Sweden'), ('SI', 'Slovenia'), ('SV', 'El Salvador'), ('TO', 'Tonga'), ('LR', 'Liberia'), ('CV', 'Cabo Verde'), ('OM', 'Oman'), ('KR', 'Korea (the Republic of)'), ('BV', 'Bouvet Island'), ('CA', 'Canada'), ('CK', 'Cook Islands'), ('BG', 'Bulgaria'), ('ZW', 'Zimbabwe'), ('FJ', 'Fiji'), ('NU', 'Niue'), ('YE', 'Yemen'), ('CM', 'Cameroon'), ('ZM', 'Zambia'), ('MC', 'Monaco'), ('SX', 'Sint Maarten (Dutch part)'), ('VG', 'Virgin Islands (British)'), ('AD', 'Andorra'), ('MF', 'Saint Martin (French part)'), ('PM', 'Saint Pierre and Miquelon'), ('RE', 'Réunion'), ('SO', 'Somalia'), ('AG', 'Antigua and Barbuda'), ('MH', 'Marshall Islands'), ('TW', 'Taiwan (Province of China)'), ('EE', 'Estonia'), ('NP', 'Nepal'), ('TV', 'Tuvalu'), ('NC', 'New Caledonia'), ('ME', 'Montenegro'), ('CX', 'Christmas Island'), ('LY', 'Libya'), ('KP', "Korea (the Democratic People's Republic of)"), ('SD', 'Sudan'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('KZ', 'Kazakhstan'), ('MV', 'Maldives'), ('CI', "Côte d'Ivoire"), ('VN', 'Viet Nam'), ('IN', 'India'), ('GN', 'Guinea'), ('AX', 'Ã…land Islands'), ('ST', 'Sao Tome and Principe'), ('HR', 'Croatia'), ('HM', 'Heard Island and McDonald Islands'), ('MO', 'Macao'), ('MS', 'Montserrat'), ('AT', 'Austria'), ('NO', 'Norway'), ('BT', 'Bhutan'), ('PT', 'Portugal'), ('IO', 'British Indian Ocean Territory'), ('KN', 'Saint Kitts and Nevis'), ('PL', 'Poland'), ('MW', 'Malawi'), ('LS', 'Lesotho'), ('SZ', 'Swaziland'), ('TJ', 'Tajikistan'), ('NR', 'Nauru'), ('MT', 'Malta'), ('ES', 'Spain'), ('BB', 'Barbados'), ('CG', 'Congo'), ('PG', 'Papua New Guinea'), ('GS', 'South Georgia and the South Sandwich Islands'), ('TN', 'Tunisia'), ('MU', 'Mauritius'), ('TH', 'Thailand'), ('BJ', 'Benin'), ('KY', 'Cayman Islands'), ('ID', 'Indonesia'), ('CD', 'Congo (the Democratic Republic of the)'), ('KI', 'Kiribati'), ('IL', 'Israel'), ('CN', 'China'), ('SA', 'Saudi Arabia'), ('AS', 'American Samoa'), ('KH', 'Cambodia'), ('JO', 'Jordan'), ('SY', 'Syrian Arab Republic'), ('LI', 'Liechtenstein'), ('GF', 'French Guiana'), ('EC', 'Ecuador'), ('CC', 'Cocos (Keeling) Islands'), ('TL', 'Timor-Leste'), ('BE', 'Belgium'), ('PR', 'Puerto Rico'), ('GA', 'Gabon'), ('BM', 'Bermuda'), ('GG', 'Guernsey')], max_length=20, default='ES', verbose_name='country', blank=True)), - ('account', models.ForeignKey(null=True, related_name='contacts', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('account', models.ForeignKey(null=True, related_name='contacts', on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Account')), ], ), ] diff --git a/orchestra/contrib/contacts/migrations/0001_squashed_0012_auto_20210422_1108.py b/orchestra/contrib/contacts/migrations/0001_squashed_0012_auto_20210422_1108.py new file mode 100644 index 00000000..31a58dbd --- /dev/null +++ b/orchestra/contrib/contacts/migrations/0001_squashed_0012_auto_20210422_1108.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:08 +from __future__ import unicode_literals + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import orchestra.contrib.contacts.validators +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('contacts', '0001_initial'), ('contacts', '0002_auto_20170528_2011'), ('contacts', '0003_auto_20170625_1813'), ('contacts', '0004_auto_20170625_1840'), ('contacts', '0005_auto_20170625_1840'), ('contacts', '0006_auto_20170625_1840'), ('contacts', '0007_auto_20170625_1841'), ('contacts', '0008_auto_20190805_1134'), ('contacts', '0009_auto_20200204_1217'), ('contacts', '0010_auto_20200204_1218'), ('contacts', '0011_auto_20210330_1049'), ('contacts', '0012_auto_20210422_1108')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Contact', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('short_name', models.CharField(max_length=128, verbose_name='short name')), + ('full_name', models.CharField(blank=True, max_length=256, verbose_name='full name')), + ('email', models.EmailField(max_length=254)), + ('email_usage', orchestra.models.fields.MultiSelectField(blank=True, choices=[('SUPPORT', 'Support tickets'), ('ADMIN', 'Administrative'), ('BILLING', 'Billing'), ('TECH', 'Technical'), ('ADDS', 'Announcements'), ('EMERGENCY', 'Emergency contact')], default=('SUPPORT', 'ADMIN', 'BILLING', 'TECH', 'ADDS', 'EMERGENCY'), max_length=256, verbose_name='email usage')), + ('phone', models.CharField(blank=True, max_length=32, validators=[orchestra.contrib.contacts.validators.validate_phone], verbose_name='phone')), + ('phone2', models.CharField(blank=True, max_length=32, validators=[orchestra.contrib.contacts.validators.validate_phone], verbose_name='alternative phone')), + ('address', models.TextField(blank=True, verbose_name='address')), + ('city', models.CharField(blank=True, max_length=128, verbose_name='city')), + ('zipcode', models.CharField(blank=True, max_length=10, validators=[django.core.validators.RegexValidator('^[0-9,A-Z]{3,10}$', 'Enter a valid zipcode.', 'invalid')], verbose_name='zip code')), + ('country', models.CharField(blank=True, choices=[('AF', 'Afghanistan'), ('AX', 'Ã…land Islands'), ('AL', 'Albania'), ('DZ', 'Algeria'), ('AS', 'American Samoa'), ('AD', 'Andorra'), ('AO', 'Angola'), ('AI', 'Anguilla'), ('AQ', 'Antarctica'), ('AG', 'Antigua and Barbuda'), ('AR', 'Argentina'), ('AM', 'Armenia'), ('AW', 'Aruba'), ('AU', 'Australia'), ('AT', 'Austria'), ('AZ', 'Azerbaijan'), ('BS', 'Bahamas'), ('BH', 'Bahrain'), ('BD', 'Bangladesh'), ('BB', 'Barbados'), ('BY', 'Belarus'), ('BE', 'Belgium'), ('BZ', 'Belize'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BT', 'Bhutan'), ('BO', 'Bolivia (Plurinational State of)'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('BA', 'Bosnia and Herzegovina'), ('BW', 'Botswana'), ('BV', 'Bouvet Island'), ('BR', 'Brazil'), ('IO', 'British Indian Ocean Territory'), ('BN', 'Brunei Darussalam'), ('BG', 'Bulgaria'), ('BF', 'Burkina Faso'), ('BI', 'Burundi'), ('CV', 'Cabo Verde'), ('KH', 'Cambodia'), ('CM', 'Cameroon'), ('CA', 'Canada'), ('KY', 'Cayman Islands'), ('CF', 'Central African Republic'), ('TD', 'Chad'), ('CL', 'Chile'), ('CN', 'China'), ('CX', 'Christmas Island'), ('CC', 'Cocos (Keeling) Islands'), ('CO', 'Colombia'), ('KM', 'Comoros'), ('CG', 'Congo'), ('CD', 'Congo (the Democratic Republic of the)'), ('CK', 'Cook Islands'), ('CR', 'Costa Rica'), ('CI', "Côte d'Ivoire"), ('HR', 'Croatia'), ('CU', 'Cuba'), ('CW', 'Curaçao'), ('CY', 'Cyprus'), ('CZ', 'Czechia'), ('DK', 'Denmark'), ('DJ', 'Djibouti'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('EC', 'Ecuador'), ('EG', 'Egypt'), ('SV', 'El Salvador'), ('GQ', 'Equatorial Guinea'), ('ER', 'Eritrea'), ('EE', 'Estonia'), ('SZ', 'Eswatini'), ('ET', 'Ethiopia'), ('FK', 'Falkland Islands (Malvinas)'), ('FO', 'Faroe Islands'), ('FJ', 'Fiji'), ('FI', 'Finland'), ('FR', 'France'), ('GF', 'French Guiana'), ('PF', 'French Polynesia'), ('TF', 'French Southern Territories'), ('GA', 'Gabon'), ('GM', 'Gambia'), ('GE', 'Georgia'), ('DE', 'Germany'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GR', 'Greece'), ('GL', 'Greenland'), ('GD', 'Grenada'), ('GP', 'Guadeloupe'), ('GU', 'Guam'), ('GT', 'Guatemala'), ('GG', 'Guernsey'), ('GN', 'Guinea'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HT', 'Haiti'), ('HM', 'Heard Island and McDonald Islands'), ('VA', 'Holy See'), ('HN', 'Honduras'), ('HK', 'Hong Kong'), ('HU', 'Hungary'), ('IS', 'Iceland'), ('IN', 'India'), ('ID', 'Indonesia'), ('IR', 'Iran (Islamic Republic of)'), ('IQ', 'Iraq'), ('IE', 'Ireland'), ('IM', 'Isle of Man'), ('IL', 'Israel'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JP', 'Japan'), ('JE', 'Jersey'), ('JO', 'Jordan'), ('KZ', 'Kazakhstan'), ('KE', 'Kenya'), ('KI', 'Kiribati'), ('KP', "Korea (the Democratic People's Republic of)"), ('KR', 'Korea (the Republic of)'), ('KW', 'Kuwait'), ('KG', 'Kyrgyzstan'), ('LA', "Lao People's Democratic Republic"), ('LV', 'Latvia'), ('LB', 'Lebanon'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('LY', 'Libya'), ('LI', 'Liechtenstein'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('MO', 'Macao'), ('MG', 'Madagascar'), ('MW', 'Malawi'), ('MY', 'Malaysia'), ('MV', 'Maldives'), ('ML', 'Mali'), ('MT', 'Malta'), ('MH', 'Marshall Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MU', 'Mauritius'), ('YT', 'Mayotte'), ('MX', 'Mexico'), ('FM', 'Micronesia (Federated States of)'), ('MD', 'Moldova (the Republic of)'), ('MC', 'Monaco'), ('MN', 'Mongolia'), ('ME', 'Montenegro'), ('MS', 'Montserrat'), ('MA', 'Morocco'), ('MZ', 'Mozambique'), ('MM', 'Myanmar'), ('NA', 'Namibia'), ('NR', 'Nauru'), ('NP', 'Nepal'), ('NL', 'Netherlands'), ('NC', 'New Caledonia'), ('NZ', 'New Zealand'), ('NI', 'Nicaragua'), ('NE', 'Niger'), ('NG', 'Nigeria'), ('NU', 'Niue'), ('NF', 'Norfolk Island'), ('MK', 'North Macedonia'), ('MP', 'Northern Mariana Islands'), ('NO', 'Norway'), ('OM', 'Oman'), ('PK', 'Pakistan'), ('PW', 'Palau'), ('PS', 'Palestine, State of'), ('PA', 'Panama'), ('PG', 'Papua New Guinea'), ('PY', 'Paraguay'), ('PE', 'Peru'), ('PH', 'Philippines'), ('PN', 'Pitcairn'), ('PL', 'Poland'), ('PT', 'Portugal'), ('PR', 'Puerto Rico'), ('QA', 'Qatar'), ('RE', 'Réunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('BL', 'Saint Barthélemy'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('KN', 'Saint Kitts and Nevis'), ('LC', 'Saint Lucia'), ('MF', 'Saint Martin (French part)'), ('PM', 'Saint Pierre and Miquelon'), ('VC', 'Saint Vincent and the Grenadines'), ('WS', 'Samoa'), ('SM', 'San Marino'), ('ST', 'Sao Tome and Principe'), ('SA', 'Saudi Arabia'), ('SN', 'Senegal'), ('RS', 'Serbia'), ('SC', 'Seychelles'), ('SL', 'Sierra Leone'), ('SG', 'Singapore'), ('SX', 'Sint Maarten (Dutch part)'), ('SK', 'Slovakia'), ('SI', 'Slovenia'), ('SB', 'Solomon Islands'), ('SO', 'Somalia'), ('ZA', 'South Africa'), ('GS', 'South Georgia and the South Sandwich Islands'), ('SS', 'South Sudan'), ('ES', 'Spain'), ('LK', 'Sri Lanka'), ('SD', 'Sudan'), ('SR', 'Suriname'), ('SJ', 'Svalbard and Jan Mayen'), ('SE', 'Sweden'), ('CH', 'Switzerland'), ('SY', 'Syrian Arab Republic'), ('TW', 'Taiwan (Province of China)'), ('TJ', 'Tajikistan'), ('TZ', 'Tanzania, the United Republic of'), ('TH', 'Thailand'), ('TL', 'Timor-Leste'), ('TG', 'Togo'), ('TK', 'Tokelau'), ('TO', 'Tonga'), ('TT', 'Trinidad and Tobago'), ('TN', 'Tunisia'), ('TR', 'Turkey'), ('TM', 'Turkmenistan'), ('TC', 'Turks and Caicos Islands'), ('TV', 'Tuvalu'), ('UG', 'Uganda'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VU', 'Vanuatu'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('VN', 'Viet Nam'), ('VG', 'Virgin Islands (British)'), ('VI', 'Virgin Islands (U.S.)'), ('WF', 'Wallis and Futuna'), ('EH', 'Western Sahara'), ('YE', 'Yemen'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe')], default='ES', max_length=20, verbose_name='country')), + ('account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='contacts', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ], + ), + ] diff --git a/orchestra/contrib/contacts/migrations/0011_auto_20210330_1049.py b/orchestra/contrib/contacts/migrations/0011_auto_20210330_1049.py new file mode 100644 index 00000000..d4664e93 --- /dev/null +++ b/orchestra/contrib/contacts/migrations/0011_auto_20210330_1049.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contacts', '0010_auto_20200204_1218'), + ] + + operations = [ + migrations.AlterField( + model_name='contact', + name='country', + field=models.CharField(blank=True, choices=[('AF', 'Afghanistan'), ('AX', 'Ã…land Islands'), ('AL', 'Albania'), ('DZ', 'Algeria'), ('AS', 'American Samoa'), ('AD', 'Andorra'), ('AO', 'Angola'), ('AI', 'Anguilla'), ('AQ', 'Antarctica'), ('AG', 'Antigua and Barbuda'), ('AR', 'Argentina'), ('AM', 'Armenia'), ('AW', 'Aruba'), ('AU', 'Australia'), ('AT', 'Austria'), ('AZ', 'Azerbaijan'), ('BS', 'Bahamas'), ('BH', 'Bahrain'), ('BD', 'Bangladesh'), ('BB', 'Barbados'), ('BY', 'Belarus'), ('BE', 'Belgium'), ('BZ', 'Belize'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BT', 'Bhutan'), ('BO', 'Bolivia (Plurinational State of)'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('BA', 'Bosnia and Herzegovina'), ('BW', 'Botswana'), ('BV', 'Bouvet Island'), ('BR', 'Brazil'), ('IO', 'British Indian Ocean Territory'), ('BN', 'Brunei Darussalam'), ('BG', 'Bulgaria'), ('BF', 'Burkina Faso'), ('BI', 'Burundi'), ('CV', 'Cabo Verde'), ('KH', 'Cambodia'), ('CM', 'Cameroon'), ('CA', 'Canada'), ('KY', 'Cayman Islands'), ('CF', 'Central African Republic'), ('TD', 'Chad'), ('CL', 'Chile'), ('CN', 'China'), ('CX', 'Christmas Island'), ('CC', 'Cocos (Keeling) Islands'), ('CO', 'Colombia'), ('KM', 'Comoros'), ('CG', 'Congo'), ('CD', 'Congo (the Democratic Republic of the)'), ('CK', 'Cook Islands'), ('CR', 'Costa Rica'), ('CI', "Côte d'Ivoire"), ('HR', 'Croatia'), ('CU', 'Cuba'), ('CW', 'Curaçao'), ('CY', 'Cyprus'), ('CZ', 'Czechia'), ('DK', 'Denmark'), ('DJ', 'Djibouti'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('EC', 'Ecuador'), ('EG', 'Egypt'), ('SV', 'El Salvador'), ('GQ', 'Equatorial Guinea'), ('ER', 'Eritrea'), ('EE', 'Estonia'), ('SZ', 'Eswatini'), ('ET', 'Ethiopia'), ('FK', 'Falkland Islands (Malvinas)'), ('FO', 'Faroe Islands'), ('FJ', 'Fiji'), ('FI', 'Finland'), ('FR', 'France'), ('GF', 'French Guiana'), ('PF', 'French Polynesia'), ('TF', 'French Southern Territories'), ('GA', 'Gabon'), ('GM', 'Gambia'), ('GE', 'Georgia'), ('DE', 'Germany'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GR', 'Greece'), ('GL', 'Greenland'), ('GD', 'Grenada'), ('GP', 'Guadeloupe'), ('GU', 'Guam'), ('GT', 'Guatemala'), ('GG', 'Guernsey'), ('GN', 'Guinea'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HT', 'Haiti'), ('HM', 'Heard Island and McDonald Islands'), ('VA', 'Holy See'), ('HN', 'Honduras'), ('HK', 'Hong Kong'), ('HU', 'Hungary'), ('IS', 'Iceland'), ('IN', 'India'), ('ID', 'Indonesia'), ('IR', 'Iran (Islamic Republic of)'), ('IQ', 'Iraq'), ('IE', 'Ireland'), ('IM', 'Isle of Man'), ('IL', 'Israel'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JP', 'Japan'), ('JE', 'Jersey'), ('JO', 'Jordan'), ('KZ', 'Kazakhstan'), ('KE', 'Kenya'), ('KI', 'Kiribati'), ('KP', "Korea (the Democratic People's Republic of)"), ('KR', 'Korea (the Republic of)'), ('KW', 'Kuwait'), ('KG', 'Kyrgyzstan'), ('LA', "Lao People's Democratic Republic"), ('LV', 'Latvia'), ('LB', 'Lebanon'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('LY', 'Libya'), ('LI', 'Liechtenstein'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('MO', 'Macao'), ('MG', 'Madagascar'), ('MW', 'Malawi'), ('MY', 'Malaysia'), ('MV', 'Maldives'), ('ML', 'Mali'), ('MT', 'Malta'), ('MH', 'Marshall Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MU', 'Mauritius'), ('YT', 'Mayotte'), ('MX', 'Mexico'), ('FM', 'Micronesia (Federated States of)'), ('MD', 'Moldova (the Republic of)'), ('MC', 'Monaco'), ('MN', 'Mongolia'), ('ME', 'Montenegro'), ('MS', 'Montserrat'), ('MA', 'Morocco'), ('MZ', 'Mozambique'), ('MM', 'Myanmar'), ('NA', 'Namibia'), ('NR', 'Nauru'), ('NP', 'Nepal'), ('NL', 'Netherlands'), ('NC', 'New Caledonia'), ('NZ', 'New Zealand'), ('NI', 'Nicaragua'), ('NE', 'Niger'), ('NG', 'Nigeria'), ('NU', 'Niue'), ('NF', 'Norfolk Island'), ('MK', 'North Macedonia'), ('MP', 'Northern Mariana Islands'), ('NO', 'Norway'), ('OM', 'Oman'), ('PK', 'Pakistan'), ('PW', 'Palau'), ('PS', 'Palestine, State of'), ('PA', 'Panama'), ('PG', 'Papua New Guinea'), ('PY', 'Paraguay'), ('PE', 'Peru'), ('PH', 'Philippines'), ('PN', 'Pitcairn'), ('PL', 'Poland'), ('PT', 'Portugal'), ('PR', 'Puerto Rico'), ('QA', 'Qatar'), ('RE', 'Réunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('BL', 'Saint Barthélemy'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('KN', 'Saint Kitts and Nevis'), ('LC', 'Saint Lucia'), ('MF', 'Saint Martin (French part)'), ('PM', 'Saint Pierre and Miquelon'), ('VC', 'Saint Vincent and the Grenadines'), ('WS', 'Samoa'), ('SM', 'San Marino'), ('ST', 'Sao Tome and Principe'), ('SA', 'Saudi Arabia'), ('SN', 'Senegal'), ('RS', 'Serbia'), ('SC', 'Seychelles'), ('SL', 'Sierra Leone'), ('SG', 'Singapore'), ('SX', 'Sint Maarten (Dutch part)'), ('SK', 'Slovakia'), ('SI', 'Slovenia'), ('SB', 'Solomon Islands'), ('SO', 'Somalia'), ('ZA', 'South Africa'), ('GS', 'South Georgia and the South Sandwich Islands'), ('SS', 'South Sudan'), ('ES', 'Spain'), ('LK', 'Sri Lanka'), ('SD', 'Sudan'), ('SR', 'Suriname'), ('SJ', 'Svalbard and Jan Mayen'), ('SE', 'Sweden'), ('CH', 'Switzerland'), ('SY', 'Syrian Arab Republic'), ('TW', 'Taiwan (Province of China)'), ('TJ', 'Tajikistan'), ('TZ', 'Tanzania, the United Republic of'), ('TH', 'Thailand'), ('TL', 'Timor-Leste'), ('TG', 'Togo'), ('TK', 'Tokelau'), ('TO', 'Tonga'), ('TT', 'Trinidad and Tobago'), ('TN', 'Tunisia'), ('TR', 'Turkey'), ('TM', 'Turkmenistan'), ('TC', 'Turks and Caicos Islands'), ('TV', 'Tuvalu'), ('UG', 'Uganda'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VU', 'Vanuatu'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('VN', 'Viet Nam'), ('VG', 'Virgin Islands (British)'), ('VI', 'Virgin Islands (U.S.)'), ('WF', 'Wallis and Futuna'), ('EH', 'Western Sahara'), ('YE', 'Yemen'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe')], default='ES', max_length=20, verbose_name='country'), + ), + ] diff --git a/orchestra/contrib/contacts/models.py b/orchestra/contrib/contacts/models.py index 42fee3fe..a8de1580 100644 --- a/orchestra/contrib/contacts/models.py +++ b/orchestra/contrib/contacts/models.py @@ -29,11 +29,11 @@ class Contact(models.Model): ('ADDS', _("Announcements")), ('EMERGENCY', _("Emergency contact")), ) - + objects = ContactQuerySet.as_manager() - + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='contacts', null=True) + related_name='contacts', null=True, on_delete=models.SET_NULL) short_name = models.CharField(_("short name"), max_length=128) full_name = models.CharField(_("full name"), max_length=256, blank=True) email = models.EmailField() @@ -54,10 +54,10 @@ class Contact(models.Model): country = models.CharField(_("country"), max_length=20, blank=True, choices=settings.CONTACTS_COUNTRIES, default=settings.CONTACTS_DEFAULT_COUNTRY) - + def __str__(self): return self.full_name or self.short_name - + def clean(self): self.short_name = self.short_name.strip() self.full_name = self.full_name.strip() diff --git a/orchestra/contrib/databases/migrations/0001_initial.py b/orchestra/contrib/databases/migrations/0001_initial.py index 47b3ff1b..e25696ac 100644 --- a/orchestra/contrib/databases/migrations/0001_initial.py +++ b/orchestra/contrib/databases/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings +import django.db.models.deletion import orchestra.core.validators @@ -19,7 +20,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(serialize=False, primary_key=True, verbose_name='ID', auto_created=True)), ('name', models.CharField(verbose_name='name', max_length=64, validators=[orchestra.core.validators.validate_name])), ('type', models.CharField(default='mysql', choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], verbose_name='type', max_length=32)), - ('account', models.ForeignKey(related_name='databases', verbose_name='Account', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databases', verbose_name='Account', to=settings.AUTH_USER_MODEL)), ], ), migrations.CreateModel( @@ -29,7 +30,7 @@ class Migration(migrations.Migration): ('username', models.CharField(verbose_name='username', max_length=16, validators=[orchestra.core.validators.validate_name])), ('password', models.CharField(verbose_name='password', max_length=256)), ('type', models.CharField(default='mysql', choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], verbose_name='type', max_length=32)), - ('account', models.ForeignKey(related_name='databaseusers', verbose_name='Account', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databaseusers', verbose_name='Account', to=settings.AUTH_USER_MODEL)), ], options={ 'verbose_name_plural': 'DB users', diff --git a/orchestra/contrib/databases/migrations/0001_squashed_0004_auto_20210330_1049.py b/orchestra/contrib/databases/migrations/0001_squashed_0004_auto_20210330_1049.py new file mode 100644 index 00000000..2c12381c --- /dev/null +++ b/orchestra/contrib/databases/migrations/0001_squashed_0004_auto_20210330_1049.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:25 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('databases', '0001_initial'), ('databases', '0002_auto_20170528_2005'), ('databases', '0003_database_comments'), ('databases', '0004_auto_20210330_1049')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Database', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=64, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('type', models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databases', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ], + ), + migrations.CreateModel( + name='DatabaseUser', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('username', models.CharField(max_length=16, validators=[orchestra.core.validators.validate_name], verbose_name='username')), + ('password', models.CharField(max_length=256, verbose_name='password')), + ('type', models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databaseusers', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ], + options={ + 'verbose_name_plural': 'DB users', + }, + ), + migrations.AddField( + model_name='database', + name='users', + field=models.ManyToManyField(blank=True, related_name='databases', to='databases.DatabaseUser', verbose_name='users'), + ), + migrations.AlterUniqueTogether( + name='databaseuser', + unique_together=set([('username', 'type')]), + ), + migrations.AlterUniqueTogether( + name='database', + unique_together=set([('name', 'type')]), + ), + migrations.AlterField( + model_name='database', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL')], default='mysql', max_length=32, verbose_name='type'), + ), + migrations.AlterField( + model_name='databaseuser', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL')], default='mysql', max_length=32, verbose_name='type'), + ), + migrations.AddField( + model_name='database', + name='comments', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterField( + model_name='database', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type'), + ), + migrations.AlterField( + model_name='databaseuser', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type'), + ), + ] diff --git a/orchestra/contrib/databases/migrations/0004_auto_20210330_1049.py b/orchestra/contrib/databases/migrations/0004_auto_20210330_1049.py new file mode 100644 index 00000000..9259bfa4 --- /dev/null +++ b/orchestra/contrib/databases/migrations/0004_auto_20210330_1049.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('databases', '0003_database_comments'), + ] + + operations = [ + migrations.AlterField( + model_name='database', + name='comments', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterField( + model_name='database', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type'), + ), + migrations.AlterField( + model_name='databaseuser', + name='type', + field=models.CharField(choices=[('mysql', 'MySQL'), ('postgres', 'PostgreSQL')], default='mysql', max_length=32, verbose_name='type'), + ), + ] diff --git a/orchestra/contrib/databases/models.py b/orchestra/contrib/databases/models.py index ab452358..207fe63d 100644 --- a/orchestra/contrib/databases/models.py +++ b/orchestra/contrib/databases/models.py @@ -12,7 +12,7 @@ class Database(models.Model): """ Represents a basic database for a web application """ MYSQL = 'mysql' POSTGRESQL = 'postgresql' - + name = models.CharField(_("name"), max_length=64, # MySQL limit validators=[validators.validate_name]) users = models.ManyToManyField('databases.DatabaseUser', blank=True, @@ -20,16 +20,16 @@ class Database(models.Model): type = models.CharField(_("type"), max_length=32, choices=settings.DATABASES_TYPE_CHOICES, default=settings.DATABASES_DEFAULT_TYPE) - account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='databases') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='databases') comments = models.TextField(default="", blank=True) - + class Meta: unique_together = ('name', 'type') - + def __str__(self): return "%s" % self.name - + @property def owner(self): """ database owner is the first user related to it """ @@ -39,7 +39,7 @@ class Database(models.Model): if user is not None: return user.databaseuser return None - + @property def active(self): return self.account.is_active @@ -53,26 +53,26 @@ Database.users.through._meta.unique_together = ( class DatabaseUser(models.Model): MYSQL = Database.MYSQL POSTGRESQL = Database.POSTGRESQL - + username = models.CharField(_("username"), max_length=16, # MySQL usernames 16 char long validators=[validators.validate_name]) password = models.CharField(_("password"), max_length=256) type = models.CharField(_("type"), max_length=32, choices=settings.DATABASES_TYPE_CHOICES, default=settings.DATABASES_DEFAULT_TYPE) - account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='databaseusers') - + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='databaseusers') + class Meta: verbose_name_plural = _("DB users") unique_together = ('username', 'type') - + def __str__(self): return self.username - + def get_username(self): return self.username - + def set_password(self, password): if self.type == self.MYSQL: # MySQL stores sha1(sha1(password).binary).hex diff --git a/orchestra/contrib/databases/tests/functional_tests/tests.py b/orchestra/contrib/databases/tests/functional_tests/tests.py index c13ad946..3d89bff1 100644 --- a/orchestra/contrib/databases/tests/functional_tests/tests.py +++ b/orchestra/contrib/databases/tests/functional_tests/tests.py @@ -1,22 +1,24 @@ -import MySQLdb import os import socket import time +import unittest +import MySQLdb from django.conf import settings as djsettings from django.core.management.base import CommandError -from django.core.urlresolvers import reverse -from selenium.webdriver.support.select import Select - +from django.urls import reverse from orchestra.admin.utils import change_url -from orchestra.contrib.orchestration.models import Server, Route +from orchestra.contrib.orchestration.models import Route, Server from orchestra.utils.sys import sshrun -from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, save_response_on_error, - snapshot_on_error) +from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, + save_response_on_error, snapshot_on_error) +from selenium.webdriver.support.select import Select from ... import backends, settings from ...models import Database, DatabaseUser +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + class DatabaseTestMixin(object): MASTER_SERVER = os.environ.get('ORCHESTRA_SECOND_SERVER', 'localhost') @@ -24,40 +26,40 @@ class DatabaseTestMixin(object): 'orchestra.contrib.orchestration', 'orcgestra.apps.databases', ) - + def setUp(self): super(DatabaseTestMixin, self).setUp() self.add_route() djsettings.DEBUG = True - + def add_route(self): raise NotImplementedError - + def save(self): raise NotImplementedError - + def add(self): raise NotImplementedError - + def delete(self): raise NotImplementedError - + def update(self): raise NotImplementedError - + def disable(self): raise NotImplementedError - + def add_group(self, username, groupname): raise NotImplementedError - + def test_add(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) password = '@!?%spppP001' % random_ascii(5) self.add(dbname, username, password) self.validate_create_table(dbname, username, password) - + def test_delete(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) @@ -68,7 +70,7 @@ class DatabaseTestMixin(object): self.delete_user(username) self.validate_delete(dbname, username, password) self.validate_delete_user(dbname, username) - + def test_change_password(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) @@ -81,7 +83,7 @@ class DatabaseTestMixin(object): self.change_password(username, new_password) self.validate_login_error(dbname, username, password) self.validate_create_table(dbname, username, new_password) - + def test_add_user(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) @@ -98,7 +100,7 @@ class DatabaseTestMixin(object): self.add_user_to_db(username2, dbname) self.validate_create_table(dbname, username, password) self.validate_create_table(dbname, username2, password2) - + def test_delete_user(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) @@ -117,7 +119,7 @@ class DatabaseTestMixin(object): self.delete_user(username2) self.validate_login_error(dbname, username2, password2) self.validate_delete_user(username2, password2) - + def test_swap_user(self): dbname = '%s_database' % random_ascii(5) username = '%s_dbuser' % random_ascii(5) @@ -137,7 +139,7 @@ class DatabaseTestMixin(object): class MySQLControllerMixin(object): db_type = 'mysql' - + def setUp(self): super(MySQLControllerMixin, self).setUp() # Get local ip address used to reach self.MASTER_SERVER @@ -145,7 +147,7 @@ class MySQLControllerMixin(object): s.connect((self.MASTER_SERVER, 22)) settings.DATABASES_DEFAULT_HOST = s.getsockname()[0] s.close() - + def add_route(self): server = Server.objects.create(name=self.MASTER_SERVER) backend = backends.MySQLController.get_name() @@ -154,22 +156,22 @@ class MySQLControllerMixin(object): match = "databaseuser.type == '%s'" % self.db_type backend = backends.MySQLUserController.get_name() Route.objects.create(backend=backend, match=match, host=server) - + def validate_create_table(self, name, username, password): db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name) cur = db.cursor() cur.execute('CREATE TABLE table_%s ( id INT ) ;' % random_ascii(10)) - + def validate_login_error(self, dbname, username, password): self.assertRaises(MySQLdb.OperationalError, self.validate_create_table, dbname, username, password ) - + def validate_delete(self, dbname, username, password): self.validate_login_error(dbname, username, password) self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, 'mysql %s' % dbname, display=False) - + def validate_delete_user(self, name, username): context = { 'name': name, @@ -181,11 +183,12 @@ class MySQLControllerMixin(object): """mysql mysql -e 'SELECT * FROM user WHERE user="%(username)s";'""" % context, display=False).stdout) +@unittest.skipUnless(TEST_REST_API, "REST API tests") class RESTDatabaseMixin(DatabaseTestMixin): def setUp(self): super(RESTDatabaseMixin, self).setUp() self.rest_login() - + @save_response_on_error def add(self, dbname, username, password): user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type) @@ -193,31 +196,31 @@ class RESTDatabaseMixin(DatabaseTestMixin): 'username': user.username }] self.rest.databases.create(name=dbname, users=users, type=self.db_type) - + @save_response_on_error def delete(self, dbname): self.rest.databases.retrieve(name=dbname).delete() - + @save_response_on_error def change_password(self, username, password): user = self.rest.databaseusers.retrieve(username=username).get() user.set_password(password) - + @save_response_on_error def add_user(self, username, password): self.rest.databaseusers.create(username=username, password=password, type=self.db_type) - + @save_response_on_error def add_user_to_db(self, username, dbname): user = self.rest.databaseusers.retrieve(username=username).get() db = self.rest.databases.retrieve(name=dbname).get() db.users.append(user) db.save() - + @save_response_on_error def delete_user(self, username): self.rest.databaseusers.retrieve(username=username).delete() - + @save_response_on_error def swap_user(self, username, username2, dbname): user = self.rest.databaseusers.retrieve(username=username2).get() @@ -231,84 +234,84 @@ class AdminDatabaseMixin(DatabaseTestMixin): def setUp(self): super(AdminDatabaseMixin, self).setUp() self.admin_login() - + @snapshot_on_error def add(self, dbname, username, password): url = self.live_server_url + reverse('admin:databases_database_add') self.selenium.get(url) - + type_input = self.selenium.find_element_by_id('id_type') type_select = Select(type_input) type_select.select_by_value(self.db_type) - + name_field = self.selenium.find_element_by_id('id_name') name_field.send_keys(dbname) - + username_field = self.selenium.find_element_by_id('id_username') username_field.send_keys(username) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) - + name_field.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete(self, dbname): db = Database.objects.get(name=dbname) self.admin_delete(db) - + @snapshot_on_error def change_password(self, username, password): user = DatabaseUser.objects.get(username=username) self.admin_change_password(user, password) - + @snapshot_on_error def add_user(self, username, password): url = self.live_server_url + reverse('admin:databases_databaseuser_add') self.selenium.get(url) - + type_input = self.selenium.find_element_by_id('id_type') type_select = Select(type_input) type_select.select_by_value(self.db_type) - + username_field = self.selenium.find_element_by_id('id_username') username_field.send_keys(username) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) - + username_field.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def add_user_to_db(self, username, dbname): database = Database.objects.get(name=dbname, type=self.db_type) url = self.live_server_url + change_url(database) self.selenium.get(url) - + user = DatabaseUser.objects.get(username=username, type=self.db_type) users_from = self.selenium.find_element_by_id('id_users_from') users_select = Select(users_from) users_select.select_by_value(str(user.pk)) - + add_user = self.selenium.find_element_by_id('id_users_add_link') add_user.click() - + save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def swap_user(self, username, username2, dbname): database = Database.objects.get(name=dbname, type=self.db_type) url = self.live_server_url + change_url(database) self.selenium.get(url) - + # remove user "username" user = DatabaseUser.objects.get(username=username, type=self.db_type) users_to = self.selenium.find_element_by_id('id_users_to') @@ -317,7 +320,7 @@ class AdminDatabaseMixin(DatabaseTestMixin): remove_user = self.selenium.find_element_by_id('id_users_remove_link') remove_user.click() time.sleep(0.2) - + # add user "username2" user = DatabaseUser.objects.get(username=username2, type=self.db_type) users_from = self.selenium.find_element_by_id('id_users_from') @@ -326,11 +329,11 @@ class AdminDatabaseMixin(DatabaseTestMixin): add_user = self.selenium.find_element_by_id('id_users_add_link') add_user.click() time.sleep(0.2) - + save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete_user(self, username): user = DatabaseUser.objects.get(username=username) diff --git a/orchestra/contrib/domains/admin.py b/orchestra/contrib/domains/admin.py index af054de4..47b5e14d 100644 --- a/orchestra/contrib/domains/admin.py +++ b/orchestra/contrib/domains/admin.py @@ -1,5 +1,5 @@ from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.db.models.functions import Concat, Coalesce from django.templatetags.static import static @@ -32,18 +32,18 @@ class DomainInline(admin.TabularInline): readonly_fields = ('domain_link', 'display_records', 'account_link') extra = 0 verbose_name_plural = _("Subdomains") - + domain_link = admin_link('__str__') domain_link.short_description = _("Name") account_link = admin_link('account') - + def display_records(self, domain): return ', '.join([record.type for record in domain.records.all()]) display_records.short_description = _("Declared records") - + def has_add_permission(self, *args, **kwargs): return False - + def get_queryset(self, request): """ Order by structured name and imporve performance """ qs = super(DomainInline, self).get_queryset(request) @@ -66,9 +66,9 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): add_form = BatchDomainCreationAdminForm actions = (edit_records, set_soa, list_accounts) change_view_actions = (view_zone, edit_records) - + top_link = admin_link('top') - + def structured_name(self, domain): if domain.is_top: return domain.name @@ -76,13 +76,13 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): structured_name.short_description = _("name") structured_name.allow_tags = True structured_name.admin_order_field = 'structured_name' - + def display_is_top(self, domain): return domain.is_top display_is_top.short_description = _("Is top") display_is_top.boolean = True display_is_top.admin_order_field = 'top' - + def display_websites(self, domain): if apps.isinstalled('orchestra.contrib.websites'): websites = domain.websites.all() @@ -107,7 +107,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): display_websites.admin_order_field = 'websites__name' display_websites.short_description = _("Websites") display_websites.allow_tags = True - + def display_addresses(self, domain): if apps.isinstalled('orchestra.contrib.mailboxes'): add_url = reverse('admin:mailboxes_address_add') @@ -127,7 +127,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): display_addresses.short_description = _("Addresses") display_addresses.admin_order_field = 'addresses__count' display_addresses.allow_tags = True - + def implicit_records(self, domain): defaults = [] types = set(domain.records.values_list('type', flat=True)) @@ -149,7 +149,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): return '
'.join(lines) implicit_records.short_description = _("Implicit records") implicit_records.allow_tags = True - + def get_fieldsets(self, request, obj=None): """ Add SOA fields when domain is top """ fieldsets = super(DomainAdmin, self).get_fieldsets(request, obj) @@ -175,13 +175,13 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): if 'top_link' not in existing: fieldsets[0][1]['fields'].insert(2, 'top_link') return fieldsets - + def get_inline_instances(self, request, obj=None): inlines = super(DomainAdmin, self).get_inline_instances(request, obj) if not obj or not obj.is_top: return [inline for inline in inlines if type(inline) != DomainInline] return inlines - + def get_queryset(self, request): """ Order by structured name and imporve performance """ qs = super(DomainAdmin, self).get_queryset(request) @@ -196,7 +196,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): if apps.isinstalled('orchestra.contrib.mailboxes'): qs = qs.annotate(models.Count('addresses')) return qs - + def save_model(self, request, obj, form, change): """ batch domain creation support """ super(DomainAdmin, self).save_model(request, obj, form, change) @@ -205,7 +205,7 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): for name in form.extra_names: domain = Domain.objects.create(name=name, account_id=obj.account_id) self.extra_domains.append(domain) - + def save_related(self, request, form, formsets, change): """ batch domain creation support """ super(DomainAdmin, self).save_related(request, form, formsets, change) diff --git a/orchestra/contrib/domains/migrations/0001_initial.py b/orchestra/contrib/domains/migrations/0001_initial.py index cb58ca78..f0877cd7 100644 --- a/orchestra/contrib/domains/migrations/0001_initial.py +++ b/orchestra/contrib/domains/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.contrib.domains.utils import orchestra.contrib.domains.validators from django.conf import settings @@ -20,8 +21,8 @@ class Migration(migrations.Migration): ('id', models.AutoField(serialize=False, primary_key=True, auto_created=True, verbose_name='ID')), ('name', models.CharField(unique=True, max_length=256, validators=[orchestra.contrib.domains.validators.validate_domain_name, orchestra.contrib.domains.validators.validate_allowed_domain], verbose_name='name', help_text='Domain or subdomain name.')), ('serial', models.IntegerField(default=orchestra.contrib.domains.utils.generate_zone_serial, verbose_name='serial', help_text='Serial number')), - ('account', models.ForeignKey(related_name='domains', help_text='Automatically selected for subdomains.', to=settings.AUTH_USER_MODEL, verbose_name='Account', blank=True)), - ('top', models.ForeignKey(null=True, to='domains.Domain', editable=False, related_name='subdomain_set')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domains', help_text='Automatically selected for subdomains.', to=settings.AUTH_USER_MODEL, verbose_name='Account', blank=True)), + ('top', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, null=True, to='domains.Domain', editable=False, related_name='subdomain_set')), ], ), migrations.CreateModel( @@ -31,7 +32,7 @@ class Migration(migrations.Migration): ('ttl', models.CharField(help_text='Record TTL, defaults to 1h', max_length=8, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='TTL', blank=True)), ('type', models.CharField(max_length=32, verbose_name='type', choices=[('MX', 'MX'), ('NS', 'NS'), ('CNAME', 'CNAME'), ('A', 'A (IPv4 address)'), ('AAAA', 'AAAA (IPv6 address)'), ('SRV', 'SRV'), ('TXT', 'TXT'), ('SOA', 'SOA')])), ('value', models.CharField(max_length=256, verbose_name='value')), - ('domain', models.ForeignKey(related_name='records', to='domains.Domain', verbose_name='domain')), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='domains.Domain', verbose_name='domain')), ], ), ] diff --git a/orchestra/contrib/domains/migrations/0001_squashed_0010_auto_20210330_1049.py b/orchestra/contrib/domains/migrations/0001_squashed_0010_auto_20210330_1049.py new file mode 100644 index 00000000..ee8cfb2f --- /dev/null +++ b/orchestra/contrib/domains/migrations/0001_squashed_0010_auto_20210330_1049.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.contrib.domains.utils +import orchestra.contrib.domains.validators + + +class Migration(migrations.Migration): + + replaces = [('domains', '0001_initial'), ('domains', '0002_auto_20150715_1017'), ('domains', '0003_auto_20150720_1121'), ('domains', '0004_auto_20150720_1121'), ('domains', '0005_auto_20160219_1034'), ('domains', '0006_auto_20170528_2011'), ('domains', '0007_auto_20190805_1134'), ('domains', '0008_domain_dns2136_address_match_list'), ('domains', '0009_auto_20200204_1217'), ('domains', '0010_auto_20210330_1049')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Domain', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Domain or subdomain name.', max_length=256, unique=True, validators=[orchestra.contrib.domains.validators.validate_domain_name, orchestra.contrib.domains.validators.validate_allowed_domain], verbose_name='name')), + ('serial', models.IntegerField(default=orchestra.contrib.domains.utils.generate_zone_serial, help_text='Serial number', verbose_name='serial')), + ('account', models.ForeignKey(blank=True, help_text='Automatically selected for subdomains.', on_delete=django.db.models.deletion.CASCADE, related_name='domains', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('top', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='subdomain_set', to='domains.Domain')), + ], + ), + migrations.CreateModel( + name='Record', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ttl', models.CharField(blank=True, help_text='Record TTL, defaults to 1h', max_length=8, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='TTL')), + ('type', models.CharField(choices=[('MX', 'MX'), ('NS', 'NS'), ('CNAME', 'CNAME'), ('A', 'A (IPv4 address)'), ('AAAA', 'AAAA (IPv6 address)'), ('SRV', 'SRV'), ('TXT', 'TXT'), ('SPF', 'SPF')], max_length=32, verbose_name='type')), + ('value', models.CharField(help_text='MX, NS and CNAME records sould end with a dot.', max_length=1024, verbose_name='value')), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='domains.Domain', verbose_name='domain')), + ], + ), + migrations.AlterField( + model_name='domain', + name='serial', + field=models.IntegerField(default=orchestra.contrib.domains.utils.generate_zone_serial, editable=False, help_text='A revision number that changes whenever this domain is updated.', verbose_name='serial'), + ), + migrations.AddField( + model_name='domain', + name='expire', + field=models.CharField(blank=True, help_text='The time that a secondary server will keep trying to complete a zone transfer. If this time expires prior to a successful zone transfer, the secondary server will expire its zone file. This means the secondary will stop answering queries. The default value is 4w.', max_length=16, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='expire'), + ), + migrations.AddField( + model_name='domain', + name='min_ttl', + field=models.CharField(blank=True, help_text='The minimum time-to-live value applies to all resource records in the zone file. This value is supplied in query responses to inform other servers how long they should keep the data in cache. The default value is 1h.', max_length=16, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='min TTL'), + ), + migrations.AddField( + model_name='domain', + name='refresh', + field=models.CharField(blank=True, help_text="The time a secondary DNS server waits before querying the primary DNS server's SOA record to check for changes. When the refresh time expires, the secondary DNS server requests a copy of the current SOA record from the primary. The primary DNS server complies with this request. The secondary DNS server compares the serial number of the primary DNS server's current SOA record and the serial number in it's own SOA record. If they are different, the secondary DNS server will request a zone transfer from the primary DNS server. The default value is 1d.", max_length=16, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='refresh'), + ), + migrations.AddField( + model_name='domain', + name='retry', + field=models.CharField(blank=True, help_text='The time a secondary server waits before retrying a failed zone transfer. Normally, the retry time is less than the refresh time. The default value is 2h.', max_length=16, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='retry'), + ), + migrations.AlterField( + model_name='domain', + name='name', + field=models.CharField(db_index=True, help_text='Domain or subdomain name.', max_length=256, unique=True, validators=[orchestra.contrib.domains.validators.validate_domain_name, orchestra.contrib.domains.validators.validate_allowed_domain], verbose_name='name'), + ), + migrations.AlterField( + model_name='domain', + name='top', + field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='subdomain_set', to='domains.Domain', verbose_name='top domain'), + ), + migrations.AddField( + model_name='domain', + name='dns2136_address_match_list', + field=models.CharField(blank=True, default='key pangea.key;', help_text="A bind-9 'address_match_list' that will be granted permission to perform dns2136 updates. Chiefly used to enable Let's Encrypt self-service validation.", max_length=80), + ), + ] diff --git a/orchestra/contrib/domains/migrations/0005_auto_20160219_1034.py b/orchestra/contrib/domains/migrations/0005_auto_20160219_1034.py index 3d8a3c5c..7ef16e9b 100644 --- a/orchestra/contrib/domains/migrations/0005_auto_20160219_1034.py +++ b/orchestra/contrib/domains/migrations/0005_auto_20160219_1034.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import migrations, models +import django.db.models.deletion import orchestra.contrib.domains.validators @@ -20,7 +21,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='domain', name='top', - field=models.ForeignKey(editable=False, verbose_name='top domain', related_name='subdomain_set', to='domains.Domain', null=True), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, editable=False, verbose_name='top domain', related_name='subdomain_set', to='domains.Domain', null=True), ), migrations.AlterField( model_name='record', diff --git a/orchestra/contrib/domains/migrations/0010_auto_20210330_1049.py b/orchestra/contrib/domains/migrations/0010_auto_20210330_1049.py new file mode 100644 index 00000000..37f81aaa --- /dev/null +++ b/orchestra/contrib/domains/migrations/0010_auto_20210330_1049.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models +import orchestra.contrib.domains.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('domains', '0009_auto_20200204_1217'), + ] + + operations = [ + migrations.AlterField( + model_name='domain', + name='min_ttl', + field=models.CharField(blank=True, help_text='The minimum time-to-live value applies to all resource records in the zone file. This value is supplied in query responses to inform other servers how long they should keep the data in cache. The default value is 1h.', max_length=16, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='min TTL'), + ), + migrations.AlterField( + model_name='record', + name='ttl', + field=models.CharField(blank=True, help_text='Record TTL, defaults to 1h', max_length=8, validators=[orchestra.contrib.domains.validators.validate_zone_interval], verbose_name='TTL'), + ), + ] diff --git a/orchestra/contrib/domains/models.py b/orchestra/contrib/domains/models.py index 4cbcfc93..05434559 100644 --- a/orchestra/contrib/domains/models.py +++ b/orchestra/contrib/domains/models.py @@ -31,9 +31,9 @@ class Domain(models.Model): validators.validate_allowed_domain ]) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True, - related_name='domains', help_text=_("Automatically selected for subdomains.")) + related_name='domains', on_delete=models.CASCADE, help_text=_("Automatically selected for subdomains.")) top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set', - editable=False, verbose_name=_("top domain")) + editable=False, verbose_name=_("top domain"), on_delete=models.CASCADE) serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False, help_text=_("A revision number that changes whenever this domain is updated.")) refresh = models.CharField(_("refresh"), max_length=16, blank=True, @@ -69,16 +69,16 @@ class Domain(models.Model): blank=True, help_text="A bind-9 'address_match_list' that will be granted permission to perform " "dns2136 updates. Chiefly used to enable Let's Encrypt self-service validation.") - + objects = DomainQuerySet.as_manager() - + def __str__(self): return self.name - + @property def origin(self): return self.top or self - + @property def is_top(self): # don't cache, don't replace by top_id @@ -86,14 +86,14 @@ class Domain(models.Model): return not bool(self.top) except Domain.DoesNotExist: return False - + @property def subdomains(self): return Domain.objects.filter(name__regex='\.%s$' % self.name) - + def clean(self): self.name = self.name.lower() - + def save(self, *args, **kwargs): """ create top relation """ update = False @@ -110,7 +110,7 @@ class Domain(models.Model): # queryset.update() is not used because we want to trigger backend to delete ex-topdomains domain.top = self domain.save(update_fields=('top',)) - + def get_description(self): if self.is_top: num = self.subdomains.count() @@ -119,21 +119,21 @@ class Domain(models.Model): _("top domain with %d subdomains") % num, num) return _("subdomain") - + def get_absolute_url(self): return 'http://%s' % self.name - + def get_declared_records(self): """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.records.all() - + def get_subdomains(self): """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.origin.subdomain_set.all().prefetch_related('records') - + def get_parent(self, top=False): return type(self).objects.get_parent(self.name, top=top) - + def render_zone(self): origin = self.origin zone = origin.render_records() @@ -147,7 +147,7 @@ class Domain(models.Model): for subdomain in sorted(tail, key=lambda x: len(x.name), reverse=True): zone += subdomain.render_records() return zone.strip() - + def refresh_serial(self): """ Increases the domain serial number by one """ serial = utils.generate_zone_serial() @@ -159,7 +159,7 @@ class Domain(models.Model): serial = int(serial) self.serial = serial self.save(update_fields=('serial',)) - + def get_default_soa(self): return ' '.join([ "%s." % settings.DOMAINS_DEFAULT_NAME_SERVER, @@ -170,7 +170,7 @@ class Domain(models.Model): self.expire or settings.DOMAINS_DEFAULT_EXPIRE, self.min_ttl or settings.DOMAINS_DEFAULT_MIN_TTL, ]) - + def get_default_records(self): defaults = [] if self.is_top: @@ -202,7 +202,7 @@ class Domain(models.Model): value=default_aaaa )) return defaults - + def record_is_implicit(self, record, types): if record.type not in types: if record.type is Record.NS: @@ -221,7 +221,7 @@ class Domain(models.Model): elif not has_a and not has_aaaa: return True return False - + def get_records(self): types = set() records = utils.RecordStorage() @@ -249,7 +249,7 @@ class Domain(models.Model): else: records.append(record) return records - + def render_records(self): result = '' for record in self.get_records(): @@ -273,7 +273,7 @@ class Domain(models.Model): value=record.value ) return result - + def has_default_mx(self): records = self.get_records() for record in records.by_type('MX'): @@ -294,7 +294,7 @@ class Record(models.Model): TXT = 'TXT' SPF = 'SPF' SOA = 'SOA' - + TYPE_CHOICES = ( (MX, "MX"), (NS, "NS"), @@ -305,7 +305,7 @@ class Record(models.Model): (TXT, "TXT"), (SPF, "SPF"), ) - + VALIDATORS = { MX: (validators.validate_mx_record,), NS: (validators.validate_zone_label,), @@ -317,8 +317,8 @@ class Record(models.Model): SRV: (validators.validate_srv_record,), SOA: (validators.validate_soa_record,), } - - domain = models.ForeignKey(Domain, verbose_name=_("domain"), related_name='records') + + domain = models.ForeignKey(Domain, verbose_name=_("domain"), related_name='records', on_delete=models.CASCADE) ttl = models.CharField(_("TTL"), max_length=8, blank=True, help_text=_("Record TTL, defaults to %s") % settings.DOMAINS_DEFAULT_TTL, validators=[validators.validate_zone_interval]) @@ -326,10 +326,10 @@ class Record(models.Model): # max_length bumped from 256 to 1024 (arbitrary) on August 2019. value = models.CharField(_("value"), max_length=1024, help_text=_("MX, NS and CNAME records sould end with a dot.")) - + def __str__(self): return "%s %s IN %s %s" % (self.domain, self.get_ttl(), self.type, self.value) - + def clean(self): """ validates record value based on its type """ # validate value @@ -343,6 +343,6 @@ class Record(models.Model): raise ValidationError({ 'value': error, }) - + def get_ttl(self): return self.ttl or settings.DOMAINS_DEFAULT_TTL diff --git a/orchestra/contrib/domains/tests/functional_tests/tests.py b/orchestra/contrib/domains/tests/functional_tests/tests.py index fbc92478..f13342e4 100644 --- a/orchestra/contrib/domains/tests/functional_tests/tests.py +++ b/orchestra/contrib/domains/tests/functional_tests/tests.py @@ -4,7 +4,7 @@ import socket from functools import partial from django.conf import settings as djsettings -from django.core.urlresolvers import reverse +from django.urls import reverse from selenium.webdriver.support.select import Select from orchestra.contrib.orchestration.models import Server, Route @@ -23,7 +23,7 @@ class DomainTestMixin(object): SLAVE_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') MASTER_SERVER_ADDR = socket.gethostbyname(MASTER_SERVER) SLAVE_SERVER_ADDR = socket.gethostbyname(SLAVE_SERVER) - + def setUp(self): djsettings.DEBUG = True super(DomainTestMixin, self).setUp() @@ -53,19 +53,19 @@ class DomainTestMixin(object): (Record.CNAME, 'external.server.org.'), ) self.django_domain_name = 'django%s.lan' % random_ascii(10) - + def add_route(self): raise NotImplementedError - + def add(self, domain_name, records): raise NotImplementedError - + def delete(self, domain_name, records): raise NotImplementedError - + def update(self, domain_name, records): raise NotImplementedError - + def validate_add(self, server_addr, domain_name): context = { 'domain_name': domain_name, @@ -81,7 +81,7 @@ class DomainTestMixin(object): self.assertEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) self.assertEqual(hostmaster, soa[5]) - + dig_ns = 'dig @%(server_addr)s %(domain_name)s NS|grep "\sNS\s"' name_servers = run(dig_ns % context).stdout # testdomain.org. 3600 IN NS ns1.orchestra.lan. @@ -95,7 +95,7 @@ class DomainTestMixin(object): self.assertEqual('IN', ns[2]) self.assertEqual('NS', ns[3]) self.assertIn(ns[4], ns_records) - + dig_mx = 'dig @%(server_addr)s %(domain_name)s MX|grep "\sMX\s"' mail_servers = run(dig_mx % context).stdout for mx in mail_servers.splitlines(): @@ -107,7 +107,7 @@ class DomainTestMixin(object): self.assertEqual('MX', mx[3]) self.assertIn(mx[4], ['10', '20']) self.assertIn(mx[5], ['mail2.orchestra.lan.', 'mail.orchestra.lan.']) - + def validate_delete(self, server_addr, domain_name): context = { 'domain_name': domain_name, @@ -122,7 +122,7 @@ class DomainTestMixin(object): self.assertNotEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) self.assertNotEqual(hostmaster, soa[5]) - + def validate_update(self, server_addr, domain_name): context = { 'domain_name': domain_name, @@ -138,7 +138,7 @@ class DomainTestMixin(object): self.assertEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) self.assertEqual(hostmaster, soa[5]) - + dig_ns = 'dig @%(server_addr)s %(domain_name)s NS |grep "\sNS\s"' name_servers = run(dig_ns % context).stdout ns_records = ['ns1.%s.' % self.domain_name, 'ns2.%s.' % self.domain_name] @@ -151,7 +151,7 @@ class DomainTestMixin(object): self.assertEqual('IN', ns[2]) self.assertEqual('NS', ns[3]) self.assertIn(ns[4], ns_records) - + dig_mx = 'dig @%(server_addr)s %(domain_name)s MX | grep "\sMX\s"' mx = run(dig_mx % context).stdout.split() # testdomain.org. 3600 IN MX 10 orchestra.lan. @@ -161,7 +161,7 @@ class DomainTestMixin(object): self.assertEqual('MX', mx[3]) self.assertIn(mx[4], ['30', '40']) self.assertIn(mx[5], ['mail3.orchestra.lan.', 'mail4.orchestra.lan.']) - + def validate_www_update(self, server_addr, domain_name): context = { 'domain_name': domain_name, @@ -175,7 +175,7 @@ class DomainTestMixin(object): self.assertEqual('IN', cname[2]) self.assertEqual('CNAME', cname[3]) self.assertEqual('external.server.org.', cname[4]) - + def test_add(self): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) @@ -184,7 +184,7 @@ class DomainTestMixin(object): self.validate_add(self.MASTER_SERVER_ADDR, self.domain_name) time.sleep(1) self.validate_add(self.SLAVE_SERVER_ADDR, self.domain_name) - + def test_delete(self): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) @@ -193,7 +193,7 @@ class DomainTestMixin(object): for name in [self.domain_name, self.ns1_name, self.ns2_name]: self.validate_delete(self.MASTER_SERVER_ADDR, name) self.validate_delete(self.SLAVE_SERVER_ADDR, name) - + def test_update(self): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) @@ -209,7 +209,7 @@ class DomainTestMixin(object): self.validate_www_update(self.MASTER_SERVER_ADDR, self.domain_name) time.sleep(5) self.validate_www_update(self.SLAVE_SERVER_ADDR, self.domain_name) - + def test_add_add_delete_delete(self): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) @@ -221,7 +221,7 @@ class DomainTestMixin(object): self.delete(self.django_domain_name) self.validate_delete(self.MASTER_SERVER_ADDR, self.django_domain_name) self.validate_delete(self.SLAVE_SERVER_ADDR, self.django_domain_name) - + def test_bad_creation(self): self.assertRaises((self.rest.ResponseStatusError, AssertionError), self.add, self.domain_name, self.domain_records) @@ -232,7 +232,7 @@ class AdminDomainMixin(DomainTestMixin): super(AdminDomainMixin, self).setUp() self.add_route() self.admin_login() - + def _add_records(self, records): self.selenium.find_element_by_link_text('Add another Record').click() for i, record in zip(range(0, len(records)), records): @@ -244,29 +244,29 @@ class AdminDomainMixin(DomainTestMixin): value_input.clear() value_input.send_keys(value) return value_input - + @snapshot_on_error def add(self, domain_name, records): add = reverse('admin:domains_domain_add') url = self.live_server_url + add self.selenium.get(url) - + name = self.selenium.find_element_by_id('id_name') name.send_keys(domain_name) - + account_input = self.selenium.find_element_by_id('id_account') account_select = Select(account_input) account_select.select_by_value(str(self.account.pk)) - + value_input = self._add_records(records) value_input.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete(self, domain_name): domain = Domain.objects.get(name=domain_name) self.admin_delete(domain) - + @snapshot_on_error def update(self, domain_name, records): domain = Domain.objects.get(name=domain_name) @@ -283,18 +283,18 @@ class RESTDomainMixin(DomainTestMixin): super(RESTDomainMixin, self).setUp() self.rest_login() self.add_route() - + @save_response_on_error def add(self, domain_name, records): records = [ dict(type=type, value=value) for type,value in records ] self.rest.domains.create(name=domain_name, records=records) - + @save_response_on_error def delete(self, domain_name): domain = Domain.objects.get(name=domain_name) domain = self.rest.domains.retrieve(id=domain.pk) domain.delete() - + @save_response_on_error def update(self, domain_name, records): records = [ dict(type=type, value=value) for type,value in records ] @@ -307,7 +307,7 @@ class Bind9BackendMixin(object): DEPENDENCIES = ( 'orchestra.contrib.orchestration', ) - + def add_route(self): master = Server.objects.create(name=self.MASTER_SERVER, address=self.MASTER_SERVER_ADDR) backend = backends.Bind9MasterDomainController.get_name() diff --git a/orchestra/contrib/history/admin.py b/orchestra/contrib/history/admin.py index 09a35e07..ebd80e46 100644 --- a/orchestra/contrib/history/admin.py +++ b/orchestra/contrib/history/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin from django.utils.translation import ugettext_lazy as _ -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.contrib.admin.templatetags.admin_urls import add_preserved_filters from django.http import HttpResponseRedirect from django.contrib.admin.utils import unquote @@ -30,10 +30,10 @@ class LogEntryAdmin(admin.ModelAdmin): actions = None list_select_related = ('user', 'content_type') list_display_links = None - + user_link = admin_link('user') display_action_time = admin_date('action_time', short_description=_("Time")) - + def display_message(self, log): edit = '' % { 'url': reverse('admin:admin_logentry_change', args=(log.pk,)), @@ -58,7 +58,7 @@ class LogEntryAdmin(admin.ModelAdmin): display_message.short_description = _("Message") display_message.admin_order_field = 'action_flag' display_message.allow_tags = True - + def display_action(self, log): if log.is_addition(): return _("Added") @@ -67,7 +67,7 @@ class LogEntryAdmin(admin.ModelAdmin): return _("Deleted") display_action.short_description = _("Action") display_action.admin_order_field = 'action_flag' - + def content_object_link(self, log): ct = log.content_type view = 'admin:%s_%s_change' % (ct.app_label, ct.model) @@ -79,7 +79,7 @@ class LogEntryAdmin(admin.ModelAdmin): content_object_link.short_description = _("Content object") content_object_link.admin_order_field = 'object_repr' content_object_link.allow_tags = True - + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): """ Add rel_opts and object to context """ if not add and 'edit' in request.GET.urlencode(): @@ -89,14 +89,14 @@ class LogEntryAdmin(admin.ModelAdmin): }) return super(LogEntryAdmin, self).render_change_form( request, context, add, change, form_url, obj) - + def response_change(self, request, obj): """ save and continue preserve edit query string """ response = super(LogEntryAdmin, self).response_change(request, obj) if 'edit' in request.GET.urlencode() and 'edit' not in response.url: return HttpResponseRedirect(response.url + '?edit=True') return response - + def response_post_save_change(self, request, obj): """ save redirect to object history """ if 'edit' in request.GET.urlencode(): @@ -109,19 +109,19 @@ class LogEntryAdmin(admin.ModelAdmin): }, post_url) return HttpResponseRedirect(post_url) return super(LogEntryAdmin, self).response_post_save_change(request, obj) - + def has_add_permission(self, *args, **kwargs): return False - + def has_delete_permission(self, *args, **kwargs): return False - + def log_addition(self, *args, **kwargs): pass - + def log_change(self, *args, **kwargs): pass - + def log_deletion(self, *args, **kwargs): pass diff --git a/orchestra/contrib/issues/admin.py b/orchestra/contrib/issues/admin.py index 27580708..5dcc32ac 100644 --- a/orchestra/contrib/issues/admin.py +++ b/orchestra/contrib/issues/admin.py @@ -1,7 +1,7 @@ from django import forms from django.conf.urls import url from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.http import HttpResponse from django.shortcuts import get_object_or_404 @@ -21,14 +21,14 @@ from .helpers import get_ticket_changes, markdown_formated_changes, filter_actio from .models import Ticket, Queue, Message -PRIORITY_COLORS = { +PRIORITY_COLORS = { Ticket.HIGH: 'red', Ticket.MEDIUM: 'darkorange', Ticket.LOW: 'green', } -STATE_COLORS = { +STATE_COLORS = { Ticket.NEW: 'grey', Ticket.IN_PROGRESS: 'darkorange', Ticket.FEEDBACK: 'purple', @@ -44,12 +44,12 @@ class MessageReadOnlyInline(admin.TabularInline): can_delete = False fields = ('content_html',) readonly_fields = ('content_html',) - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + def content_html(self, msg): context = { 'number': msg.number, @@ -64,10 +64,10 @@ class MessageReadOnlyInline(admin.TabularInline): return header + content content_html.short_description = _("Content") content_html.allow_tags = True - + def has_add_permission(self, request): return False - + def has_delete_permission(self, request, obj=None): return False @@ -79,12 +79,12 @@ class MessageInline(admin.TabularInline): form = MessageInlineForm can_delete = False fields = ('content',) - + def get_formset(self, request, obj=None, **kwargs): """ hook request.user on the inline form """ self.form.user = request.user return super(MessageInline, self).get_formset(request, obj, **kwargs) - + def get_queryset(self, request): """ Don't show any message """ qs = super(MessageInline, self).get_queryset(request) @@ -103,14 +103,14 @@ class TicketInline(admin.TabularInline): model = Ticket extra = 0 max_num = 0 - + creator_link = admin_link('creator') owner_link = admin_link('owner') created = admin_link('created_at') updated = admin_link('updated_at') colored_state = admin_colored('state', colors=STATE_COLORS, bold=False) colored_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False) - + def ticket_id(self, instance): return '%s' % admin_link()(instance) ticket_id.short_description = '#' @@ -176,7 +176,7 @@ class TicketAdmin(ExtendedModelAdmin): }), ) list_select_related = ('queue', 'owner', 'creator') - + class Media: css = { 'all': ('issues/css/ticket-admin.css',) @@ -184,14 +184,14 @@ class TicketAdmin(ExtendedModelAdmin): js = ( 'issues/js/ticket-admin.js', ) - + display_creator = admin_link('creator') display_queue = admin_link('queue') display_owner = admin_link('owner') updated = admin_date('updated_at') display_state = admin_colored('state', colors=STATE_COLORS, bold=False) display_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False) - + def display_summary(self, ticket): context = { 'creator': admin_link('creator')(self, ticket) if ticket.creator else ticket.creator_name, @@ -208,7 +208,7 @@ class TicketAdmin(ExtendedModelAdmin): return '

Added by %(creator)s about %(created)s%(updated)s

' % context display_summary.short_description = 'Summary' display_summary.allow_tags = True - + def unbold_id(self, ticket): """ Unbold id if ticket is read """ if ticket.is_read_by(self.user): @@ -217,7 +217,7 @@ class TicketAdmin(ExtendedModelAdmin): unbold_id.allow_tags = True unbold_id.short_description = "#" unbold_id.admin_order_field = 'id' - + def bold_subject(self, ticket): """ Bold subject when tickets are unread for request.user """ if ticket.is_read_by(self.user): @@ -226,31 +226,31 @@ class TicketAdmin(ExtendedModelAdmin): bold_subject.allow_tags = True bold_subject.short_description = _("Subject") bold_subject.admin_order_field = 'subject' - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'subject': kwargs['widget'] = forms.TextInput(attrs={'size':'120'}) return super(TicketAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def save_model(self, request, obj, *args, **kwargs): """ Define creator for new tickets """ if not obj.pk: obj.creator = request.user super(TicketAdmin, self).save_model(request, obj, *args, **kwargs) obj.mark_as_read_by(request.user) - + def get_urls(self): """ add markdown preview url """ return [ url(r'^preview/$', wrap_admin_view(self, self.message_preview_view)) ] + super(TicketAdmin, self).get_urls() - + def add_view(self, request, form_url='', extra_context=None): """ Do not sow message inlines """ return super(TicketAdmin, self).add_view(request, form_url, extra_context) - + def change_view(self, request, object_id, form_url='', extra_context=None): """ Change view actions based on ticket state """ ticket = get_object_or_404(Ticket, pk=object_id) @@ -269,12 +269,12 @@ class TicketAdmin(ExtendedModelAdmin): context.update(extra_context or {}) return super(TicketAdmin, self).change_view(request, object_id, form_url=form_url, extra_context=context) - + def changelist_view(self, request, extra_context=None): # Hook user for bold_subject self.user = request.user return super(TicketAdmin,self).changelist_view(request, extra_context=extra_context) - + def message_preview_view(self, request): """ markdown preview render via ajax """ data = request.POST.get("data") @@ -287,12 +287,12 @@ class QueueAdmin(admin.ModelAdmin): actions = (set_default_queue,) inlines = (TicketInline,) ordering = ('name',) - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + def num_tickets(self, queue): num = queue.tickets__count url = reverse('admin:issues_ticket_changelist') @@ -301,7 +301,7 @@ class QueueAdmin(admin.ModelAdmin): num_tickets.short_description = _("Tickets") num_tickets.admin_order_field = 'tickets__count' num_tickets.allow_tags = True - + def get_list_display(self, request): """ show notifications """ list_display = list(self.list_display) @@ -312,7 +312,7 @@ class QueueAdmin(admin.ModelAdmin): display_notify.boolean = True list_display.append(display_notify) return list_display - + def get_queryset(self, request): qs = super(QueueAdmin, self).get_queryset(request) qs = qs.annotate(models.Count('tickets')) diff --git a/orchestra/contrib/issues/migrations/0001_initial.py b/orchestra/contrib/issues/migrations/0001_initial.py index 1dd5d46a..0c76499c 100644 --- a/orchestra/contrib/issues/migrations/0001_initial.py +++ b/orchestra/contrib/issues/migrations/0001_initial.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import django.db.models.deletion from django.db import models, migrations import orchestra.models.fields from django.conf import settings @@ -20,7 +21,7 @@ class Migration(migrations.Migration): ('author_name', models.CharField(blank=True, max_length=256, verbose_name='author name')), ('content', models.TextField(verbose_name='content')), ('created_on', models.DateTimeField(auto_now_add=True, verbose_name='created on')), - ('author', models.ForeignKey(related_name='ticket_messages', to=settings.AUTH_USER_MODEL, verbose_name='author')), + ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ticket_messages', to=settings.AUTH_USER_MODEL, verbose_name='author')), ], options={ 'get_latest_by': 'id', @@ -48,9 +49,9 @@ class Migration(migrations.Migration): ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created')), ('updated_at', models.DateTimeField(auto_now=True, verbose_name='modified')), ('cc', models.TextField(blank=True, help_text='emails to send a carbon copy to', verbose_name='CC')), - ('creator', models.ForeignKey(related_name='tickets_created', null=True, to=settings.AUTH_USER_MODEL, verbose_name='created by')), - ('owner', models.ForeignKey(blank=True, related_name='tickets_owned', null=True, to=settings.AUTH_USER_MODEL, verbose_name='assigned to')), - ('queue', models.ForeignKey(blank=True, related_name='tickets', null=True, to='issues.Queue')), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tickets_created', null=True, to=settings.AUTH_USER_MODEL, verbose_name='created by')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, blank=True, related_name='tickets_owned', null=True, to=settings.AUTH_USER_MODEL, verbose_name='assigned to')), + ('queue', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, blank=True, related_name='tickets', null=True, to='issues.Queue')), ], options={ 'ordering': ['-updated_at'], @@ -60,14 +61,14 @@ class Migration(migrations.Migration): name='TicketTracker', fields=[ ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')), - ('ticket', models.ForeignKey(related_name='trackers', to='issues.Ticket', verbose_name='ticket')), - ('user', models.ForeignKey(related_name='ticket_trackers', to=settings.AUTH_USER_MODEL, verbose_name='user')), + ('ticket', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='trackers', to='issues.Ticket', verbose_name='ticket')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ticket_trackers', to=settings.AUTH_USER_MODEL, verbose_name='user')), ], ), migrations.AddField( model_name='message', name='ticket', - field=models.ForeignKey(related_name='messages', to='issues.Ticket', verbose_name='ticket'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='issues.Ticket', verbose_name='ticket'), ), migrations.AlterUniqueTogether( name='tickettracker', diff --git a/orchestra/contrib/issues/migrations/0001_squashed_0004_auto_20170528_2011.py b/orchestra/contrib/issues/migrations/0001_squashed_0004_auto_20170528_2011.py new file mode 100644 index 00000000..2763cf79 --- /dev/null +++ b/orchestra/contrib/issues/migrations/0001_squashed_0004_auto_20170528_2011.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +import datetime +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +from django.utils.timezone import utc +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('issues', '0001_initial'), ('issues', '0002_auto_20150709_1018'), ('issues', '0003_auto_20160320_1127'), ('issues', '0004_auto_20170528_2011')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Message', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('author_name', models.CharField(blank=True, max_length=256, verbose_name='author name')), + ('content', models.TextField(verbose_name='content')), + ('created_on', models.DateTimeField(auto_now_add=True, verbose_name='created on')), + ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ticket_messages', to=settings.AUTH_USER_MODEL, verbose_name='author')), + ], + options={ + 'get_latest_by': 'id', + }, + ), + migrations.CreateModel( + name='Queue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, unique=True, verbose_name='name')), + ('verbose_name', models.CharField(blank=True, max_length=128, verbose_name='verbose_name')), + ('default', models.BooleanField(default=False, verbose_name='default')), + ('notify', orchestra.models.fields.MultiSelectField(blank=True, choices=[('SUPPORT', 'Support tickets'), ('ADMIN', 'Administrative'), ('BILLING', 'Billing'), ('TECH', 'Technical'), ('ADDS', 'Announcements'), ('EMERGENCY', 'Emergency contact')], default=('SUPPORT', 'ADMIN', 'BILLING', 'TECH', 'ADDS', 'EMERGENCY'), help_text='Contacts to notify by email', max_length=256, verbose_name='notify')), + ], + ), + migrations.CreateModel( + name='Ticket', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creator_name', models.CharField(blank=True, max_length=256, verbose_name='creator name')), + ('subject', models.CharField(max_length=256, verbose_name='subject')), + ('description', models.TextField(verbose_name='description')), + ('priority', models.CharField(choices=[('HIGH', 'High'), ('MEDIUM', 'Medium'), ('LOW', 'Low')], default='MEDIUM', max_length=32, verbose_name='priority')), + ('state', models.CharField(choices=[('NEW', 'New'), ('IN_PROGRESS', 'In Progress'), ('RESOLVED', 'Resolved'), ('FEEDBACK', 'Feedback'), ('REJECTED', 'Rejected'), ('CLOSED', 'Closed')], default='NEW', max_length=32, verbose_name='state')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='modified')), + ('cc', models.TextField(blank=True, help_text='emails to send a carbon copy to', verbose_name='CC')), + ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tickets_created', to=settings.AUTH_USER_MODEL, verbose_name='created by')), + ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tickets_owned', to=settings.AUTH_USER_MODEL, verbose_name='assigned to')), + ('queue', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tickets', to='issues.Queue')), + ], + options={ + 'ordering': ['-updated_at'], + }, + ), + migrations.CreateModel( + name='TicketTracker', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ticket', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='trackers', to='issues.Ticket', verbose_name='ticket')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ticket_trackers', to=settings.AUTH_USER_MODEL, verbose_name='user')), + ], + ), + migrations.AddField( + model_name='message', + name='ticket', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='issues.Ticket', verbose_name='ticket'), + ), + migrations.AlterUniqueTogether( + name='tickettracker', + unique_together=set([('ticket', 'user')]), + ), + migrations.AlterField( + model_name='ticket', + name='created_at', + field=models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='created'), + ), + migrations.RemoveField( + model_name='message', + name='created_on', + ), + migrations.AddField( + model_name='message', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=datetime.datetime(2016, 3, 20, 10, 27, 45, 766388, tzinfo=utc), verbose_name='created at'), + preserve_default=False, + ), + migrations.AlterField( + model_name='ticket', + name='creator', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tickets_created', to=settings.AUTH_USER_MODEL, verbose_name='created by'), + ), + migrations.AlterField( + model_name='ticket', + name='owner', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tickets_owned', to=settings.AUTH_USER_MODEL, verbose_name='assigned to'), + ), + migrations.AlterField( + model_name='ticket', + name='queue', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tickets', to='issues.Queue'), + ), + ] diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py index f2a75f27..8c783ea3 100644 --- a/orchestra/contrib/issues/models.py +++ b/orchestra/contrib/issues/models.py @@ -19,10 +19,10 @@ class Queue(models.Model): choices=Contact.EMAIL_USAGES, default=contacts_settings.CONTACTS_DEFAULT_EMAIL_USAGES, help_text=_("Contacts to notify by email")) - + def __str__(self): return self.verbose_name or self.name - + def save(self, *args, **kwargs): """ mark as default queue if needed """ existing_default = Queue.objects.filter(default=True) @@ -48,7 +48,7 @@ class Ticket(models.Model): (MEDIUM, 'Medium'), (LOW, 'Low'), ) - + NEW = 'NEW' IN_PROGRESS = 'IN_PROGRESS' RESOLVED = 'RESOLVED' @@ -63,7 +63,7 @@ class Ticket(models.Model): (REJECTED, 'Rejected'), (CLOSED, 'Closed'), ) - + creator = models.ForeignKey(djsettings.AUTH_USER_MODEL, verbose_name=_("created by"), related_name='tickets_created', null=True, on_delete=models.SET_NULL) creator_name = models.CharField(_("creator name"), max_length=256, blank=True) @@ -79,15 +79,15 @@ class Ticket(models.Model): created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) updated_at = models.DateTimeField(_("modified"), auto_now=True) cc = models.TextField("CC", help_text=_("emails to send a carbon copy to"), blank=True) - + objects = TicketQuerySet.as_manager() - + class Meta: ordering = ['-updated_at'] - + def __str__(self): return str(self.pk) - + def get_notification_emails(self): """ Get emails of the users related to the ticket """ emails = list(settings.ISSUES_SUPPORT_EMAILS) @@ -100,7 +100,7 @@ class Ticket(models.Model): for message in self.messages.distinct('author'): emails.append(message.author.email) return set(emails + self.get_cc_emails()) - + def notify(self, message=None, content=None): """ Send an email to ticket stakeholders notifying an state update """ emails = self.get_notification_emails() @@ -111,7 +111,7 @@ class Ticket(models.Model): 'ticket_message': message } send_email_template(template, context, emails, html=html_template) - + def save(self, *args, **kwargs): """ notify stakeholders of new ticket """ new_issue = not self.pk @@ -121,60 +121,60 @@ class Ticket(models.Model): if new_issue: # PK should be available for rendering the template self.notify() - + def is_involved_by(self, user): """ returns whether user has participated or is referenced on the ticket as owner or member of the group """ return Ticket.objects.filter(pk=self.pk).involved_by(user).exists() - + def get_cc_emails(self): return self.cc.split(',') if self.cc else [] - + def mark_as_read_by(self, user): self.trackers.get_or_create(user=user) - + def mark_as_unread_by(self, user): self.trackers.filter(user=user).delete() - + def mark_as_unread(self): self.trackers.all().delete() - + def is_read_by(self, user): return self.trackers.filter(user=user).exists() - + def reject(self): self.state = Ticket.REJECTED self.save(update_fields=('state', 'updated_at')) - + def resolve(self): self.state = Ticket.RESOLVED self.save(update_fields=('state', 'updated_at')) - + def close(self): self.state = Ticket.CLOSED self.save(update_fields=('state', 'updated_at')) - + def take(self, user): self.owner = user self.save(update_fields=('state', 'updated_at')) class Message(models.Model): - ticket = models.ForeignKey('issues.Ticket', verbose_name=_("ticket"), - related_name='messages') - author = models.ForeignKey(djsettings.AUTH_USER_MODEL, verbose_name=_("author"), - related_name='ticket_messages') + ticket = models.ForeignKey('issues.Ticket', on_delete=models.CASCADE, + verbose_name=_("ticket"), related_name='messages') + author = models.ForeignKey(djsettings.AUTH_USER_MODEL, on_delete=models.CASCADE, + verbose_name=_("author"), related_name='ticket_messages') author_name = models.CharField(_("author name"), max_length=256, blank=True) content = models.TextField(_("content")) created_at = models.DateTimeField(_("created at"), auto_now_add=True) - + class Meta: get_latest_by = 'id' - + def __str__(self): return "#%i" % self.id - + def save(self, *args, **kwargs): """ notify stakeholders of ticket update """ if not self.pk: @@ -183,7 +183,7 @@ class Message(models.Model): self.ticket.notify(message=self) self.author_name = self.author.get_full_name() super(Message, self).save(*args, **kwargs) - + @property def number(self): return self.ticket.messages.filter(id__lte=self.id).count() @@ -191,10 +191,11 @@ class Message(models.Model): class TicketTracker(models.Model): """ Keeps track of user read tickets """ - ticket = models.ForeignKey(Ticket, verbose_name=_("ticket"), related_name='trackers') - user = models.ForeignKey(djsettings.AUTH_USER_MODEL, verbose_name=_("user"), - related_name='ticket_trackers') - + ticket = models.ForeignKey(Ticket, on_delete=models.CASCADE, + verbose_name=_("ticket"), related_name='trackers') + user = models.ForeignKey(djsettings.AUTH_USER_MODEL, on_delete=models.CASCADE, + verbose_name=_("user"), related_name='ticket_trackers') + class Meta: unique_together = ( ('ticket', 'user'), diff --git a/orchestra/contrib/lists/migrations/0001_initial.py b/orchestra/contrib/lists/migrations/0001_initial.py index 73497c53..b6821076 100644 --- a/orchestra/contrib/lists/migrations/0001_initial.py +++ b/orchestra/contrib/lists/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings +import django.db.models.deletion import orchestra.core.validators @@ -22,8 +23,8 @@ class Migration(migrations.Migration): ('address_name', models.CharField(max_length=128, validators=[orchestra.core.validators.validate_name], verbose_name='address name', blank=True)), ('admin_email', models.EmailField(max_length=254, verbose_name='admin email', help_text='Administration email address')), ('is_active', models.BooleanField(default=True, verbose_name='active', help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.')), - ('account', models.ForeignKey(related_name='lists', to=settings.AUTH_USER_MODEL, verbose_name='Account')), - ('address_domain', models.ForeignKey(null=True, blank=True, to='domains.Domain', verbose_name='address domain')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='lists', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('address_domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, null=True, blank=True, to='domains.Domain', verbose_name='address domain')), ], ), migrations.AlterUniqueTogether( diff --git a/orchestra/contrib/lists/migrations/0001_squashed_0004_auto_20210330_1049.py b/orchestra/contrib/lists/migrations/0001_squashed_0004_auto_20210330_1049.py new file mode 100644 index 00000000..b630dd27 --- /dev/null +++ b/orchestra/contrib/lists/migrations/0001_squashed_0004_auto_20210330_1049.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('lists', '0001_initial'), ('lists', '0002_auto_20160912_1221'), ('lists', '0003_auto_20160912_1241'), ('lists', '0004_auto_20210330_1049')] + + initial = True + + dependencies = [ + ('domains', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='List', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Default list address <name>@lists.orchestra.lan', max_length=128, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('address_name', models.CharField(blank=True, max_length=128, validators=[orchestra.core.validators.validate_name], verbose_name='address name')), + ('admin_email', models.EmailField(help_text='Administration email address', max_length=254, verbose_name='admin email')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='lists', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('address_domain', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='domains.Domain', verbose_name='address domain')), + ], + ), + migrations.AlterUniqueTogether( + name='list', + unique_together=set([('address_name', 'address_domain')]), + ), + migrations.AlterField( + model_name='list', + name='address_domain', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='domains.Domain', verbose_name='address domain'), + ), + migrations.AlterField( + model_name='list', + name='address_name', + field=models.CharField(blank=True, max_length=52, validators=[orchestra.core.validators.validate_name], verbose_name='address name'), + ), + migrations.AlterField( + model_name='list', + name='name', + field=models.CharField(help_text='Default list address <name>@grups.pangea.org', max_length=52, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name'), + ), + migrations.AlterField( + model_name='list', + name='address_name', + field=models.CharField(blank=True, max_length=64, validators=[orchestra.core.validators.validate_name], verbose_name='address name'), + ), + migrations.AlterField( + model_name='list', + name='name', + field=models.CharField(help_text='Default list address <name>@grups.pangea.org', max_length=64, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name'), + ), + migrations.AlterField( + model_name='list', + name='name', + field=models.CharField(help_text='Default list address <name>@lists.orchestra.lan', max_length=64, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name'), + ), + ] diff --git a/orchestra/contrib/lists/migrations/0004_auto_20210330_1049.py b/orchestra/contrib/lists/migrations/0004_auto_20210330_1049.py new file mode 100644 index 00000000..94055a3d --- /dev/null +++ b/orchestra/contrib/lists/migrations/0004_auto_20210330_1049.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models +import orchestra.core.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('lists', '0003_auto_20160912_1241'), + ] + + operations = [ + migrations.AlterField( + model_name='list', + name='name', + field=models.CharField(help_text='Default list address <name>@lists.orchestra.lan', max_length=64, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name'), + ), + ] diff --git a/orchestra/contrib/lists/models.py b/orchestra/contrib/lists/models.py index 8dc50912..a78580d9 100644 --- a/orchestra/contrib/lists/models.py +++ b/orchestra/contrib/lists/models.py @@ -30,54 +30,54 @@ class List(models.Model): admin_email = models.EmailField(_("admin email"), help_text=_("Administration email address")) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='lists') + related_name='lists', on_delete=models.CASCADE) # TODO also admin is_active = models.BooleanField(_("active"), default=True, help_text=_("Designates whether this account should be treated as active. " "Unselect this instead of deleting accounts.")) password = None - + objects = ListQuerySet.as_manager() - + class Meta: unique_together = ('address_name', 'address_domain') - + def __str__(self): return self.name - + @property def address(self): if self.address_name and self.address_domain: return "%s@%s" % (self.address_name, self.address_domain) return '' - + @cached_property def active(self): return self.is_active and self.account.is_active - + def clean(self): if self.address_name and not self.address_domain_id: raise ValidationError({ 'address_domain': _("Domain should be selected for provided address name."), }) - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def get_address_name(self): return self.address_name or self.name - + def get_username(self): return self.name - + def set_password(self, password): self.password = password - + def get_absolute_url(self): context = { 'name': self.name diff --git a/orchestra/contrib/lists/serializers.py b/orchestra/contrib/lists/serializers.py index c40dc010..c4a666fd 100644 --- a/orchestra/contrib/lists/serializers.py +++ b/orchestra/contrib/lists/serializers.py @@ -12,7 +12,7 @@ from .models import List class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): class Meta: - model = List.address_domain.field.rel.to + model = List.address_domain.field.model fields = ('url', 'id', 'name') @@ -26,14 +26,14 @@ class ListSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): 'This value may contain any ascii character except for ' ' \'/"/\\/ characters.'), 'invalid'), ]) - + address_domain = RelatedDomainSerializer(required=False) - + class Meta: model = List fields = ('url', 'id', 'name', 'password', 'address_name', 'address_domain', 'admin_email', 'is_active',) postonly_fields = ('name', 'password') - + def validate_address_domain(self, address_name): if self.instance: address_domain = address_domain or self.instance.address_domain diff --git a/orchestra/contrib/lists/tests/functional_tests/tests.py b/orchestra/contrib/lists/tests/functional_tests/tests.py index 5ccc6acd..447a8249 100644 --- a/orchestra/contrib/lists/tests/functional_tests/tests.py +++ b/orchestra/contrib/lists/tests/functional_tests/tests.py @@ -1,24 +1,26 @@ import os import smtplib import time -import requests +import unittest from email.mime.text import MIMEText +import requests from django.conf import settings as djsettings from django.core.management.base import CommandError -from django.core.urlresolvers import reverse -from selenium.webdriver.support.select import Select - +from django.urls import reverse from orchestra.admin.utils import change_url from orchestra.contrib.domains.models import Domain -from orchestra.contrib.orchestration.models import Server, Route +from orchestra.contrib.orchestration.models import Route, Server from orchestra.utils.sys import sshrun -from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, snapshot_on_error, - save_response_on_error) +from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, + save_response_on_error, snapshot_on_error) +from selenium.webdriver.support.select import Select from ... import backends, settings from ...models import List +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + class ListMixin(object): MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') @@ -27,12 +29,12 @@ class ListMixin(object): 'orchestra.contrib.domains', 'orchestra.contrib.lists', ) - + def setUp(self): super(ListMixin, self).setUp() self.add_route() djsettings.DEBUG = True - + def validate_add(self, name, address=None): sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False) if not address: @@ -44,11 +46,11 @@ class ListMixin(object): sshrun(self.MASTER_SERVER, 'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"' % request_address, display=False) - + def validate_login(self, name, password): url = 'http://%s/cgi-bin/mailman/admin/%s' % (settings.LISTS_DEFAULT_DOMAIN, name) self.assertEqual(200, requests.post(url, data={'adminpw': password}).status_code) - + def validate_delete(self, name): context = { 'name': name, @@ -62,7 +64,7 @@ class ListMixin(object): 'grep "^\s*$(domain)s\s*$" %(virtual_domain)s' % context, display=False) self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, 'list_lists | grep -i "^\s*%(name)s\s"' % context, display=False) - + def subscribe(self, subscribe_address): msg = MIMEText('') msg['To'] = subscribe_address @@ -76,12 +78,12 @@ class ListMixin(object): server.sendmail(msg['From'], msg['To'], msg.as_string()) finally: server.quit() - + def add_route(self): server = Server.objects.create(name=self.MASTER_SERVER) backend = backends.MailmanController.get_name() Route.objects.create(backend=backend, match=True, host=server) - + def test_add(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -90,7 +92,7 @@ class ListMixin(object): self.validate_add(name) self.validate_login(name, password) self.addCleanup(self.delete, name) - + def test_add_with_address(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -102,7 +104,7 @@ class ListMixin(object): self.addCleanup(self.delete, name) # Mailman doesn't support changing the address, only the domain self.validate_add(name, address="%s@%s" % (address_name, address_domain)) - + def test_change_password(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -113,7 +115,7 @@ class ListMixin(object): new_password = '@!?%spppP001' % random_ascii(5) self.change_password(name, new_password) self.validate_login(name, new_password) - + def test_change_domain(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -128,7 +130,7 @@ class ListMixin(object): address_domain = Domain.objects.create(name=domain_name, account=self.account) self.update_domain(name, domain_name) self.validate_add(name, address="%s@%s" % (address_name, address_domain)) - + def test_change_address_name(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -142,7 +144,7 @@ class ListMixin(object): address_name = '%s_name' % random_ascii(10) self.update_address_name(name, address_name) self.validate_add(name, address="%s@%s" % (address_name, address_domain)) - + def test_delete(self): name = '%s_list' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -158,11 +160,12 @@ class ListMixin(object): self.validate_delete(name) +@unittest.skipUnless(TEST_REST_API, "REST API tests") class RESTListMixin(ListMixin): def setUp(self): super(RESTListMixin, self).setUp() self.rest_login() - + @save_response_on_error def add(self, name, password, admin_email, address_name=None, address_domain=None): extra = {} @@ -172,22 +175,22 @@ class RESTListMixin(ListMixin): 'address_domain': self.rest.domains.retrieve(name=address_domain.name).get(), }) self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra) - + @save_response_on_error def delete(self, name): self.rest.lists.retrieve(name=name).delete() - + @save_response_on_error def change_password(self, name, password): mail_list = self.rest.lists.retrieve(name=name).get() mail_list.set_password(password) - + @save_response_on_error def update_domain(self, name, domain_name): mail_list = self.rest.lists.retrieve(name=name).get() domain = self.rest.domains.retrieve(name=domain_name).get() mail_list.update(address_domain=domain) - + @save_response_on_error def update_address_name(self, name, address_name): mail_list = self.rest.lists.retrieve(name=name).get() @@ -198,70 +201,70 @@ class AdminListMixin(ListMixin): def setUp(self): super(AdminListMixin, self).setUp() self.admin_login() - + @snapshot_on_error def add(self, name, password, admin_email, address_name=None, address_domain=None): url = self.live_server_url + reverse('admin:lists_list_add') self.selenium.get(url) - + name_field = self.selenium.find_element_by_id('id_name') name_field.send_keys(name) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) - + admin_email_field = self.selenium.find_element_by_id('id_admin_email') admin_email_field.send_keys(admin_email) - + if address_name: address_name_field = self.selenium.find_element_by_id('id_address_name') address_name_field.send_keys(address_name) - + domain = Domain.objects.get(name=address_domain) domain_input = self.selenium.find_element_by_id('id_address_domain') domain_select = Select(domain_input) domain_select.select_by_value(str(domain.pk)) - + name_field.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete(self, name): mail_list = List.objects.get(name=name) self.admin_delete(mail_list) - + @snapshot_on_error def change_password(self, name, password): mail_list = List.objects.get(name=name) self.admin_change_password(mail_list, password) - + @snapshot_on_error def update_domain(self, name, domain_name): mail_list = List.objects.get(name=name) url = self.live_server_url + change_url(mail_list) self.selenium.get(url) - + domain = Domain.objects.get(name=domain_name) domain_input = self.selenium.find_element_by_id('id_address_domain') domain_select = Select(domain_input) domain_select.select_by_value(str(domain.pk)) - + save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def update_address_name(self, name, address_name): mail_list = List.objects.get(name=name) url = self.live_server_url + change_url(mail_list) self.selenium.get(url) - + address_name_field = self.selenium.find_element_by_id('id_address_name') address_name_field.clear() address_name_field.send_keys(address_name) - + save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) diff --git a/orchestra/contrib/mailboxes/admin.py b/orchestra/contrib/mailboxes/admin.py index d6965655..0336c052 100644 --- a/orchestra/contrib/mailboxes/admin.py +++ b/orchestra/contrib/mailboxes/admin.py @@ -3,7 +3,7 @@ from urllib.parse import parse_qs from django import forms from django.contrib import admin, messages -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import F, Count, Value as V from django.db.models.functions import Concat from django.utils.safestring import mark_safe @@ -28,7 +28,7 @@ from .widgets import OpenCustomFilteringOnSelect class AutoresponseInline(admin.StackedInline): model = Autoresponse verbose_name_plural = _("autoresponse") - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'subject': kwargs['widget'] = forms.TextInput(attrs={'size':'118'}) @@ -76,12 +76,12 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo form = MailboxChangeForm list_prefetch_related = ('addresses__domain',) actions = (disable, enable, list_accounts) - + def __init__(self, *args, **kwargs): super(MailboxAdmin, self).__init__(*args, **kwargs) if settings.MAILBOXES_LOCAL_DOMAIN: type(self).actions = self.actions + (SendMailboxEmail(),) - + def display_addresses(self, mailbox): # Get from forwards cache = caches.get_request_cache() @@ -111,7 +111,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo return '
'.join(addresses+forwards) display_addresses.short_description = _("Addresses") display_addresses.allow_tags = True - + def display_forwards(self, mailbox): forwards = [] for addr in mailbox.get_forwards(): @@ -120,19 +120,19 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo return '
'.join(forwards) display_forwards.short_description = _("Forward from") display_forwards.allow_tags = True - + def display_filtering(self, mailbox): """ becacuse of allow_tags = True """ return mailbox.get_filtering_display() display_filtering.short_description = _("Filtering") display_filtering.admin_order_field = 'filtering' display_filtering.allow_tags = True - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'filtering': kwargs['widget'] = OpenCustomFilteringOnSelect() return super(MailboxAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def get_fieldsets(self, request, obj=None): fieldsets = super(MailboxAdmin, self).get_fieldsets(request, obj) if obj and obj.filtering == obj.CUSTOM: @@ -144,31 +144,31 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo fieldsets = list(copy.deepcopy(fieldsets)) fieldsets.pop(-1) return fieldsets - + def get_form(self, *args, **kwargs): form = super(MailboxAdmin, self).get_form(*args, **kwargs) form.modeladmin = self return form - + def get_search_results(self, request, queryset, search_term): # Remove local domain from the search term if present (implicit local addreç) search_term = search_term.replace('@'+settings.MAILBOXES_LOCAL_DOMAIN, '') # Split address name from domain in order to support address searching search_term = search_term.replace('@', ' ') return super(MailboxAdmin, self).get_search_results(request, queryset, search_term) - + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): if not add: self.check_unrelated_address(request, obj) self.check_matching_address(request, obj) return super(MailboxAdmin, self).render_change_form( request, context, add, change, form_url, obj) - + def log_addition(self, request, object, *args, **kwargs): self.check_unrelated_address(request, object) self.check_matching_address(request, object) return super(MailboxAdmin, self).log_addition(request, object, *args, **kwargs) - + def check_matching_address(self, request, obj): local_domain = settings.MAILBOXES_LOCAL_DOMAIN if obj.name and local_domain: @@ -183,7 +183,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo "selecting it makes sense.") % (obj, addr) if msg not in (m.message for m in messages.get_messages(request)): self.message_user(request, msg, level=messages.WARNING) - + def check_unrelated_address(self, request, obj): # Check if there exists an unrelated local Address for this mbox local_domain = settings.MAILBOXES_LOCAL_DOMAIN @@ -204,7 +204,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo # Prevent duplication (add_view+continue) if msg not in (m.message for m in messages.get_messages(request)): self.message_user(request, msg, level=messages.WARNING) - + def save_model(self, request, obj, form, change): """ save hacky mailbox.addresses and local domain clashing """ if obj.filtering != obj.CUSTOM: @@ -237,20 +237,20 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): filter_horizontal = ['mailboxes'] form = AddressForm list_prefetch_related = ('mailboxes', 'domain') - + domain_link = admin_link('domain', order='domain__name') - + def display_email(self, address): return address.computed_email display_email.short_description = _("Email") display_email.admin_order_field = 'computed_email' - + def email_link(self, address): link = self.domain_link(address) return "%s@%s" % (address.name, link) email_link.short_description = _("Email") email_link.allow_tags = True - + def display_mailboxes(self, address): boxes = [] for mailbox in address.mailboxes.all(): @@ -260,7 +260,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): display_mailboxes.short_description = _("Mailboxes") display_mailboxes.allow_tags = True display_mailboxes.admin_order_field = 'mailboxes__count' - + def display_all_mailboxes(self, address): boxes = [] for mailbox in address.get_mailboxes(): @@ -269,7 +269,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): return '
'.join(boxes) display_all_mailboxes.short_description = _("Mailboxes links") display_all_mailboxes.allow_tags = True - + def display_forward(self, address): forward_mailboxes = {m.name: m for m in address.get_forward_mailboxes()} values = [] @@ -283,12 +283,12 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): display_forward.short_description = _("Forward") display_forward.allow_tags = True display_forward.admin_order_field = 'forward' - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'forward': kwargs['widget'] = forms.TextInput(attrs={'size':'118'}) return super(AddressAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def get_fields(self, request, obj=None): """ Remove mailboxes field when creating address from a popup i.e. from mailbox add form """ fields = super(AddressAdmin, self).get_fields(request, obj) @@ -297,22 +297,22 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): fields = list(fields) fields.remove('mailboxes') return fields - + def get_queryset(self, request): qs = super(AddressAdmin, self).get_queryset(request) qs = qs.annotate(computed_email=Concat(F('name'), V('@'), F('domain__name'))) return qs.annotate(Count('mailboxes')) - + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): if not add: self.check_matching_mailbox(request, obj) return super(AddressAdmin, self).render_change_form( request, context, add, change, form_url, obj) - + def log_addition(self, request, object, *args, **kwargs): self.check_matching_mailbox(request, object) return super(AddressAdmin, self).log_addition(request, object, *args, **kwargs) - + def check_matching_mailbox(self, request, obj): # Check if new addresse matches with a mbox because of having a local domain if obj.name and obj.domain and obj.domain.name == settings.MAILBOXES_LOCAL_DOMAIN: diff --git a/orchestra/contrib/mailboxes/migrations/0001_initial.py b/orchestra/contrib/mailboxes/migrations/0001_initial.py index 93dcc5fc..c2434c15 100644 --- a/orchestra/contrib/mailboxes/migrations/0001_initial.py +++ b/orchestra/contrib/mailboxes/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings +import django.db.models.deletion import orchestra.contrib.mailboxes.validators import django.core.validators @@ -21,8 +22,8 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(verbose_name='name', validators=[orchestra.contrib.mailboxes.validators.validate_emailname], blank=True, help_text='Address name, left blank for a catch-all address', max_length=64)), ('forward', models.CharField(verbose_name='forward', validators=[orchestra.contrib.mailboxes.validators.validate_forward], blank=True, help_text='Space separated email addresses or mailboxes', max_length=256)), - ('account', models.ForeignKey(to=settings.AUTH_USER_MODEL, related_name='addresses', verbose_name='Account')), - ('domain', models.ForeignKey(to='domains.Domain', related_name='addresses', verbose_name='domain')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, related_name='addresses', verbose_name='Account')), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='domains.Domain', related_name='addresses', verbose_name='domain')), ], options={ 'verbose_name_plural': 'addresses', @@ -35,7 +36,7 @@ class Migration(migrations.Migration): ('subject', models.CharField(verbose_name='subject', max_length=256)), ('message', models.TextField(verbose_name='message')), ('enabled', models.BooleanField(verbose_name='enabled', default=False)), - ('address', models.OneToOneField(to='mailboxes.Address', related_name='autoresponse', verbose_name='address')), + ('address', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='mailboxes.Address', related_name='autoresponse', verbose_name='address')), ], ), migrations.CreateModel( @@ -47,7 +48,7 @@ class Migration(migrations.Migration): ('filtering', models.CharField(choices=[('CUSTOM', 'Custom filtering'), ('REDIRECT', 'Archive spam (X-Spam-Score≥9)'), ('DISABLE', 'Disable'), ('REJECT', 'Reject spam (X-Spam-Score≥9)')], max_length=16, default='REDIRECT')), ('custom_filtering', models.TextField(verbose_name='filtering', validators=[orchestra.contrib.mailboxes.validators.validate_sieve], blank=True, help_text='Arbitrary email filtering in sieve language. This overrides any automatic junk email filtering')), ('is_active', models.BooleanField(verbose_name='active', default=True)), - ('account', models.ForeignKey(to=settings.AUTH_USER_MODEL, related_name='mailboxes', verbose_name='account')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, related_name='mailboxes', verbose_name='account')), ], options={ 'verbose_name_plural': 'mailboxes', diff --git a/orchestra/contrib/mailboxes/migrations/0001_squashed_0003_auto_20170528_2011.py b/orchestra/contrib/mailboxes/migrations/0001_squashed_0003_auto_20170528_2011.py new file mode 100644 index 00000000..c66017bf --- /dev/null +++ b/orchestra/contrib/mailboxes/migrations/0001_squashed_0003_auto_20170528_2011.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import orchestra.contrib.mailboxes.validators + + +class Migration(migrations.Migration): + + replaces = [('mailboxes', '0001_initial'), ('mailboxes', '0002_auto_20160219_1032'), ('mailboxes', '0003_auto_20170528_2011')] + + initial = True + + dependencies = [ + ('domains', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Address', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, help_text='Address name, left blank for a catch-all address', max_length=64, validators=[orchestra.contrib.mailboxes.validators.validate_emailname], verbose_name='name')), + ('forward', models.CharField(blank=True, help_text='Space separated email addresses or mailboxes', max_length=256, validators=[orchestra.contrib.mailboxes.validators.validate_forward], verbose_name='forward')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='addresses', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='addresses', to='domains.Domain', verbose_name='domain')), + ], + options={ + 'verbose_name_plural': 'addresses', + }, + ), + migrations.CreateModel( + name='Autoresponse', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('subject', models.CharField(max_length=256, verbose_name='subject')), + ('message', models.TextField(verbose_name='message')), + ('enabled', models.BooleanField(default=False, verbose_name='enabled')), + ('address', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='autoresponse', to='mailboxes.Address', verbose_name='address')), + ], + ), + migrations.CreateModel( + name='Mailbox', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(db_index=True, help_text='Required. 32 characters or fewer. Letters, digits and ./-/_ only.', max_length=32, unique=True, validators=[django.core.validators.RegexValidator('^[\\w.-]+$', 'Enter a valid mailbox name.')], verbose_name='name')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('filtering', models.CharField(choices=[('CUSTOM', 'Custom filtering'), ('DISABLE', 'Disable'), ('REDIRECT', 'Archive spam (Score≥8)'), ('REDIRECT5', 'Archive spam (Score≥5)'), ('REJECT', 'Reject spam (Score≥8)'), ('REJECT5', 'Reject spam (Score≥5)')], default='REDIRECT', max_length=16)), + ('custom_filtering', models.TextField(blank=True, help_text="Arbitrary email filtering in sieve language. This overrides any automatic junk email filtering", validators=[orchestra.contrib.mailboxes.validators.validate_sieve], verbose_name='filtering')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailboxes', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + options={ + 'verbose_name_plural': 'mailboxes', + }, + ), + migrations.AddField( + model_name='address', + name='mailboxes', + field=models.ManyToManyField(blank=True, related_name='addresses', to='mailboxes.Mailbox', verbose_name='mailboxes'), + ), + migrations.AlterUniqueTogether( + name='address', + unique_together=set([('name', 'domain')]), + ), + ] diff --git a/orchestra/contrib/mailboxes/models.py b/orchestra/contrib/mailboxes/models.py index 265519c6..8581d0ec 100644 --- a/orchestra/contrib/mailboxes/models.py +++ b/orchestra/contrib/mailboxes/models.py @@ -13,7 +13,7 @@ from . import validators, settings class Mailbox(models.Model): CUSTOM = 'CUSTOM' - + name = models.CharField(_("name"), unique=True, db_index=True, max_length=settings.MAILBOXES_NAME_MAX_LENGTH, help_text=_("Required. %s characters or fewer. Letters, digits and ./-/_ only.") % @@ -23,7 +23,7 @@ class Mailbox(models.Model): ]) password = models.CharField(_("password"), max_length=128) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='mailboxes') + related_name='mailboxes', on_delete=models.CASCADE) filtering = models.CharField(max_length=16, default=settings.MAILBOXES_MAILBOX_DEFAULT_FILTERING, choices=[(k, v[0]) for k,v in sorted(settings.MAILBOXES_MAILBOX_FILTERINGS.items())]) @@ -33,59 +33,59 @@ class Mailbox(models.Model): "sieve language. " "This overrides any automatic junk email filtering")) is_active = models.BooleanField(_("active"), default=True) - + class Meta: verbose_name_plural = _("mailboxes") - + def __str__(self): return self.name - + @cached_property def active(self): try: return self.is_active and self.account.is_active - except type(self).account.field.rel.to.DoesNotExist: + except type(self).account.field.model.DoesNotExist: return self.is_active - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def set_password(self, raw_password): self.password = make_password(raw_password) - + def get_home(self): context = { 'name': self.name, 'username': self.name, } return os.path.normpath(settings.MAILBOXES_HOME % context) - + def clean(self): if self.filtering == self.CUSTOM and not self.custom_filtering: raise ValidationError({ 'custom_filtering': _("Custom filtering is selected but not provided.") }) - + def get_filtering(self): name, content = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering] if callable(content): # Custom filtering content = content(self) return (name, content) - + def get_local_address(self): if not settings.MAILBOXES_LOCAL_DOMAIN: raise AttributeError("Mailboxes do not have a defined local address domain.") return '@'.join((self.name, settings.MAILBOXES_LOCAL_DOMAIN)) - + def get_forwards(self): return Address.objects.filter(forward__regex=r'(^|.*\s)%s(\s.*|$)' % self.name) - + def get_addresses(self): mboxes = self.addresses.all() forwards = self.get_forwards() @@ -97,33 +97,33 @@ class Address(models.Model): validators=[validators.validate_emailname], help_text=_("Address name, left blank for a catch-all address")) domain = models.ForeignKey(settings.MAILBOXES_DOMAIN_MODEL, - verbose_name=_("domain"), related_name='addresses') + verbose_name=_("domain"), related_name='addresses', on_delete=models.CASCADE) mailboxes = models.ManyToManyField(Mailbox, verbose_name=_("mailboxes"), related_name='addresses', blank=True) forward = models.CharField(_("forward"), max_length=256, blank=True, validators=[validators.validate_forward], help_text=_("Space separated email addresses or mailboxes")) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='addresses') - + related_name='addresses', on_delete=models.CASCADE) + class Meta: verbose_name_plural = _("addresses") unique_together = ('name', 'domain') - + def __str__(self): return self.email - + @property def email(self): return "%s@%s" % (self.name, self.domain) - + @cached_property def destination(self): destinations = list(self.mailboxes.values_list('name', flat=True)) if self.forward: destinations += self.forward.split() return ' '.join(destinations) - + def clean(self): errors = defaultdict(list) local_domain = settings.MAILBOXES_LOCAL_DOMAIN @@ -149,7 +149,7 @@ class Address(models.Model): ) if errors: raise ValidationError(errors) - + def get_forward_mailboxes(self): rm_local_domain = re.compile(r'@%s$' % settings.MAILBOXES_LOCAL_DOMAIN) mailboxes = [] @@ -158,7 +158,7 @@ class Address(models.Model): if '@' not in forward: mailboxes.append(forward) return Mailbox.objects.filter(name__in=mailboxes) - + def get_mailboxes(self): for mailbox in self.mailboxes.all(): yield mailbox @@ -168,11 +168,11 @@ class Address(models.Model): class Autoresponse(models.Model): address = models.OneToOneField(Address, verbose_name=_("address"), - related_name='autoresponse') + related_name='autoresponse', on_delete=models.CASCADE) # TODO initial_date subject = models.CharField(_("subject"), max_length=256) message = models.TextField(_("message")) enabled = models.BooleanField(_("enabled"), default=False) - + def __str__(self): return self.address diff --git a/orchestra/contrib/mailboxes/serializers.py b/orchestra/contrib/mailboxes/serializers.py index 18bc09ce..264afac8 100644 --- a/orchestra/contrib/mailboxes/serializers.py +++ b/orchestra/contrib/mailboxes/serializers.py @@ -8,17 +8,17 @@ from .models import Mailbox, Address class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): class Meta: - model = Address.domain.field.rel.to + model = Address.domain.field.model fields = ('url', 'id', 'name') class RelatedAddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): domain = RelatedDomainSerializer() - + class Meta: model = Address fields = ('url', 'id', 'name', 'domain', 'forward') -# +# # def from_native(self, data, files=None): # queryset = self.opts.model.objects.filter(account=self.account) # return get_object_or_404(queryset, name=data['name']) @@ -26,7 +26,7 @@ class RelatedAddressSerializer(AccountSerializerMixin, serializers.HyperlinkedMo class MailboxSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): addresses = RelatedAddressSerializer(many=True, read_only=True) - + class Meta: model = Mailbox fields = ( @@ -44,11 +44,11 @@ class RelatedMailboxSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSe class AddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): domain = RelatedDomainSerializer() mailboxes = RelatedMailboxSerializer(many=True, required=False) #allow_add_remove=True - + class Meta: model = Address fields = ('url', 'id', 'name', 'domain', 'mailboxes', 'forward') - + def validate(self, attrs): attrs = super(AddressSerializer, self).validate(attrs) if not attrs['mailboxes'] and not attrs['forward']: diff --git a/orchestra/contrib/mailboxes/signals.py b/orchestra/contrib/mailboxes/signals.py index 8bb45533..5cfdcef0 100644 --- a/orchestra/contrib/mailboxes/signals.py +++ b/orchestra/contrib/mailboxes/signals.py @@ -27,7 +27,7 @@ def create_local_address(sender, *args, **kwargs): mbox = kwargs['instance'] local_domain = settings.MAILBOXES_LOCAL_DOMAIN if not mbox.pk and local_domain: - Domain = Address._meta.get_field('domain').rel.to + Domain = Address._meta.get_field('domain').remote_field.model try: domain = Domain.objects.get(name=local_domain) except Domain.DoesNotExist: diff --git a/orchestra/contrib/mailboxes/tests/functional_tests/tests.py b/orchestra/contrib/mailboxes/tests/functional_tests/tests.py index fb37208f..6ae693bb 100644 --- a/orchestra/contrib/mailboxes/tests/functional_tests/tests.py +++ b/orchestra/contrib/mailboxes/tests/functional_tests/tests.py @@ -4,13 +4,14 @@ import poplib import smtplib import time import textwrap +import unittest from email.mime.text import MIMEText from django.apps import apps from django.conf import settings as djsettings from django.contrib.contenttypes.models import ContentType from django.core.management.base import CommandError -from django.core.urlresolvers import reverse +from django.urls import reverse from selenium.webdriver.support.select import Select from orchestra.contrib.orchestration.models import Server, Route @@ -21,6 +22,8 @@ from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot from ... import backends, settings from ...models import Mailbox +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + class MailboxMixin(object): MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') @@ -29,21 +32,21 @@ class MailboxMixin(object): 'orchestra.contrib.mails', 'orchestra.contrib.resources', ) - + def setUp(self): super(MailboxMixin, self).setUp() self.add_route() # clean resource relation from other tests apps.get_app_config('resources').reload_relations() djsettings.DEBUG = True - + def add_route(self): server = Server.objects.create(name=self.MASTER_SERVER) - backend = backends.PasswdVirtualUserBackend.get_name() + backend = backends.RoundcubeIdentityController.get_name() Route.objects.create(backend=backend, match=True, host=server) backend = backends.PostfixAddressController.get_name() Route.objects.create(backend=backend, match=True, host=server) - + def add_quota_resource(self): Resource.objects.create( name='disk', @@ -55,38 +58,38 @@ class MailboxMixin(object): on_demand=False, default_allocation=2000 ) - + def save(self): raise NotImplementedError - + def add(self): raise NotImplementedError - + def delete(self): raise NotImplementedError - + def update(self): raise NotImplementedError - + def disable(self): raise NotImplementedError - + def add_group(self, username, groupname): raise NotImplementedError - + def login_imap(self, username, password): mail = imaplib.IMAP4_SSL(self.MASTER_SERVER) status, msg = mail.login(username, password) self.assertEqual('OK', status) self.assertEqual(['Logged in'], msg) return mail - + def login_pop3(self, username, password): pop = poplib.POP3(self.MASTER_SERVER) pop.user(username) pop.pass_(password) return pop - + def send_email(self, to, token): msg = MIMEText(token) msg['To'] = to @@ -100,14 +103,14 @@ class MailboxMixin(object): server.sendmail(msg['From'], msg['To'], msg.as_string()) finally: server.quit() - + def validate_mailbox(self, username): sshrun(self.MASTER_SERVER, "doveadm search -u %s ALL" % username, display=False) - + def validate_email(self, username, token): home = Mailbox.objects.get(name=username).get_home() sshrun(self.MASTER_SERVER, "grep '%s' %s/Maildir/new/*" % (token, home), display=False) - + def test_add(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -115,7 +118,7 @@ class MailboxMixin(object): self.addCleanup(self.delete, username) imap = self.login_imap(username, password) self.validate_mailbox(username) - + def test_change_password(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -125,7 +128,7 @@ class MailboxMixin(object): new_password = '@!?%spppP001' % random_ascii(5) self.change_password(username, new_password) imap = self.login_imap(username, new_password) - + def test_quota(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -139,7 +142,7 @@ class MailboxMixin(object): imap = self.login_imap(username, password) imap_quota = int(imap.getquotaroot("INBOX")[1][1][0].split(' ')[-1].split(')')[0]) self.assertEqual(quota*1024, imap_quota) - + def test_send_email(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -155,7 +158,7 @@ class MailboxMixin(object): server.sendmail(msg['From'], msg['To'], msg.as_string()) finally: server.quit() - + def test_address(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -168,7 +171,7 @@ class MailboxMixin(object): token = random_ascii(100) self.send_email("%s@%s" % (name, domain), token) self.validate_email(username, token) - + def test_disable(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -178,7 +181,7 @@ class MailboxMixin(object): imap = self.login_imap(username, password) self.disable(username) self.assertRaises(imap.error, self.login_imap, username, password) - + def test_delete(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%sppppP001' % random_ascii(5) @@ -193,7 +196,7 @@ class MailboxMixin(object): self.assertRaises(imap.error, self.login_imap, username, password) self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, 'ls %s' % home, display=False) - + def test_delete_address(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -209,14 +212,14 @@ class MailboxMixin(object): self.delete_address(username) self.send_email("%s@%s" % (name, domain), token) self.validate_email(username, token) - + def test_custom_filtering(self): username = '%s_mailbox' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) folder = random_ascii(5) filtering = textwrap.dedent(""" require "fileinto"; - if true { + if true { fileinto "%s"; stop; }""" % folder) @@ -235,11 +238,12 @@ class MailboxMixin(object): # TODO test autoreply +@unittest.skipUnless(TEST_REST_API, "REST API tests") class RESTMailboxMixin(MailboxMixin): def setUp(self): super(RESTMailboxMixin, self).setUp() self.rest_login() - + @save_response_on_error def add(self, username, password, quota=None, filtering=None): extra = {} @@ -258,28 +262,28 @@ class RESTMailboxMixin(MailboxMixin): 'custom_filtering': filtering, }) self.rest.mailboxes.create(name=username, password=password, **extra) - + @save_response_on_error def delete(self, username): mailbox = self.rest.mailboxes.retrieve(name=username).get() mailbox.delete() - + @save_response_on_error def change_password(self, username, password): mailbox = self.rest.mailboxes.retrieve(name=username).get() mailbox.change_password(password) - + @save_response_on_error def add_address(self, username, name, domain): mailbox = self.rest.mailboxes.retrieve(name=username).get() domain = self.rest.domains.retrieve(name=domain.name).get() self.rest.addresses.create(name=name, domain=domain, mailboxes=[mailbox]) - + @save_response_on_error def delete_address(self, username): mailbox = self.rest.mailboxes.retrieve(name=username).get() self.rest.addresses.delete() - + @save_response_on_error def disable(self, username): mailbox = self.rest.mailboxes.retrieve(name=username).get() @@ -290,30 +294,30 @@ class AdminMailboxMixin(MailboxMixin): def setUp(self): super(AdminMailboxMixin, self).setUp() self.admin_login() - + @snapshot_on_error def add(self, username, password, quota=None, filtering=None): url = self.live_server_url + reverse('admin:mailboxes_mailbox_add') self.selenium.get(url) - + # account_input = self.selenium.find_element_by_id('id_account') # account_select = Select(account_input) # account_select.select_by_value(str(self.account.pk)) - + name_field = self.selenium.find_element_by_id('id_name') name_field.send_keys(username) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) - + if quota is not None: quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated' quota_field = self.selenium.find_element_by_id(quota_id) quota_field.clear() quota_field.send_keys(quota) - + if filtering is not None: filtering_input = self.selenium.find_element_by_id('id_filtering') filtering_select = Select(filtering_input) @@ -323,45 +327,45 @@ class AdminMailboxMixin(MailboxMixin): time.sleep(0.5) filtering_field = self.selenium.find_element_by_id('id_custom_filtering') filtering_field.send_keys(filtering) - + name_field.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete(self, username): mailbox = Mailbox.objects.get(name=username) self.admin_delete(mailbox) - + @snapshot_on_error def change_password(self, username, password): mailbox = Mailbox.objects.get(name=username) self.admin_change_password(mailbox, password) - + @snapshot_on_error def add_address(self, username, name, domain): url = self.live_server_url + reverse('admin:mailboxes_address_add') self.selenium.get(url) - + name_field = self.selenium.find_element_by_id('id_name') name_field.send_keys(name) - + domain_input = self.selenium.find_element_by_id('id_domain') domain_select = Select(domain_input) domain_select.select_by_value(str(domain.pk)) - + mailboxes = self.selenium.find_element_by_id('id_mailboxes_add_all_link') mailboxes.click() time.sleep(0.5) name_field.submit() - + self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete_address(self, username): mailbox = Mailbox.objects.get(name=username) address = mailbox.addresses.get() self.admin_delete(address) - + @snapshot_on_error def disable(self, username): mailbox = Mailbox.objects.get(name=username) diff --git a/orchestra/contrib/mailer/actions.py b/orchestra/contrib/mailer/actions.py index d4a50f58..1ba1b90a 100644 --- a/orchestra/contrib/mailer/actions.py +++ b/orchestra/contrib/mailer/actions.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import reverse +from django.urls import reverse from django.shortcuts import redirect diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index b979c9aa..1eaabad0 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -3,7 +3,7 @@ import email from django import forms from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import Count from django.shortcuts import redirect from django.utils.translation import ugettext_lazy as _ @@ -52,11 +52,11 @@ class MessageAdmin(ExtendedModelAdmin): ) date_hierarchy = 'created_at' change_view_actions = (last,) - + colored_state = admin_colored('state', colors=COLORS) created_at_delta = admin_date('created_at') last_try_delta = admin_date('last_try') - + def display_subject(self, instance): subject = instance.subject if len(subject) > 64: @@ -65,7 +65,7 @@ class MessageAdmin(ExtendedModelAdmin): display_subject.short_description = _("Subject") display_subject.admin_order_field = 'subject' display_subject.allow_tags = True - + def display_retries(self, instance): num_logs = instance.logs__count if num_logs == 1: @@ -78,7 +78,7 @@ class MessageAdmin(ExtendedModelAdmin): display_retries.short_description = _("Retries") display_retries.admin_order_field = 'retries' display_retries.allow_tags = True - + def display_content(self, instance): part = email.message_from_string(instance.content) payload = part.get_payload() @@ -102,19 +102,19 @@ class MessageAdmin(ExtendedModelAdmin): return payload display_content.short_description = _("Content") display_content.allow_tags = True - + def display_full_subject(self, instance): return instance.subject display_full_subject.short_description = _("Subject") - + def display_from(self, instance): return instance.from_address display_from.short_description = _("From") - + def display_to(self, instance): return instance.to_address display_to.short_description = _("To") - + def get_urls(self): from django.conf.urls import url urls = super().get_urls() @@ -125,16 +125,16 @@ class MessageAdmin(ExtendedModelAdmin): name='%s_%s_send_pending' % info) ) return urls - + def get_queryset(self, request): qs = super().get_queryset(request) return qs.annotate(Count('logs')).defer('content') - + def send_pending_view(self, request): task(send_pending).apply_async() self.message_user(request, _("Pending messages are being sent on the background.")) return redirect('..') - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'subject': kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) @@ -148,7 +148,7 @@ class SMTPLogAdmin(admin.ModelAdmin): list_filter = ('result',) fields = ('message_link', 'colored_result', 'date_delta', 'log_message') readonly_fields = fields - + message_link = admin_link('message') colored_result = admin_colored('result', colors=COLORS, bold=False) date_delta = admin_date('date') diff --git a/orchestra/contrib/mailer/migrations/0001_initial.py b/orchestra/contrib/mailer/migrations/0001_initial.py index 4b19ca88..ca6dcd3b 100644 --- a/orchestra/contrib/mailer/migrations/0001_initial.py +++ b/orchestra/contrib/mailer/migrations/0001_initial.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import django.db.models.deletion from django.db import models, migrations @@ -32,7 +33,7 @@ class Migration(migrations.Migration): ('result', models.CharField(choices=[('SUCCESS', 'Success'), ('FAILURE', 'Failure')], default='SUCCESS', max_length=16)), ('date', models.DateTimeField(auto_now_add=True)), ('log_message', models.TextField()), - ('message', models.ForeignKey(to='mailer.Message', editable=False, related_name='logs')), + ('message', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mailer.Message', editable=False, related_name='logs')), ], ), ] diff --git a/orchestra/contrib/mailer/migrations/0001_squashed_0005_auto_20160219_1056.py b/orchestra/contrib/mailer/migrations/0001_squashed_0005_auto_20160219_1056.py new file mode 100644 index 00000000..576ec469 --- /dev/null +++ b/orchestra/contrib/mailer/migrations/0001_squashed_0005_auto_20160219_1056.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:28 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('mailer', '0001_initial'), ('mailer', '0002_auto_20150617_1021'), ('mailer', '0003_auto_20150617_1024'), ('mailer', '0004_auto_20150805_1328'), ('mailer', '0005_auto_20160219_1056')] + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Message', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('state', models.CharField(choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failes')], default='QUEUED', max_length=16, verbose_name='State')), + ('priority', models.PositiveIntegerField(choices=[(0, 'Critical (not queued)'), (1, 'High'), (2, 'Normal'), (3, 'Low')], default=2, verbose_name='Priority')), + ('to_address', models.CharField(max_length=256)), + ('from_address', models.CharField(max_length=256)), + ('subject', models.CharField(max_length=256, verbose_name='subject')), + ('content', models.TextField(verbose_name='content')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), + ('retries', models.PositiveIntegerField(default=0, verbose_name='retries')), + ('last_retry', models.DateTimeField(auto_now=True, verbose_name='last try')), + ], + ), + migrations.CreateModel( + name='SMTPLog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('result', models.CharField(choices=[('SUCCESS', 'Success'), ('FAILURE', 'Failure')], default='SUCCESS', max_length=16)), + ('date', models.DateTimeField(auto_now_add=True)), + ('log_message', models.TextField()), + ('message', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='mailer.Message')), + ], + ), + migrations.RenameField( + model_name='message', + old_name='last_retry', + new_name='last_try', + ), + migrations.AlterField( + model_name='message', + name='last_try', + field=models.DateTimeField(verbose_name='last try'), + ), + migrations.AlterField( + model_name='message', + name='subject', + field=models.TextField(verbose_name='subject'), + ), + migrations.AlterField( + model_name='message', + name='last_try', + field=models.DateTimeField(null=True, verbose_name='last try'), + ), + migrations.AlterField( + model_name='message', + name='state', + field=models.CharField(choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], default='QUEUED', max_length=16, verbose_name='State'), + ), + migrations.AlterField( + model_name='message', + name='last_try', + field=models.DateTimeField(db_index=True, null=True, verbose_name='last try'), + ), + migrations.AlterField( + model_name='message', + name='priority', + field=models.PositiveIntegerField(choices=[(0, 'Critical (not queued)'), (1, 'High'), (2, 'Normal'), (3, 'Low')], db_index=True, default=2, verbose_name='Priority'), + ), + migrations.AlterField( + model_name='message', + name='retries', + field=models.PositiveIntegerField(db_index=True, default=0, verbose_name='retries'), + ), + migrations.AlterField( + model_name='message', + name='state', + field=models.CharField(choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], db_index=True, default='QUEUED', max_length=16, verbose_name='State'), + ), + ] diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py index 5b43e756..f7c61aff 100644 --- a/orchestra/contrib/mailer/models.py +++ b/orchestra/contrib/mailer/models.py @@ -15,7 +15,7 @@ class Message(models.Model): (DEFERRED, _("Deferred")), (FAILED, _("Failed")), ) - + CRITICAL = 0 HIGH = 1 NORMAL = 2 @@ -26,7 +26,7 @@ class Message(models.Model): (NORMAL, _("Normal")), (LOW, _("Low")), ) - + state = models.CharField(_("State"), max_length=16, choices=STATES, default=QUEUED, db_index=True) priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL, @@ -38,21 +38,21 @@ class Message(models.Model): created_at = models.DateTimeField(_("created at"), auto_now_add=True) retries = models.PositiveIntegerField(_("retries"), default=0, db_index=True) last_try = models.DateTimeField(_("last try"), null=True, db_index=True) - + def __str__(self): return '%s to %s' % (self.subject, self.to_address) - + def defer(self): self.state = self.DEFERRED # Max tries if self.retries >= len(settings.MAILER_DEFERE_SECONDS): self.state = self.FAILED self.save(update_fields=('state',)) - + def sent(self): self.state = self.SENT self.save(update_fields=('state',)) - + def log(self, error): result = SMTPLog.SUCCESS if error: @@ -67,7 +67,7 @@ class SMTPLog(models.Model): (SUCCESS, _("Success")), (FAILURE, _("Failure")), ) - message = models.ForeignKey(Message, editable=False, related_name='logs') + message = models.ForeignKey(Message, editable=False, related_name='logs', on_delete=models.CASCADE) result = models.CharField(max_length=16, choices=RESULTS, default=SUCCESS) date = models.DateTimeField(auto_now_add=True) log_message = models.TextField() diff --git a/orchestra/contrib/miscellaneous/admin.py b/orchestra/contrib/miscellaneous/admin.py index a54b8be3..2f7d699c 100644 --- a/orchestra/contrib/miscellaneous/admin.py +++ b/orchestra/contrib/miscellaneous/admin.py @@ -1,6 +1,6 @@ from django import forms from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ @@ -36,19 +36,19 @@ class MiscServiceAdmin(ExtendedModelAdmin): prepopulated_fields = {'name': ('verbose_name',)} change_readonly_fields = ('name',) actions = (disable, enable) - + def display_name(self, misc): return '%s' % (misc.description, misc.name) display_name.short_description = _("name") display_name.allow_tags = True display_name.admin_order_field = 'name' - + def display_verbose_name(self, misc): return '%s' % (misc.description, misc.verbose_name) display_verbose_name.short_description = _("verbose name") display_verbose_name.allow_tags = True display_verbose_name.admin_order_field = 'verbose_name' - + def num_instances(self, misc): """ return num slivers as a link to slivers changelist view """ num = misc.instances__count @@ -57,11 +57,11 @@ class MiscServiceAdmin(ExtendedModelAdmin): return mark_safe('{1}'.format(url, num)) num_instances.short_description = _("Instances") num_instances.admin_order_field = 'instances__count' - + def get_queryset(self, request): qs = super(MiscServiceAdmin, self).get_queryset(request) return qs.annotate(models.Count('instances', distinct=True)) - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'description': @@ -83,21 +83,21 @@ class MiscellaneousAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedMode actions = (disable, enable) plugin_field = 'service' plugin = MiscServicePlugin - + service_link = admin_link('service') - + def dispaly_active(self, instance): return instance.active dispaly_active.short_description = _("Active") dispaly_active.boolean = True dispaly_active.admin_order_field = 'is_active' - + def get_service(self, obj): if obj is None: return self.plugin.get(self.plugin_value).related_instance else: return obj.service - + def get_fieldsets(self, request, obj=None): fieldsets = super().get_fieldsets(request, obj) fields = list(fieldsets[0][1]['fields']) @@ -110,7 +110,7 @@ class MiscellaneousAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedMode fields.insert(2, 'identifier') fieldsets[0][1]['fields'] = fields return fieldsets - + def get_form(self, request, obj=None, **kwargs): if obj: plugin = self.plugin.get(obj.service.name)() @@ -127,16 +127,16 @@ class MiscellaneousAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedMode validator = import_class(validator_path) validator(identifier) return identifier - + form.clean_identifier = clean_identifier return form - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'description': kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4}) return super(MiscellaneousAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def save_model(self, request, obj, form, change): if not change: plugin = self.plugin diff --git a/orchestra/contrib/miscellaneous/migrations/0001_initial.py b/orchestra/contrib/miscellaneous/migrations/0001_initial.py index 57f112e2..53b560e6 100644 --- a/orchestra/contrib/miscellaneous/migrations/0001_initial.py +++ b/orchestra/contrib/miscellaneous/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.core.validators from django.conf import settings import orchestra.models.fields @@ -22,7 +23,7 @@ class Migration(migrations.Migration): ('description', models.TextField(blank=True, verbose_name='description')), ('amount', models.PositiveIntegerField(default=1, verbose_name='amount')), ('is_active', models.BooleanField(default=True, help_text='Designates whether this service should be treated as active. Unselect this instead of deleting services.', verbose_name='active')), - ('account', models.ForeignKey(related_name='miscellaneous', verbose_name='account', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='miscellaneous', verbose_name='account', to=settings.AUTH_USER_MODEL)), ], options={ 'verbose_name_plural': 'miscellaneous', @@ -43,6 +44,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='miscellaneous', name='service', - field=models.ForeignKey(related_name='instances', verbose_name='service', to='miscellaneous.MiscService'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', verbose_name='service', to='miscellaneous.MiscService'), ), ] diff --git a/orchestra/contrib/miscellaneous/migrations/0001_squashed_0002_auto_20150723_1252.py b/orchestra/contrib/miscellaneous/migrations/0001_squashed_0002_auto_20150723_1252.py new file mode 100644 index 00000000..e5d5e6d0 --- /dev/null +++ b/orchestra/contrib/miscellaneous/migrations/0001_squashed_0002_auto_20150723_1252.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:28 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('miscellaneous', '0001_initial'), ('miscellaneous', '0002_auto_20150723_1252')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Miscellaneous', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('identifier', orchestra.models.fields.NullableCharField(help_text='A unique identifier for this service.', max_length=256, null=True, unique=True, verbose_name='identifier')), + ('description', models.TextField(blank=True, verbose_name='description')), + ('amount', models.PositiveIntegerField(default=1, verbose_name='amount')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this service should be treated as active. Unselect this instead of deleting services.', verbose_name='active')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='miscellaneous', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + options={ + 'verbose_name_plural': 'miscellaneous', + }, + ), + migrations.CreateModel( + name='MiscService', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Raw name used for internal referenciation, i.e. service match definition', max_length=32, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('verbose_name', models.CharField(blank=True, help_text='Human readable name', max_length=256, verbose_name='verbose name')), + ('description', models.TextField(blank=True, help_text='Optional description', verbose_name='description')), + ('has_identifier', models.BooleanField(default=True, help_text='Designates if this service has a unique text field that identifies it or not.', verbose_name='has identifier')), + ('has_amount', models.BooleanField(default=False, help_text='Designates whether this service has amount property or not.', verbose_name='has amount')), + ('is_active', models.BooleanField(default=True, help_text='Whether new instances of this service can be created or not. Unselect this instead of deleting services.', verbose_name='active')), + ], + ), + migrations.AddField( + model_name='miscellaneous', + name='service', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='miscellaneous.MiscService', verbose_name='service'), + ), + migrations.AlterField( + model_name='miscellaneous', + name='identifier', + field=orchestra.models.fields.NullableCharField(db_index=True, help_text='A unique identifier for this service.', max_length=256, null=True, unique=True, verbose_name='identifier'), + ), + ] diff --git a/orchestra/contrib/miscellaneous/models.py b/orchestra/contrib/miscellaneous/models.py index c5ab30f2..e5fe5f0f 100644 --- a/orchestra/contrib/miscellaneous/models.py +++ b/orchestra/contrib/miscellaneous/models.py @@ -22,30 +22,30 @@ class MiscService(models.Model): is_active = models.BooleanField(_("active"), default=True, help_text=_("Whether new instances of this service can be created " "or not. Unselect this instead of deleting services.")) - + def __str__(self): return self.name - + def clean(self): self.verbose_name = self.verbose_name.strip() - + def get_verbose_name(self): return self.verbose_name or self.name - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) class Miscellaneous(models.Model): - service = models.ForeignKey(MiscService, verbose_name=_("service"), - related_name='instances') - account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='miscellaneous') + service = models.ForeignKey(MiscService, on_delete=models.CASCADE, + verbose_name=_("service"), related_name='instances') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='miscellaneous') identifier = NullableCharField(_("identifier"), max_length=256, null=True, unique=True, db_index=True, help_text=_("A unique identifier for this service.")) description = models.TextField(_("description"), blank=True) @@ -53,32 +53,32 @@ class Miscellaneous(models.Model): is_active = models.BooleanField(_("active"), default=True, help_text=_("Designates whether this service should be treated as " "active. Unselect this instead of deleting services.")) - + class Meta: verbose_name_plural = _("miscellaneous") - + def __str__(self): return self.identifier or self.description[:32] or str(self.service) - + @cached_property def active(self): return self.is_active and self.service.is_active and self.account.is_active - + def get_description(self): return ' '.join((str(self.amount), self.service.description or self.service.verbose_name)) - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) - + @cached_property def service_class(self): return self.service - + def clean(self): if self.identifier: self.identifier = self.identifier.strip().lower() diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py index b5aded20..c4774481 100644 --- a/orchestra/contrib/orchestration/__init__.py +++ b/orchestra/contrib/orchestration/__init__.py @@ -15,21 +15,21 @@ class Operation(): MONITOR = 'monitor' EXCEEDED = 'exceeded' RECOVERY = 'recovery' - + def __str__(self): return '%s.%s(%s)' % (self.backend, self.action, self.instance) - + def __repr__(self): return str(self) - + def __hash__(self): """ set() """ return hash((self.backend, self.instance, self.action)) - + def __eq__(self, operation): """ set() """ return hash(self) == hash(operation) - + def __init__(self, backend, instance, action, routes=None): self.backend = backend # instance should maintain any dynamic attribute until backend execution @@ -37,13 +37,13 @@ class Operation(): self.instance = copy.deepcopy(instance) self.action = action self.routes = routes - + @classmethod - def execute(cls, operations, serialize=False, async=None): + def execute(cls, operations, serialize=False, run_async=None): from . import manager scripts, backend_serialize = manager.generate(operations) - return manager.execute(scripts, serialize=(serialize or backend_serialize), async=async) - + return manager.execute(scripts, serialize=(serialize or backend_serialize), run_async=run_async) + @classmethod def create_for_action(cls, instances, action): if not isinstance(instances, collections.Iterable): @@ -56,13 +56,13 @@ class Operation(): cls(backend_cls, instance, action) ) return operations - + @classmethod def execute_action(cls, instances, action): """ instances can be an object or an iterable for batch processing """ operations = cls.create_for_action(instances, action) return cls.execute(operations) - + def preload_context(self): """ Heuristic: Running get_context will prevent most of related objects do not exist errors @@ -70,7 +70,7 @@ class Operation(): if self.action == self.DELETE: if hasattr(self.backend, 'get_context'): self.backend().get_context(self.instance) - + def store(self, log): from .models import BackendOperation return BackendOperation.objects.create( @@ -79,7 +79,7 @@ class Operation(): instance=self.instance, action=self.action, ) - + @classmethod def load(cls, operation, log=None): routes = None @@ -88,4 +88,4 @@ class Operation(): (operation.backend, operation.action): AttrDict(host=log.server) } return cls(operation.backend_class, operation.instance, operation.action, routes=routes) - + diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index da9cfe01..60737d09 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -30,25 +30,25 @@ STATE_COLORS = { class RouteAdmin(ExtendedModelAdmin): list_display = ( - 'display_backend', 'host', 'match', 'display_model', 'display_actions', 'async', + 'display_backend', 'host', 'match', 'display_model', 'display_actions', 'run_async', 'is_active' ) - list_editable = ('host', 'match', 'async', 'is_active') - list_filter = ('host', 'is_active', 'async', 'backend') + list_editable = ('host', 'match', 'run_async', 'is_active') + list_filter = ('host', 'is_active', 'run_async', 'backend') list_prefetch_related = ('host',) ordering = ('backend',) - add_fields = ('backend', 'host', 'match', 'async', 'is_active') + add_fields = ('backend', 'host', 'match', 'run_async', 'is_active') change_form = RouteForm actions = (orchestrate,) change_view_actions = actions - + BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends()) DEFAULT_MATCH = { backend.get_name(): backend.default_route_match for backend in ServiceBackend.get_backends() } - + display_backend = display_plugin_field('backend') - + def display_model(self, route): try: return escape(route.backend_class.model) @@ -56,7 +56,7 @@ class RouteAdmin(ExtendedModelAdmin): return "NOT AVAILABLE" display_model.short_description = _("model") display_model.allow_tags = True - + def display_actions(self, route): try: return '
'.join(route.backend_class.get_actions()) @@ -64,7 +64,7 @@ class RouteAdmin(ExtendedModelAdmin): return "NOT AVAILABLE" display_actions.short_description = _("actions") display_actions.allow_tags = True - + def formfield_for_dbfield(self, db_field, **kwargs): """ Provides dynamic help text on backend form field """ if db_field.name == 'backend': @@ -79,23 +79,23 @@ class RouteAdmin(ExtendedModelAdmin): request._host_choices_cache = choices = list(field.choices) field.choices = choices return field - + def get_form(self, request, obj=None, **kwargs): """ Include dynamic help text for existing objects """ form = super(RouteAdmin, self).get_form(request, obj, **kwargs) if obj: form.base_fields['backend'].help_text = self.BACKEND_HELP_TEXT.get(obj.backend, '') return form - + def show_orchestration_disabled(self, request): if settings.ORCHESTRATION_DISABLE_EXECUTION: msg = _("Orchestration execution is disabled by ORCHESTRATION_DISABLE_EXECUTION setting.") self.message_user(request, mark_safe(msg), messages.WARNING) - + def changelist_view(self, request, extra_context=None): self.show_orchestration_disabled(request) return super(RouteAdmin, self).changelist_view(request, extra_context) - + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): self.show_orchestration_disabled(request) return super(RouteAdmin, self).changeform_view( @@ -108,12 +108,12 @@ class BackendOperationInline(admin.TabularInline): readonly_fields = ('action', 'instance_link') extra = 0 can_delete = False - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + def instance_link(self, operation): link = admin_link('instance')(self, operation) if link == '---': @@ -122,10 +122,10 @@ class BackendOperationInline(admin.TabularInline): return link instance_link.allow_tags = True instance_link.short_description = _("Instance") - + def has_add_permission(self, *args, **kwargs): return False - + def get_queryset(self, request): queryset = super(BackendOperationInline, self).get_queryset(request) return queryset.prefetch_related('instance') @@ -149,7 +149,7 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin): readonly_fields = fields actions = (retry_backend,) change_view_actions = actions - + server_link = admin_link('server') display_created = admin_date('created_at', short_description=_("Created")) display_state = admin_colored('state', colors=STATE_COLORS) @@ -157,17 +157,17 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin): mono_stdout = display_mono('stdout') mono_stderr = display_mono('stderr') mono_traceback = display_mono('traceback') - + class Media: css = { 'all': ('orchestra/css/pygments/github.css',) } - + def get_queryset(self, request): """ Order by structured name and imporve performance """ qs = super(BackendLogAdmin, self).get_queryset(request) return qs.select_related('server').defer('script', 'stdout') - + def has_add_permission(self, *args, **kwargs): return False @@ -177,17 +177,17 @@ class ServerAdmin(ExtendedModelAdmin): list_filter = ('os',) actions = (orchestrate,) change_view_actions = actions - + def display_ping(self, instance): return self._remote_state[instance.pk][0] display_ping.short_description = _("Ping") display_ping.allow_tags = True - + def display_uptime(self, instance): return self._remote_state[instance.pk][1] display_uptime.short_description = _("Uptime") display_uptime.allow_tags = True - + def get_queryset(self, request): """ Order by structured name and imporve performance """ qs = super(ServerAdmin, self).get_queryset(request) diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py index cf861102..b2f22267 100644 --- a/orchestra/contrib/orchestration/backends.py +++ b/orchestra/contrib/orchestration/backends.py @@ -31,7 +31,7 @@ class ServiceMount(plugins.PluginMount): class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): """ Service management backend base class - + It uses the _unit of work_ design principle, which allows bulk operations to be conviniently supported. Each backend generates the configuration for all the changes of all modified objects, reloading the daemon just once. @@ -52,15 +52,15 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): # By default backend will not run if actions do not generate insctructions, # If your backend uses prepare() or commit() only then you should set force_empty_action_execution = True force_empty_action_execution = False - + def __str__(self): return type(self).__name__ - + def __init__(self): self.head = [] self.content = [] self.tail = [] - + def __getattribute__(self, attr): """ Select head, content or tail section depending on the method name """ IGNORE_ATTRS = ( @@ -83,29 +83,29 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): elif attr not in IGNORE_ATTRS and attr in self.actions: self.set_content() return super(ServiceBackend, self).__getattribute__(attr) - + def set_head(self): self.cmd_section = self.head - + def set_tail(self): self.cmd_section = self.tail - + def set_content(self): self.cmd_section = self.content - + @classmethod def get_actions(cls): return [ action for action in cls.actions if action in dir(cls) ] - + @classmethod def get_name(cls): return cls.__name__ - + @classmethod def is_main(cls, obj): opts = obj._meta return cls.model == '%s.%s' % (opts.app_label, opts.object_name) - + @classmethod def get_related(cls, obj): opts = obj._meta @@ -122,7 +122,7 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): return related.all() return [related] return [] - + @classmethod def get_backends(cls, instance=None, action=None): backends = cls.get_plugins() @@ -140,15 +140,15 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): if include: included.append(backend) return included - + @classmethod def get_backend(cls, name): return cls.get(name) - + @classmethod def model_class(cls): return apps.get_model(cls.model) - + @property def scripts(self): """ group commands based on their method """ @@ -163,12 +163,12 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): except KeyError: pass return list(scripts.items()) - + def get_banner(self): now = timezone.localtime(timezone.now()) time = now.strftime("%h %d, %Y %I:%M:%S %Z") return "Generated by Orchestra at %s" % time - + def create_log(self, server, **kwargs): from .models import BackendLog state = BackendLog.RECEIVED @@ -181,8 +181,8 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): manager = manager.using(using) log = manager.create(backend=self.get_name(), state=state, server=server) return log - - def execute(self, server, async=False, log=None): + + def execute(self, server, run_async=False, log=None): from .models import BackendLog if log is None: log = self.create_log(server) @@ -190,11 +190,11 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): if run: scripts = self.scripts for method, commands in scripts: - method(log, server, commands, async) + method(log, server, commands, run_async) if log.state != BackendLog.SUCCESS: break return log - + def append(self, *cmd): # aggregate commands acording to its execution method if isinstance(cmd[0], str): @@ -207,10 +207,10 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): self.cmd_section.append((method, [cmd])) else: self.cmd_section[-1][1].append(cmd) - + def get_context(self, obj): return {} - + def prepare(self): """ hook for executing something at the beging @@ -221,7 +221,7 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): set -o pipefail exit_code=0""") ) - + def commit(self): """ hook for executing something at the end @@ -235,11 +235,11 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): class ServiceController(ServiceBackend): actions = ('save', 'delete') abstract = True - + @classmethod def get_verbose_name(cls): return _("[S] %s") % super(ServiceController, cls).get_verbose_name() - + @classmethod def get_backends(cls): """ filter controller classes """ diff --git a/orchestra/contrib/orchestration/helpers.py b/orchestra/contrib/orchestration/helpers.py index 7f0deae8..768e9997 100644 --- a/orchestra/contrib/orchestration/helpers.py +++ b/orchestra/contrib/orchestration/helpers.py @@ -2,7 +2,7 @@ import textwrap from django.contrib import messages from django.core.mail import mail_admins -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import ungettext, ugettext_lazy as _ @@ -105,7 +105,7 @@ def get_backend_url(ids): def get_messages(logs): messages = [] - total, successes, async = 0, 0, 0 + total, successes, run_async = 0, 0, 0 ids = [] async_ids = [] for log in logs: @@ -118,17 +118,17 @@ def get_messages(logs): if log.is_success: successes += 1 elif not log.has_finished: - async += 1 + run_async += 1 async_ids.append(log.id) - errors = total-successes-async + errors = total-successes-run_async url = get_backend_url(ids) async_url = get_backend_url(async_ids) async_msg = '' - if async: + if run_async: async_msg = ungettext( _('{name} is running on the background'), - _('{async} backends are running on the background'), - async) + _('{run_async} backends are running on the background'), + run_async) if errors: if total == 1: msg = _('{name} has fail to execute') @@ -139,7 +139,7 @@ def get_messages(logs): errors) if async_msg: msg += ', ' + str(async_msg) - msg = msg.format(errors=errors, async=async, async_url=async_url, total=total, url=url, + msg = msg.format(errors=errors, run_async=run_async, async_url=async_url, total=total, url=url, name=log.backend) messages.append(('error', msg + '.')) elif successes: @@ -158,12 +158,12 @@ def get_messages(logs): _('{total} backends have been executed'), total) msg = msg.format( - total=total, url=url, async_url=async_url, async=async, successes=successes, + total=total, url=url, async_url=async_url, run_async=run_async, successes=successes, name=log.backend ) messages.append(('success', msg + '.')) else: - msg = async_msg.format(url=url, async_url=async_url, async=async, name=log.backend) + msg = async_msg.format(url=url, async_url=async_url, run_async=run_async, name=log.backend) messages.append(('success', msg + '.')) return messages diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index 4b076f73..211f1b59 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -12,7 +12,7 @@ from orchestra.utils.sys import confirm class Command(BaseCommand): help = 'Runs orchestration backends.' - + def add_arguments(self, parser): parser.add_argument('model', nargs='?', help='Label of a model to execute the orchestration.') @@ -30,8 +30,8 @@ class Command(BaseCommand): help='List available baclends.') parser.add_argument('--dry-run', action='store_true', dest='dry', default=False, help='Only prints scrtipt.') - - + + def collect_operations(self, **options): model = options.get('model') backends = options.get('backends') or set() @@ -66,7 +66,7 @@ class Command(BaseCommand): model = apps.get_model(*model.split('.')) queryset = model.objects.filter(**kwargs).order_by('id') querysets = [queryset] - + operations = OrderedSet() route_cache = {} for queryset in querysets: @@ -88,7 +88,7 @@ class Command(BaseCommand): result.append(operation) operations = result return operations - + def handle(self, *args, **options): list_backends = options.get('list_backends') if list_backends: @@ -116,7 +116,7 @@ class Command(BaseCommand): if not confirm("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context): return if not dry: - logs = manager.execute(scripts, serialize=serialize, async=True) + logs = manager.execute(scripts, serialize=serialize, run_async=True) running = list(logs) stdout = 0 stderr = 0 diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index eafe8422..62572d04 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -97,12 +97,12 @@ def generate(operations): return scripts, serialize -def execute(scripts, serialize=False, async=None): +def execute(scripts, serialize=False, run_async=None): """ executes the operations on the servers - + serialize: execute one backend at a time - async: do not join threads (overrides route.async) + run_async: do not join threads (overrides route.run_async) """ if settings.ORCHESTRATION_DISABLE_EXECUTION: logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION.') @@ -115,12 +115,12 @@ def execute(scripts, serialize=False, async=None): route, __, async_action = key backend, operations = value args = (route.host,) - if async is None: - is_async = not serialize and (route.async or async_action) + if run_async is None: + is_async = not serialize and (route.run_async or async_action) else: - is_async = not serialize and (async or async_action) + is_async = not serialize and (run_async or async_action) kwargs = { - 'async': is_async, + 'run_async': is_async, } # we clone the connection just in case we are isolated inside a transaction with db.clone(model=BackendLog) as handle: diff --git a/orchestra/contrib/orchestration/methods.py b/orchestra/contrib/orchestration/methods.py index db665a0d..cd3d7a22 100644 --- a/orchestra/contrib/orchestration/methods.py +++ b/orchestra/contrib/orchestration/methods.py @@ -17,7 +17,7 @@ from . import settings logger = logging.getLogger(__name__) -def Paramiko(backend, log, server, cmds, async=False, paramiko_connections={}): +def Paramiko(backend, log, server, cmds, run_async=False, paramiko_connections={}): """ Executes cmds to remote server using Pramaiko """ @@ -55,7 +55,7 @@ def Paramiko(backend, log, server, cmds, async=False, paramiko_connections={}): channel.shutdown_write() # Log results logger.debug('%s running on %s' % (backend, server)) - if async: + if run_async: second = False while True: # Non-blocking is the secret ingridient in the async sauce @@ -78,7 +78,7 @@ def Paramiko(backend, log, server, cmds, async=False, paramiko_connections={}): else: log.stdout += channel.makefile('rb', -1).read().decode('utf-8') log.stderr += channel.makefile_stderr('rb', -1).read().decode('utf-8') - + log.exit_code = channel.recv_exit_status() log.state = log.SUCCESS if log.exit_code == 0 else log.FAILURE logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) @@ -97,7 +97,7 @@ def Paramiko(backend, log, server, cmds, async=False, paramiko_connections={}): channel.close() -def OpenSSH(backend, log, server, cmds, async=False): +def OpenSSH(backend, log, server, cmds, run_async=False): """ Executes cmds to remote server using SSH with connection resuse for maximum performance """ @@ -110,9 +110,9 @@ def OpenSSH(backend, log, server, cmds, async=False): return try: ssh = sshrun(server.get_address(), script, executable=backend.script_executable, - persist=True, async=async, silent=True) + persist=True, run_async=run_async, silent=True) logger.debug('%s running on %s' % (backend, server)) - if async: + if run_async: for state in ssh: log.stdout += state.stdout.decode('utf8') log.stderr += state.stderr.decode('utf8') @@ -148,7 +148,7 @@ def SSH(*args, **kwargs): return method(*args, **kwargs) -def Python(backend, log, server, cmds, async=False): +def Python(backend, log, server, cmds, run_async=False): script = '' functions = set() for cmd in cmds: @@ -170,7 +170,7 @@ def Python(backend, log, server, cmds, async=False): log.stdout += line + '\n' if result: log.stdout += '# Result: %s\n' % result - if async: + if run_async: log.save(update_fields=('stdout', 'updated_at')) except: log.exit_code = 1 diff --git a/orchestra/contrib/orchestration/middlewares.py b/orchestra/contrib/orchestration/middlewares.py index afab235e..2daefe19 100644 --- a/orchestra/contrib/orchestration/middlewares.py +++ b/orchestra/contrib/orchestration/middlewares.py @@ -1,7 +1,7 @@ from threading import local from django.contrib.admin.models import LogEntry -from django.core.urlresolvers import resolve +from django.urls import resolve from django.db import transaction from django.db.models.signals import pre_delete, post_save, m2m_changed from django.dispatch import receiver @@ -39,12 +39,12 @@ class OperationsMiddleware(object): """ Stores all the operations derived from save and delete signals and executes them at the end of the request/response cycle - + It also works as a transaction middleware, making requets to run within an atomic block. """ # Thread local is used because request object is not available on model signals thread_locals = local() - + @classmethod def get_pending_operations(cls): # Check if an error poped up before OperationsMiddleware.process_request() @@ -54,7 +54,7 @@ class OperationsMiddleware(object): request.pending_operations = OrderedSet() return request.pending_operations return set() - + @classmethod def get_route_cache(cls): """ chache the routes to save sql queries """ @@ -64,7 +64,7 @@ class OperationsMiddleware(object): request.route_cache = {} return request.route_cache return {} - + @classmethod def collect(cls, action, **kwargs): """ Collects all pending operations derived from model signals """ @@ -75,26 +75,26 @@ class OperationsMiddleware(object): kwargs['route_cache'] = cls.get_route_cache() instance = kwargs.pop('instance') manager.collect(instance, action, **kwargs) - + def enter_transaction_management(self): type(self).thread_locals.transaction = transaction.atomic() type(self).thread_locals.transaction.__enter__() - + def leave_transaction_management(self, exception=None): locals = type(self).thread_locals if hasattr(locals, 'transaction'): # Don't fucking know why sometimes thread_locals does not contain a transaction locals.transaction.__exit__(exception, None, None) - + def process_request(self, request): """ Store request on a thread local variable """ type(self).thread_locals.request = request self.enter_transaction_management() - + def process_exception(self, request, exception): """Rolls back the database and leaves transaction management""" self.leave_transaction_management(exception) - + def process_response(self, request, response): """ Processes pending backend operations """ if response.status_code != 500: diff --git a/orchestra/contrib/orchestration/migrations/0001_initial.py b/orchestra/contrib/orchestration/migrations/0001_initial.py index 3431fdda..e14d15d9 100644 --- a/orchestra/contrib/orchestration/migrations/0001_initial.py +++ b/orchestra/contrib/orchestration/migrations/0001_initial.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import django.db.models.deletion from django.db import models, migrations import orchestra.models.fields @@ -38,8 +39,8 @@ class Migration(migrations.Migration): ('backend', models.CharField(max_length=256, verbose_name='backend')), ('action', models.CharField(max_length=64, verbose_name='action')), ('object_id', models.PositiveIntegerField()), - ('content_type', models.ForeignKey(to='contenttypes.ContentType')), - ('log', models.ForeignKey(related_name='operations', to='orchestration.BackendLog')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('log', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='operations', to='orchestration.BackendLog')), ], options={ 'verbose_name_plural': 'Operations', @@ -68,12 +69,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='route', name='host', - field=models.ForeignKey(to='orchestration.Server', verbose_name='host'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='orchestration.Server', verbose_name='host'), ), migrations.AddField( model_name='backendlog', name='server', - field=models.ForeignKey(related_name='execution_logs', to='orchestration.Server', verbose_name='server'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='execution_logs', to='orchestration.Server', verbose_name='server'), ), migrations.AlterUniqueTogether( name='route', diff --git a/orchestra/contrib/orchestration/migrations/0001_squashed_0009_rename_route_async_run_async.py b/orchestra/contrib/orchestration/migrations/0001_squashed_0009_rename_route_async_run_async.py new file mode 100644 index 00000000..6e5d1c72 --- /dev/null +++ b/orchestra/contrib/orchestration/migrations/0001_squashed_0009_rename_route_async_run_async.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('orchestration', '0001_initial'), ('orchestration', '0002_auto_20150506_1420'), ('orchestration', '0003_auto_20150512_1512'), ('orchestration', '0004_route_async_actions'), ('orchestration', '0005_auto_20150709_1016'), ('orchestration', '0006_auto_20160219_1110'), ('orchestration', '0007_auto_20170528_2011'), ('orchestration', '0008_auto_20190805_1134'), ('orchestration', '0009_rename_route_async_run_async')] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='BackendLog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('backend', models.CharField(max_length=256, verbose_name='backend')), + ('state', models.CharField(choices=[('RECEIVED', 'RECEIVED'), ('TIMEOUT', 'TIMEOUT'), ('STARTED', 'STARTED'), ('SUCCESS', 'SUCCESS'), ('FAILURE', 'FAILURE'), ('ERROR', 'ERROR'), ('ABORTED', 'ABORTED'), ('REVOKED', 'REVOKED')], default='RECEIVED', max_length=16, verbose_name='state')), + ('script', models.TextField(verbose_name='script')), + ('stdout', models.TextField(verbose_name='stdout')), + ('stderr', models.TextField(verbose_name='stdin')), + ('traceback', models.TextField(verbose_name='traceback')), + ('exit_code', models.IntegerField(null=True, verbose_name='exit code')), + ('task_id', models.CharField(help_text='Celery task ID when used as execution backend', max_length=36, null=True, unique=True, verbose_name='task ID')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='updated')), + ], + options={ + 'get_latest_by': 'id', + }, + ), + migrations.CreateModel( + name='BackendOperation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('backend', models.CharField(max_length=256, verbose_name='backend')), + ('action', models.CharField(max_length=64, verbose_name='action')), + ('object_id', models.PositiveIntegerField(null=True)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('log', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='operations', to='orchestration.BackendLog')), + ('instance_repr', models.CharField(default='', max_length=256, verbose_name='instance representation')), + ], + options={ + 'verbose_name_plural': 'Operations', + 'verbose_name': 'Operation', + }, + ), + migrations.CreateModel( + name='Route', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('backend', models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DovecotPostfixPasswdVirtualUserController', '[S] Dovecot-Postfix virtualuser'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailmanController', '[S] Mailman'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PostfixAddressController', '[S] Postfix address'), ('uWSGIPythonController', '[S] Python uWSGI'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend')), + ('match', models.CharField(blank=True, default='True', help_text='Python expression used for selecting the targe host, instance referes to the current object.', max_length=256, verbose_name='match')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ], + ), + migrations.CreateModel( + name='Server', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Verbose name or hostname of this server.', max_length=256, unique=True, verbose_name='name')), + ('address', orchestra.models.fields.NullableCharField(blank=True, help_text='Optional IP address or domain name. If blank, name field will be used for address resolution.
If the IP address never changes you can set this field and save DNS requests.', max_length=256, null=True, unique=True, validators=[orchestra.core.validators.OrValidator(orchestra.core.validators.validate_ip_address, orchestra.core.validators.validate_hostname)], verbose_name='address')), + ('description', models.TextField(blank=True, verbose_name='description')), + ('os', models.CharField(choices=[('LINUX', 'Linux')], default='LINUX', max_length=32, verbose_name='operative system')), + ], + ), + migrations.AddField( + model_name='route', + name='host', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='orchestration.Server', verbose_name='host'), + ), + migrations.AddField( + model_name='backendlog', + name='server', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='execution_logs', to='orchestration.Server', verbose_name='server'), + ), + migrations.AddField( + model_name='route', + name='run_async', + field=models.BooleanField(default=False, help_text='Whether or not block the request/response cycle waitting this backend to finish its execution. Usually you want slave servers to run asynchronously.'), + ), + migrations.AlterUniqueTogether( + name='route', + unique_together=set([('backend', 'host')]), + ), + migrations.AlterField( + model_name='backendlog', + name='state', + field=models.CharField(choices=[('RECEIVED', 'RECEIVED'), ('TIMEOUT', 'TIMEOUT'), ('STARTED', 'STARTED'), ('SUCCESS', 'SUCCESS'), ('FAILURE', 'FAILURE'), ('ERROR', 'ERROR'), ('ABORTED', 'ABORTED'), ('REVOKED', 'REVOKED'), ('NOTHING', 'NOTHING')], default='RECEIVED', max_length=16, verbose_name='state'), + ), + migrations.AlterField( + model_name='backendlog', + name='stderr', + field=models.TextField(verbose_name='stderr'), + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DovecotPostfixPasswdVirtualUserController', '[S] Dovecot-Postfix virtualuser'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('uWSGIPythonController', '[S] Python uWSGI'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('SyncBind9MasterDomainController', '[S] Sync Bind9 master domain'), ('SyncBind9SlaveDomainController', '[S] Sync Bind9 slave domain'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + migrations.AddField( + model_name='route', + name='async_actions', + field=orchestra.models.fields.MultiSelectField(blank=True, help_text='Specify individual actions to be executed asynchronoulsy.', max_length=256), + ), + migrations.AlterField( + model_name='backendlog', + name='created_at', + field=models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='created'), + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailScannerSpamRuleController', '[S] MailScanner ruleset'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MoodleController', '[S] Moodle'), ('MoodleWWWRootController', '[S] Moodle WWWRoot (required)'), ('MoodleMuController', '[S] Moodle multisite'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PangeaProxmoxOVZ', '[S] PangeaProxmoxOVZ'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('PostfixRecipientAccessController', '[S] Postfix recipient access'), ('ProxmoxOVZ', '[S] ProxmoxOVZ'), ('uWSGIPythonController', '[S] Python uWSGI'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('SyncBind9MasterDomainController', '[S] Sync Bind9 master domain'), ('SyncBind9SlaveDomainController', '[S] Sync Bind9 slave domain'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressURLController', '[S] WordPress URL'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('OwnCloudController', '[S] ownCloud SaaS'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + migrations.AlterIndexTogether( + name='backendoperation', + index_together=set([('content_type', 'object_id')]), + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('LetsEncryptController', "[S] Let's encrypt!"), ('LxcController', '[S] LxcController'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailScannerSpamRuleController', '[S] MailScanner ruleset'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MoodleController', '[S] Moodle'), ('MoodleWWWRootController', '[S] Moodle WWWRoot (required)'), ('MoodleMuController', '[S] Moodle multisite'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PangeaProxmoxOVZ', '[S] PangeaProxmoxOVZ'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('PostfixRecipientAccessController', '[S] Postfix recipient access'), ('ProxmoxOVZ', '[S] ProxmoxOVZ'), ('uWSGIPythonController', '[S] Python uWSGI'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('SyncBind9MasterDomainController', '[S] Sync Bind9 master domain'), ('SyncBind9SlaveDomainController', '[S] Sync Bind9 slave domain'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressForceSSLController', '[S] WordPress Force SSL'), ('WordPressURLController', '[S] WordPress URL'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('NextCloudController', '[S] nextCloud SaaS'), ('OwnCloudController', '[S] ownCloud SaaS'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + migrations.AlterField( + model_name='route', + name='host', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='routes', to='orchestration.Server', verbose_name='host'), + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('LetsEncryptController', "[S] Let's encrypt!"), ('LxcController', '[S] LxcController'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailScannerSpamRuleController', '[S] MailScanner ruleset'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MoodleController', '[S] Moodle'), ('MoodleWWWRootController', '[S] Moodle WWWRoot (required)'), ('MoodleMuController', '[S] Moodle multisite'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PangeaProxmoxOVZ', '[S] PangeaProxmoxOVZ'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('PostfixRecipientAccessController', '[S] Postfix recipient access'), ('ProxmoxOVZ', '[S] ProxmoxOVZ'), ('uWSGIPythonController', '[S] Python uWSGI'), ('RoundcubeIdentityController', '[S] Roundcube Identity Controller'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('SyncBind9MasterDomainController', '[S] Sync Bind9 master domain'), ('SyncBind9SlaveDomainController', '[S] Sync Bind9 slave domain'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressForceSSLController', '[S] WordPress Force SSL'), ('WordPressURLController', '[S] WordPress URL'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('NextCloudController', '[S] nextCloud SaaS'), ('OwnCloudController', '[S] ownCloud SaaS'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('LetsEncryptController', "[S] Let's encrypt!"), ('LxcController', '[S] LxcController'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MoodleController', '[S] Moodle'), ('MoodleWWWRootController', '[S] Moodle WWWRoot (required)'), ('MoodleMuController', '[S] Moodle multisite'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('ProxmoxOVZ', '[S] ProxmoxOVZ'), ('uWSGIPythonController', '[S] Python uWSGI'), ('RoundcubeIdentityController', '[S] Roundcube Identity Controller'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressForceSSLController', '[S] WordPress Force SSL'), ('WordPressURLController', '[S] WordPress URL'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('NextCloudController', '[S] nextCloud SaaS'), ('OwnCloudController', '[S] ownCloud SaaS'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + ] diff --git a/orchestra/contrib/orchestration/migrations/0009_rename_route_async_run_async.py b/orchestra/contrib/orchestration/migrations/0009_rename_route_async_run_async.py new file mode 100644 index 00000000..f70deeeb --- /dev/null +++ b/orchestra/contrib/orchestration/migrations/0009_rename_route_async_run_async.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '0008_auto_20190805_1134'), + ] + + operations = [ + migrations.RenameField( + model_name='route', + old_name='async', + new_name='run_async', + ), + migrations.AlterField( + model_name='route', + name='backend', + field=models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic'), ('Apache2Controller', '[S] Apache 2'), ('BSCWController', '[S] BSCW SaaS'), ('Bind9MasterDomainController', '[S] Bind9 master domain'), ('Bind9SlaveDomainController', '[S] Bind9 slave domain'), ('DokuWikiMuController', '[S] DokuWiki multisite'), ('DrupalMuController', '[S] Drupal multisite'), ('GitLabSaaSController', '[S] GitLab SaaS'), ('LetsEncryptController', "[S] Let's encrypt!"), ('LxcController', '[S] LxcController'), ('AutoresponseController', '[S] Mail autoresponse'), ('MailmanController', '[S] Mailman'), ('MailmanVirtualDomainController', '[S] Mailman virtdomain-only'), ('MoodleController', '[S] Moodle'), ('MoodleWWWRootController', '[S] Moodle WWWRoot (required)'), ('MoodleMuController', '[S] Moodle multisite'), ('MySQLController', '[S] MySQL database'), ('MySQLUserController', '[S] MySQL user'), ('PHPController', '[S] PHP FPM/FCGID'), ('PostfixAddressController', '[S] Postfix address'), ('PostfixAddressVirtualDomainController', '[S] Postfix address virtdomain-only'), ('ProxmoxOVZ', '[S] ProxmoxOVZ'), ('uWSGIPythonController', '[S] Python uWSGI'), ('RoundcubeIdentityController', '[S] Roundcube Identity Controller'), ('StaticController', '[S] Static'), ('SymbolicLinkController', '[S] Symbolic link webapp'), ('UNIXUserMaildirController', '[S] UNIX maildir user'), ('UNIXUserController', '[S] UNIX user'), ('WebalizerAppController', '[S] Webalizer App'), ('WebalizerController', '[S] Webalizer Content'), ('WordPressForceSSLController', '[S] WordPress Force SSL'), ('WordPressURLController', '[S] WordPress URL'), ('WordPressController', '[S] Wordpress'), ('WordpressMuController', '[S] Wordpress multisite'), ('NextCloudController', '[S] nextCloud SaaS'), ('OwnCloudController', '[S] ownCloud SaaS'), ('PhpListSaaSController', '[S] phpList SaaS')], max_length=256, verbose_name='backend'), + ), + ] diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index 2952b72f..95e0e4d4 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -33,22 +33,22 @@ class Server(models.Model): os = models.CharField(_("operative system"), max_length=32, choices=settings.ORCHESTRATION_OS_CHOICES, default=settings.ORCHESTRATION_DEFAULT_OS) - + def __str__(self): return self.name or str(self.address) - + def get_address(self): if self.address: return self.address return self.name - + def get_ip(self): address = self.get_address() try: return validate_ip_address(address) except ValidationError: return socket.gethostbyname(self.name) - + def clean(self): self.name = self.name.strip() self.address = self.address.strip() @@ -75,7 +75,7 @@ class BackendLog(models.Model): NOTHING = 'NOTHING' # Special state for mocked backendlogs EXCEPTION = 'EXCEPTION' - + STATES = ( (RECEIVED, RECEIVED), (TIMEOUT, TIMEOUT), @@ -87,10 +87,10 @@ class BackendLog(models.Model): (REVOKED, REVOKED), (NOTHING, NOTHING), ) - + backend = models.CharField(_("backend"), max_length=256) state = models.CharField(_("state"), max_length=16, choices=STATES, default=RECEIVED) - server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs') + server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs', on_delete=models.CASCADE) script = models.TextField(_("script")) stdout = models.TextField(_("stdout")) stderr = models.TextField(_("stderr")) @@ -100,25 +100,25 @@ class BackendLog(models.Model): help_text="Celery task ID when used as execution backend") created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) updated_at = models.DateTimeField(_("updated"), auto_now=True) - + class Meta: get_latest_by = 'id' - + def __str__(self): return "%s@%s" % (self.backend, self.server) - + @property def execution_time(self): return (self.updated_at-self.created_at).total_seconds() - + @property def has_finished(self): return self.state not in (self.STARTED, self.RECEIVED) - + @property def is_success(self): return self.state in (self.SUCCESS, self.NOTHING) - + def backend_class(self): return ServiceBackend.get_backend(self.backend) @@ -135,26 +135,26 @@ class BackendOperation(models.Model): """ Encapsulates an operation, storing its related object, the action and the backend. """ - log = models.ForeignKey('orchestration.BackendLog', related_name='operations') + log = models.ForeignKey('orchestration.BackendLog', related_name='operations', on_delete=models.CASCADE) backend = models.CharField(_("backend"), max_length=256) action = models.CharField(_("action"), max_length=64) - content_type = models.ForeignKey(ContentType) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(null=True) instance_repr = models.CharField(_("instance representation"), max_length=256) - + instance = GenericForeignKey('content_type', 'object_id') objects = BackendOperationQuerySet.as_manager() - + class Meta: verbose_name = _("Operation") verbose_name_plural = _("Operations") index_together = ( ('content_type', 'object_id'), ) - + def __str__(self): return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.instance_repr) - + @cached_property def backend_class(self): return ServiceBackend.get_backend(self.backend) @@ -199,11 +199,11 @@ class Route(models.Model): """ backend = models.CharField(_("backend"), max_length=256, choices=ServiceBackend.get_choices()) - host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes') + host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes', on_delete=models.CASCADE) match = models.CharField(_("match"), max_length=256, blank=True, default='True', help_text=_("Python expression used for selecting the targe host, " "instance referes to the current object.")) - async = models.BooleanField(default=False, + run_async = models.BooleanField(default=False, help_text=_("Whether or not block the request/response cycle waitting this backend to " "finish its execution. Usually you want slave servers to run asynchronously.")) async_actions = MultiSelectField(max_length=256, blank=True, @@ -211,19 +211,19 @@ class Route(models.Model): # method = models.CharField(_("method"), max_lenght=32, choices=method_choices, # default=MethodBackend.get_default()) is_active = models.BooleanField(_("active"), default=True) - + objects = RouteQuerySet.as_manager() - + class Meta: unique_together = ('backend', 'host') - + def __str__(self): return "%s@%s" % (self.backend, self.host) - + @cached_property def backend_class(self): return ServiceBackend.get_backend(self.backend) - + def clean(self): if not self.match: self.match = 'True' @@ -244,10 +244,10 @@ class Route(models.Model): except Exception as exception: name = type(exception).__name__ raise ValidationError(': '.join((name, str(exception)))) - + def action_is_async(self, action): return action in self.async_actions - + def matches(self, instance): safe_locals = { 'instance': instance, @@ -255,11 +255,11 @@ class Route(models.Model): instance._meta.model_name: instance, } return eval(self.match, safe_locals) - + def enable(self): self.is_active = True self.save() - + def disable(self): self.is_active = False self.save() diff --git a/orchestra/contrib/orchestration/tests/test_route.py b/orchestra/contrib/orchestration/tests/test_route.py index 4b7ee121..390661ee 100644 --- a/orchestra/contrib/orchestration/tests/test_route.py +++ b/orchestra/contrib/orchestration/tests/test_route.py @@ -12,7 +12,7 @@ class RouterTests(BaseTestCase): def test_list_backends(self): # TODO count actual, register and compare - choices = list(Route._meta.get_field('backend')._choices) + choices = list(Route._meta.get_field('backend').choices) self.assertLess(1, len(choices)) def test_get_instances(self): @@ -25,7 +25,7 @@ class RouterTests(BaseTestCase): pass choices = backends.ServiceBackend.get_choices() - Route._meta.get_field('backend')._choices = choices + Route._meta.get_field('backend').choices = choices backend = TestBackend.get_name() route = Route.objects.create(backend=backend, host=self.host, match='True') diff --git a/orchestra/contrib/orchestration/utils.py b/orchestra/contrib/orchestration/utils.py index 5d9d2886..9e4dd51d 100644 --- a/orchestra/contrib/orchestration/utils.py +++ b/orchestra/contrib/orchestration/utils.py @@ -6,11 +6,11 @@ def retrieve_state(servers): pings = [] for server in servers: address = server.get_address() - ping = run('ping -c 1 -w 1 %s' % address, async=True) + ping = run('ping -c 1 -w 1 %s' % address, run_async=True) pings.append(ping) - uptime = sshrun(address, 'uptime', persist=True, async=True, options={'ConnectTimeout': 1}) + uptime = sshrun(address, 'uptime', persist=True, run_async=True, options={'ConnectTimeout': 1}) uptimes.append(uptime) - + state = {} for server, ping, uptime in zip(servers, pings, uptimes): ping = join(ping, silent=True) @@ -19,7 +19,7 @@ def retrieve_state(servers): ping = '%s ms' % ping.split('/')[4] else: ping = 'Offline' - + uptime = join(uptime, silent=True) uptime_stderr = uptime.stderr.decode() uptime = uptime.stdout.decode().split() @@ -28,5 +28,5 @@ def retrieve_state(servers): else: uptime = '%s' % uptime_stderr state[server.pk] = (ping, uptime) - + return state diff --git a/orchestra/contrib/orders/actions.py b/orchestra/contrib/orders/actions.py index fbfb9369..5a91a3f7 100644 --- a/orchestra/contrib/orders/actions.py +++ b/orchestra/contrib/orders/actions.py @@ -1,5 +1,5 @@ from django.contrib import admin, messages -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import transaction from django.utils import timezone from django.utils.safestring import mark_safe @@ -17,7 +17,7 @@ class BillSelectedOrders(object): verbose_name = _("Bill") template = 'admin/orders/order/bill_selected_options.html' __name__ = 'bill_selected_orders' - + def __call__(self, modeladmin, request, queryset): """ make this monster behave like a function """ self.modeladmin = modeladmin @@ -34,7 +34,7 @@ class BillSelectedOrders(object): del(self.queryset) del(self.context) return ret - + def set_options(self, request): form = BillSelectedOptionsForm() if request.POST.get('step'): @@ -56,7 +56,7 @@ class BillSelectedOrders(object): 'form': form, }) return render(request, self.template, self.context) - + def select_related(self, request): # TODO use changelist ? related = self.queryset.get_related().select_related('account', 'service') @@ -76,7 +76,7 @@ class BillSelectedOrders(object): 'form': form, }) return render(request, self.template, self.context) - + @transaction.atomic def confirmation(self, request): form = BillSelectConfirmationForm(initial=self.options) diff --git a/orchestra/contrib/orders/admin.py b/orchestra/contrib/orders/admin.py index a2918f11..79076067 100644 --- a/orchestra/contrib/orders/admin.py +++ b/orchestra/contrib/orders/admin.py @@ -1,13 +1,13 @@ from django import forms from django.contrib import admin -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.db.models import Prefetch from django.utils import timezone from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ -from orchestra.admin import ExtendedModelAdmin +from orchestra.admin import ExtendedModelAdmin from orchestra.admin.utils import admin_link, admin_date, change_url from orchestra.contrib.accounts.actions import list_accounts from orchestra.contrib.accounts.admin import AccountAdminMixin @@ -22,10 +22,10 @@ class MetricStorageInline(admin.TabularInline): model = MetricStorage readonly_fields = ('value', 'created_on', 'updated_on') extra = 0 - + def has_add_permission(self, request, obj=None): return False - + def get_fieldsets(self, request, obj=None): if obj: url = reverse('admin:orders_metricstorage_changelist') @@ -33,7 +33,7 @@ class MetricStorageInline(admin.TabularInline): title = _('Metric storage, last 10 entries, (See all)') self.verbose_name_plural = mark_safe(title % url) return super(MetricStorageInline, self).get_fieldsets(request, obj) - + def get_queryset(self, request): qs = super(MetricStorageInline, self).get_queryset(request) change_view = bool(self.parent_object and self.parent_object.pk) @@ -106,17 +106,17 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): 'content_object_repr', 'content_object_link', 'bills_links', 'account_link', 'service_link' ) - + service_link = admin_link('service') display_registered_on = admin_date('registered_on') display_cancelled_on = admin_date('cancelled_on') - + def display_description(self, order): return order.description[:64] display_description.short_description = _("Description") display_description.allow_tags = True display_description.admin_order_field = 'description' - + def content_object_link(self, order): if order.content_object: try: @@ -131,7 +131,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): content_object_link.short_description = _("Content object") content_object_link.allow_tags = True content_object_link.admin_order_field = 'content_object_repr' - + def bills_links(self, order): bills = [] make_link = admin_link() @@ -140,7 +140,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): return '
'.join(bills) bills_links.short_description = _("Bills") bills_links.allow_tags = True - + def display_billed_until(self, order): billed_until = order.billed_until red = False @@ -163,7 +163,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): display_billed_until.short_description = _("billed until") display_billed_until.allow_tags = True display_billed_until.admin_order_field = 'billed_until' - + def display_metric(self, order): """ dispalys latest metric value, don't uses latest() because not loosing prefetch_related @@ -174,7 +174,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): return '' return metric.value display_metric.short_description = _("Metric") - + def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ if db_field.name == 'description': diff --git a/orchestra/contrib/orders/helpers.py b/orchestra/contrib/orders/helpers.py index b5e1487a..ce0d1917 100644 --- a/orchestra/contrib/orders/helpers.py +++ b/orchestra/contrib/orders/helpers.py @@ -6,7 +6,7 @@ from orchestra.core import services def get_related_object(origin, max_depth=2): """ Introspects origin object and return the first related service object - + WARNING this is NOT an exhaustive search but a compromise between cost and flexibility. A more comprehensive approach may be considered if a use-case calls for it. @@ -16,12 +16,12 @@ def get_related_object(origin, max_depth=2): if hasattr(field, 'ct_field'): yield getattr(node, field.name) for field in node._meta.fields: - if field.rel: + if field.remote_field: try: yield getattr(node, field.name) except ObjectDoesNotExist: pass - + # BFS model relation transversal queue = [[origin]] while queue: diff --git a/orchestra/contrib/orders/migrations/0001_initial.py b/orchestra/contrib/orders/migrations/0001_initial.py index 7d4f9173..adc37bc1 100644 --- a/orchestra/contrib/orders/migrations/0001_initial.py +++ b/orchestra/contrib/orders/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations import django.utils.timezone +import django.db.models.deletion from django.conf import settings @@ -38,9 +39,9 @@ class Migration(migrations.Migration): ('billed_until', models.DateField(blank=True, verbose_name='billed until', null=True)), ('ignore', models.BooleanField(default=False, verbose_name='ignore')), ('description', models.TextField(blank=True, verbose_name='description')), - ('account', models.ForeignKey(verbose_name='account', related_name='orders', to=settings.AUTH_USER_MODEL)), - ('content_type', models.ForeignKey(to='contenttypes.ContentType')), - ('service', models.ForeignKey(verbose_name='service', related_name='orders', to='services.Service')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='account', related_name='orders', to=settings.AUTH_USER_MODEL)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='service', related_name='orders', to='services.Service')), ], options={ 'get_latest_by': 'id', @@ -49,6 +50,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='metricstorage', name='order', - field=models.ForeignKey(verbose_name='order', related_name='metrics', to='orders.Order'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='order', related_name='metrics', to='orders.Order'), ), ] diff --git a/orchestra/contrib/orders/migrations/0001_squashed_0006_auto_20210422_1108.py b/orchestra/contrib/orders/migrations/0001_squashed_0006_auto_20210422_1108.py new file mode 100644 index 00000000..b9091268 --- /dev/null +++ b/orchestra/contrib/orders/migrations/0001_squashed_0006_auto_20210422_1108.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:09 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + replaces = [('orders', '0001_initial'), ('orders', '0002_auto_20150709_1018'), ('orders', '0003_order_content_object_repr'), ('orders', '0004_auto_20150729_0945'), ('orders', '0005_auto_20160219_1107'), ('orders', '0006_auto_20210422_1108')] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('services', '__first__'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='MetricStorage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.DecimalField(decimal_places=2, max_digits=16, verbose_name='value')), + ('created_on', models.DateField(auto_now_add=True, verbose_name='created')), + ('updated_on', models.DateTimeField(verbose_name='updated')), + ], + options={ + 'get_latest_by': 'id', + }, + ), + migrations.CreateModel( + name='Order', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(null=True)), + ('registered_on', models.DateField(db_index=True, default=django.utils.timezone.now, verbose_name='registered')), + ('cancelled_on', models.DateField(blank=True, null=True, verbose_name='cancelled')), + ('billed_on', models.DateField(blank=True, null=True, verbose_name='billed')), + ('billed_until', models.DateField(blank=True, null=True, verbose_name='billed until')), + ('ignore', models.BooleanField(default=False, verbose_name='ignore')), + ('description', models.TextField(blank=True, verbose_name='description')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='orders', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='orders', to='services.Service', verbose_name='service')), + ('content_object_repr', models.CharField(editable=False, help_text='Used for searches.', max_length=256, verbose_name='content object representation')), + ('billed_metric', models.DecimalField(blank=True, decimal_places=2, max_digits=16, null=True, verbose_name='billed metric')), + ], + options={ + 'get_latest_by': 'id', + }, + ), + migrations.AddField( + model_name='metricstorage', + name='order', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metrics', to='orders.Order', verbose_name='order'), + ), + migrations.AlterIndexTogether( + name='order', + index_together=set([('content_type', 'object_id')]), + ), + ] diff --git a/orchestra/contrib/orders/models.py b/orchestra/contrib/orders/models.py index 28ceb4fd..69e5fa3b 100644 --- a/orchestra/contrib/orders/models.py +++ b/orchestra/contrib/orders/models.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) class OrderQuerySet(models.QuerySet): group_by = queryset.group_by - + def bill(self, **options): bills = [] bill_backend = Order.get_bill_backend() @@ -46,17 +46,17 @@ class OrderQuerySet(models.QuerySet): if commit: return list(set(bills)) return bills - + def givers(self, ini, end): return self.cancelled_and_billed().filter(billed_until__gt=ini, registered_on__lt=end) - + def cancelled_and_billed(self, exclude=False): qs = dict(cancelled_on__isnull=False, billed_until__isnull=False, cancelled_on__lte=F('billed_until')) if exclude: return self.exclude(**qs) return self.filter(**qs) - + def get_related(self, **options): """ returns related orders that could have a pricing effect """ Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) @@ -86,25 +86,25 @@ class OrderQuerySet(models.QuerySet): return self.model.objects.none() ids = self.values_list('id', flat=True) return self.model.objects.filter(qs).exclude(id__in=ids) - + def pricing_orders(self, ini, end): return self.filter(billed_until__isnull=False, billed_until__gt=ini, registered_on__lt=end) - + def by_object(self, obj, **kwargs): ct = ContentType.objects.get_for_model(obj) return self.filter(object_id=obj.pk, content_type=ct, **kwargs) - + def active(self, **kwargs): """ return active orders """ return self.filter( Q(cancelled_on__isnull=True) | Q(cancelled_on__gt=timezone.now()) ).filter(**kwargs) - + def inactive(self, **kwargs): """ return inactive orders """ return self.filter(cancelled_on__lte=timezone.now(), **kwargs) - + def update_by_instance(self, instance, service=None, commit=True): updates = [] if service is None: @@ -150,12 +150,12 @@ class OrderQuerySet(models.QuerySet): class Order(models.Model): - account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='orders') - content_type = models.ForeignKey(ContentType) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='orders') + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField(null=True) - service = models.ForeignKey(settings.ORDERS_SERVICE_MODEL, verbose_name=_("service"), - related_name='orders') + service = models.ForeignKey(settings.ORDERS_SERVICE_MODEL, on_delete=models.PROTECT, + verbose_name=_("service"), related_name='orders') registered_on = models.DateField(_("registered"), default=timezone.now, db_index=True) cancelled_on = models.DateField(_("cancelled"), null=True, blank=True) billed_on = models.DateField(_("billed"), null=True, blank=True) @@ -166,29 +166,29 @@ class Order(models.Model): description = models.TextField(_("description"), blank=True) content_object_repr = models.CharField(_("content object representation"), max_length=256, editable=False, help_text=_("Used for searches.")) - + content_object = GenericForeignKey() objects = OrderQuerySet.as_manager() - + class Meta: get_latest_by = 'id' index_together = ( ('content_type', 'object_id'), ) - + def __str__(self): return str(self.service) - + @classmethod def get_bill_backend(cls): return import_class(settings.ORDERS_BILLING_BACKEND)() - + def clean(self): if self.billed_on and self.billed_on < self.registered_on: raise ValidationError(_("Billed date can not be earlier than registered on.")) if self.billed_until and not self.billed_on: raise ValidationError(_("Billed on is missing while billed until is being provided.")) - + def update(self): instance = self.content_object if instance is None: @@ -214,22 +214,22 @@ class Order(models.Model): update_fields.append('content_object_repr') if update_fields: self.save(update_fields=update_fields) - + def cancel(self, commit=True): self.cancelled_on = timezone.now() self.ignore = self.service.handler.get_order_ignore(self) if commit: self.save(update_fields=['cancelled_on', 'ignore']) logger.info("CANCELLED order id: {id}".format(id=self.id)) - + def mark_as_ignored(self): self.ignore = True self.save(update_fields=['ignore']) - + def mark_as_not_ignored(self): self.ignore = False self.save(update_fields=['ignore']) - + def get_metric(self, *args, **kwargs): if kwargs.pop('changes', False): ini, end = args @@ -294,16 +294,17 @@ class MetricStorageQuerySet(models.QuerySet): class MetricStorage(models.Model): """ Stores metric state for future billing """ - order = models.ForeignKey(Order, verbose_name=_("order"), related_name='metrics') + order = models.ForeignKey(Order, on_delete=models.CASCADE, + verbose_name=_("order"), related_name='metrics') value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) created_on = models.DateField(_("created"), auto_now_add=True, editable=True) # TODO time field? updated_on = models.DateTimeField(_("updated")) - + objects = MetricStorageQuerySet.as_manager() - + class Meta: get_latest_by = 'id' - + def __str__(self): return str(self.order) diff --git a/orchestra/contrib/orders/signals.py b/orchestra/contrib/orders/signals.py index 5ddc818f..c1b541e8 100644 --- a/orchestra/contrib/orders/signals.py +++ b/orchestra/contrib/orders/signals.py @@ -15,7 +15,7 @@ def cancel_orders(sender, **kwargs): if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS: instance = kwargs['instance'] # Account delete will delete all related orders, no need to maintain order consistency - if isinstance(instance, Order.account.field.rel.to): + if isinstance(instance, Order.account.field.model): return if type(instance) in services: for order in Order.objects.by_object(instance).active(): diff --git a/orchestra/contrib/payments/actions.py b/orchestra/contrib/payments/actions.py index 1a904c64..e6b4b348 100644 --- a/orchestra/contrib/payments/actions.py +++ b/orchestra/contrib/payments/actions.py @@ -2,7 +2,7 @@ from functools import partial from django.contrib import messages from django.contrib.admin import actions -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import transaction from django.shortcuts import render, redirect from django.utils.safestring import mark_safe diff --git a/orchestra/contrib/payments/admin.py b/orchestra/contrib/payments/admin.py index 37fe7c7d..f753231b 100644 --- a/orchestra/contrib/payments/admin.py +++ b/orchestra/contrib/payments/admin.py @@ -1,5 +1,5 @@ from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponseRedirect from django.utils.translation import ugettext_lazy as _ @@ -48,20 +48,20 @@ class TransactionInline(admin.TabularInline): 'amount', 'currency' ) readonly_fields = fields - + transaction_link = admin_link('__str__', short_description=_("ID")) bill_link = admin_link('bill') source_link = admin_link('source') display_state = admin_colored('state', colors=STATE_COLORS) - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + def has_add_permission(self, *args, **kwargs): return False - + def get_queryset(self, *args, **kwargs): qs = super().get_queryset(*args, **kwargs) return qs.select_related('source', 'bill') @@ -116,28 +116,28 @@ class TransactionAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): ) list_select_related = ('source', 'bill__account', 'process') date_hierarchy = 'created_at' - + bill_link = admin_link('bill') source_link = admin_link('source') process_link = admin_link('process', short_description=_("proc")) account_link = admin_link('bill__account') display_created_at = admin_date('created_at', short_description=_("Created")) display_modified_at = admin_date('modified_at', short_description=_("Modified")) - + def has_delete_permission(self, *args, **kwargs): return False - + def get_actions(self, request): actions = super().get_actions(request) if 'delete_selected' in actions: del actions['delete_selected'] return actions - + def get_change_readonly_fields(self, request, obj): if obj.state in (Transaction.WAITTING_PROCESSING, Transaction.WAITTING_EXECUTION): return () return ('amount', 'currency') - + def get_change_view_actions(self, obj=None): actions = super(TransactionAdmin, self).get_change_view_actions() exclude = [] @@ -153,7 +153,7 @@ class TransactionAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): elif obj.state == Transaction.SECURED: return [] return [action for action in actions if action.__name__ not in exclude] - + def display_state(self, obj): state = admin_colored('state', colors=STATE_COLORS)(obj) help_text = obj.get_state_help() @@ -178,16 +178,16 @@ class TransactionProcessAdmin(ChangeViewActionsMixin, admin.ModelAdmin): actions.mark_process_as_executed, actions.abort, actions.commit, actions.report ) actions = change_view_actions + (actions.delete_selected,) - + display_state = admin_colored('state', colors=PROCESS_STATE_COLORS) display_created_at = admin_date('created_at', short_description=_("Created")) - + def file_url(self, process): if process.file: return '%s' % (process.file.url, process.file.name) file_url.allow_tags = True file_url.admin_order_field = 'file' - + def display_transactions(self, process): ids = [] lines = [] @@ -208,10 +208,10 @@ class TransactionProcessAdmin(ChangeViewActionsMixin, admin.ModelAdmin): return '%s' % (url, transactions) display_transactions.short_description = _("Transactions") display_transactions.allow_tags = True - + def has_add_permission(self, *args, **kwargs): return False - + def get_change_view_actions(self, obj=None): actions = super().get_change_view_actions() exclude = [] @@ -223,7 +223,7 @@ class TransactionProcessAdmin(ChangeViewActionsMixin, admin.ModelAdmin): elif obj.state == TransactionProcess.ABORTED: exclude = ['mark_process_as_executed', 'abort', 'commit'] return [action for action in actions if action.__name__ not in exclude] - + def delete_view(self, request, object_id, extra_context=None): queryset = self.model.objects.filter(id=object_id) related_transactions = helpers.pre_delete_processes(self, request, queryset) diff --git a/orchestra/contrib/payments/migrations/0001_initial.py b/orchestra/contrib/payments/migrations/0001_initial.py index dce1ac84..d63a7988 100644 --- a/orchestra/contrib/payments/migrations/0001_initial.py +++ b/orchestra/contrib/payments/migrations/0001_initial.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import django.db.models.deletion from django.db import models, migrations import jsonfield.fields from django.conf import settings @@ -21,7 +22,7 @@ class Migration(migrations.Migration): ('method', models.CharField(choices=[('CreditCard', 'Credit card'), ('SEPADirectDebit', 'SEPA Direct Debit')], verbose_name='method', max_length=32)), ('data', jsonfield.fields.JSONField(verbose_name='data', default={})), ('is_active', models.BooleanField(verbose_name='active', default=True)), - ('account', models.ForeignKey(verbose_name='account', related_name='paymentsources', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='account', related_name='paymentsources', to=settings.AUTH_USER_MODEL)), ], ), migrations.CreateModel( @@ -33,7 +34,7 @@ class Migration(migrations.Migration): ('currency', models.CharField(max_length=10, default='Eur')), ('created_at', models.DateTimeField(verbose_name='created', auto_now_add=True)), ('modified_at', models.DateTimeField(verbose_name='modified', auto_now=True)), - ('bill', models.ForeignKey(verbose_name='bill', related_name='transactions', to='bills.Bill')), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='bill', related_name='transactions', to='bills.Bill')), ], ), migrations.CreateModel( @@ -53,11 +54,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='transaction', name='process', - field=models.ForeignKey(verbose_name='process', null=True, blank=True, related_name='transactions', to='payments.TransactionProcess'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='process', null=True, blank=True, related_name='transactions', to='payments.TransactionProcess'), ), migrations.AddField( model_name='transaction', name='source', - field=models.ForeignKey(verbose_name='source', null=True, blank=True, related_name='transactions', to='payments.PaymentSource'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='source', null=True, blank=True, related_name='transactions', to='payments.PaymentSource'), ), ] diff --git a/orchestra/contrib/payments/migrations/0001_squashed_0004_auto_20210330_1049.py b/orchestra/contrib/payments/migrations/0001_squashed_0004_auto_20210330_1049.py new file mode 100644 index 00000000..d4b1b8b8 --- /dev/null +++ b/orchestra/contrib/payments/migrations/0001_squashed_0004_auto_20210330_1049.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:27 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.fields +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('payments', '0001_initial'), ('payments', '0002_auto_20150709_1018'), ('payments', '0003_auto_20170528_2011'), ('payments', '0004_auto_20210330_1049')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('bills', '0002_auto_20150429_1417'), + ] + + operations = [ + migrations.CreateModel( + name='PaymentSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('method', models.CharField(choices=[('CreditCard', 'Credit card'), ('SEPADirectDebit', 'SEPA Direct Debit')], max_length=32, verbose_name='method')), + ('data', jsonfield.fields.JSONField(default={}, verbose_name='data')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='paymentsources', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + ), + migrations.CreateModel( + name='Transaction', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('state', models.CharField(choices=[('WAITTING_PROCESSING', 'Waitting processing'), ('WAITTING_EXECUTION', 'Waitting execution'), ('EXECUTED', 'Executed'), ('SECURED', 'Secured'), ('REJECTED', 'Rejected')], default='WAITTING_PROCESSING', max_length=32, verbose_name='state')), + ('amount', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='amount')), + ('currency', models.CharField(default='Eur', max_length=10)), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created')), + ('modified_at', models.DateTimeField(auto_now=True, verbose_name='modified')), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transactions', to='bills.Bill', verbose_name='bill')), + ], + ), + migrations.CreateModel( + name='TransactionProcess', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', jsonfield.fields.JSONField(blank=True, verbose_name='data')), + ('file', orchestra.models.fields.PrivateFileField(blank=True, upload_to='', verbose_name='file')), + ('state', models.CharField(choices=[('CREATED', 'Created'), ('EXECUTED', 'Executed'), ('ABORTED', 'Aborted'), ('COMMITED', 'Commited')], default='CREATED', max_length=16, verbose_name='state')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='created')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='updated')), + ], + options={ + 'verbose_name_plural': 'Transaction processes', + }, + ), + migrations.AddField( + model_name='transaction', + name='process', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='transactions', to='payments.TransactionProcess', verbose_name='process'), + ), + migrations.AddField( + model_name='transaction', + name='source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='transactions', to='payments.PaymentSource', verbose_name='source'), + ), + ] diff --git a/orchestra/contrib/payments/migrations/0004_auto_20210330_1049.py b/orchestra/contrib/payments/migrations/0004_auto_20210330_1049.py new file mode 100644 index 00000000..8519ff36 --- /dev/null +++ b/orchestra/contrib/payments/migrations/0004_auto_20210330_1049.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('payments', '0003_auto_20170528_2011'), + ] + + operations = [ + migrations.AlterField( + model_name='paymentsource', + name='method', + field=models.CharField(choices=[('CreditCard', 'Credit card'), ('SEPADirectDebit', 'SEPA Direct Debit')], max_length=32, verbose_name='method'), + ), + ] diff --git a/orchestra/contrib/payments/models.py b/orchestra/contrib/payments/models.py index 82f2ce22..4d7770c8 100644 --- a/orchestra/contrib/payments/models.py +++ b/orchestra/contrib/payments/models.py @@ -18,50 +18,50 @@ class PaymentSourcesQueryset(models.QuerySet): class PaymentSource(models.Model): account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='paymentsources') + related_name='paymentsources', on_delete=models.CASCADE) method = models.CharField(_("method"), max_length=32, choices=PaymentMethod.get_choices()) data = JSONField(_("data"), default={}) is_active = models.BooleanField(_("active"), default=True) - + objects = PaymentSourcesQueryset.as_manager() - + def __str__(self): return "%s (%s)" % (self.label, self.method_class.verbose_name) - + @cached_property def method_class(self): return PaymentMethod.get(self.method) - + @cached_property def method_instance(self): """ Per request lived method_instance """ return self.method_class(self) - + @cached_property def label(self): return self.method_instance.get_label() - + @cached_property def number(self): return self.method_instance.get_number() - + def get_bill_context(self): method = self.method_instance return { 'message': method.get_bill_message(), } - + def get_due_delta(self): return self.method_instance.due_delta - + def clean(self): self.data = self.method_instance.clean_data() class TransactionQuerySet(models.QuerySet): group_by = group_by - + def create(self, **kwargs): source = kwargs.get('source') if source is None or not hasattr(source.method_class, 'process'): @@ -71,16 +71,16 @@ class TransactionQuerySet(models.QuerySet): if amount == 0: kwargs['state'] = self.model.SECURED return super(TransactionQuerySet, self).create(**kwargs) - + def secured(self): return self.filter(state=Transaction.SECURED) - + def exclude_rejected(self): return self.exclude(state=Transaction.REJECTED) - + def amount(self): return next(iter(self.aggregate(models.Sum('amount')).values())) or 0 - + def processing(self): return self.filter(state__in=[Transaction.EXECUTED, Transaction.WAITTING_EXECUTION]) @@ -108,8 +108,8 @@ class Transaction(models.Model): REJECTED: _("The transaction has failed and the ammount is lost, a new transaction " "should be created for recharging."), } - - bill = models.ForeignKey('bills.bill', verbose_name=_("bill"), + + bill = models.ForeignKey('bills.bill', on_delete=models.CASCADE, verbose_name=_("bill"), related_name='transactions') source = models.ForeignKey(PaymentSource, null=True, blank=True, on_delete=models.SET_NULL, verbose_name=_("source"), related_name='transactions') @@ -121,16 +121,16 @@ class Transaction(models.Model): currency = models.CharField(max_length=10, default=settings.PAYMENT_CURRENCY) created_at = models.DateTimeField(_("created"), auto_now_add=True) modified_at = models.DateTimeField(_("modified"), auto_now=True) - + objects = TransactionQuerySet.as_manager() - + def __str__(self): return "#%i" % self.id - + @property def account(self): return self.bill.account - + def clean(self): if not self.pk: amount = self.bill.transactions.exclude(state=self.REJECTED).amount() @@ -141,24 +141,24 @@ class Transaction(models.Model): 'amount': amount, } ) - + def get_state_help(self): if self.source: return self.source.method_instance.state_help.get(self.state) or self.STATE_HELP.get(self.state) return self.STATE_HELP.get(self.state) - + def mark_as_processed(self): self.state = self.WAITTING_EXECUTION self.save(update_fields=('state', 'modified_at')) - + def mark_as_executed(self): self.state = self.EXECUTED self.save(update_fields=('state', 'modified_at')) - + def mark_as_secured(self): self.state = self.SECURED self.save(update_fields=('state', 'modified_at')) - + def mark_as_rejected(self): self.state = self.REJECTED self.save(update_fields=('state', 'modified_at')) @@ -178,31 +178,31 @@ class TransactionProcess(models.Model): (ABORTED, _("Aborted")), (COMMITED, _("Commited")), ) - + data = JSONField(_("data"), blank=True) file = PrivateFileField(_("file"), blank=True) state = models.CharField(_("state"), max_length=16, choices=STATES, default=CREATED) created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) updated_at = models.DateTimeField(_("updated"), auto_now=True) - + class Meta: verbose_name_plural = _("Transaction processes") - + def __str__(self): return '#%i' % self.id - + def mark_as_executed(self): self.state = self.EXECUTED for transaction in self.transactions.all(): transaction.mark_as_executed() self.save(update_fields=('state', 'updated_at')) - + def abort(self): self.state = self.ABORTED for transaction in self.transactions.all(): transaction.mark_as_rejected() self.save(update_fields=('state', 'updated_at')) - + def commit(self): self.state = self.COMMITED for transaction in self.transactions.processing(): diff --git a/orchestra/contrib/plans/admin.py b/orchestra/contrib/plans/admin.py index 224865f2..c283d5e4 100644 --- a/orchestra/contrib/plans/admin.py +++ b/orchestra/contrib/plans/admin.py @@ -1,5 +1,5 @@ from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -28,7 +28,7 @@ class PlanAdmin(ExtendedModelAdmin): } change_readonly_fields = ('name',) inlines = [RateInline] - + def num_contracts(self, plan): num = plan.contracts__count url = reverse('admin:plans_contractedplan_changelist') @@ -37,7 +37,7 @@ class PlanAdmin(ExtendedModelAdmin): num_contracts.short_description = _("Contracts") num_contracts.admin_order_field = 'contracts__count' num_contracts.allow_tags = True - + def get_queryset(self, request): qs = super(PlanAdmin, self).get_queryset(request) return qs.annotate(models.Count('contracts', distinct=True)) @@ -49,7 +49,7 @@ class ContractedPlanAdmin(AccountAdminMixin, admin.ModelAdmin): list_select_related = ('plan', 'account') search_fields = ('account__username', 'plan__name', 'id') actions = (list_accounts,) - + plan_link = admin_link('plan') diff --git a/orchestra/contrib/plans/migrations/0001_initial.py b/orchestra/contrib/plans/migrations/0001_initial.py index f7ec2ba2..b48f70b8 100644 --- a/orchestra/contrib/plans/migrations/0001_initial.py +++ b/orchestra/contrib/plans/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.core.validators from django.conf import settings @@ -18,7 +19,7 @@ class Migration(migrations.Migration): name='ContractedPlan', fields=[ ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')), - ('account', models.ForeignKey(to=settings.AUTH_USER_MODEL, related_name='plans', verbose_name='account')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, related_name='plans', verbose_name='account')), ], options={ 'verbose_name_plural': 'plans', @@ -42,14 +43,14 @@ class Migration(migrations.Migration): ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')), ('quantity', models.PositiveIntegerField(help_text='See rate algorihm help text.', blank=True, verbose_name='quantity', null=True)), ('price', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='price')), - ('plan', models.ForeignKey(to='plans.Plan', related_name='rates', verbose_name='plan')), - ('service', models.ForeignKey(to='services.Service', related_name='rates', verbose_name='service')), + ('plan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plans.Plan', related_name='rates', verbose_name='plan')), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='services.Service', related_name='rates', verbose_name='service')), ], ), migrations.AddField( model_name='contractedplan', name='plan', - field=models.ForeignKey(to='plans.Plan', related_name='contracts', verbose_name='plan'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plans.Plan', related_name='contracts', verbose_name='plan'), ), migrations.AlterUniqueTogether( name='rate', diff --git a/orchestra/contrib/plans/migrations/0001_squashed_0003_auto_20210422_1108.py b/orchestra/contrib/plans/migrations/0001_squashed_0003_auto_20210422_1108.py new file mode 100644 index 00000000..292fc0cc --- /dev/null +++ b/orchestra/contrib/plans/migrations/0001_squashed_0003_auto_20210422_1108.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:09 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('plans', '0001_initial'), ('plans', '0002_auto_20160114_1713'), ('plans', '0003_auto_20210422_1108')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('services', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ContractedPlan', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='plans', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ], + options={ + 'verbose_name_plural': 'plans', + }, + ), + migrations.CreateModel( + name='Plan', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=32, unique=True, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('verbose_name', models.CharField(blank=True, max_length=128, verbose_name='verbose_name')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('is_default', models.BooleanField(default=False, help_text='Designates whether this plan is used by default or not.', verbose_name='default')), + ('is_combinable', models.BooleanField(default=True, help_text='Designates whether this plan can be combined with other plans or not.', verbose_name='combinable')), + ('allow_multiple', models.BooleanField(default=False, help_text='Designates whether this plan allow for multiple contractions.', verbose_name='allow multiple')), + ], + ), + migrations.CreateModel( + name='Rate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('quantity', models.PositiveIntegerField(blank=True, help_text='See rate algorihm help text.', null=True, verbose_name='quantity')), + ('price', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='price')), + ('plan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rates', to='plans.Plan', verbose_name='plan')), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rates', to='services.Service', verbose_name='service')), + ], + ), + migrations.AddField( + model_name='contractedplan', + name='plan', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='contracts', to='plans.Plan', verbose_name='plan'), + ), + migrations.AlterUniqueTogether( + name='rate', + unique_together=set([('service', 'plan', 'quantity')]), + ), + migrations.AlterField( + model_name='rate', + name='plan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rates', to='plans.Plan', verbose_name='plan'), + ), + migrations.AlterField( + model_name='rate', + name='plan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='rates', to='plans.Plan', verbose_name='plan'), + ), + ] diff --git a/orchestra/contrib/plans/migrations/0002_auto_20160114_1713.py b/orchestra/contrib/plans/migrations/0002_auto_20160114_1713.py index f9a08d9f..58e2ea14 100644 --- a/orchestra/contrib/plans/migrations/0002_auto_20160114_1713.py +++ b/orchestra/contrib/plans/migrations/0002_auto_20160114_1713.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -14,6 +15,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='rate', name='plan', - field=models.ForeignKey(related_name='rates', to='plans.Plan', blank=True, null=True, verbose_name='plan'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rates', to='plans.Plan', blank=True, null=True, verbose_name='plan'), ), ] diff --git a/orchestra/contrib/plans/models.py b/orchestra/contrib/plans/models.py index 3c87da00..22b698f3 100644 --- a/orchestra/contrib/plans/models.py +++ b/orchestra/contrib/plans/models.py @@ -25,32 +25,33 @@ class Plan(models.Model): help_text=_("Designates whether this plan can be combined with other plans or not.")) allow_multiple = models.BooleanField(_("allow multiple"), default=False, help_text=_("Designates whether this plan allow for multiple contractions.")) - + def __str__(self): return self.get_verbose_name() - + def clean(self): self.verbose_name = self.verbose_name.strip() - + def get_verbose_name(self): return self.verbose_name or self.name class ContractedPlan(models.Model): - plan = models.ForeignKey(Plan, verbose_name=_("plan"), related_name='contracts') - account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='plans') - + plan = models.ForeignKey(Plan, on_delete=models.CASCADE, + verbose_name=_("plan"), related_name='contracts') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='plans') + class Meta: verbose_name_plural = _("plans") - + def __str__(self): return str(self.plan) - + @cached_property def active(self): return self.plan.is_active and self.account.is_active - + def clean(self): if not self.pk and not self.plan.allow_multiple: if ContractedPlan.objects.filter(plan=self.plan, account=self.account).exists(): @@ -59,7 +60,7 @@ class ContractedPlan(models.Model): class RateQuerySet(models.QuerySet): group_by = queryset.group_by - + def by_account(self, account): # Default allways selected return self.filter( @@ -69,27 +70,27 @@ class RateQuerySet(models.QuerySet): class Rate(models.Model): - service = models.ForeignKey('services.Service', verbose_name=_("service"), - related_name='rates') - plan = models.ForeignKey(Plan, verbose_name=_("plan"), related_name='rates', null=True, - blank=True) + service = models.ForeignKey('services.Service', on_delete=models.CASCADE, + verbose_name=_("service"), related_name='rates') + plan = models.ForeignKey(Plan, on_delete=models.SET_NULL, null=True, blank=True, + verbose_name=_("plan"), related_name='rates') quantity = models.PositiveIntegerField(_("quantity"), null=True, blank=True, help_text=_("See rate algorihm help text.")) price = models.DecimalField(_("price"), max_digits=12, decimal_places=2) - + objects = RateQuerySet.as_manager() - + class Meta: unique_together = ('service', 'plan', 'quantity') - + def __str__(self): return "{}-{}".format(str(self.price), self.quantity) - + @classmethod @lru_cache() def get_methods(cls): return dict((method, import_class(method)) for method in settings.PLANS_RATE_METHODS) - + @classmethod @lru_cache() def get_choices(cls): @@ -97,7 +98,7 @@ class Rate(models.Model): for name, method in cls.get_methods().items(): choices.append((name, method.verbose_name)) return choices - + @classmethod def get_default(cls): return settings.PLANS_DEFAULT_RATE_METHOD diff --git a/orchestra/contrib/plans/ratings.py b/orchestra/contrib/plans/ratings.py index ec8d9494..e55c7a27 100644 --- a/orchestra/contrib/plans/ratings.py +++ b/orchestra/contrib/plans/ratings.py @@ -65,7 +65,7 @@ def _standardize(rates): def step_price(rates, metric): - if rates.query.order_by != ['plan', 'quantity']: + if rates.query.order_by != ('plan', 'quantity'): raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") # Step price group = [] @@ -127,7 +127,7 @@ step_price.help_text = _("All rates with a quantity lower or equal than the metr def match_price(rates, metric): - if rates.query.order_by != ['plan', 'quantity']: + if rates.query.order_by != ('plan', 'quantity'): raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") candidates = [] selected = False @@ -159,7 +159,7 @@ match_price.help_text = _("Only the rate with a) inmediate inferior metri def best_price(rates, metric): - if rates.query.order_by != ['plan', 'quantity']: + if rates.query.order_by != ('plan', 'quantity'): raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") candidates = [] for plan, rates in rates.group_by('plan').items(): diff --git a/orchestra/contrib/resources/actions.py b/orchestra/contrib/resources/actions.py index a40e7aa1..c355da2b 100644 --- a/orchestra/contrib/resources/actions.py +++ b/orchestra/contrib/resources/actions.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import reverse +from django.urls import reverse from django.shortcuts import redirect, render from django.utils.safestring import mark_safe from django.utils.translation import ungettext, ugettext_lazy as _ @@ -7,14 +7,14 @@ from django.utils.translation import ungettext, ugettext_lazy as _ def run_monitor(modeladmin, request, queryset): """ Resource and ResourceData run monitors """ referer = request.META.get('HTTP_REFERER') - async = modeladmin.model.monitor.__defaults__[0] + run_async = modeladmin.model.monitor.__defaults__[0] logs = set() for resource in queryset: rlogs = resource.monitor() - if not async: + if not run_async: logs = logs.union(set([str(log.pk) for log in rlogs])) modeladmin.log_change(request, resource, _("Run monitors")) - if async: + if run_async: num = len(queryset) # TODO listfilter by uuid: task.request.id + ?task_id__in=ids link = reverse('admin:djcelery_taskstate_changelist') diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py index fe3b0d05..f758851e 100644 --- a/orchestra/contrib/resources/admin.py +++ b/orchestra/contrib/resources/admin.py @@ -6,7 +6,7 @@ from django.contrib import admin, messages from django.contrib.contenttypes.admin import GenericTabularInline from django.contrib.contenttypes.forms import BaseGenericInlineFormSet from django.contrib.admin.utils import unquote -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import Q from django.shortcuts import redirect from django.templatetags.static import static @@ -58,7 +58,7 @@ class ResourceAdmin(ExtendedModelAdmin): 'name': ('verbose_name',) } list_select_related = ('content_type', 'crontab',) - + def change_view(self, request, object_id, form_url='', extra_context=None): """ Remaind user when monitor routes are not configured """ if request.method == 'GET': @@ -78,7 +78,7 @@ class ResourceAdmin(ExtendedModelAdmin): }) return super(ResourceAdmin, self).change_view(request, object_id, form_url=form_url, extra_context=extra_context) - + def save_model(self, request, obj, form, change): super(ResourceAdmin, self).save_model(request, obj, form, change) # best-effort @@ -93,12 +93,12 @@ class ResourceAdmin(ExtendedModelAdmin): modeladmin.inlines = inlines # reload Not always work sys.touch_wsgi() - + def formfield_for_dbfield(self, db_field, **kwargs): """ filter service content_types """ if db_field.name == 'content_type': models = [ model._meta.model_name for model in services.get() ] - kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models) + kwargs['queryset'] = db_field.remote_field.model.objects.filter(model__in=models) return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs) @@ -127,10 +127,10 @@ class ResourceDataAdmin(ExtendedModelAdmin): change_view_actions = actions ordering = ('-updated_at',) list_select_related = ('resource__content_type', 'content_type') - + resource_link = admin_link('resource') display_updated = admin_date('updated_at', short_description=_("Updated")) - + def get_urls(self): """Returns the additional urls for the change view links""" urls = super(ResourceDataAdmin, self).get_urls() @@ -150,7 +150,7 @@ class ResourceDataAdmin(ExtendedModelAdmin): name='%s_%s_list_related' % (opts.app_label, opts.model_name) ), ] + urls - + def display_used(self, rdata): if rdata.used is None: return '' @@ -159,15 +159,15 @@ class ResourceDataAdmin(ExtendedModelAdmin): display_used.short_description = _("Used") display_used.admin_order_field = 'used' display_used.allow_tags = True - + def has_add_permission(self, *args, **kwargs): return False - + def used_monitordata_view(self, request, object_id): url = reverse('admin:resources_monitordata_changelist') url += '?resource_data=%s' % object_id return redirect(url) - + def list_related_view(self, request, app_name, model_name, object_id): resources = Resource.objects.select_related('content_type') resource_models = {r.content_type.model_class(): r.content_type_id for r in resources} @@ -203,9 +203,9 @@ class MonitorDataAdmin(ExtendedModelAdmin): list_select_related = ('content_type',) search_fields = ('content_object_repr',) date_hierarchy = 'created_at' - + display_created = admin_date('created_at', short_description=_("Created")) - + def filter_used_monitordata(self, request, queryset): query_string = parse_qs(request.META['QUERY_STRING']) resource_data = query_string.get('resource_data') @@ -221,7 +221,7 @@ class MonitorDataAdmin(ExtendedModelAdmin): ids += dataset.values_list('id', flat=True) return queryset.filter(id__in=ids) return queryset - + def get_queryset(self, request): queryset = super(MonitorDataAdmin, self).get_queryset(request) queryset = self.filter_used_monitordata(request, queryset) @@ -239,13 +239,13 @@ def resource_inline_factory(resources): class ResourceInlineFormSet(BaseGenericInlineFormSet): def total_form_count(self, resources=resources): return len(resources) - + @cached def get_queryset(self): """ Filter disabled resources """ queryset = super(ResourceInlineFormSet, self).get_queryset() return queryset.filter(resource__is_active=True).select_related('resource') - + @cached_property def forms(self, resources=resources): forms = [] @@ -276,7 +276,7 @@ def resource_inline_factory(resources): for i, resource in enumerate(resources_copy, len(queryset)): forms.append(self._construct_form(i, resource=resource)) return forms - + class ResourceInline(GenericTabularInline): model = ResourceData verbose_name_plural = _("resources") @@ -287,14 +287,14 @@ def resource_inline_factory(resources): 'verbose_name', 'display_used', 'display_updated', 'allocated', 'unit', ) readonly_fields = ('display_used', 'display_updated',) - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + display_updated = admin_date('updated_at', default=_("Never")) - + def get_fieldsets(self, request, obj=None): if obj: opts = self.parent_model._meta @@ -303,7 +303,7 @@ def resource_inline_factory(resources): link = '%s' % (url, _("List related")) self.verbose_name_plural = mark_safe(_("Resources") + ' ' + link) return super(ResourceInline, self).get_fieldsets(request, obj) - + def display_used(self, rdata): update = '' history = '' @@ -330,11 +330,11 @@ def resource_inline_factory(resources): return _("No monitor") display_used.short_description = _("Used") display_used.allow_tags = True - + def has_add_permission(self, *args, **kwargs): """ Hidde add another """ return False - + return ResourceInline diff --git a/orchestra/contrib/resources/migrations/0001_initial.py b/orchestra/contrib/resources/migrations/0001_initial.py index 2496191e..ca718163 100644 --- a/orchestra/contrib/resources/migrations/0001_initial.py +++ b/orchestra/contrib/resources/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.contrib.resources.validators import orchestra.models.fields import django.utils.timezone @@ -24,7 +25,7 @@ class Migration(migrations.Migration): ('object_id', models.PositiveIntegerField(verbose_name='object id')), ('created_at', models.DateTimeField(default=django.utils.timezone.now, verbose_name='created')), ('value', models.DecimalField(decimal_places=2, max_digits=16, verbose_name='value')), - ('content_type', models.ForeignKey(verbose_name='content type', to='contenttypes.ContentType')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='content type', to='contenttypes.ContentType')), ], options={ 'get_latest_by': 'id', @@ -45,8 +46,8 @@ class Migration(migrations.Migration): ('disable_trigger', models.BooleanField(help_text='Disables monitors exeeded and recovery triggers', default=False, verbose_name='disable trigger')), ('monitors', orchestra.models.fields.MultiSelectField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic')], blank=True, help_text='Monitor backends used for monitoring this resource.', max_length=256, verbose_name='monitors')), ('is_active', models.BooleanField(default=True, verbose_name='active')), - ('content_type', models.ForeignKey(help_text='Model where this resource will be hooked.', to='contenttypes.ContentType')), - ('crontab', models.ForeignKey(help_text='Crontab for periodic execution. Leave it empty to disable periodic monitoring', to='djcelery.CrontabSchedule', verbose_name='crontab', blank=True, null=True)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, help_text='Model where this resource will be hooked.', to='contenttypes.ContentType')), + ('crontab', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, help_text='Crontab for periodic execution. Leave it empty to disable periodic monitoring', to='djcelery.CrontabSchedule', verbose_name='crontab', blank=True, null=True)), ], ), migrations.CreateModel( @@ -57,8 +58,8 @@ class Migration(migrations.Migration): ('used', models.DecimalField(decimal_places=3, editable=False, max_digits=16, null=True, verbose_name='used')), ('updated_at', models.DateTimeField(editable=False, null=True, verbose_name='updated')), ('allocated', models.DecimalField(decimal_places=2, max_digits=8, blank=True, null=True, verbose_name='allocated')), - ('content_type', models.ForeignKey(verbose_name='content type', to='contenttypes.ContentType')), - ('resource', models.ForeignKey(related_name='dataset', to='resources.Resource', verbose_name='resource')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='content type', to='contenttypes.ContentType')), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dataset', to='resources.Resource', verbose_name='resource')), ], options={ 'verbose_name_plural': 'resource data', diff --git a/orchestra/contrib/resources/migrations/0001_squashed_0011_auto_20170528_2005.py b/orchestra/contrib/resources/migrations/0001_squashed_0011_auto_20170528_2005.py new file mode 100644 index 00000000..b0f8b8e6 --- /dev/null +++ b/orchestra/contrib/resources/migrations/0001_squashed_0011_auto_20170528_2005.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:26 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import orchestra.contrib.resources.validators +import orchestra.core.validators +import orchestra.models.fields + + +class Migration(migrations.Migration): + + replaces = [('resources', '0001_initial'), ('resources', '0002_auto_20150502_1429'), ('resources', '0003_auto_20150502_1433'), ('resources', '0004_auto_20150503_1559'), ('resources', '0005_auto_20150723_0940'), ('resources', '0006_auto_20150723_1249'), ('resources', '0007_auto_20150723_1251'), ('resources', '0008_monitordata_state'), ('resources', '0009_auto_20150804_1450'), ('resources', '0010_auto_20160219_1108'), ('resources', '0011_auto_20170528_2005')] + + initial = True + + dependencies = [ + ('djcelery', '__first__'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='MonitorData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('monitor', models.CharField(choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic')], db_index=True, max_length=256, verbose_name='monitor')), + ('object_id', models.PositiveIntegerField(verbose_name='object id')), + ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='created')), + ('value', models.DecimalField(decimal_places=2, max_digits=16, verbose_name='value')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='content type')), + ('content_object_repr', models.CharField(default='', editable=False, max_length=256, verbose_name='content object representation')), + ('state', models.DecimalField(decimal_places=2, help_text='Optional field used to store current state needed for diff-based monitoring.', max_digits=16, null=True, verbose_name='state')), + ], + options={ + 'get_latest_by': 'id', + 'verbose_name_plural': 'monitor data', + }, + ), + migrations.CreateModel( + name='Resource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Required. 32 characters or fewer. Lowercase letters, digits and hyphen only.', max_length=32, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('verbose_name', models.CharField(max_length=256, verbose_name='verbose name')), + ('aggregation', models.CharField(choices=[('last-10-days-avg', 'Last 10 days AVG'), ('last', 'Last value'), ('monthly-avg', 'Monthly AVG'), ('monthly-sum', 'Monthly Sum')], default='last-10-days-avg', help_text='Method used for aggregating this resource monitored data.', max_length=16, verbose_name='aggregation')), + ('on_demand', models.BooleanField(default=False, help_text='If enabled the resource will not be pre-allocated, but allocated under the application demand', verbose_name='on demand')), + ('default_allocation', models.PositiveIntegerField(blank=True, help_text='Default allocation value used when this is not an on demand resource', null=True, verbose_name='default allocation')), + ('unit', models.CharField(help_text='The unit in which this resource is represented. For example GB, KB or subscribers', max_length=16, verbose_name='unit')), + ('scale', models.CharField(help_text='Scale in which this resource monitoring resoults should be prorcessed to match with unit. e.g. 10**9', max_length=32, validators=[orchestra.contrib.resources.validators.validate_scale], verbose_name='scale')), + ('disable_trigger', models.BooleanField(default=False, help_text='Disables monitors exeeded and recovery triggers', verbose_name='disable trigger')), + ('monitors', orchestra.models.fields.MultiSelectField(blank=True, choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic')], help_text='Monitor backends used for monitoring this resource.', max_length=256, verbose_name='monitors')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ('content_type', models.ForeignKey(help_text='Model where this resource will be hooked.', on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('crontab', models.ForeignKey(blank=True, help_text='Crontab for periodic execution. Leave it empty to disable periodic monitoring', null=True, on_delete=django.db.models.deletion.CASCADE, to='djcelery.CrontabSchedule', verbose_name='crontab')), + ], + ), + migrations.CreateModel( + name='ResourceData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(verbose_name='object id')), + ('used', models.DecimalField(decimal_places=3, editable=False, max_digits=16, null=True, verbose_name='used')), + ('updated_at', models.DateTimeField(editable=False, null=True, verbose_name='updated')), + ('allocated', models.DecimalField(blank=True, decimal_places=2, max_digits=8, null=True, verbose_name='allocated')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='content type')), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dataset', to='resources.Resource', verbose_name='resource')), + ('content_object_repr', models.CharField(default='', editable=False, max_length=256, verbose_name='content object representation')), + ], + options={ + 'verbose_name_plural': 'resource data', + }, + ), + migrations.AlterUniqueTogether( + name='resourcedata', + unique_together=set([('resource', 'content_type', 'object_id')]), + ), + migrations.AlterField( + model_name='resource', + name='disable_trigger', + field=models.BooleanField(default=True, help_text='Disables monitors exeeded and recovery triggers', verbose_name='disable trigger'), + ), + migrations.AlterField( + model_name='resource', + name='monitors', + field=orchestra.models.fields.MultiSelectField(blank=True, choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('OpenVZTraffic', '[M] OpenVZTraffic'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic')], help_text='Monitor backends used for monitoring this resource.', max_length=256, verbose_name='monitors'), + ), + migrations.AlterField( + model_name='resource', + name='crontab', + field=models.ForeignKey(blank=True, help_text='Crontab for periodic execution. Leave it empty to disable periodic monitoring', null=True, on_delete=django.db.models.deletion.SET_NULL, to='djcelery.CrontabSchedule', verbose_name='crontab'), + ), + migrations.AlterField( + model_name='resource', + name='monitors', + field=orchestra.models.fields.MultiSelectField(blank=True, choices=[('Apache2Traffic', '[M] Apache 2 Traffic'), ('ApacheTrafficByName', '[M] ApacheTrafficByName'), ('DokuWikiMuTraffic', '[M] DokuWiki MU Traffic'), ('DovecotMaildirDisk', '[M] Dovecot Maildir size'), ('Exim4Traffic', '[M] Exim4 traffic'), ('MailmanSubscribers', '[M] Mailman subscribers'), ('MailmanTraffic', '[M] Mailman traffic'), ('MysqlDisk', '[M] MySQL disk'), ('PostfixMailscannerTraffic', '[M] Postfix-Mailscanner traffic'), ('ProxmoxOpenVZTraffic', '[M] ProxmoxOpenVZTraffic'), ('UNIXUserDisk', '[M] UNIX user disk'), ('VsFTPdTraffic', '[M] VsFTPd traffic'), ('WordpressMuTraffic', '[M] Wordpress MU Traffic'), ('NextCloudDiskQuota', '[M] nextCloud SaaS Disk Quota'), ('NextcloudTraffic', '[M] nextCloud SaaS Traffic'), ('OwnCloudDiskQuota', '[M] ownCloud SaaS Disk Quota'), ('OwncloudTraffic', '[M] ownCloud SaaS Traffic'), ('PhpListTraffic', '[M] phpList SaaS Traffic')], help_text='Monitor backends used for monitoring this resource.', max_length=256, verbose_name='monitors'), + ), + migrations.AlterUniqueTogether( + name='resource', + unique_together=set([('name', 'content_type'), ('verbose_name', 'content_type')]), + ), + migrations.AlterField( + model_name='resourcedata', + name='object_id', + field=models.PositiveIntegerField(db_index=True, verbose_name='object id'), + ), + migrations.AlterField( + model_name='resourcedata', + name='allocated', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='allocated'), + ), + migrations.AlterField( + model_name='resourcedata', + name='object_id', + field=models.PositiveIntegerField(verbose_name='object id'), + ), + migrations.AlterIndexTogether( + name='monitordata', + index_together=set([('content_type', 'object_id')]), + ), + migrations.AlterIndexTogether( + name='resourcedata', + index_together=set([('content_type', 'object_id')]), + ), + ] diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index 143d0496..50c1b14a 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -26,7 +26,7 @@ class Resource(models.Model): Defines a resource, a resource is basically an interpretation of data gathered by a Monitor """ - + LAST = 'LAST' MONTHLY_SUM = 'MONTHLY_SUM' MONTHLY_AVG = 'MONTHLY_AVG' @@ -36,13 +36,13 @@ class Resource(models.Model): (MONTHLY_AVG, _("Monthly avg")), ) _related = set() # keeps track of related models for resource cleanup - + name = models.CharField(_("name"), max_length=32, help_text=_("Required. 32 characters or fewer. Lowercase letters, " "digits and hyphen only."), validators=[validators.validate_name]) verbose_name = models.CharField(_("verbose name"), max_length=256) - content_type = models.ForeignKey(ContentType, + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, help_text=_("Model where this resource will be hooked.")) aggregation = models.CharField(_("aggregation"), max_length=16, choices=Aggregation.get_choices(), default=Aggregation.get_choices()[0][0], @@ -70,27 +70,27 @@ class Resource(models.Model): choices=ServiceMonitor.get_choices(), help_text=_("Monitor backends used for monitoring this resource.")) is_active = models.BooleanField(_("active"), default=True) - + objects = ResourceQuerySet.as_manager() - + class Meta: unique_together = ( ('name', 'content_type'), ('verbose_name', 'content_type') ) - + def __str__(self): return "%s-%s" % (self.content_type, self.name) - + @cached_property def aggregation_class(self): return Aggregation.get(self.aggregation) - + @cached_property def aggregation_instance(self): """ Per request lived type_instance """ return self.aggregation_class(self) - + def clean(self): self.verbose_name = self.verbose_name.strip() if self.on_demand and self.default_allocation: @@ -114,12 +114,12 @@ class Resource(models.Model): model_name, ) for error in monitor_errors ]}) - + def save(self, *args, **kwargs): super(Resource, self).save(*args, **kwargs) # This only works on tests (multiprocessing used on real deployments) apps.get_app_config('resources').reload_relations() - + def sync_periodic_task(self, delete=False): """ sync periodic task on save/delete resource operations """ name = 'monitor.%s' % self @@ -140,21 +140,21 @@ class Resource(models.Model): if task.crontab != self.crontab: task.crontab = self.crontab task.save(update_fields=['crontab']) - + def get_model_path(self, monitor): """ returns a model path between self.content_type and monitor.model """ resource_model = self.content_type.model_class() monitor_model = ServiceMonitor.get_backend(monitor).model_class() return get_model_field_path(monitor_model, resource_model) - + def get_scale(self): return eval(self.scale) - + def get_verbose_name(self): return self.verbose_name or self.name - - def monitor(self, async=True): - if async: + + def monitor(self, run_async=True): + if run_async: return tasks.monitor.apply_async(self.pk) return tasks.monitor(self.pk) @@ -178,8 +178,8 @@ class ResourceDataQuerySet(models.QuerySet): class ResourceData(models.Model): """ Stores computed resource usage and allocation """ - resource = models.ForeignKey(Resource, related_name='dataset', verbose_name=_("resource")) - content_type = models.ForeignKey(ContentType, verbose_name=_("content type")) + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name='dataset', verbose_name=_("resource")) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, verbose_name=_("content type")) object_id = models.PositiveIntegerField(_("object id")) used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True, editable=False) @@ -187,28 +187,28 @@ class ResourceData(models.Model): allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True) content_object_repr = models.CharField(_("content object representation"), max_length=256, editable=False) - + content_object = GenericForeignKey() objects = ResourceDataQuerySet.as_manager() - + class Meta: unique_together = ('resource', 'content_type', 'object_id') verbose_name_plural = _("resource data") index_together = ( ('content_type', 'object_id'), ) - + def __str__(self): return "%s: %s" % (self.resource, self.content_object) - + @property def unit(self): return self.resource.unit - + @property def verbose_name(self): return self.resource.verbose_name - + def get_used(self): resource = self.resource total = 0 @@ -220,7 +220,7 @@ class ResourceData(models.Model): has_result = True total += usage return float(total)/resource.get_scale() if has_result else None - + def update(self, current=None): if current is None: current = self.get_used() @@ -228,13 +228,13 @@ class ResourceData(models.Model): self.updated_at = timezone.now() self.content_object_repr = str(self.content_object) self.save(update_fields=('used', 'updated_at', 'content_object_repr')) - - def monitor(self, async=False): + + def monitor(self, run_async=False): ids = (self.object_id,) - if async: + if run_async: return tasks.monitor.delay(self.resource_id, ids=ids) return tasks.monitor(self.resource_id, ids=ids) - + def get_monitor_datasets(self): resource = self.resource for monitor in resource.monitors: @@ -267,7 +267,7 @@ class MonitorData(models.Model): """ Stores monitored data """ monitor = models.CharField(_("monitor"), max_length=256, db_index=True, choices=ServiceMonitor.get_choices()) - content_type = models.ForeignKey(ContentType, verbose_name=_("content type")) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, verbose_name=_("content type")) object_id = models.PositiveIntegerField(_("object id")) created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True) value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) @@ -275,20 +275,20 @@ class MonitorData(models.Model): help_text=_("Optional field used to store current state needed for diff-based monitoring.")) content_object_repr = models.CharField(_("content object representation"), max_length=256, editable=False) - + content_object = GenericForeignKey() objects = MonitorDataQuerySet.as_manager() - + class Meta: get_latest_by = 'id' verbose_name_plural = _("monitor data") index_together = ( ('content_type', 'object_id'), ) - + def __str__(self): return str(self.monitor) - + @cached_property def unit(self): return self.resource.unit @@ -324,15 +324,15 @@ def create_resource_relation(): ) self.obj.__resource_cache[attr] = rdata return rdata - + def __get__(self, obj, cls): """ proxy handled object """ self.obj = obj return self - + def __iter__(self): return iter(self.obj.resource_set.all()) - + # Clean previous state for related in Resource._related: try: @@ -342,9 +342,9 @@ def create_resource_relation(): pass else: related._meta.private_fields = [ - field for field in related._meta.private_fields if field.rel.to != ResourceData + field for field in related._meta.private_fields if field.remote_field.model != ResourceData ] - + for ct, resources in Resource.objects.group_by('content_type').items(): model = ct.model_class() relation = GenericRelation('resources.ResourceData') diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py index 4b176fda..10f80729 100644 --- a/orchestra/contrib/resources/tasks.py +++ b/orchestra/contrib/resources/tasks.py @@ -36,8 +36,8 @@ def monitor(resource_id, ids=None): for obj in model.objects.filter(**kwargs): op = Operation(backend, obj, Operation.MONITOR) monitorings.append(op) - logs += Operation.execute(monitorings, async=False) - + logs += Operation.execute(monitorings, run_async=False) + kwargs = {'id__in': ids} if ids else {} # Update used resources and trigger resource exceeded and revovery triggers = [] diff --git a/orchestra/contrib/saas/fields.py b/orchestra/contrib/saas/fields.py index 785e5eda..74bf8b6a 100644 --- a/orchestra/contrib/saas/fields.py +++ b/orchestra/contrib/saas/fields.py @@ -11,5 +11,4 @@ class VirtualDatabaseRelation(GenericRelation): pks.append(obj.database_id) if not pks: return [] - # TODO renamed to self.remote_field in django 1.8 - return self.rel.to._base_manager.db_manager(using).filter(pk__in=pks) + return self.remote_field.model._base_manager.db_manager(using).filter(pk__in=pks) diff --git a/orchestra/contrib/saas/migrations/0001_initial.py b/orchestra/contrib/saas/migrations/0001_initial.py index 5ed402ed..94ff9390 100644 --- a/orchestra/contrib/saas/migrations/0001_initial.py +++ b/orchestra/contrib/saas/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import jsonfield.fields from django.conf import settings import orchestra.core.validators @@ -23,8 +24,8 @@ class Migration(migrations.Migration): ('name', models.CharField(max_length=64, validators=[orchestra.core.validators.validate_username], verbose_name='Name', help_text='Required. 64 characters or fewer. Letters, digits and ./-/_ only.')), ('is_active', models.BooleanField(help_text='Designates whether this service should be treated as active. ', verbose_name='active', default=True)), ('data', jsonfield.fields.JSONField(help_text='Extra information dependent of each service.', verbose_name='data', default={})), - ('account', models.ForeignKey(verbose_name='account', to=settings.AUTH_USER_MODEL, related_name='saas')), - ('database', models.ForeignKey(null=True, blank=True, to='databases.Database')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='account', to=settings.AUTH_USER_MODEL, related_name='saas')), + ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, null=True, blank=True, to='databases.Database')), ], options={ 'verbose_name_plural': 'SaaS', diff --git a/orchestra/contrib/saas/migrations/0001_squashed_0004_auto_20210422_1108.py b/orchestra/contrib/saas/migrations/0001_squashed_0004_auto_20210422_1108.py new file mode 100644 index 00000000..419dea27 --- /dev/null +++ b/orchestra/contrib/saas/migrations/0001_squashed_0004_auto_20210422_1108.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:09 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.fields +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('saas', '0001_initial'), ('saas', '0002_auto_20151001_0923'), ('saas', '0003_auto_20170528_2011'), ('saas', '0004_auto_20210422_1108')] + + initial = True + + dependencies = [ + ('databases', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='SaaS', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service', models.CharField(choices=[('bscw', 'BSCW'), ('DokuWikiService', 'Dowkuwiki'), ('DrupalService', 'Drupal'), ('gitlab', 'GitLab'), ('MoodleService', 'Moodle'), ('seafile', 'SeaFile'), ('WordPressService', 'WordPress'), ('phplist', 'phpList')], max_length=32, verbose_name='service')), + ('name', models.CharField(help_text='Required. 64 characters or fewer. Letters, digits and ./-/_ only.', max_length=64, validators=[orchestra.core.validators.validate_username], verbose_name='Name')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this service should be treated as active. ', verbose_name='active')), + ('data', jsonfield.fields.JSONField(default={}, help_text='Extra information dependent of each service.', verbose_name='data')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='saas', to=settings.AUTH_USER_MODEL, verbose_name='account')), + ('database', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='databases.Database')), + ('custom_url', models.URLField(blank=True, help_text='Optional and alternative URL for accessing this service instance. A related website will be automatically configured if needed.', verbose_name='custom URL')), + ], + options={ + 'verbose_name_plural': 'SaaS', + 'verbose_name': 'SaaS', + }, + ), + migrations.AlterUniqueTogether( + name='saas', + unique_together=set([('name', 'service')]), + ), + migrations.AlterField( + model_name='saas', + name='service', + field=models.CharField(choices=[('bscw', 'BSCW'), ('dokuwiki', 'Dowkuwiki'), ('drupal', 'Drupal'), ('gitlab', 'GitLab'), ('moodle', 'Moodle'), ('seafile', 'SeaFile'), ('wordpress', 'WordPress'), ('phplist', 'phpList')], max_length=32, verbose_name='service'), + ), + migrations.AlterField( + model_name='saas', + name='custom_url', + field=models.URLField(blank=True, help_text='Optional and alternative URL for accessing this service instance. i.e. https://wiki.mydomain/doku/
A related website will be automatically configured if needed.', verbose_name='custom URL'), + ), + migrations.AlterField( + model_name='saas', + name='name', + field=models.CharField(help_text='Required. 64 characters or fewer. Letters, digits and ./- only.', max_length=64, validators=[orchestra.core.validators.validate_hostname], verbose_name='Name'), + ), + migrations.AlterField( + model_name='saas', + name='service', + field=models.CharField(choices=[('bscw', 'BSCW'), ('dokuwiki', 'Dowkuwiki'), ('drupal', 'Drupal'), ('gitlab', 'GitLab'), ('moodle', 'Moodle'), ('wordpress', 'WordPress'), ('nextcloud', 'nextCloud'), ('owncloud', 'ownCloud'), ('phplist', 'phpList')], max_length=32, verbose_name='service'), + ), + migrations.AlterField( + model_name='saas', + name='database', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='databases.Database'), + ), + ] diff --git a/orchestra/contrib/saas/models.py b/orchestra/contrib/saas/models.py index ee1423b1..c8a31f60 100644 --- a/orchestra/contrib/saas/models.py +++ b/orchestra/contrib/saas/models.py @@ -26,8 +26,8 @@ class SaaS(models.Model): name = models.CharField(_("Name"), max_length=64, help_text=_("Required. 64 characters or fewer. Letters, digits and ./- only."), validators=[validators.validate_hostname]) - account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='saas') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='saas') is_active = models.BooleanField(_("active"), default=True, help_text=_("Designates whether this service should be treated as active. ")) data = JSONField(_("data"), default={}, @@ -36,51 +36,52 @@ class SaaS(models.Model): help_text=_("Optional and alternative URL for accessing this service instance. " "i.e. https://wiki.mydomain/doku/
" "A related website will be automatically configured if needed.")) - database = models.ForeignKey('databases.Database', null=True, blank=True) - + database = models.ForeignKey('databases.Database', + on_delete=models.SET_NULL, null=True, blank=True) + # Some SaaS sites may need a database, with this virtual field we tell the ORM to delete them databases = VirtualDatabaseRelation('databases.Database') objects = SaaSQuerySet.as_manager() - + class Meta: verbose_name = "SaaS" verbose_name_plural = "SaaS" unique_together = ( ('name', 'service'), ) - + def __str__(self): return "%s@%s" % (self.name, self.service) - + @cached_property def service_class(self): return SoftwareService.get(self.service) - + @cached_property def service_instance(self): """ Per request lived service_instance """ return self.service_class(self) - + @cached_property def active(self): return self.is_active and self.account.is_active - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = True self.save(update_fields=('is_active',)) - + def clean(self): if not self.pk: self.name = self.name.lower() self.service_instance.clean() self.data = self.service_instance.clean_data() - + def get_site_domain(self): return self.service_instance.get_site_domain() - + def set_password(self, password): self.password = password diff --git a/orchestra/contrib/saas/services/helpers.py b/orchestra/contrib/saas/services/helpers.py index bf081995..7418bc9d 100644 --- a/orchestra/contrib/saas/services/helpers.py +++ b/orchestra/contrib/saas/services/helpers.py @@ -42,7 +42,7 @@ def clean_custom_url(saas): ) except Website.DoesNotExist: # get or create domain - Domain = Website.domains.field.rel.to + Domain = Website.domains.field.model try: domain = Domain.objects.get(name=url.netloc) except Domain.DoesNotExist: @@ -51,7 +51,7 @@ def clean_custom_url(saas): }) if domain.account != account: raise ValidationError({ - 'custom_url': _("Domain %s does not belong to account %s, it's from %s.") % + 'custom_url': _("Domain %s does not belong to account %s, it's from %s.") % (url.netloc, account, domain.account), }) # Create new website for custom_url @@ -110,7 +110,7 @@ def create_or_update_directive(saas): account=account, ) except Website.DoesNotExist: - Domain = Website.domains.field.rel.to + Domain = Website.domains.field.model domain = Domain.objects.get(name=url.netloc) # Create new website for custom_url tgt_server = Server.objects.get(name='web.pangea.lan') diff --git a/orchestra/contrib/saas/services/options.py b/orchestra/contrib/saas/services/options.py index 56a8dfef..168aecce 100644 --- a/orchestra/contrib/saas/services/options.py +++ b/orchestra/contrib/saas/services/options.py @@ -24,7 +24,7 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): 'http': (Website.HTTP, (Website.HTTP, Website.HTTP_AND_HTTPS)), 'https': (Website.HTTPS_ONLY, (Website.HTTPS, Website.HTTP_AND_HTTPS, Website.HTTPS_ONLY)), } - + name = None verbose_name = None form = SaaSPasswordForm @@ -34,7 +34,7 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): class_verbose_name = _("Software as a Service") plugin_field = 'service' allow_custom_url = False - + @classmethod @lru_cache() def get_plugins(cls, all=False): @@ -48,18 +48,18 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): for cls in settings.SAAS_ENABLED_SERVICES: plugins.append(import_class(cls)) return plugins - + def get_change_readonly_fields(cls): fields = super(SoftwareService, cls).get_change_readonly_fields() return fields + ('name',) - + def get_site_domain(self): context = { 'site_name': self.instance.name, 'name': self.instance.name, } return self.site_domain % context - + def clean(self): if self.allow_custom_url: if self.instance.custom_url: @@ -69,7 +69,7 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): raise ValidationError({ 'custom_url': _("Custom URL not allowed for this service."), }) - + def clean_data(self): data = super(SoftwareService, self).clean_data() if not self.instance.pk: @@ -88,10 +88,10 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): if errors: raise ValidationError(errors) return data - + def get_directive_name(self): return '%s-saas' % self.name - + def get_directive(self, *args): if not args: instance = self.instance @@ -106,7 +106,7 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): website__domains__name=url.netloc, website__account=account, ) - + def get_website(self): url = urlparse(self.instance.custom_url) account = self.instance.account @@ -117,10 +117,10 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): directives__name=self.get_directive_name(), directives__value=url.path, ) - + def create_or_update_directive(self): return helpers.create_or_update_directive(self) - + def delete_directive(self): directive = None try: @@ -131,7 +131,7 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): return if directive is not None: directive.delete() - + def save(self): # pre instance.save() if isinstalled('orchestra.contrib.websites'): @@ -139,11 +139,11 @@ class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): self.create_or_update_directive() elif self.instance.pk: self.delete_directive() - + def delete(self): if isinstalled('orchestra.contrib.websites'): self.delete_directive() - + def get_related(self): return [] @@ -152,7 +152,7 @@ class DBSoftwareService(SoftwareService): db_name = None db_user = None abstract = True - + def get_db_name(self): context = { 'name': self.instance.name, @@ -161,15 +161,15 @@ class DBSoftwareService(SoftwareService): db_name = self.db_name % context # Limit for mysql database names return db_name[:65] - + def get_db_user(self): return self.db_user - + @cached def get_account(self): account_model = self.instance._meta.get_field('account') - return account_model.rel.to.objects.get_main() - + return account_model.remote_field.model.objects.get_main() + def validate(self): super(DBSoftwareService, self).validate() create = not self.instance.pk @@ -192,7 +192,7 @@ class DBSoftwareService(SoftwareService): raise ValidationError({ 'name': e.messages, }) - + def save(self): super(DBSoftwareService, self).save() account = self.get_account() diff --git a/orchestra/contrib/saas/services/phplist.py b/orchestra/contrib/saas/services/phplist.py index 7b265b18..7b803f04 100644 --- a/orchestra/contrib/saas/services/phplist.py +++ b/orchestra/contrib/saas/services/phplist.py @@ -1,7 +1,7 @@ from django import forms from django.core import validators from django.core.exceptions import ValidationError -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import Q from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ @@ -25,7 +25,7 @@ class PHPListForm(SaaSPasswordForm): help_text=_("Dedicated mailbox used for reciving bounces."), widget=SpanWidget(display=settings.SAAS_PHPLIST_BOUNCES_MAILBOX_NAME.replace( '%(', '<').replace(')s', '>'))) - + def __init__(self, *args, **kwargs): super(PHPListForm, self).__init__(*args, **kwargs) self.fields['name'].label = _("Site name") @@ -76,14 +76,14 @@ class PHPListService(DBSoftwareService): allow_custom_url = settings.SAAS_PHPLIST_ALLOW_CUSTOM_URL db_name = settings.SAAS_PHPLIST_DB_NAME db_user = settings.SAAS_PHPLIST_DB_USER - + def get_mailbox_name(self): context = { 'name': self.instance.name, 'site_name': self.instance.name, } return settings.SAAS_PHPLIST_BOUNCES_MAILBOX_NAME % context - + def validate(self): super(PHPListService, self).validate() create = not self.instance.pk @@ -97,7 +97,7 @@ class PHPListService(DBSoftwareService): raise ValidationError({ 'name': e.messages, }) - + def save(self): super(PHPListService, self).save() account = self.get_account() @@ -111,7 +111,7 @@ class PHPListService(DBSoftwareService): 'mailbox_id': mailbox.pk, 'mailbox_name': mailbox_name, }) - + def delete(self): super(PHPListService, self).save() account = self.get_account() diff --git a/orchestra/contrib/services/actions.py b/orchestra/contrib/services/actions.py index b11b9b49..96d38e3d 100644 --- a/orchestra/contrib/services/actions.py +++ b/orchestra/contrib/services/actions.py @@ -1,5 +1,5 @@ from django.contrib.admin import helpers -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import transaction from django.shortcuts import render, redirect from django.template.response import TemplateResponse diff --git a/orchestra/contrib/services/admin.py b/orchestra/contrib/services/admin.py index 3d0f05a0..b8e20634 100644 --- a/orchestra/contrib/services/admin.py +++ b/orchestra/contrib/services/admin.py @@ -1,7 +1,7 @@ from django import forms from django.conf.urls import url from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.template.response import TemplateResponse from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -42,7 +42,7 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): actions = (update_orders, clone, disable, enable) change_view_actions = actions + (view_help,) change_form_template = 'admin/services/service/change_form.html' - + def get_urls(self): """Returns the additional urls for the change view links""" urls = super(ServiceAdmin, self).get_urls() @@ -54,17 +54,17 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): name='%s_%s_help' % (opts.app_label, opts.model_name) ) ] + urls - + def formfield_for_dbfield(self, db_field, **kwargs): """ Improve performance of account field and filter by account """ if db_field.name == 'content_type': models = [model._meta.model_name for model in services.get()] - queryset = db_field.rel.to.objects + queryset = db_field.remote_field.model.objects kwargs['queryset'] = queryset.filter(model__in=models) if db_field.name in ['match', 'metric', 'order_description']: kwargs['widget'] = forms.TextInput(attrs={'size':'160'}) return super(ServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs) - + def num_orders(self, service): num = service.orders__count url = reverse('admin:orders_order_changelist') @@ -73,7 +73,7 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): num_orders.short_description = _("Orders") num_orders.admin_order_field = 'orders__count' num_orders.allow_tags = True - + def get_queryset(self, request): qs = super(ServiceAdmin, self).get_queryset(request) # Count active orders @@ -88,7 +88,7 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): ) }) return qs - + def help_view(self, request, *args): opts = self.model._meta context = { diff --git a/orchestra/contrib/services/handlers.py b/orchestra/contrib/services/handlers.py index 29675e76..af2671bb 100644 --- a/orchestra/contrib/services/handlers.py +++ b/orchestra/contrib/services/handlers.py @@ -21,29 +21,29 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): """ Separates all the logic of billing handling from the model allowing to better customize its behaviout - + Relax and enjoy the journey. """ _PLAN = 'plan' _COMPENSATION = 'compensation' _PREPAY = 'prepay' - + model = None - + def __init__(self, service): self.service = service - + def __getattr__(self, attr): return getattr(self.service, attr) - + @classmethod def get_choices(cls): choices = super(ServiceHandler, cls).get_choices() return [('', _("Default"))] + choices - + def validate_content_type(self, service): pass - + def validate_expression(self, service, method): try: obj = service.content_type.model_class().objects.all()[0] @@ -53,24 +53,24 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): bool(getattr(self, method)(obj)) except Exception as exc: raise ValidationError(format_exception(exc)) - + def validate_match(self, service): if not service.match: service.match = 'True' self.validate_expression(service, 'matches') - + def validate_metric(self, service): self.validate_expression(service, 'get_metric') - + def validate_order_description(self, service): self.validate_expression(service, 'get_order_description') - + def get_content_type(self): if not self.model: return self.content_type app_label, model = self.model.split('.') return ContentType.objects.get_by_natural_key(app_label, model.lower()) - + def get_expression_context(self, instance): return { 'instance': instance, @@ -85,14 +85,14 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): 'log10': math.log10, 'Decimal': decimal.Decimal, } - + def matches(self, instance): if not self.match: # Blank expressions always evaluate True return True safe_locals = self.get_expression_context(instance) return eval(self.match, safe_locals) - + def get_ignore_delta(self): if self.ignore_period == self.NEVER: return None @@ -104,14 +104,14 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): return datetime.timedelta(months=value) else: raise ValueError("Unknown unit %s" % unit) - + def get_order_ignore(self, order): """ service trial delta """ ignore_delta = self.get_ignore_delta() if ignore_delta and (order.cancelled_on-ignore_delta).date() <= order.registered_on: return True return order.ignore - + def get_ignore(self, instance): if self.ignore_superusers: account = getattr(instance, 'account', instance) @@ -120,7 +120,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if 'superuser' in settings.SERVICES_IGNORE_ACCOUNT_TYPE and account.is_superuser: return True return False - + def get_metric(self, instance): if self.metric: safe_locals = self.get_expression_context(instance) @@ -128,7 +128,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): return eval(self.metric, safe_locals) except Exception as exc: raise type(exc)("'%s' evaluating metric for '%s' service" % (exc, self.service)) - + def get_order_description(self, instance): safe_locals = self.get_expression_context(instance) account = getattr(instance, 'account', instance) @@ -136,7 +136,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if not self.order_description: return '%s: %s' % (ugettext(self.description), instance) return eval(self.order_description, safe_locals) - + def get_billing_point(self, order, bp=None, **options): cachable = bool(self.billing_point == self.FIXED_DATE and not options.get('fixed_point')) if not cachable or bp is None: @@ -151,7 +151,10 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): else: date = timezone.now().date() if self.billing_point == self.ON_REGISTER: - day = order.registered_on.day + # handle edge cases of last day of the month: + # e.g. on March is 31 but on April 30 + last_day_of_month = calendar.monthrange(date.year, date.month)[1] + day = min(last_day_of_month, order.registered_on.day) elif self.billing_point == self.FIXED_DATE: day = 1 else: @@ -171,6 +174,11 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): year = bp.year - relativedelta.relativedelta(years=1) if bp.month >= month: year = bp.year + 1 + + # handle edge cases of last day of the month: + # e.g. on March is 31 but on April 30 + last_day_of_month = calendar.monthrange(year,month)[1] + day = min(last_day_of_month, day) bp = datetime.date(year=year, month=month, day=day) elif self.billing_period == self.NEVER: bp = order.registered_on @@ -179,7 +187,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if self.on_cancel != self.NOTHING and order.cancelled_on and order.cancelled_on < bp: bp = order.cancelled_on return bp - + # def aligned(self, date): # if self.granularity == self.DAILY: # return date @@ -188,7 +196,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): # elif self.granularity == self.ANUAL: # return datetime.date(year=date.year, month=1, day=1) # raise NotImplementedError - + def get_price_size(self, ini, end): rdelta = relativedelta.relativedelta(end, ini) anual_prepay_of_monthly_pricing = bool( @@ -211,7 +219,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): raise NotImplementedError size = round(size, 2) return decimal.Decimal(str(size)) - + def get_pricing_slots(self, ini, end): day = 1 month = settings.SERVICES_SERVICE_ANUAL_BILLING_MONTH @@ -235,7 +243,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if next >= end: break ini = next - + def get_pricing_rdelta(self): period = self.get_pricing_period() if period == self.MONTHLY: @@ -244,13 +252,13 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): return relativedelta.relativedelta(years=1) elif period == self.NEVER: return None - + def generate_discount(self, line, dtype, price): line.discounts.append(AttrDict(**{ 'type': dtype, 'total': price, })) - + def generate_line(self, order, price, *dates, metric=1, discounts=None, computed=False): """ discounts: extra discounts to apply @@ -263,7 +271,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): else: raise AttributeError("WTF is '%s'?" % dates) discounts = discounts or () - + size = self.get_price_size(ini, end) if not computed: price = price * size @@ -277,7 +285,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): 'metric': metric, 'discounts': [], }) - + if subtotal > price: plan_discount = price-subtotal self.generate_discount(line, self._PLAN, plan_discount) @@ -290,7 +298,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if dprice: self.generate_discount(line, dtype, dprice) return line - + def assign_compensations(self, givers, receivers, **options): compensations = [] for order in givers: @@ -313,7 +321,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if hasattr(order, 'new_billed_until'): order.billed_until = order.new_billed_until order.save(update_fields=['billed_until']) - + def apply_compensations(self, order, only_beyond=False): dsize = 0 ini = order.billed_until or order.registered_on @@ -339,7 +347,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): new_end = cend dsize += self.get_price_size(comp.ini, cend) return dsize, new_end - + def get_register_or_renew_events(self, porders, ini, end): counter = 0 for order in porders: @@ -354,7 +362,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if registered != order.billed_until and order.billed_until > ini and order.billed_until <= end: counter += 1 return counter - + def bill_concurrent_orders(self, account, porders, rates, ini, end): # Concurrent # Get pricing orders @@ -403,7 +411,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): order, price, ini, end, discounts=discounts, computed=True) lines.append(line) return lines - + def bill_registered_or_renew_events(self, account, porders, rates): # Before registration lines = [] @@ -431,7 +439,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): line = self.generate_line(order, price, ini, end, discounts=discounts) lines.append(line) return lines - + def bill_with_orders(self, orders, account, **options): # For the "boundary conditions" just think that: # date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0) @@ -458,7 +466,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): end = max(end, bp) orders_.append(order) orders = orders_ - + # Compensation related_orders = account.orders.filter(service=self.service) if self.payment_style == self.PREPAY and self.on_cancel == self.COMPENSATE: @@ -504,7 +512,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): line = self.generate_line(order, price, ini, end, discounts=discounts) lines.append(line) return lines - + def bill_with_metric(self, orders, account, **options): lines = [] bp = None @@ -512,7 +520,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): prepay_discount = 0 bp = self.get_billing_point(order, bp=bp, **options) recharged_until = datetime.date.min - + if (self.billing_period != self.NEVER and self.get_pricing_period() == self.NEVER and self.payment_style == self.PREPAY and order.billed_on): @@ -633,7 +641,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): # Last processed metric for futrue recharges order.new_billed_metric = metric return lines - + def generate_bill_lines(self, orders, account, **options): if options.get('proforma', False): options['commit'] = False diff --git a/orchestra/contrib/services/migrations/0001_initial.py b/orchestra/contrib/services/migrations/0001_initial.py index f376f4e3..50f9d1d5 100644 --- a/orchestra/contrib/services/migrations/0001_initial.py +++ b/orchestra/contrib/services/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion class Migration(migrations.Migration): @@ -32,7 +33,7 @@ class Migration(migrations.Migration): ('rate_algorithm', models.CharField(default='MATCH_PRICE', help_text='Algorithm used to interprete the rating table.
  Match price: Only the rate with a) inmediate inferior metric and b) lower price is applied. Nominal price will be used when initial block is missing.
  Step price: All rates with a quantity lower than the metric are applied. Nominal price will be used when initial block is missing.', max_length=16, choices=[('MATCH_PRICE', 'Match price'), ('STEP_PRICE', 'Step price')], verbose_name='rate algorithm')), ('on_cancel', models.CharField(default='DISCOUNT', help_text='Defines the cancellation behaviour of this service.', max_length=16, choices=[('NOTHING', 'Nothing'), ('DISCOUNT', 'Discount'), ('COMPENSATE', 'Compensat'), ('REFUND', 'Refund')], verbose_name='on cancel')), ('payment_style', models.CharField(default='PREPAY', help_text='Designates whether this service should be paid after consumtion (postpay/on demand) or prepaid.', max_length=16, choices=[('PREPAY', 'Prepay'), ('POSTPAY', 'Postpay (on demand)')], verbose_name='payment style')), - ('content_type', models.ForeignKey(help_text='Content type of the related service objects.', to='contenttypes.ContentType', verbose_name='content type')), + ('content_type', models.ForeignKey(help_text='Content type of the related service objects.', on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='content type')), ], ), ] diff --git a/orchestra/contrib/services/migrations/0001_squashed_0015_auto_20210330_1049.py b/orchestra/contrib/services/migrations/0001_squashed_0015_auto_20210330_1049.py new file mode 100644 index 00000000..a09be3dd --- /dev/null +++ b/orchestra/contrib/services/migrations/0001_squashed_0015_auto_20210330_1049.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:26 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('services', '0001_initial'), ('services', '0002_auto_20150509_1501'), ('services', '0003_auto_20150917_0942'), ('services', '0004_auto_20160405_1133'), ('services', '0005_auto_20160427_1531'), ('services', '0006_auto_20170528_2005'), ('services', '0007_auto_20170528_2011'), ('services', '0008_auto_20170625_1813'), ('services', '0009_auto_20170625_1840'), ('services', '0010_auto_20170625_1840'), ('services', '0011_auto_20170625_1840'), ('services', '0012_auto_20170625_1841'), ('services', '0013_auto_20190805_1134'), ('services', '0014_auto_20200204_1218'), ('services', '0015_auto_20210330_1049')] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Service', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.CharField(max_length=256, unique=True, verbose_name='description')), + ('match', models.CharField(blank=True, help_text="Python expression that designates wheter a content_type object is related to this service or not, always evaluates True when left blank. Related instance can be instantiated with instance keyword or content_type.model_name.
 databaseuser.type == 'MYSQL'
 miscellaneous.active and str(miscellaneous.identifier).endswith(('.org', '.net', '.com'))
 contractedplan.plan.name == 'association_fee''
 instance.active", max_length=256, verbose_name='match')), + ('handler_type', models.CharField(blank=True, choices=[('', 'Default')], help_text='Handler used for processing this Service. A handler enables customized behaviour far beyond what options here allow to.', max_length=256, verbose_name='handler')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ('ignore_superusers', models.BooleanField(default=True, help_text='Designates whether superuser, staff and friend orders are marked as ignored by default or not.', verbose_name='ignore superuser, staff and friend')), + ('billing_period', models.CharField(blank=True, choices=[('', 'One time service'), ('MONTHLY', 'Monthly billing'), ('ANUAL', 'Anual billing')], default='ANUAL', help_text='Renewal period for recurring invoicing.', max_length=16, verbose_name='billing period')), + ('billing_point', models.CharField(choices=[('ON_REGISTER', 'Registration date'), ('ON_FIXED_DATE', 'Every January')], default='ON_FIXED_DATE', help_text='Reference point for calculating the renewal date on recurring invoices', max_length=16, verbose_name='billing point')), + ('is_fee', models.BooleanField(default=False, help_text='Designates whether this service should be billed as membership fee or not', verbose_name='fee')), + ('order_description', models.CharField(blank=True, help_text="Python expression used for generating the description for the bill lines of this services.
Defaults to '%s: %s' % (ugettext(handler.description), instance)", max_length=256, verbose_name='Order description')), + ('ignore_period', models.CharField(blank=True, choices=[('', 'Never'), ('ONE_DAY', 'One day'), ('TWO_DAYS', 'Two days'), ('TEN_DAYS', 'Ten days'), ('ONE_MONTH', 'One month')], default='TEN_DAYS', help_text='Period in which orders will be ignored if cancelled. Useful for designating trial periods', max_length=16, verbose_name='ignore period')), + ('metric', models.CharField(blank=True, help_text="Python expression used for obtinging the metric value for the pricing rate computation. Number of orders is used when left blank. Related instance can be instantiated with instance keyword or content_type.model_name.
 max((mailbox.resources.disk.allocated or 0) -1, 0)
 miscellaneous.amount
 max((account.resources.traffic.used or 0) - getattr(account.miscellaneous.filter(is_active=True, service__name='traffic-prepay').last(), 'amount', 0), 0)", max_length=256, verbose_name='metric')), + ('nominal_price', models.DecimalField(decimal_places=2, max_digits=12, verbose_name='nominal price')), + ('tax', models.PositiveIntegerField(choices=[(0, 'Duty free'), (21, '21%')], default=0, verbose_name='tax')), + ('pricing_period', models.CharField(blank=True, choices=[('', 'Current value'), ('BILLING_PERIOD', 'Same as billing period'), ('MONTHLY', 'Monthly data'), ('ANUAL', 'Anual data')], default='BILLING_PERIOD', help_text='Time period that is used for computing the rate metric.', max_length=16, verbose_name='pricing period')), + ('rate_algorithm', models.CharField(choices=[('orchestra.contrib.plans.ratings.step_price', 'Step price'), ('orchestra.contrib.plans.ratings.match_price', 'Match price'), ('orchestra.contrib.plans.ratings.best_price', 'Best price')], default='orchestra.contrib.plans.ratings.step_price', help_text='Algorithm used to interprete the rating table.
  Step price: All rates with a quantity lower or equal than the metric are applied. Nominal price will be used when initial block is missing.
  Match price: Only the rate with a) inmediate inferior metric and b) lower price is applied. Nominal price will be used when initial block is missing.
  Best price: Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).', max_length=64, verbose_name='rate algorithm')), + ('on_cancel', models.CharField(choices=[('NOTHING', 'Nothing'), ('DISCOUNT', 'Discount'), ('COMPENSATE', 'Compensat'), ('REFUND', 'Refund')], default='DISCOUNT', help_text='Defines the cancellation behaviour of this service.', max_length=16, verbose_name='on cancel')), + ('payment_style', models.CharField(choices=[('PREPAY', 'Prepay'), ('POSTPAY', 'Postpay (on demand)')], default='PREPAY', help_text='Designates whether this service should be paid after consumtion (postpay/on demand) or prepaid.', max_length=16, verbose_name='payment style')), + ('content_type', models.ForeignKey(help_text='Content type of the related service objects.', on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='content type')), + ('periodic_update', models.BooleanField(default=False, help_text='Whether a periodic update of this service orders should be performed or not. Needed for match definitions that depend on complex model interactions, where content type model save and delete operations are not enought.', verbose_name='periodic update')), + ], + ), + ] diff --git a/orchestra/contrib/services/migrations/0015_auto_20210330_1049.py b/orchestra/contrib/services/migrations/0015_auto_20210330_1049.py new file mode 100644 index 00000000..c1402f79 --- /dev/null +++ b/orchestra/contrib/services/migrations/0015_auto_20210330_1049.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('services', '0014_auto_20200204_1218'), + ] + + operations = [ + migrations.AlterField( + model_name='service', + name='billing_point', + field=models.CharField(choices=[('ON_REGISTER', 'Registration date'), ('ON_FIXED_DATE', 'Every January')], default='ON_FIXED_DATE', help_text='Reference point for calculating the renewal date on recurring invoices', max_length=16, verbose_name='billing point'), + ), + migrations.AlterField( + model_name='service', + name='rate_algorithm', + field=models.CharField(choices=[('orchestra.contrib.plans.ratings.step_price', 'Step price'), ('orchestra.contrib.plans.ratings.match_price', 'Match price'), ('orchestra.contrib.plans.ratings.best_price', 'Best price')], default='orchestra.contrib.plans.ratings.step_price', help_text='Algorithm used to interprete the rating table.
  Step price: All rates with a quantity lower or equal than the metric are applied. Nominal price will be used when initial block is missing.
  Match price: Only the rate with a) inmediate inferior metric and b) lower price is applied. Nominal price will be used when initial block is missing.
  Best price: Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).', max_length=64, verbose_name='rate algorithm'), + ), + migrations.AlterField( + model_name='service', + name='tax', + field=models.PositiveIntegerField(choices=[(0, 'Duty free'), (21, '21%')], default=0, verbose_name='tax'), + ), + ] diff --git a/orchestra/contrib/services/models.py b/orchestra/contrib/services/models.py index 1d3377e7..961c5c6b 100644 --- a/orchestra/contrib/services/models.py +++ b/orchestra/contrib/services/models.py @@ -53,12 +53,13 @@ class Service(models.Model): REFUND = 'REFUND' PREPAY = 'PREPAY' POSTPAY = 'POSTPAY' - + _ignore_types = ' and '.join( ', '.join(settings.SERVICES_IGNORE_ACCOUNT_TYPE).rsplit(', ', 1)).lower() - + description = models.CharField(_("description"), max_length=256, unique=True) - content_type = models.ForeignKey(ContentType, verbose_name=_("content type"), + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, + verbose_name=_("content type"), help_text=_("Content type of the related service objects.")) match = models.CharField(_("match"), max_length=256, blank=True, help_text=_( @@ -168,19 +169,19 @@ class Service(models.Model): (POSTPAY, _("Postpay (on demand)")), ), default=PREPAY) - + objects = ServiceQuerySet.as_manager() - + def __str__(self): return self.description - + @cached_property def handler(self): """ Accessor of this service handler instance """ if self.handler_type: return ServiceHandler.get(self.handler_type)(self) return ServiceHandler(self) - + def clean(self): self.description = self.description.strip() if hasattr(self, 'content_type'): @@ -190,12 +191,12 @@ class Service(models.Model): 'metric': (self.handler.validate_metric, self), 'order_description': (self.handler.validate_order_description, self), }) - + def get_pricing_period(self): if self.pricing_period == self.BILLING_PERIOD: return self.billing_period return self.pricing_period - + def get_price(self, account, metric, rates=None, position=None): """ if position is provided an specific price for that position is returned, @@ -233,7 +234,7 @@ class Service(models.Model): price = round(rate['price'], 2) return decimal.Decimal(str(rate['price'])) raise RuntimeError("Rating algorithm bad result") - + def get_rates(self, account, cache=True): # rates are cached per account if not cache: @@ -246,11 +247,11 @@ class Service(models.Model): rates = self.rates.by_account(account) self.__cached_rates[account.id] = rates return rates - + @property def rate_method(self): return rate_class.get_methods()[self.rate_algorithm] - + def update_orders(self, commit=True): order_model = apps.get_model(settings.SERVICES_ORDER_MODEL) manager = order_model.objects diff --git a/orchestra/contrib/services/tests/functional_tests/test_domain.py b/orchestra/contrib/services/tests/functional_tests/test_domain.py index 423a166e..a45c3ec2 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_domain.py +++ b/orchestra/contrib/services/tests/functional_tests/test_domain.py @@ -44,95 +44,95 @@ class DomainBillingTest(BaseTestCase): account = self.create_account() self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(0, bills[0].get_total()) + self.assertEqual(0, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(20, bills[0].get_total()) + self.assertEqual(20, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(29, bills[0].get_total()) + self.assertEqual(29, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(38, bills[0].get_total()) + self.assertEqual(38, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(44, bills[0].get_total()) + self.assertEqual(44, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(50, bills[0].get_total()) + self.assertEqual(50, bills[0].total) self.create_domain(account=account) bills = account.orders.bill() - self.assertEqual(56, bills[0].get_total()) + self.assertEqual(56, bills[0].total) def test_domain_proforma(self): self.create_domain_service() account = self.create_account() self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(0, bills[0].get_total()) + self.assertEqual(0, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(20, bills[0].get_total()) + self.assertEqual(20, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(29, bills[0].get_total()) + self.assertEqual(29, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(38, bills[0].get_total()) + self.assertEqual(38, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(44, bills[0].get_total()) + self.assertEqual(44, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(50, bills[0].get_total()) + self.assertEqual(50, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True, new_open=True) - self.assertEqual(56, bills[0].get_total()) + self.assertEqual(56, bills[0].total) def test_domain_cumulative(self): self.create_domain_service() account = self.create_account() self.create_domain(account=account) bills = account.orders.bill(proforma=True) - self.assertEqual(0, bills[0].get_total()) + self.assertEqual(0, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True) - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(proforma=True) - self.assertEqual(30, bills[0].get_total()) + self.assertEqual(30, bills[0].total) def test_domain_new_open(self): self.create_domain_service() account = self.create_account() self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(0, bills[0].get_total()) + self.assertEqual(0, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(9, bills[0].get_total()) + self.assertEqual(9, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(9, bills[0].get_total()) + self.assertEqual(9, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(6, bills[0].get_total()) + self.assertEqual(6, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(6, bills[0].get_total()) + self.assertEqual(6, bills[0].total) self.create_domain(account=account) bills = account.orders.bill(new_open=True) - self.assertEqual(6, bills[0].get_total()) + self.assertEqual(6, bills[0].total) diff --git a/orchestra/contrib/services/tests/functional_tests/test_ftp.py b/orchestra/contrib/services/tests/functional_tests/test_ftp.py index a7fdc81b..03f51be6 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_ftp.py +++ b/orchestra/contrib/services/tests/functional_tests/test_ftp.py @@ -48,21 +48,21 @@ class FTPBillingTest(BaseTestCase): self.assertEqual(1, service.orders.count()) bp = timezone.now().date() + relativedelta(years=1) bills = service.orders.bill(billing_point=bp, fixed_point=True) - self.assertEqual(10, bills[0].get_total()) + self.assertEqual(10, bills[0].total) def test_ftp_account_2_year_fiexed(self): service = self.create_ftp_service() self.create_ftp() bp = timezone.now().date() + relativedelta(years=2) bills = service.orders.bill(billing_point=bp, fixed_point=True) - self.assertEqual(20, bills[0].get_total()) + self.assertEqual(20, bills[0].total) def test_ftp_account_6_month_fixed(self): service = self.create_ftp_service() self.create_ftp() bp = timezone.now().date() + relativedelta(months=6) bills = service.orders.bill(billing_point=bp, fixed_point=True) - self.assertEqual(5, bills[0].get_total()) + self.assertEqual(5, bills[0].total) def test_ftp_account_next_billing_point(self): service = self.create_ftp_service() @@ -76,8 +76,8 @@ class FTPBillingTest(BaseTestCase): bills = service.orders.bill(billing_point=now, fixed_point=False) size = decimal.Decimal((bp - now).days)/365 error = decimal.Decimal(0.05) - self.assertGreater(10*size+error*(10*size), bills[0].get_total()) - self.assertLess(10*size-error*(10*size), bills[0].get_total()) + self.assertGreater(10*size+error*(10*size), bills[0].total) + self.assertLess(10*size-error*(10*size), bills[0].total) def test_ftp_account_with_compensation(self): account = self.create_account() @@ -99,4 +99,4 @@ class FTPBillingTest(BaseTestCase): self.assertEqual(order.cancelled_on, order.billed_until) order = account.orders.order_by('-id').first() self.assertEqual(first_bp, order.billed_until) - self.assertEqual(decimal.Decimal(0), bills[0].get_total()) + self.assertEqual(decimal.Decimal(0), bills[0].total) diff --git a/orchestra/contrib/services/tests/functional_tests/test_job.py b/orchestra/contrib/services/tests/functional_tests/test_job.py index db8b23e4..4e81d7ed 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_job.py +++ b/orchestra/contrib/services/tests/functional_tests/test_job.py @@ -42,8 +42,8 @@ class JobBillingTest(BaseTestCase): self.create_job(5, account=account) bill = account.orders.bill()[0] - self.assertEqual(5*20, bill.get_total()) + self.assertEqual(5*20, bill.total) self.create_job(100, account=account) bill = account.orders.bill(new_open=True)[0] - self.assertEqual(100*15, bill.get_total()) + self.assertEqual(100*15, bill.total) diff --git a/orchestra/contrib/services/tests/functional_tests/test_mailbox.py b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py index 32bd8afc..5756cf62 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_mailbox.py +++ b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py @@ -85,10 +85,10 @@ class MailboxBillingTest(BaseTestCase): mailbox = self.create_mailbox(account=account) self.allocate_disk(mailbox, 10) bill = service.orders.bill()[0] - self.assertEqual(0, bill.get_total()) + self.assertEqual(0, bill.total) bp = timezone.now().date() + relativedelta(years=1) bill = disk_service.orders.bill(billing_point=bp, fixed_point=True)[0] - self.assertEqual(90, bill.get_total()) + self.assertEqual(90, bill.total) mailbox = self.create_mailbox(account=account) mailbox = self.create_mailbox(account=account) mailbox = self.create_mailbox(account=account) @@ -96,7 +96,7 @@ class MailboxBillingTest(BaseTestCase): mailbox = self.create_mailbox(account=account) mailbox = self.create_mailbox(account=account) bill = service.orders.bill(billing_point=bp, fixed_point=True)[0] - self.assertEqual(120, bill.get_total()) + self.assertEqual(120, bill.total) def test_mailbox_size_with_changes(self): service = self.create_mailbox_disk_service() @@ -109,25 +109,25 @@ class MailboxBillingTest(BaseTestCase): self.allocate_disk(mailbox, 10) bill = service.orders.bill(**options).pop() - self.assertEqual(9*10, bill.get_total()) + self.assertEqual(9*10, bill.total) with freeze_time(now+relativedelta(months=6)): self.allocate_disk(mailbox, 20) bill = service.orders.bill(**options).pop() total = 9*10*0.5 + 19*10*0.5 - self.assertEqual(total, bill.get_total()) + self.assertEqual(total, bill.total) with freeze_time(now+relativedelta(months=9)): self.allocate_disk(mailbox, 30) bill = service.orders.bill(**options).pop() total = 9*10*0.5 + 19*10*0.25 + 29*10*0.25 - self.assertEqual(total, bill.get_total()) + self.assertEqual(total, bill.total) with freeze_time(now+relativedelta(years=1)): self.allocate_disk(mailbox, 10) bill = service.orders.bill(**options).pop() total = 9*10*0.5 + 19*10*0.25 + 29*10*0.25 - self.assertEqual(total, bill.get_total()) + self.assertEqual(total, bill.total) def test_mailbox_with_recharge(self): service = self.create_mailbox_disk_service() @@ -140,8 +140,12 @@ class MailboxBillingTest(BaseTestCase): self.allocate_disk(mailbox, 100) bill = service.orders.bill(**options).pop() - self.assertEqual(99*10, bill.get_total()) + self.assertEqual(99*10, bill.total) + with freeze_time(now+relativedelta(months=6)): + bills = service.orders.bill(new_open=True, **options) + self.assertEqual([], bills) + with freeze_time(now+relativedelta(months=6)): self.allocate_disk(mailbox, 50) bills = service.orders.bill(**options) @@ -150,11 +154,8 @@ class MailboxBillingTest(BaseTestCase): with freeze_time(now+relativedelta(months=6)): self.allocate_disk(mailbox, 200) bill = service.orders.bill(new_open=True, **options).pop() - self.assertEqual((199-99)*10*0.5, bill.get_total()) + self.assertEqual((199-99)*10*0.5, bill.total) - with freeze_time(now+relativedelta(months=6)): - bills = service.orders.bill(new_open=True, **options) - self.assertEqual([], bills) def test_mailbox_second_billing(self): service = self.create_mailbox_disk_service() diff --git a/orchestra/contrib/services/tests/functional_tests/test_traffic.py b/orchestra/contrib/services/tests/functional_tests/test_traffic.py index 1edbcad3..e7334de9 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_traffic.py +++ b/orchestra/contrib/services/tests/functional_tests/test_traffic.py @@ -52,7 +52,7 @@ class BaseTrafficBillingTest(BaseTestCase): scale='10**9', on_demand=True, # TODO - monitors=FTPTrafficMonitor.get_name(), + monitors=[FTPTrafficMonitor.get_name()], ) return self.resource @@ -77,11 +77,11 @@ class TrafficBillingTest(BaseTrafficBillingTest): with freeze_time(now+relativedelta(months=1)): bill = account.orders.bill(proforma=True)[0] self.report_traffic(account, 10**10*9) - self.assertEqual(0, bill.get_total()) + self.assertEqual(0, bill.total) with freeze_time(now+relativedelta(months=3)): bill = account.orders.bill(proforma=True)[0] - self.assertEqual((90-10)*10, bill.get_total()) + self.assertEqual((90-10)*10, bill.total) def test_multiple_traffics(self): self.create_traffic_service() @@ -93,7 +93,7 @@ class TrafficBillingTest(BaseTrafficBillingTest): with freeze_time(timezone.now()+relativedelta(months=1)): bill1 = account1.orders.bill().pop() bill2 = account2.orders.bill().pop() - self.assertNotEqual(bill1.get_total(), bill2.get_total()) + self.assertNotEqual(bill1.total, bill2.total) class TrafficPrepayBillingTest(BaseTrafficBillingTest): @@ -139,31 +139,31 @@ class TrafficPrepayBillingTest(BaseTrafficBillingTest): self.create_prepay(10, account=account) bill = account.orders.bill(proforma=True)[0] - self.assertEqual(10*50, bill.get_total()) + self.assertEqual(10*50, bill.total) self.report_traffic(account, 10**10) with freeze_time(now+relativedelta(months=1)): bill = account.orders.bill(proforma=True, new_open=True)[0] - self.assertEqual(2*10*50 + 0*10, bill.get_total()) + self.assertEqual(2*10*50 + 0*10, bill.total) # TODO RuntimeWarning: DateTimeField MetricStorage.updated_on received a naive self.report_traffic(account, 10**10) with freeze_time(now+relativedelta(months=1)): bill = account.orders.bill(proforma=True, new_open=True)[0] - self.assertEqual(2*10*50 + 0*10, bill.get_total()) + self.assertEqual(2*10*50 + 0*10, bill.total) self.report_traffic(account, 10**10) with freeze_time(now+relativedelta(months=1)): bill = account.orders.bill(proforma=True, new_open=True)[0] - self.assertEqual(2*10*50 + (30-10-10)*10, bill.get_total()) + self.assertEqual(2*10*50 + (30-10-10)*10, bill.total) with freeze_time(now+relativedelta(months=2)): self.report_traffic(account, 10**11) with freeze_time(now+relativedelta(months=1)): bill = account.orders.bill(proforma=True, new_open=True)[0] - self.assertEqual(2*10*50 + (30-10-10)*10, bill.get_total()) + self.assertEqual(2*10*50 + (30-10-10)*10, bill.total) with freeze_time(now+relativedelta(months=3)): bill = account.orders.bill(proforma=True, new_open=True)[0] - self.assertEqual(4*10*50 + (30-10-10)*10 + (100-10-10)*10, bill.get_total()) + self.assertEqual(4*10*50 + (30-10-10)*10 + (100-10-10)*10, bill.total) diff --git a/orchestra/contrib/services/tests/test_handler.py b/orchestra/contrib/services/tests/test_handler.py index d6bfad81..17d5d154 100644 --- a/orchestra/contrib/services/tests/test_handler.py +++ b/orchestra/contrib/services/tests/test_handler.py @@ -225,7 +225,7 @@ class HandlerTests(BaseTestCase): account = self.create_account() superplan = Plan.objects.create( name='SUPER', allow_multiple=False, is_combinable=True) - service.rates.create(plan=superplan, quantity=1, price=0) + service.rates.create(plan=superplan, quantity=0, price=0) service.rates.create(plan=superplan, quantity=3, price=10) service.rates.create(plan=superplan, quantity=4, price=9) service.rates.create(plan=superplan, quantity=10, price=1) @@ -269,7 +269,7 @@ class HandlerTests(BaseTestCase): hyperplan = Plan.objects.create( name='HYPER', allow_multiple=False, is_combinable=False) - service.rates.create(plan=hyperplan, quantity=1, price=0) + service.rates.create(plan=hyperplan, quantity=0, price=0) service.rates.create(plan=hyperplan, quantity=20, price=5) account.plans.create(plan=hyperplan) results = service.get_rates(account, cache=False) @@ -362,7 +362,7 @@ class HandlerTests(BaseTestCase): dupeplan = Plan.objects.create( name='DUPE', allow_multiple=True, is_combinable=True) account.plans.create(plan=dupeplan) - service.rates.create(plan=dupeplan, quantity=1, price=0) + service.rates.create(plan=dupeplan, quantity=0, price=0) service.rates.create(plan=dupeplan, quantity=3, price=9) results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) diff --git a/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413.py b/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413.py index d71721e9..663a7d9b 100644 --- a/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413.py +++ b/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.core.validators from django.conf import settings @@ -17,7 +18,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='systemuser', name='account', - field=models.ForeignKey(related_name='systemusers', to=settings.AUTH_USER_MODEL, default=1, verbose_name='Account'), + field=models.ForeignKey(related_name='systemusers', to=settings.AUTH_USER_MODEL, default=1, on_delete=django.db.models.deletion.CASCADE, verbose_name='Account'), preserve_default=False, ), ] diff --git a/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413_squashed_0004_auto_20210330_1049.py b/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413_squashed_0004_auto_20210330_1049.py new file mode 100644 index 00000000..b311dca3 --- /dev/null +++ b/orchestra/contrib/systemusers/migrations/0002_auto_20150429_1413_squashed_0004_auto_20210330_1049.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:28 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('systemusers', '0002_auto_20150429_1413'), ('systemusers', '0003_auto_20170528_2011'), ('systemusers', '0004_auto_20210330_1049')] + + dependencies = [ + ('systemusers', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='systemuser', + name='account', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='systemusers', to=settings.AUTH_USER_MODEL, verbose_name='Account'), + preserve_default=False, + ), + migrations.AlterField( + model_name='systemuser', + name='shell', + field=models.CharField(choices=[('/dev/null', 'No shell, FTP only'), ('/bin/rssh', 'No shell, SFTP/RSYNC only'), ('/usr/bin/git-shell', 'No shell, GIT only'), ('/bin/bash', '/bin/bash')], default='/dev/null', max_length=32, verbose_name='shell'), + ), + migrations.AlterField( + model_name='systemuser', + name='username', + field=models.CharField(help_text='Required. 32 characters or fewer. Letters, digits and ./-/_ only.', max_length=32, unique=True, validators=[orchestra.core.validators.validate_username], verbose_name='username'), + ), + migrations.AlterField( + model_name='systemuser', + name='shell', + field=models.CharField(choices=[('/dev/null', 'No shell, FTP only'), ('/bin/rssh', 'No shell, SFTP/RSYNC only'), ('/bin/bash', '/bin/bash')], default='/dev/null', max_length=32, verbose_name='shell'), + ), + ] diff --git a/orchestra/contrib/systemusers/migrations/0004_auto_20210330_1049.py b/orchestra/contrib/systemusers/migrations/0004_auto_20210330_1049.py new file mode 100644 index 00000000..d495e8bd --- /dev/null +++ b/orchestra/contrib/systemusers/migrations/0004_auto_20210330_1049.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('systemusers', '0003_auto_20170528_2011'), + ] + + operations = [ + migrations.AlterField( + model_name='systemuser', + name='shell', + field=models.CharField(choices=[('/dev/null', 'No shell, FTP only'), ('/bin/rssh', 'No shell, SFTP/RSYNC only'), ('/bin/bash', '/bin/bash')], default='/dev/null', max_length=32, verbose_name='shell'), + ), + ] diff --git a/orchestra/contrib/systemusers/models.py b/orchestra/contrib/systemusers/models.py index de60a698..92576646 100644 --- a/orchestra/contrib/systemusers/models.py +++ b/orchestra/contrib/systemusers/models.py @@ -19,7 +19,7 @@ class SystemUserQuerySet(models.QuerySet): user.set_password(password) user.save(update_fields=['password']) return user - + def by_is_main(self, is_main=True, **kwargs): if is_main: return self.filter(account__main_systemuser_id=F('id')) @@ -30,7 +30,7 @@ class SystemUserQuerySet(models.QuerySet): class SystemUser(models.Model): """ System users - + Username max_length determined by LINUX system user/group lentgh: 32 """ username = models.CharField(_("username"), max_length=32, unique=True, @@ -38,7 +38,7 @@ class SystemUser(models.Model): validators=[validators.validate_username]) password = models.CharField(_("password"), max_length=128) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='systemusers') + related_name='systemusers', on_delete=models.CASCADE) home = models.CharField(_("home"), max_length=256, blank=True, help_text=_("Starting location when login with this no-shell user.")) directory = models.CharField(_("directory"), max_length=256, blank=True, @@ -51,19 +51,19 @@ class SystemUser(models.Model): is_active = models.BooleanField(_("active"), default=True, help_text=_("Designates whether this account should be treated as active. " "Unselect this instead of deleting accounts.")) - + objects = SystemUserQuerySet.as_manager() - + def __str__(self): return self.username - + @cached_property def active(self): try: return self.is_active and self.account.is_active - except type(self).account.field.rel.to.DoesNotExist: + except type(self).account.field.model.DoesNotExist: return self.is_active - + @cached_property def is_main(self): # TODO on account delete @@ -71,34 +71,34 @@ class SystemUser(models.Model): if self.account.main_systemuser_id: return self.account.main_systemuser_id == self.pk return self.account.username == self.username - + @cached_property def main(self): # On account creation main_systemuser_id is still None if self.account.main_systemuser_id: return self.account.main_systemuser return type(self).objects.get(username=self.account.username) - + @property def has_shell(self): return self.shell not in settings.SYSTEMUSERS_DISABLED_SHELLS - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = True self.save(update_fields=('is_active',)) - + def get_description(self): return self.get_shell_display() - + def save(self, *args, **kwargs): if not self.home: self.home = self.get_base_home() super(SystemUser, self).save(*args, **kwargs) - + def clean(self): self.directory = self.directory.lstrip('/') if self.home: @@ -123,16 +123,16 @@ class SystemUser(models.Model): raise ValidationError({ 'home': _("Shell users should use their own home."), }) - + def set_password(self, raw_password): self.password = make_password(raw_password) - + def get_base_home(self): context = { 'user': self.username, 'username': self.username, } return os.path.normpath(settings.SYSTEMUSERS_HOME % context) - + def get_home(self): return os.path.normpath(os.path.join(self.home, self.directory)) diff --git a/orchestra/contrib/systemusers/tests/functional_tests/tests.py b/orchestra/contrib/systemusers/tests/functional_tests/tests.py index 681326bf..9d3cf2e0 100644 --- a/orchestra/contrib/systemusers/tests/functional_tests/tests.py +++ b/orchestra/contrib/systemusers/tests/functional_tests/tests.py @@ -2,12 +2,13 @@ import ftplib import os import re import time +import unittest from functools import partial import paramiko from django.conf import settings as djsettings from django.core.management.base import CommandError -from django.core.urlresolvers import reverse +from django.urls import reverse from selenium.webdriver.support.select import Select from orchestra.admin.utils import change_url @@ -21,6 +22,7 @@ from ... import backends from ...models import SystemUser +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) r = partial(run, silent=True, display=False) sshr = partial(sshrun, silent=True, display=False) @@ -31,35 +33,35 @@ class SystemUserMixin(object): 'orchestra.contrib.orchestration', 'orcgestra.apps.systemusers', ) - + def setUp(self): super(SystemUserMixin, self).setUp() self.add_route() djsettings.DEBUG = True - + def add_route(self): master = Server.objects.create(name=self.MASTER_SERVER) - backend = backends.SystemUserBackend.get_name() + backend = backends.UNIXUserController.get_name() Route.objects.create(backend=backend, match=True, host=master) - + def save(self): raise NotImplementedError - + def add(self): raise NotImplementedError - + def delete(self): raise NotImplementedError - + def update(self): raise NotImplementedError - + def disable(self): raise NotImplementedError - + def add_group(self, username, groupname): raise NotImplementedError - + def validate_user(self, username): idcmd = sshr(self.MASTER_SERVER, "id %s" % username) self.assertEqual(0, idcmd.exit_code) @@ -69,7 +71,7 @@ class SystemUserMixin(object): idgroups = idcmd.stdout.strip().split(' ')[2] idgroups = re.findall(r'\d+\((\w+)\)', idgroups) self.assertEqual(set(groups), set(idgroups)) - + def validate_delete(self, username): self.assertRaises(SystemUser.DoesNotExist, SystemUser.objects.get, username=username) self.assertRaises(CommandError, @@ -81,19 +83,19 @@ class SystemUserMixin(object): self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/shadow' % username, display=False) # Home will be deleted on account delete, see test_delete_account - + def validate_ftp(self, username, password): ftp = ftplib.FTP(self.MASTER_SERVER) ftp.login(user=username, passwd=password) ftp.close() - + def validate_sftp(self, username, password): transport = paramiko.Transport((self.MASTER_SERVER, 22)) transport.connect(username=username, password=password) sftp = paramiko.SFTPClient.from_transport(transport) sftp.listdir() sftp.close() - + def validate_ssh(self, username, password): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -103,14 +105,14 @@ class SystemUserMixin(object): channel.exec_command('ls') self.assertEqual(0, channel.recv_exit_status()) channel.close() - + def test_add(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) self.add(username, password) self.addCleanup(self.delete, username) self.validate_user(username) - + def test_ftp(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -120,7 +122,7 @@ class SystemUserMixin(object): self.validate_sftp, username, password) self.assertRaises(paramiko.AuthenticationException, self.validate_ssh, username, password) - + def test_sftp(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -128,14 +130,14 @@ class SystemUserMixin(object): self.addCleanup(self.delete, username) self.validate_sftp(username, password) self.assertRaises(AssertionError, self.validate_ssh, username, password) - + def test_ssh(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) self.add(username, password, shell='/bin/bash') self.addCleanup(self.delete, username) self.validate_ssh(username, password) - + def test_delete(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%sppppP001' % random_ascii(5) @@ -144,7 +146,7 @@ class SystemUserMixin(object): self.delete(username) self.validate_delete(username) self.assertRaises(Exception, self.delete, self.account.username) - + def test_add_group(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -161,7 +163,7 @@ class SystemUserMixin(object): groups = list(user.groups.values_list('username', flat=True)) self.assertIn(username2, groups) self.validate_user(username) - + def test_disable(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -171,7 +173,7 @@ class SystemUserMixin(object): self.disable(username) self.validate_user(username) self.assertRaises(ftplib.error_perm, self.validate_ftp, username, password) - + def test_change_password(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) @@ -185,6 +187,7 @@ class SystemUserMixin(object): # TODO test resources +@unittest.skipUnless(TEST_REST_API, "REST API tests") class RESTSystemUserMixin(SystemUserMixin): def setUp(self): super(RESTSystemUserMixin, self).setUp() @@ -192,38 +195,38 @@ class RESTSystemUserMixin(SystemUserMixin): # create main user self.save(self.account.username) self.addCleanup(self.delete_account, self.account.username) - + @save_response_on_error def add(self, username, password, shell='/dev/null'): self.rest.systemusers.create(username=username, password=password, shell=shell) - + @save_response_on_error def delete(self, username): user = self.rest.systemusers.retrieve(username=username).get() user.delete() - + @save_response_on_error def add_group(self, username, groupname): user = self.rest.systemusers.retrieve(username=username).get() user.groups.append({'username': groupname}) user.save() - + @save_response_on_error def disable(self, username): user = self.rest.systemusers.retrieve(username=username).get() user.is_active = False user.save() - + @save_response_on_error def save(self, username): user = self.rest.systemusers.retrieve(username=username).get() user.save() - + @save_response_on_error def change_password(self, username, password): user = self.rest.systemusers.retrieve(username=username).get() user.set_password(password) - + def delete_account(self, username): self.rest.account.delete() @@ -235,42 +238,42 @@ class AdminSystemUserMixin(SystemUserMixin): # create main user self.save(self.account.username) self.addCleanup(self.delete_account, self.account.username) - + @snapshot_on_error def add(self, username, password, shell='/dev/null'): url = self.live_server_url + reverse('admin:systemusers_systemuser_add') self.selenium.get(url) - + username_field = self.selenium.find_element_by_id('id_username') username_field.send_keys(username) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) - + shell_input = self.selenium.find_element_by_id('id_shell') shell_select = Select(shell_input) shell_select.select_by_value(shell) - + username_field.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def delete(self, username): user = SystemUser.objects.get(username=username) self.admin_delete(user) - + @snapshot_on_error def delete_account(self, username): account = Account.objects.get(username=username) self.admin_delete(account) - + @snapshot_on_error def disable(self, username): user = SystemUser.objects.get(username=username) self.admin_disable(user) - + @snapshot_on_error def add_group(self, username, groupname): user = SystemUser.objects.get(username=username) @@ -282,7 +285,7 @@ class AdminSystemUserMixin(SystemUserMixin): save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def save(self, username): user = SystemUser.objects.get(username=username) @@ -291,7 +294,7 @@ class AdminSystemUserMixin(SystemUserMixin): save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + @snapshot_on_error def change_password(self, username, password): user = SystemUser.objects.get(username=username) @@ -307,37 +310,37 @@ class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): def test_create_account(self): url = self.live_server_url + reverse('admin:accounts_account_add') self.selenium.get(url) - + account_username = '%s_account' % random_ascii(10) username = self.selenium.find_element_by_id('id_username') username.send_keys(account_username) - + account_password = '@!?%spppP001' % random_ascii(5) password = self.selenium.find_element_by_id('id_password1') password.send_keys(account_password) password = self.selenium.find_element_by_id('id_password2') password.send_keys(account_password) - + full_name = random_ascii(10) full_name_field = self.selenium.find_element_by_id('id_full_name') full_name_field.send_keys(full_name) - + account_email = 'orchestra@orchestra.lan' email = self.selenium.find_element_by_id('id_email') email.send_keys(account_email) - + contact_short_name = random_ascii(10) short_name = self.selenium.find_element_by_id('id_contacts-0-short_name') short_name.send_keys(contact_short_name) - + email = self.selenium.find_element_by_id('id_contacts-0-email') email.send_keys(account_email) email.submit() self.assertNotEqual(url, self.selenium.current_url) - + self.addCleanup(self.delete_account, account_username) self.assertEqual(0, sshr(self.MASTER_SERVER, "id %s" % account_username).exit_code) - + @snapshot_on_error def test_delete_account(self): home = self.account.main_systemuser.get_home() @@ -347,7 +350,7 @@ class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): self.account = self.create_account(username=self.account.username, superuser=True) self.selenium.delete_all_cookies() self.admin_login() - + @snapshot_on_error def test_disable_account(self): username = '%s_systemuser' % random_ascii(10) @@ -357,18 +360,18 @@ class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): self.validate_ftp(username, password) self.disable(username) self.validate_user(username) - + disable = reverse('admin:accounts_account_disable', args=(self.account.pk,)) url = self.live_server_url + disable self.selenium.get(url) confirmation = self.selenium.find_element_by_name('post') confirmation.submit() self.assertNotEqual(url, self.selenium.current_url) - + self.assertRaises(ftplib.error_perm, self.validate_ftp, username, password) self.selenium.get(url) self.assertNotEqual(url, self.selenium.current_url) - + # Reenable for test cleanup self.account.is_active = True self.account.save() diff --git a/orchestra/contrib/tasks/beat.py b/orchestra/contrib/tasks/beat.py index 81732b6d..7a4772ac 100644 --- a/orchestra/contrib/tasks/beat.py +++ b/orchestra/contrib/tasks/beat.py @@ -23,11 +23,11 @@ def is_due(task, time=None): ) -def run_task(task, thread=True, process=False, async=False): +def run_task(task, thread=True, process=False, run_async=False): args = json.loads(task.args) kwargs = json.loads(task.kwargs) task_fn = current_app.tasks.get(task.task) - if async: + if run_async: method = 'process' if process else 'thread' return apply_async(task_fn, method=method).apply_async(*args, **kwargs) return task_fn(*args, **kwargs) @@ -38,6 +38,6 @@ def run(): procs = [] for task in PeriodicTask.objects.enabled().select_related('crontab'): if is_due(task, now): - proc = run_task(task, process=True, async=True) + proc = run_task(task, process=True, run_async=True) procs.append(proc) [proc.join() for proc in procs] diff --git a/orchestra/contrib/tasks/utils.py b/orchestra/contrib/tasks/utils.py index 19972696..96b5bf0b 100644 --- a/orchestra/contrib/tasks/utils.py +++ b/orchestra/contrib/tasks/utils.py @@ -13,7 +13,7 @@ def get_name(fn): def run(method, *args, **kwargs): - async = kwargs.pop('async', True) + run_async = kwargs.pop('run_async', True) thread = threading.Thread(target=close_connection(method), args=args, kwargs=kwargs) thread = Process(target=close_connection(counter)) thread.start() diff --git a/orchestra/contrib/vps/migrations/0001_initial.py b/orchestra/contrib/vps/migrations/0001_initial.py index 21a3638f..aa1e2133 100644 --- a/orchestra/contrib/vps/migrations/0001_initial.py +++ b/orchestra/contrib/vps/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.core.validators from django.conf import settings @@ -21,7 +22,7 @@ class Migration(migrations.Migration): ('type', models.CharField(choices=[('openvz', 'OpenVZ container')], verbose_name='type', default='openvz', max_length=64)), ('template', models.CharField(choices=[('debian7', 'Debian 7 - Wheezy')], verbose_name='template', default='debian7', max_length=64)), ('password', models.CharField(verbose_name='password', help_text='root password of this virtual machine', max_length=128)), - ('account', models.ForeignKey(verbose_name='Account', related_name='vpss', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='Account', related_name='vpss', to=settings.AUTH_USER_MODEL)), ], options={ 'verbose_name': 'VPS', diff --git a/orchestra/contrib/vps/migrations/0001_squashed_0004_auto_20170528_2005.py b/orchestra/contrib/vps/migrations/0001_squashed_0004_auto_20170528_2005.py new file mode 100644 index 00000000..1284ea89 --- /dev/null +++ b/orchestra/contrib/vps/migrations/0001_squashed_0004_auto_20170528_2005.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:26 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('vps', '0001_initial'), ('vps', '0002_auto_20150804_1524'), ('vps', '0003_vps_is_active'), ('vps', '0004_auto_20170528_2005')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='VPS', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hostname', models.CharField(max_length=256, unique=True, validators=[orchestra.core.validators.validate_hostname], verbose_name='hostname')), + ('type', models.CharField(choices=[('openvz', 'OpenVZ container'), ('lxc', 'LXC container')], default='lxc', max_length=64, verbose_name='type')), + ('template', models.CharField(choices=[('debian7', 'Debian 7 - Wheezy'), ('placeholder', 'LXC placeholder')], default='placeholder', help_text='Initial template.', max_length=64, verbose_name='template')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='vpss', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ], + options={ + 'verbose_name': 'VPS', + 'verbose_name_plural': 'VPSs', + }, + ), + ] diff --git a/orchestra/contrib/vps/models.py b/orchestra/contrib/vps/models.py index 4c7cc155..fe3c62c9 100644 --- a/orchestra/contrib/vps/models.py +++ b/orchestra/contrib/vps/models.py @@ -15,31 +15,31 @@ class VPS(models.Model): template = models.CharField(_("template"), max_length=64, choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE, help_text=_("Initial template.")) - account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='vpss') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='vpss') is_active = models.BooleanField(_("active"), default=True) - + class Meta: verbose_name = "VPS" verbose_name_plural = "VPSs" - + def __str__(self): return self.hostname - + def set_password(self, raw_password): self.password = make_password(raw_password) - + def get_username(self): return self.hostname - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) - + @property def active(self): return self.is_active and self.account.is_active diff --git a/orchestra/contrib/webapps/admin.py b/orchestra/contrib/webapps/admin.py index 0297ae8b..9d93ee6a 100644 --- a/orchestra/contrib/webapps/admin.py +++ b/orchestra/contrib/webapps/admin.py @@ -1,6 +1,6 @@ from django import forms from django.contrib import admin -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.encoding import force_text from django.utils.translation import ugettext, ugettext_lazy as _ @@ -21,16 +21,16 @@ from .types import AppType class WebAppOptionInline(admin.TabularInline): model = WebAppOption extra = 1 - + OPTIONS_HELP_TEXT = { op.name: force_text(op.help_text) for op in AppOption.get_plugins() } - + class Media: css = { 'all': ('orchestra/css/hide-inline-id.css',) } - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'value': kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) @@ -63,9 +63,9 @@ class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin) plugin_field = 'type' plugin_title = _("Web application type") actions = (list_accounts,) - + display_type = display_plugin_field('type') - + def display_websites(self, webapp): websites = [] for content in webapp.content_set.all(): @@ -83,7 +83,7 @@ class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin) return '
'.join(websites) display_websites.short_description = _("web sites") display_websites.allow_tags = True - + def display_detail(self, webapp): try: return webapp.type_instance.get_detail() @@ -91,11 +91,11 @@ class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin) return "Not available" display_detail.short_description = _("detail") display_detail.allow_tags = True - + # def get_form(self, request, obj=None, **kwargs): # form = super(WebAppAdmin, self).get_form(request, obj, **kwargs) # if obj: -# +# # def formfield_for_dbfield(self, db_field, **kwargs): # """ Make value input widget bigger """ diff --git a/orchestra/contrib/webapps/backends/python.py b/orchestra/contrib/webapps/backends/python.py index 8c7ee35e..fa0b9881 100644 --- a/orchestra/contrib/webapps/backends/python.py +++ b/orchestra/contrib/webapps/backends/python.py @@ -72,7 +72,7 @@ class uWSGIPythonController(WebAppServiceMixin, ServiceController): return context def get_context(self, webapp): - context = super(PHPController, self).get_context(webapp) + context = super(uWSGIPythonController, self).get_context(webapp) options = webapp.get_options() context.update({ 'python_version': webapp.type_instance.get_python_version(), diff --git a/orchestra/contrib/webapps/fields.py b/orchestra/contrib/webapps/fields.py index 7efd52b8..d430a414 100644 --- a/orchestra/contrib/webapps/fields.py +++ b/orchestra/contrib/webapps/fields.py @@ -12,8 +12,7 @@ class VirtualDatabaseRelation(GenericRelation): pks.append(db_id) if not pks: return [] - # TODO renamed to self.remote_field in django 1.8 - return self.rel.to._base_manager.db_manager(using).filter(pk__in=pks) + return self.remote_field.model._base_manager.db_manager(using).filter(pk__in=pks) class VirtualDatabaseUserRelation(GenericRelation): @@ -26,5 +25,4 @@ class VirtualDatabaseUserRelation(GenericRelation): pks.append(db_id) if not pks: return [] - # TODO renamed to self.remote_field in django 1.8 - return self.rel.to._base_manager.db_manager(using).filter(pk__in=pks) + return self.remote_field.model._base_manager.db_manager(using).filter(pk__in=pks) diff --git a/orchestra/contrib/webapps/migrations/0001_initial.py b/orchestra/contrib/webapps/migrations/0001_initial.py index 7f0c5087..54a329d5 100644 --- a/orchestra/contrib/webapps/migrations/0001_initial.py +++ b/orchestra/contrib/webapps/migrations/0001_initial.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import models, migrations +import django.db.models.deletion import orchestra.core.validators import jsonfield.fields from django.conf import settings @@ -21,7 +22,7 @@ class Migration(migrations.Migration): ('name', models.CharField(verbose_name='name', validators=[orchestra.core.validators.validate_name], help_text='The app will be installed in %(home)s/webapps/%(app_name)s', max_length=128)), ('type', models.CharField(verbose_name='type', max_length=32, choices=[('php', 'PHP'), ('python', 'Python'), ('static', 'Static'), ('symbolic-link', 'Symbolic link'), ('webalizer', 'Webalizer'), ('wordpress-php', 'WordPress')])), ('data', jsonfield.fields.JSONField(verbose_name='data', blank=True, help_text='Extra information dependent of each service.', default={})), - ('account', models.ForeignKey(verbose_name='Account', related_name='webapps', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='Account', related_name='webapps', to=settings.AUTH_USER_MODEL)), ], options={ 'verbose_name': 'Web App', @@ -34,7 +35,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', auto_created=True, serialize=False, primary_key=True)), ('name', models.CharField(verbose_name='name', max_length=128, choices=[(None, '-------'), ('FileSystem', [('public-root', 'Public root')]), ('Process', [('timeout', 'Process timeout'), ('processes', 'Number of processes')]), ('PHP', [('enable_functions', 'Enable functions'), ('allow_url_include', 'Allow URL include'), ('allow_url_fopen', 'Allow URL fopen'), ('auto_append_file', 'Auto append file'), ('auto_prepend_file', 'Auto prepend file'), ('date.timezone', 'date.timezone'), ('default_socket_timeout', 'Default socket timeout'), ('display_errors', 'Display errors'), ('extension', 'Extension'), ('magic_quotes_gpc', 'Magic quotes GPC'), ('magic_quotes_runtime', 'Magic quotes runtime'), ('magic_quotes_sybase', 'Magic quotes sybase'), ('max_input_time', 'Max input time'), ('max_input_vars', 'Max input vars'), ('memory_limit', 'Memory limit'), ('mysql.connect_timeout', 'Mysql connect timeout'), ('output_buffering', 'Output buffering'), ('register_globals', 'Register globals'), ('post_max_size', 'Post max size'), ('sendmail_path', 'Sendmail path'), ('session.bug_compat_warn', 'Session bug compat warning'), ('session.auto_start', 'Session auto start'), ('safe_mode', 'Safe mode'), ('suhosin.post.max_vars', 'Suhosin POST max vars'), ('suhosin.get.max_vars', 'Suhosin GET max vars'), ('suhosin.request.max_vars', 'Suhosin request max vars'), ('suhosin.session.encrypt', 'Suhosin session encrypt'), ('suhosin.simulation', 'Suhosin simulation'), ('suhosin.executor.include.whitelist', 'Suhosin executor include whitelist'), ('upload_max_filesize', 'Upload max filesize'), ('zend_extension', 'Zend extension')])])), ('value', models.CharField(verbose_name='value', max_length=256)), - ('webapp', models.ForeignKey(verbose_name='Web application', related_name='options', to='webapps.WebApp')), + ('webapp', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='Web application', related_name='options', to='webapps.WebApp')), ], options={ 'verbose_name': 'option', diff --git a/orchestra/contrib/webapps/migrations/0001_squashed_0006_auto_20210330_1049.py b/orchestra/contrib/webapps/migrations/0001_squashed_0006_auto_20210330_1049.py new file mode 100644 index 00000000..0074262c --- /dev/null +++ b/orchestra/contrib/webapps/migrations/0001_squashed_0006_auto_20210330_1049.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:25 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.fields +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('webapps', '0001_initial'), ('webapps', '0002_auto_20170528_2011'), ('webapps', '0003_webapp_target_server'), ('webapps', '0004_webapp_comments'), ('webapps', '0005_auto_20200204_1218'), ('webapps', '0006_auto_20210330_1049')] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('orchestration', '0007_auto_20170528_2011'), + ] + + operations = [ + migrations.CreateModel( + name='WebApp', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='The app will be installed in %(home)s/webapps/%(app_name)s', max_length=128, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('type', models.CharField(choices=[('php', 'PHP'), ('python', 'Python'), ('static', 'Static'), ('symbolic-link', 'Symbolic link'), ('webalizer', 'Webalizer'), ('wordpress-php', 'WordPress')], max_length=32, verbose_name='type')), + ('data', jsonfield.fields.JSONField(blank=True, default={}, help_text='Extra information dependent of each service.', verbose_name='data')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webapps', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ], + options={ + 'verbose_name': 'Web App', + 'verbose_name_plural': 'Web Apps', + }, + ), + migrations.CreateModel( + name='WebAppOption', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(choices=[(None, '-------'), ('FileSystem', [('public-root', 'Public root')]), ('Process', [('timeout', 'Process timeout'), ('processes', 'Number of processes')]), ('PHP', [('enable_functions', 'Enable functions'), ('allow_url_include', 'Allow URL include'), ('allow_url_fopen', 'Allow URL fopen'), ('auto_append_file', 'Auto append file'), ('auto_prepend_file', 'Auto prepend file'), ('date.timezone', 'date.timezone'), ('default_socket_timeout', 'Default socket timeout'), ('display_errors', 'Display errors'), ('extension', 'Extension'), ('magic_quotes_gpc', 'Magic quotes GPC'), ('magic_quotes_runtime', 'Magic quotes runtime'), ('magic_quotes_sybase', 'Magic quotes sybase'), ('max_input_time', 'Max input time'), ('max_input_vars', 'Max input vars'), ('memory_limit', 'Memory limit'), ('mysql.connect_timeout', 'Mysql connect timeout'), ('output_buffering', 'Output buffering'), ('register_globals', 'Register globals'), ('post_max_size', 'Post max size'), ('sendmail_path', 'Sendmail path'), ('session.bug_compat_warn', 'Session bug compat warning'), ('session.auto_start', 'Session auto start'), ('safe_mode', 'Safe mode'), ('suhosin.post.max_vars', 'Suhosin POST max vars'), ('suhosin.get.max_vars', 'Suhosin GET max vars'), ('suhosin.request.max_vars', 'Suhosin request max vars'), ('suhosin.session.encrypt', 'Suhosin session encrypt'), ('suhosin.simulation', 'Suhosin simulation'), ('suhosin.executor.include.whitelist', 'Suhosin executor include whitelist'), ('upload_max_filesize', 'Upload max filesize'), ('zend_extension', 'Zend extension')])], max_length=128, verbose_name='name')), + ('value', models.CharField(max_length=256, verbose_name='value')), + ('webapp', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='options', to='webapps.WebApp', verbose_name='Web application')), + ], + options={ + 'verbose_name': 'option', + 'verbose_name_plural': 'options', + }, + ), + migrations.AlterUniqueTogether( + name='webappoption', + unique_together=set([('webapp', 'name')]), + ), + migrations.AlterField( + model_name='webapp', + name='type', + field=models.CharField(choices=[('moodle-php', 'Moodle'), ('php', 'PHP'), ('python', 'Python'), ('static', 'Static'), ('symbolic-link', 'Symbolic link'), ('webalizer', 'Webalizer'), ('wordpress-php', 'WordPress')], max_length=32, verbose_name='type'), + ), + migrations.AddField( + model_name='webapp', + name='target_server', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='webapps', to='orchestration.Server', verbose_name='Target Server'), + preserve_default=False, + ), + migrations.AddField( + model_name='webapp', + name='comments', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterUniqueTogether( + name='webapp', + unique_together=set([('name', 'account')]), + ), + migrations.AlterField( + model_name='webappoption', + name='name', + field=models.CharField(choices=[(None, '-------'), ('FileSystem', [('public-root', 'Public root')]), ('Process', [('timeout', 'Process timeout'), ('processes', 'Number of processes')]), ('PHP', [('enable_functions', 'Enable functions'), ('disable_functions', 'Disable functions'), ('allow_url_include', 'Allow URL include'), ('allow_url_fopen', 'Allow URL fopen'), ('auto_append_file', 'Auto append file'), ('auto_prepend_file', 'Auto prepend file'), ('date.timezone', 'date.timezone'), ('default_socket_timeout', 'Default socket timeout'), ('display_errors', 'Display errors'), ('extension', 'Extension'), ('include_path', 'Include path'), ('magic_quotes_gpc', 'Magic quotes GPC'), ('magic_quotes_runtime', 'Magic quotes runtime'), ('magic_quotes_sybase', 'Magic quotes sybase'), ('max_input_time', 'Max input time'), ('max_input_vars', 'Max input vars'), ('memory_limit', 'Memory limit'), ('mysql.connect_timeout', 'Mysql connect timeout'), ('output_buffering', 'Output buffering'), ('register_globals', 'Register globals'), ('post_max_size', 'Post max size'), ('sendmail_path', 'Sendmail path'), ('session.bug_compat_warn', 'Session bug compat warning'), ('session.auto_start', 'Session auto start'), ('safe_mode', 'Safe mode'), ('suhosin.post.max_vars', 'Suhosin POST max vars'), ('suhosin.get.max_vars', 'Suhosin GET max vars'), ('suhosin.request.max_vars', 'Suhosin request max vars'), ('suhosin.session.encrypt', 'Suhosin session encrypt'), ('suhosin.simulation', 'Suhosin simulation'), ('suhosin.executor.include.whitelist', 'Suhosin executor include whitelist'), ('upload_max_filesize', 'Upload max filesize'), ('upload_tmp_dir', 'Upload tmp dir'), ('zend_extension', 'Zend extension')])], max_length=128, verbose_name='name'), + ), + ] diff --git a/orchestra/contrib/webapps/migrations/0006_auto_20210330_1049.py b/orchestra/contrib/webapps/migrations/0006_auto_20210330_1049.py new file mode 100644 index 00000000..1c27407d --- /dev/null +++ b/orchestra/contrib/webapps/migrations/0006_auto_20210330_1049.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('webapps', '0005_auto_20200204_1218'), + ] + + operations = [ + migrations.AlterField( + model_name='webapp', + name='comments', + field=models.TextField(blank=True, default=''), + ), + ] diff --git a/orchestra/contrib/webapps/models.py b/orchestra/contrib/webapps/models.py index 2180e01b..42b3a227 100644 --- a/orchestra/contrib/webapps/models.py +++ b/orchestra/contrib/webapps/models.py @@ -19,44 +19,44 @@ class WebApp(models.Model): name = models.CharField(_("name"), max_length=128, validators=[validators.validate_name], help_text=_("The app will be installed in %s") % settings.WEBAPPS_BASE_DIR) type = models.CharField(_("type"), max_length=32, choices=AppType.get_choices()) - account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='webapps') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='webapps') data = JSONField(_("data"), blank=True, default={}, help_text=_("Extra information dependent of each service.")) - target_server = models.ForeignKey('orchestration.Server', verbose_name=_("Target Server"), - related_name='webapps') + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Target Server"), related_name='webapps') comments = models.TextField(default="", blank=True) - + # CMS webapps usually need a database and dbuser, with these virtual fields we tell the ORM to delete them databases = VirtualDatabaseRelation('databases.Database') databaseusers = VirtualDatabaseUserRelation('databases.DatabaseUser') - + class Meta: unique_together = ('name', 'account') verbose_name = _("Web App") verbose_name_plural = _("Web Apps") - + def __str__(self): return self.name - + def get_description(self): return self.get_type_display() - + @cached_property def type_class(self): return AppType.get(self.type) - + @cached_property def type_instance(self): """ Per request lived type_instance """ return self.type_class(self) - + def clean(self): apptype = self.type_instance apptype.validate() a = apptype.clean_data() self.data = apptype.clean_data() - + def get_options(self, **kwargs): options = OrderedDict() qs = WebAppOption.objects.filter(**kwargs) @@ -69,57 +69,57 @@ class WebApp(models.Model): else: options[name] = value return options - + def get_directive(self): return self.type_instance.get_directive() - + def get_base_path(self): context = { 'home': self.get_user().get_home(), 'app_name': self.name, } return settings.WEBAPPS_BASE_DIR % context - + def get_path(self): path = self.get_base_path() public_root = self.options.filter(name='public-root').first() if public_root: path = os.path.join(path, public_root.value) return os.path.normpath(path.replace('//', '/')) - + def get_user(self): return self.account.main_systemuser - + def get_username(self): return self.get_user().username - + def get_groupname(self): return self.get_username() class WebAppOption(models.Model): - webapp = models.ForeignKey(WebApp, verbose_name=_("Web application"), - related_name='options') + webapp = models.ForeignKey(WebApp, on_delete=models.CASCADE, + verbose_name=_("Web application"), related_name='options') name = models.CharField(_("name"), max_length=128, choices=AppType.get_group_options_choices()) value = models.CharField(_("value"), max_length=256) - + class Meta: unique_together = ('webapp', 'name') verbose_name = _("option") verbose_name_plural = _("options") - + def __str__(self): return self.name - + @cached_property def option_class(self): return AppOption.get(self.name) - + @cached_property def option_instance(self): """ Per request lived option instance """ return self.option_class(self) - + def clean(self): self.option_instance.validate() diff --git a/orchestra/contrib/webapps/tests/functional_tests/tests.py b/orchestra/contrib/webapps/tests/functional_tests/tests.py index 414a2316..72536f15 100644 --- a/orchestra/contrib/webapps/tests/functional_tests/tests.py +++ b/orchestra/contrib/webapps/tests/functional_tests/tests.py @@ -1,16 +1,19 @@ import ftplib import os +import unittest from io import StringIO from django.conf import settings as djsettings - -from orchestra.contrib.orchestration.models import Server, Route -from orchestra.contrib.systemusers.backends import SystemUserBackend -from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error +from orchestra.contrib.orchestration.models import Route, Server +from orchestra.contrib.systemusers.backends import UNIXUserController +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, save_response_on_error, snapshot_on_error from ... import backends +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + + class WebAppMixin(object): MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') DEPENDENCIES = ( @@ -18,20 +21,20 @@ class WebAppMixin(object): 'orchestra.contrib.systemusers', 'orchestra.contrib.webapps', ) - + def setUp(self): super(WebAppMixin, self).setUp() self.add_route() djsettings.DEBUG = True - + def add_route(self): server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER) - backend = SystemUserBackend.get_name() + backend = UNIXUserController.get_name() Route.objects.get_or_create(backend=backend, match=True, host=server) backend = self.backend.get_name() match = 'webapp.type == "%s"' % self.type_value Route.objects.create(backend=backend, match=match, host=server) - + def upload_webapp(self, name): try: ftp = ftplib.FTP(self.MASTER_SERVER) @@ -44,7 +47,7 @@ class WebAppMixin(object): index.close() finally: ftp.close() - + def test_add(self): name = '%s_%s_webapp' % (random_ascii(10), self.type_value) self.add_webapp(name) @@ -63,9 +66,9 @@ class StaticWebAppMixin(object): ) -class PHPFcidWebAppMixin(StaticWebAppMixin): - backend = backends.phpfcgid.PHPFcgidBackend - type_value = 'php5.2' +class PHPFPMWebAppMixin(StaticWebAppMixin): + backend = backends.php.PHPController + type_value = 'php5.5' token = random_ascii(100) page = ( 'index.php', @@ -74,27 +77,23 @@ class PHPFcidWebAppMixin(StaticWebAppMixin): ) -class PHPFPMWebAppMixin(PHPFcidWebAppMixin): - backend = backends.phpfpm.PHPFPMBackend - type_value = 'php5.5' - - +@unittest.skipUnless(TEST_REST_API, "REST API tests") class RESTWebAppMixin(object): def setUp(self): super(RESTWebAppMixin, self).setUp() self.rest_login() # create main user self.save_systemuser() - + @save_response_on_error def save_systemuser(self): systemuser = self.rest.systemusers.retrieve().get() systemuser.update(is_active=True) - + @save_response_on_error def add_webapp(self, name, options=[]): self.rest.webapps.create(name=name, type=self.type_value, options=options) - + @save_response_on_error def delete_webapp(self, name): self.rest.webapps.retrieve(name=name).delete() @@ -106,11 +105,11 @@ class AdminWebAppMixin(WebAppMixin): self.admin_login() # create main user self.save_systemuser() - + @snapshot_on_error def save_systemuser(self): url = '' - + @snapshot_on_error def add(self, name, password, admin_email): pass @@ -120,10 +119,6 @@ class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, Base pass -class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): - pass - - class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): pass diff --git a/orchestra/contrib/webapps/types/cms.py b/orchestra/contrib/webapps/types/cms.py index 80822cc2..0343ce15 100644 --- a/orchestra/contrib/webapps/types/cms.py +++ b/orchestra/contrib/webapps/types/cms.py @@ -1,6 +1,6 @@ from django import forms from django.core.exceptions import ValidationError -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers @@ -20,7 +20,7 @@ class CMSAppForm(PHPAppForm): password = forms.CharField(label=_("Password"), help_text=_("Initial database and WordPress admin password.
" "Subsequent changes to the admin password will not be reflected.")) - + def __init__(self, *args, **kwargs): super(CMSAppForm, self).__init__(*args, **kwargs) if self.instance: @@ -55,20 +55,20 @@ class CMSApp(PHPApp): db_type = Database.MYSQL abstract = True db_prefix = 'cms_' - + def get_db_name(self): db_name = '%s%s_%s' % (self.db_prefix, self.instance.name, self.instance.account) # Limit for mysql database names return db_name[:65] - + def get_db_user(self): db_name = self.get_db_name() # Limit for mysql user names return db_name[:16] - + def get_password(self): return random_ascii(10) - + def validate(self): super(CMSApp, self).validate() create = not self.instance.pk @@ -83,7 +83,7 @@ class CMSApp(PHPApp): raise ValidationError({ 'name': e.messages, }) - + def save(self): db_name = self.get_db_name() db_user = self.get_db_user() diff --git a/orchestra/contrib/websites/admin.py b/orchestra/contrib/websites/admin.py index e447683a..fc54b4bc 100644 --- a/orchestra/contrib/websites/admin.py +++ b/orchestra/contrib/websites/admin.py @@ -1,6 +1,6 @@ from django import forms from django.contrib import admin -from django.core.urlresolvers import resolve +from django.urls import resolve from django.db.models import Q from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -24,11 +24,11 @@ class WebsiteDirectiveInline(admin.TabularInline): model = WebsiteDirective formset = WebsiteDirectiveInlineFormSet extra = 1 - + DIRECTIVES_HELP_TEXT = { op.name: force_text(op.help_text) for op in SiteDirective.get_plugins() } - + def formfield_for_dbfield(self, db_field, **kwargs): if db_field.name == 'value': kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) @@ -45,10 +45,10 @@ class ContentInline(AccountAdminMixin, admin.TabularInline): fields = ('webapp', 'webapp_link', 'webapp_type', 'path') readonly_fields = ('webapp_link', 'webapp_type') filter_by_account_fields = ['webapp'] - + webapp_link = admin_link('webapp', popup=True) webapp_link.short_description = _("Web App") - + def webapp_type(self, content): if not content.pk: return '' @@ -77,7 +77,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): list_prefetch_related = ('domains', 'content_set__webapp') search_fields = ('name', 'account__username', 'domains__name', 'content__webapp__name') actions = (disable, enable, list_accounts) - + def display_domains(self, website): domains = [] for domain in website.domains.all(): @@ -87,7 +87,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): display_domains.short_description = _("domains") display_domains.allow_tags = True display_domains.admin_order_field = 'domains' - + def display_webapps(self, website): webapps = [] for content in website.content_set.all(): @@ -104,7 +104,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): return '
'.join(webapps) display_webapps.allow_tags = True display_webapps.short_description = _("Web apps") - + def formfield_for_dbfield(self, db_field, **kwargs): """ Exclude domains with exhausted ports @@ -124,7 +124,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): qset = Q(qset & ~Q(websites__pk=object_id)) formfield.queryset = formfield.queryset.exclude(qset) return formfield - + def _create_formsets(self, request, obj, change): """ bind contents formset to directive formset for unique location cross-validation """ formsets, inline_instances = super(WebsiteAdmin, self)._create_formsets(request, obj, change) diff --git a/orchestra/contrib/websites/migrations/0001_initial.py b/orchestra/contrib/websites/migrations/0001_initial.py index 083875de..f58a4210 100644 --- a/orchestra/contrib/websites/migrations/0001_initial.py +++ b/orchestra/contrib/websites/migrations/0001_initial.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings +import django.db.models.deletion import orchestra.core.validators @@ -20,7 +21,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True, serialize=False)), ('path', models.CharField(validators=[orchestra.core.validators.validate_url_path], verbose_name='path', max_length=256, blank=True)), - ('webapp', models.ForeignKey(verbose_name='web application', to='webapps.WebApp')), + ('webapp', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='web application', to='webapps.WebApp')), ], ), migrations.CreateModel( @@ -30,7 +31,7 @@ class Migration(migrations.Migration): ('name', models.CharField(validators=[orchestra.core.validators.validate_name], verbose_name='name', max_length=128)), ('protocol', models.CharField(verbose_name='protocol', default='http', choices=[('http', 'HTTP'), ('https', 'HTTPS'), ('http/https', 'HTTP and HTTPS'), ('https-only', 'HTTPS only')], help_text='Select the protocol(s) for this website
HTTPS only performs a redirection from http to https.', max_length=16)), ('is_active', models.BooleanField(verbose_name='active', default=True)), - ('account', models.ForeignKey(related_name='websites', verbose_name='Account', to=settings.AUTH_USER_MODEL)), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='websites', verbose_name='Account', to=settings.AUTH_USER_MODEL)), ('contents', models.ManyToManyField(through='websites.Content', to='webapps.WebApp')), ('domains', models.ManyToManyField(verbose_name='domains', related_name='websites', to='domains.Domain')), ], @@ -41,13 +42,13 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True, serialize=False)), ('name', models.CharField(verbose_name='name', choices=[(None, '-------'), ('SSL', [('ssl-ca', 'SSL CA'), ('ssl-cert', 'SSL cert'), ('ssl-key', 'SSL key')]), ('HTTPD', [('redirect', 'Redirection'), ('proxy', 'Proxy'), ('error-document', 'ErrorDocumentRoot')]), ('ModSecurity', [('sec-rule-remove', 'SecRuleRemoveById'), ('sec-engine', 'SecRuleEngine Off')]), ('SaaS', [('wordpress-saas', 'WordPress SaaS'), ('dokuwiki-saas', 'DokuWiki SaaS'), ('drupal-saas', 'Drupdal SaaS')])], max_length=128)), ('value', models.CharField(verbose_name='value', max_length=256)), - ('website', models.ForeignKey(related_name='directives', verbose_name='web site', to='websites.Website')), + ('website', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='directives', verbose_name='web site', to='websites.Website')), ], ), migrations.AddField( model_name='content', name='website', - field=models.ForeignKey(verbose_name='web site', to='websites.Website'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, verbose_name='web site', to='websites.Website'), ), migrations.AlterUniqueTogether( name='website', diff --git a/orchestra/contrib/websites/migrations/0001_squashed_0017_auto_20210330_1049.py b/orchestra/contrib/websites/migrations/0001_squashed_0017_auto_20210330_1049.py new file mode 100644 index 00000000..e6847bc4 --- /dev/null +++ b/orchestra/contrib/websites/migrations/0001_squashed_0017_auto_20210330_1049.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-04-22 11:25 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + replaces = [('websites', '0001_initial'), ('websites', '0002_auto_20160219_1036'), ('websites', '0003_auto_20170528_2011'), ('websites', '0004_auto_20170625_1813'), ('websites', '0005_auto_20170625_1840'), ('websites', '0006_auto_20170625_1840'), ('websites', '0007_auto_20170625_1840'), ('websites', '0008_auto_20170625_1841'), ('websites', '0009_auto_20170625_2206'), ('websites', '0010_auto_20170625_2214'), ('websites', '0011_auto_20170704_1117'), ('websites', '0012_auto_20190805_1134'), ('websites', '0013_auto_20200204_1217'), ('websites', '0014_auto_20200204_1218'), ('websites', '0015_auto_20200204_1219'), ('websites', '0016_auto_20200204_1221'), ('websites', '0017_auto_20210330_1049')] + + initial = True + + dependencies = [ + ('domains', '0001_initial'), + ('webapps', '0001_initial'), + ('orchestration', '0007_auto_20170528_2011'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Content', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(blank=True, max_length=256, validators=[orchestra.core.validators.validate_url_path], verbose_name='path')), + ('webapp', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='webapps.WebApp', verbose_name='web application')), + ], + ), + migrations.CreateModel( + name='Website', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('protocol', models.CharField(choices=[('http', 'HTTP'), ('https', 'HTTPS'), ('http/https', 'HTTP and HTTPS'), ('https-only', 'HTTPS only')], default='http', help_text='Select the protocol(s) for this website
HTTPS only performs a redirection from http to https.', max_length=16, verbose_name='protocol')), + ('is_active', models.BooleanField(default=True, verbose_name='active')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='websites', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('contents', models.ManyToManyField(through='websites.Content', to='webapps.WebApp')), + ('domains', models.ManyToManyField(related_name='websites', to='domains.Domain', verbose_name='domains')), + ], + ), + migrations.CreateModel( + name='WebsiteDirective', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(choices=[(None, '-------'), ('HTTPD', [('redirect', 'Redirection'), ('proxy', 'Proxy'), ('error-document', 'ErrorDocumentRoot')]), ('SSL', [('ssl-ca', 'SSL CA'), ('ssl-cert', 'SSL cert'), ('ssl-key', 'SSL key')]), ('ModSecurity', [('sec-rule-remove', 'SecRuleRemoveById'), ('sec-engine', 'SecRuleEngine Off')]), ('SaaS', [('wordpress-saas', 'WordPress SaaS'), ('dokuwiki-saas', 'DokuWiki SaaS'), ('drupal-saas', 'Drupdal SaaS'), ('moodle-saas', 'Moodle SaaS')])], db_index=True, max_length=128, verbose_name='name')), + ('value', models.CharField(blank=True, max_length=256, verbose_name='value')), + ('website', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='directives', to='websites.Website', verbose_name='web site')), + ], + ), + migrations.AddField( + model_name='content', + name='website', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='websites.Website', verbose_name='web site'), + ), + migrations.AlterField( + model_name='website', + name='domains', + field=models.ManyToManyField(blank=True, related_name='websites', to='domains.Domain', verbose_name='domains'), + ), + migrations.AddField( + model_name='website', + name='target_server', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='websites', to='orchestration.Server', verbose_name='Target Server'), + preserve_default=False, + ), + migrations.AddField( + model_name='website', + name='comments', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterUniqueTogether( + name='website', + unique_together=set([('name', 'account')]), + ), + migrations.AlterUniqueTogether( + name='content', + unique_together=set([('website', 'path')]), + ), + ] diff --git a/orchestra/contrib/websites/migrations/0017_auto_20210330_1049.py b/orchestra/contrib/websites/migrations/0017_auto_20210330_1049.py new file mode 100644 index 00000000..cfb047e6 --- /dev/null +++ b/orchestra/contrib/websites/migrations/0017_auto_20210330_1049.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2021-03-30 10:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('websites', '0016_auto_20200204_1221'), + ] + + operations = [ + migrations.AlterField( + model_name='website', + name='comments', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterField( + model_name='websitedirective', + name='name', + field=models.CharField(choices=[(None, '-------'), ('HTTPD', [('redirect', 'Redirection'), ('proxy', 'Proxy'), ('error-document', 'ErrorDocumentRoot')]), ('SSL', [('ssl-ca', 'SSL CA'), ('ssl-cert', 'SSL cert'), ('ssl-key', 'SSL key')]), ('ModSecurity', [('sec-rule-remove', 'SecRuleRemoveById'), ('sec-engine', 'SecRuleEngine Off')]), ('SaaS', [('wordpress-saas', 'WordPress SaaS'), ('dokuwiki-saas', 'DokuWiki SaaS'), ('drupal-saas', 'Drupdal SaaS'), ('moodle-saas', 'Moodle SaaS')])], db_index=True, max_length=128, verbose_name='name'), + ), + ] diff --git a/orchestra/contrib/websites/models.py b/orchestra/contrib/websites/models.py index 80e1bf1f..f71d1a1f 100644 --- a/orchestra/contrib/websites/models.py +++ b/orchestra/contrib/websites/models.py @@ -18,11 +18,11 @@ class Website(models.Model): HTTPS = 'https' HTTP_AND_HTTPS = 'http/https' HTTPS_ONLY = 'https-only' - + name = models.CharField(_("name"), max_length=128, validators=[validators.validate_name]) - account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='websites') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='websites') protocol = models.CharField(_("protocol"), max_length=16, choices=settings.WEBSITES_PROTOCOL_CHOICES, default=settings.WEBSITES_DEFAULT_PROTOCOL, @@ -34,34 +34,34 @@ class Website(models.Model): domains = models.ManyToManyField(settings.WEBSITES_DOMAIN_MODEL, blank=True, related_name='websites', verbose_name=_("domains")) contents = models.ManyToManyField('webapps.WebApp', through='websites.Content') - target_server = models.ForeignKey('orchestration.Server', verbose_name=_("Target Server"), - related_name='websites') + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Target Server"), related_name='websites') is_active = models.BooleanField(_("active"), default=True) comments = models.TextField(default="", blank=True) - + class Meta: unique_together = ('name', 'account') - + def __str__(self): return self.name - + @property def unique_name(self): context = self.get_settings_context() return settings.WEBSITES_UNIQUE_NAME_FORMAT % context - + @cached_property def active(self): return self.is_active and self.account.is_active - + def disable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def enable(self): self.is_active = False self.save(update_fields=('is_active',)) - + def get_settings_context(self): """ format settings strings """ return { @@ -73,12 +73,12 @@ class Website(models.Model): 'site_name': self.name, 'protocol': self.protocol, } - + def get_protocol(self): if self.protocol in (self.HTTP, self.HTTP_AND_HTTPS): return self.HTTP return self.HTTPS - + @cached def get_directives(self): directives = OrderedDict() @@ -88,7 +88,7 @@ class Website(models.Model): except KeyError: directives[opt.name] = [opt.value] return directives - + def get_absolute_url(self): try: domain = self.domains.all()[0] @@ -96,22 +96,22 @@ class Website(models.Model): return else: return '%s://%s' % (self.get_protocol(), domain) - + def get_user(self): return self.account.main_systemuser - + def get_username(self): return self.get_user().username - + def get_groupname(self): return self.get_username() - + def get_www_access_log_path(self): context = self.get_settings_context() context['unique_name'] = self.unique_name path = settings.WEBSITES_WEBSITE_WWW_ACCESS_LOG_PATH % context return os.path.normpath(path) - + def get_www_error_log_path(self): context = self.get_settings_context() context['unique_name'] = self.unique_name @@ -120,52 +120,54 @@ class Website(models.Model): class WebsiteDirective(models.Model): - website = models.ForeignKey(Website, verbose_name=_("web site"), - related_name='directives') + website = models.ForeignKey(Website, on_delete=models.CASCADE, + verbose_name=_("web site"), related_name='directives') name = models.CharField(_("name"), max_length=128, db_index=True, choices=SiteDirective.get_choices()) value = models.CharField(_("value"), max_length=256, blank=True) - + def __str__(self): return self.name - + @cached_property def directive_class(self): return SiteDirective.get(self.name) - + @cached_property def directive_instance(self): """ Per request lived directive instance """ return self.directive_class() - + def clean(self): self.directive_instance.validate(self) class Content(models.Model): # related_name is content_set to differentiate between website.content -> webapp - webapp = models.ForeignKey('webapps.WebApp', verbose_name=_("web application")) - website = models.ForeignKey('websites.Website', verbose_name=_("web site")) + webapp = models.ForeignKey('webapps.WebApp', on_delete=models.CASCADE, + verbose_name=_("web application")) + website = models.ForeignKey('websites.Website', on_delete=models.CASCADE, + verbose_name=_("web site")) path = models.CharField(_("path"), max_length=256, blank=True, validators=[validators.validate_url_path]) - + class Meta: unique_together = ('website', 'path') - + def __str__(self): try: return self.website.name + self.path except Website.DoesNotExist: return self.path - + def clean_fields(self, *args, **kwargs): self.path = self.path.strip() return super(Content, self).clean_fields(*args, **kwargs) - + def clean(self): if not self.path: self.path = '/' - + def get_absolute_url(self): try: domain = self.website.domains.all()[0] diff --git a/orchestra/contrib/websites/serializers.py b/orchestra/contrib/websites/serializers.py index 89fd6ce8..f38203a3 100644 --- a/orchestra/contrib/websites/serializers.py +++ b/orchestra/contrib/websites/serializers.py @@ -13,23 +13,23 @@ from .validators import validate_domain_protocol class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): class Meta: - model = Website.domains.field.rel.to + model = Website.domains.field.model fields = ('url', 'id', 'name') class RelatedWebAppSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): class Meta: - model = Content.webapp.field.rel.to + model = Content.webapp.field.model fields = ('url', 'id', 'name', 'type') class ContentSerializer(serializers.ModelSerializer): webapp = RelatedWebAppSerializer() - + class Meta: model = Content fields = ('webapp', 'path') - + def get_identity(self, data): return '%s-%s' % (data.get('website'), data.get('path')) @@ -38,10 +38,10 @@ class DirectiveSerializer(serializers.ModelSerializer): class Meta: model = WebsiteDirective fields = ('name', 'value') - + def to_representation(self, instance): return {prop.name: prop.value for prop in instance.all()} - + def to_internal_value(self, data): return data @@ -50,12 +50,12 @@ class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): domains = RelatedDomainSerializer(many=True, required=False) contents = ContentSerializer(required=False, many=True, source='content_set') directives = DirectiveSerializer(required=False) - + class Meta: model = Website fields = ('url', 'id', 'name', 'protocol', 'domains', 'is_active', 'contents', 'directives') postonly_fields = ('name',) - + def validate(self, data): """ Prevent multiples domains on the same protocol """ # Validate location and directive uniqueness @@ -87,14 +87,14 @@ class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): if errors: raise ValidationError(errors) return data - + def create(self, validated_data): directives_data = validated_data.pop('directives') webapp = super(WebsiteSerializer, self).create(validated_data) for key, value in directives_data.items(): WebsiteDirective.objects.create(webapp=webapp, name=key, value=value) return webap - + def update_directives(self, instance, directives_data): existing = {} for obj in instance.directives.all(): @@ -112,13 +112,13 @@ class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): directive.save(update_fields=('value',)) for to_delete in set(existing.keys())-posted: existing[to_delete].delete() - + def update_contents(self, instance, contents_data): raise NotImplementedError - + def update_domains(self, instance, domains_data): raise NotImplementedError - + def update(self, instance, validated_data): directives_data = validated_data.pop('directives') domains_data = validated_data.pop('domains') diff --git a/orchestra/contrib/websites/tests/functional_tests/tests.py b/orchestra/contrib/websites/tests/functional_tests/tests.py index 61e31786..1a9bd3ad 100644 --- a/orchestra/contrib/websites/tests/functional_tests/tests.py +++ b/orchestra/contrib/websites/tests/functional_tests/tests.py @@ -6,7 +6,7 @@ import requests from orchestra.contrib.domains.models import Domain, Record from orchestra.contrib.domains.backends import Bind9MasterDomainController from orchestra.contrib.orchestration.models import Server, Route -from orchestra.contrib.webapps.tests.functional_tests.tests import StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, PHPFcidWebAppMixin, PHPFPMWebAppMixin +from orchestra.contrib.webapps.tests.functional_tests.tests import StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, PHPFPMWebAppMixin from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, save_response_on_error from ... import backends @@ -22,7 +22,7 @@ class WebsiteMixin(WebAppMixin): 'orchestra.contrib.webapps', 'orchestra.contrib.systemusers', ) - + def add_route(self): super(WebsiteMixin, self).add_route() server = Server.objects.get() @@ -30,11 +30,11 @@ class WebsiteMixin(WebAppMixin): Route.objects.get_or_create(backend=backend, match=True, host=server) backend = Bind9MasterDomainController.get_name() Route.objects.get_or_create(backend=backend, match=True, host=server) - + def validate_add_website(self, name, domain): url = 'http://%s/%s' % (domain.name, self.page[0]) self.assertEqual(self.page[2], requests.get(url).content) - + def test_add(self): # TODO domains with "_" bad name! domain_name = '%sdomain.lan' % random_ascii(10) @@ -55,7 +55,7 @@ class RESTWebsiteMixin(RESTWebAppMixin): @save_response_on_error def save_domain(self, domain): self.rest.domains.retrieve().get().save() - + @save_response_on_error def add_website(self, name, domain, webapp, path='/'): domain = self.rest.domains.retrieve(name=domain).get() @@ -65,11 +65,11 @@ class RESTWebsiteMixin(RESTWebAppMixin): 'path': path }] self.rest.websites.create(name=name, domains=[domain], contents=contents) - + @save_response_on_error def delete_website(self, name): self.rest.websites.retrieve(name=name).delete() - + @save_response_on_error def add_content(self, website, webapp, path): website = self.rest.websites.retrieve(name=website).get() @@ -101,20 +101,7 @@ class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, B self.add_website(website, domain, webapp) self.addCleanup(self.delete_website, website) self.validate_add_website(website, domain) - - self.type_value = PHPFcidWebAppMixin.type_value - self.backend = PHPFcidWebAppMixin.backend - self.page = PHPFcidWebAppMixin.page - self.add_route() - webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) - self.add_webapp(webapp) - self.addCleanup(self.delete_webapp, webapp) - self.upload_webapp(webapp) - path = '/%s' % webapp - self.add_content(website, webapp, path) - url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) - self.assertEqual(self.page[2], requests.get(url).content) - + self.type_value = PHPFPMWebAppMixin.type_value self.backend = PHPFPMWebAppMixin.backend self.page = PHPFPMWebAppMixin.page @@ -124,14 +111,23 @@ class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, B self.addCleanup(self.delete_webapp, webapp) self.upload_webapp(webapp) path = '/%s' % webapp - self.add_content(website, webapp, path) url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) self.assertEqual(self.page[2], requests.get(url).content) + self.type_value = PHPFPMWebAppMixin.type_value + self.backend = PHPFPMWebAppMixin.backend + self.page = PHPFPMWebAppMixin.page + self.add_route() + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + path = '/%s' % webapp -class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): - pass + self.add_content(website, webapp, path) + url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): diff --git a/orchestra/models/fields.py b/orchestra/models/fields.py index 44331180..81ab5796 100644 --- a/orchestra/models/fields.py +++ b/orchestra/models/fields.py @@ -1,7 +1,7 @@ import os from django.core import exceptions -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.db.models.fields.files import FileField, FieldFile from django.utils.text import capfirst @@ -21,34 +21,34 @@ class MultiSelectField(models.CharField): defaults['initial'] = self.get_default() defaults.update(kwargs) return MultiSelectFormField(**defaults) - + def get_db_prep_value(self, value, connection=None, prepared=False): if isinstance(value, str): return value else: return ','.join(value) - + def to_python(self, value): if value: if isinstance(value, str): return value.split(',') return value return [] - + def from_db_value(self, value, expression, connection, context): if value: if isinstance(value, str): return value.split(',') return value return [] - + def contribute_to_class(self, cls, name): super(MultiSelectField, self).contribute_to_class(cls, name) if self.choices: def func(self, field=name, choices=dict(self.choices)): return ','.join([choices.get(value, value) for value in getattr(self, field)]) setattr(cls, 'get_%s_display' % self.name, func) - + def validate(self, value, model_instance): if self.choices: arr_choices = self.get_choices_selected(self.get_choices()) @@ -56,7 +56,7 @@ class MultiSelectField(models.CharField): if (opt_select not in arr_choices): msg = self.error_messages['invalid_choice'] % {'value': opt_select} raise exceptions.ValidationError(msg) - + def get_choices_selected(self, arr_choices=''): if not arr_choices: return False @@ -79,11 +79,11 @@ class PrivateFieldFile(FieldFile): filename = os.path.basename(self.path) args = [app_label, model_name, field_name, pk, filename] return reverse('private-media', args=args) - + @property def condition(self): return self.field.condition - + @property def attachment(self): return self.field.attachment @@ -91,7 +91,7 @@ class PrivateFieldFile(FieldFile): class PrivateFileField(FileField): attr_class = PrivateFieldFile - + def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, attachment=True, condition=lambda request, instance: request.user.is_superuser, **kwargs): super(PrivateFileField, self).__init__(verbose_name, name, upload_to, storage, **kwargs) diff --git a/orchestra/models/utils.py b/orchestra/models/utils.py index e9514fe8..e0facdd1 100644 --- a/orchestra/models/utils.py +++ b/orchestra/models/utils.py @@ -50,9 +50,9 @@ def get_model_field_path(origin, target): if node == target: return path for field in node._meta.fields: - if field.rel: + if field.remote_field: new_model = list(model) - new_model.append(field.rel.to) + new_model.append(field.remote_field.model) new_path = list(path) new_path.append(field.name) queue.append((new_model, new_path)) diff --git a/orchestra/permissions/api.py b/orchestra/permissions/api.py index abfcdb99..5e72aae8 100644 --- a/orchestra/permissions/api.py +++ b/orchestra/permissions/api.py @@ -1,28 +1,28 @@ -from django.core.urlresolvers import resolve +from django.urls import resolve from rest_framework.permissions import DjangoModelPermissions class OrchestraPermissionBackend(DjangoModelPermissions): """ Permissions according to each user """ - + def has_permission(self, request, view): queryset = getattr(view, 'queryset', None) if queryset is None: name = resolve(request.path).url_name return name == 'api-root' - + model_cls = queryset.model perms = self.get_required_permissions(request.method, model_cls) if (request.user and - request.user.is_authenticated() and + request.user.is_authenticated and request.user.has_perms(perms, model_cls)): return True return False - + def has_object_permission(self, request, view, obj): perms = self.get_required_permissions(request.method, type(obj)) if (request.user and - request.user.is_authenticated() and + request.user.is_authenticated and request.user.has_perms(perms, obj)): return True return False diff --git a/orchestra/permissions/options.py b/orchestra/permissions/options.py index 74806885..b37d2363 100644 --- a/orchestra/permissions/options.py +++ b/orchestra/permissions/options.py @@ -7,24 +7,24 @@ import inspect class Permission(object): - """ + """ Base class used for defining class and instance permissions. Enabling an ''intuitive'' interface for checking permissions: - + # Define permissions class NodePermission(Permission): def change(self, obj, cls, user): return obj.user == user - + # Provide permissions Node.has_permission = NodePermission() - + # Check class permission by passing it as string Node.has_permission(user, 'change') - + # Check class permission by calling it Node.has_permission.change(user) - + # Check instance permissions node = Node() node.has_permission(user, 'change') @@ -35,7 +35,7 @@ class Permission(object): # call interface: has_permission(user, 'perm') def call(user, perm): return getattr(self, perm)(obj, cls, user) - + # has_permission.perm(user) for func in inspect.getmembers(type(self), predicate=inspect.ismethod): if not isinstance(self, func[1].__self__.__class__): @@ -45,7 +45,7 @@ class Permission(object): # self methods setattr(call, func[0], functools.partial(func[1], self, obj, cls)) return call - + def _aggregate(self, obj, cls, perm): """ Aggregates cls methods to self class""" for method in inspect.getmembers(perm, predicate=inspect.ismethod): @@ -68,7 +68,7 @@ class AllowAllPermission(object): """ Fake object that always returns True """ def __call__(self, *args): return True - + def __getattr__(self, name): return lambda n: True @@ -76,13 +76,13 @@ class AllowAllPermission(object): class RelatedPermission(Permission): """ Inherit permissions of a related object - + The following example will inherit permissions from sliver_iface.sliver.slice SliverIfaces.has_permission = RelatedPermission('sliver.slices') """ def __init__(self, relation): self.relation = relation - + def __get__(self, obj, cls): """ Hacking object internals to provide means for the mentioned interface """ # Walk through FK relations @@ -90,17 +90,17 @@ class RelatedPermission(Permission): if obj is None: parent = cls for relation in relations: - parent = getattr(parent, relation).field.rel.to + parent = getattr(parent, relation).field.model else: parent = functools.reduce(getattr, relations, obj) - + # call interface: has_permission(user, 'perm') def call(user, perm): return parent.has_permission(user, perm) - + # method interface: has_permission.perm(user) for name, func in parent.has_permission.__dict__.items(): if not name.startswith('_'): setattr(call, name, func) - + return call diff --git a/orchestra/templatetags/utils.py b/orchestra/templatetags/utils.py index 9a88fd32..cadde7f9 100644 --- a/orchestra/templatetags/utils.py +++ b/orchestra/templatetags/utils.py @@ -4,7 +4,7 @@ import re from django import template from django.contrib.contenttypes.models import ContentType -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse, NoReverseMatch from django.forms import CheckboxInput from django.template.base import Node from django.template.defaultfilters import date @@ -54,7 +54,7 @@ def rest_to_admin_url(context): class OneLinerNode(Node): def __init__(self, nodelist): self.nodelist = nodelist - + def render(self, context): line = self.nodelist.render(context).replace('\n', ' ') return re.sub(r'\s\s+', '', line) diff --git a/orchestra/urls.py b/orchestra/urls.py index 619192b2..e3f7cefb 100644 --- a/orchestra/urls.py +++ b/orchestra/urls.py @@ -14,7 +14,7 @@ api.autodiscover() urlpatterns = [ # Admin - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), url(r'^admin_tools/', include('admin_tools.urls')), # REST API url(r'^api/', include(api.router.urls)), diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py index 07b0fcb4..fc732343 100644 --- a/orchestra/utils/sys.py +++ b/orchestra/utils/sys.py @@ -71,43 +71,43 @@ def runiterator(command, display=False, stdin=b''): """ Subprocess wrapper for running commands concurrently """ if display: sys.stderr.write("\n\033[1m $ %s\033[0m\n" % command) - + p = subprocess.Popen(command, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - + p.stdin.write(stdin) p.stdin.close() yield - + make_async(p.stdout) make_async(p.stderr) - + # Async reading of stdout and sterr while True: stdout = b'' stderr = b'' # Get complete unicode chunks select.select([p.stdout, p.stderr], [], []) - + stdoutPiece = read_async(p.stdout) stderrPiece = read_async(p.stderr) - + stdout += (stdoutPiece or b'') #.decode('ascii'), errors='replace') stderr += (stderrPiece or b'') #.decode('ascii'), errors='replace') - + if display and stdout: sys.stdout.write(stdout.decode('utf8')) if display and stderr: sys.stderr.write(stderr.decode('utf8')) - + state = _Attribute(stdout) state.stderr = stderr state.exit_code = p.poll() state.command = command yield state - + if state.exit_code != None: p.stdout.close() p.stderr.close() @@ -121,12 +121,12 @@ def join(iterator, display=False, silent=False, valid_codes=(0,)): for state in iterator: stdout += state.stdout stderr += state.stderr - + exit_code = state.exit_code - + out = _Attribute(stdout.strip()) err = stderr.strip() - + out.failed = False out.exit_code = exit_code out.stderr = err @@ -138,7 +138,7 @@ def join(iterator, display=False, silent=False, valid_codes=(0,)): sys.stderr.write("\n\033[1;31mCommandError: %s %s\033[m\n" % (msg, err)) if not silent: raise CommandError("%s %s" % (msg, err)) - + out.succeeded = not out.failed return out @@ -151,10 +151,10 @@ def joinall(iterators, **kwargs): return results -def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', async=False): +def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', run_async=False): iterator = runiterator(command, display, stdin) next(iterator) - if async: + if run_async: return iterator return join(iterator, display=display, silent=silent, valid_codes=valid_codes) @@ -213,7 +213,7 @@ class LockFile(object): self.lockfile = lockfile self.expire = expire self.unlocked = unlocked - + def acquire(self): if os.path.exists(self.lockfile): lock_time = os.path.getmtime(self.lockfile) @@ -222,17 +222,17 @@ class LockFile(object): return False touch(self.lockfile) return True - + def release(self): os.remove(self.lockfile) - + def __enter__(self): if not self.unlocked: if not self.acquire(): raise OperationLocked("%s lock file exists and its mtime is less than %s seconds" % (self.lockfile, self.expire)) return True - + def __exit__(self, type, value, traceback): if not self.unlocked: self.release() @@ -240,4 +240,4 @@ class LockFile(object): def touch_wsgi(delay=0): from . import paths - run('{ sleep %i && touch %s/wsgi.py; } &' % (delay, paths.get_project_dir()), async=True) + run('{ sleep %i && touch %s/wsgi.py; } &' % (delay, paths.get_project_dir()), run_async=True) diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index 98b75ff5..ae37bd6f 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -5,7 +5,7 @@ from functools import wraps from django.conf import settings from django.contrib.auth import BACKEND_SESSION_KEY, SESSION_KEY from django.contrib.sessions.backends.db import SessionStore -from django.core.urlresolvers import reverse +from django.urls import reverse from django.test import LiveServerTestCase, TestCase from selenium.webdriver.firefox.webdriver import WebDriver from xvfbwrapper import Xvfb @@ -17,7 +17,7 @@ from .python import random_ascii class AppDependencyMixin(object): DEPENDENCIES = () - + @classmethod def setUpClass(cls): current_app = cls.__module__.split('.tests.')[0] @@ -70,13 +70,13 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): cls.vdisplay.start() cls.selenium = WebDriver() super(BaseLiveServerTestCase, cls).setUpClass() - + @classmethod def tearDownClass(cls): cls.selenium.quit() cls.vdisplay.stop() super(BaseLiveServerTestCase, cls).tearDownClass() - + def create_account(self, username='', superuser=False): if not username: username = '%s_superaccount' % random_ascii(5) @@ -85,17 +85,17 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): if superuser: return Account.objects.create_superuser(username, password=password, email='orchestra@orchestra.org') return Account.objects.create_user(username, password=password, email='orchestra@orchestra.org') - + def setUp(self): from orm.api import Api super(BaseLiveServerTestCase, self).setUp() self.rest = Api(self.live_server_url + '/api/') self.rest.enable_logging() self.account = self.create_account(superuser=True) - + def admin_login(self): session = SessionStore() - session[SESSION_KEY] = self.account_id + session[SESSION_KEY] = self.account.id session[BACKEND_SESSION_KEY] = settings.AUTHENTICATION_BACKENDS[0] session.save() ## to set a cookie we need to first visit the domain. @@ -105,16 +105,16 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): value=session.session_key, # path='/', )) - + def rest_login(self): self.rest.login(username=self.account.username, password=self.account_password) - + def take_screenshot(self): timestamp = datetime.datetime.now().isoformat().replace(':', '') filename = 'screenshot_%s_%s.png' % (self.id(), timestamp) path = '/home/orchestra/snapshots' self.selenium.save_screenshot(os.path.join(path, filename)) - + def admin_delete(self, obj): opts = obj._meta app_label, model_name = opts.app_label, opts.model_name @@ -124,7 +124,7 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): confirmation = self.selenium.find_element_by_name('post') confirmation.submit() self.assertNotEqual(url, self.selenium.current_url) - + def admin_disable(self, obj): opts = obj._meta app_label, model_name = opts.app_label, opts.model_name @@ -136,20 +136,20 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): save = self.selenium.find_element_by_name('_save') save.submit() self.assertNotEqual(url, self.selenium.current_url) - + def admin_change_password(self, obj, password): opts = obj._meta app_label, model_name = opts.app_label, opts.model_name change_password = reverse('admin:%s_%s_change_password' % (app_label, model_name), args=(obj.pk,)) url = self.live_server_url + change_password self.selenium.get(url) - + password_field = self.selenium.find_element_by_id('id_password1') password_field.send_keys(password) password_field = self.selenium.find_element_by_id('id_password2') password_field.send_keys(password) password_field.submit() - + self.assertNotEqual(url, self.selenium.current_url) def snapshot_on_error(test): @@ -174,7 +174,7 @@ def save_response_on_error(test): timestamp = datetime.datetime.now().isoformat().replace(':', '') filename = '%s_%s.html' % (self.id(), timestamp) path = '/home/orchestra/snapshots' - with open(os.path.join(path, filename), 'w') as dumpfile: + with open(os.path.join(path, filename), 'wb') as dumpfile: dumpfile.write(self.rest.last_response.content) raise return inner diff --git a/requirements.txt b/requirements.txt index 7a96243f..4a828a28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -Django==1.10.5 -django-fluent-dashboard==0.6.1 -django-admin-tools==0.8.0 -django-extensions==1.7.4 -django-celery==3.1.17 +Django==2.0 +django-fluent-dashboard==1.0.1 +django-admin-tools==0.9.1 +django-extensions==2.1.1 +django-celery==3.2.1 celery==3.1.23 kombu==3.0.35 billiard==3.3.0.23 Markdown==2.4 -djangorestframework==3.4.7 +djangorestframework==3.9.3 ecdsa==0.11 Pygments==1.6 -django-filter==0.15.2 +django-filter==1.1 jsonfield==0.9.22 python-dateutil==2.2 https://github.com/glic3rinu/passlib/archive/master.zip diff --git a/total_requirements.txt b/total_requirements.txt index 98f23183..f360a3ef 100644 --- a/total_requirements.txt +++ b/total_requirements.txt @@ -21,17 +21,17 @@ django-localflavor amqp anyjson pytz -cracklib +cracklib lxml==3.3.5 -selenium -xvfbwrapper -freezegun -coverage -flake8 -django-debug-toolbar==1.3.0 -django-nose==1.4.4 -sqlparse -pyinotify +selenium +xvfbwrapper +freezegun==0.3.14 +coverage +flake8 +django-debug-toolbar==1.3.0 +django-nose==1.4.4 +sqlparse +pyinotify PyMySQL dj_database_url==0.5.0 psycopg2-binary