From cf2215f6045a00e1530b9bda7de555ce7c4a7956 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 14 May 2015 13:28:54 +0000 Subject: [PATCH] Fixes on billing and added support for async backend actions --- TODO.md | 31 ++- orchestra/admin/options.py | 9 +- orchestra/contrib/mailboxes/models.py | 2 +- orchestra/contrib/mailboxes/settings.py | 16 +- orchestra/contrib/orchestration/admin.py | 18 +- orchestra/contrib/orchestration/manager.py | 10 +- .../migrations/0004_route_async_actions.py | 20 ++ orchestra/contrib/orchestration/models.py | 8 +- orchestra/contrib/orders/actions.py | 1 + orchestra/contrib/orders/billing.py | 20 +- orchestra/contrib/orders/forms.py | 4 +- orchestra/contrib/plans/models.py | 2 +- orchestra/contrib/plans/ratings.py | 67 +++--- orchestra/contrib/services/models.py | 2 + .../contrib/services/tests/test_handler.py | 201 +++++++++++++++--- .../contrib/webapps/backends/__init__.py | 4 +- .../contrib/webapps/backends/wordpress.py | 7 +- orchestra/models/fields.py | 15 +- 18 files changed, 329 insertions(+), 108 deletions(-) create mode 100644 orchestra/contrib/orchestration/migrations/0004_route_async_actions.py diff --git a/TODO.md b/TODO.md index 6b0ff87a..c3da63d0 100644 --- a/TODO.md +++ b/TODO.md @@ -174,7 +174,7 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * allow empty metric pack for default rates? changes on rating algo # don't produce lines with cost == 0 or quantity 0 ? maybe minimal quantity for billing? like 0.1 ? or minimal price? per line or per bill? -# lines too long on invoice, double lines or cut, and make margin wider +# lines too long on invoice, double lines or cut * payment methods icons * use server.name | server.address on python backends, like gitlab instead of settings? @@ -183,11 +183,11 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * update service orders on a celery task? because it take alot # FIXME do more test, make sure billed until doesn't get uodated whhen services are billed with les metric, and don't upgrade billed_until when undoing under this circumstances - * line 513: change threshold and one time service metric change should update last value if not billed, only record for recurring invoicing. postpay services should store the last metric for pricing period. - * add ini, end dates on bill lines and breakup quanity into size(defaut:1) and metric - * threshold for significative metric accountancy on services.handler - * http://orchestra.pangea.org/admin/orders/order/6418/ - * http://orchestra.pangea.org/admin/orders/order/6495/bill_selected_orders/ +# * line 513: change threshold and one time service metric change should update last value if not billed, only record for recurring invoicing. postpay services should store the last metric for pricing period. +# * add ini, end dates on bill lines and breakup quanity into size(defaut:1) and metric +# * threshold for significative metric accountancy on services.handler +# * http://orchestra.pangea.org/admin/orders/order/6418/ +# * http://orchestra.pangea.org/admin/orders/order/6495/bill_selected_orders/ * move normurlpath to orchestra.utils from websites.utils @@ -254,8 +254,6 @@ https://code.djangoproject.com/ticket/24576 * move all tests to django-orchestra/tests * *natural keys: those fields that uniquely identify a service, list.name, website.name, webapp.name+account, make sure rest api can not edit thos things -# migrations accounts, bill, orders, auth -> migrate the rest (contacts lambda error) - * MultiCHoiceField proper serialization @@ -275,19 +273,14 @@ https://code.djangoproject.com/ticket/24576 # bill.totals make it 100% computed? * joomla: wget https://github.com/joomla/joomla-cms/releases/download/3.4.1/Joomla_3.4.1-Stable-Full_Package.tar.gz -O - | tar xvfz - - -# bill confirmation: show total # Amend lines??? # orders currency setting # Determine the difference between data serializer used for validation and used for the rest API! # Make PluginApiView that fills metadata and other stuff like modeladmin plugin support -# custom validation for settings # TODO orchestra related services code reload: celery/uwsgi reloading find aonther way without root and implement reload -# insert settings on dashboard dynamically -# convert all complex settings to string # size monitor of @002 @003 database names # password validation cracklib on change password form=????? # reset setting button @@ -353,18 +346,20 @@ make django admin taskstate uncollapse fucking traceback, ( if exists ?) resorce monitoring more efficient, less mem an better queries for calc current data -# test best_price rating method - # bill this https://orchestra.pangea.org/admin/orders/order/8236/ should be already billed, <= vs < # Convert rating method from function to PluginClass # Tests can not run because django.db.utils.ProgrammingError: relation "accounts_account" does not exist # autoresponses on mailboxes, not addresses or remove them -# Async specific backend actions? systemusers.set_permission - - +# ACL don't give exec permissions to files! +# force save and continue on routes (and others?) # gevent for python3 apt-get install cython3 export CYTHON='cython3' pip3 install https://github.com/fantix/gevent/archive/master.zip + + +# SIgnal handler for notify workers to reload stuff, like resource sync: https://docs.python.org/2/library/signal.html + +# INVOICE fucking Id based on order ID or what? diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py index fb0b9a3c..3396b97e 100644 --- a/orchestra/admin/options.py +++ b/orchestra/admin/options.py @@ -113,6 +113,7 @@ class ChangeAddFieldsMixin(object): add_form = None add_prepopulated_fields = {} change_readonly_fields = () + change_form = None add_inlines = None def get_prepopulated_fields(self, request, obj=None): @@ -151,8 +152,12 @@ class ChangeAddFieldsMixin(object): def get_form(self, request, obj=None, **kwargs): """ Use special form during user creation """ defaults = {} - if obj is None and self.add_form: - defaults['form'] = self.add_form + if obj is None: + if self.add_form: + defaults['form'] = self.add_form + else: + if self.change_form: + defaults['form'] = self.change_form defaults.update(kwargs) return super(ChangeAddFieldsMixin, self).get_form(request, obj, **defaults) diff --git a/orchestra/contrib/mailboxes/models.py b/orchestra/contrib/mailboxes/models.py index 41617c00..ac0e2704 100644 --- a/orchestra/contrib/mailboxes/models.py +++ b/orchestra/contrib/mailboxes/models.py @@ -22,7 +22,7 @@ class Mailbox(models.Model): related_name='mailboxes') filtering = models.CharField(max_length=16, default=settings.MAILBOXES_MAILBOX_DEFAULT_FILTERING, - choices=[(k, v[0]) for k,v in settings.MAILBOXES_MAILBOX_FILTERINGS.items()]) + choices=[(k, v[0]) for k,v in sorted(settings.MAILBOXES_MAILBOX_FILTERINGS.items())]) custom_filtering = models.TextField(_("filtering"), blank=True, validators=[validators.validate_sieve], help_text=_("Arbitrary email filtering in sieve language. " diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py index 3cbc3f39..790678e0 100644 --- a/orchestra/contrib/mailboxes/settings.py +++ b/orchestra/contrib/mailboxes/settings.py @@ -82,18 +82,30 @@ MAILBOXES_MAILBOX_FILTERINGS = Setting('MAILBOXES_MAILBOX_FILTERINGS', { # value: (verbose_name, filter) 'DISABLE': (_("Disable"), ''), - 'REJECT': (mark_safe_lazy(_("Reject spam (Score≥9)")), textwrap.dedent(""" + 'REJECT': (mark_safe_lazy(_("Reject spam (Score≥9)")), textwrap.dedent("""\ require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "9" { discard; stop; }""")), - 'REDIRECT': (mark_safe_lazy(_("Archive spam (Score≥9)")), textwrap.dedent(""" + 'REJECT5': (mark_safe_lazy(_("Reject spam (Score≥5)")), textwrap.dedent("""\ + require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; + if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" { + discard; + stop; + }""")), + 'REDIRECT': (mark_safe_lazy(_("Archive spam (Score≥9)")), textwrap.dedent("""\ require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "9" { fileinto "Spam"; stop; }""")), + 'REDIRECT5': (mark_safe_lazy(_("Archive spam (Score≥5)")), textwrap.dedent("""\ + require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"]; + if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" { + fileinto "Spam"; + stop; + }""")), 'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering), } ) diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index dc655d8b..cac1404d 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -3,6 +3,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ +from orchestra.admin import ExtendedModelAdmin from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono from . import settings, helpers @@ -23,13 +24,28 @@ STATE_COLORS = { } -class RouteAdmin(admin.ModelAdmin): +from django import forms +from orchestra.forms.widgets import SpanWidget +from orchestra.forms.widgets import paddingCheckboxSelectMultiple +class RouteForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super(RouteForm, self).__init__(*args, **kwargs) + if self.instance: + self.fields['backend'].widget = SpanWidget() + self.fields['backend'].required = False + self.fields['async_actions'].widget = paddingCheckboxSelectMultiple(45) + self.fields['async_actions'].choices = ((action, action) for action in self.instance.backend_class.actions) + + +class RouteAdmin(ExtendedModelAdmin): list_display = ( 'backend', 'host', 'match', 'display_model', 'display_actions', 'async', 'is_active' ) list_editable = ('host', 'match', 'async', 'is_active') list_filter = ('host', 'is_active', 'async', 'backend') ordering = ('backend',) + add_fields = ('backend', 'host', 'match', 'async', 'is_active') + change_form = RouteForm BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends()) DEFAULT_MATCH = { diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index 1824c15c..65159cff 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -62,7 +62,9 @@ def generate(operations): if operation.routes is None: operation.routes = router.get_routes(operation, cache=cache) for route in operation.routes: - key = (route, operation.backend) + # TODO key by action.async + async_action = route.action_is_async(operation.action) + key = (route, operation.backend, async_action) if key not in scripts: backend, operations = (operation.backend(), [operation]) scripts[key] = (backend, operations) @@ -111,13 +113,13 @@ def execute(scripts, serialize=False, async=None): threads_to_join = [] logs = [] for key, value in scripts.items(): - route, __ = key + route, __, async_action = key backend, operations = value args = (route.host,) if async is None: - is_async = not serialize and route.async + is_async = not serialize and (route.async or async_action) else: - is_async = not serialize and async + is_async = not serialize and (async or async_action) kwargs = { 'async': is_async, } diff --git a/orchestra/contrib/orchestration/migrations/0004_route_async_actions.py b/orchestra/contrib/orchestration/migrations/0004_route_async_actions.py new file mode 100644 index 00000000..25349b84 --- /dev/null +++ b/orchestra/contrib/orchestration/migrations/0004_route_async_actions.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import orchestra.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '0003_auto_20150512_1512'), + ] + + operations = [ + migrations.AddField( + model_name='route', + name='async_actions', + field=orchestra.models.fields.MultiSelectField(blank=True, max_length=256), + ), + ] diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index 9a9482fa..1e1f3644 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -8,7 +8,7 @@ from django.utils.module_loading import autodiscover_modules from django.utils.translation import ugettext_lazy as _ from orchestra.core.validators import validate_ip_address, ValidationError -from orchestra.models.fields import NullableCharField +from orchestra.models.fields import NullableCharField, MultiSelectField #from orchestra.utils.apps import autodiscover from . import settings @@ -157,10 +157,13 @@ class Route(models.Model): async = models.BooleanField(default=False, help_text=_("Whether or not block the request/response cycle waitting this backend to " "finish its execution. Usually you want slave servers to run asynchronously.")) + async_actions = MultiSelectField(max_length=256, blank=True, + help_text=_("Specify individual actions to be executed asynchronoulsy.")) # method = models.CharField(_("method"), max_lenght=32, choices=method_choices, # default=MethodBackend.get_default()) is_active = models.BooleanField(_("active"), default=True) + class Meta: unique_together = ('backend', 'host') @@ -210,6 +213,9 @@ class Route(models.Model): name = type(exception).__name__ raise ValidationError(': '.join((name, exception))) + def action_is_async(self, action): + return action in self.async_actions + def matches(self, instance): safe_locals = { 'instance': instance, diff --git a/orchestra/contrib/orders/actions.py b/orchestra/contrib/orders/actions.py index 4e1023da..090baa9b 100644 --- a/orchestra/contrib/orders/actions.py +++ b/orchestra/contrib/orders/actions.py @@ -42,6 +42,7 @@ class BillSelectedOrders(object): proforma=form.cleaned_data['proforma'], new_open=form.cleaned_data['new_open'], ) + print(self.options) if int(request.POST.get('step')) != 3: return self.select_related(request) else: diff --git a/orchestra/contrib/orders/billing.py b/orchestra/contrib/orders/billing.py index 56ab1fcb..fb9aa7b4 100644 --- a/orchestra/contrib/orders/billing.py +++ b/orchestra/contrib/orders/billing.py @@ -8,6 +8,7 @@ from orchestra.contrib.bills.models import Invoice, Fee, ProForma class BillsBackend(object): def create_bills(self, account, lines, **options): bill = None + ant_bill = None bills = [] create_new = options.get('new_open', False) proforma = options.get('proforma', False) @@ -17,24 +18,33 @@ class BillsBackend(object): continue service = line.order.service # Create bill if needed - if bill is None or service.is_fee: - if proforma: + if proforma: + if ant_bill is None: if create_new: bill = ProForma.objects.create(account=account) else: bill = ProForma.objects.filter(account=account, is_open=True).last() if not bill: bill = ProForma.objects.create(account=account, is_open=True) - elif service.is_fee: - bill = Fee.objects.create(account=account) + bills.append(bill) else: + bill = ant_bill + ant_bill = bill + elif service.is_fee: + bill = Fee.objects.create(account=account) + bills.append(bill) + else: + if ant_bill is None: if create_new: bill = Invoice.objects.create(account=account) else: bill = Invoice.objects.filter(account=account, is_open=True).last() if not bill: bill = Invoice.objects.create(account=account, is_open=True) - bills.append(bill) + bills.append(bill) + else: + bill = ant_bill + ant_bill = bill # Create bill line billine = bill.lines.create( rate=service.nominal_price, diff --git a/orchestra/contrib/orders/forms.py b/orchestra/contrib/orders/forms.py index 410bad4c..2cdde9f8 100644 --- a/orchestra/contrib/orders/forms.py +++ b/orchestra/contrib/orders/forms.py @@ -50,7 +50,7 @@ class BillSelectRelatedForm(AdminFormMixin, forms.Form): billing_point = forms.DateField(widget=forms.HiddenInput()) fixed_point = forms.BooleanField(widget=forms.HiddenInput(), required=False) proforma = forms.BooleanField(widget=forms.HiddenInput(), required=False) - create_new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) + new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) def __init__(self, *args, **kwargs): super(BillSelectRelatedForm, self).__init__(*args, **kwargs) @@ -64,4 +64,4 @@ class BillSelectConfirmationForm(AdminFormMixin, forms.Form): billing_point = forms.DateField(widget=forms.HiddenInput()) fixed_point = forms.BooleanField(widget=forms.HiddenInput(), required=False) proforma = forms.BooleanField(widget=forms.HiddenInput(), required=False) - create_new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) + new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) diff --git a/orchestra/contrib/plans/models.py b/orchestra/contrib/plans/models.py index b083a570..cd7d6a6a 100644 --- a/orchestra/contrib/plans/models.py +++ b/orchestra/contrib/plans/models.py @@ -64,7 +64,7 @@ class RateQuerySet(models.QuerySet): return self.filter( Q(plan__is_default=True) | Q(plan__contracts__account=account) - ).order_by('plan', 'quantity').select_related('plan') + ).order_by('plan', 'quantity').select_related('plan', 'service') class Rate(models.Model): diff --git a/orchestra/contrib/plans/ratings.py b/orchestra/contrib/plans/ratings.py index 18606f7d..d9a69903 100644 --- a/orchestra/contrib/plans/ratings.py +++ b/orchestra/contrib/plans/ratings.py @@ -44,24 +44,24 @@ def _compute_steps(rates, metric): return value, steps -def _prepend_missing(rates): +def _standardize(rates): """ Support for incomplete rates When first rate (quantity=5, price=10) defaults to nominal_price """ - if rates: - first = rates[0] - if first.quantity == 0: - first.quantity = 1 - elif first.quantity > 1: - if not isinstance(rates, list): - rates = list(rates) - service = first.service - rate_class = type(first) - rates.insert(0, - rate_class(service=service, plan=first.plan, quantity=1, price=service.nominal_price) + std_rates = [] + minimal = rates[0].quantity + for rate in rates: + if rate.quantity == 0: + rate.quantity = 1 + elif rate.quantity == minimal and rate.quantity > 1: + service = rate.service + rate_class = type(rate) + std_rates.append( + rate_class(service=service, plan=rate.plan, quantity=1, price=service.nominal_price) ) - return rates + std_rates.append(rate) + return std_rates def step_price(rates, metric): @@ -71,7 +71,7 @@ def step_price(rates, metric): group = [] minimal = (sys.maxsize, []) for plan, rates in rates.group_by('plan').items(): - rates = _prepend_missing(rates) + rates = _standardize(rates) value, steps = _compute_steps(rates, metric) if plan.is_combinable: group.append(steps) @@ -122,7 +122,7 @@ def step_price(rates, metric): minimal = min(minimal, (value, result), key=lambda v: v[0]) return minimal[1] step_price.verbose_name = _("Step price") -step_price.help_text = _("All rates with a quantity lower than the metric are applied. " +step_price.help_text = _("All rates with a quantity lower or equal than the metric are applied. " "Nominal price will be used when initial block is missing.") @@ -132,7 +132,7 @@ def match_price(rates, metric): candidates = [] selected = False prev = None - rates = _prepend_missing(rates.distinct()) + rates = _standardize(rates.distinct()) for rate in rates: if prev: if prev.plan != rate.plan: @@ -163,25 +163,42 @@ def best_price(rates, metric): raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") candidates = [] for plan, rates in rates.group_by('plan').items(): - rates = _prepend_missing(rates) + rates = _standardize(rates) plan_candidates = [] for rate in rates: if rate.quantity > metric: break if plan_candidates: - plan_candidates[-1].barrier = rate.quantity - plan_candidates.append(AttrDict( - price=rate.price, - barrier=metric, - )) + ant = plan_candidates[-1] + if ant.price == rate.price: + # Multiple plans support + ant.fold += 1 + else: + ant.quantity = rate.quantity-1 + plan_candidates.append(AttrDict( + price=rate.price, + quantity=metric, + fold=1, + )) + else: + plan_candidates.append(AttrDict( + price=rate.price, + quantity=metric, + fold=1, + )) candidates.extend(plan_candidates) results = [] accumulated = 0 for candidate in sorted(candidates, key=lambda c: c.price): - if accumulated+candidate.barrier > metric: + if candidate.quantity < accumulated: + # Out of barrier + continue + candidate.quantity *= candidate.fold + if accumulated+candidate.quantity > metric: quantity = metric - accumulated else: - quantity = candidate.barrier + quantity = candidate.quantity + accumulated += quantity if quantity: if results and results[-1].price == candidate.price: results[-1].quantity += quantity @@ -192,4 +209,4 @@ def best_price(rates, metric): })) return results best_price.verbose_name = _("Best price") -best_price.help_text = _("Produces the best possible price given all active rating lines.") +best_price.help_text = _("Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).") diff --git a/orchestra/contrib/services/models.py b/orchestra/contrib/services/models.py index 2a6f2574..5bfd55f4 100644 --- a/orchestra/contrib/services/models.py +++ b/orchestra/contrib/services/models.py @@ -214,6 +214,7 @@ class Service(models.Model): return decimal.Decimal(str(accumulated)) ant_counter = counter accumulated += rate['price'] * rate['quantity'] + raise RuntimeError("Rating algorithm bad result") else: if metric < position: raise ValueError("Metric can not be less than the position.") @@ -221,6 +222,7 @@ class Service(models.Model): counter += rate['quantity'] if counter >= position: return decimal.Decimal(str(rate['price'])) + raise RuntimeError("Rating algorithm bad result") def get_rates(self, account, cache=True): # rates are cached per account diff --git a/orchestra/contrib/services/tests/test_handler.py b/orchestra/contrib/services/tests/test_handler.py index fc3a611e..7cc4650a 100644 --- a/orchestra/contrib/services/tests/test_handler.py +++ b/orchestra/contrib/services/tests/test_handler.py @@ -32,8 +32,8 @@ class HandlerTests(BaseTestCase): 'orchestra.contrib.systemusers', ) - def create_ftp_service(self): - service = Service.objects.create( + def create_ftp_service(self, **kwargs): + default = dict( description="FTP Account", content_type=ContentType.objects.get_for_model(SystemUser), match='not systemuser.is_main', @@ -46,10 +46,18 @@ class HandlerTests(BaseTestCase): on_cancel=Service.DISCOUNT, payment_style=Service.PREPAY, tax=0, - nominal_price=10, + nominal_price=10 ) + default.update(kwargs) + service = Service.objects.create(**default) return service + def validate_results(self, rates, results): + self.assertEqual(len(rates), len(results)) + for rate, result in zip(rates, results): + self.assertEqual(rate['price'], result.price) + self.assertEqual(rate['quantity'], result.quantity) + def test_get_chunks(self): service = self.create_ftp_service() handler = service.handler @@ -239,9 +247,7 @@ class HandlerTests(BaseTestCase): 'quantity': 21 } ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) dupeplan = Plan.objects.create( name='DUPE', allow_multiple=True, is_combinable=True) @@ -249,9 +255,7 @@ class HandlerTests(BaseTestCase): service.rates.create(plan=dupeplan, quantity=3, price=9) results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) account.plans.create(plan=dupeplan) results = service.get_rates(account, cache=False) @@ -261,9 +265,7 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('9.00'), 'quantity': 5}, {'price': decimal.Decimal('1.00'), 'quantity': 21}, ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) hyperplan = Plan.objects.create( name='HYPER', allow_multiple=False, is_combinable=False) @@ -276,9 +278,8 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('0.00'), 'quantity': 19}, {'price': decimal.Decimal('5.00'), 'quantity': 11} ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) + hyperplan.is_combinable = True hyperplan.save() results = service.get_rates(account, cache=False) @@ -287,9 +288,7 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('0.00'), 'quantity': 23}, {'price': decimal.Decimal('1.00'), 'quantity': 7} ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) service.rate_algorithm = 'orchestra.contrib.plans.ratings.match_price' service.save() @@ -335,9 +334,7 @@ class HandlerTests(BaseTestCase): 'quantity': 21 } ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) def test_zero_rates(self): service = self.create_ftp_service() @@ -357,9 +354,7 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('9.00'), 'quantity': 6}, {'price': decimal.Decimal('1.00'), 'quantity': 21} ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) def test_rates_allow_multiple(self): service = self.create_ftp_service() @@ -375,9 +370,7 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('0.00'), 'quantity': 2}, {'price': decimal.Decimal('9.00'), 'quantity': 28}, ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) account.plans.create(plan=dupeplan) results = service.get_rates(account, cache=False) @@ -386,9 +379,7 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('0.00'), 'quantity': 4}, {'price': decimal.Decimal('9.00'), 'quantity': 26}, ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) account.plans.create(plan=dupeplan) results = service.get_rates(account, cache=False) @@ -397,6 +388,150 @@ class HandlerTests(BaseTestCase): {'price': decimal.Decimal('0.00'), 'quantity': 6}, {'price': decimal.Decimal('9.00'), 'quantity': 24}, ] - for rate, result in zip(rates, results): - self.assertEqual(rate['price'], result.price) - self.assertEqual(rate['quantity'], result.quantity) + self.validate_results(rates, results) + + def test_best_price(self): + service = self.create_ftp_service(rate_algorithm='orchestra.contrib.plans.ratings.best_price') + account = self.create_account() + dupeplan = Plan.objects.create(name='DUPE') + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=0, price=0) + service.rates.create(plan=dupeplan, quantity=2, price=9) + service.rates.create(plan=dupeplan, quantity=3, price=8) + service.rates.create(plan=dupeplan, quantity=4, price=7) + service.rates.create(plan=dupeplan, quantity=5, price=10) + service.rates.create(plan=dupeplan, quantity=10, price=5) + raw_rates = service.get_rates(account, cache=False) + results = service.rate_method(raw_rates, 2) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('9.00'), + 'quantity': 1 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('8.00'), + 'quantity': 2 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 5) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('7.00'), + 'quantity': 4 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 9) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('7.00'), + 'quantity': 4 + }, + { + 'price': decimal.Decimal('10.00'), + 'quantity': 4 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 9 + }, + ] + self.validate_results(rates, results) + + def test_best_price_multiple(self): + service = self.create_ftp_service(rate_algorithm='orchestra.contrib.plans.ratings.best_price') + account = self.create_account() + dupeplan = Plan.objects.create(name='DUPE') + account.plans.create(plan=dupeplan) + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=0, price=0) + service.rates.create(plan=dupeplan, quantity=2, price=9) + service.rates.create(plan=dupeplan, quantity=3, price=8) + service.rates.create(plan=dupeplan, quantity=4, price=7) + service.rates.create(plan=dupeplan, quantity=5, price=10) + service.rates.create(plan=dupeplan, quantity=10, price=5) + raw_rates = service.get_rates(account, cache=False) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 2 + }, + { + 'price': decimal.Decimal('8.00'), + 'quantity': 1 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 2 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 8 + }, + ] + self.validate_results(rates, results) + + account.plans.create(plan=dupeplan) + raw_rates = service.get_rates(account, cache=False) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 3 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 3 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 7 + }, + ] + self.validate_results(rates, results) diff --git a/orchestra/contrib/webapps/backends/__init__.py b/orchestra/contrib/webapps/backends/__init__.py index b750b114..bcf63ffe 100644 --- a/orchestra/contrib/webapps/backends/__init__.py +++ b/orchestra/contrib/webapps/backends/__init__.py @@ -29,9 +29,9 @@ class WebAppServiceMixin(object): if context['under_construction_path']: self.append(textwrap.dedent("""\ if [[ $CREATED == 1 && ! $(ls -A %(app_path)s) ]]; then - # Async wait for other backends to do their thing or cp under construction + # Async wait 2 more seconds for other backends to lock app_path or cp under construction nohup bash -c ' - sleep 10 + sleep 2 if [[ ! $(ls -A %(app_path)s) ]]; then cp -r %(under_construction_path)s %(app_path)s chown -R %(user)s:%(group)s %(app_path)s diff --git a/orchestra/contrib/webapps/backends/wordpress.py b/orchestra/contrib/webapps/backends/wordpress.py index 9a31cb6f..5a2454ed 100644 --- a/orchestra/contrib/webapps/backends/wordpress.py +++ b/orchestra/contrib/webapps/backends/wordpress.py @@ -43,7 +43,8 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): die("App directory not empty."); } shell_exec("mkdir -p %(app_path)s - rm -f %(app_path)s/index.html + # Prevent other backends from writting here + touch %(app_path)s/.lock filename=\\$(wget https://wordpress.org/latest.tar.gz --server-response --spider --no-check-certificate 2>&1 | grep filename | cut -d'=' -f2) mkdir -p %(cms_cache_dir)s if [ \\$(basename \\$(readlink %(cms_cache_dir)s/wordpress) 2> /dev/null ) != \\$filename ]; then @@ -54,7 +55,9 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): tar -xzvf %(cms_cache_dir)s/wordpress -C %(app_path)s --strip-components=1 fi mkdir %(app_path)s/wp-content/uploads - chmod 750 %(app_path)s/wp-content/uploads"); + chmod 750 %(app_path)s/wp-content/uploads + rm %(app_path)s/.lock + "); $config_file = file('%(app_path)s/' . 'wp-config-sample.php'); $secret_keys = file_get_contents('https://api.wordpress.org/secret-key/1.1/salt/'); diff --git a/orchestra/models/fields.py b/orchestra/models/fields.py index 3b12deb1..3b34c934 100644 --- a/orchestra/models/fields.py +++ b/orchestra/models/fields.py @@ -27,10 +27,6 @@ class MultiSelectField(models.CharField, metaclass=models.SubfieldBase): def to_python(self, value): if value: -# if isinstance(value, tuple) and value[0].startswith('('): -# # Workaround unknown bug on default model values -# # [u"('SUPPORT'", u" 'ADMIN'", u" 'BILLING'", u" 'TECH'", u" 'ADDS'", u" 'EMERGENCY')"] -# value = list(eval(', '.join(value))) if isinstance(value, str): return value.split(',') return value @@ -44,11 +40,12 @@ class MultiSelectField(models.CharField, metaclass=models.SubfieldBase): setattr(cls, 'get_%s_display' % self.name, func) def validate(self, value, model_instance): - arr_choices = self.get_choices_selected(self.get_choices_default()) - for opt_select in value: - if (opt_select not in arr_choices): - msg = self.error_messages['invalid_choice'] % value - raise exceptions.ValidationError(msg) + if self.choices: + arr_choices = self.get_choices_selected(self.get_choices_default()) + for opt_select in value: + if (opt_select not in arr_choices): + msg = self.error_messages['invalid_choice'] % {'value': opt_select} + raise exceptions.ValidationError(msg) def get_choices_selected(self, arr_choices=''): if not arr_choices: