From 3963f6ce86e176d320147e062ea6f38de093c3f8 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Fri, 15 May 2015 14:19:24 +0000 Subject: [PATCH] Fixes on billing --- TODO.md | 8 ++- orchestra/bin/orchestra-admin | 11 +++- .../bills/templates/bills/microspective.css | 2 +- orchestra/contrib/domains/forms.py | 3 +- orchestra/contrib/mailer/backends.py | 3 +- orchestra/contrib/orders/filters.py | 57 ++++++++++++------- orchestra/contrib/orders/models.py | 24 +++++--- .../orders/order/bill_selected_options.html | 4 +- orchestra/utils/html.py | 17 ++++-- 9 files changed, 84 insertions(+), 45 deletions(-) diff --git a/TODO.md b/TODO.md index c3da63d0..03330caa 100644 --- a/TODO.md +++ b/TODO.md @@ -13,7 +13,10 @@ * backend logs with hal logo -* LAST version of this shit http://wkhtmltopdf.org/downloads.h otml +# LAST version of this shit http://wkhtmltopdf.org/downloads.h otml +#apt-get install xfonts-75dpi +#wget http://downloads.sourceforge.net/wkhtmltopdf/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb +#dpkg -i wkhtmltox-0.12.2.1_linux-jessie-amd64.deb * help_text on readonly_fields specialy Bill.state. (eg. A bill is in OPEN state when bla bla ) @@ -363,3 +366,6 @@ pip3 install https://github.com/fantix/gevent/archive/master.zip # SIgnal handler for notify workers to reload stuff, like resource sync: https://docs.python.org/2/library/signal.html # INVOICE fucking Id based on order ID or what? + +# user order_id as bill line id +# BUG Delete related services also deletes account! diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index a55e78a4..dd9f1e6c 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -123,6 +123,7 @@ function install_requirements () { ORCHESTRA_PATH=$(get_orchestra_dir) || true # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev + # wkhtmltopdf: xfonts-75dpi, xvfb APT="python3 \ python3-pip \ python3-dev \ @@ -131,12 +132,13 @@ function install_requirements () { zlib1g-dev \ bind9utils \ wkhtmltopdf \ + xfonts-75dpi \ xvfb \ ca-certificates \ gettext \ libcrack2-dev" - # cracklib and lxml are excluded on the requirements because they are hard to build + # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies PIP="$(wget https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt -q -O - | tr '\n' ' ') \ cracklib \ lxml==3.3.5" @@ -153,12 +155,17 @@ function install_requirements () { coverage \ flake8 \ django-debug-toolbar==1.3.0 \ - https://github.com/django-nose/django-nose/archive/master.zip \ + django-nose==1.4 \ sqlparse \ pyinotify \ PyMySQL" fi + # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support) + wkhtmltox=$(mktemp) + wget http://downloads.sourceforge.net/wkhtmltopdf/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb -O ${wkhtmltox} + dpkg -i ${wkhtmltox} + # Make sure locales are in place before installing postgres if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen diff --git a/orchestra/contrib/bills/templates/bills/microspective.css b/orchestra/contrib/bills/templates/bills/microspective.css index 628ac261..15d76266 100644 --- a/orchestra/contrib/bills/templates/bills/microspective.css +++ b/orchestra/contrib/bills/templates/bills/microspective.css @@ -4,7 +4,7 @@ body { margin: 40 auto !important; /* margin-bottom: 30 !important;*/ float: none !important; - font-family: Arial, 'Liberation Sans', 'DejaVu Sans', sans-serif; + font-family: sans; } a { diff --git a/orchestra/contrib/domains/forms.py b/orchestra/contrib/domains/forms.py index f1c2ad66..3bbf466f 100644 --- a/orchestra/contrib/domains/forms.py +++ b/orchestra/contrib/domains/forms.py @@ -35,8 +35,7 @@ class BatchDomainCreationAdminForm(forms.ModelForm): if not cleaned_data['account']: account = None for name in [cleaned_data['name']] + self.extra_names: - domain = Domain(name=name) - parent = domain.get_parent() + parent = Domain.get_parent_domain(name) if not parent: # Fake an account to make django validation happy account_model = self.fields['account']._queryset.model diff --git a/orchestra/contrib/mailer/backends.py b/orchestra/contrib/mailer/backends.py index e7cc46a8..af5efc0c 100644 --- a/orchestra/contrib/mailer/backends.py +++ b/orchestra/contrib/mailer/backends.py @@ -1,3 +1,4 @@ +from django.conf import settings as djsettings from django.core.mail.backends.base import BaseEmailBackend from .models import Message @@ -20,7 +21,7 @@ class EmailBackend(BaseEmailBackend): message = Message.objects.create( priority=priority, to_address=to_email, - from_address=message.from_email, + from_address=getattr(message, 'from_email', djsettings.DEFAULT_FROM_EMAIL), subject=message.subject, content=content, ) diff --git a/orchestra/contrib/orders/filters.py b/orchestra/contrib/orders/filters.py index c9f7070c..aa8cd972 100644 --- a/orchestra/contrib/orders/filters.py +++ b/orchestra/contrib/orders/filters.py @@ -38,35 +38,48 @@ class BilledOrderListFilter(SimpleListFilter): return ( ('yes', _("Billed")), ('no', _("Not billed")), + ('pending', _("Pending (re-evaluate metric)")), + ('not_pending', _("Not pending (re-evaluate metric)")), ) + def get_pending_metric_pks(self, queryset): + mindelta = timedelta(days=2) # TODO + metric_pks = [] + prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics', + queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'), + created_on__lte=(F('updated_on')-mindelta)) + ) + prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric', + queryset=MetricStorage.objects.filter(order__billed_on__isnull=False, + created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on')) + ) + metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True) + for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric): + if len(order.billed_metric) != 1: + raise ValueError("Data inconsistency #metrics %i != 1." % len(order.billed_metric)) + billed_metric = order.billed_metric[0].value + for metric in order.valid_metrics: + if metric.created_on <= order.billed_on: + raise ValueError("This value should already be filtered on the prefetch query.") + if metric.value > billed_metric: + metric_pks.append(order.pk) + break + return metric_pks + def queryset(self, request, queryset): if self.value() == 'yes': return queryset.filter(billed_until__isnull=False, billed_until__gte=timezone.now()) elif self.value() == 'no': - mindelta = timedelta(days=2) # TODO - metric_pks = [] - prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics', - queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'), - created_on__lte=(F('updated_on')-mindelta)) - ) - prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric', - queryset=MetricStorage.objects.filter(order__billed_on__isnull=False, - created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on')) - ) - metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True) - for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric): - if len(order.billed_metric) != 1: - raise ValueError("Data inconsistency.") - billed_metric = order.billed_metric[0].value - for metric in order.valid_metrics: - if metric.created_on <= order.billed_on: - raise ValueError("This value should already be filtered on the prefetch query.") - if metric.value > billed_metric: - metric_pks.append(order.pk) - break + return queryset.exclude(billed_until__isnull=False, billed_until__gte=timezone.now()) + elif self.value() == 'pending': return queryset.filter( - Q(pk__in=metric_pks) | Q( + Q(pk__in=self.get_pending_metric_pks(queryset)) | Q( + Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now()) + ) + ) + elif self.value() == 'not_pending': + return queryset.exclude( + Q(pk__in=self.get_pending_metric_pks(queryset)) | Q( Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now()) ) ) diff --git a/orchestra/contrib/orders/models.py b/orchestra/contrib/orders/models.py index 51ff7f44..744f8f37 100644 --- a/orchestra/contrib/orders/models.py +++ b/orchestra/contrib/orders/models.py @@ -59,15 +59,15 @@ class OrderQuerySet(models.QuerySet): def get_related(self, **options): """ returns related orders that could have a pricing effect """ + # TODO for performance reasons get missing from queryset: + # TODO optimize this shit, don't get related if all objects are here Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) conflictive = self.filter(service__metric='') - conflictive = conflictive.exclude(service__billing_period=Service.NEVER) - conflictive = conflictive.select_related('service').group_by('account_id', 'service') + conflictive = conflictive.exclude(service__billing_period=Service.NEVER).exclude(service__rates__isnull=True) + conflictive = conflictive.select_related('service').distinct().group_by('account_id', 'service') qs = Q() for account_id, services in conflictive.items(): for service, orders in services.items(): - if not service.rates.exists(): - continue ini = datetime.date.max end = datetime.date.min bp = None @@ -265,9 +265,15 @@ class MetricStorage(models.Model): except cls.DoesNotExist: cls.objects.create(order=order, value=value, updated_on=now) else: - error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR)) - if value > last.value+error or value < last.value-error: - cls.objects.create(order=order, value=value, updated_on=now) - else: + # Metric storage has per-day granularity (last value of the day is what counts) + if last.created_on == now.date(): + last.value = value last.updated_on = now - last.save(update_fields=['updated_on']) + last.save() + else: + error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR)) + if value > last.value+error or value < last.value-error: + cls.objects.create(order=order, value=value, updated_on=now) + else: + last.updated_on = now + last.save(update_fields=['updated_on']) diff --git a/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html b/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html index b27b2594..18222cad 100644 --- a/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html +++ b/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html @@ -35,7 +35,7 @@

{{ account }}{{ total | floatformat:"-2" }} €

- + {% for line in lines %} @@ -47,7 +47,7 @@ {% endfor %} - +
Description Period Quantity Price
Description Period Size×Quantity Price
{{ line.ini | date }} to {{ line.end | date }}{{ line.size | floatformat:"-2" }}{{ line.size | floatformat:"-2" }}×{{ line.metric | floatformat:"-2"}}  {{ line.subtotal | floatformat:"-2" }} € {% for discount in line.discounts %}
{{ discount.total | floatformat:"-2" }} €{% endfor %} diff --git a/orchestra/utils/html.py b/orchestra/utils/html.py index 12faebf7..79bee5a6 100644 --- a/orchestra/utils/html.py +++ b/orchestra/utils/html.py @@ -1,12 +1,19 @@ +import textwrap + from orchestra.utils.sys import run def html_to_pdf(html): """ converts HTL to PDF using wkhtmltopdf """ - return run( - 'PATH=$PATH:/usr/local/bin/\n' - 'xvfb-run -a -s "-screen 0 640x4800x16" ' - 'wkhtmltopdf -q --footer-center "Page [page] of [topage]" ' - ' --footer-font-size 9 --margin-bottom 20 --margin-top 20 - -', + return run(textwrap.dedent("""\ + PATH=$PATH:/usr/local/bin/ + xvfb-run -a -s "-screen 0 2480x3508x16" wkhtmltopdf -q \\ + --use-xserver \\ + --footer-center "Page [page] of [topage]" \\ + --footer-font-name sans \\ + --footer-font-size 7 \\ + --footer-spacing 7 \\ + --margin-bottom 22 \\ + --margin-top 20 - - """), stdin=html.encode('utf-8') ).stdout