Fixes on the billing system

TODO.md

@@ -102,3 +102,6 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon
     * transaction.ABORTED -> bill.bad_debt
     - Issue new transaction when current transaction is ABORTED
 * underescore *every* private function
+
+
+* create log file at /var/log/orchestra.log and rotate

orchestra/apps/bills/admin.py

@@ -61,7 +61,7 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
     fieldsets = (
         (None, {
             'fields': ('number', 'display_total', 'account_link', 'type',
-                       'is_open', 'display_payment_state', 'is_sent', 'due_on', 'comments'),
+                       'display_payment_state', 'is_sent', 'due_on', 'comments'),
         }),
         (_("Raw"), {
             'classes': ('collapse',),
@@ -71,7 +71,7 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
     actions = [download_bills, close_bills, send_bills]
     change_view_actions = [view_bill, download_bills, send_bills, close_bills]
     change_readonly_fields = ('account_link', 'type', 'is_open')
-    readonly_fields = ('number', 'display_total', 'display_payment_state')
+    readonly_fields = ('number', 'display_total', 'is_sent', 'display_payment_state')
     inlines = [BillLineInline]
     
     created_on_display = admin_date('created_on')

orchestra/apps/bills/migrations/0007_auto_20140918_1454.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bills', '0006_auto_20140911_1238'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='bill',
+            name='status',
+        ),
+        migrations.RemoveField(
+            model_name='billline',
+            name='amount',
+        ),
+        migrations.RemoveField(
+            model_name='billline',
+            name='total',
+        ),
+        migrations.AddField(
+            model_name='bill',
+            name='is_open',
+            field=models.BooleanField(default=True, verbose_name='is open'),
+            preserve_default=True,
+        ),
+        migrations.AddField(
+            model_name='bill',
+            name='is_sent',
+            field=models.BooleanField(default=False, verbose_name='is sent'),
+            preserve_default=True,
+        ),
+        migrations.AddField(
+            model_name='billline',
+            name='quantity',
+            field=models.DecimalField(default=10, verbose_name='quantity', max_digits=12, decimal_places=2),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='billline',
+            name='subtotal',
+            field=models.DecimalField(default=20, verbose_name='subtotal', max_digits=12, decimal_places=2),
+            preserve_default=False,
+        ),
+    ]

orchestra/apps/bills/models.py

@@ -51,6 +51,7 @@ class Bill(models.Model):
     type = models.CharField(_("type"), max_length=16, choices=TYPES)
     created_on = models.DateTimeField(_("created on"), auto_now_add=True)
     closed_on = models.DateTimeField(_("closed on"), blank=True, null=True)
+    # TODO rename to is_closed
     is_open = models.BooleanField(_("is open"), default=True)
     is_sent = models.BooleanField(_("is sent"), default=False)
     due_on = models.DateField(_("due on"), null=True, blank=True)
@@ -130,7 +131,8 @@ class Bill(models.Model):
             self.due_on = self.get_due_date(payment=payment)
         self.total = self.get_total()
         self.html = self.render(payment=payment)
-        self.transactions.create(bill=self, source=payment, amount=self.total)
+        if self.get_type() != 'PROFORMA':
+            self.transactions.create(bill=self, source=payment, amount=self.total)
         self.closed_on = timezone.now()
         self.is_open = False
         self.is_sent = False

orchestra/apps/orders/actions.py

@@ -39,7 +39,7 @@ class BillSelectedOrders(object):
                     billing_point=form.cleaned_data['billing_point'],
                     fixed_point=form.cleaned_data['fixed_point'],
                     is_proforma=form.cleaned_data['is_proforma'],
-                    create_new_open=form.cleaned_data['create_new_open'],
+                    new_open=form.cleaned_data['new_open'],
                 )
                 if int(request.POST.get('step')) != 3:
                     return self.select_related(request)

orchestra/apps/orders/billing.py

@@ -9,7 +9,7 @@ class BillsBackend(object):
     def create_bills(self, account, lines, **options):
         bill = None
         bills = []
-        create_new = options.get('create_new_open', False)
+        create_new = options.get('new_open', False)
         is_proforma = options.get('is_proforma', False)
         for line in lines:
             service = line.order.service
@@ -19,16 +19,14 @@ class BillsBackend(object):
                     if create_new:
                         bill = ProForma.objects.create(account=account)
                     else:
-                        bill, __ = ProForma.objects.get_or_create(account=account,
+                        bill, __ = ProForma.objects.get_or_create(account=account, is_open=True)
-                                status=ProForma.OPEN)
                 elif service.is_fee:
                     bill = Fee.objects.create(account=account)
                 else:
                     if create_new:
                         bill = Invoice.objects.create(account=account)
                     else:
-                        bill, __ = Invoice.objects.get_or_create(account=account,
+                        bill, __ = Invoice.objects.get_or_create(account=account, is_open=True)
-                                status=Invoice.OPEN)
                 bills.append(bill)
             # Create bill line
             billine = bill.lines.create(

orchestra/apps/orders/forms.py

@@ -21,7 +21,7 @@ class BillSelectedOptionsForm(AdminFormMixin, forms.Form):
     is_proforma = forms.BooleanField(initial=False, required=False,
             label=_("Pro-forma, billing simulation"),
             help_text=_("O."))
-    create_new_open = forms.BooleanField(initial=False, required=False,
+    new_open = forms.BooleanField(initial=False, required=False,
             label=_("Create a new open bill"),
             help_text=_("Deisgnates whether you want to put this orders on a new "
                         "open bill, or allow to reuse an existing one."))

orchestra/apps/orders/models.py

@@ -1,3 +1,4 @@
+import logging
 import sys
 
 from django.db import models
@@ -22,10 +23,14 @@ from . import helpers, settings
 from .handlers import ServiceHandler
 
 
+logger = logging.getLogger(__name__)
+
+
 class OrderQuerySet(models.QuerySet):
     group_by = queryset.group_by
     
     def bill(self, **options):
+        # TODO classmethod?
         bills = []
         bill_backend = Order.get_bill_backend()
         qs = self.select_related('account', 'service')
@@ -41,13 +46,17 @@ class OrderQuerySet(models.QuerySet):
                 bills += [(account, bill_lines)]
         return bills
     
-    def filter_givers(self, ini, end):
+    def givers(self, ini, end):
-        return self.filter(
+        return self.cancelled_and_billed().filter(billed_until__gt=ini, registered_on__lt=end)
-            cancelled_on__isnull=False, billed_until__isnull=False,
-            cancelled_on__lte=F('billed_until'), billed_until__gt=ini,
-            registered_on__lt=end)
     
-    def filter_pricing_orders(self, ini, end):
+    def cancelled_and_billed(self, exclude=False):
+        qs = dict(cancelled_on__isnull=False, billed_until__isnull=False,
+                  cancelled_on__lte=F('billed_until'))
+        if exclude:
+            return self.exclude(**qs)
+        return self.filter(**qs)
+    
+    def pricing_orders(self, ini, end):
         return self.filter(billed_until__isnull=False, billed_until__gt=ini,
             registered_on__lt=end)
     
@@ -86,18 +95,6 @@ class Order(models.Model):
     def __unicode__(self):
         return str(self.service)
     
-    def update(self):
-        instance = self.content_object
-        handler = self.service.handler
-        if handler.metric:
-            metric = handler.get_metric(instance)
-            if metric is not None:
-                MetricStorage.store(self, metric)
-        description = "{}: {}".format(handler.description, str(instance))
-        if self.description != description:
-            self.description = description
-            self.save()
-    
     @classmethod
     def update_orders(cls, instance):
         Service = get_model(*settings.ORDERS_SERVICE_MODEL.split('.'))
@@ -111,6 +108,7 @@ class Order(models.Model):
                         continue
                     order = cls.objects.create(content_object=instance,
                             service=service, account_id=account_id)
+                    logger.info("CREATED new order id: {id}".format(id=order.id))
                 else:
                     order = orders.get()
                 order.update()
@@ -121,9 +119,24 @@ class Order(models.Model):
     def get_bill_backend(cls):
         return import_class(settings.ORDERS_BILLING_BACKEND)()
     
+    def update(self):
+        instance = self.content_object
+        handler = self.service.handler
+        if handler.metric:
+            metric = handler.get_metric(instance)
+            if metric is not None:
+                MetricStorage.store(self, metric)
+        description = "{}: {}".format(handler.description, str(instance))
+        logger.info("UPDATED order id: {id} description:{description}".format(
+                    id=self.id, description=description))
+        if self.description != description:
+            self.description = description
+            self.save()
+    
     def cancel(self):
         self.cancelled_on = timezone.now()
         self.save()
+        logger.info("CANCELLED order id: {id}".format(id=self.id))
     
     def get_metric(self, ini, end):
         return MetricStorage.get(self, ini, end)
@@ -162,30 +175,31 @@ class MetricStorage(models.Model):
             return 0
 
 
-# TODO If this happens to be very costly then, consider an additional
+_excluded_models = (MetricStorage, LogEntry, Order, ContentType, MigrationRecorder.Migration)
-#      implementation when runnning within a request/Response cycle, more efficient :)
+
-@receiver(pre_delete, dispatch_uid="orders.cancel_orders")
+@receiver(post_delete, dispatch_uid="orders.cancel_orders")
 def cancel_orders(sender, **kwargs):
-    if sender in services:
+    if sender not in _excluded_models:
         instance = kwargs['instance']
-        for order in Order.objects.by_object(instance).active():
+        if hasattr(instance, 'account'):
-            order.cancel()
+            for order in Order.objects.by_object(instance).active():
+                order.cancel()
+        else:
+            related = helpers.get_related_objects(instance)
+            if related and related != instance:
+                Order.update_orders(related)
 
 
 @receiver(post_save, dispatch_uid="orders.update_orders")
-@receiver(post_delete, dispatch_uid="orders.update_orders_post_delete")
 def update_orders(sender, **kwargs):
-    exclude = (
+    if sender not in _excluded_models:
-        MetricStorage, LogEntry, Order, ContentType, MigrationRecorder.Migration
-    )
-    if sender not in exclude:
         instance = kwargs['instance']
-        if instance.pk:
+        if hasattr(instance, 'account'):
-            # post_save
             Order.update_orders(instance)
-        related = helpers.get_related_objects(instance)
+        else:
-        if related:
+            related = helpers.get_related_objects(instance)
-            Order.update_orders(related)
+            if related and related != instance:
+                Order.update_orders(related)
 
 
 accounts.register(Order)

orchestra/apps/orders/tests/functional_tests/tests.py

@@ -96,13 +96,16 @@ class BillingTests(BaseTestCase):
         account = self.create_account()
         service = self.create_ftp_service()
         user = self.create_ftp(account=account)
-        bp = timezone.now().date() + relativedelta.relativedelta(years=2)
+        first_bp = timezone.now().date() + relativedelta.relativedelta(years=2)
-        bills = service.orders.bill(billing_point=bp, fixed_point=True)
+        bills = service.orders.bill(billing_point=first_bp, fixed_point=True)
         user.delete()
         user = self.create_ftp(account=account)
         bp = timezone.now().date() + relativedelta.relativedelta(years=1)
-        bills = service.orders.bill(billing_point=bp, fixed_point=True)
+        bills = service.orders.bill(billing_point=bp, fixed_point=True, new_open=True)
-        for line in bills[0].lines.all():
+        discount = bills[0].lines.order_by('id')[0].sublines.get()
-            print line
+        self.assertEqual(decimal.Decimal(-20), discount.total)
-            print line.sublines.all()
+        order = service.orders.order_by('id').first()
-        # TODO asserts
+        self.assertEqual(order.cancelled_on, order.billed_until)
+        order = service.orders.order_by('-id').first()
+        self.assertEqual(first_bp, order.billed_until)
+        self.assertEqual(decimal.Decimal(0), bills[0].get_total())

orchestra/apps/payments/admin.py

@@ -127,12 +127,14 @@ class TransactionAdmin(ChangeViewActionsMixin, AccountAdminMixin, admin.ModelAdm
         actions = super(TransactionAdmin, self).get_change_view_actions()
         exclude = []
         if obj:
+            if obj.state == Transaction.WAITTING_PROCESSING:
+                exclude = ['mark_as_executed', 'mark_as_secured', 'mark_as_rejected']
+            elif obj.state == Transaction.WAITTING_EXECUTION:
+                exclude = ['process_transactions', 'mark_as_secured', 'mark_as_rejected']
             if obj.state == Transaction.EXECUTED:
-                exclude.append('mark_as_executed')
+                exclude = ['process_transactions', 'mark_as_executed']
-            elif obj.state == Transaction.REJECTED:
+            elif obj.state in [Transaction.REJECTED, Transaction.SECURED]:
-                exclude.append('mark_as_rejected')
+                return []
-            elif obj.state == Transaction.SECURED:
-                exclude.append('mark_as_secured')
         return [action for action in actions if action.__name__ not in exclude]
 
 

orchestra/apps/payments/models.py

@@ -118,18 +118,22 @@ class Transaction(models.Model):
                 raise ValidationError(_("New transactions can not be allocated for this bill"))
     
     def mark_as_processed(self):
+        assert self.state == self.WAITTING_PROCESSING
         self.state = self.WAITTING_EXECUTION
         self.save()
     
     def mark_as_executed(self):
+        assert self.state == self.WAITTING_EXECUTION
         self.state = self.EXECUTED
         self.save()
     
     def mark_as_secured(self):
+        assert self.state == self.EXECUTED
         self.state = self.SECURED
         self.save()
     
     def mark_as_rejected(self):
+        assert self.state == self.EXECUTED
         self.state = self.REJECTED
         self.save()
 

orchestra/apps/services/handlers.py

@@ -154,12 +154,12 @@ class ServiceHandler(plugins.Plugin):
         for dtype, dprice in discounts:
             self.generate_discount(line, dtype, dprice)
             discounted += dprice
-        subtotal -= discounted
+        subtotal += discounted
         if subtotal > price:
             self.generate_discount(line, 'volume', price-subtotal)
         return line
-
+    
-    def compensate(self, givers, receivers, commit=True):
+    def assign_compensations(self, givers, receivers, commit=True):
         compensations = []
         for order in givers:
             if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until:
@@ -170,17 +170,43 @@ class ServiceHandler(plugins.Plugin):
                 # receiver
                 ini = order.billed_until or order.registered_on
                 end = order.cancelled_on or datetime.date.max
-                order_interval = helpers.Interval(ini, order.new_billed_until)
+                interval = helpers.Interval(ini, end)
-                compensations, used_compensations = helpers.compensate(order_interval, compensations)
+                compensations, used_compensations = helpers.compensate(interval, compensations)
                 order._compensations = used_compensations
                 for comp in used_compensations:
-                    comp.order.new_billed_until = min(comp.order.billed_until, comp.end)
+                    # TODO get min right
+                    comp.order.new_billed_until = min(comp.order.billed_until, comp.ini,
+                            getattr(comp.order, 'new_billed_until', datetime.date.max))
         if commit:
             for order in givers:
                 if hasattr(order, 'new_billed_until'):
                     order.billed_until = order.new_billed_until
                     order.save()
     
+    def apply_compensations(self, order, only_beyond=False):
+        dsize = 0
+        discounts = ()
+        ini = order.billed_until or order.registered_on
+        end = order.new_billed_until
+        beyond = end
+        cend = None
+        for comp in getattr(order, '_compensations', []):
+            intersect = comp.intersect(helpers.Interval(ini=ini, end=end))
+            if intersect:
+                cini, cend = intersect.ini, intersect.end
+                if comp.end > beyond:
+                    cend = comp.end
+                    if only_beyond:
+                        cini = beyond
+                elif not only_beyond:
+                    continue
+                dsize += self.get_price_size(cini, cend)
+            # Extend billing point a little bit to benefit from a substantial discount
+            elif comp.end > beyond and (comp.end-comp.ini).days > 3*(comp.ini-beyond).days:
+                cend = comp.end
+                dsize += self.get_price_size(comp.ini, cend)
+        return dsize, cend
+    
     def get_register_or_renew_events(self, porders, ini, end):
         # TODO count intermediat billing points too
         counter = 0
@@ -207,6 +233,7 @@ class ServiceHandler(plugins.Plugin):
             for position, order in enumerate(orders):
                 csize = 0
                 compensations = getattr(order, '_compensations', [])
+                # Compensations < new_billed_until
                 for comp in compensations:
                     intersect = comp.intersect(interval)
                     if intersect:
@@ -223,8 +250,15 @@ class ServiceHandler(plugins.Plugin):
         for order, prices in priced.iteritems():
             # Generate lines and discounts from order.nominal_price
             price, cprice = prices
+            # Compensations > new_billed_until
+            dsize, new_end = self.apply_compensations(order, only_beyond=True)
+            cprice += dsize*price
             if cprice:
-                discounts = (('compensation', cprice),)
+                discounts = (('compensation', -cprice),)
+                if new_end:
+                    size = self.get_price_size(order.new_billed_until, new_end)
+                    price += price*size
+                    order.new_billed_until = new_end
             line = self.generate_line(order, price, size, ini, end, discounts=discounts)
             lines.append(line)
             if commit:
@@ -232,7 +266,7 @@ class ServiceHandler(plugins.Plugin):
                 order.save()
         return lines
     
-    def bill_registered_or_renew_events(self, account, porders, rates, ini, end, commit=True):
+    def bill_registered_or_renew_events(self, account, porders, rates, commit=True):
         # Before registration
         lines = []
         perido = self.get_pricing_period()
@@ -242,16 +276,24 @@ class ServiceHandler(plugins.Plugin):
             rdelta = relativedelta.relativedelta(years=1)
         elif period == self.NEVER:
             raise NotImplementedError("Rates with no pricing period?")
-        ini -= rdelta
         for position, order in enumerate(porders):
             if hasattr(order, 'new_billed_until'):
-                cend = order.billed_until or order.registered_on
+                pend = order.billed_until or order.registered_on
-                cini = cend - rdelta
+                pini = pend - rdelta
-                metric = self.get_register_or_renew_events(porders, cini, cend)
+                metric = self.get_register_or_renew_events(porders, pini, pend)
-                size = self.get_price_size(ini, end)
                 price = self.get_price(account, metric, position=position, rates=rates)
+                ini = order.billed_until or order.registered_on
+                end = order.new_billed_until
+                discounts = ()
+                dsize, new_end = self.apply_compensations(order)
+                if dsize:
+                    discounts=(('compensation', -dsize*price),)
+                    if new_end:
+                        order.new_billed_until = new_end
+                        end = new_end
+                size = self.get_price_size(ini, end)
                 price = price * size
-                line = self.generate_line(order, price, size, ini, end)
+                line = self.generate_line(order, price, size, ini, end, discounts=discounts)
                 lines.append(line)
                 if commit:
                     order.billed_until = order.new_billed_until
@@ -262,38 +304,47 @@ class ServiceHandler(plugins.Plugin):
         #   date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0)
         #   In most cases:
         #       ini >= registered_date, end < registered_date
-        bp = None
-        lines = []
         commit = options.get('commit', True)
+        
+        # boundary lookup and exclude cancelled and billed
+        orders_ = []
+        bp = None
         ini = datetime.date.max
         end = datetime.date.min
-        # boundary lookup
         for order in orders:
             cini = order.registered_on
             if order.billed_until:
+                # exclude cancelled and billed
+                if self.on_cancel != self.REFOUND:
+                    if order.cancelled_on and order.billed_until > order.cancelled_on:
+                        continue
                 cini = order.billed_until
             bp = self.get_billing_point(order, bp=bp, **options)
             order.new_billed_until = bp
             ini = min(ini, cini)
             end = max(end, bp)
+            orders_.append(order)
+        orders = orders_
+        
+        # Compensation
         related_orders = account.orders.filter(service=self.service)
         if self.on_cancel == self.DISCOUNT:
             # Get orders pending for compensation
-            givers = list(related_orders.filter_givers(ini, end))
+            givers = list(related_orders.givers(ini, end))
-            print givers
             givers.sort(cmp=helpers.cmp_billed_until_or_registered_on)
             orders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
-            self.compensate(givers, orders, commit=commit)
+            self.assign_compensations(givers, orders, commit=commit)
         
         rates = self.get_rates(account)
         if rates:
-            porders = related_orders.filter_pricing_orders(ini, end)
+            porders = related_orders.pricing_orders(ini, end)
             porders = list(set(orders).union(set(porders)))
             porders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
             if self.billing_period != self.NEVER and self.get_pricing_period != self.NEVER:
                 liens = self.bill_concurrent_orders(account, porders, rates, ini, end, commit=commit)
             else:
-                lines = self.bill_registered_or_renew_events(account, porders, rates, ini, end, commit=commit)
+                # TODO compensation in this case?
+                lines = self.bill_registered_or_renew_events(account, porders, rates, commit=commit)
         else:
             lines = []
             price = self.nominal_price
@@ -301,9 +352,15 @@ class ServiceHandler(plugins.Plugin):
             for order in orders:
                 ini = order.billed_until or order.registered_on
                 end = order.new_billed_until
+                discounts = ()
+                dsize, new_end = self.apply_compensations(order)
+                if dsize:
+                    discounts=(('compensation', -dsize*price),)
+                    if new_end:
+                        order.new_billed_until = new_end
+                        end = new_end
                 size = self.get_price_size(ini, end)
-                order.nominal_price = price * size
+                line = self.generate_line(order, price*size, size, ini, end, discounts=discounts)
-                line = self.generate_line(order, price*size, size, ini, end)
                 lines.append(line)
                 if commit:
                     order.billed_until = order.new_billed_until
@@ -311,6 +368,7 @@ class ServiceHandler(plugins.Plugin):
         return lines
     
     def bill_with_metric(self, orders, account, **options):
+        # TODO filter out orders with cancelled_on < billed_until ?
         lines = []
         commit = options.get('commit', True)
         for order in orders:
@@ -342,7 +400,6 @@ class ServiceHandler(plugins.Plugin):
         return lines
     
     def generate_bill_lines(self, orders, account, **options):
-        # TODO filter out orders with cancelled_on < billed_until ?
         if not self.metric:
             lines = self.bill_with_orders(orders, account, **options)
         else:

orchestra/apps/services/helpers.py

@@ -1,6 +1,3 @@
-from django.utils import timezone
-
-
 def get_chunks(porders, ini, end, ix=0):
     if ix >= len(porders):
         return [[ini, end, []]]
@@ -57,8 +54,10 @@ class Interval(object):
         return remaining
     
     def __repr__(self):
-        now = timezone.now()
+        return "Start: {ini}    End: {end}".format(
-        return "Start: %s    End: %s" % ((self.ini-now).days, (self.end-now).days)
+            ini=self.ini.strftime('%Y-%-m-%-d'),
+            end=self.end.strftime('%Y-%-m-%-d')
+        )
     
     def intersect(self, other, remaining_self=None, remaining_other=None):
         if remaining_self is not None:

orchestra/apps/services/models.py

@@ -172,6 +172,7 @@ class Service(models.Model):
             choices=(
                 (NOTHING, _("Nothing")),
                 (DISCOUNT, _("Discount")),
+                (REFOUND, _("Refound")),
             ),
             default=DISCOUNT)
     payment_style = models.CharField(_("payment style"), max_length=16,

orchestra/apps/users/models.py

@@ -38,7 +38,8 @@ class User(auth.AbstractBaseUser):
     
     @property
     def is_main(self):
-        return self.account.user == self
+        # TODO chicken and egg
+        return not self.account.user_id or self.account.user == self
     
     def get_full_name(self):
         full_name = '%s %s' % (self.first_name, self.last_name)