Added dokuwiki traffic accountancy backend

orchestra/admin/menu.py

@@ -29,30 +29,33 @@ def api_link(context):
 
 
 def process_registry(register):
-    def get_item(model, options, parent=False):
+    def get_item(model, options, name=None):
-        name = options.get('verbose_name_plural')
+        if name is None:
+            name = capfirst(options.get('verbose_name_plural'))
         if isinstance(model, str):
             url = reverse('admin:'+model)
         else:
             opts = model._meta
             url = reverse('admin:{}_{}_changelist'.format(
-                opts.app_label, opts.model_name))
+                opts.app_label, opts.model_name)
-            if parent:
+            )
-                name = opts.app_label
+        item = items.MenuItem(name, url)
-        name = capfirst(name)
+        item.options = options
-        return items.MenuItem(name, url)
+        return item
     
     childrens = {}
     for model, options in register.get().items():
         if options.get('menu', True):
             parent = options.get('parent')
             if parent:
+                name = capfirst(model._meta.app_label)
                 parent_item = childrens.get(parent)
                 if parent_item:
                     if not parent_item.children:
                         parent_item.children.append(deepcopy(parent_item))
+                        parent_item.title = name
                 else:
-                    parent_item = get_item(parent, register[parent], parent=True)
+                    parent_item = get_item(parent, register[parent], name=name)
                     parent_item.children = []
                 parent_item.children.append(get_item(model, options))
                 childrens[parent] = parent_item

orchestra/contrib/bills/models.py

@@ -458,3 +458,6 @@ class BillSubline(models.Model):
     description = models.CharField(_("description"), max_length=256)
     total = models.DecimalField(max_digits=12, decimal_places=2)
     type = models.CharField(_("type"), max_length=16, choices=TYPES, default=OTHER)
+    
+    def __str__(self):
+        return "%s %i" % (self.description, self.total)

orchestra/contrib/bills/templates/bills/microspective.html

@@ -79,7 +79,7 @@
     {% for line in lines %}
         {% with sublines=line.sublines.all description=line.description|slice:"38:" %}
         <span class="{% if not sublines and not description %}last {% endif %}column-id">{% if not line.order_id %}L{% endif %}{{ line.order_id|default:line.pk }}</span>
-        <span class="{% if not sublines and not description %}last {% endif %}column-description">{{ line.description|slice:":38" }}</span>
+        <span class="{% if not sublines and not description %}last {% endif %}column-description">{{ line.description|safe|slice:":38" }}</span>
         <span class="{% if not sublines and not description %}last {% endif %}column-period">{{ line.get_verbose_period }}</span>
         <span class="{% if not sublines and not description %}last {% endif %}column-quantity">{{ line.get_verbose_quantity|default:"&nbsp;"|safe }}</span>
         <span class="{% if not sublines and not description %}last {% endif %}column-rate">{% if line.rate %}{{ line.rate }} &{{ currency.lower }};{% else %}&nbsp;{% endif %}</span>
@@ -87,7 +87,7 @@
         <br>
             {% if description %}
                 <span class="{% if not sublines %}last {% endif %}subline column-id">&nbsp;</span>
-                <span class="{% if not sublines %}last {% endif %}subline column-description">{{ description|truncatechars:39 }}</span>
+                <span class="{% if not sublines %}last {% endif %}subline column-description">{{ description|safe|truncatechars:39 }}</span>
                 <span class="{% if not sublines %}last {% endif %}subline column-period">&nbsp;</span>
                 <span class="{% if not sublines %}last {% endif %}subline column-quantity">&nbsp;</span>
                 <span class="{% if not sublines %}last {% endif %}subline column-rate">&nbsp;</span>
@@ -95,7 +95,7 @@
             {% endif %}
             {% for subline in sublines %}
                 <span class="{% if forloop.last %}last {% endif %}subline column-id">&nbsp;</span>
-                <span class="{% if forloop.last %}last {% endif %}subline column-description">{{ subline.description|truncatechars:39 }}</span>
+                <span class="{% if forloop.last %}last {% endif %}subline column-description">{{ subline.description|safe|truncatechars:39 }}</span>
                 <span class="{% if forloop.last %}last {% endif %}subline column-period">&nbsp;</span>
                 <span class="{% if forloop.last %}last {% endif %}subline column-quantity">&nbsp;</span>
                 <span class="{% if forloop.last %}last {% endif %}subline column-rate">&nbsp;</span>
@@ -126,9 +126,11 @@
 <div class="footer">
     <div id="footer-column-1">
         <div id="comments">
+            {% block comments %}
             {% if bill.comments %}
             <span class="title">{% trans "COMMENTS" %}</span> {{ bill.comments|linebreaksbr }}
             {% endif %}
+            {% endblock %}
         </div>
     </div>
     <div id="footer-column-2">

orchestra/contrib/lists/backends.py

@@ -302,7 +302,7 @@ class MailmanTraffic(ServiceMonitor):
                                             # anonymized post
                                             pass
                     except IOError as e:
-                        sys.stderr.write(str(e))
+                        sys.stderr.write(str(e)+'\\n')
                 
                 for list_name, opts in lists.items():
                     __, object_id, size = opts

orchestra/contrib/mailboxes/backends.py

@@ -549,7 +549,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
                                                     except KeyError:
                                                         counter[req_id] = 1
                     except IOError as e:
-                        sys.stderr.write(str(e))
+                        sys.stderr.write(str(e)+'\\n')
                     
                 for username, opts in users.iteritems():
                     size = 0

orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html

@@ -62,7 +62,7 @@ $(document).ready( function () {
                 {% for line in lines %}
                 <tr class="form-row {% if forloop.counter|divisibleby:2 %}row2{% else %}row1{% endif %}">
                     <td>
-                        <a href="{{ line.order | admin_url }}">{{ line.order.description }}</a>
+                        <a href="{{ line.order | admin_url }}">{{ line.order.description|safe }}</a>
                         {% for discount in line.discounts %}
                             <br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Discount per {{ discount.type }}
                         {% endfor %}

orchestra/contrib/resources/apps.py

@@ -21,6 +21,7 @@ class ResourcesConfig(AppConfig):
         administration.register(Resource, icon='gauge.png')
         administration.register(ResourceData, parent=Resource, icon='monitor.png')
         administration.register(MonitorData, parent=Resource, dashboard=False)
+        from . import signals
     
     def reload_relations(self):
         from .admin import insert_resource_inlines

orchestra/contrib/resources/models.py

@@ -116,17 +116,12 @@ class Resource(models.Model):
                 ]})
     
     def save(self, *args, **kwargs):
-        created = not self.pk
         super(Resource, self).save(*args, **kwargs)
-        self.sync_periodic_task()
         # This only works on tests (multiprocessing used on real deployments)
         apps.get_app_config('resources').reload_relations()
     
-    def delete(self, *args, **kwargs):
-        super(Resource, self).delete(*args, **kwargs)
-        self.sync_periodic_task()
-    
     def sync_periodic_task(self):
+        """ sync periodic task on save/delete resource operations """
         name = 'monitor.%s' % str(self)
         if self.pk and self.crontab and self.is_active:
             try:

orchestra/contrib/resources/signals.py

@@ -0,0 +1,12 @@
+from django.db.models.signals import post_delete, post_save
+from django.dispatch import receiver
+
+from .models import Resource
+
+
+@receiver(post_save, sender=Resource, dispatch_uid="resources.sync_periodic_task")
+@receiver(post_delete, sender=Resource, dispatch_uid="resources.sync_periodic_task")
+def sync_periodic_task(sender, **kwargs):
+    """ useing signals instead of Model.delete() override beucause of admin bulk delete() """
+    instance = kwargs['instance']
+    instance.sync_periodic_task()

orchestra/contrib/saas/backends/__init__.py

@@ -1,13 +1,110 @@
 import pkgutil
+import textwrap
+
+from orchestra.contrib.resources import ServiceMonitor
+
+from .. import settings
 
 
-class SaaSServiceMixin(object):
+class SaaSWebTraffic(ServiceMonitor):
+    """
+    Parses apache logs,
+    looking for the size of each request on the last word of the log line.
+    
+    Compatible log format:
+    <tt>LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host</tt>
+    <tt>CustomLog /home/pangea/logs/apache/host_blog.pangea.org.log host</tt>
+    """
     model = 'saas.SaaS'
-    # TODO Match definition support on backends (mysql) and saas
+    script_executable = '/usr/bin/python'
+    monthly_sum_old_values = True
+    abstract = True
     
-    def get_context(self, webapp):
+    def prepare(self):
-        # TODO
+        access_log = self.log_path
+        context = {
+            'access_logs': str((access_log, access_log+'.1')),
+            'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
+        }
+        self.append(textwrap.dedent("""\
+            import sys
+            from datetime import datetime
+            from dateutil import tz
+            
+            def to_local_timezone(date, tzlocal=tz.tzlocal()):
+                date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
+                date = date.replace(tzinfo=tz.tzutc())
+                date = date.astimezone(tzlocal)
+                return date
+            
+            # Use local timezone
+            end_date = to_local_timezone('{current_date}')
+            end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
+            access_logs = {access_logs}
+            sites = {{}}
+            months = {{
+                'Jan': '01',
+                'Feb': '02',
+                'Mar': '03',
+                'Apr': '04',
+                'May': '05',
+                'Jun': '06',
+                'Jul': '07',
+                'Aug': '08',
+                'Sep': '09',
+                'Oct': '10',
+                'Nov': '11',
+                'Dec': '12',
+            }}
+            
+            def prepare(object_id, site_domain, ini_date):
+                global sites
+                ini_date = to_local_timezone(ini_date)
+                ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
+                sites[site_domain] = [ini_date, object_id, 0]
+            
+            def monitor(sites, end_date, months, access_logs):
+                for access_log in access_logs:
+                    try:
+                        with open(access_log, 'r') as handler:
+                            for line in handler.readlines():
+                                meta, request, response, hostname, __ = line.split('"')
+                                host, __, __, date, tz = meta.split()
+                                if host in {ignore_hosts}:
+                                    continue
+                                try:
+                                    site = sites[hostname]
+                                except KeyError:
+                                    continue
+                                else:
+                                    # [16/Sep/2015:11:40:38 +0200]
+                                    day, month, date = date[1:].split('/')
+                                    year, hour, min, sec = date.split(':')
+                                    date = year + months[month] + day + hour + min + sec
+                                    if site[0] < int(date) < end_date:
+                                        status, size = response.split()
+                                        site[2] += int(size)
+                    except IOError as e:
+                        sys.stderr.write(str(e)+'\\n')
+                for opts in sites.values():
+                    ini_date, object_id, size = opts
+                    print object_id, size
+            """).format(**context)
+        )
+    
+    def monitor(self, saas):
+        context = self.get_context(saas)
+        self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
+    
+    def commit(self):
+        self.append('monitor(sites, end_date, months, access_logs)')
+    
+    def get_context(self, saas):
         return {
+            'site_domain': saas.get_site_domain(),
+            'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'object_id': saas.pk,
         }
 
 

orchestra/contrib/saas/backends/dokuwikimu.py

@@ -7,6 +7,7 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.contrib.orchestration import ServiceController
 from orchestra.utils.python import random_ascii
 
+from . import SaaSWebTraffic
 from .. import settings
 
 
@@ -64,3 +65,13 @@ class DokuWikiMuBackend(ServiceController):
             'users_path': os.path.join(context['app_path'], 'conf/users.auth.php'),
         })
         return context
+
+
+class DokuWikiMuTraffic(SaaSWebTraffic):
+    __doc__ = SaaSWebTraffic.__doc__
+    verbose_name = _("DokuWiki MU Traffic")
+    default_route_match = "saas.service == 'dokuwiki'"
+    doc_settings = (settings,
+        ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_DOKUWIKI_LOG_PATH')
+    )
+    log_path = settings.SAAS_DOKUWIKI_LOG_PATH

orchestra/contrib/saas/backends/phplist.py

@@ -213,7 +213,7 @@ class PhpListTraffic(ServiceMonitor):
                                         size = int(size[5:-1])
                                         opts[2] += size
                     except IOError as e:
-                        sys.stderr.write(str(e))
+                        sys.stderr.write(str(e)+'\\n')
                 for opts in lists.values():
                     print opts[1], opts[2]
             """).format(**context)

orchestra/contrib/saas/backends/wordpressmu.py

@@ -1,12 +1,11 @@
 import re
-import textwrap
 
 import requests
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.orchestration import ServiceController
-from orchestra.contrib.resources import ServiceMonitor
 
+from . import SaaSWebTraffic
 from .. import settings
 
 
@@ -123,103 +122,11 @@ class WordpressMuBackend(ServiceController):
         self.append(self.delete_blog, saas)
 
 
-class WordpressMuTraffic(ServiceMonitor):
+class WordpressMuTraffic(SaaSWebTraffic):
-    """
+    __doc__ = SaaSWebTraffic.__doc__
-    Parses apache logs,
-    looking for the size of each request on the last word of the log line.
-    """
     verbose_name = _("Wordpress MU Traffic")
-    model = 'saas.SaaS'
     default_route_match = "saas.service == 'wordpress'"
-    script_executable = '/usr/bin/python'
-    monthly_sum_old_values = True
     doc_settings = (settings,
         ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_WORDPRESS_LOG_PATH')
     )
-    
+    log_path = settings.SAAS_WORDPRESS_LOG_PATH
-    def prepare(self):
-        access_log = settings.SAAS_WORDPRESS_LOG_PATH
-        context = {
-            'access_logs': str((access_log, access_log+'.1')),
-            'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
-            'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
-        }
-        self.append(textwrap.dedent("""\
-            import sys
-            from datetime import datetime
-            from dateutil import tz
-            
-            def to_local_timezone(date, tzlocal=tz.tzlocal()):
-                date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
-                date = date.replace(tzinfo=tz.tzutc())
-                date = date.astimezone(tzlocal)
-                return date
-            
-            # Use local timezone
-            end_date = to_local_timezone('{current_date}')
-            end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
-            access_logs = {access_logs}
-            blogs = {{}}
-            months = {{
-                'Jan': '01',
-                'Feb': '02',
-                'Mar': '03',
-                'Apr': '04',
-                'May': '05',
-                'Jun': '06',
-                'Jul': '07',
-                'Aug': '08',
-                'Sep': '09',
-                'Oct': '10',
-                'Nov': '11',
-                'Dec': '12',
-            }}
-            
-            def prepare(object_id, site_domain, ini_date):
-                global blogs
-                ini_date = to_local_timezone(ini_date)
-                ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
-                blogs[site_domain] = [ini_date, object_id, 0]
-            
-            def monitor(blogs, end_date, months, access_logs):
-                for access_log in access_logs:
-                    try:
-                        with open(access_log, 'r') as handler:
-                            for line in handler.readlines():
-                                meta, request, response, hostname, __ = line.split('"')
-                                host, __, __, date, tz = meta.split()
-                                if host in {ignore_hosts}:
-                                    continue
-                                try:
-                                    blog = blogs[hostname]
-                                except KeyError:
-                                    continue
-                                else:
-                                    # [16/Sep/2015:11:40:38 +0200]
-                                    day, month, date = date[1:].split('/')
-                                    year, hour, min, sec = date.split(':')
-                                    date = year + months[month] + day + hour + min + sec
-                                    if blog[0] < int(date) < end_date:
-                                        status, size = response.split()
-                                        blog[2] += int(size)
-                    except IOError as e:
-                        sys.stderr.write(str(e))
-                for opts in blogs.values():
-                    ini_date, object_id, size = opts
-                    print object_id, size
-            """).format(**context)
-        )
-    
-    def monitor(self, saas):
-        context = self.get_context(saas)
-        self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
-    
-    def commit(self):
-        self.append('monitor(blogs, end_date, months, access_logs)')
-    
-    def get_context(self, saas):
-        return {
-            'site_domain': saas.get_site_domain(),
-            'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
-            'object_id': saas.pk,
-        }

orchestra/contrib/saas/settings.py

@@ -25,11 +25,14 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
 
 SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS',
     (),
+    help_text=_("IP addresses to ignore during traffic accountability."),
 )
 
 
 SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH',
     '',
+    help_text=_('Filesystem path for the webserver access logs.<br>'
+                '<tt>LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host</tt>'),
 )
 
 
@@ -83,6 +86,11 @@ SAAS_DOKUWIKI_GROUP = Setting('SAAS_DOKUWIKI_GROUP',
 )
 
 
+SAAS_DOKUWIKI_LOG_PATH = Setting('SAAS_DOKUWIKI_LOG_PATH',
+    '',
+)
+
+
 SAAS_DRUPAL_SITES_PATH = Setting('WEBSITES_DRUPAL_SITES_PATH',
     '/home/httpd/htdocs/drupal-mu/sites/%(site_name)s',
 )

orchestra/contrib/services/migrations/0003_auto_20150917_0942.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('services', '0002_auto_20150509_1501'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='service',
+            name='billing_point',
+            field=models.CharField(choices=[('ON_REGISTER', 'Registration date'), ('ON_FIXED_DATE', 'Every April')], help_text='Reference point for calculating the renewal date on recurring invoices', verbose_name='billing point', max_length=16, default='ON_FIXED_DATE'),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='is_fee',
+            field=models.BooleanField(help_text='Designates whether this service should be billed as membership fee or not', verbose_name='fee', default=False),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='order_description',
+            field=models.CharField(help_text="Python <a href='https://docs.python.org/2/library/functions.html#eval'>expression</a> used for generating the description for the bill lines of this services.<br>Defaults to <tt>'%s: %s' % (ugettext(handler.description), instance)</tt>", blank=True, max_length=256, verbose_name='Order description'),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='rate_algorithm',
+            field=models.CharField(choices=[('orchestra.contrib.plans.ratings.step_price', 'Step price'), ('orchestra.contrib.plans.ratings.best_price', 'Best price'), ('orchestra.contrib.plans.ratings.match_price', 'Match price')], help_text='Algorithm used to interprete the rating table.<br>&nbsp;&nbsp;Step price: All rates with a quantity lower or equal than the metric are applied. Nominal price will be used when initial block is missing.<br>&nbsp;&nbsp;Best price: Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).<br>&nbsp;&nbsp;Match price: Only <b>the rate</b> with a) inmediate inferior metric and b) lower price is applied. Nominal price will be used when initial block is missing.', verbose_name='rate algorithm', max_length=64, default='orchestra.contrib.plans.ratings.step_price'),
+        ),
+    ]

orchestra/contrib/services/models.py

@@ -97,7 +97,7 @@ class Service(models.Model):
         default=FIXED_DATE)
     is_fee = models.BooleanField(_("fee"), default=False,
         help_text=_("Designates whether this service should be billed as membership fee or not"))
-    order_description = models.CharField(_("Order description"), max_length=128, blank=True,
+    order_description = models.CharField(_("Order description"), max_length=256, blank=True,
         help_text=_(
             "Python <a href='https://docs.python.org/2/library/functions.html#eval'>expression</a> "
             "used for generating the description for the bill lines of this services.<br>"