From 5b5d62ef3831291ccfb35138a536fa5176e551ee Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 16 Sep 2015 12:15:05 +0000 Subject: [PATCH] Added phplist and wordpress SaaS traffic backends --- TODO.md | 3 + orchestra/contrib/lists/backends.py | 2 +- orchestra/contrib/mailboxes/backends.py | 2 +- orchestra/contrib/resources/models.py | 2 +- orchestra/contrib/saas/backends/phplist.py | 112 ++++++++++++++++++ .../contrib/saas/backends/wordpressmu.py | 104 ++++++++++++++++ orchestra/contrib/saas/services/phplist.py | 2 +- orchestra/contrib/saas/settings.py | 15 +++ 8 files changed, 238 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 374f2010..47f1fca1 100644 --- a/TODO.md +++ b/TODO.md @@ -393,3 +393,6 @@ Case # Don't enforce one contact per account? remove account.email in favour of contacts? # Mailer: mark as sent + + +# Implement wordpressmu change password or remove password from the form diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index 6ca9f3a0..7ba7da33 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -302,7 +302,7 @@ class MailmanTraffic(ServiceMonitor): # anonymized post pass except IOError as e: - sys.stderr.write(e) + sys.stderr.write(str(e)) for list_name, opts in lists.items(): __, object_id, size = opts diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 54c0c2c3..eba9a765 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -549,7 +549,7 @@ class PostfixMailscannerTraffic(ServiceMonitor): except KeyError: counter[req_id] = 1 except IOError as e: - sys.stderr.write(e) + sys.stderr.write(str(e)) for username, opts in users.iteritems(): size = 0 diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index acb41c3e..3fdecc77 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -60,7 +60,7 @@ class Resource(models.Model): scale = models.CharField(_("scale"), max_length=32, validators=[validate_scale], help_text=_("Scale in which this resource monitoring resoults should " "be prorcessed to match with unit. e.g. 10**9")) - disable_trigger = models.BooleanField(_("disable trigger"), default=False, + disable_trigger = models.BooleanField(_("disable trigger"), default=True, help_text=_("Disables monitors exeeded and recovery triggers")) crontab = models.ForeignKey('djcelery.CrontabSchedule', verbose_name=_("crontab"), null=True, blank=True, diff --git a/orchestra/contrib/saas/backends/phplist.py b/orchestra/contrib/saas/backends/phplist.py index 03bd5c0b..0ffeee93 100644 --- a/orchestra/contrib/saas/backends/phplist.py +++ b/orchestra/contrib/saas/backends/phplist.py @@ -7,6 +7,7 @@ import requests from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor from orchestra.utils.sys import sshrun from .. import settings @@ -121,3 +122,114 @@ class PhpListSaaSBackend(ServiceController): 'db_name': context['db_name'] % context, }) return context + + +class PhpListTraffic(ServiceMonitor): + verbose_name = _("phpList SaaS Traffic") + model = 'saas.SaaS' + default_route_match = "saas.service == 'phplist'" + resource = ServiceMonitor.TRAFFIC + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('SAAS_PHPLIST_MAIL_LOG_PATH',) + ) + + def prepare(self): + mail_log = settings.SAAS_PHPLIST_MAIL_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mail_logs': str((mail_log, mail_log+'.1')), + } + self.append(textwrap.dedent("""\ + import sys + from datetime import datetime + from dateutil import tz + + def prepare(object_id, list_domain, ini_date): + global lists + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + lists[list_domain] = [ini_date, object_id, 0] + + def inside_period(month, day, time, ini_date): + global months + global end_datetime + # Mar 9 17:13:22 + month = months[month] + year = end_datetime.year + if month == '12' and end_datetime.month == 1: + year = year+1 + if len(day) == 1: + day = '0' + day + date = str(year) + month + day + date += time.replace(':', '') + return ini_date < int(date) < end_date + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + # Converts orchestra's UTC dates to local timezone + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + maillogs = {mail_logs} + end_datetime = to_local_timezone('{current_date}') + end_date = int(end_datetime.strftime('%Y%m%d%H%M%S')) + months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + months = dict((m, '%02d' % n) for n, m in enumerate(months, 1)) + + lists = {{}} + id_to_domain = {{}} + + def monitor(lists, id_to_domain, maillogs): + for maillog in maillogs: + try: + with open(maillog, 'r') as maillog: + for line in maillog.readlines(): + if ': message-id=<' in line: + # Sep 15 09:36:51 web postfix/cleanup[8138]: C20FF244283: message-id= + month, day, time, __, __, id, message_id = line.split()[:7] + list_domain = message_id.split('@')[1][:-1] + try: + opts = lists[list_domain] + except KeyError: + pass + else: + ini_date = opts[0] + if inside_period(month, day, time, ini_date): + id = id[:-1] + id_to_domain[id] = list_domain + elif '>, size=' in line: + # Sep 15 09:36:51 web postfix/qmgr[2296]: C20FF244283: from=, size=12252, nrcpt=1 (queue active) + month, day, time, __, __, id, __, size = line.split()[:8] + id = id[:-1] + try: + list_domain = id_to_domain[id] + except KeyError: + pass + else: + opts = lists[list_domain] + size = int(size[5:-1]) + opts[2] += size + except IOError as e: + sys.stderr.write(str(e)) + for opts in lists.values(): + print opts[1], opts[2] + """).format(**context) + ) + + def commit(self): + self.append('monitor(lists, id_to_domain, maillogs)') + + def monitor(self, saas): + context = self.get_context(saas) + self.append("prepare(%(object_id)s, '%(list_domain)s', '%(last_date)s')" % context) + + def get_context(self, saas): + context = { + 'list_domain': saas.get_site_domain(), + 'object_id': saas.pk, + 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + return context diff --git a/orchestra/contrib/saas/backends/wordpressmu.py b/orchestra/contrib/saas/backends/wordpressmu.py index be8e8def..c8e4ab0d 100644 --- a/orchestra/contrib/saas/backends/wordpressmu.py +++ b/orchestra/contrib/saas/backends/wordpressmu.py @@ -1,9 +1,11 @@ import re +import textwrap import requests from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor from .. import settings @@ -119,3 +121,105 @@ class WordpressMuBackend(ServiceController): def delete(self, saas): self.append(self.delete_blog, saas) + + +class WordpressMuTraffic(ServiceMonitor): + """ + Parses apache logs, + looking for the size of each request on the last word of the log line. + """ + verbose_name = _("Wordpress MU Traffic") + model = 'saas.SaaS' + default_route_match = "saas.service == 'wordpress'" + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_WORDPRESS_LOG_PATH') + ) + + def prepare(self): + access_log = settings.SAAS_WORDPRESS_LOG_PATH + context = { + 'access_logs': str((access_log, access_log+'.1')), + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS), + } + self.append(textwrap.dedent("""\ + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + access_logs = {access_logs} + blogs = {{}} + months = {{ + 'Jan': '01', + 'Feb': '02', + 'Mar': '03', + 'Apr': '04', + 'May': '05', + 'Jun': '06', + 'Jul': '07', + 'Aug': '08', + 'Sep': '09', + 'Oct': '10', + 'Nov': '11', + 'Dec': '12', + }} + + def prepare(object_id, site_domain, ini_date): + global blogs + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + blogs[site_domain] = [ini_date, object_id, 0] + + def monitor(blogs, end_date, months, access_logs): + for access_log in access_logs: + try: + with open(access_log, 'r') as handler: + for line in handler.readlines(): + meta, request, response, hostname, __ = line.split('"') + host, __, __, date, tz = meta.split() + if host in {ignore_hosts}: + continue + try: + blog = blogs[hostname] + except KeyError: + continue + else: + # [16/Sep/2015:11:40:38 +0200] + day, month, date = date[1:].split('/') + year, hour, min, sec = date.split(':') + date = year + months[month] + day + hour + min + sec + if blog[0] < int(date) < end_date: + status, size = response.split() + blog[2] += int(size) + except IOError as e: + sys.stderr.write(str(e)) + for opts in blogs.values(): + ini_date, object_id, size = opts + print object_id, size + """).format(**context) + ) + + def monitor(self, saas): + context = self.get_context(saas) + self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(blogs, end_date, months, access_logs)') + + def get_context(self, saas): + return { + 'site_domain': saas.get_site_domain(), + 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': saas.pk, + } diff --git a/orchestra/contrib/saas/services/phplist.py b/orchestra/contrib/saas/services/phplist.py index 7bc11c6c..0bcd4d4a 100644 --- a/orchestra/contrib/saas/services/phplist.py +++ b/orchestra/contrib/saas/services/phplist.py @@ -58,7 +58,7 @@ class PHPListService(SoftwareService): def get_account(self): account_model = self.instance._meta.get_field_by_name('account')[0] - return account_model.objects.get_main() + return account_model.rel.to.objects.get_main() def validate(self): super(PHPListService, self).validate() diff --git a/orchestra/contrib/saas/settings.py b/orchestra/contrib/saas/settings.py index 2105bcf5..c75ebcb7 100644 --- a/orchestra/contrib/saas/settings.py +++ b/orchestra/contrib/saas/settings.py @@ -23,6 +23,16 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES', ) +SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS', + (), +) + + +SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH', + '', +) + + SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD', 'secret' ) @@ -132,6 +142,11 @@ SAAS_PHPLIST_CRONTAB = Setting('SAAS_PHPLIST_CRONTAB', "Left blank if you don't want crontab to be configured") ) +SAAS_PHPLIST_MAIL_LOG_PATH = Setting('SAAS_PHPLIST_MAIL_LOG_PATH', + '/var/log/mail.log', +) + + SAAS_SEAFILE_DOMAIN = Setting('SAAS_SEAFILE_DOMAIN', 'seafile.{}'.format(ORCHESTRA_BASE_DOMAIN), help_text="Uses ORCHESTRA_BASE_DOMAIN by default.",