From 2d3e925c3638d28bc374f32ebf1e39405cafb987 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Fri, 20 Mar 2015 15:13:08 +0000 Subject: [PATCH] Reimplemented traffic monitors, order of magnitude faster --- TODO.md | 20 +++ orchestra/apps/lists/backends.py | 117 ++++++++++++- orchestra/apps/mailboxes/backends.py | 168 +++++++++++++++++- orchestra/apps/orchestration/methods.py | 10 +- orchestra/apps/resources/actions.py | 6 +- orchestra/apps/resources/admin.py | 26 ++- orchestra/apps/resources/backends.py | 6 +- orchestra/apps/resources/forms.py | 1 - orchestra/apps/resources/models.py | 7 + orchestra/apps/systemusers/backends.py | 194 ++++++++++++++++++++- orchestra/apps/webapps/admin.py | 6 +- orchestra/apps/webapps/settings.py | 6 + orchestra/apps/webapps/types/__init__.py | 3 + orchestra/apps/webapps/types/misc.py | 6 +- orchestra/apps/webapps/types/php.py | 3 + orchestra/apps/websites/backends/apache.py | 2 +- 16 files changed, 543 insertions(+), 38 deletions(-) diff --git a/TODO.md b/TODO.md index a4c63f66..38030156 100644 --- a/TODO.md +++ b/TODO.md @@ -215,3 +215,23 @@ Php binaries should have this format: /usr/bin/php5.2-cgi * show details data on webapp changelist + +* lock resource monitoring + +* Optimize backends like mail backend (log files single read), single "/var/log/vsftpd.log{,.1}" on ftp traffic + + +* -EXecCGI in common CMS upload locations /wp-upload/upload/uploads +* cgi user / pervent shell access +* merge php wrapper configuration to optimize process classes + + +* prevent stderr when users exists on backend i.e. mysql user create + +* disable anonymized list options (mailman) + +* webapps directory protection and disable excecgi + +* php-fpm disable execCGI + +* SuexecUserGroup needs to be per app othewise wrapper/fpm user can't be correct diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index 6baa4322..c8cfa495 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -155,22 +155,25 @@ class MailmanBackend(ServiceController): return context -class MailmanTraffic(ServiceMonitor): +class MailmanTrafficBash(ServiceMonitor): model = 'lists.List' resource = ServiceMonitor.TRAFFIC - verbose_name = _("Mailman traffic") + verbose_name = _("Mailman traffic (Bash)") def prepare(self): super(MailmanTraffic, self).prepare() - current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") + context = { + 'mailman_log': '%s{,.1}' % settings.LISTS_MAILMAN_POST_LOG_PATH, + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + } self.append(textwrap.dedent("""\ function monitor () { OBJECT_ID=$1 # Dates convertions are done server-side because of timezone discrepancies INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") - END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s') + END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') LIST_NAME="$3" - MAILMAN_LOG="$4" + MAILMAN_LOG=%(mailman_log)s SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l) { @@ -203,17 +206,115 @@ class MailmanTraffic(ServiceMonitor): print sum * subs }' || [[ $? == 1 ]] && true } | xargs echo ${OBJECT_ID} - }""") % current_date) + }""") % context) def monitor(self, mail_list): context = self.get_context(mail_list) self.append( - 'monitor %(object_id)i "%(last_date)s" "%(list_name)s" %(mailman_log)s{,.1}' % context + 'monitor %(object_id)i "%(last_date)s" "%(list_name)s"' % context ) def get_context(self, mail_list): return { - 'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH, + 'list_name': mail_list.name, + 'object_id': mail_list.pk, + 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + + +class MailmanTraffic(ServiceMonitor): + model = 'lists.List' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Mailman traffic") + script_executable = '/usr/bin/python' + + def prepare(self): + postlog = settings.LISTS_MAILMAN_POST_LOG_PATH + context = { + 'postlogs': str((postlog, postlog+'.1')), + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + } + self.append(textwrap.dedent("""\ + import re + import subprocess + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + postlogs = {postlogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + lists = {{}} + months = {{ + 'Jan': '01', + 'Feb': '02', + 'Mar': '03', + 'Apr': '04', + 'May': '05', + 'Jun': '06', + 'Jul': '07', + 'Aug': '08', + 'Sep': '09', + 'Oct': '10', + 'Nov': '11', + 'Dec': '12', + }} + + def prepare(object_id, list_name, ini_date): + global lists + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + lists[list_name] = [ini_date, object_id, 0] + + def monitor(lists, end_date, months, postlogs): + for postlog in postlogs: + try: + with open(postlog, 'r') as postlog: + for line in postlog.readlines(): + month, day, time, year, __, __, __, list_name, __, __, size = line.split()[:11] + try: + list = lists[list_name] + except KeyError: + continue + else: + date = year + months[month] + day + time.replace(':', '') + if list[0] < int(date) < end_date: + size = size[5:-1] + try: + list[2] += int(size) + except ValueError: + # anonymized post + pass + except IOError as e: + sys.stderr.write(e) + + for list_name, opts in lists.iteritems(): + __, object_id, size = opts + if size: + cmd = ' '.join(('list_members', list_name, '| wc -l')) + ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subscribers = ps.communicate()[0].strip() + size *= int(subscribers) + print object_id, size + """).format(**context) + ) + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(list_name)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(lists, end_date, months, postlogs)') + + def get_context(self, mail_list): + return { 'list_name': mail_list.name, 'object_id': mail_list.pk, 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), diff --git a/orchestra/apps/mailboxes/backends.py b/orchestra/apps/mailboxes/backends.py index fb752539..603d85f3 100644 --- a/orchestra/apps/mailboxes/backends.py +++ b/orchestra/apps/mailboxes/backends.py @@ -9,7 +9,7 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController from orchestra.apps.systemusers.backends import SystemUserBackend from orchestra.apps.resources import ServiceMonitor -from orchestra.utils.humanize import unit_to_bytes +#from orchestra.utils.humanize import unit_to_bytes from . import settings from .models import Address @@ -42,7 +42,8 @@ class MailSystemUserBackend(ServiceController): self.set_quota(mailbox, context) def set_quota(self, mailbox, context): - context['quota'] = mailbox.resources.disk.allocated * unit_to_bytes(mailbox.resources.disk.unit) + context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale() + #unit_to_bytes(mailbox.resources.disk.unit) self.append(textwrap.dedent(""" mkdir -p %(home)s/Maildir chown %(user)s:%(group)s %(home)s/Maildir @@ -294,3 +295,166 @@ class MaildirDisk(ServiceMonitor): } context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context return context + + +class PostfixTraffic(ServiceMonitor): + """ + A high-performance log parser + Reads the mail.log file only once, for all users + """ + model = 'mailboxes.Mailbox' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Postfix traffic usage") + script_executable = '/usr/bin/python' + + def prepare(self): + mail_log = '/var/log/mail.log' + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mail_logs': str((mail_log, mail_log+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + # Converts orchestra's UTC dates to local timezone + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + maillogs = {mail_logs} + end_datetime = to_local_timezone('{current_date}') + end_date = int(end_datetime.strftime('%Y%m%d%H%M%S')) + months = {{ + "Jan": "01", + "Feb": "02", + "Mar": "03", + "Apr": "04", + "May": "05", + "Jun": "06", + "Jul": "07", + "Aug": "08", + "Sep": "09", + "Oct": "10", + "Nov": "11", + "Dec": "12", + }} + + def inside_period(month, day, time, ini_date): + global months + global end_datetime + # Mar 19 17:13:22 + month = months[month] + year = end_datetime.year + if month == '12' and end_datetime.month == 1: + year = year+1 + date = str(year) + month + day + date += time.replace(':', '') + return ini_date < int(date) < end_date + + users = {{}} + delivers = {{}} + reverse = {{}} + + def prepare(object_id, mailbox, ini_date): + global users + global delivers + global reverse + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[mailbox] = (ini_date, object_id) + delivers[mailbox] = set() + reverse[mailbox] = set() + + def monitor(users, delivers, reverse, maillogs): + targets = {{}} + counter = {{}} + user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)') + for maillog in maillogs: + try: + with open(maillog, 'r') as maillog: + for line in maillog.readlines(): + # Only search for Authenticated sendings + if '(Authenticated sender: ' in line: + username = user_regex.search(line).groups()[0] + try: + sender = users[username] + except KeyError: + continue + else: + month, day, time, __, proc, id = line.split()[:6] + if inside_period(month, day, time, sender[0]): + # Add new email + delivers[id[:-1]] = username + # Look for a MailScanner requeue ID + elif ' Requeue: ' in line: + id, __, req_id = line.split()[6:9] + id = id.split('.')[0] + try: + username = delivers[id] + except KeyError: + pass + else: + targets[req_id] = (username, None) + reverse[username].add(req_id) + # Look for the mail size and count the number of recipients of each email + else: + try: + month, day, time, __, proc, req_id, __, msize = line.split()[:8] + except ValueError: + # not interested in this line + continue + if proc.startswith('postfix/'): + req_id = req_id[:-1] + if msize.startswith('size='): + try: + target = targets[req_id] + except KeyError: + pass + else: + targets[req_id] = (target[0], int(msize[5:-1])) + elif proc.startswith('postfix/smtp'): + try: + target = targets[req_id] + except KeyError: + pass + else: + if inside_period(month, day, time, users[target[0]][0]): + try: + counter[req_id] += 1 + except KeyError: + counter[req_id] = 1 + except IOError as e: + sys.stderr.write(e) + + for username, opts in users.iteritems(): + size = 0 + for req_id in reverse[username]: + size += targets[req_id][1] * counter.get(req_id, 0) + print opts[1], size + """).format(**context) + ) + + def commit(self): + self.append('monitor(users, delivers, reverse, maillogs)') + + def monitor(self, mailbox): + context = self.get_context(mailbox) + self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context) + + def get_context(self, mailbox): + return { +# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH, + 'mailbox': mailbox.name, + 'object_id': mailbox.pk, + 'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + + + + + diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index c8b4f682..346fe60b 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -88,9 +88,15 @@ def SSH(backend, log, server, cmds, async=False): # Non-blocking is the secret ingridient in the async sauce select.select([channel], [], []) if channel.recv_ready(): - log.stdout += channel.recv(1024) + part = channel.recv(1024) + while part: + log.stdout += part + part = channel.recv(1024) if channel.recv_stderr_ready(): - log.stderr += channel.recv_stderr(1024) + part = channel.recv_stderr(1024) + while part: + log.stderr += part + part = channel.recv_stderr(1024) log.save(update_fields=['stdout', 'stderr']) if channel.exit_status_ready(): break diff --git a/orchestra/apps/resources/actions.py b/orchestra/apps/resources/actions.py index db2c4aa4..bf1b2735 100644 --- a/orchestra/apps/resources/actions.py +++ b/orchestra/apps/resources/actions.py @@ -16,7 +16,7 @@ def run_monitor(modeladmin, request, queryset): if not async: for result in results: if hasattr(result, 'log'): - logs.add(result.log.pk) + logs.add(str(result.log.pk)) modeladmin.log_change(request, resource, _("Run monitors")) if async: num = len(queryset) @@ -28,8 +28,8 @@ def run_monitor(modeladmin, request, queryset): else: num = len(logs) if num == 1: - log = logs.pop() - link = reverse('admin:orchestration_backendlog_change', args=(log,)) + log_pk = int(logs.pop()) + link = reverse('admin:orchestration_backendlog_change', args=(log_pk,)) msg = _("One related monitor has been executed.") % link elif num >= 1: link = reverse('admin:orchestration_backendlog_changelist') diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 5ab17e88..a61b449b 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -179,25 +179,37 @@ admin.site.register(MonitorData, MonitorDataAdmin) def resource_inline_factory(resources): class ResourceInlineFormSet(generic.BaseGenericInlineFormSet): def total_form_count(self, resources=resources): - return len(resources) + return len(resources) + + def get_queryset(self): + queryset = super(ResourceInlineFormSet, self).get_queryset() + return queryset.order_by('-id').filter(resource__is_active=True) @cached_property def forms(self, resources=resources): forms = [] resources_copy = list(resources) - queryset = self.queryset + # Remove queryset disabled objects + queryset = [data for data in self.queryset if data.resource in resources] if self.instance.pk: # Create missing resource data - queryset = list(queryset) queryset_resources = [data.resource for data in queryset] for resource in resources: if resource not in queryset_resources: - data = resource.dataset.create(content_object=self.instance) + kwargs = { + 'content_object': self.instance, + } + if resource.default_allocation: + kwargs['allocated'] = resource.default_allocation + data = resource.dataset.create(**kwargs) queryset.append(data) # Existing dataset for i, data in enumerate(queryset): forms.append(self._construct_form(i, resource=data.resource)) - resources_copy.remove(data.resource) + try: + resources_copy.remove(data.resource) + except ValueError: + pass # Missing dataset for i, resource in enumerate(resources_copy, len(queryset)): forms.append(self._construct_form(i, resource=resource)) @@ -246,8 +258,8 @@ def insert_resource_inlines(): for inline in getattr(modeladmin_class, 'inlines', []): if inline.__name__ == 'ResourceInline': modeladmin_class.inlines.remove(inline) - - for ct, resources in Resource.objects.group_by('content_type').iteritems(): + resources = Resource.objects.filter(is_active=True) + for ct, resources in resources.group_by('content_type').iteritems(): inline = resource_inline_factory(resources) model = ct.model_class() insertattr(model, 'inlines', inline) diff --git a/orchestra/apps/resources/backends.py b/orchestra/apps/resources/backends.py index 04f70409..754751f1 100644 --- a/orchestra/apps/resources/backends.py +++ b/orchestra/apps/resources/backends.py @@ -64,7 +64,11 @@ class ServiceMonitor(ServiceBackend): ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) for line in log.stdout.splitlines(): line = line.strip() - object_id, value = self.process(line) + try: + object_id, value = self.process(line) + except ValueError: + cls_name = self.__class__.__name__ + raise ValueError("%s expected ' ' got '%s'" % (cls_name, line)) MonitorData.objects.create(monitor=name, object_id=object_id, content_type=ct, value=value, created_at=self.current_date) diff --git a/orchestra/apps/resources/forms.py b/orchestra/apps/resources/forms.py index d1815a72..2a90caad 100644 --- a/orchestra/apps/resources/forms.py +++ b/orchestra/apps/resources/forms.py @@ -25,7 +25,6 @@ class ResourceForm(forms.ModelForm): else: self.fields['allocated'].required = True self.fields['allocated'].initial = self.resource.default_allocation - # def has_changed(self): # """ Make sure resourcedata objects are created for all resources """ # if not self.instance.pk: diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index b4ea8bc7..24e0d3fd 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -86,6 +86,10 @@ class Resource(models.Model): def clean(self): self.verbose_name = self.verbose_name.strip() + if self.on_demand and self.default_allocation: + raise validators.ValidationError({ + 'default_allocation': _("Default allocation can not be set for 'on demand' services") + }) # Validate that model path exists between ct and each monitor.model monitor_errors = [] for monitor in self.monitors: @@ -172,6 +176,9 @@ class ResourceData(models.Model): unique_together = ('resource', 'content_type', 'object_id') verbose_name_plural = _("resource data") + def __unicode__(self): + return "%s: %s" % (str(self.resource), str(self.content_object)) + @classmethod def get_or_create(cls, obj, resource): ct = ContentType.objects.get_for_model(type(obj)) diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index adcf4e01..f9ffee97 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -99,21 +99,24 @@ class SystemUserDisk(ServiceMonitor): } -class FTPTraffic(ServiceMonitor): +class FTPTrafficBash(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.TRAFFIC - verbose_name = _('Systemuser FTP traffic') + verbose_name = _('Systemuser FTP traffic (Bash)') def prepare(self): - super(FTPTraffic, self).prepare() - current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") + super(FTPTrafficBash, self).prepare() + context = { + 'log_file': '%s{,.1}' % settings.SYSTEMUSERS_FTP_LOG_PATH, + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + } self.append(textwrap.dedent("""\ function monitor () { OBJECT_ID=$1 INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") - END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s') + END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') USERNAME="$3" - LOG_FILE="$4" + LOG_FILE=%(log_file)s { grep " bytes, " ${LOG_FILE} \\ | grep " \\[${USERNAME}\\] " \\ @@ -145,18 +148,191 @@ class FTPTraffic(ServiceMonitor): print sum }' || [[ $? == 1 ]] && true } | xargs echo ${OBJECT_ID} - }""") % current_date) + }""") % context) def monitor(self, user): context = self.get_context(user) self.append( - 'monitor {object_id} "{last_date}" "{username}" {log_file}'.format(**context) + 'monitor {object_id} "{last_date}" "{username}"'.format(**context) ) def get_context(self, user): return { - 'log_file': '%s{,.1}' % settings.SYSTEMUSERS_FTP_LOG_PATH, 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': user.pk, 'username': user.username, } + + +class Exim4Traffic(ServiceMonitor): + model = 'systemusers.SystemUser' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Exim4 traffic usage") + script_executable = '/usr/bin/python' + + def prepare(self): + mainlog = '/var/log/exim4/mainlog' + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mainlogs': str((mainlog, mainlog+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + mainlogs = {mainlogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + users = {{}} + + def prepare(object_id, username, ini_date): + global users + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[username] = [ini_date, object_id, 0] + + def monitor(users, end_date, mainlogs): + user_regex = re.compile(r' U=([^ ]+) ') + for mainlog in mainlogs: + try: + with open(mainlog, 'r') as mainlog: + for line in mainlog.readlines(): + if ' <= ' in line and 'P=local' in line: + username = user_regex.search(line).groups()[0] + try: + sender = users[username] + except KeyError: + continue + else: + date, time, id, __, __, user, protocol, size = line.split()[:8] + date = date.replace('-', '') + date += time.replace(':', '') + if sender[0] < int(date) < end_date: + sender[2] += int(size[2:]) + except IOError as e: + sys.stderr.write(e) + + for username, opts in users.iteritems(): + __, object_id, size = opts + print object_id, size + """).format(**context) + ) + + def commit(self): + self.append('monitor(users, end_date, mainlogs)') + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context) + + def get_context(self, user): + return { +# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH, + 'username': user.username, + 'object_id': user.pk, + 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + + + +class FTPTraffic(ServiceMonitor): + model = 'systemusers.SystemUser' + resource = ServiceMonitor.TRAFFIC + verbose_name = _('Systemuser FTP traffic') + script_executable = '/usr/bin/python' + + def prepare(self): + vsftplog = settings.SYSTEMUSERS_FTP_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'vsftplogs': str((vsftplog, vsftplog+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + vsftplogs = {vsftplogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + users = {{}} + months = {{ + 'Jan': '01', + 'Feb': '02', + 'Mar': '03', + 'Apr': '04', + 'May': '05', + 'Jun': '06', + 'Jul': '07', + 'Aug': '08', + 'Sep': '09', + 'Oct': '10', + 'Nov': '11', + 'Dec': '12', + }} + + def prepare(object_id, username, ini_date): + global users + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[username] = [ini_date, object_id, 0] + + def monitor(users, end_date, months, vsftplogs): + user_regex = re.compile(r'\] \[([^ ]+)\] OK ') + bytes_regex = re.compile(r', ([0-9]+) bytes, ') + for vsftplog in vsftplogs: + try: + with open(vsftplog, 'r') as vsftplog: + for line in vsftplog.readlines(): + if ' bytes, ' in line: + username = user_regex.search(line).groups()[0] + try: + user = users[username] + except KeyError: + continue + else: + __, month, day, time, year = line.split()[:5] + date = year + months[month] + day + time.replace(':', '') + if user[0] < int(date) < end_date: + bytes = bytes_regex.search(line).groups()[0] + user[2] += int(bytes) + except IOError as e: + sys.stderr.write(e) + + for username, opts in users.iteritems(): + __, object_id, size = opts + print object_id, size + """).format(**context) + ) + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(users, end_date, months, vsftplogs)') + + def get_context(self, user): + return { + 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': user.pk, + 'username': user.username, + } + diff --git a/orchestra/apps/webapps/admin.py b/orchestra/apps/webapps/admin.py index 4b8aa003..b151a0ff 100644 --- a/orchestra/apps/webapps/admin.py +++ b/orchestra/apps/webapps/admin.py @@ -46,7 +46,7 @@ class WebAppOptionInline(admin.TabularInline): class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): - list_display = ('name', 'type', 'display_websites', 'account_link') + list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link') list_filter = ('type',) # add_fields = ('account', 'name', 'type') # fields = ('account_link', 'name', 'type') @@ -80,6 +80,10 @@ class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin) display_websites.short_description = _("web sites") display_websites.allow_tags = True + def display_detail(self, webapp): + return webapp.type_instance.get_detail() + display_detail.short_description = _("detail") + # def formfield_for_dbfield(self, db_field, **kwargs): # """ Make value input widget bigger """ # if db_field.name == 'type': diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py index fa4e77e0..c951ae90 100644 --- a/orchestra/apps/webapps/settings.py +++ b/orchestra/apps/webapps/settings.py @@ -28,6 +28,12 @@ WEBAPPS_PHP_ERROR_LOG_PATH = getattr(settings, 'WEBAPPS_PHP_ERROR_LOG_PATH', '') +WEBAPPS_MERGE_PHP_WEBAPPS = getattr(settings, 'WEBAPPS_MERGE_PHP_WEBAPPS', + # Combine all fcgid-wrappers/fpm-pools into one per account-php_version + # to better control num processes per account and save memory + False) + + WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', ( 'orchestra.apps.webapps.types.php.PHPApp', 'orchestra.apps.webapps.types.misc.StaticApp', diff --git a/orchestra/apps/webapps/types/__init__.py b/orchestra/apps/webapps/types/__init__.py index 17fe6aa9..933e23bc 100644 --- a/orchestra/apps/webapps/types/__init__.py +++ b/orchestra/apps/webapps/types/__init__.py @@ -100,6 +100,9 @@ class AppType(plugins.Plugin): else: yield (group, [(op.name, op.verbose_name) for op in options]) + def get_detail(self): + return '' + def save(self): pass diff --git a/orchestra/apps/webapps/types/misc.py b/orchestra/apps/webapps/types/misc.py index 82829e3f..93b555be 100644 --- a/orchestra/apps/webapps/types/misc.py +++ b/orchestra/apps/webapps/types/misc.py @@ -9,7 +9,7 @@ from orchestra.plugins.forms import PluginDataForm from ..options import AppOption from . import AppType -from .php import PHPApp +from .php import PHPApp, PHPAppForm, PHPAppSerializer class StaticApp(AppType): @@ -39,12 +39,12 @@ class WebalizerApp(AppType): return ('static', webalizer_path) -class SymbolicLinkForm(PluginDataForm): +class SymbolicLinkForm(PHPAppForm): path = forms.CharField(label=_("Path"), widget=forms.TextInput(attrs={'size':'100'}), help_text=_("Path for the origin of the symbolic link.")) -class SymbolicLinkSerializer(serializers.Serializer): +class SymbolicLinkSerializer(PHPAppSerializer): path = serializers.CharField(label=_("Path")) diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index dc0cdc12..bba1ad38 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -54,6 +54,9 @@ class PHPApp(AppType): def is_fcgid(self): return self.get_php_version().endswith('-cgi') + def get_detail(self): + return self.instance.data.get('php_version', '') + def get_context(self): """ context used to format settings """ return { diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 0d65b626..46470c87 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -343,7 +343,7 @@ class Apache2Traffic(ServiceMonitor): def get_context(self, site): return { - 'log_file': '%s{,.1}' % site.get_www_log_path(), + 'log_file': '%s{,.1}' % site.get_www_access_log_path(), 'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': site.pk, }