import pkgutil import textwrap import requests import time import sys from orchestra.contrib.resources import ServiceMonitor from orchestra.contrib.orchestration import replace from django.utils.translation import gettext_lazy as _ import xml.etree.ElementTree as ET from .. import settings class ApacheTrafficByHost(ServiceMonitor): """ Parses apache logs, looking for the size of each request on the last word of the log line. Compatible log format: LogFormat "%h %l %u %t \"%r\" %>s %O %{Host}i" host or if include_received_bytes: LogFormat "%h %l %u %t \"%r\" %>s %I %O %{Host}i" host CustomLog /home/pangea/logs/apache/host_blog.pangea.org.log host """ model = 'saas.SaaS' script_executable = '/usr/bin/python' monthly_sum_old_values = True abstract = True include_received_bytes = False def prepare(self): access_log = self.log_path context = { 'access_logs': str((access_log, access_log+'.1')), 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), 'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS), 'include_received_bytes': str(self.include_received_bytes), } self.append(textwrap.dedent("""\ import sys from datetime import datetime from dateutil import tz def to_local_timezone(date, tzlocal=tz.tzlocal()): date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') date = date.replace(tzinfo=tz.tzutc()) date = date.astimezone(tzlocal) return date # Use local timezone end_date = to_local_timezone('{current_date}') end_date = int(end_date.strftime('%Y%m%d%H%M%S')) access_logs = {access_logs} sites = {{}} months = {{ 'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12', }} def prepare(object_id, site_domain, ini_date): global sites ini_date = to_local_timezone(ini_date) ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) sites[site_domain] = [ini_date, object_id, 0] def monitor(sites, end_date, months, access_logs): include_received = {include_received_bytes} for access_log in access_logs: try: with open(access_log, 'r') as handler: for line in handler.readlines(): line = line.split() host, __, __, date = line[:4] if host in {ignore_hosts}: continue size, hostname = line[-2:] size = int(size) if include_received: size += int(line[-3]) try: site = sites[hostname] except KeyError: continue else: # [16/Sep/2015:11:40:38 day, month, date = date[1:].split('/') year, hour, min, sec = date.split(':') date = year + months[month] + day + hour + min + sec if site[0] < int(date) < end_date: site[2] += size except IOError as e: sys.stderr.write(str(e)+'\\n') for opts in sites.values(): ini_date, object_id, size = opts sys.stdout.write('%s %s\\n' % (object_id, size)) """).format(**context) ) def monitor(self, saas): context = self.get_context(saas) self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context) def commit(self): self.append('monitor(sites, end_date, months, access_logs)') def get_context(self, saas): return { 'site_domain': saas.get_site_domain(), 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': saas.pk, } class ApacheTrafficByName(ApacheTrafficByHost): __doc__ = ApacheTrafficByHost.__doc__ def get_context(self, saas): return { 'site_domain': saas.name, 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': saas.pk, } class NextCloudAPIMixin(object): def validate_response(self, response): request = response.request context = (request.method, response.url, request.body, response.status_code) # sys.stderr.write("%s %s '%s' HTTP %s\n" % context) if response.status_code != requests.codes.ok: raise RuntimeError("%s %s '%s' HTTP %s" % context) root = ET.fromstring(response.text) statuscode = root.find("./meta/statuscode").text if statuscode != '100': message = root.find("./meta/status").text request = response.request context = (request.method, response.url, request.body, statuscode, message) raise RuntimeError("%s %s '%s' ERROR %s, %s" % context) def api_call(self, action, url_path, *args, **kwargs): BASE_URL = settings.SAAS_NEXTCLOUD_API_URL.rstrip('/') url = '/'.join((BASE_URL, url_path)) response = action(url, headers={'OCS-APIRequest':'true'}, verify=False, *args, **kwargs) self.validate_response(response) return response def api_get(self, url_path, *args, **kwargs): return self.api_call(requests.get, url_path, *args, **kwargs) def api_post(self, url_path, *args, **kwargs): return self.api_call(requests.post, url_path, *args, **kwargs) def api_put(self, url_path, *args, **kwargs): return self.api_call(requests.put, url_path, *args, **kwargs) def api_delete(self, url_path, *args, **kwargs): return self.api_call(requests.delete, url_path, *args, **kwargs) def create(self, saas): data = { 'userid': saas.name, 'password': saas.password, } self.api_post('users', data) def update_group(self, saas): data = { 'groupid': saas.account.username } try: self.api_get('groups/%s' % saas.account.username) except RuntimeError: self.api_post('groups', data) self.api_post(f'users/{saas.name}/groups', data) def update_quota(self, saas): if hasattr(saas, 'resources') and hasattr(saas.resources, 'nextcloud-disk'): resource = getattr(saas.resources, 'nextcloud-disk') quotaValue = f"{resource.allocated}G" if resource.allocated > 0 else "default" data = { 'key': "quota", 'value': quotaValue } self.api_put(f'users/{saas.name}', data) def update_password(self, saas): """ key: email|quota|display|password value: el valor a modificar. Si es un email, tornarà un error si la direcció no te la "@" Si es una quota, sembla que algo per l'estil "5G", "100M", etc. funciona. Quota 0 = infinit "display" es el display name, no crec que el fem servir, és cosmetic """ data = { 'key': 'password', 'value': saas.password, } self.api_put('users/%s' % saas.name, data) def disable_user(self, saas): self.api_put('users/%s/disable' % saas.name) def enable_user(self, saas): self.api_put('users/%s/enable' % saas.name) def get_user(self, saas): """ { 'displayname' 'email' 'quota' => { 'free' (en Bytes) 'relative' (en tant per cent sense signe %, e.g. 68.17) 'total' (en Bytes) 'used' (en Bytes) } } """ response = self.api_get('users/%s' % saas.name) root = ET.fromstring(response.text) ret = {} for data in root.find('./data'): ret[data.tag] = data.text ret['quota'] = {} for data in root.find('.data/quota'): ret['quota'][data.tag] = data.text return ret class NextCloudDiskQuota(NextCloudAPIMixin, ServiceMonitor): model = 'saas.SaaS' verbose_name = _("nextCloud SaaS Disk Quota") default_route_match = "saas.service == 'nextcloud'" resource = ServiceMonitor.DISK delete_old_equal_values = True def monitor(self, user): context = self.get_context(user) self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) def get_context(self, user): context = { 'object_id': user.pk, 'base_home': user.get_base_home(), } return replace(context, "'", '"') def get_quota(self, saas, server): try: user = self.get_user(saas) except requests.exceptions.ConnectionError: time.sleep(2) user = self.get_user(saas) context = { 'object_id': saas.pk, 'used': int(user['quota'].get('used', 0)), } sys.stdout.write('%(object_id)i %(used)i\n' % context) def monitor(self, saas): self.append(self.get_quota, saas) for __, module_name, __ in pkgutil.walk_packages(__path__): # sorry for the exec(), but Import module function fails :( exec('from . import %s' % module_name)