From a717969926bc22c7fddf33379ab9d34f0f76c9a3 Mon Sep 17 00:00:00 2001 From: jorgepastorr Date: Sat, 3 Feb 2024 22:43:27 +0100 Subject: [PATCH 1/2] id_launch in monitordata --- orchestra/contrib/orchestration/manager.py | 3 +++ orchestra/contrib/resources/aggregations.py | 9 +++++---- orchestra/contrib/resources/backends.py | 3 ++- orchestra/contrib/resources/models.py | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index 62572d04..ebe442de 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -2,6 +2,7 @@ import logging import threading import traceback from collections import OrderedDict +import random from django.core.mail import mail_admins @@ -111,6 +112,7 @@ def execute(scripts, serialize=False, run_async=None): executions = [] threads_to_join = [] logs = [] + id_launch = random.randint(0, 2147483646) for key, value in scripts.items(): route, __, async_action = key backend, operations = value @@ -126,6 +128,7 @@ def execute(scripts, serialize=False, run_async=None): with db.clone(model=BackendLog) as handle: log = backend.create_log(*args, using=handle.target) log._state.db = handle.origin + log.id_launch = id_launch kwargs['log'] = log task = keep_log(backend.execute, log, operations) logger.debug('%s is going to be executed on %s.' % (backend, route.host)) diff --git a/orchestra/contrib/resources/aggregations.py b/orchestra/contrib/resources/aggregations.py index 0806b410..8dd0ac0a 100644 --- a/orchestra/contrib/resources/aggregations.py +++ b/orchestra/contrib/resources/aggregations.py @@ -33,10 +33,11 @@ class Last(Aggregation): def filter(self, dataset, date=None): - # dataset = dataset.order_by('object_id', '-id').distinct('monitor') - now = timezone.now() - epoch = now - datetime.timedelta(minutes=2) - dataset = dataset.filter( created_at__range=(epoch, now )) + lastdataset = dataset.order_by('-id').first() + dataset = dataset.filter( launch_id=lastdataset.launch_id) + # now = timezone.now() + # epoch = now - datetime.timedelta(minutes=2) + # dataset = dataset.filter( created_at__range=(epoch, now )) if date is not None: dataset = dataset.filter(created_at__lte=date) diff --git a/orchestra/contrib/resources/backends.py b/orchestra/contrib/resources/backends.py index a997df5f..29ecf465 100644 --- a/orchestra/contrib/resources/backends.py +++ b/orchestra/contrib/resources/backends.py @@ -72,6 +72,7 @@ class ServiceMonitor(ServiceBackend): from .models import MonitorData name = self.get_name() app_label, model_name = self.model.split('.') + id_launch = log.id_launch ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) for line in log.stdout.splitlines(): line = line.strip() @@ -83,7 +84,7 @@ class ServiceMonitor(ServiceBackend): content_object = ct.get_object_for_this_type(pk=object_id) MonitorData.objects.create( monitor=name, object_id=object_id, content_type=ct, value=value, state=state, - created_at=self.current_date, content_object_repr=str(content_object), + created_at=self.current_date, content_object_repr=str(content_object), launch_id=id_launch, ) def execute(self, *args, **kwargs): diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index f4491ab6..12e4ac51 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -280,6 +280,7 @@ class MonitorData(models.Model): content_object = GenericForeignKey() objects = MonitorDataQuerySet.as_manager() + launch_id = models.PositiveIntegerField(_("launch id"), blank=True, null=True) class Meta: get_latest_by = 'id' From 6ad5c290351800cfa1c5fd31b334afe51d799fb5 Mon Sep 17 00:00:00 2001 From: jorgepastorr Date: Fri, 9 Feb 2024 07:13:51 +0100 Subject: [PATCH 2/2] aumentar proces on resources --- orchestra/contrib/mailboxes/backends.py | 20 ++++++++++++++++++- orchestra/contrib/systemusers/backends.py | 24 ++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 8de35114..cd329dde 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -452,12 +452,30 @@ class DovecotMaildirDisk(ServiceMonitor): self.append(textwrap.dedent("""\ function monitor () { SIZE=$(du -sb $1/Maildir/ 2> /dev/null || echo 0) && echo $SIZE | awk '{print $1}' + list=() }""")) def monitor(self, mailbox): context = self.get_context(mailbox) # self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context) - self.append("echo %(object_id)s $(monitor %(home)s)" % context) + # self.append("echo %(object_id)s $(monitor %(home)s)" % context) + self.append("list[${#list[@]}]=\'echo %(object_id)s $(monitor %(home)s)\'" % context) + + def commit(self): + self.append(textwrap.dedent("""\ + proces=0 + for cmd in "${list[@]}" + do + eval $cmd & + proces=$((proces+1)) + if [ $proces -ge 10 ];then + wait + proces=0 + fi + done + wait + exit $exit_code + """)) def get_context(self, mailbox): context = { diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index 70b439c0..6f0b80ea 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -287,13 +287,31 @@ class UNIXUserDisk(ServiceMonitor): self.append(textwrap.dedent("""\ function monitor () { { SIZE=$(du -bs "$1") && echo $SIZE || echo 0; } | awk {'print $1'} - }""" - )) + } + list=() + """)) def monitor(self, user): context = self.get_context(user) - self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) + # self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) + self.append("list[${#list[@]}]=\'echo %(object_id)s $(monitor %(base_home)s)\'" % context) + def commit(self): + self.append(textwrap.dedent("""\ + proces=0 + for cmd in "${list[@]}" + do + eval $cmd & + proces=$((proces+1)) + if [ $proces -ge 5 ];then + wait + proces=0 + fi + done + wait + exit $exit_code + """)) + def get_context(self, user): context = { 'object_id': user.pk,