Merge branch 'resources' into develmain

orchestra/contrib/mailboxes/backends.py

@@ -452,12 +452,30 @@ class DovecotMaildirDisk(ServiceMonitor):
         self.append(textwrap.dedent("""\
             function monitor () {
                  SIZE=$(du -sb $1/Maildir/ 2> /dev/null || echo 0) && echo $SIZE | awk '{print $1}'
+            list=()
             }"""))
     
     def monitor(self, mailbox):
         context = self.get_context(mailbox)
         # self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context)
-        self.append("echo %(object_id)s $(monitor %(home)s)" % context)
+        # self.append("echo %(object_id)s $(monitor %(home)s)" % context)
+        self.append("list[${#list[@]}]=\'echo %(object_id)s $(monitor %(home)s)\'" % context)
+
+    def commit(self):
+        self.append(textwrap.dedent("""\
+            proces=0
+            for cmd in "${list[@]}"
+            do
+                eval $cmd & 
+                proces=$((proces+1))
+                if [ $proces -ge 10 ];then
+                    wait
+                    proces=0
+                fi
+            done
+            wait
+            exit $exit_code
+            """))
 
     def get_context(self, mailbox):
         context = {

orchestra/contrib/orchestration/manager.py

@@ -2,6 +2,7 @@ import logging
 import threading
 import traceback
 from collections import OrderedDict
+import random
 
 from django.core.mail import mail_admins
 
@@ -111,6 +112,7 @@ def execute(scripts, serialize=False, run_async=None):
     executions = []
     threads_to_join = []
     logs = []
+    id_launch = random.randint(0, 2147483646)
     for key, value in scripts.items():
         route, __, async_action = key
         backend, operations = value
@@ -126,6 +128,7 @@ def execute(scripts, serialize=False, run_async=None):
         with db.clone(model=BackendLog) as handle:
             log = backend.create_log(*args, using=handle.target)
             log._state.db = handle.origin
+            log.id_launch = id_launch
         kwargs['log'] = log
         task = keep_log(backend.execute, log, operations)
         logger.debug('%s is going to be executed on %s.' % (backend, route.host))

orchestra/contrib/resources/aggregations.py

@@ -33,10 +33,11 @@ class Last(Aggregation):
     
     def filter(self, dataset, date=None):
 
-        # dataset = dataset.order_by('object_id', '-id').distinct('monitor')
-        now = timezone.now()
-        epoch = now - datetime.timedelta(minutes=2)
-        dataset = dataset.filter( created_at__range=(epoch, now ))
+        lastdataset = dataset.order_by('-id').first()
+        dataset = dataset.filter( launch_id=lastdataset.launch_id)
+        # now = timezone.now()
+        # epoch = now - datetime.timedelta(minutes=2)
+        # dataset = dataset.filter( created_at__range=(epoch, now ))
 
         if date is not None:
             dataset = dataset.filter(created_at__lte=date)

orchestra/contrib/resources/backends.py

@@ -72,6 +72,7 @@ class ServiceMonitor(ServiceBackend):
         from .models import MonitorData
         name = self.get_name()
         app_label, model_name = self.model.split('.')
+        id_launch = log.id_launch
         ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
         for line in log.stdout.splitlines():
             line = line.strip()
@@ -83,7 +84,7 @@ class ServiceMonitor(ServiceBackend):
             content_object = ct.get_object_for_this_type(pk=object_id)
             MonitorData.objects.create(
                 monitor=name, object_id=object_id, content_type=ct, value=value, state=state,
-                created_at=self.current_date, content_object_repr=str(content_object),
+                created_at=self.current_date, content_object_repr=str(content_object), launch_id=id_launch, 
             )
     
     def execute(self, *args, **kwargs):

orchestra/contrib/resources/models.py

@@ -280,6 +280,7 @@ class MonitorData(models.Model):
 
     content_object = GenericForeignKey()
     objects = MonitorDataQuerySet.as_manager()
+    launch_id = models.PositiveIntegerField(_("launch id"), blank=True, null=True)
 
     class Meta:
         get_latest_by = 'id'

orchestra/contrib/systemusers/backends.py

@@ -287,13 +287,31 @@ class UNIXUserDisk(ServiceMonitor):
         self.append(textwrap.dedent("""\
             function monitor () {
                 { SIZE=$(du -bs "$1") && echo $SIZE || echo 0; } | awk {'print $1'}
-            }"""
-        ))
+            }
+        list=()               
+        """))
     
     def monitor(self, user):
         context = self.get_context(user)
-        self.append("echo %(object_id)s $(monitor %(base_home)s)" % context)
+        # self.append("echo %(object_id)s $(monitor %(base_home)s)" % context)
+        self.append("list[${#list[@]}]=\'echo %(object_id)s $(monitor %(base_home)s)\'" % context)
     
+    def commit(self):
+        self.append(textwrap.dedent("""\
+            proces=0
+            for cmd in "${list[@]}"
+            do
+                eval $cmd & 
+                proces=$((proces+1))
+                if [ $proces -ge 5 ];then
+                    wait
+                    proces=0
+                fi
+            done
+            wait
+            exit $exit_code
+            """))
+
     def get_context(self, user):
         context = {
             'object_id': user.pk,