django-orchestra/orchestra/apps/resources/models.py

296 lines
11 KiB
Python
Raw Normal View History

2014-07-10 15:19:06 +00:00
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
2014-07-08 15:19:15 +00:00
from django.contrib.contenttypes.models import ContentType
2014-10-06 14:57:02 +00:00
from django.apps import apps
2014-07-11 22:08:16 +00:00
from django.db import models
2014-11-13 15:34:00 +00:00
from django.db.models.loading import get_model
2014-09-22 15:59:53 +00:00
from django.utils import timezone
2014-10-27 13:29:02 +00:00
from django.utils.functional import cached_property
2014-07-08 15:19:15 +00:00
from django.utils.translation import ugettext_lazy as _
from djcelery.models import PeriodicTask, CrontabSchedule
from orchestra.core import validators
2014-07-25 13:27:31 +00:00
from orchestra.models import queryset, fields
2014-11-13 15:34:00 +00:00
from orchestra.models.utils import get_model_field_path
2014-10-23 15:38:46 +00:00
from orchestra.utils.paths import get_project_root
from orchestra.utils.system import run
2014-07-08 15:19:15 +00:00
2014-10-27 13:29:02 +00:00
from . import helpers, tasks
2014-07-09 16:17:43 +00:00
from .backends import ServiceMonitor
2014-10-23 15:38:46 +00:00
from .validators import validate_scale
2014-07-09 16:17:43 +00:00
2014-07-08 15:19:15 +00:00
2014-07-25 13:27:31 +00:00
class ResourceQuerySet(models.QuerySet):
group_by = queryset.group_by
2014-07-08 15:19:15 +00:00
class Resource(models.Model):
2014-07-09 16:17:43 +00:00
"""
Defines a resource, a resource is basically an interpretation of data
gathered by a Monitor
"""
LAST = 'LAST'
MONTHLY_SUM = 'MONTHLY_SUM'
MONTHLY_AVG = 'MONTHLY_AVG'
2014-07-08 15:19:15 +00:00
PERIODS = (
2014-07-09 16:17:43 +00:00
(LAST, _("Last")),
(MONTHLY_SUM, _("Monthly Sum")),
(MONTHLY_AVG, _("Monthly Average")),
2014-07-08 15:19:15 +00:00
)
2014-10-07 13:08:59 +00:00
_related = set() # keeps track of related models for resource cleanup
2014-07-08 15:19:15 +00:00
2014-07-16 15:20:16 +00:00
name = models.CharField(_("name"), max_length=32,
2014-10-23 15:38:46 +00:00
help_text=_("Required. 32 characters or fewer. Lowercase letters, "
"digits and hyphen only."),
validators=[validators.validate_name])
2014-07-16 15:20:16 +00:00
verbose_name = models.CharField(_("verbose name"), max_length=256)
2014-07-10 15:19:06 +00:00
content_type = models.ForeignKey(ContentType,
2014-07-16 15:20:16 +00:00
help_text=_("Model where this resource will be hooked."))
2014-07-11 14:48:46 +00:00
period = models.CharField(_("period"), max_length=16, choices=PERIODS,
default=LAST,
2014-10-23 15:38:46 +00:00
help_text=_("Operation used for aggregating this resource monitored data."))
2014-09-26 15:05:20 +00:00
on_demand = models.BooleanField(_("on demand"), default=False,
2014-07-10 15:19:06 +00:00
help_text=_("If enabled the resource will not be pre-allocated, "
"but allocated under the application demand"))
2014-07-09 16:17:43 +00:00
default_allocation = models.PositiveIntegerField(_("default allocation"),
2014-07-11 14:48:46 +00:00
null=True, blank=True,
2014-07-10 15:19:06 +00:00
help_text=_("Default allocation value used when this is not an "
2014-07-11 14:48:46 +00:00
"on demand resource"))
2014-07-16 15:20:16 +00:00
unit = models.CharField(_("unit"), max_length=16,
2014-10-27 17:34:14 +00:00
help_text=_("The unit in which this resource is represented. "
2014-10-23 15:38:46 +00:00
"For example GB, KB or subscribers"))
scale = models.CharField(_("scale"), max_length=32, validators=[validate_scale],
2014-07-16 15:20:16 +00:00
help_text=_("Scale in which this resource monitoring resoults should "
"be prorcessed to match with unit. e.g. <tt>10**9</tt>"))
2014-07-10 15:19:06 +00:00
disable_trigger = models.BooleanField(_("disable trigger"), default=False,
2014-07-11 14:48:46 +00:00
help_text=_("Disables monitors exeeded and recovery triggers"))
2014-07-10 10:03:22 +00:00
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
2014-07-11 14:48:46 +00:00
null=True, blank=True,
help_text=_("Crontab for periodic execution. "
"Leave it empty to disable periodic monitoring"))
2014-07-25 13:27:31 +00:00
monitors = fields.MultiSelectField(_("monitors"), max_length=256, blank=True,
2014-07-21 12:20:04 +00:00
choices=ServiceMonitor.get_plugin_choices(),
2014-07-11 14:48:46 +00:00
help_text=_("Monitor backends used for monitoring this resource."))
2014-09-30 10:20:11 +00:00
is_active = models.BooleanField(_("active"), default=True)
2014-07-16 15:20:16 +00:00
2014-07-25 13:27:31 +00:00
objects = ResourceQuerySet.as_manager()
2014-07-16 15:20:16 +00:00
class Meta:
unique_together = (
('name', 'content_type'),
('verbose_name', 'content_type')
)
2014-07-08 15:19:15 +00:00
def __unicode__(self):
2014-07-16 15:20:16 +00:00
return "{}-{}".format(str(self.content_type), self.name)
2014-07-08 15:19:15 +00:00
2014-10-23 15:38:46 +00:00
def clean(self):
self.verbose_name = self.verbose_name.strip()
2014-07-10 10:03:22 +00:00
def save(self, *args, **kwargs):
2014-09-23 11:13:50 +00:00
created = not self.pk
2014-07-10 10:03:22 +00:00
super(Resource, self).save(*args, **kwargs)
2014-11-13 15:34:00 +00:00
self.sync_periodic_task()
2014-10-09 17:04:12 +00:00
# This only work on tests (multiprocessing used on real deployments)
apps.get_app_config('resources').reload_relations()
2014-11-13 15:34:00 +00:00
run('sleep 2 && touch %s/wsgi.py' % get_project_root(), async=True, display=True)
2014-07-10 15:19:06 +00:00
2014-07-10 10:03:22 +00:00
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)
name = 'monitor.%s' % str(self)
2014-11-13 15:34:00 +00:00
def sync_periodic_task(self):
name = 'monitor.%s' % str(self)
if self.pk and self.crontab:
try:
task = PeriodicTask.objects.get(name=name)
except PeriodicTask.DoesNotExist:
if self.is_active:
PeriodicTask.objects.create(
name=name,
task='resources.Monitor',
args=[self.pk],
crontab=self.crontab
)
else:
if task.crontab != self.crontab:
task.crontab = self.crontab
task.save(update_fields=['crontab'])
else:
PeriodicTask.objects.filter(
name=name,
).delete()
def get_scale(self):
return eval(self.scale)
2014-11-10 15:40:51 +00:00
def get_verbose_name(self):
return self.verbose_name or self.name
2014-07-10 10:03:22 +00:00
class ResourceData(models.Model):
""" Stores computed resource usage and allocation """
2014-09-26 15:05:20 +00:00
resource = models.ForeignKey(Resource, related_name='dataset', verbose_name=_("resource"))
content_type = models.ForeignKey(ContentType, verbose_name=_("content type"))
object_id = models.PositiveIntegerField(_("object id"))
2014-11-09 10:16:07 +00:00
used = models.DecimalField(_("used"), max_digits=16, decimal_places=2, null=True,
editable=False)
updated_at = models.DateTimeField(_("updated"), null=True, editable=False)
allocated = models.DecimalField(_("allocated"), max_digits=8, decimal_places=2,
null=True, blank=True)
2014-07-10 15:19:06 +00:00
content_object = GenericForeignKey()
2014-07-10 10:03:22 +00:00
class Meta:
unique_together = ('resource', 'content_type', 'object_id')
verbose_name_plural = _("resource data")
@classmethod
def get_or_create(cls, obj, resource):
2014-07-11 21:09:17 +00:00
ct = ContentType.objects.get_for_model(type(obj))
2014-07-10 10:03:22 +00:00
try:
2014-11-13 15:34:00 +00:00
return cls.objects.get(
content_type=ct,
object_id=obj.pk,
resource=resource
)
2014-07-11 21:09:17 +00:00
except cls.DoesNotExist:
2014-11-13 15:34:00 +00:00
return cls.objects.create(
content_object=obj,
resource=resource,
allocated=resource.default_allocation
)
2014-07-08 15:19:15 +00:00
2014-10-06 14:57:02 +00:00
@property
def unit(self):
return self.resource.unit
2014-07-10 10:03:22 +00:00
def get_used(self):
2014-07-16 15:20:16 +00:00
return helpers.compute_resource_usage(self)
2014-09-22 15:59:53 +00:00
def update(self, current=None):
if current is None:
current = self.get_used()
self.used = current or 0
2014-09-26 15:05:20 +00:00
self.updated_at = timezone.now()
self.save(update_fields=['used', 'updated_at'])
2014-10-27 13:29:02 +00:00
def monitor(self):
tasks.monitor(self.resource_id, ids=(self.object_id,))
2014-11-13 15:34:00 +00:00
def get_monitor_datasets(self):
resource = self.resource
today = timezone.now()
datasets = []
for monitor in resource.monitors:
resource_model = self.content_type.model_class()
model_path = ServiceMonitor.get_backend(monitor).model
monitor_model = get_model(model_path)
if resource_model == monitor_model:
dataset = MonitorData.objects.filter(
monitor=monitor,
content_type=self.content_type_id,
object_id=self.object_id
)
else:
path = get_model_field_path(monitor_model, resource_model)
fields = '__'.join(path)
objects = monitor_model.objects.filter(**{fields: self.object_id})
pks = objects.values_list('id', flat=True)
ct = ContentType.objects.get_for_model(monitor_model)
dataset = MonitorData.objects.filter(
monitor=monitor,
content_type=ct,
object_id__in=pks
)
if resource.period in (resource.MONTHLY_AVG, resource.MONTHLY_SUM):
datasets.append(
dataset.filter(
created_at__year=today.year,
created_at__month=today.month
)
)
elif resource.period == resource.LAST:
try:
datasets.append(dataset.latest())
except MonitorData.DoesNotExist:
continue
else:
raise NotImplementedError("%s support not implemented" % self.period)
return datasets
2014-07-08 15:19:15 +00:00
class MonitorData(models.Model):
2014-07-09 16:17:43 +00:00
""" Stores monitored data """
monitor = models.CharField(_("monitor"), max_length=256,
2014-07-21 12:20:04 +00:00
choices=ServiceMonitor.get_plugin_choices())
2014-09-26 15:05:20 +00:00
content_type = models.ForeignKey(ContentType, verbose_name=_("content type"))
object_id = models.PositiveIntegerField(_("object id"))
2014-10-27 17:34:14 +00:00
created_at = models.DateTimeField(_("created"), default=timezone.now)
2014-07-16 15:20:16 +00:00
value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
2014-07-08 15:19:15 +00:00
2014-07-10 15:19:06 +00:00
content_object = GenericForeignKey()
2014-07-08 15:19:15 +00:00
2014-07-09 16:17:43 +00:00
class Meta:
2014-09-24 20:09:41 +00:00
get_latest_by = 'id'
2014-07-09 16:17:43 +00:00
verbose_name_plural = _("monitor data")
2014-07-08 15:19:15 +00:00
def __unicode__(self):
return str(self.monitor)
2014-10-27 13:29:02 +00:00
@cached_property
def unit(self):
return self.resource.unit
2014-07-10 10:03:22 +00:00
def create_resource_relation():
2014-07-18 15:32:27 +00:00
class ResourceHandler(object):
""" account.resources.web """
def __getattr__(self, attr):
2014-07-22 21:47:01 +00:00
""" get or build ResourceData """
2014-11-09 10:16:07 +00:00
try:
return self.obj.__resource_cache[attr]
except AttributeError:
self.obj.__resource_cache = {}
except KeyError:
pass
2014-07-21 15:43:36 +00:00
try:
2014-07-22 21:47:01 +00:00
data = self.obj.resource_set.get(resource__name=attr)
2014-07-21 15:43:36 +00:00
except ResourceData.DoesNotExist:
model = self.obj._meta.model_name
2014-11-13 15:34:00 +00:00
resource = Resource.objects.get(
content_type__model=model,
name=attr,
is_active=True
)
data = ResourceData(
content_object=self.obj,
resource=resource,
allocated=resource.default_allocation
)
2014-11-09 10:16:07 +00:00
self.obj.__resource_cache[attr] = data
2014-07-22 21:47:01 +00:00
return data
2014-07-18 15:32:27 +00:00
def __get__(self, obj, cls):
2014-07-25 15:17:50 +00:00
""" proxy handled object """
2014-07-18 15:32:27 +00:00
self.obj = obj
return self
2014-10-07 13:08:59 +00:00
# Clean previous state
for related in Resource._related:
try:
delattr(related, 'resource_set')
delattr(related, 'resources')
except AttributeError:
pass
else:
related._meta.virtual_fields = [
field for field in related._meta.virtual_fields if field.rel.to != ResourceData
]
2014-07-10 15:19:06 +00:00
relation = GenericRelation('resources.ResourceData')
for ct, resources in Resource.objects.group_by('content_type').iteritems():
2014-07-25 13:27:31 +00:00
model = ct.model_class()
2014-07-18 15:32:27 +00:00
model.add_to_class('resource_set', relation)
model.resources = ResourceHandler()
2014-10-07 13:08:59 +00:00
Resource._related.add(model)