django-orchestra/orchestra/apps/orchestration/manager.py

128 lines
4.5 KiB
Python
Raw Normal View History

import logging
2014-05-08 16:59:35 +00:00
import threading
import traceback
2015-03-23 15:36:51 +00:00
from collections import OrderedDict
2014-05-08 16:59:35 +00:00
from django import db
from django.core.mail import mail_admins
2014-05-08 16:59:35 +00:00
2014-05-13 13:46:40 +00:00
from orchestra.utils.python import import_class
2014-05-08 16:59:35 +00:00
from . import settings
from .helpers import send_report
from .models import BackendLog
2015-03-12 14:05:23 +00:00
from .signals import pre_action, post_action
2014-05-08 16:59:35 +00:00
logger = logging.getLogger(__name__)
2015-03-04 21:06:16 +00:00
router = import_class(settings.ORCHESTRATION_ROUTER)
2014-05-08 16:59:35 +00:00
def as_task(execute):
def wrapper(*args, **kwargs):
""" send report """
# Tasks run on a separate transaction pool (thread), no need to temper with the transaction
2014-10-27 14:31:04 +00:00
try:
log = execute(*args, **kwargs)
if log.state != log.SUCCESS:
send_report(execute, args, log)
except Exception as e:
subject = 'EXCEPTION executing backend(s) %s %s' % (str(args), str(kwargs))
message = traceback.format_exc()
logger.error(subject)
logger.error(message)
mail_admins(subject, message)
# We don't propagate the exception further to avoid transaction rollback
2014-10-27 14:31:04 +00:00
else:
# Using the wrapper function as threader messenger for the execute output
# Absense of it will indicate a failure at this stage
2014-10-27 14:31:04 +00:00
wrapper.log = log
return log
return wrapper
def close_connection(execute):
""" Threads have their own connection pool, closing it when finishing """
def wrapper(*args, **kwargs):
try:
log = execute(*args, **kwargs)
except:
pass
else:
wrapper.log = log
2014-10-27 14:31:04 +00:00
finally:
db.connection.close()
2014-05-08 16:59:35 +00:00
return wrapper
def execute(operations, async=False):
2014-05-08 16:59:35 +00:00
""" generates and executes the operations on the servers """
2015-03-23 15:36:51 +00:00
scripts = OrderedDict()
2014-07-17 16:09:24 +00:00
cache = {}
2015-03-23 15:36:51 +00:00
block = False
2014-10-04 09:29:18 +00:00
# Generate scripts per server+backend
2014-05-08 16:59:35 +00:00
for operation in operations:
2014-10-02 15:58:27 +00:00
logger.debug("Queued %s" % str(operation))
2015-03-04 21:06:16 +00:00
if operation.servers is None:
operation.servers = router.get_servers(operation, cache=cache)
for server in operation.servers:
2014-05-08 16:59:35 +00:00
key = (server, operation.backend)
if key not in scripts:
scripts[key] = (operation.backend(), [operation])
2014-07-25 15:17:50 +00:00
scripts[key][0].prepare()
2014-05-08 16:59:35 +00:00
else:
scripts[key][1].append(operation)
2014-10-04 09:29:18 +00:00
# Get and call backend action method
2015-03-12 14:05:23 +00:00
backend = scripts[key][0]
method = getattr(backend, operation.action)
kwargs = {
'sender': backend.__class__,
'backend': backend,
'instance': operation.instance,
'action': operation.action,
}
pre_action.send(**kwargs)
2014-05-08 16:59:35 +00:00
method(operation.instance)
2015-03-12 14:05:23 +00:00
post_action.send(**kwargs)
2015-03-23 15:36:51 +00:00
if backend.block:
block = True
2014-05-08 16:59:35 +00:00
# Execute scripts on each server
threads = []
executions = []
for key, value in scripts.iteritems():
server, __ = key
backend, operations = value
backend.commit()
execute = as_task(backend.execute)
2015-03-23 15:36:51 +00:00
logger.debug('%s is going to be executed on %s' % (backend, server))
if block:
# Execute one bakend at a time, no need for threads
execute(server, async=async)
else:
execute = close_connection(execute)
thread = threading.Thread(target=execute, args=(server,), kwargs={'async': async})
thread.start()
threads.append(thread)
2014-05-08 16:59:35 +00:00
executions.append((execute, operations))
[ thread.join() for thread in threads ]
logs = []
2014-10-04 09:29:18 +00:00
# collect results
2014-05-08 16:59:35 +00:00
for execution, operations in executions:
# There is no log if an exception has been rised at the very end of the execution
if hasattr(execution, 'log'):
for operation in operations:
logger.info("Executed %s" % str(operation))
operation.log = execution.log
if operation.object_id:
# Not all backends are call with objects saved on the database
operation.save()
stdout = execution.log.stdout.strip()
stdout and logger.debug('STDOUT %s', stdout)
stderr = execution.log.stderr.strip()
stderr and logger.debug('STDERR %s', stderr)
logs.append(execution.log)
else:
mocked_log = BackendLog(state=BackendLog.EXCEPTION)
logs.append(mocked_log)
2014-05-08 16:59:35 +00:00
return logs