diff --git a/TODO.md b/TODO.md index 73826208..25c2bc65 100644 --- a/TODO.md +++ b/TODO.md @@ -236,7 +236,6 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * display subline links on billlines, to show that they exists. * update service orders on a celery task? because it take alot -* * billline quantity eval('10x100') instead of miningless description '(10*100)' * IMPORTANT do more test, make sure billed until doesn't get uodated whhen services are billed with les metric, and don't upgrade billed_until when undoing under this circumstances @@ -245,8 +244,6 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * threshold for significative metric accountancy on services.handler * http://orchestra.pangea.org/admin/orders/order/6418/ * http://orchestra.pangea.org/admin/orders/order/6495/bill_selected_orders/ - * >>> round(float(decimal.Decimal('2.63'))/0.5)*0.5 - * >>> round(float(str(decimal.Decimal('2.99')).split('.')[0]))/1*1 * move normurlpath to orchestra.utils from websites.utils @@ -286,23 +283,13 @@ translation.activate('ca') ugettext("Description") -Object = disk*15 -bscw quota -root@web:/home/pangea/bscw/bin ./bsadmin quota report - Disk Objects -User usage soft hard time usage soft hard time -xxx2 -- 0 20M 22M 9 200 300 -xxxxxxxxxxxxx -- 0 20M 22M 8 200 300 -xxxxx -- 0 20M 22M 7 200 300 -xxxxx -- 0 20M 22M 7 200 300 - * saas validate_creation generic approach, for all backends. standard output * html code x: × -* cleanup backendlogs, monitor data and metricstorage +* periodic task to cleanup backendlogs, monitor data and metricstorage * create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help * uwsgi --max-requests=5000 \ # respawn processes after serving 5000 requests and @@ -313,3 +300,19 @@ celery max-tasks-per-child * postupgradeorchestra send signals in order to hook custom stuff * make base home for systemusers that ara homed into main account systemuser + + +* user force_text instead of unicode for _() + +* autoscale celery workers http://docs.celeryproject.org/en/latest/userguide/workers.html#autoscaling + + +* Delete transaction middleware + + +* webapp has_website list filter + + +apt-get install python3 python3-pip +cp /usr/local/lib/python2.7/dist-packages/orchestra.pth /usr/local/lib/python3.4/dist-packages/ +glic3rinu's django-fluent-dashboard diff --git a/docs/conf.py b/docs/conf.py index 7b99e9cd..1bd8e078 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + # -*- coding: utf-8 -*- # # django-orchestra documentation build configuration file, created by diff --git a/orchestra/admin/__init__.py b/orchestra/admin/__init__.py index 89dd130c..8885990f 100644 --- a/orchestra/admin/__init__.py +++ b/orchestra/admin/__init__.py @@ -1,2 +1,2 @@ -from options import * -from dashboard import * +from .options import * +from .dashboard import * diff --git a/orchestra/admin/actions.py b/orchestra/admin/actions.py index 7d5e00de..1f5a32d1 100644 --- a/orchestra/admin/actions.py +++ b/orchestra/admin/actions.py @@ -100,7 +100,7 @@ class SendEmail(object): 'content_message': _( "Are you sure you want to send the following message to the following %s?" ) % self.opts.verbose_name_plural, - 'display_objects': [u"%s (%s)" % (contact, contact.email) for contact in self.queryset], + 'display_objects': ["%s (%s)" % (contact, contact.email) for contact in self.queryset], 'form': form, 'subject': subject, 'message': message, diff --git a/orchestra/admin/dashboard.py b/orchestra/admin/dashboard.py index 722f2511..03586b66 100644 --- a/orchestra/admin/dashboard.py +++ b/orchestra/admin/dashboard.py @@ -5,7 +5,7 @@ from orchestra.core import services def generate_services_group(): models = [] - for model, options in services.get().iteritems(): + for model, options in services.get().items(): if options.get('menu', True): models.append("%s.%s" % (model.__module__, model._meta.object_name)) diff --git a/orchestra/admin/forms.py b/orchestra/admin/forms.py index 5d84fe1d..18ca411c 100644 --- a/orchestra/admin/forms.py +++ b/orchestra/admin/forms.py @@ -17,7 +17,7 @@ class AdminFormMixin(object): def as_admin(self): prepopulated_fields = {} fieldsets = [ - (None, {'fields': self.fields.keys()}) + (None, {'fields': list(self.fields.keys())}) ] adminform = helpers.AdminForm(self, fieldsets, prepopulated_fields) template = Template( @@ -32,7 +32,7 @@ class AdminFormSet(BaseModelFormSet): def as_admin(self): prepopulated = {} fieldsets = [ - (None, {'fields': self.form().fields.keys()}) + (None, {'fields': list(self.form().fields.keys())}) ] readonly = getattr(self.form.Meta, 'readonly_fields', ()) if not hasattr(self.modeladmin, 'verbose_name_plural'): @@ -114,7 +114,11 @@ class AdminPasswordChangeForm(forms.Form): if password: self.user.set_password(password) if commit: - self.user.save(update_fields=['password']) + try: + self.user.save(update_fields=['password']) + except ValueError: + # password is not a field but an attribute + self.user.save() # Trigger the backend for ix, rel in enumerate(self.related): password = self.cleaned_data['password1_%s' % ix] if password: diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py index 1bacdf64..bf268378 100644 --- a/orchestra/admin/menu.py +++ b/orchestra/admin/menu.py @@ -32,7 +32,7 @@ def api_link(context): def get_services(): childrens = [] - for model, options in services.get().iteritems(): + for model, options in services.get().items(): if options.get('menu', True): opts = model._meta url = reverse('admin:{}_{}_changelist'.format( @@ -50,7 +50,7 @@ def get_accounts(): if isinstalled('orchestra.apps.issues'): url = reverse('admin:issues_ticket_changelist') childrens.append(items.MenuItem(_("Tickets"), url)) - for model, options in accounts.get().iteritems(): + for model, options in accounts.get().items(): if options.get('menu', True): opts = model._meta url = reverse('admin:{}_{}_changelist'.format( diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py index fd95132f..cd936cf9 100644 --- a/orchestra/admin/options.py +++ b/orchestra/admin/options.py @@ -80,7 +80,7 @@ class ChangeViewActionsMixin(object): """ allow customization on modelamdin """ views = [] for action in self.change_view_actions: - if isinstance(action, basestring): + if isinstance(action, str): action = getattr(self, action) view = action_to_view(action, self) view.url_name = getattr(action, 'url_name', action.__name__) diff --git a/orchestra/admin/utils.py b/orchestra/admin/utils.py index b9fdf334..b3cc03e5 100644 --- a/orchestra/admin/utils.py +++ b/orchestra/admin/utils.py @@ -20,7 +20,7 @@ from .decorators import admin_field def get_modeladmin(model, import_module=True): """ returns the modeladmin registred for model """ - for k,v in admin.site._registry.iteritems(): + for k,v in admin.site._registry.items(): if k is model: return v if import_module: @@ -97,7 +97,7 @@ def change_url(obj): @admin_field def admin_link(*args, **kwargs): instance = args[-1] - if kwargs['field'] in ['id', 'pk', '__unicode__']: + if kwargs['field'] in ['id', 'pk', '__str__']: obj = instance else: try: diff --git a/orchestra/api/__init__.py b/orchestra/api/__init__.py index c5895bfd..9f59e74c 100644 --- a/orchestra/api/__init__.py +++ b/orchestra/api/__init__.py @@ -1,2 +1,2 @@ -from options import * -from actions import * +from .options import * +from .actions import * diff --git a/orchestra/api/actions.py b/orchestra/api/actions.py index 257f138c..4eed0b37 100644 --- a/orchestra/api/actions.py +++ b/orchestra/api/actions.py @@ -10,7 +10,7 @@ class SetPasswordApiMixin(object): def set_password(self, request, pk): obj = self.get_object() data = request.DATA - if isinstance(data, basestring): + if isinstance(data, str): data = { 'password': data } diff --git a/orchestra/api/fields.py b/orchestra/api/fields.py index 16d42f80..6183898b 100644 --- a/orchestra/api/fields.py +++ b/orchestra/api/fields.py @@ -20,17 +20,17 @@ class OptionField(serializers.WritableField): properties = serializers.RelationsList() if value: model = getattr(parent.opts.model, self.source or 'options').related.model - if isinstance(value, basestring): + if isinstance(value, str): try: value = json.loads(value) except: raise exceptions.ParseError("Malformed property: %s" % str(value)) if not related_manager: # POST (new parent object) - return [ model(name=n, value=v) for n,v in value.iteritems() ] + return [ model(name=n, value=v) for n,v in value.items() ] # PUT to_save = [] - for (name, value) in value.iteritems(): + for (name, value) in value.items(): try: # Update existing property prop = related_manager.get(name=name) diff --git a/orchestra/api/serializers.py b/orchestra/api/serializers.py index ef9ba2fd..21f17727 100644 --- a/orchestra/api/serializers.py +++ b/orchestra/api/serializers.py @@ -24,7 +24,7 @@ class HyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer): """ removes postonly_fields from attrs when not posting """ model_attrs = dict(**attrs) if instance is not None: - for attr, value in attrs.iteritems(): + for attr, value in attrs.items(): if attr in self.opts.postonly_fields: model_attrs.pop(attr) return super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance) diff --git a/orchestra/apps/accounts/actions.py b/orchestra/apps/accounts/actions.py index 69e92d3b..31994b02 100644 --- a/orchestra/apps/accounts/actions.py +++ b/orchestra/apps/accounts/actions.py @@ -44,7 +44,7 @@ def service_report(modeladmin, request, queryset): accounts = [] fields = [] # First we get related manager names to fire a prefetch related - for name, field in queryset.model._meta._name_map.iteritems(): + for name, field in queryset.model._meta._name_map.items(): model = field[0].model if model in services.get() and model != queryset.model: fields.append((model, name)) @@ -63,3 +63,7 @@ def service_report(modeladmin, request, queryset): 'date': timezone.now().today() } return render(request, settings.ACCOUNTS_SERVICE_REPORT_TEMPLATE, context) + + +def delete_related_services(modeladmin, request, queryset): + pass diff --git a/orchestra/apps/accounts/forms.py b/orchestra/apps/accounts/forms.py index 1a9eb9ca..faeb842f 100644 --- a/orchestra/apps/accounts/forms.py +++ b/orchestra/apps/accounts/forms.py @@ -62,12 +62,12 @@ def create_account_creation_form(): field_name = 'create_%s' % model._meta.model_name if self.cleaned_data[field_name]: kwargs = { - key: eval(value, {'account': account}) for key, value in related_kwargs.iteritems() + key: eval(value, {'account': account}) for key, value in related_kwargs.items() } model.objects.create(account=account, **kwargs) fields.update({ - 'create_related_fields': fields.keys(), + 'create_related_fields': list(fields.keys()), 'clean': clean, 'save_model': save_model, 'save_related': save_related, diff --git a/orchestra/apps/accounts/models.py b/orchestra/apps/accounts/models.py index 76890be8..e68aaa2e 100644 --- a/orchestra/apps/accounts/models.py +++ b/orchestra/apps/accounts/models.py @@ -44,7 +44,7 @@ class Account(auth.AbstractBaseUser): USERNAME_FIELD = 'username' REQUIRED_FIELDS = ['email'] - def __unicode__(self): + def __str__(self): return self.name @property diff --git a/orchestra/apps/accounts/settings.py b/orchestra/apps/accounts/settings.py index 838020b6..b5d380f5 100644 --- a/orchestra/apps/accounts/settings.py +++ b/orchestra/apps/accounts/settings.py @@ -11,6 +11,7 @@ ACCOUNTS_TYPES = getattr(settings, 'ACCOUNTS_TYPES', ( ('COMPANY', _("Company")), ('PUBLICBODY', _("Public body")), ('STAFF', _("Staff")), + ('FRIEND', _("Friend")), )) diff --git a/orchestra/apps/bills/actions.py b/orchestra/apps/bills/actions.py index ebec9709..ffd3c083 100644 --- a/orchestra/apps/bills/actions.py +++ b/orchestra/apps/bills/actions.py @@ -1,5 +1,5 @@ -import StringIO import zipfile +from io import StringIO from django.contrib import messages from django.contrib.admin import helpers @@ -20,7 +20,7 @@ from .helpers import validate_contact def download_bills(modeladmin, request, queryset): if queryset.count() > 1: - stringio = StringIO.StringIO() + stringio = StringIO() archive = zipfile.ZipFile(stringio, 'w') for bill in queryset: pdf = html_to_pdf(bill.html or bill.render()) @@ -122,7 +122,7 @@ def undo_billing(modeladmin, request, queryset): except KeyError: group[line.order] = [line] # TODO force incomplete info - for order, lines in group.iteritems(): + for order, lines in group.items(): # Find path from ini to end for attr in ['order_id', 'order_billed_on', 'order_billed_until']: if not getattr(self, attr): @@ -131,7 +131,7 @@ def undo_billing(modeladmin, request, queryset): if 'a' != order.billed_on: raise ValidationError(_("Dates don't match")) prev = order.billed_on - for ix in xrange(0, len(lines)): + for ix in range(0, len(lines)): if lines[ix].order_b: # TODO we need to look at the periods here pass order.billed_until = self.order_billed_until diff --git a/orchestra/apps/bills/admin.py b/orchestra/apps/bills/admin.py index fce1bf44..79be1ef8 100644 --- a/orchestra/apps/bills/admin.py +++ b/orchestra/apps/bills/admin.py @@ -210,8 +210,8 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin): def get_inline_instances(self, request, obj=None): inlines = super(BillAdmin, self).get_inline_instances(request, obj) if obj and not obj.is_open: - return [inline for inline in inlines if not type(inline) == BillLineInline] - return [inline for inline in inlines if not type(inline) == ClosedBillLineInline] + return [inline for inline in inlines if not isinstance(inline, BillLineInline)] + return [inline for inline in inlines if not isinstance(inline, ClosedBillLineInline)] def formfield_for_dbfield(self, db_field, **kwargs): """ Make value input widget bigger """ diff --git a/orchestra/apps/bills/forms.py b/orchestra/apps/bills/forms.py index 599c642a..71378909 100644 --- a/orchestra/apps/bills/forms.py +++ b/orchestra/apps/bills/forms.py @@ -33,7 +33,7 @@ class SelectSourceForm(forms.ModelForm): choices.append((source.pk, str(source))) self.fields['source'].choices = choices self.fields['source'].initial = choices[-1][0] - self.fields['bill_link'].initial = admin_link('__unicode__')(bill) + self.fields['bill_link'].initial = admin_link('__str__')(bill) self.fields['display_type'].initial = bill.get_type_display() def clean_source(self): diff --git a/orchestra/apps/bills/models.py b/orchestra/apps/bills/models.py index ddee9f28..2336c23b 100644 --- a/orchestra/apps/bills/models.py +++ b/orchestra/apps/bills/models.py @@ -31,7 +31,7 @@ class BillContact(models.Model): default=settings.BILLS_CONTACT_DEFAULT_COUNTRY) vat = models.CharField(_("VAT number"), max_length=64) - def __unicode__(self): + def __str__(self): return self.name def get_name(self): @@ -98,7 +98,7 @@ class Bill(models.Model): class Meta: get_latest_by = 'id' - def __unicode__(self): + def __str__(self): return self.number @cached_property @@ -235,7 +235,7 @@ class Bill(models.Model): def get_total(self): total = 0 - for tax, subtotal in self.get_subtotals().iteritems(): + for tax, subtotal in self.get_subtotals().items(): subtotal, taxes = subtotal total += subtotal + taxes return total @@ -287,7 +287,7 @@ class BillLine(models.Model): amended_line = models.ForeignKey('self', verbose_name=_("amended line"), related_name='amendment_lines', null=True, blank=True) - def __unicode__(self): + def __str__(self): return "#%i" % self.number @cached_property diff --git a/orchestra/apps/bills/settings.py b/orchestra/apps/bills/settings.py index 6b116ccb..c6696089 100644 --- a/orchestra/apps/bills/settings.py +++ b/orchestra/apps/bills/settings.py @@ -90,7 +90,7 @@ BILLS_CONTACT_DEFAULT_CITY = getattr(settings, 'BILLS_CONTACT_DEFAULT_CITY', BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES', - ((k,v) for k,v in data.COUNTRIES.iteritems()) + ((k,v) for k,v in data.COUNTRIES.items()) ) diff --git a/orchestra/apps/contacts/admin.py b/orchestra/apps/contacts/admin.py index b7dc5d5c..d5ea86f4 100644 --- a/orchestra/apps/contacts/admin.py +++ b/orchestra/apps/contacts/admin.py @@ -62,7 +62,7 @@ class ContactAdmin(AccountAdminMixin, ExtendedModelAdmin): actions = [SendEmail(),] def dispaly_name(self, contact): - return unicode(contact) + return str(contact) dispaly_name.short_description = _("Name") dispaly_name.admin_order_field = 'short_name' diff --git a/orchestra/apps/contacts/models.py b/orchestra/apps/contacts/models.py index 0584b73e..1001a93d 100644 --- a/orchestra/apps/contacts/models.py +++ b/orchestra/apps/contacts/models.py @@ -55,7 +55,7 @@ class Contact(models.Model): choices=settings.CONTACTS_COUNTRIES, default=settings.CONTACTS_DEFAULT_COUNTRY) - def __unicode__(self): + def __str__(self): return self.full_name or self.short_name def clean(self): diff --git a/orchestra/apps/contacts/settings.py b/orchestra/apps/contacts/settings.py index 82138af5..331a6f3a 100644 --- a/orchestra/apps/contacts/settings.py +++ b/orchestra/apps/contacts/settings.py @@ -18,7 +18,7 @@ CONTACTS_DEFAULT_CITY = getattr(settings, 'CONTACTS_DEFAULT_CITY', CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', ( - (k,v) for k,v in data.COUNTRIES.iteritems() + (k,v) for k,v in data.COUNTRIES.items() )) diff --git a/orchestra/apps/databases/models.py b/orchestra/apps/databases/models.py index 76972437..68ae8cac 100644 --- a/orchestra/apps/databases/models.py +++ b/orchestra/apps/databases/models.py @@ -26,7 +26,7 @@ class Database(models.Model): class Meta: unique_together = ('name', 'type') - def __unicode__(self): + def __str__(self): return "%s" % self.name @property @@ -59,7 +59,7 @@ class DatabaseUser(models.Model): verbose_name_plural = _("DB users") unique_together = ('username', 'type') - def __unicode__(self): + def __str__(self): return self.username def get_username(self): @@ -68,7 +68,7 @@ class DatabaseUser(models.Model): def set_password(self, password): if self.type == self.MYSQL: # MySQL stores sha1(sha1(password).binary).hex - binary = hashlib.sha1(password).digest() + binary = hashlib.sha1(password.encode('utf-8')).digest() hexdigest = hashlib.sha1(binary).hexdigest() self.password = '*%s' % hexdigest.upper() else: diff --git a/orchestra/apps/domains/admin.py b/orchestra/apps/domains/admin.py index f91104d4..e0ab1e01 100644 --- a/orchestra/apps/domains/admin.py +++ b/orchestra/apps/domains/admin.py @@ -41,7 +41,7 @@ class DomainInline(admin.TabularInline): extra = 0 verbose_name_plural = _("Subdomains") - domain_link = admin_link('__unicode__') + domain_link = admin_link('__str__') domain_link.short_description = _("Name") account_link = admin_link('account') diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 4dda4944..87ad0b80 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -23,7 +23,7 @@ class Domain(models.Model): serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, help_text=_("Serial number")) - def __unicode__(self): + def __str__(self): return self.name @classmethod @@ -228,7 +228,7 @@ class Record(models.Model): type = models.CharField(_("type"), max_length=32, choices=TYPE_CHOICES) value = models.CharField(_("value"), max_length=256) - def __unicode__(self): + def __str__(self): return "%s %s IN %s %s" % (self.domain, self.get_ttl(), self.type, self.value) def clean(self): diff --git a/orchestra/apps/issues/admin.py b/orchestra/apps/issues/admin.py index 1f9772c4..a80d3d5d 100644 --- a/orchestra/apps/issues/admin.py +++ b/orchestra/apps/issues/admin.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from django import forms from django.conf.urls import patterns from django.contrib import admin @@ -267,8 +265,8 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin): changes = get_ticket_changes(self, request, ticket) if changes: content = markdown_formated_changes(changes) - content += request.POST[u'messages-2-0-content'] - request.POST[u'messages-2-0-content'] = content + content += request.POST['messages-2-0-content'] + request.POST['messages-2-0-content'] = content ticket.mark_as_read_by(request.user) context = {'title': "Issue #%i - %s" % (ticket.id, ticket.subject)} context.update(extra_context or {}) diff --git a/orchestra/apps/issues/filters.py b/orchestra/apps/issues/filters.py index 982ac7b1..f8943645 100644 --- a/orchestra/apps/issues/filters.py +++ b/orchestra/apps/issues/filters.py @@ -22,7 +22,7 @@ class MyTicketsListFilter(SimpleListFilter): def choices(self, cl): """ Remove default All """ choices = iter(super(MyTicketsListFilter, self).choices(cl)) - choices.next() + next(choices) return choices @@ -52,6 +52,6 @@ class TicketStateListFilter(SimpleListFilter): def choices(self, cl): """ Remove default All """ choices = iter(super(TicketStateListFilter, self).choices(cl)) - choices.next() + next(choices) return choices diff --git a/orchestra/apps/issues/helpers.py b/orchestra/apps/issues/helpers.py index 68df9c49..a7cceb30 100644 --- a/orchestra/apps/issues/helpers.py +++ b/orchestra/apps/issues/helpers.py @@ -11,7 +11,7 @@ def filter_actions(modeladmin, ticket, request): del_actions.append('take') exclude = lambda a: not (a == action or a.url_name == action) for action in del_actions: - actions = filter(exclude, actions) + actions = list(filter(exclude, actions)) return actions diff --git a/orchestra/apps/issues/models.py b/orchestra/apps/issues/models.py index 7f608991..9613a79c 100644 --- a/orchestra/apps/issues/models.py +++ b/orchestra/apps/issues/models.py @@ -20,7 +20,7 @@ class Queue(models.Model): default=contacts_settings.CONTACTS_DEFAULT_EMAIL_USAGES, help_text=_("Contacts to notify by email")) - def __unicode__(self): + def __str__(self): return self.verbose_name or self.name def save(self, *args, **kwargs): @@ -77,8 +77,8 @@ class Ticket(models.Model): class Meta: ordering = ['-updated_at'] - def __unicode__(self): - return unicode(self.pk) + def __str__(self): + return str(self.pk) def get_notification_emails(self): """ Get emails of the users related to the ticket """ @@ -164,8 +164,8 @@ class Message(models.Model): class Meta: get_latest_by = 'id' - def __unicode__(self): - return u"#%i" % self.id + def __str__(self): + return "#%i" % self.id def save(self, *args, **kwargs): """ notify stakeholders of ticket update """ diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index c8cfa495..a166cda2 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -295,7 +295,7 @@ class MailmanTraffic(ServiceMonitor): except IOError as e: sys.stderr.write(e) - for list_name, opts in lists.iteritems(): + for list_name, opts in lists.items(): __, object_id, size = opts if size: cmd = ' '.join(('list_members', list_name, '| wc -l')) diff --git a/orchestra/apps/lists/models.py b/orchestra/apps/lists/models.py index 372c560f..2bbef0e4 100644 --- a/orchestra/apps/lists/models.py +++ b/orchestra/apps/lists/models.py @@ -30,7 +30,7 @@ class List(models.Model): class Meta: unique_together = ('address_name', 'address_domain') - def __unicode__(self): + def __str__(self): return self.name @property diff --git a/orchestra/apps/mailboxes/admin.py b/orchestra/apps/mailboxes/admin.py index ec527880..c3860eed 100644 --- a/orchestra/apps/mailboxes/admin.py +++ b/orchestra/apps/mailboxes/admin.py @@ -1,5 +1,5 @@ import copy -from urlparse import parse_qs +from urllib.parse import urlparse from django import forms from django.contrib import admin diff --git a/orchestra/apps/mailboxes/backends.py b/orchestra/apps/mailboxes/backends.py index 7bfbd20a..957dd99f 100644 --- a/orchestra/apps/mailboxes/backends.py +++ b/orchestra/apps/mailboxes/backends.py @@ -431,7 +431,7 @@ class PostfixTraffic(ServiceMonitor): except IOError as e: sys.stderr.write(e) - for username, opts in users.iteritems(): + for username, opts in users.items(): size = 0 for req_id in reverse[username]: size += targets[req_id][1] * counter.get(req_id, 0) diff --git a/orchestra/apps/mailboxes/models.py b/orchestra/apps/mailboxes/models.py index 20463de0..0fa123a0 100644 --- a/orchestra/apps/mailboxes/models.py +++ b/orchestra/apps/mailboxes/models.py @@ -25,7 +25,7 @@ class Mailbox(models.Model): filtering = models.CharField(max_length=16, default=settings.MAILBOXES_MAILBOX_DEFAULT_FILTERING, choices=[ - (k, v[0]) for k,v in settings.MAILBOXES_MAILBOX_FILTERINGS.iteritems() + (k, v[0]) for k,v in settings.MAILBOXES_MAILBOX_FILTERINGS.items() ]) custom_filtering = models.TextField(_("filtering"), blank=True, validators=[validators.validate_sieve], @@ -36,7 +36,7 @@ class Mailbox(models.Model): class Meta: verbose_name_plural = _("mailboxes") - def __unicode__(self): + def __str__(self): return self.name @cached_property @@ -62,7 +62,7 @@ class Mailbox(models.Model): def get_filtering(self): __, filtering = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering] - if isinstance(filtering, basestring): + if isinstance(filtering, str): return filtering return filtering(self) @@ -104,7 +104,7 @@ class Address(models.Model): verbose_name_plural = _("addresses") unique_together = ('name', 'domain') - def __unicode__(self): + def __str__(self): return self.email @property @@ -154,7 +154,7 @@ class Autoresponse(models.Model): message = models.TextField(_("message")) enabled = models.BooleanField(_("enabled"), default=False) - def __unicode__(self): + def __str__(self): return self.address diff --git a/orchestra/apps/miscellaneous/admin.py b/orchestra/apps/miscellaneous/admin.py index 334553d5..35b35085 100644 --- a/orchestra/apps/miscellaneous/admin.py +++ b/orchestra/apps/miscellaneous/admin.py @@ -54,7 +54,7 @@ class MiscServiceAdmin(ExtendedModelAdmin): class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelAdmin): list_display = ( - '__unicode__', 'service_link', 'amount', 'dispaly_active', 'account_link' + '__str__', 'service_link', 'amount', 'dispaly_active', 'account_link' ) list_filter = ('service__name', 'is_active') list_select_related = ('service', 'account') diff --git a/orchestra/apps/miscellaneous/migrations/0001_initial.py b/orchestra/apps/miscellaneous/migrations/0001_initial.py index b9dbcf8b..f3ed1690 100644 --- a/orchestra/apps/miscellaneous/migrations/0001_initial.py +++ b/orchestra/apps/miscellaneous/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import orchestra.core.validators diff --git a/orchestra/apps/miscellaneous/migrations/0002_auto_20141112_1853.py b/orchestra/apps/miscellaneous/migrations/0002_auto_20141112_1853.py index aac1cc9c..d1fd7de1 100644 --- a/orchestra/apps/miscellaneous/migrations/0002_auto_20141112_1853.py +++ b/orchestra/apps/miscellaneous/migrations/0002_auto_20141112_1853.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import orchestra.models.fields diff --git a/orchestra/apps/miscellaneous/models.py b/orchestra/apps/miscellaneous/models.py index db5171f5..bd567eb1 100644 --- a/orchestra/apps/miscellaneous/models.py +++ b/orchestra/apps/miscellaneous/models.py @@ -25,7 +25,7 @@ class MiscService(models.Model): help_text=_("Whether new instances of this service can be created " "or not. Unselect this instead of deleting services.")) - def __unicode__(self): + def __str__(self): return self.name def clean(self): @@ -51,7 +51,7 @@ class Miscellaneous(models.Model): class Meta: verbose_name_plural = _("miscellaneous") - def __unicode__(self): + def __str__(self): return self.identifier or self.description[:32] or str(self.service) @cached_property diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index a55ef611..f508d69e 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -18,7 +18,7 @@ class ServiceMount(plugins.PluginMount): super(ServiceMount, cls).__init__(name, bases, attrs) -class ServiceBackend(plugins.Plugin): +class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): """ Service management backend base class @@ -37,13 +37,8 @@ class ServiceBackend(plugins.Plugin): default_route_match = 'True' block = False # Force the backend manager to block in multiple backend executions and execute them synchronously - __metaclass__ = ServiceMount - - def __unicode__(self): - return type(self).__name__ - def __str__(self): - return unicode(self) + return type(self).__name__ def __init__(self): self.head = [] @@ -138,7 +133,7 @@ class ServiceBackend(plugins.Plugin): scripts[method] += commands except KeyError: pass - return list(scripts.iteritems()) + return list(scripts.items()) def get_banner(self): time = timezone.now().strftime("%h %d, %Y %I:%M:%S %Z") @@ -159,7 +154,7 @@ class ServiceBackend(plugins.Plugin): def append(self, *cmd): # aggregate commands acording to its execution method - if isinstance(cmd[0], basestring): + if isinstance(cmd[0], str): method = self.script_method cmd = cmd[0] else: diff --git a/orchestra/apps/orchestration/helpers.py b/orchestra/apps/orchestration/helpers.py index d0f1671a..edb03318 100644 --- a/orchestra/apps/orchestration/helpers.py +++ b/orchestra/apps/orchestration/helpers.py @@ -7,10 +7,9 @@ from django.utils.translation import ungettext, ugettext_lazy as _ def send_report(method, args, log): - backend = method.im_class().get_name() server = args[0] - subject = '[Orchestra] %s execution %s on %s' - subject = subject % (backend, log.state, server) + backend = method.__self__.__class__.__name__ + subject = '[Orchestra] %s execution %s on %s' % (backend, log.state, server) separator = "\n%s\n\n" % ('~ '*40,) message = separator.join([ "[EXIT CODE] %s" % log.exit_code, diff --git a/orchestra/apps/orchestration/management/commands/orchestrate.py b/orchestra/apps/orchestration/management/commands/orchestrate.py index bcbcfb13..e25c7039 100644 --- a/orchestra/apps/orchestration/management/commands/orchestrate.py +++ b/orchestra/apps/orchestration/management/commands/orchestrate.py @@ -32,7 +32,7 @@ class Command(BaseCommand): scripts, block = manager.generate(operations) servers = [] # Print scripts - for key, value in scripts.iteritems(): + for key, value in scripts.items(): server, __ = key backend, operations = value servers.append(server.name) diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index 28a6150c..47b06bc6 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -1,3 +1,4 @@ + import logging import threading import traceback @@ -47,7 +48,7 @@ def close_connection(execute): def wrapper(*args, **kwargs): try: log = execute(*args, **kwargs) - except: + except Exception as e: pass else: wrapper.log = log @@ -86,7 +87,7 @@ def generate(operations): post_action.send(**kwargs) if backend.block: block = True - for value in scripts.itervalues(): + for value in scripts.values(): backend, operations = value backend.commit() return scripts, block @@ -97,13 +98,13 @@ def execute(scripts, block=False, async=False): # Execute scripts on each server threads = [] executions = [] - for key, value in scripts.iteritems(): + for key, value in scripts.items(): server, __ = key backend, operations = value execute = as_task(backend.execute) logger.debug('%s is going to be executed on %s' % (backend, server)) if block: - # Execute one bakend at a time, no need for threads + # Execute one backend at a time, no need for threads execute(server, async=async) else: execute = close_connection(execute) diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index d2ad1606..133d340e 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -29,7 +29,8 @@ def SSH(backend, log, server, cmds, async=False): """ script = '\n'.join(cmds) script = script.replace('\r', '') - digest = hashlib.md5(script).hexdigest() + bscript = script.encode('utf-8') + digest = hashlib.md5(bscript).hexdigest() path = os.path.join(settings.ORCHESTRATION_TEMP_SCRIPT_PATH, digest) remote_path = "%s.remote" % path log.script = '# %s\n%s' % (remote_path, script) @@ -41,8 +42,8 @@ def SSH(backend, log, server, cmds, async=False): try: # Avoid "Argument list too long" on large scripts by genereting a file # and scping it to the remote server - with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as handle: - handle.write(script) + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as handle: + handle.write(bscript) # ssh connection ssh = paramiko.SSHClient() @@ -62,7 +63,7 @@ def SSH(backend, log, server, cmds, async=False): # Copy script to remote server sftp = paramiko.SFTPClient.from_transport(transport) sftp.put(path, remote_path) - sftp.chmod(remote_path, 0600) + sftp.chmod(remote_path, 0o600) sftp.close() os.remove(path) @@ -124,7 +125,7 @@ def SSH(backend, log, server, cmds, async=False): def Python(backend, log, server, cmds, async=False): # TODO collect stdout? - script = [ str(cmd.func.func_name) + str(cmd.args) for cmd in cmds ] + script = [ str(cmd.func.__name__) + str(cmd.args) for cmd in cmds ] script = json.dumps(script, indent=4).replace('"', '') log.script = '\n'.join([log.script, script]) log.save(update_fields=['script']) @@ -133,7 +134,7 @@ def Python(backend, log, server, cmds, async=False): with CaptureStdout() as stdout: result = cmd(server) for line in stdout: - log.stdout += unicode(line, errors='replace') + '\n' + log.stdout += line + '\n' if async: log.save(update_fields=['stdout']) except: diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index 35fe0bca..062bee33 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -1,6 +1,7 @@ from threading import local from django.core.urlresolvers import resolve +from django.db import connection, transaction from django.db.models.signals import pre_delete, post_save, m2m_changed from django.dispatch import receiver from django.http.response import HttpResponseServerError @@ -36,6 +37,11 @@ class OperationsMiddleware(object): """ Stores all the operations derived from save and delete signals and executes them at the end of the request/response cycle + + It also works as a transaction middleware. Each view function will be run + with commit_on_response activated - that way a save() doesn't do a direct + commit, the commit is done when a successful response is created. If an + exception happens, the database is rolled back. """ # Thread local is used because request object is not available on model signals thread_locals = local() @@ -71,16 +77,55 @@ class OperationsMiddleware(object): instance = kwargs.pop('instance') manager.collect(instance, action, **kwargs) + def commit_transaction(self): + if not transaction.get_autocommit(): + if transaction.is_dirty(): + # Note: it is possible that the commit fails. If the reason is + # closed connection or some similar reason, then there is + # little hope to proceed nicely. However, in some cases ( + # deferred foreign key checks for exampl) it is still possible + # to rollback(). + try: + transaction.commit() + except Exception: + # If the rollback fails, the transaction state will be + # messed up. It doesn't matter, the connection will be set + # to clean state after the request finishes. And, we can't + # clean the state here properly even if we wanted to, the + # connection is in transaction but we can't rollback... + transaction.rollback() + transaction.leave_transaction_management() + raise + transaction.leave_transaction_management() + def process_request(self, request): """ Store request on a thread local variable """ type(self).thread_locals.request = request + # Enters transaction management + transaction.enter_transaction_management() + + def process_exception(self, request, exception): + """Rolls back the database and leaves transaction management""" + if transaction.is_dirty(): + # This rollback might fail because of network failure for example. + # If rollback isn't possible it is impossible to clean the + # connection's state. So leave the connection in dirty state and + # let request_finished signal deal with cleaning the connection. + transaction.rollback() + transaction.leave_transaction_management() def process_response(self, request, response): """ Processes pending backend operations """ if not isinstance(response, HttpResponseServerError): operations = type(self).get_pending_operations() if operations: - logs = Operation.execute(operations) + scripts, block = manager.generate(operations) + # We commit transaction just before executing operations + # because here is when IntegrityError show up + self.commit_transaction() + logs = manager.execute(scripts, block=block) if logs and resolve(request.path).app_name == 'admin': message_user(request, logs) + return response + self.commit_transaction() return response diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index 4385a2b6..370538af 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -25,7 +25,7 @@ class Server(models.Model): choices=settings.ORCHESTRATION_OS_CHOICES, default=settings.ORCHESTRATION_DEFAULT_OS) - def __unicode__(self): + def __str__(self): return self.name def get_address(self): @@ -83,7 +83,7 @@ class BackendLog(models.Model): class Meta: get_latest_by = 'id' - def __unicode__(self): + def __str__(self): return "%s@%s" % (self.backend, self.server) @property @@ -116,7 +116,7 @@ class BackendOperation(models.Model): verbose_name = _("Operation") verbose_name_plural = _("Operations") - def __unicode__(self): + def __str__(self): return '%s.%s(%s)' % (self.backend, self.action, self.instance) def __hash__(self): @@ -184,7 +184,7 @@ class Route(models.Model): class Meta: unique_together = ('backend', 'host') - def __unicode__(self): + def __str__(self): return "%s@%s" % (self.backend, self.host) @property @@ -218,7 +218,7 @@ class Route(models.Model): if not self.match: self.match = 'True' if self.backend: - backend_model = self.backend_class.model + backend_model = self.backend_class.model_class() try: obj = backend_model.objects.all()[0] except IndexError: @@ -227,8 +227,7 @@ class Route(models.Model): bool(self.matches(obj)) except Exception as exception: name = type(exception).__name__ - message = exception.message - raise ValidationError(': '.join((name, message))) + raise ValidationError(': '.join((name, exception))) def matches(self, instance): safe_locals = { diff --git a/orchestra/apps/orders/forms.py b/orchestra/apps/orders/forms.py index e7e71bdc..c96da35a 100644 --- a/orchestra/apps/orders/forms.py +++ b/orchestra/apps/orders/forms.py @@ -29,8 +29,8 @@ class BillSelectedOptionsForm(AdminFormMixin, forms.Form): def selected_related_choices(queryset): for order in queryset: - verbose = u'{description} ' - verbose += u'{account}' + verbose = '{description} ' + verbose += '{account}' verbose = verbose.format( order_url=change_url(order), description=order.description, account_url=change_url(order.account), account=str(order.account) diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index dc795f02..c5df12d3 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -6,7 +6,7 @@ from django.db import models from django.db.migrations.recorder import MigrationRecorder from django.db.models import F, Q from django.db.models.loading import get_model -from django.db.models.signals import post_delete, post_save +from django.db.models.signals import post_delete, post_save, pre_delete from django.dispatch import receiver from django.contrib.admin.models import LogEntry from django.contrib.contenttypes import generic @@ -32,9 +32,9 @@ class OrderQuerySet(models.QuerySet): bill_backend = Order.get_bill_backend() qs = self.select_related('account', 'service') commit = options.get('commit', True) - for account, services in qs.group_by('account', 'service').iteritems(): + for account, services in qs.group_by('account', 'service').items(): bill_lines = [] - for service, orders in services.iteritems(): + for service, orders in services.items(): for order in orders: # Saved for undoing support order.old_billed_on = order.billed_on @@ -65,8 +65,8 @@ class OrderQuerySet(models.QuerySet): conflictive = conflictive.exclude(service__billing_period=Service.NEVER) conflictive = conflictive.select_related('service').group_by('account_id', 'service') qs = Q() - for account_id, services in conflictive.iteritems(): - for service, orders in services.iteritems(): + for account_id, services in conflictive.items(): + for service, orders in services.items(): if not service.rates.exists(): continue ini = datetime.date.max @@ -127,8 +127,8 @@ class Order(models.Model): class Meta: get_latest_by = 'id' - def __unicode__(self): - return unicode(self.service) + def __str__(self): + return str(self.service) @classmethod def update_orders(cls, instance, service=None, commit=True): @@ -178,7 +178,7 @@ class Order(models.Model): MetricStorage.store(self, metric) metric = ', metric:{}'.format(metric) description = handler.get_order_description(instance) - logger.info(u"UPDATED order id:{id}, description:{description}{metric}".format( + logger.info("UPDATED order id:{id}, description:{description}{metric}".format( id=self.id, description=description, metric=metric).encode('ascii', 'ignore') ) if self.description != description: @@ -247,8 +247,8 @@ class MetricStorage(models.Model): class Meta: get_latest_by = 'id' - def __unicode__(self): - return unicode(self.order) + def __str__(self): + return str(self.order) @classmethod def store(cls, order, value): @@ -268,12 +268,27 @@ class MetricStorage(models.Model): accounts.register(Order) +@receiver(pre_delete, dispatch_uid="orders.account_orders") +def account_orders(sender, **kwargs): + account = kwargs['instance'] + if isinstance(account, Order.account.field.rel.to): + account._deleted = True + # TODO build a cache hash table {model: related, model: None} @receiver(post_delete, dispatch_uid="orders.cancel_orders") def cancel_orders(sender, **kwargs): if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS: instance = kwargs['instance'] + # Account delete will delete all related orders, no need to maintain order consistency + if isinstance(instance, Order.account.field.rel.to): +# print 'aaaaaaaaaaaaaAAAAAAAAAAAAAAAAaa' + return +# print 'delete', sender, kwargs + try: + print(instance.account.pk) + except Exception as e: + pass if type(instance) in services: for order in Order.objects.by_object(instance).active(): order.cancel() @@ -286,6 +301,7 @@ def cancel_orders(sender, **kwargs): def update_orders(sender, **kwargs): if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS: instance = kwargs['instance'] +# print 'save', sender, kwargs if type(instance) in services: Order.update_orders(instance) elif not hasattr(instance, 'account'): diff --git a/orchestra/apps/payments/actions.py b/orchestra/apps/payments/actions.py index 4dfd3afa..d4dce049 100644 --- a/orchestra/apps/payments/actions.py +++ b/orchestra/apps/payments/actions.py @@ -21,7 +21,7 @@ def process_transactions(modeladmin, request, queryset): msg = _("Selected transactions must be on '{state}' state") messages.error(request, msg.format(state=Transaction.WAITTING_PROCESSING)) return - for method, transactions in queryset.group_by('source__method').iteritems(): + for method, transactions in queryset.group_by('source__method').items(): if method is not None: method = PaymentMethod.get(method) procs = method.process(transactions) diff --git a/orchestra/apps/payments/admin.py b/orchestra/apps/payments/admin.py index 83f27029..4496eb85 100644 --- a/orchestra/apps/payments/admin.py +++ b/orchestra/apps/payments/admin.py @@ -39,7 +39,7 @@ class TransactionInline(admin.TabularInline): ) readonly_fields = fields - transaction_link = admin_link('__unicode__', short_description=_("ID")) + transaction_link = admin_link('__str__', short_description=_("ID")) bill_link = admin_link('bill') source_link = admin_link('source') display_state = admin_colored('state', colors=STATE_COLORS) diff --git a/orchestra/apps/payments/methods/sepadirectdebit.py b/orchestra/apps/payments/methods/sepadirectdebit.py index bfd8979e..3d46c71b 100644 --- a/orchestra/apps/payments/methods/sepadirectdebit.py +++ b/orchestra/apps/payments/methods/sepadirectdebit.py @@ -3,7 +3,7 @@ import lxml.builder import os from lxml import etree from lxml.builder import E -from StringIO import StringIO +from io import StringIO from django import forms from django.utils import timezone diff --git a/orchestra/apps/payments/models.py b/orchestra/apps/payments/models.py index ecf1afff..e803e147 100644 --- a/orchestra/apps/payments/models.py +++ b/orchestra/apps/payments/models.py @@ -26,7 +26,7 @@ class PaymentSource(models.Model): objects = PaymentSourcesQueryset.as_manager() - def __unicode__(self): + def __str__(self): return "%s (%s)" % (self.label, self.method_class.verbose_name) @cached_property @@ -76,7 +76,7 @@ class TransactionQuerySet(models.QuerySet): return self.exclude(state=Transaction.REJECTED) def amount(self): - return self.aggregate(models.Sum('amount')).values()[0] + return next(iter(self.aggregate(models.Sum('amount')).values())) def processing(self): return self.filter(state__in=[Transaction.EXECUTED, Transaction.WAITTING_EXECUTION]) @@ -111,7 +111,7 @@ class Transaction(models.Model): objects = TransactionQuerySet.as_manager() - def __unicode__(self): + def __str__(self): return "Transaction #{}".format(self.id) @property @@ -173,7 +173,7 @@ class TransactionProcess(models.Model): class Meta: verbose_name_plural = _("Transaction processes") - def __unicode__(self): + def __str__(self): return '#%i' % self.id def check_state(*args): diff --git a/orchestra/apps/plans/models.py b/orchestra/apps/plans/models.py index 3f1ca958..1db219d3 100644 --- a/orchestra/apps/plans/models.py +++ b/orchestra/apps/plans/models.py @@ -23,7 +23,7 @@ class Plan(models.Model): allow_multiple = models.BooleanField(_("allow multiple"), default=False, help_text=_("Designates whether this plan allow for multiple contractions.")) - def __unicode__(self): + def __str__(self): return self.get_verbose_name() def clean(self): @@ -41,7 +41,7 @@ class ContractedPlan(models.Model): class Meta: verbose_name_plural = _("plans") - def __unicode__(self): + def __str__(self): return str(self.plan) def clean(self): @@ -80,7 +80,7 @@ class Rate(models.Model): class Meta: unique_together = ('service', 'plan', 'quantity') - def __unicode__(self): + def __str__(self): return "{}-{}".format(str(self.price), self.quantity) @classmethod @@ -90,7 +90,7 @@ class Rate(models.Model): @classmethod def get_choices(cls): choices = [] - for name, method in cls.RATE_METHODS.iteritems(): + for name, method in cls.RATE_METHODS.items(): choices.append((name, method.verbose_name)) return choices diff --git a/orchestra/apps/plans/rating.py b/orchestra/apps/plans/rating.py index 2ba7db04..30e7e6a2 100644 --- a/orchestra/apps/plans/rating.py +++ b/orchestra/apps/plans/rating.py @@ -67,8 +67,8 @@ def _prepend_missing(rates): def step_price(rates, metric): # Step price group = [] - minimal = (sys.maxint, []) - for plan, rates in rates.group_by('plan').iteritems(): + minimal = (sys.maxsize, []) + for plan, rates in rates.group_by('plan').items(): rates = _prepend_missing(rates) value, steps = _compute(rates, metric) if plan.is_combinable: diff --git a/orchestra/apps/resources/actions.py b/orchestra/apps/resources/actions.py index bf1b2735..1cf3d0aa 100644 --- a/orchestra/apps/resources/actions.py +++ b/orchestra/apps/resources/actions.py @@ -9,7 +9,7 @@ from django.utils.translation import ungettext, ugettext_lazy as _ def run_monitor(modeladmin, request, queryset): """ Resource and ResourceData run monitors """ referer = request.META.get('HTTP_REFERER') - async = modeladmin.model.monitor.func_defaults[0] + async = modeladmin.model.monitor.__defaults__[0] logs = set() for resource in queryset: results = resource.monitor() diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index e1bd6793..daa76c3a 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -262,7 +262,7 @@ def insert_resource_inlines(): if inline.__name__ == 'ResourceInline': modeladmin_class.inlines.remove(inline) resources = Resource.objects.filter(is_active=True) - for ct, resources in resources.group_by('content_type').iteritems(): + for ct, resources in resources.group_by('content_type').items(): inline = resource_inline_factory(resources) model = ct.model_class() insertattr(model, 'inlines', inline) diff --git a/orchestra/apps/resources/methods.py b/orchestra/apps/resources/methods.py index e7fc7070..4d6cf009 100644 --- a/orchestra/apps/resources/methods.py +++ b/orchestra/apps/resources/methods.py @@ -7,10 +7,8 @@ from django.utils.translation import ugettext_lazy as _ from orchestra import plugins -class DataMethod(plugins.Plugin): +class DataMethod(plugins.Plugin, metaclass=plugins.PluginMount): """ filters and computes dataset usage """ - __metaclass__ = plugins.PluginMount - def filter(self, dataset): """ Filter the dataset to get the relevant data according to the period """ raise NotImplementedError diff --git a/orchestra/apps/resources/migrations/0001_initial.py b/orchestra/apps/resources/migrations/0001_initial.py index 6c0451a6..33ca30e5 100644 --- a/orchestra/apps/resources/migrations/0001_initial.py +++ b/orchestra/apps/resources/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import orchestra.core.validators diff --git a/orchestra/apps/resources/migrations/0002_auto_20141117_1415.py b/orchestra/apps/resources/migrations/0002_auto_20141117_1415.py index 4168ee24..4a056065 100644 --- a/orchestra/apps/resources/migrations/0002_auto_20141117_1415.py +++ b/orchestra/apps/resources/migrations/0002_auto_20141117_1415.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index 64d75eb4..a00177ac 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -83,7 +83,7 @@ class Resource(models.Model): ('verbose_name', 'content_type') ) - def __unicode__(self): + def __str__(self): return "{}-{}".format(str(self.content_type), self.name) @cached_property @@ -188,7 +188,7 @@ class ResourceData(models.Model): unique_together = ('resource', 'content_type', 'object_id') verbose_name_plural = _("resource data") - def __unicode__(self): + def __str__(self): return "%s: %s" % (str(self.resource), str(self.content_object)) @classmethod @@ -278,7 +278,7 @@ class MonitorData(models.Model): get_latest_by = 'id' verbose_name_plural = _("monitor data") - def __unicode__(self): + def __str__(self): return str(self.monitor) @cached_property @@ -331,7 +331,7 @@ def create_resource_relation(): field for field in related._meta.virtual_fields if field.rel.to != ResourceData ] - for ct, resources in Resource.objects.group_by('content_type').iteritems(): + for ct, resources in Resource.objects.group_by('content_type').items(): model = ct.model_class() relation = GenericRelation('resources.ResourceData') model.add_to_class('resource_set', relation) diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 4dc2ee68..07e4ff94 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -41,7 +41,7 @@ def insert_resource_serializers(): pass viewset.serializer_class.Meta.fields = fields # Create nested serializers on target models - for ct, resources in Resource.objects.group_by('content_type').iteritems(): + for ct, resources in Resource.objects.group_by('content_type').items(): model = ct.model_class() try: router.insert(model, 'resources', ResourceSerializer, required=False, many=True, source='resource_set') diff --git a/orchestra/apps/saas/admin.py b/orchestra/apps/saas/admin.py index d8670899..c125f730 100644 --- a/orchestra/apps/saas/admin.py +++ b/orchestra/apps/saas/admin.py @@ -2,7 +2,7 @@ from django.contrib import admin from django.core.urlresolvers import reverse from django.utils.translation import ugettext, ugettext_lazy as _ -from orchestra.admin import ExtendedModelAdmin +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.plugins.admin import SelectPluginAdminMixin @@ -10,7 +10,7 @@ from .models import SaaS from .services import SoftwareService -class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): +class SaaSAdmin(SelectPluginAdminMixin, ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin): list_display = ('name', 'service', 'display_site_domain', 'account_link', 'is_active') list_filter = ('service', 'is_active') change_readonly_fields = ('service',) diff --git a/orchestra/apps/saas/backends/bscw.py b/orchestra/apps/saas/backends/bscw.py index 85c3d63c..137a6e9c 100644 --- a/orchestra/apps/saas/backends/bscw.py +++ b/orchestra/apps/saas/backends/bscw.py @@ -1,3 +1,5 @@ +import textwrap + from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController @@ -26,11 +28,11 @@ class BSCWBackend(ServiceController): if hasattr(saas, 'password'): self.append(textwrap.dedent("""\ if [[ ! $(%(bsadmin)s register %(email)s) && ! $(%(bsadmin)s users -n %(username)s) ]]; then - # Change password - %(bsadmin)s chpwd %(username)s '%(password)s' - else # Create new user %(bsadmin)s register -r %(email)s %(username)s '%(password)s' + else + # Change password + %(bsadmin)s chpwd %(username)s '%(password)s' fi """) % context ) diff --git a/orchestra/apps/saas/backends/gitlab.py b/orchestra/apps/saas/backends/gitlab.py index 2925504a..70ab5d91 100644 --- a/orchestra/apps/saas/backends/gitlab.py +++ b/orchestra/apps/saas/backends/gitlab.py @@ -84,7 +84,7 @@ class GitLabSaaSBackend(ServiceController): user_url = self.get_user_url(saas) response = requests.delete(user_url, headers=self.headers) user = self.validate_response(response, 200, 404) - print json.dumps(user, indent=4) + print(json.dumps(user, indent=4)) def _validate_creation(self, saas, server): """ checks if a saas object is valid for creation on the server side """ @@ -95,9 +95,9 @@ class GitLabSaaSBackend(ServiceController): users = json.loads(requests.get(users_url, headers=self.headers).content) for user in users: if user['username'] == username: - print 'ValidationError: user-exists' + print('ValidationError: user-exists') if user['email'] == email: - print 'ValidationError: email-exists' + print('ValidationError: email-exists') def validate_creation(self, saas): self.append(self._validate_creation, saas) diff --git a/orchestra/apps/saas/backends/phplist.py b/orchestra/apps/saas/backends/phplist.py index 2b072eff..71b233cc 100644 --- a/orchestra/apps/saas/backends/phplist.py +++ b/orchestra/apps/saas/backends/phplist.py @@ -34,7 +34,7 @@ class PhpListSaaSBackend(ServiceController): 'adminpassword': saas.password, } response = requests.post(install_link, data=post) - print response.content + print(response.content) if response.status_code != 200: raise RuntimeError("Bad status code %i" % response.status_code) else: diff --git a/orchestra/apps/saas/models.py b/orchestra/apps/saas/models.py index 7366865f..85f72f1f 100644 --- a/orchestra/apps/saas/models.py +++ b/orchestra/apps/saas/models.py @@ -36,7 +36,7 @@ class SaaS(models.Model): ('name', 'service'), ) - def __unicode__(self): + def __str__(self): return "%s@%s" % (self.name, self.service) @cached_property diff --git a/orchestra/apps/saas/services/bscw.py b/orchestra/apps/saas/services/bscw.py index 5215d528..6ced9863 100644 --- a/orchestra/apps/saas/services/bscw.py +++ b/orchestra/apps/saas/services/bscw.py @@ -7,18 +7,12 @@ from .. import settings from .options import SoftwareService, SoftwareServiceForm -# TODO monitor quota since out of sync? - class BSCWForm(SoftwareServiceForm): email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) - quota = forms.IntegerField(label=_("Quota"), initial=settings.SAAS_BSCW_DEFAULT_QUOTA, - help_text=_("Disk quota in MB.")) class BSCWDataSerializer(serializers.Serializer): email = serializers.EmailField(label=_("Email")) - quota = serializers.IntegerField(label=_("Quota"), default=settings.SAAS_BSCW_DEFAULT_QUOTA, - help_text=_("Disk quota in MB.")) class BSCWService(SoftwareService): diff --git a/orchestra/apps/saas/services/gitlab.py b/orchestra/apps/saas/services/gitlab.py index a3bf6a54..2e124291 100644 --- a/orchestra/apps/saas/services/gitlab.py +++ b/orchestra/apps/saas/services/gitlab.py @@ -3,7 +3,6 @@ from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from orchestra.apps.orchestration.models import BackendOperation as Operation from orchestra.forms import widgets from .options import SoftwareService, SoftwareServiceForm @@ -35,16 +34,4 @@ class GitLabService(SoftwareService): change_readonly_fileds = ('email', 'user_id',) verbose_name = "GitLab" icon = 'orchestra/icons/apps/gitlab.png' - - def clean_data(self): - data = super(GitLabService, self).clean_data() - if not self.instance.pk: - log = Operation.execute_action(self.instance, 'validate_creation')[0] - errors = {} - if 'user-exists' in log.stdout: - errors['name'] = _("User with this username already exists.") - elif 'email-exists' in log.stdout: - errors['email'] = _("User with this email address already exists.") - if errors: - raise ValidationError(errors) - return data + diff --git a/orchestra/apps/saas/services/options.py b/orchestra/apps/saas/services/options.py index 57dc8fa7..a355c54d 100644 --- a/orchestra/apps/saas/services/options.py +++ b/orchestra/apps/saas/services/options.py @@ -4,9 +4,10 @@ from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from orchestra import plugins -from orchestra.plugins.forms import PluginDataForm +from orchestra.apps.orchestration.models import BackendOperation as Operation from orchestra.core import validators from orchestra.forms import widgets +from orchestra.plugins.forms import PluginDataForm from orchestra.utils.functional import cached from orchestra.utils.python import import_class, random_ascii @@ -91,6 +92,23 @@ class SoftwareService(plugins.Plugin): (self.instance.name, self.site_base_domain) ) + def clean_data(self): + data = super(SoftwareService, self).clean_data() + if not self.instance.pk: + try: + log = Operation.execute_action(self.instance, 'validate_creation')[0] + except IndexError: + pass + else: + errors = {} + if 'user-exists' in log.stdout: + errors['name'] = _("User with this username already exists.") + elif 'email-exists' in log.stdout: + errors['email'] = _("User with this email address already exists.") + if errors: + raise ValidationError(errors) + return data + def save(self): pass diff --git a/orchestra/apps/services/handlers.py b/orchestra/apps/services/handlers.py index 0ef71930..a329e087 100644 --- a/orchestra/apps/services/handlers.py +++ b/orchestra/apps/services/handlers.py @@ -1,6 +1,7 @@ import calendar import datetime import decimal +import math from dateutil import relativedelta from django.contrib.contenttypes.models import ContentType @@ -15,7 +16,7 @@ from orchestra.utils.python import AttrDict from . import settings, helpers -class ServiceHandler(plugins.Plugin): +class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): """ Separates all the logic of billing handling from the model allowing to better customize its behaviout @@ -27,8 +28,6 @@ class ServiceHandler(plugins.Plugin): model = None - __metaclass__ = plugins.PluginMount - def __init__(self, service): self.service = service @@ -54,8 +53,7 @@ class ServiceHandler(plugins.Plugin): bool(self.matches(obj)) except Exception as exception: name = type(exception).__name__ - message = exception.message - raise ValidationError(': '.join((name, message))) + raise ValidationError(': '.join((name, exception))) def validate_metric(self, service): try: @@ -66,8 +64,7 @@ class ServiceHandler(plugins.Plugin): bool(self.get_metric(obj)) except Exception as exception: name = type(exception).__name__ - message = exception.message - raise ValidationError(': '.join((name, message))) + raise ValidationError(': '.join((name, exception))) def get_content_type(self): if not self.model: @@ -106,18 +103,26 @@ class ServiceHandler(plugins.Plugin): return order.ignore def get_ignore(self, instance): - ignore = False - account = getattr(instance, 'account', instance) - if account.is_superuser: - ignore = self.ignore_superusers - return ignore + if self.ignore_superusers: + account = getattr(instance, 'account', instance) + if (account.type in settings.SERVICES_IGNORE_ACCOUNT_TYPE or + 'superuser' in settings.SERVICES_IGNORE_ACCOUNT_TYPE): + return True + return False def get_metric(self, instance): if self.metric: safe_locals = { - instance._meta.model_name: instance + instance._meta.model_name: instance, + 'instance': instance, + 'math': math, + 'log10': math.log10, + 'Decimal': decimal.Decimal, } - return eval(self.metric, safe_locals) + try: + return eval(self.metric, safe_locals) + except Exception as error: + raise type(error)("%s on '%s'" %(error, self.service)) def get_order_description(self, instance): safe_locals = { @@ -126,7 +131,7 @@ class ServiceHandler(plugins.Plugin): instance._meta.model_name: instance, } if not self.order_description: - return u'%s: %s' % (self.description, instance) + return '%s: %s' % (self.description, instance) return eval(self.order_description, safe_locals) def get_billing_point(self, order, bp=None, **options): @@ -359,7 +364,7 @@ class ServiceHandler(plugins.Plugin): else: priced[order] = (price, cprice) lines = [] - for order, prices in priced.iteritems(): + for order, prices in priced.items(): discounts = () # Generate lines and discounts from order.nominal_price price, cprice = prices diff --git a/orchestra/apps/services/models.py b/orchestra/apps/services/models.py index 66bb013b..ade905aa 100644 --- a/orchestra/apps/services/models.py +++ b/orchestra/apps/services/models.py @@ -24,6 +24,7 @@ autodiscover_modules('handlers') rate_class = import_class(settings.SERVICES_RATE_CLASS) + class Service(models.Model): NEVER = '' # DAILY = 'DAILY' @@ -46,6 +47,8 @@ class Service(models.Model): PREPAY = 'PREPAY' POSTPAY = 'POSTPAY' + _ignore_types = ' and '.join(', '.join(settings.SERVICES_IGNORE_ACCOUNT_TYPE).rsplit(', ', 1)).lower() + description = models.CharField(_("description"), max_length=256, unique=True) content_type = models.ForeignKey(ContentType, verbose_name=_("content type"), help_text=_("Content type of the related service objects.")) @@ -66,8 +69,8 @@ class Service(models.Model): "here allow to."), choices=ServiceHandler.get_choices()) is_active = models.BooleanField(_("active"), default=True) - ignore_superusers = models.BooleanField(_("ignore superusers"), default=True, - help_text=_("Designates whether superuser orders are marked as ignored by default or not.")) + ignore_superusers = models.BooleanField(_("ignore %s") % _ignore_types, default=True, + help_text=_("Designates whether %s orders are marked as ignored by default or not.") % _ignore_types) # Billing billing_period = models.CharField(_("billing period"), max_length=16, help_text=_("Renewal period for recurring invoicing."), @@ -133,7 +136,7 @@ class Service(models.Model): rate_algorithm = models.CharField(_("rate algorithm"), max_length=16, help_text=string_concat(_("Algorithm used to interprete the rating table."), *[ string_concat('
  ', method.verbose_name, ': ', method.help_text) - for name, method in rate_class.get_methods().iteritems() + for name, method in rate_class.get_methods().items() ]), choices=rate_class.get_choices(), default=rate_class.get_choices()[0][0]) on_cancel = models.CharField(_("on cancel"), max_length=16, help_text=_("Defines the cancellation behaviour of this service."), @@ -153,7 +156,7 @@ class Service(models.Model): ), default=PREPAY) - def __unicode__(self): + def __str__(self): return self.description @classmethod diff --git a/orchestra/apps/services/settings.py b/orchestra/apps/services/settings.py index 2acde264..db6dc00e 100644 --- a/orchestra/apps/services/settings.py +++ b/orchestra/apps/services/settings.py @@ -33,3 +33,10 @@ SERVICES_RATE_CLASS = getattr(settings, 'SERVICES_RATE_CLASS', SERVICES_DEFAULT_IGNORE_PERIOD = getattr(settings, 'SERVICES_DEFAULT_IGNORE_PERIOD', 'TEN_DAYS' ) + + +SERVICES_IGNORE_ACCOUNT_TYPE = getattr(settings, 'SERVICES_IGNORE_ACCOUNT_TYPE', ( + 'superuser', + 'STAFF', + 'FRIEND', +)) diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index 12214b71..ba2f7aa7 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -222,7 +222,7 @@ class Exim4Traffic(ServiceMonitor): except IOError as e: sys.stderr.write(e) - for username, opts in users.iteritems(): + for username, opts in users.items(): __, object_id, size = opts print object_id, size """).format(**context) @@ -317,7 +317,7 @@ class FTPTraffic(ServiceMonitor): except IOError as e: sys.stderr.write(e) - for username, opts in users.iteritems(): + for username, opts in users.items(): __, object_id, size = opts print object_id, size """).format(**context) diff --git a/orchestra/apps/systemusers/migrations/0001_initial.py b/orchestra/apps/systemusers/migrations/0001_initial.py index 54626e32..7bb97b0a 100644 --- a/orchestra/apps/systemusers/migrations/0001_initial.py +++ b/orchestra/apps/systemusers/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/orchestra/apps/systemusers/migrations/0002_systemuser_relative_to_main.py b/orchestra/apps/systemusers/migrations/0002_systemuser_relative_to_main.py index 6bb47008..1cbba0f7 100644 --- a/orchestra/apps/systemusers/migrations/0002_systemuser_relative_to_main.py +++ b/orchestra/apps/systemusers/migrations/0002_systemuser_relative_to_main.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/orchestra/apps/systemusers/migrations/0003_auto_20141114_1340.py b/orchestra/apps/systemusers/migrations/0003_auto_20141114_1340.py index a3a38e57..bfb7f7fa 100644 --- a/orchestra/apps/systemusers/migrations/0003_auto_20141114_1340.py +++ b/orchestra/apps/systemusers/migrations/0003_auto_20141114_1340.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/orchestra/apps/systemusers/models.py b/orchestra/apps/systemusers/models.py index df2d905b..7f39b09e 100644 --- a/orchestra/apps/systemusers/models.py +++ b/orchestra/apps/systemusers/models.py @@ -48,7 +48,7 @@ class SystemUser(models.Model): objects = SystemUserQuerySet.as_manager() - def __unicode__(self): + def __str__(self): return self.username @cached_property diff --git a/orchestra/apps/vps/models.py b/orchestra/apps/vps/models.py index 476acad9..f0777c9c 100644 --- a/orchestra/apps/vps/models.py +++ b/orchestra/apps/vps/models.py @@ -24,7 +24,7 @@ class VPS(models.Model): verbose_name = "VPS" verbose_name_plural = "VPSs" - def __unicode__(self): + def __str__(self): return self.hostname def set_password(self, raw_password): diff --git a/orchestra/apps/webapps/admin.py b/orchestra/apps/webapps/admin.py index fd4e3d14..e8f6c916 100644 --- a/orchestra/apps/webapps/admin.py +++ b/orchestra/apps/webapps/admin.py @@ -1,6 +1,7 @@ from django import forms from django.contrib import admin from django.core.urlresolvers import reverse +from django.utils.encoding import force_text from django.utils.translation import ugettext, ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin @@ -20,7 +21,7 @@ class WebAppOptionInline(admin.TabularInline): extra = 1 OPTIONS_HELP_TEXT = { - op.name: str(unicode(op.help_text)) for op in AppOption.get_plugins() + op.name: force_text(op.help_text) for op in AppOption.get_plugins() } class Media: diff --git a/orchestra/apps/webapps/backends/php.py b/orchestra/apps/webapps/backends/php.py index 5f181cf4..bc509fd5 100644 --- a/orchestra/apps/webapps/backends/php.py +++ b/orchestra/apps/webapps/backends/php.py @@ -118,6 +118,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): listen.owner = {{ user }} listen.group = {{ group }} pm = ondemand + pm.max_requests = {{ max_requests }} {% if max_children %}pm.max_children = {{ max_children }}{% endif %} {% if request_terminate_timeout %}request_terminate_timeout = {{ request_terminate_timeout }}{% endif %} {% for name, value in init_vars.iteritems %} @@ -131,7 +132,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): # Format PHP init vars init_vars = opt.get_php_init_vars(merge=self.MERGE) if init_vars: - init_vars = [ '-d %s="%s"' % (k,v) for k,v in init_vars.iteritems() ] + init_vars = [ '-d %s="%s"' % (k,v) for k,v in init_vars.items() ] init_vars = ', '.join(init_vars) context.update({ 'php_binary': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context), @@ -144,6 +145,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): # %(banner)s export PHPRC=%(php_rc)s export PHP_INI_SCAN_DIR=%(php_ini_scan)s + export PHP_FCGI_MAX_REQUESTS=%(max_requests)s exec %(php_binary)s %(php_init_vars)s""") % context def get_fcgid_cmd_options(self, webapp, context): @@ -152,7 +154,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): 'IOTimeout': webapp.get_options().get('timeout', None), } cmd_options = [] - for directive, value in maps.iteritems(): + for directive, value in maps.items(): if value: cmd_options.append("%s %s" % (directive, value)) if cmd_options: @@ -187,6 +189,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): context.update({ 'php_version': webapp.type_instance.get_php_version(), 'php_version_number': webapp.type_instance.get_php_version_number(), + 'max_requests': settings.WEBAPPS_PHP_MAX_REQUESTS, }) self.update_fcgid_context(webapp, context) self.update_fpm_context(webapp, context) diff --git a/orchestra/apps/webapps/migrations/0001_initial.py b/orchestra/apps/webapps/migrations/0001_initial.py index c6c501b0..5b0a2aab 100644 --- a/orchestra/apps/webapps/migrations/0001_initial.py +++ b/orchestra/apps/webapps/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import orchestra.core.validators diff --git a/orchestra/apps/webapps/migrations/0002_webapp_data.py b/orchestra/apps/webapps/migrations/0002_webapp_data.py index 868b478d..2158f75f 100644 --- a/orchestra/apps/webapps/migrations/0002_webapp_data.py +++ b/orchestra/apps/webapps/migrations/0002_webapp_data.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import jsonfield.fields diff --git a/orchestra/apps/webapps/migrations/0003_auto_20150310_2103.py b/orchestra/apps/webapps/migrations/0003_auto_20150310_2103.py index 348d3793..31101602 100644 --- a/orchestra/apps/webapps/migrations/0003_auto_20150310_2103.py +++ b/orchestra/apps/webapps/migrations/0003_auto_20150310_2103.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/orchestra/apps/webapps/models.py b/orchestra/apps/webapps/models.py index c6cbfbe8..24ff6b8c 100644 --- a/orchestra/apps/webapps/models.py +++ b/orchestra/apps/webapps/models.py @@ -1,3 +1,4 @@ + import os import re @@ -37,7 +38,7 @@ class WebApp(models.Model): verbose_name = _("Web App") verbose_name_plural = _("Web Apps") - def __unicode__(self): + def __str__(self): return self.name def get_description(self): @@ -98,7 +99,7 @@ class WebAppOption(models.Model): verbose_name = _("option") verbose_name_plural = _("options") - def __unicode__(self): + def __str__(self): return self.name @cached_property diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py index ba01d275..90616b06 100644 --- a/orchestra/apps/webapps/settings.py +++ b/orchestra/apps/webapps/settings.py @@ -30,6 +30,13 @@ WEBAPPS_FCGID_CMD_OPTIONS_PATH = getattr(settings, 'WEBAPPS_FCGID_CMD_OPTIONS_PA ) +# Greater or equal to your FcgidMaxRequestsPerProcess +# http://httpd.apache.org/mod_fcgid/mod/mod_fcgid.html#examples +WEBAPPS_PHP_MAX_REQUESTS = getattr(settings, 'WEBAPPS_PHP_MAX_REQUESTS', + 400 +) + + WEBAPPS_PHP_ERROR_LOG_PATH = getattr(settings, 'WEBAPPS_PHP_ERROR_LOG_PATH', '' ) @@ -92,7 +99,7 @@ WEBAPPS_UNDER_CONSTRUCTION_PATH = getattr(settings, 'WEBAPPS_UNDER_CONSTRUCTION_ #WEBAPPS_TYPES_OVERRIDE = getattr(settings, 'WEBAPPS_TYPES_OVERRIDE', {}) -#for webapp_type, value in WEBAPPS_TYPES_OVERRIDE.iteritems(): +#for webapp_type, value in WEBAPPS_TYPES_OVERRIDE.items(): # if value is None: # WEBAPPS_TYPES.pop(webapp_type, None) # else: diff --git a/orchestra/apps/webapps/tests/functional_tests/tests.py b/orchestra/apps/webapps/tests/functional_tests/tests.py index 27466aba..ab64d74e 100644 --- a/orchestra/apps/webapps/tests/functional_tests/tests.py +++ b/orchestra/apps/webapps/tests/functional_tests/tests.py @@ -2,7 +2,7 @@ import ftplib import os import time import textwrap -from StringIO import StringIO +from io import StringIO from django.conf import settings as djsettings from django.contrib.contenttypes.models import ContentType diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index 2bd11b59..d7526c7a 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -61,7 +61,7 @@ class PHPApp(AppType): @cached def get_php_options(self): - php_version = self.get_php_version() + php_version = self.get_php_version_number() php_options = AppOption.get_option_groups()[AppOption.PHP] return [op for op in php_options if getattr(self, 'deprecated', 999) > php_version] @@ -93,6 +93,9 @@ class PHPApp(AppType): if function not in enabled_functions: disabled_functions.append(function) init_vars['dissabled_functions'] = ','.join(disabled_functions) + timeout = self.instance.options.filter(name='timeout').first() + if timeout: + init_vars['max_execution_time'] = timeout.value if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars: context = self.get_directive_context() error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context) @@ -128,4 +131,4 @@ class PHPApp(AppType): raise ValueError("No version number matches for '%s'" % php_version) if len(number) > 1: raise ValueError("Multiple version number matches for '%s'" % php_version) - return number[0] + return float(number[0]) diff --git a/orchestra/apps/websites/admin.py b/orchestra/apps/websites/admin.py index 98906dbc..5a2a28db 100644 --- a/orchestra/apps/websites/admin.py +++ b/orchestra/apps/websites/admin.py @@ -2,6 +2,7 @@ from django import forms from django.contrib import admin from django.core.urlresolvers import resolve from django.db.models import Q +from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -22,7 +23,7 @@ class WebsiteDirectiveInline(admin.TabularInline): extra = 1 DIRECTIVES_HELP_TEXT = { - op.name: str(unicode(op.help_text)) for op in SiteDirective.get_plugins() + op.name: force_text(op.help_text) for op in SiteDirective.get_plugins() } def formfield_for_dbfield(self, db_field, **kwargs): diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 918480aa..a5f42951 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -219,7 +219,7 @@ class Apache2Backend(ServiceController): def get_saas(self, directives): saas = [] - for name, values in directives.iteritems(): + for name, values in directives.items(): if name.endswith('-saas'): for value in values: context = { diff --git a/orchestra/apps/websites/directives.py b/orchestra/apps/websites/directives.py index c2f1777e..92af8a3b 100644 --- a/orchestra/apps/websites/directives.py +++ b/orchestra/apps/websites/directives.py @@ -47,7 +47,7 @@ class SiteDirective(Plugin): options = cls.get_option_groups() for option in options.pop(None, ()): yield (option.name, option.verbose_name) - for group, options in options.iteritems(): + for group, options in options.items(): yield (group, [(op.name, op.verbose_name) for op in options]) def validate(self, website): diff --git a/orchestra/apps/websites/forms.py b/orchestra/apps/websites/forms.py index ee0599c8..b557d664 100644 --- a/orchestra/apps/websites/forms.py +++ b/orchestra/apps/websites/forms.py @@ -1,5 +1,6 @@ from django import forms from django.core.exceptions import ValidationError +from django.utils.encoding import force_text from .validators import validate_domain_protocol @@ -36,7 +37,7 @@ class WebsiteDirectiveInlineFormSet(forms.models.BaseInlineFormSet): if value is not None: if directive.unique_value and value in values.get(name, []): form.add_error('value', ValidationError( - _("This value is already used by other %s.") % unicode(directive.get_verbose_name()) + _("This value is already used by other %s.") % force_text(directive.get_verbose_name()) )) try: values[name].append(value) diff --git a/orchestra/apps/websites/models.py b/orchestra/apps/websites/models.py index 8c33caf7..33089c8c 100644 --- a/orchestra/apps/websites/models.py +++ b/orchestra/apps/websites/models.py @@ -40,7 +40,7 @@ class Website(models.Model): class Meta: unique_together = ('name', 'account') - def __unicode__(self): + def __str__(self): return self.name @property @@ -107,7 +107,7 @@ class WebsiteDirective(models.Model): choices=SiteDirective.get_choices()) value = models.CharField(_("value"), max_length=256) - def __unicode__(self): + def __str__(self): return self.name @cached_property @@ -133,7 +133,7 @@ class Content(models.Model): class Meta: unique_together = ('website', 'path') - def __unicode__(self): + def __str__(self): try: return self.website.name + self.path except Website.DoesNotExist: diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index ad35a522..a86531ad 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -136,10 +136,10 @@ function install_requirements () { PIP="django==1.7.7 \ django-celery-email==1.0.4 \ - django-fluent-dashboard==0.3.5 \ + django-fluent-dashboard==0.4 \ https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip \ IPy==0.81 \ - django-extensions==1.1.1 \ + django-extensions==1.5.2 \ django-transaction-signals==1.0.0 \ django-celery==3.1.16 \ celery==3.1.16 \ @@ -209,10 +209,10 @@ function install_requirements () { # Patch passlib IMPORT="from django.contrib.auth.hashers import mask_hash, _" COLLECTIONS="from collections import OrderedDict" - sed -i "s/${IMPORT}, SortedDict/${IMPORT}\n ${COLLECTIONS}/" \ - /usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py - sed -i "s/SortedDict/OrderedDict/g" \ - /usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py + ls /usr/local/lib/python*/dist-packages/passlib/ext/django/utils.py \ + | xargs sed -i "s/${IMPORT}, SortedDict/${IMPORT}\n ${COLLECTIONS}/" + ls /usr/local/lib/python*/dist-packages/passlib/ext/django/utils.py \ + | xargs sed -i "s/SortedDict/OrderedDict/g" # Patch dateutil sed -i "s/elif not isinstance(dt2, datetime.datetime):/else:/" \ diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 5bf1cb87..4a411de8 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -48,8 +48,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'orchestra.core.caches.RequestCacheMiddleware', - # ATOMIC REQUESTS do not wrap middlewares - 'orchestra.core.middlewares.TransactionMiddleware', + # also handles transations, ATOMIC REQUESTS does not wrap middlewares 'orchestra.apps.orchestration.middlewares.OperationsMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', diff --git a/orchestra/core/middlewares.py b/orchestra/core/middlewares.py index fb6823ff..7e96a46a 100644 --- a/orchestra/core/middlewares.py +++ b/orchestra/core/middlewares.py @@ -8,40 +8,40 @@ class TransactionMiddleware(object): commit, the commit is done when a successful response is created. If an exception happens, the database is rolled back. """ - - def process_request(self, request): - """Enters transaction management""" - transaction.enter_transaction_management() - - def process_exception(self, request, exception): - """Rolls back the database and leaves transaction management""" - if transaction.is_dirty(): - # This rollback might fail because of network failure for example. - # If rollback isn't possible it is impossible to clean the - # connection's state. So leave the connection in dirty state and - # let request_finished signal deal with cleaning the connection. - transaction.rollback() - transaction.leave_transaction_management() - - def process_response(self, request, response): - """Commits and leaves transaction management.""" - if not transaction.get_autocommit(): - if transaction.is_dirty(): - # Note: it is possible that the commit fails. If the reason is - # closed connection or some similar reason, then there is - # little hope to proceed nicely. However, in some cases ( - # deferred foreign key checks for exampl) it is still possible - # to rollback(). - try: - transaction.commit() - except Exception: - # If the rollback fails, the transaction state will be - # messed up. It doesn't matter, the connection will be set - # to clean state after the request finishes. And, we can't - # clean the state here properly even if we wanted to, the - # connection is in transaction but we can't rollback... - transaction.rollback() - transaction.leave_transaction_management() - raise - transaction.leave_transaction_management() - return response + pass +# def process_request(self, request): +# """Enters transaction management""" +# transaction.enter_transaction_management() +# +# def process_exception(self, request, exception): +# """Rolls back the database and leaves transaction management""" +# if transaction.is_dirty(): +# # This rollback might fail because of network failure for example. +# # If rollback isn't possible it is impossible to clean the +# # connection's state. So leave the connection in dirty state and +# # let request_finished signal deal with cleaning the connection. +# transaction.rollback() +# transaction.leave_transaction_management() +# +# def process_response(self, request, response): +# """Commits and leaves transaction management.""" +# if not transaction.get_autocommit(): +# if transaction.is_dirty(): +# # Note: it is possible that the commit fails. If the reason is +# # closed connection or some similar reason, then there is +# # little hope to proceed nicely. However, in some cases ( +# # deferred foreign key checks for exampl) it is still possible +# # to rollback(). +# try: +# transaction.commit() +# except Exception: +# # If the rollback fails, the transaction state will be +# # messed up. It doesn't matter, the connection will be set +# # to clean state after the request finishes. And, we can't +# # clean the state here properly even if we wanted to, the +# # connection is in transaction but we can't rollback... +# transaction.rollback() +# transaction.leave_transaction_management() +# raise +# transaction.leave_transaction_management() +# return response diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index 7914a8bf..080bd551 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -15,7 +15,7 @@ from ..utils.python import import_class def all_valid(kwargs): """ helper function to merge multiple validators at once """ errors = {} - for field, validator in kwargs.iteritems(): + for field, validator in kwargs.items(): try: validator[0](*validator[1:]) except ValidationError as error: diff --git a/orchestra/forms/widgets.py b/orchestra/forms/widgets.py index 5ffbe0e1..8b94bee9 100644 --- a/orchestra/forms/widgets.py +++ b/orchestra/forms/widgets.py @@ -19,16 +19,16 @@ class ShowTextWidget(forms.Widget): if hasattr(self, 'initial'): value = self.initial if self.bold: - final_value = u'%s' % (value) + final_value = '%s' % (value) else: final_value = '
'.join(value.split('\n')) if self.warning: final_value = ( - u'' + '' % final_value) if self.hidden: final_value = ( - u'%s' + '%s' % (final_value, name, value)) return mark_safe(final_value) diff --git a/orchestra/management/commands/makemessages.py b/orchestra/management/commands/makemessages.py index 38a06588..88f6dcd6 100644 --- a/orchestra/management/commands/makemessages.py +++ b/orchestra/management/commands/makemessages.py @@ -18,13 +18,13 @@ class Command(makemessages.Command): self.remove_database_files() def get_contents(self): - for model, fields in ModelTranslation._registry.iteritems(): + for model, fields in ModelTranslation._registry.items(): for field in fields: contents = [] for content in model.objects.values_list('id', field): pk, value = content contents.append( - (pk, u"_(u'%s')" % value) + (pk, "_(u'%s')" % value) ) if contents: yield ('_'.join((model._meta.db_table, field)), contents) @@ -38,7 +38,7 @@ class Command(makemessages.Command): 2) Django's makemessages will work with no modifications """ for name, contents in self.get_contents(): - name = unicode(name) + name = str(name) maximum = None content = {} for pk, value in contents: @@ -46,9 +46,9 @@ class Command(makemessages.Command): maximum = pk content[pk] = value tmpcontent = [] - for ix in xrange(maximum+1): + for ix in range(maximum+1): tmpcontent.append(content.get(ix, '')) - tmpcontent = u'\n'.join(tmpcontent) + '\n' + tmpcontent = '\n'.join(tmpcontent) + '\n' filename = 'database_%s.sql.py' % name self.database_files.append(filename) with open(filename, 'w') as tmpfile: diff --git a/orchestra/management/commands/staticcheck.py b/orchestra/management/commands/staticcheck.py index 05c74398..83e4791e 100644 --- a/orchestra/management/commands/staticcheck.py +++ b/orchestra/management/commands/staticcheck.py @@ -96,7 +96,7 @@ class Command(BaseCommand): filenames = [get_orchestra_dir(), '.'] warnings = checkPaths(filenames) for warning in warnings: - print warning + print(warning) if warnings: - print 'Total warnings: %d' % len(warnings) + print('Total warnings: %d' % len(warnings)) raise SystemExit(1) diff --git a/orchestra/models/fields.py b/orchestra/models/fields.py index e3cf40e8..fb1992ec 100644 --- a/orchestra/models/fields.py +++ b/orchestra/models/fields.py @@ -6,9 +6,7 @@ from ..forms.fields import MultiSelectFormField from ..utils.apps import isinstalled -class MultiSelectField(models.CharField): - __metaclass__ = models.SubfieldBase - +class MultiSelectField(models.CharField, metaclass=models.SubfieldBase): def formfield(self, **kwargs): defaults = { 'required': not self.blank, @@ -22,7 +20,7 @@ class MultiSelectField(models.CharField): return MultiSelectFormField(**defaults) def get_db_prep_value(self, value, connection=None, prepared=False): - if isinstance(value, basestring): + if isinstance(value, str): return value elif isinstance(value, list): return ','.join(value) diff --git a/orchestra/permissions/options.py b/orchestra/permissions/options.py index 1838ecd5..42523eee 100644 --- a/orchestra/permissions/options.py +++ b/orchestra/permissions/options.py @@ -38,7 +38,7 @@ class Permission(object): # has_permission.perm(user) for func in inspect.getmembers(type(self), predicate=inspect.ismethod): - if func[1].im_class is not type(self): + if not isinstance(self, func[1].__self__.__class__): # aggregated methods setattr(call, func[0], functools.partial(func[1], obj, cls)) else: @@ -92,14 +92,14 @@ class RelatedPermission(Permission): for relation in relations: parent = getattr(parent, relation).field.rel.to else: - parent = reduce(getattr, relations, obj) + parent = functools.reduce(getattr, relations, obj) # call interface: has_permission(user, 'perm') def call(user, perm): return parent.has_permission(user, perm) # method interface: has_permission.perm(user) - for name, func in parent.has_permission.__dict__.iteritems(): + for name, func in parent.has_permission.__dict__.items(): if not name.startswith('_'): setattr(call, name, func) diff --git a/orchestra/plugins/forms.py b/orchestra/plugins/forms.py index a747dc61..273b6a49 100644 --- a/orchestra/plugins/forms.py +++ b/orchestra/plugins/forms.py @@ -1,4 +1,5 @@ from django import forms +from django.utils.encoding import force_text from orchestra.forms.widgets import ReadOnlyWidget @@ -10,7 +11,7 @@ class PluginDataForm(forms.ModelForm): super(PluginDataForm, self).__init__(*args, **kwargs) if self.plugin_field in self.fields: value = self.plugin.get_name() - display = '%s change' % unicode(self.plugin.verbose_name) + display = '%s change' % force_text(self.plugin.verbose_name) self.fields[self.plugin_field].widget = ReadOnlyWidget(value, display) self.fields[self.plugin_field].help_text = getattr(self.plugin, 'help_text', '') if self.instance: @@ -29,9 +30,17 @@ class PluginDataForm(forms.ModelForm): def clean(self): data = {} - for field, value in self.instance.data.iteritems(): + # Update data fields + for field in self.declared_fields: try: data[field] = self.cleaned_data[field] except KeyError: - data[field] = value + data[field] = self.data[field] + # Keep old data fields + for field, value in self.instance.data.items(): + if field not in data: + try: + data[field] = self.cleaned_data[field] + except KeyError: + data[field] = value self.cleaned_data['data'] = data diff --git a/orchestra/templatetags/markdown.py b/orchestra/templatetags/markdown.py index ad7d6faf..2e5a9d99 100644 --- a/orchestra/templatetags/markdown.py +++ b/orchestra/templatetags/markdown.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from django import template from markdown import markdown diff --git a/orchestra/urls.py b/orchestra/urls.py index 62360a44..455d2d9c 100644 --- a/orchestra/urls.py +++ b/orchestra/urls.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from django.contrib import admin from django.conf import settings from django.conf.urls import patterns, include, url diff --git a/orchestra/utils/humanize.py b/orchestra/utils/humanize.py index da34d075..73505b4a 100644 --- a/orchestra/utils/humanize.py +++ b/orchestra/utils/humanize.py @@ -174,7 +174,7 @@ UNITS_CONVERSIONS = { } def unit_to_bytes(unit): - for bytes, units in UNITS_CONVERSIONS.iteritems(): + for bytes, units in UNITS_CONVERSIONS.items(): if unit in units: return bytes raise KeyError("%s is not a valid unit." % unit) diff --git a/orchestra/utils/options.py b/orchestra/utils/options.py index a6961bd1..48d58f9d 100644 --- a/orchestra/utils/options.py +++ b/orchestra/utils/options.py @@ -1,5 +1,5 @@ import sys -import urlparse +from urllib.parse import urlparse from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string @@ -17,7 +17,7 @@ def send_email_template(template, context, to, email_from=None, html=None, attac if isinstance(context, dict): context = Context(context) - if type(to) in [str, unicode]: + if isinstance(to, str): to = [to] if not 'site' in context: @@ -49,13 +49,13 @@ def database_ready(): def dict_setting_to_choices(choices): return sorted( - [ (name, opt.get('verbose_name', 'name')) for name, opt in choices.iteritems() ], + [ (name, opt.get('verbose_name', 'name')) for name, opt in choices.items() ], key=lambda e: e[0] ) def tuple_setting_to_choices(choices): return sorted( - tuple((name, opt[0]) for name, opt in choices.iteritems()), + tuple((name, opt[0]) for name, opt in choices.items()), key=lambda e: e[0] ) diff --git a/orchestra/utils/python.py b/orchestra/utils/python.py index fa5e37eb..95f18e9d 100644 --- a/orchestra/utils/python.py +++ b/orchestra/utils/python.py @@ -2,7 +2,7 @@ import sys import collections import random import string -from cStringIO import StringIO +from io import StringIO def import_class(cls): diff --git a/orchestra/utils/system.py b/orchestra/utils/system.py index f35bd437..994f8c2d 100644 --- a/orchestra/utils/system.py +++ b/orchestra/utils/system.py @@ -42,7 +42,7 @@ def read_async(fd): if e.errno != errno.EAGAIN: raise e else: - return u'' + return '' def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', force_unicode=True): @@ -53,7 +53,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', p = subprocess.Popen(command, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - p.stdin.write(stdin) + p.stdin.write(bytes(stdin, 'utf-8')) p.stdin.close() yield @@ -61,23 +61,18 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', make_async(p.stderr) # Async reading of stdout and sterr - # TODO cleanup while True: - # TODO https://github.com/isagalaev/ijson/issues/15 - stdout = unicode() if force_unicode else '' - sdterr = unicode() if force_unicode else '' + stdout = '' + sdterr = '' # Get complete unicode chunks - while True: - select.select([p.stdout, p.stderr], [], []) - stdoutPiece = read_async(p.stdout) - stderrPiece = read_async(p.stderr) - try: - stdout += unicode(stdoutPiece.decode("utf8")) if force_unicode else stdoutPiece - sdterr += unicode(stderrPiece.decode("utf8")) if force_unicode else stderrPiece - except UnicodeDecodeError as e: - pass - else: - break + select.select([p.stdout, p.stderr], [], []) + + stdoutPiece = read_async(p.stdout) + stderrPiece = read_async(p.stderr) + + stdout += (stdoutPiece or b'').decode('utf8') + sdterr += (stderrPiece or b'').decode('utf8') + if display and stdout: sys.stdout.write(stdout) if display and stderr: @@ -96,7 +91,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', def run(command, display=False, error_codes=[0], silent=False, stdin='', async=False, force_unicode=True): iterator = runiterator(command, display, error_codes, silent, stdin, force_unicode) - iterator.next() + next(iterator) if async: return iterator diff --git a/scripts/migration/apache2.py b/scripts/migration/apache2.py index 27bfc8b5..e6d24c60 100644 --- a/scripts/migration/apache2.py +++ b/scripts/migration/apache2.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import re import glob diff --git a/setup.py b/setup.py index 10202805..2ca94c1e 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os, sys from distutils.sysconfig import get_python_lib from setuptools import setup, find_packages