Added support for Let's encrypt

TODO.md

@@ -430,3 +430,13 @@ mkhomedir_helper or create ssh homes with bash.rc and such
 # Automatically re-run backends until success? only timedout executions?
 # TODO save serialized versions ob backendoperation.instance in order to allow backend reexecution of deleted objects
 
+# websites active list_display
+# account for account.is_active on service is_active filters like systemusers
+
+# upgrade to django 1.9 and make margins wider
+# lets encrypt: DNS vs HTTP challange
+
+# Warning websites with ssl options without https protocol
+
+
+

orchestra/admin/html.py

@@ -6,7 +6,7 @@ MONOSPACE_FONTS = ('Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans M
 
 
 def monospace_format(text):
-    style="font-family:%s;padding-left:110px;" % MONOSPACE_FONTS
+    style="font-family:%s;padding-left:110px;white-space:pre-wrap;" % MONOSPACE_FONTS
     return mark_safe('<pre style="%s">%s</pre>' % (style, text))
 
 

orchestra/admin/utils.py

@@ -169,7 +169,8 @@ def get_object_from_url(modeladmin, request):
 
 def display_mono(field):
     def display(self, log):
-        return monospace_format(escape(getattr(log, field)))
+        content = getattr(log, field)
+        return monospace_format(escape(content))
     display.short_description = field
     return display
 

orchestra/contrib/domains/admin.py

@@ -1,6 +1,8 @@
 from django.contrib import admin
+from django.core.urlresolvers import reverse
 from django.db.models.functions import Concat, Coalesce
-from django.utils.translation import ugettext_lazy as _
+from django.templatetags.static import static
+from django.utils.translation import ugettext, ugettext_lazy as _
 
 from orchestra.admin import ExtendedModelAdmin
 from orchestra.admin.utils import admin_link, change_url
@@ -91,8 +93,16 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin):
                         admin_url, title, website.name, site_link)
                     links.append(link)
                 return '<br>'.join(links)
+        add_url = reverse('admin:websites_website_add')
+        add_url += '?account=%i&domains=%i' % (domain.account_id, domain.pk)
+        context = {
+            'title': _("Add website"),
+            'url': add_url,
+            'image': '<img src="%s"></img>' % static('orchestra/images/add.png'),
+        }
+        add_link = '<a href="%(url)s" title="%(title)s">%(image)s</a>' % context
         site_link = get_on_site_link('http://%s' % domain.name)
-        return _("No website %s") % site_link
+        return _("No website %s %s") % (add_link, site_link)
     display_websites.admin_order_field = 'websites__name'
     display_websites.short_description = _("Websites")
     display_websites.allow_tags = True

orchestra/contrib/letsencrypt/actions.py

@@ -0,0 +1,98 @@
+from django.contrib import messages, admin
+from django.template.response import TemplateResponse
+from django.utils.translation import ungettext, ugettext_lazy as _
+
+from orchestra.contrib.orchestration import Operation, helpers
+
+from .helpers import is_valid_domain, read_live_lineages, configure_cert
+from .forms import LetsEncryptForm
+
+
+def letsencrypt(modeladmin, request, queryset):
+    wildcards = set()
+    domains = set()
+    queryset = queryset.prefetch_related('domains')
+    for website in queryset:
+        for domain in website.domains.all():
+            if domain.name.startswith('*.'):
+                wildcards.add(domain.name)
+            else:
+                domains.add(domain.name)
+    form = LetsEncryptForm(domains, wildcards, initial={'domains': '\n'.join(domains)})
+    action_value = 'letsencrypt'
+    if request.POST.get('post') == 'generic_confirmation':
+        form = LetsEncryptForm(domains, wildcards, request.POST)
+        if form.is_valid():
+            cleaned_data = form.cleaned_data
+            domains = set(cleaned_data['domains'])
+            operations = []
+            for website in queryset:
+                website_domains = [d.name for d in website.domains.all()]
+                encrypt_domains = set()
+                for domain in domains:
+                    if is_valid_domain(domain, website_domains, wildcards):
+                        encrypt_domains.add(domain)
+                website.encrypt_domains = encrypt_domains
+                operations.extend(Operation.create_for_action(website, 'encrypt'))
+                modeladmin.log_change(request, request.user, _("Encrypted!"))
+            if not operations:
+                messages.error(request, _("No backend operation has been executed."))
+            else:
+                logs = Operation.execute(operations)
+                helpers.message_user(request, logs)
+                live_lineages = read_live_lineages(logs)
+                errors = 0
+                successes = 0
+                no_https = 0
+                for website in queryset:
+                    try:
+                        configure_cert(website, live_lineages)
+                    except LookupError:
+                        errors += 1
+                        messages.error(request, _("No lineage found for website %s") % website.name)
+                    else:
+                        if website.protocol == website.HTTP:
+                            no_https += 1
+                        website.save(update_fields=('name',))
+                    successes += 1
+                context = {
+                    'name': website.name,
+                    'errors': errors,
+                    'successes': successes,
+                    'no_https': no_https
+                }
+                if errors:
+                    msg = ungettext(
+                        _("No lineages found for websites {name}."),
+                        _("No lineages found for {errors} websites."),
+                        errors)
+                    messages.error(request, msg % context)
+                if successes:
+                    msg = ungettext(
+                        _("{name} website has successfully been encrypted."),
+                        _("{successes} websites have been successfully encrypted."),
+                        successes)
+                    messages.success(request, msg.format(**context))
+                if no_https:
+                    msg = ungettext(
+                        _("{name} website does not have HTTPS protocol enabled."),
+                        _("{no_https} websites do not have HTTPS protocol enabled."),
+                        no_https)
+                    messages.warning(request, msg.format(**context))
+                return
+    opts = modeladmin.model._meta
+    app_label = opts.app_label
+    context = {
+        'title': _("Let's encrypt!"),
+        'action_name': _("Encrypt"),
+        'action_value': action_value,
+        'queryset': queryset,
+        'opts': opts,
+        'obj': website if len(queryset) == 1 else None,
+        'app_label': app_label,
+        'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME,
+        'form': form,
+    }
+    return TemplateResponse(request, 'admin/orchestra/generic_confirmation.html',
+        context, current_app=modeladmin.admin_site.name)
+letsencrypt.short_description = "Let's encrypt!"

orchestra/contrib/letsencrypt/admin.py

@@ -0,0 +1,8 @@
+from orchestra.admin.utils import insertattr
+from orchestra.contrib.websites.admin import WebsiteAdmin
+
+from .import actions
+
+
+insertattr(WebsiteAdmin, 'change_view_actions', actions.letsencrypt)
+insertattr(WebsiteAdmin, 'actions', actions.letsencrypt)

orchestra/contrib/letsencrypt/backends.py

@@ -0,0 +1,57 @@
+import os
+import textwrap
+
+from orchestra.contrib.orchestration import ServiceController
+
+from . import settings
+
+
+class LetsEncryptController(ServiceController):
+    model = 'websites.Website'
+    verbose_name = "Let's encrypt!"
+    actions = ('encrypt',)
+    
+    def prepare(self):
+        super().prepare()
+        self.cleanup = []
+        context = {
+            'letsencrypt_auto': settings.LETSENCRYPT_AUTO_PATH,
+        }
+        self.append(textwrap.dedent("""
+            %(letsencrypt_auto)s --non-interactive --no-self-upgrade \\
+                --keep --expand --agree-tos certonly --webroot \\""") % context
+        )
+    
+    def encrypt(self, website):
+        context = self.get_context(website)
+        self.append("    --webroot-path %(webroot)s \\" % context)
+        self.append("        --email %(email)s \\" % context)
+        self.append("        -d %(domains)s \\" % context)
+        self.cleanup.append("rm -rf %(webroot)s/.well-known" % context)
+    
+    def commit(self):
+        self.append("    || exit_code=$?")
+        for cleanup in self.cleanup:
+            self.append(cleanup)
+        context = {
+            'letsencrypt_live': os.path.normpath(settings.LETSENCRYPT_LIVE_PATH),
+        }
+        self.append(textwrap.dedent("""
+            # Report back the lineages in order to infere each certificate path
+            echo '<live-lineages>'
+            find %(letsencrypt_live)s/* -maxdepth 0
+            echo '</live-lineages>'""") % context
+        )
+        super().commit()
+    
+    def get_context(self, website):
+        try:
+            content = website.content_set.get(path='/')
+        except website.content_set.model.DoesNotExist:
+            raise
+        return {
+            'letsencrypt_auto': settings.LETSENCRYPT_AUTO_PATH,
+            'webroot': content.webapp.get_path(),
+            'email': website.account.email,
+            'domains': ' \\\n        -d '.join(website.encrypt_domains),
+        }

orchestra/contrib/letsencrypt/forms.py

@@ -0,0 +1,32 @@
+from django import forms
+from django.core.exceptions import ValidationError
+from django.utils.translation import ungettext, ugettext_lazy as _
+
+from .helpers import is_valid_domain
+
+
+class LetsEncryptForm(forms.Form):
+    domains = forms.CharField(widget=forms.Textarea)
+    
+    def __init__(self, domains, wildcards, *args, **kwargs):
+        self.domains = domains
+        self.wildcards = wildcards
+        super().__init__(*args, **kwargs)
+        if wildcards:
+            help_text = _("You can add domains maching the following wildcards: %s")
+            self.fields['domains'].help_text += help_text % ', '.join(wildcards)
+    
+    def clean_domains(self):
+        domains = self.cleaned_data['domains'].split()
+        cleaned_domains = set()
+        for domain in domains:
+            domain = domain.strip()
+            if domain not in self.domains:
+                domain = domain.strip()
+                if not is_valid_domain(domain, self.domains, self.wildcards):
+                    raise ValidationError(_(
+                        "%s domain is not included on selected websites, "
+                        "nor matches with any wildcard domain.") % domain
+                    )
+            cleaned_domains.add(domain)
+        return cleaned_domains

orchestra/contrib/letsencrypt/helpers.py

@@ -0,0 +1,48 @@
+import os
+
+
+def is_valid_domain(domain, existing, wildcards):
+    if domain in existing:
+        return True
+    for wildcard in wildcards:
+        if domain.startswith(wildcard.lstrip('*')) and domain.count('.') == wildcard.count('.'):
+            return True
+    return False
+
+
+def read_live_lineages(logs):
+    live_lineages = {}
+    for log in logs:
+        reading = False
+        for line in log.stdout.splitlines():
+            line = line.strip()
+            if line == '</live-lineages>':
+                break
+            if reading:
+                live_lineages[line.split('/')[-1]] = line
+            elif line == '<live-lineages>':
+                reading = True
+    return live_lineages
+
+
+def configure_cert(website, live_lineages):
+    for domain in website.domains.all():
+        try:
+            path = live_lineages[domain.name]
+        except KeyError:
+            pass
+        else:
+            maps = (
+                ('ssl-ca', os.path.join(path, 'chain.pem')),
+                ('ssl-cert', os.path.join(path, 'cert.pem')),
+                ('ssl-key', os.path.join(path, 'privkey.pem')),
+            )
+            for directive, path in maps:
+                try:
+                    directive = website.directives.get(name=directive)
+                except website.directives.model.DoesNotExist:
+                    directive = website.directives.model(name=directive, website=website)
+                directive.value = path
+                directive.save()
+            return
+    raise LookupError("Lineage not found")

orchestra/contrib/letsencrypt/settings.py

@@ -0,0 +1,11 @@
+from orchestra.contrib.settings import Setting
+
+
+LETSENCRYPT_AUTO_PATH = Setting('LETSENCRYPT_AUTO_PATH',
+    '/home/httpd/letsencrypt/letsencrypt-auto'
+)
+
+
+LETSENCRYPT_LIVE_PATH = Setting('LETSENCRYPT_LIVE_PATH',
+    '/etc/letsencrypt/live'
+)

orchestra/contrib/orchestration/backends.py

@@ -113,8 +113,10 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount):
                 related = obj
                 for attribute in field.split('__'):
                     related = getattr(related, attribute)
-                return related
+                if type(related).__name__ == 'RelatedManager':
-        return None
+                    return related.all()
+                return [related]
+        return []
     
     @classmethod
     def get_backends(cls, instance=None, action=None):

orchestra/contrib/orchestration/helpers.py

@@ -123,32 +123,40 @@ def message_user(request, logs):
     async_msg = ''
     if async:
         async_msg = ungettext(
-            _('<a href="{async_url}">{async} backend</a> is running on the background'),
+            _('<a href="{async_url}">{name}</a> is running on the background'),
             _('<a href="{async_url}">{async} backends</a> are running on the background'),
             async)
     if errors:
-        msg = ungettext(
+        if total == 1:
-            _('<a href="{url}">{errors} out of {total} backend</a> has fail to execute'),
+            msg = _('<a href="{url}">{name}</a> has fail to execute'),
-            _('<a href="{url}">{errors} out of {total} backends</a> have fail to execute'),
+        else:
-            errors)
+            msg = ungettext(
+                _('<a href="{url}">{errors} out of {total} backends</a> has fail to execute'),
+                _('<a href="{url}">{errors} out of {total} backends</a> have fail to execute'),
+                errors)
         if async_msg:
             msg += ', ' + str(async_msg)
-        msg = msg.format(errors=errors, async=async, async_url=async_url, total=total, url=url)
+        msg = msg.format(errors=errors, async=async, async_url=async_url, total=total, url=url,
+            name=log.backend)
         messages.error(request, mark_safe(msg + '.'))
     elif successes:
         if async_msg:
-            msg = ungettext(
+            if total == 1:
-                _('<a href="{url}">{successes} out of {total} backend</a> has been executed'),
+                msg = _('<a href="{url}">{name}</a> has been executed')
-                _('<a href="{url}">{successes} out of {total} backends</a> have been executed'),
+            else:
-                successes)
+                msg = ungettext(
+                    _('<a href="{url}">{successes} out of {total} backends</a> has been executed'),
+                    _('<a href="{url}">{successes} out of {total} backends</a> have been executed'),
+                    successes)
             msg += ', ' + str(async_msg)
         else:
             msg = ungettext(
-                _('<a href="{url}">{total} backend</a> has been executed'),
+                _('<a href="{url}">{name}</a> has been executed'),
                 _('<a href="{url}">{total} backends</a> have been executed'),
                 total)
         msg = msg.format(
-            total=total, url=url, async_url=async_url, async=async, successes=successes
+            total=total, url=url, async_url=async_url, async=async, successes=successes,
+            name=log.backend
         )
         messages.success(request, mark_safe(msg + '.'))
     else:

orchestra/contrib/orchestration/manager.py

@@ -154,8 +154,7 @@ def collect(instance, action, **kwargs):
             if backend_cls.is_main(instance):
                 instances = [(instance, action)]
             else:
-                candidate = backend_cls.get_related(instance)
+                for candidate in backend_cls.get_related(instance):
-                if candidate:
                     if candidate.__class__.__name__ == 'ManyRelatedManager':
                         if 'pk_set' in kwargs:
                             # m2m_changed signal

orchestra/contrib/websites/backends/wordpress.py

@@ -1,3 +1,4 @@
+import os
 import textwrap
 
 from orchestra.contrib.orchestration import ServiceController
@@ -35,3 +36,31 @@ class WordPressURLController(ServiceController):
             'url': content.get_absolute_url(),
             'db_name': content.webapp.data.get('db_name'),
         }
+
+
+class WordPressForceSSLController(ServiceController):
+    """ sets FORCE_SSL_ADMIN to true when website supports HTTPS """
+    verbose_name = "WordPress Force SSL"
+    model = 'websites.Content'
+    related_models = (
+        ('websites.Website', 'content_set'),
+    )
+    default_route_match = "content.webapp.type == 'wordpress-php'"
+    
+    def save(self, content):
+        context = self.get_context(content)
+        site = content.website
+        if site.protocol in (site.HTTP_AND_HTTPS, site.HTTPS_ONLY, site.HTTPS):
+            self.append(textwrap.dedent("""
+                if [[ ! $(grep FORCE_SSL_ADMIN %(wp_conf_path)s) ]]; then
+                    echo "Enabling FORCE_SSL_ADMIN for %(webapp_name)s webapp"
+                    sed -i -E "s#^(define\('NONCE_SALT.*)#\\1\\n\\ndefine\('FORCE_SSL_ADMIN', true\);#" \\
+                        %(wp_conf_path)s
+                fi""") % context
+            )
+    
+    def get_context(self, content):
+        return {
+            'webapp_name': content.webapp.name,
+            'wp_conf_path': os.path.join(content.webapp.get_path(), 'wp-config.php'),
+        }

orchestra/contrib/websites/directives.py

@@ -21,7 +21,7 @@ class SiteDirective(plugins.Plugin, metaclass=plugins.PluginMount):
     help_text = ""
     unique_name = False
     unique_value = False
-    unique_location = False
+    is_location = False
     
     @classmethod
     @lru_cache()
@@ -62,8 +62,10 @@ class SiteDirective(plugins.Plugin, metaclass=plugins.PluginMount):
         value = directive.get('value', None)
         # location uniqueness
         location = None
-        if self.unique_location and value is not None:
+        if self.is_location and value is not None:
-            location = normurlpath(directive['value'].split()[0])
+            if not value and self.is_location:
+                value = '/'
+            location = normurlpath(value.split()[0])
         if location is not None and location in locations:
             errors['value'].append(ValidationError(
                 "Location '%s' already in use by other content/directive." % location
@@ -89,6 +91,8 @@ class SiteDirective(plugins.Plugin, metaclass=plugins.PluginMount):
     
     def validate(self, directive):
         directive.value = directive.value.strip()
+        if not directive.value and self.is_location:
+            directive.value = '/'
         if self.regex and not re.match(self.regex, directive.value):
             raise ValidationError({
                 'value': ValidationError(_("'%(value)s' does not match %(regex)s."),
@@ -106,7 +110,7 @@ class Redirect(SiteDirective):
     regex = r'^[^ ]*\s[^ ]+$'
     group = SiteDirective.HTTPD
     unique_value = True
-    unique_location = True
+    is_location = True
     
     def validate(self, directive):
         """ inserts default url-path if not provided """
@@ -164,7 +168,7 @@ class SecRuleRemove(SiteDirective):
     help_text = _("Space separated ModSecurity rule IDs.")
     regex = r'^[0-9\s]+$'
     group = SiteDirective.SEC
-    unique_location = True
+    is_location = True
 
 
 class SecEngine(SecRuleRemove):
@@ -172,7 +176,7 @@ class SecEngine(SecRuleRemove):
     verbose_name = _("SecRuleEngine Off")
     help_text = _("URL-path with disabled modsecurity engine.")
     regex = r'^/[^ ]*$'
-    unique_location = False
+    is_location = False
 
 
 class WordPressSaaS(SiteDirective):
@@ -182,7 +186,7 @@ class WordPressSaaS(SiteDirective):
     group = SiteDirective.SAAS
     regex = r'^/[^ ]*$'
     unique_value = True
-    unique_location = True
+    is_location = True
 
 
 class DokuWikiSaaS(WordPressSaaS):

orchestra/contrib/websites/models.py

@@ -117,7 +117,7 @@ class WebsiteDirective(models.Model):
         related_name='directives')
     name = models.CharField(_("name"), max_length=128, db_index=True,
         choices=SiteDirective.get_choices())
-    value = models.CharField(_("value"), max_length=256)
+    value = models.CharField(_("value"), max_length=256, blank=True)
     
     def __str__(self):
         return self.name

orchestra/static/orchestra/images/add.svg

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="10"
+   height="10"
+   id="svg3898"
+   version="1.1"
+   inkscape:version="0.48.4 r9939"
+   inkscape:export-filename="/home/glic3rinu/orchestra/django-orchestra/orchestra/static/orchestra/images/add.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"
+   sodipodi:docname="New document 9">
+  <defs
+     id="defs3900" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="45.254834"
+     inkscape:cx="12.180788"
+     inkscape:cy="3.5068203"
+     inkscape:current-layer="layer1"
+     showgrid="true"
+     inkscape:grid-bbox="true"
+     inkscape:document-units="px"
+     inkscape:window-width="1920"
+     inkscape:window-height="1014"
+     inkscape:window-x="0"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata3903">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     id="layer1"
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     transform="translate(0,-6)">
+    <path
+       sodipodi:type="arc"
+       style="fill:#447e9b;fill-opacity:1;stroke:#447e9b;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="path3767"
+       sodipodi:cx="5.237927"
+       sodipodi:cy="5.8947067"
+       sodipodi:rx="4.7884302"
+       sodipodi:ry="4.7884302"
+       d="m 10.026357,5.8947067 a 4.7884302,4.7884302 0 1 1 -9.57686025,0 4.7884302,4.7884302 0 1 1 9.57686025,0 z"
+       transform="matrix(0.99135867,0,0,0.99135867,-0.18664494,5.1502121)" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.49999982;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect3763"
+       width="1.305508"
+       height="6.2165585"
+       x="4.3477392"
+       y="7.8912253"
+       rx="0.17796597"
+       ry="0.17796597" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.49999976;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect3763-9"
+       width="1.3055079"
+       height="6.2165575"
+       x="10.346749"
+       y="-8.1087723"
+       rx="0.17796594"
+       ry="0.17796594"
+       transform="matrix(0,1,-1,0,0,0)" />
+  </g>
+</svg>