diff --git a/INSTALL.md b/INSTALL.md index 01ba3a51..54f82731 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -38,22 +38,38 @@ Django-orchestra can be installed on any Linux system, however it is **strongly 5. Create and configure a Postgres database ```bash + sudo apt-get install python3-psycopg2 postgresql sudo python3 manage.py setuppostgres --db_password # admin_tools needs accounts and does not have migrations python3 manage.py migrate accounts python3 manage.py migrate ``` -7. Configure celeryd + +6. See the Django deployment checklist ```bash - sudo python3 manage.py setupcelery --username orchestra + python3 panel/manage.py check --deploy ``` + +6. Configure periodic execution of tasks (choose one) + 1. Use cron + ```bash + sudo python3 manage.py setupcronbeat + ``` + + 2. Use celeryd + ```bash + sudo apt-get install rabbitmq + sudo python3 manage.py setupcelery --username orchestra + ``` + + 8. Configure the web server: ```bash python3 manage.py collectstatic --noinput sudo apt-get install nginx-full uwsgi uwsgi-plugin-python3 - sudo python3 manage.py setupnginx + sudo python3 manage.py setupnginx --user orchestra ``` 9. Start all services: diff --git a/TODO.md b/TODO.md index a0c88898..f6ea79d3 100644 --- a/TODO.md +++ b/TODO.md @@ -361,9 +361,10 @@ Collecting lxml==3.3.5 (from -r re (line 22)) # project settings modified copy of django's default project settings -# migrate accounts break on superuser insert because of orders signals: read() + db_ready() +# migrate accounts break on superuser insert because of orders signals: ready() + db_ready() -# if backend.async: don't join +# if backend.async: don't join. +# RELATED: domains.sync to ns3 make it async # ngnix setup certificate from orchestra.contrib.tasks import task @@ -377,46 +378,18 @@ Collecting lxml==3.3.5 (from -r re (line 22)) time.sleep(1) counter.apply_async(10, '/tmp/kakas') -# standard django deployment pracices (run checks) # setup main systemuser on post_migrate SystemUser # Provide some fixtures with mocked data -# don't make hard dependencies strict dependencies, fail when needed. +don't make hard dependencies strict dependencies, fail when needed. # on project_settings add debug settings but commented # rename context processes varbailes to its original name -# TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall -# TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix -# TODO mount the filesystem with "nosuid" option +TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall +TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix +TODO mount the filesystem with "nosuid" option # execute Make after postfix update -# setupuwsgi + setupnginx - - -if not cert: - sudo mkdir /etc/nginx/ssl - sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/ssl/nginx.key -out /etc/nginx/ssl/nginx.crt - if --noinput - openssl req \ - -new \ - -newkey rsa:4096 \ - -days 365 \ - -nodes \ - -x509 \ - -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" \ - -keyout www.example.com.key \ - -out www.example.com.cert - - - ssl_certificate /etc/nginx/ssl/nginx.crt; - ssl_certificate_key /etc/nginx/ssl/nginx.key; - -if server_name: - at sites-enabled - server_name your_domain.com; - -else conf-enabled - - +# wkhtmltopdf -> reportlab diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index a6eb8375..90df6bbc 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -122,56 +122,23 @@ function install_requirements () { check_root || true ORCHESTRA_PATH=$(get_orchestra_dir) || true - # TODO reduce this list to 0 - # include /usr/sbin/named-checkzone - # wkhtmltopdf -> reportlab - # remove rabbit, postgres - # uwsgi –py-autoreload for devel APT="python3 \ python3-pip \ - python3-psycopg2 \ - python3-lxml \ python3-dev \ - bind9utils \ - python3-cracklib \ - libz-dev \ + libxml2-dev \ + libxslt1-dev \ wkhtmltopdf \ xvfb \ ca-certificates \ gettext" - # TODO remove celery deps, django 1.8.1, glic3rinu fork, celery email - PIP="django==1.8.1 \ - django-celery-email==1.0.4 \ - https://github.com/glic3rinu/django-fluent-dashboard/archive/master.zip \ - https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip \ - IPy==0.81 \ - django-extensions==1.5.2 \ - django-transaction-signals==1.0.0 \ - django-celery==3.1.16 \ - celery==3.1.16 \ - kombu==3.0.23 \ - billiard==3.3.0.18 \ - Markdown==2.4 \ - djangorestframework==3.1.1 \ - paramiko==1.15.1 \ - ecdsa==0.11 \ - Pygments==1.6 \ - django-filter==0.9.2 \ - https://github.com/glic3rinu/passlib/archive/master.zip \ - jsonfield==0.9.22 \ - python-dateutil==2.4.2 \ - django-iban==0.3.0 \ - requests \ - phonenumbers \ - django-countries \ - django-localflavor \ - pip==6.0.8" + PIP=$(wget https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt -q -O - | tr '\n' ' ') if $testing; then APT="${APT} \ iceweasel \ dnsutils" + PIP="${PIP} \ selenium \ xvfbwrapper \ @@ -205,13 +172,6 @@ function install_requirements () { fi run pip3 install $PIP - - # Patch passlib -# IMPORT="from django.contrib.auth.hashers import mask_hash, _" -# COLLECTIONS="from collections import OrderedDict" -# PASSLIB_PATH=$(python3 -c "from passlib.ext.django import utils; print(utils.__file__)") -# sed -i "s/${IMPORT}, SortedDict/${IMPORT}\n ${COLLECTIONS}/" $PASSLIB_PATH -# sed -i "s/SortedDict/OrderedDict/g" $PASSLIB_PATH } export -f install_requirements diff --git a/orchestra/contrib/domains/bin/named-checkzone b/orchestra/contrib/domains/bin/named-checkzone new file mode 100755 index 00000000..3d2961c8 Binary files /dev/null and b/orchestra/contrib/domains/bin/named-checkzone differ diff --git a/orchestra/contrib/domains/settings.py b/orchestra/contrib/domains/settings.py index 62afd2fa..2e9c6b73 100644 --- a/orchestra/contrib/domains/settings.py +++ b/orchestra/contrib/domains/settings.py @@ -63,7 +63,7 @@ DOMAINS_SLAVES_PATH = Setting('DOMAINS_SLAVES_PATH', DOMAINS_CHECKZONE_BIN_PATH = Setting('DOMAINS_CHECKZONE_BIN_PATH', - '/usr/sbin/named-checkzone -i local -k fail -n fail', + 'named-checkzone -i local -k fail -n fail', ) diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index f6f38d70..1d3fb75c 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -1,3 +1,4 @@ +import os import textwrap from optparse import make_option from os.path import expanduser @@ -17,6 +18,29 @@ class Command(BaseCommand): help='Nginx SSL certificate, one will be created by default.'), make_option('--cert-key', dest='cert_key', default='', help='Nginx SSL certificate key.'), + + make_option('--cert-path', dest='cert_path', default='/etc/nginx/ssl/orchestra.crt', + help='Nginx SSL certificate, one will be created by default.'), + make_option('--cert-key-path', dest='cert_key_path', default='/etc/nginx/ssl/orchestra.key', + help='Nginx SSL certificate key.'), + # Cert options + make_option('--cert-override', dest='cert_override', action='store_true', + default=False, help='Force override cert and keys if exists.'), + make_option('--cert-country', dest='cert_country', default='ES', + help='Certificate Distinguished Name Country.'), + make_option('--cert-state', dest='cert_state', default='Spain', + help='Certificate Distinguished Name STATE.'), + make_option('--cert-locality', dest='cert_locality', default='Barcelona', + help='Certificate Distinguished Name Country.'), + make_option('--cert-org_name', dest='cert_org_name', default='Orchestra', + help='Certificate Distinguished Name Organization Name.'), + make_option('--cert-org_unit', dest='cert_org_unit', default='DevOps', + help='Certificate Distinguished Name Organization Unity.'), + make_option('--cert-email', dest='cert_email', default='orchestra@orchestra.lan', + help='Certificate Distinguished Name Email Address.'), + make_option('--cert-common_name', dest='cert_common_name', default=None, + help='Certificate Distinguished Name Common Name.'), + make_option('--server-name', dest='server_name', default='', help='Nginx SSL certificate key.'), make_option('--user', dest='user', default='', @@ -34,36 +58,121 @@ class Command(BaseCommand): option_list = BaseCommand.option_list help = 'Configures nginx + uwsgi to run with your Orchestra instance.' - @check_root - def handle(self, *args, **options): + def generate_certificate(self, **options): + override = options.get('cert_override') interactive = options.get('interactive') cert = options.get('cert') - cert_key = options.get('cert_key') - if bool(cert) != bool(cert_key): + key = options.get('cert_key') + if bool(cert) != bool(key): raise CommandError("--cert and --cert-key go in tandem") - if not cert: - run("mkdir -p /etc/nginx/ssl") - if interactive: - run("openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/ssl/nginx.key -out /etc/nginx/ssl/nginx.crt") + cert_path = options.get('cert_path') + key_path = options.get('cert_key_path') + + run('mkdir -p %s' % os.path.basename(cert_path)) + exists = os.path.isfile(cert_path) + + if not override and exists: + self.stdout.write('Your cert and keys are already in place.') + self.stdout.write('Use --override in order to regenerate them.') + return cert_path, key_path + + common_name = options.get('cert_common_name') or options.get('server_name') or 'orchestra.lan' + country = options.get('cert_country') + state = options.get('cert_state') + locality = options.get('cert_locality') + org_name = options.get('cert_org_name') + org_unit = options.get('cert_org_unit') + email = options.get('cert_email') + if interactive: + msg = ('-----\n' + 'You are about to be asked to enter information that\n' + 'will be incorporated\n' + 'into your certificate request.\n' + 'What you are about to enter is what is called a\n' + 'Distinguished Name or a DN.\n' + 'There are quite a few fields but you can leave some blank\n' + '-----\n') + self.stdout.write(msg) + + msg = 'Country Name (2 letter code) [%s]: ' % country + country = input(msg) or country + + msg = 'State or Province Name (full name) [%s]: ' % state + state = input(msg) or state + + msg = 'Locality Name (eg, city) [%s]: ' % locality + locality = input(msg) or locality + + msg = 'Organization Name (eg, company) [%s]: ' % org_name + org_name = input(msg) or org_name + + msg = 'Organizational Unit Name (eg, section) [%s]: ' % org_unit + org_unit = input(msg) or org_unit + + msg = 'Email Address [%s]: ' % email + email = input(msg) or email + + self.stdout.write('Common Name: %s' % common_name) + subject = { + 'C': country, + 'S': state, + 'L': locality, + 'O': org_name, + 'OU': org_unit, + 'Email': email, + 'CN': common_name, + } + context = { + 'subject': ''.join(('/%s=%s' % (k,v) for k,v in subject.items())), + 'key_path': key_path, + 'cert_path': cert_path, + } + self.stdout.write('writing new cert to \'%s\'' % cert_path) + self.stdout.write('writing new cert key to \'%s\'' % key_path) + run('openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout %(key_path)s -out %(cert_path)s -subj "%(subject)s"' % context, display=True) + + return cert_path, key_path + + @check_root + def handle(self, *args, **options): + user = options.get('user') + if not user: + raise CommandError("System user for running uwsgi must be provided.") + + cert_path, key_path = self.generate_certificate(**options) + server_name = options.get('server_name') context = { + 'cert_path': cert_path, + 'key_path': key_path, 'project_name': paths.get_project_name(), 'project_dir': paths.get_project_dir(), 'site_dir': paths.get_site_dir(), 'static_root': settings.STATIC_ROOT, - 'user': options.get('user'), - 'group': options.get('group') or options.get('user'), + 'user': user, + 'group': options.get('group') or user, 'home': expanduser("~%s" % options.get('user')), - 'processes': int(options.get('processes')),} + 'processes': int(options.get('processes')), + 'server_name': 'server_name %s' % server_name if server_name else '' + } nginx_conf = textwrap.dedent("""\ server { listen 80; listen [::]:80 ipv6only=on; + return 301 https://$host$request_uri; + } + + server { + listen 443 ssl; + # listen [::]:443 ssl; # add SSL support to IPv6 address + %(server_name)s + ssl_certificate %(cert_path)s; + ssl_certificate_key %(key_path)s; rewrite ^/$ /admin/; - client_max_body_size 500m; + client_max_body_size 16m; location / { uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket; include uwsgi_params; @@ -78,34 +187,42 @@ class Command(BaseCommand): uwsgi_conf = textwrap.dedent("""\ [uwsgi] - plugins = python - chdir = %(site_dir)s - module = %(project_name)s.wsgi - master = true - processes = %(processes)d - chmod-socket = 664 - stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket - vacuum = true - uid = %(user)s - gid = %(group)s - env = HOME=%(home)s - touch-reload = %(project_dir)s/wsgi.py + plugins = python + chdir = %(site_dir)s + module = %(project_name)s.wsgi + master = true + processes = %(processes)d + chmod-socket = 664 + stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket + vacuum = true + uid = %(user)s + gid = %(group)s + env = HOME=%(home)s + touch-reload = %(project_dir)s/wsgi.py + enable-threads = true + max-requests = 500 """ ) % context + nginx_file = '/etc/nginx/conf.d/%(project_name)s.conf' % context + if server_name: + context['server_name'] = server_name + nginx_file = '/etc/nginx/sites-available/%(server_name)s.conf' % context nginx = { - 'file': '/etc/nginx/conf.d/%(project_name)s.conf' % context, - 'conf': nginx_conf } + 'file': nginx_file, + 'conf': nginx_conf + } uwsgi = { 'file': '/etc/uwsgi/apps-available/%(project_name)s.ini' % context, - 'conf': uwsgi_conf } + 'conf': uwsgi_conf + } for extra_context in (nginx, uwsgi): context.update(extra_context) - diff = run("echo '%(conf)s'|diff - %(file)s" % context, error_codes=[0,1,2]) + diff = run("echo '%(conf)s' | diff - %(file)s" % context, error_codes=[0,1,2]) if diff.return_code == 2: # File does not exist - run("echo '%(conf)s' > %(file)s" % context) + run("echo '%(conf)s' > %(file)s" % context, display=True) elif diff.return_code == 1: # File is different, save the old one if interactive: @@ -119,16 +236,14 @@ class Command(BaseCommand): if confirm == 'no': return break - run("cp %(file)s %(file)s.save" % context) - run("echo '%(conf)s' > %(file)s" % context) + run("cp %(file)s %(file)s.save" % context, display=True) + run("echo '%(conf)s' > %(file)s" % context, display=True) self.stdout.write("\033[1;31mA new version of %(file)s has been installed.\n " "The old version has been placed at %(file)s.save\033[m" % context) - run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/' % context, error_codes=[0,1]) - - # nginx should start after tincd - current = "\$local_fs \$remote_fs \$network \$syslog" - run('sed -i "s/ %s$/ %s \$named/g" /etc/init.d/nginx' % (current, current)) + if server_name: + run('ln -s /etc/nginx/sites-available/%(server_name)s.conf /etc/nginx/sites-enabled/' % context, error_codes=[0,1], display=True) + run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/' % context, error_codes=[0,1], display=True) rotate = textwrap.dedent("""\ /var/log/nginx/*.log { @@ -145,7 +260,7 @@ class Command(BaseCommand): endscript }""" ) - run("echo '%s' > /etc/logrotate.d/nginx" % rotate) + run("echo '%s' > /etc/logrotate.d/nginx" % rotate, display=True) # Allow nginx to write to uwsgi socket - run('adduser www-data %(group)s' % context) + run('adduser www-data %(group)s' % context, display=True) diff --git a/requirements.txt b/requirements.txt index 3d16a2a4..610e5292 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,27 @@ cracklib django==1.8.1 -django-celery-email==1.0.4 -https://github.com/glic3rinu/django-fluent-dashboard/archive/master.zip -https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip -IPy==0.81 -django-extensions==1.5.2 -django-transaction-signals==1.0.0 -django-celery==3.1.16 -celery==3.1.16 -kombu==3.0.23 -billiard==3.3.0.18 -Markdown==2.4 +django-celery-email==1.0.4 +django-fluent-dashboard==0.5 +https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip +IPy==0.81 +django-extensions==1.5.2 +django-transaction-signals==1.0.0 +django-celery==3.1.16 +celery==3.1.16 +kombu==3.0.23 +billiard==3.3.0.18 +Markdown==2.4 djangorestframework==3.1.1 -paramiko==1.15.1 -ecdsa==0.11 -Pygments==1.6 -django-filter==0.7 +paramiko==1.15.1 +ecdsa==0.11 +Pygments==1.6 +django-filter==0.7 https://github.com/glic3rinu/passlib/archive/master.zip -jsonfield==0.9.22 -lxml==3.3.5 -python-dateutil==2.2 -django-iban==0.3.0 -requests -phonenumbers -django-countries -django-localflavor -###development -django-debug-toolbar -django-nose +jsonfield==0.9.22 +lxml==3.3.5 +python-dateutil==2.2 +django-iban==0.3.0 +requests +phonenumbers +django-countries +django-localflavor diff --git a/setup.py b/setup.py index 804e8337..11b34b53 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup( scripts=[ 'orchestra/bin/orchestra-admin', 'orchestra/contrib/tasks/bin/orchestra-beat', + 'orchestra/contrib/domains/bin/named-checkzone', ], packages = packages, classifiers = [