From 430905295d8bba2db2a8b41a45ed08a572422565 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 10 Sep 2020 00:14:59 +0200 Subject: [PATCH] root: automate system migrations, move docker to lifecycle folder --- Dockerfile | 23 +++++------ README.md | 2 +- docker/bootstrap.sh | 10 ----- e2e/test_flows_enroll.py | 6 +-- e2e/test_provider_oauth2_github.py | 8 ++-- e2e/test_provider_oauth2_oidc.py | 6 +-- e2e/test_provider_saml.py | 8 ++-- e2e/test_source_saml.py | 6 +-- e2e/test_sources_oauth.py | 6 +-- lifecycle/__init__.py | 0 lifecycle/bootstrap.sh | 14 +++++++ {docker => lifecycle}/gunicorn.conf.py | 0 lifecycle/migrate.py | 55 ++++++++++++++++++++++++++ lifecycle/system_migrations/to_0_10.py | 50 +++++++++++++++++++++++ {docker => lifecycle}/wait_for_db.py | 0 swagger.yaml | 12 ++++-- 16 files changed, 158 insertions(+), 48 deletions(-) delete mode 100755 docker/bootstrap.sh create mode 100644 lifecycle/__init__.py create mode 100755 lifecycle/bootstrap.sh rename {docker => lifecycle}/gunicorn.conf.py (100%) create mode 100755 lifecycle/migrate.py create mode 100644 lifecycle/system_migrations/to_0_10.py rename {docker => lifecycle}/wait_for_db.py (100%) diff --git a/Dockerfile b/Dockerfile index f8a5d951f..61f343de3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,27 +11,22 @@ RUN pip install pipenv && \ FROM python:3.8-slim-buster -COPY --from=locker /app/requirements.txt /app/ -COPY --from=locker /app/requirements-dev.txt /app/ - -WORKDIR /app/ +WORKDIR / +COPY --from=locker /app/requirements.txt / +COPY --from=locker /app/requirements-dev.txt / RUN apt-get update && \ apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \ rm -rf /var/lib/apt/ && \ - pip install -r requirements.txt --no-cache-dir && \ + pip install -r /requirements.txt --no-cache-dir && \ apt-get remove --purge -y build-essential && \ apt-get autoremove --purge && \ - adduser --system --no-create-home --uid 1000 --group --home /app passbook + adduser --system --no-create-home --uid 1000 --group --home /passbook passbook -COPY ./passbook/ /app/passbook -COPY ./manage.py /app/ -COPY ./docker/gunicorn.conf.py /app/ -COPY ./docker/bootstrap.sh /bootstrap.sh -COPY ./docker/wait_for_db.py /app/wait_for_db.py - -WORKDIR /app/ +COPY ./passbook/ /passbook +COPY ./manage.py / +COPY ./lifecycle/ /lifecycle USER passbook -ENTRYPOINT [ "/bootstrap.sh" ] +ENTRYPOINT [ "/lifecycle/bootstrap.sh" ] diff --git a/README.md b/README.md index 9b9ff48e2..e7527f660 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ wget https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml # export PG_PASS=$(pwgen 40 1) docker-compose pull docker-compose up -d -docker-compose exec server ./manage.py migrate +docker-compose run --rm server migrate ``` For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/) diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh deleted file mode 100755 index 8cac56846..000000000 --- a/docker/bootstrap.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e -/app/wait_for_db.py -printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@" -if [[ "$1" == "server" ]]; then - gunicorn -c gunicorn.conf.py passbook.root.asgi:application -elif [[ "$1" == "worker" ]]; then - celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule -else - ./manage.py "$@" -fi diff --git a/e2e/test_flows_enroll.py b/e2e/test_flows_enroll.py index 9a8648721..e0e767532 100644 --- a/e2e/test_flows_enroll.py +++ b/e2e/test_flows_enroll.py @@ -2,13 +2,13 @@ from time import sleep from django.test import override_settings +from docker import DockerClient, from_env +from docker.models.containers import Container +from docker.types import Healthcheck from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from docker import DockerClient, from_env -from docker.models.containers import Container -from docker.types import Healthcheck from e2e.utils import USER, SeleniumTestCase from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding from passbook.policies.expression.models import ExpressionPolicy diff --git a/e2e/test_provider_oauth2_github.py b/e2e/test_provider_oauth2_github.py index e5b2dde97..756ef7fa8 100644 --- a/e2e/test_provider_oauth2_github.py +++ b/e2e/test_provider_oauth2_github.py @@ -1,13 +1,13 @@ """test OAuth Provider flow""" from time import sleep -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys -from structlog import get_logger - from docker import DockerClient, from_env from docker.models.containers import Container from docker.types import Healthcheck +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from structlog import get_logger + from e2e.utils import USER, SeleniumTestCase from passbook.core.models import Application from passbook.flows.models import Flow diff --git a/e2e/test_provider_oauth2_oidc.py b/e2e/test_provider_oauth2_oidc.py index 43f31c0ad..829ba5e3b 100644 --- a/e2e/test_provider_oauth2_oidc.py +++ b/e2e/test_provider_oauth2_oidc.py @@ -1,14 +1,14 @@ """test OAuth2 OpenID Provider flow""" from time import sleep +from docker import DockerClient, from_env +from docker.models.containers import Container +from docker.types import Healthcheck from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from docker import DockerClient, from_env -from docker.models.containers import Container -from docker.types import Healthcheck from e2e.utils import USER, SeleniumTestCase from passbook.core.models import Application from passbook.crypto.models import CertificateKeyPair diff --git a/e2e/test_provider_saml.py b/e2e/test_provider_saml.py index 1fbd1a774..1cf9fac6a 100644 --- a/e2e/test_provider_saml.py +++ b/e2e/test_provider_saml.py @@ -1,13 +1,13 @@ """test SAML Provider flow""" from time import sleep -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys -from structlog import get_logger - from docker import DockerClient, from_env from docker.models.containers import Container from docker.types import Healthcheck +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from structlog import get_logger + from e2e.utils import USER, SeleniumTestCase from passbook.core.models import Application from passbook.crypto.models import CertificateKeyPair diff --git a/e2e/test_source_saml.py b/e2e/test_source_saml.py index 99fa37fb3..fea1ca95d 100644 --- a/e2e/test_source_saml.py +++ b/e2e/test_source_saml.py @@ -1,14 +1,14 @@ """test SAML Source""" from time import sleep +from docker import DockerClient, from_env +from docker.models.containers import Container +from docker.types import Healthcheck from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from docker import DockerClient, from_env -from docker.models.containers import Container -from docker.types import Healthcheck from e2e.utils import SeleniumTestCase from passbook.crypto.models import CertificateKeyPair from passbook.flows.models import Flow diff --git a/e2e/test_sources_oauth.py b/e2e/test_sources_oauth.py index 5cbbccec8..2db1fbea1 100644 --- a/e2e/test_sources_oauth.py +++ b/e2e/test_sources_oauth.py @@ -2,15 +2,15 @@ from os.path import abspath from time import sleep +from docker import DockerClient, from_env +from docker.models.containers import Container +from docker.types import Healthcheck from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger from yaml import safe_dump -from docker import DockerClient, from_env -from docker.models.containers import Container -from docker.types import Healthcheck from e2e.utils import SeleniumTestCase from passbook.flows.models import Flow from passbook.providers.oauth2.generators import generate_client_secret diff --git a/lifecycle/__init__.py b/lifecycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lifecycle/bootstrap.sh b/lifecycle/bootstrap.sh new file mode 100755 index 000000000..bf3471395 --- /dev/null +++ b/lifecycle/bootstrap.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e +python -m lifecycle.wait_for_db +printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@" +if [[ "$1" == "server" ]]; then + gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application +elif [[ "$1" == "worker" ]]; then + celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule +elif [[ "$1" == "migrate" ]]; then + # Run system migrations first, run normal migrations after + python -m lifecycle.migrate + python -m manage migrate +else + python -m manage "$@" +fi diff --git a/docker/gunicorn.conf.py b/lifecycle/gunicorn.conf.py similarity index 100% rename from docker/gunicorn.conf.py rename to lifecycle/gunicorn.conf.py diff --git a/lifecycle/migrate.py b/lifecycle/migrate.py new file mode 100755 index 000000000..1bbd10679 --- /dev/null +++ b/lifecycle/migrate.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""System Migration handler""" +from importlib.util import module_from_spec, spec_from_file_location +from inspect import getmembers, isclass +from pathlib import Path +from typing import Any + +from psycopg2 import connect +from structlog import get_logger + +from passbook.lib.config import CONFIG + +LOGGER = get_logger() + + +class BaseMigration: + """Base System Migration""" + + cur: Any + con: Any + + def __init__(self, cur: Any, con: Any): + self.cur = cur + self.con = con + + def needs_migration(self) -> bool: + """Return true if Migration needs to be run""" + return False + + def run(self): + """Run the actual migration""" + + +if __name__ == "__main__": + + conn = connect( + dbname=CONFIG.y("postgresql.name"), + user=CONFIG.y("postgresql.user"), + password=CONFIG.y("postgresql.password"), + host=CONFIG.y("postgresql.host"), + ) + curr = conn.cursor() + + for migration in Path(__file__).parent.absolute().glob("system_migrations/*.py"): + spec = spec_from_file_location("lifecycle.system_migrations", migration) + mod = module_from_spec(spec) + # pyright: reportGeneralTypeIssues=false + spec.loader.exec_module(mod) + + for _, sub in getmembers(mod, isclass): + migration = sub(curr, conn) + if migration.needs_migration(): + LOGGER.info("Migration needs to be applied", migration=sub) + migration.run() + LOGGER.info("Migration finished applying", migration=sub) diff --git a/lifecycle/system_migrations/to_0_10.py b/lifecycle/system_migrations/to_0_10.py new file mode 100644 index 000000000..5bdbe264b --- /dev/null +++ b/lifecycle/system_migrations/to_0_10.py @@ -0,0 +1,50 @@ +from os import system + +from lifecycle.migrate import BaseMigration + +SQL_STATEMENT = """delete from django_migrations where app = 'passbook_stages_prompt'; +drop table passbook_stages_prompt_prompt cascade; +drop table passbook_stages_prompt_promptstage cascade; +drop table passbook_stages_prompt_promptstage_fields; +drop table corsheaders_corsmodel cascade; +drop table oauth2_provider_accesstoken cascade; +drop table oauth2_provider_grant cascade; +drop table oauth2_provider_refreshtoken cascade; +drop table oidc_provider_client cascade; +drop table oidc_provider_client_response_types cascade; +drop table oidc_provider_code cascade; +drop table oidc_provider_responsetype cascade; +drop table oidc_provider_rsakey cascade; +drop table oidc_provider_token cascade; +drop table oidc_provider_userconsent cascade; +drop table passbook_providers_app_gw_applicationgatewayprovider cascade; +delete from django_migrations where app = 'passbook_flows' and name = '0008_default_flows'; +delete from django_migrations where app = 'passbook_flows' and name = '0009_source_flows'; +delete from django_migrations where app = 'passbook_flows' and name = '0010_provider_flows'; +delete from django_migrations where app = 'passbook_stages_password' and +name = '0002_passwordstage_change_flow';""" + + +class To010Migration(BaseMigration): + def needs_migration(self) -> bool: + self.cur.execute( + "select * from information_schema.tables where table_name='oidc_provider_client'" + ) + return bool(self.cur.rowcount) + + def system_crit(self, command): + retval = system(command) # nosec + if retval != 0: + raise Exception("Migration error") + + def run(self): + self.cur.execute(SQL_STATEMENT) + self.con.commit() + self.system_crit("./manage.py migrate passbook_stages_prompt") + self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake") + self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake") + self.system_crit( + "./manage.py migrate passbook_flows 0010_provider_flows --fake" + ) + self.system_crit("./manage.py migrate passbook_flows") + self.system_crit("./manage.py migrate passbook_stages_password --fake") diff --git a/docker/wait_for_db.py b/lifecycle/wait_for_db.py similarity index 100% rename from docker/wait_for_db.py rename to lifecycle/wait_for_db.py diff --git a/swagger.yaml b/swagger.yaml index 1610adf7d..b58d316da 100755 --- a/swagger.yaml +++ b/swagger.yaml @@ -5249,7 +5249,7 @@ paths: tags: - stages parameters: [] - /stages/prompt/stages/{pbm_uuid}/: + /stages/prompt/stages/{stage_uuid}/: get: operationId: stages_prompt_stages_read description: PromptStage Viewset @@ -5303,7 +5303,7 @@ paths: tags: - stages parameters: - - name: pbm_uuid + - name: stage_uuid in: path description: A UUID string identifying this Prompt Stage. required: true @@ -7463,7 +7463,7 @@ definitions: type: object properties: pk: - title: Pbm uuid + title: Stage uuid type: string format: uuid readOnly: true @@ -7477,6 +7477,12 @@ definitions: type: string format: uuid uniqueItems: true + validation_policies: + type: array + items: + type: string + format: uuid + uniqueItems: true UserDeleteStage: required: - name