diff --git a/Pipfile b/Pipfile index 93f633f01..95588a913 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ service_identity = "*" signxml = "*" urllib3 = {extras = ["secure"],version = "*"} websocket_client = "*" +structlog = "*" [requires] python_version = "3.7" @@ -57,3 +58,4 @@ unittest-xml-reporting = "*" autopep8 = "*" bandit = "*" twine = "*" +colorama = "*" diff --git a/Pipfile.lock b/Pipfile.lock index f41ca4043..f9f823cf8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "cd82871d9aca8cfd548a6a62856196b2211524f12fbd416dfe5218aad9471e44" + "sha256": "f8694b0ee03f99560e853fd24e9cd7ac987c757cd50249398346e42cdd98cbbb" }, "pipfile-spec": 6, "requires": { @@ -777,6 +777,14 @@ ], "version": "==0.3.0" }, + "structlog": { + "hashes": [ + "sha256:5feae03167620824d3ae3e8915ea8589fc28d1ad6f3edf3cc90ed7c7cb33fab5", + "sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536" + ], + "index": "pypi", + "version": "==19.1.0" + }, "tempora": { "hashes": [ "sha256:cb60b1d2b1664104e307f8e5269d7f4acdb077c82e35cd57246ae14a3427d2d6", @@ -950,6 +958,14 @@ ], "version": "==3.0.4" }, + "colorama": { + "hashes": [ + "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", + "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" + ], + "index": "pypi", + "version": "==0.4.1" + }, "coverage": { "hashes": [ "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", diff --git a/passbook/core/auth/factor.py b/passbook/core/auth/factor.py index 4af91625e..98b26232b 100644 --- a/passbook/core/auth/factor.py +++ b/passbook/core/auth/factor.py @@ -19,7 +19,7 @@ class AuthenticationFactor(TemplateView): self.authenticator = authenticator def get_context_data(self, **kwargs): - kwargs['config'] = CONFIG.get('passbook') + kwargs['config'] = CONFIG.y('passbook') kwargs['is_login'] = True kwargs['title'] = _('Log in to your account') kwargs['primary_action'] = _('Log in') diff --git a/passbook/core/management/commands/web.py b/passbook/core/management/commands/web.py index 3fdbf097c..562fe8b6f 100644 --- a/passbook/core/management/commands/web.py +++ b/passbook/core/management/commands/web.py @@ -17,7 +17,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """passbook cherrypy server""" - cherrypy.config.update(CONFIG.get('web')) + cherrypy.config.update(CONFIG.y('web')) cherrypy.tree.graft(application, '/') # Mount NullObject to serve static files cherrypy.tree.mount(None, settings.STATIC_URL, config={ diff --git a/passbook/core/views/authentication.py b/passbook/core/views/authentication.py index 3879dd7e1..119c85f61 100644 --- a/passbook/core/views/authentication.py +++ b/passbook/core/views/authentication.py @@ -40,7 +40,7 @@ class LoginView(UserPassesTestMixin, FormView): return redirect(reverse('passbook_core:overview')) def get_context_data(self, **kwargs): - kwargs['config'] = CONFIG.get('passbook') + kwargs['config'] = CONFIG.y('passbook') kwargs['is_login'] = True kwargs['title'] = _('Log in to your account') kwargs['primary_action'] = _('Log in') @@ -135,7 +135,7 @@ class SignUpView(UserPassesTestMixin, FormView): return super().get_initial() def get_context_data(self, **kwargs): - kwargs['config'] = CONFIG.get('passbook') + kwargs['config'] = CONFIG.y('passbook') kwargs['is_login'] = True kwargs['title'] = _('Sign Up') kwargs['primary_action'] = _('Sign up') diff --git a/passbook/core/views/user.py b/passbook/core/views/user.py index 0760c2b41..6de4ad003 100644 --- a/passbook/core/views/user.py +++ b/passbook/core/views/user.py @@ -66,7 +66,7 @@ class UserChangePasswordView(LoginRequiredMixin, FormView): return redirect('passbook_core:overview') def get_context_data(self, **kwargs): - kwargs['config'] = CONFIG.get('passbook') + kwargs['config'] = CONFIG.y('passbook') kwargs['is_login'] = True kwargs['title'] = _('Change Password') kwargs['primary_action'] = _('Change') diff --git a/passbook/ldap/ldap_connector.py b/passbook/ldap/ldap_connector.py index 87d0f038e..6b8b7a751 100644 --- a/passbook/ldap/ldap_connector.py +++ b/passbook/ldap/ldap_connector.py @@ -166,7 +166,7 @@ class LDAPConnector: if not self._source.enabled: return None # FIXME: Adapt user_uid - # email = filters.pop(CONFIG.get('passport').get('ldap').get, '') + # email = filters.pop(CONFIG.y('passport').get('ldap').get, '') email = filters.pop('email') user_dn = self.lookup(self.generate_filter(**{LOGIN_FIELD: email})) if not user_dn: diff --git a/passbook/lib/config.py b/passbook/lib/config.py index 4a5a1fcec..77888d74b 100644 --- a/passbook/lib/config.py +++ b/passbook/lib/config.py @@ -1,31 +1,35 @@ -"""passbook lib config loader""" +"""passbook core config loader""" import os -from collections.abc import Mapping +from collections import Mapping from contextlib import contextmanager from glob import glob -from logging import getLogger from typing import Any +from urllib.parse import urlparse import yaml from django.conf import ImproperlyConfigured from django.utils.autoreload import autoreload_started +from structlog import get_logger SEARCH_PATHS = [ 'passbook/lib/default.yml', '/etc/passbook/config.yml', - '.', + '', ] + glob('/etc/passbook/config.d/*.yml', recursive=True) -LOGGER = getLogger(__name__) -ENVIRONMENT = os.getenv('PASSBOOK_ENV', 'local') +LOGGER = get_logger() +ENV_PREFIX = 'PASSBOOK' +ENVIRONMENT = os.getenv(f'{ENV_PREFIX}_ENV', 'local') class ConfigLoader: - """Search through SEARCH_PATHS and load configuration""" + """Search through SEARCH_PATHS and load configuration. Environment variables starting with + `ENV_PREFIX` are also applied. + + A variable like PASSBOOK_POSTGRESQL__HOST would translate to postgresql.host""" loaded_file = [] __config = {} - __context_default = None __sub_dicts = [] def __init__(self): @@ -41,21 +45,13 @@ class ConfigLoader: self.update_from_file(path) elif os.path.isdir(path) and os.path.exists(path): # Path is an existing dir, so we try to read the env config from it - env_paths = [os.path.join(path, ENVIRONMENT+'.yml'), - os.path.join(path, ENVIRONMENT+'.env.yml')] + env_paths = [os.path.join(path, ENVIRONMENT + '.yml'), + os.path.join(path, ENVIRONMENT + '.env.yml')] for env_file in env_paths: if os.path.isfile(env_file) and os.path.exists(env_file): # Update config with env file self.update_from_file(env_file) - self.handle_secret_key() - - def handle_secret_key(self): - """Handle `secret_key_file`""" - if 'secret_key_file' in self.__config: - secret_key_file = self.__config.get('secret_key_file') - if os.path.isfile(secret_key_file) and os.path.exists(secret_key_file): - with open(secret_key_file) as file: - self.__config['secret_key'] = file.read().replace('\n', '') + self.update_from_env() def update(self, root, updatee): """Recursively update dictionary""" @@ -63,16 +59,25 @@ class ConfigLoader: if isinstance(value, Mapping): root[key] = self.update(root.get(key, {}), value) else: + if isinstance(value, str): + value = self.parse_uri(value) root[key] = value return root + def parse_uri(self, value): + """Parse string values which start with a URI""" + url = urlparse(value) + if url.scheme == 'env': + value = os.getenv(url.netloc, url.query) + return value + def update_from_file(self, path: str): """Update config from file contents""" try: with open(path) as file: try: self.update(self.__config, yaml.safe_load(file)) - LOGGER.debug("Loaded %s", path) + LOGGER.debug("Loaded config", file=path) self.loaded_file.append(path) except yaml.YAMLError as exc: raise ImproperlyConfigured from exc @@ -83,12 +88,26 @@ class ConfigLoader: """Update config from dict""" self.__config.update(update) - @contextmanager - def default(self, value: Any): - """Contextmanage that sets default""" - self.__context_default = value - yield - self.__context_default = None + def update_from_env(self): + """Check environment variables""" + outer = {} + idx = 0 + for key, value in os.environ.items(): + if not key.startswith(ENV_PREFIX): + continue + relative_key = key.replace(f"{ENV_PREFIX}_", '').replace('__', '.').lower() + # Recursively convert path from a.b.c into outer[a][b][c] + current_obj = outer + dot_parts = relative_key.split('.') + for dot_part in dot_parts[:-1]: + if dot_part not in current_obj: + current_obj[dot_part] = {} + current_obj = current_obj[dot_part] + current_obj[dot_parts[-1]] = value + idx += 1 + if idx > 0: + LOGGER.debug("Loaded environment variables", count=idx) + self.update(self.__config, outer) @contextmanager # pylint: disable=invalid-name @@ -98,15 +117,6 @@ class ConfigLoader: yield self.__sub_dicts.pop() - def get(self, key: str, default=None) -> Any: - """Get value from loaded config file""" - if default is None: - default = self.__context_default - config_copy = self.raw - for sub in self.__sub_dicts: - config_copy = config_copy.get(sub, None) - return config_copy.get(key, default) - @property def raw(self) -> dict: """Get raw config dictionary""" @@ -115,8 +125,6 @@ class ConfigLoader: # pylint: disable=invalid-name def y(self, path: str, default=None, sep='.') -> Any: """Access attribute by using yaml path""" - if default is None: - default = self.__context_default # Walk sub_dicts before parsing path root = self.raw for sub in self.__sub_dicts: @@ -129,11 +137,17 @@ class ConfigLoader: return default return root + def y_bool(self, path: str, default=False) -> bool: + """Wrapper for y that converts value into boolean""" + return str(self.y(path, default)).lower() == 'true' + CONFIG = ConfigLoader() # pylint: disable=unused-argument -def signal_handler(sender, **kwargs): + + +def signal_handler(sender, **_): """Add all loaded config files to autoreload watcher""" for path in CONFIG.loaded_file: sender.watch_file(path) diff --git a/passbook/lib/default.yml b/passbook/lib/default.yml index b14c25110..47dcd09eb 100644 --- a/passbook/lib/default.yml +++ b/passbook/lib/default.yml @@ -1,43 +1,20 @@ # This is the default configuration file -databases: - default: - engine: 'django.db.backends.postgresql' - name: passbook - user: passbook - password: 'EK-5jnKfjrGRm<77' - host: localhost -log: - level: - console: DEBUG - file: DEBUG - file: /dev/null - syslog: - host: 127.0.0.1 - port: 514 -email: +postgresql: host: localhost - port: 25 - user: '' + name: passbook + user: postgres password: '' - use_tls: false - use_ssl: false - from: passbook -web: - server.socket_host: 0.0.0.0 - server.socket_port: 8000 - server.thread_pool: 20 - log.screen: false - log.access_file: '' - log.error_file: '' + +redis: + host: localhost + password: '' + cache_db: 0 + message_queue_db: 1 debug: false -secure_proxy_header: - HTTP_X_FORWARDED_PROTO: https -rabbitmq: guest:guest@localhost/passbook -redis: localhost/0 + # Error reporting, sends stacktrace to sentry.services.beryju.org error_report_enabled: true -secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s domains: - passbook.local diff --git a/passbook/otp/views.py b/passbook/otp/views.py index 5f92f9e32..7918c03ba 100644 --- a/passbook/otp/views.py +++ b/passbook/otp/views.py @@ -75,7 +75,7 @@ class EnableView(LoginRequiredMixin, FormView): # TODO: Check if OTP Factor exists and applies to user def get_context_data(self, **kwargs): - kwargs['config'] = CONFIG.get('passbook') + kwargs['config'] = CONFIG.y('passbook') kwargs['is_login'] = True kwargs['title'] = _('Configue OTP') kwargs['primary_action'] = _('Setup') diff --git a/passbook/root/settings.py b/passbook/root/settings.py index 3ca8adb00..72b464b42 100644 --- a/passbook/root/settings.py +++ b/passbook/root/settings.py @@ -15,19 +15,15 @@ import logging import os import sys -from celery.schedules import crontab -from django.contrib import messages +import structlog from sentry_sdk import init as sentry_init from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.django import DjangoIntegration -from sentry_sdk.integrations.logging import LoggingIntegration from passbook import __version__ from passbook.lib.config import CONFIG from passbook.lib.sentry import before_send -VERSION = __version__ - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) STATIC_ROOT = BASE_DIR + '/static' @@ -36,12 +32,13 @@ STATIC_ROOT = BASE_DIR + '/static' # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = CONFIG.get('secret_key') +SECRET_KEY = CONFIG.y('secret_key', + "9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s") # noqa Debug # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = CONFIG.get('debug') +DEBUG = CONFIG.y_bool('debug') INTERNAL_IPS = ['127.0.0.1'] -# ALLOWED_HOSTS = CONFIG.get('domains', []) + [CONFIG.get('primary_domain')] +# ALLOWED_HOSTS = CONFIG.y('domains', []) + [CONFIG.y('primary_domain')] ALLOWED_HOSTS = ['*'] SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') @@ -53,7 +50,7 @@ AUTH_USER_MODEL = 'passbook_core.User' CSRF_COOKIE_NAME = 'passbook_csrf' SESSION_COOKIE_NAME = 'passbook_session' -SESSION_COOKIE_DOMAIN = CONFIG.get('primary_domain') +SESSION_COOKIE_DOMAIN = CONFIG.y('primary_domain') SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_CACHE_ALIAS = "default" LANGUAGE_COOKIE_NAME = 'passbook_language' @@ -72,8 +69,8 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.postgres', - 'rest_framework', - 'drf_yasg', + # 'rest_framework', + # 'drf_yasg', 'passbook.core.apps.PassbookCoreConfig', 'passbook.admin.apps.PassbookAdminConfig', 'passbook.api.apps.PassbookAPIConfig', @@ -93,16 +90,6 @@ INSTALLED_APPS = [ 'passbook.app_gw.apps.PassbookApplicationApplicationGatewayConfig', ] -# Message Tag fix for bootstrap CSS Classes -MESSAGE_TAGS = { - messages.DEBUG: 'primary', - messages.INFO: 'info', - messages.SUCCESS: 'success', - messages.WARNING: 'warning', - messages.ERROR: 'danger', -} - - REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. @@ -114,17 +101,20 @@ REST_FRAMEWORK = { CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://%s" % CONFIG.get('redis'), + "LOCATION": f"redis://{CONFIG.y('redis.host')}:6379/{CONFIG.y('redis.cache_db')}", "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", } } } +DJANGO_REDIS_IGNORE_EXCEPTIONS = True +DJANGO_REDIS_LOG_IGNORED_EXCEPTIONS = True +SESSION_ENGINE = "django.contrib.sessions.backends.cache" +SESSION_CACHE_ALIAS = "default" MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'passbook.app_gw.middleware.ApplicationGatewayMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -150,22 +140,20 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'passbook.core.wsgi.application' - +WSGI_APPLICATION = 'passbook.root.wsgi.application' # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases -DATABASES = {} -for db_alias, db_config in CONFIG.get('databases').items(): - DATABASES[db_alias] = { - 'ENGINE': db_config.get('engine'), - 'HOST': db_config.get('host'), - 'NAME': db_config.get('name'), - 'USER': db_config.get('user'), - 'PASSWORD': db_config.get('password'), - 'OPTIONS': db_config.get('options', {}), +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'HOST': CONFIG.y('postgresql.host'), + 'NAME': CONFIG.y('postgresql.name'), + 'USER': CONFIG.y('postgresql.user'), + 'PASSWORD': CONFIG.y('postgresql.password'), } +} # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators @@ -203,20 +191,13 @@ USE_TZ = True # Celery settings # Add a 10 minute timeout to all Celery tasks. CELERY_TASK_SOFT_TIME_LIMIT = 600 -CELERY_TIMEZONE = TIME_ZONE CELERY_BEAT_SCHEDULE = {} CELERY_CREATE_MISSING_QUEUES = True CELERY_TASK_DEFAULT_QUEUE = 'passbook' -CELERY_BROKER_URL = 'amqp://%s' % CONFIG.get('rabbitmq') -CELERY_RESULT_BACKEND = 'rpc://' -CELERY_ACKS_LATE = True -CELERY_BROKER_HEARTBEAT = 0 -CELERY_BEAT_SCHEDULE = { - 'cleanup-expired-nonces': { - 'task': 'passbook.core.tasks.clean_nonces', - 'schedule': crontab(hour=1, minute=1) - } -} +CELERY_BROKER_URL = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}" + f":6379/{CONFIG.y('redis.message_queue_db')}") +CELERY_RESULT_BACKEND = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}" + f":6379/{CONFIG.y('redis.message_queue_db')}") if not DEBUG: @@ -224,11 +205,7 @@ if not DEBUG: dsn="https://33cdbcb23f8b436dbe0ee06847410b67@sentry.beryju.org/3", integrations=[ DjangoIntegration(), - CeleryIntegration(), - LoggingIntegration( - level=logging.INFO, - event_level=logging.ERROR - ) + CeleryIntegration() ], send_default_pii=True, before_send=before_send, @@ -240,96 +217,77 @@ if not DEBUG: STATIC_URL = '/static/' + +structlog.configure_once( + processors=[ + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.TimeStamper(), + structlog.processors.StackInfoRenderer(), + # structlog.processors.format_exc_info, + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + context_class=structlog.threadlocal.wrap_dict(dict), + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, +) + +LOG_PRE_CHAIN = [ + # Add the log level and a timestamp to the event_dict if the log entry + # is not from structlog. + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(), +] + with CONFIG.cd('log'): + LOGGING_HANDLER_MAP = { + 'passbook': 'DEBUG', + 'django': 'WARNING', + 'celery': 'WARNING', + 'grpc': 'DEBUG', + 'oauthlib': 'DEBUG', + 'oauth2_provider': 'DEBUG', + 'daphne': 'INFO', + } LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'formatters': { - 'verbose': { - 'format': ('%(asctime)s %(levelname)-8s %(name)-55s ' - '%(funcName)-20s %(message)s'), + "plain": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.processors.JSONRenderer(), + "foreign_pre_chain": LOG_PRE_CHAIN, + }, + "colored": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.dev.ConsoleRenderer(colors=DEBUG), + "foreign_pre_chain": LOG_PRE_CHAIN, }, - 'color': { - '()': 'colorlog.ColoredFormatter', - 'format': ('%(log_color)s%(asctime)s %(levelname)-8s %(name)-55s ' - '%(funcName)-20s %(message)s'), - 'log_colors': { - 'DEBUG': 'bold_black', - 'INFO': 'white', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'bold_red', - 'SUCCESS': 'green', - }, - } }, 'handlers': { 'console': { - 'level': CONFIG.get('level').get('console'), + 'level': DEBUG, 'class': 'logging.StreamHandler', - 'formatter': 'color', - }, - 'syslog': { - 'level': CONFIG.get('level').get('file'), - 'class': 'logging.handlers.SysLogHandler', - 'formatter': 'verbose', - 'address': (CONFIG.get('syslog').get('host'), - CONFIG.get('syslog').get('port')) - }, - 'file': { - 'level': CONFIG.get('level').get('file'), - 'class': 'logging.FileHandler', - 'formatter': 'verbose', - 'filename': CONFIG.get('file'), + 'formatter': "colored" if DEBUG else "plain", }, 'queue': { - 'level': CONFIG.get('level').get('console'), + 'level': DEBUG, 'class': 'passbook.lib.log.QueueListenerHandler', 'handlers': [ 'cfg://handlers.console', - # 'cfg://handlers.syslog', - 'cfg://handlers.file', ], } }, 'loggers': { - 'passbook': { - 'handlers': ['queue'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'django': { - 'handlers': ['queue'], - 'level': 'INFO', - 'propagate': True, - }, - 'tasks': { - 'handlers': ['queue'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'cherrypy': { - 'handlers': ['queue'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'oauthlib': { - 'handlers': ['queue'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'oauth2_provider': { - 'handlers': ['queue'], - 'level': 'DEBUG', - 'propagate': True, - }, - 'daphne': { - 'handlers': ['queue'], - 'level': 'INFO', - 'propagate': True, - } } } + for handler_name, level in LOGGING_HANDLER_MAP.items(): + LOGGING['loggers'][handler_name] = { + 'handlers': ['console'], + 'level': level, + 'propagate': True, + } TEST = False TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' @@ -342,6 +300,7 @@ if any('test' in arg for arg in sys.argv): TEST = True CELERY_TASK_ALWAYS_EAGER = True + _DISALLOWED_ITEMS = ['INSTALLED_APPS', 'MIDDLEWARE', 'AUTHENTICATION_BACKENDS'] # Load subapps's INSTALLED_APPS for _app in INSTALLED_APPS: