From 1adc6948b42bbf6485a806c9cffc0d3127073704 Mon Sep 17 00:00:00 2001 From: Jens L Date: Wed, 17 Aug 2022 22:00:47 +0100 Subject: [PATCH] blueprints: allow for adding remote blueprints (#3435) * allow blueprints to be fetched from HTTP URLs Signed-off-by: Jens Langhammer * fix tests Signed-off-by: Jens Langhammer * remove os.path Signed-off-by: Jens Langhammer * add validation for blueprint path Signed-off-by: Jens Langhammer * fix tests Signed-off-by: Jens Langhammer Signed-off-by: Jens Langhammer --- authentik/blueprints/api.py | 11 +++++- .../management/commands/apply_blueprint.py | 17 +++++----- authentik/blueprints/models.py | 24 +++++++++++++ authentik/blueprints/tests/__init__.py | 7 ++-- authentik/blueprints/tests/test_bundled.py | 8 +++-- authentik/blueprints/tests/test_v1_tasks.py | 20 +++++++---- authentik/blueprints/v1/tasks.py | 34 +++++++++++++++---- authentik/outposts/tasks.py | 7 ++-- tests/e2e/test_source_oauth.py | 4 +-- 9 files changed, 97 insertions(+), 35 deletions(-) diff --git a/authentik/blueprints/api.py b/authentik/blueprints/api.py index 7f6e153e3..ae976f48e 100644 --- a/authentik/blueprints/api.py +++ b/authentik/blueprints/api.py @@ -1,6 +1,7 @@ """Serializer mixin for managed models""" from drf_spectacular.utils import extend_schema, inline_serializer from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError from rest_framework.fields import CharField, DateTimeField, JSONField from rest_framework.permissions import IsAdminUser from rest_framework.request import Request @@ -9,7 +10,7 @@ from rest_framework.serializers import ListSerializer, ModelSerializer from rest_framework.viewsets import ModelViewSet from authentik.api.decorators import permission_required -from authentik.blueprints.models import BlueprintInstance +from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed from authentik.blueprints.v1.tasks import apply_blueprint, blueprints_find_dict from authentik.core.api.used_by import UsedByMixin from authentik.core.api.utils import PassiveSerializer @@ -31,6 +32,14 @@ class MetadataSerializer(PassiveSerializer): class BlueprintInstanceSerializer(ModelSerializer): """Info about a single blueprint instance file""" + def validate_path(self, path: str) -> str: + """Ensure the path specified is retrievable""" + try: + BlueprintInstance(path=path).retrieve() + except BlueprintRetrievalFailed as exc: + raise ValidationError(exc) from exc + return path + class Meta: model = BlueprintInstance diff --git a/authentik/blueprints/management/commands/apply_blueprint.py b/authentik/blueprints/management/commands/apply_blueprint.py index d21ecf571..e00e0e0a0 100644 --- a/authentik/blueprints/management/commands/apply_blueprint.py +++ b/authentik/blueprints/management/commands/apply_blueprint.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand, no_translations from structlog.stdlib import get_logger +from authentik.blueprints.models import BlueprintInstance from authentik.blueprints.v1.importer import Importer LOGGER = get_logger() @@ -14,14 +15,14 @@ class Command(BaseCommand): def handle(self, *args, **options): """Apply all blueprints in order, abort when one fails to import""" for blueprint_path in options.get("blueprints", []): - with open(blueprint_path, "r", encoding="utf8") as blueprint_file: - importer = Importer(blueprint_file.read()) - valid, logs = importer.validate() - if not valid: - for log in logs: - LOGGER.debug(**log) - raise ValueError("blueprint invalid") - importer.apply() + content = BlueprintInstance(path=blueprint_path).retrieve() + importer = Importer(content) + valid, logs = importer.validate() + if not valid: + for log in logs: + LOGGER.debug(**log) + raise ValueError("blueprint invalid") + importer.apply() def add_arguments(self, parser): parser.add_argument("blueprints", nargs="+", type=str) diff --git a/authentik/blueprints/models.py b/authentik/blueprints/models.py index 59da385f3..74f2fcb76 100644 --- a/authentik/blueprints/models.py +++ b/authentik/blueprints/models.py @@ -1,12 +1,23 @@ """Managed Object models""" +from pathlib import Path +from urllib.parse import urlparse from uuid import uuid4 from django.contrib.postgres.fields import ArrayField from django.db import models from django.utils.translation import gettext_lazy as _ +from requests import RequestException from rest_framework.serializers import Serializer +from authentik.lib.config import CONFIG from authentik.lib.models import CreatedUpdatedModel, SerializerModel +from authentik.lib.sentry import SentryIgnoredException +from authentik.lib.utils.http import get_http_session + + +class BlueprintRetrievalFailed(SentryIgnoredException): + """Error raised when we're unable to fetch the blueprint contents, whether it be HTTP files + not being accessible or local files not being readable""" class ManagedModel(models.Model): @@ -60,6 +71,19 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel): enabled = models.BooleanField(default=True) managed_models = ArrayField(models.TextField(), default=list) + def retrieve(self) -> str: + """Retrieve blueprint contents""" + if urlparse(self.path).scheme != "": + try: + res = get_http_session().get(self.path, timeout=3, allow_redirects=True) + res.raise_for_status() + return res.text + except RequestException as exc: + raise BlueprintRetrievalFailed(exc) from exc + path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path)) + with path.open("r", encoding="utf-8") as _file: + return _file.read() + @property def serializer(self) -> Serializer: from authentik.blueprints.api import BlueprintInstanceSerializer diff --git a/authentik/blueprints/tests/__init__.py b/authentik/blueprints/tests/__init__.py index 9d83d7703..e7ee2597e 100644 --- a/authentik/blueprints/tests/__init__.py +++ b/authentik/blueprints/tests/__init__.py @@ -6,6 +6,7 @@ from typing import Callable from django.apps import apps from authentik.blueprints.manager import ManagedAppConfig +from authentik.blueprints.models import BlueprintInstance from authentik.lib.config import CONFIG @@ -19,11 +20,9 @@ def apply_blueprint(*files: str): @wraps(func) def wrapper(*args, **kwargs): - base_path = Path(CONFIG.y("blueprints_dir")) for file in files: - full_path = Path(base_path, file) - with full_path.open("r", encoding="utf-8") as _file: - Importer(_file.read()).apply() + content = BlueprintInstance(path=file).retrieve() + Importer(content).apply() return func(*args, **kwargs) return wrapper diff --git a/authentik/blueprints/tests/test_bundled.py b/authentik/blueprints/tests/test_bundled.py index 889770919..300cd16c1 100644 --- a/authentik/blueprints/tests/test_bundled.py +++ b/authentik/blueprints/tests/test_bundled.py @@ -4,6 +4,7 @@ from typing import Callable from django.test import TransactionTestCase +from authentik.blueprints.models import BlueprintInstance from authentik.blueprints.tests import apply_blueprint from authentik.blueprints.v1.importer import Importer from authentik.tenants.models import Tenant @@ -18,12 +19,13 @@ class TestBundled(TransactionTestCase): self.assertTrue(Tenant.objects.filter(domain="authentik-default").exists()) -def blueprint_tester(file_name: str) -> Callable: +def blueprint_tester(file_name: Path) -> Callable: """This is used instead of subTest for better visibility""" def tester(self: TestBundled): - with open(file_name, "r", encoding="utf8") as blueprint: - importer = Importer(blueprint.read()) + base = Path("blueprints/") + rel_path = Path(file_name).relative_to(base) + importer = Importer(BlueprintInstance(path=str(rel_path)).retrieve()) self.assertTrue(importer.validate()[0]) self.assertTrue(importer.apply()) diff --git a/authentik/blueprints/tests/test_v1_tasks.py b/authentik/blueprints/tests/test_v1_tasks.py index ca6a94741..909b12d5f 100644 --- a/authentik/blueprints/tests/test_v1_tasks.py +++ b/authentik/blueprints/tests/test_v1_tasks.py @@ -1,4 +1,5 @@ """Test blueprints v1 tasks""" +from hashlib import sha512 from tempfile import NamedTemporaryFile, mkdtemp from django.test import TransactionTestCase @@ -36,25 +37,32 @@ class TestBlueprintsV1Tasks(TransactionTestCase): @CONFIG.patch("blueprints_dir", TMP) def test_valid(self): """Test valid file""" + blueprint_id = generate_id() with NamedTemporaryFile(mode="w+", suffix=".yaml", dir=TMP) as file: file.write( dump( { "version": 1, "entries": [], + "metadata": { + "name": blueprint_id, + }, } ) ) + file.seek(0) + file_hash = sha512(file.read().encode()).hexdigest() file.flush() blueprints_discover() # pylint: disable=no-value-for-parameter + instance = BlueprintInstance.objects.filter(name=blueprint_id).first() + self.assertEqual(instance.last_applied_hash, file_hash) self.assertEqual( - BlueprintInstance.objects.first().last_applied_hash, - ( - "e52bb445b03cd36057258dc9f0ce0fbed8278498ee1470e45315293e5f026d1b" - "d1f9b3526871c0003f5c07be5c3316d9d4a08444bd8fed1b3f03294e51e44522" - ), + instance.metadata, + { + "name": blueprint_id, + "labels": {}, + }, ) - self.assertEqual(BlueprintInstance.objects.first().metadata, {}) @CONFIG.patch("blueprints_dir", TMP) def test_valid_updated(self): diff --git a/authentik/blueprints/v1/tasks.py b/authentik/blueprints/v1/tasks.py index 4b7b41af8..9ad6810a4 100644 --- a/authentik/blueprints/v1/tasks.py +++ b/authentik/blueprints/v1/tasks.py @@ -8,10 +8,15 @@ from dacite import from_dict from django.db import DatabaseError, InternalError, ProgrammingError from django.utils.timezone import now from django.utils.translation import gettext_lazy as _ +from structlog.stdlib import get_logger from yaml import load from yaml.error import YAMLError -from authentik.blueprints.models import BlueprintInstance, BlueprintInstanceStatus +from authentik.blueprints.models import ( + BlueprintInstance, + BlueprintInstanceStatus, + BlueprintRetrievalFailed, +) from authentik.blueprints.v1.common import BlueprintLoader, BlueprintMetadata from authentik.blueprints.v1.importer import Importer from authentik.blueprints.v1.labels import LABEL_AUTHENTIK_INSTANTIATE @@ -25,6 +30,8 @@ from authentik.events.utils import sanitize_dict from authentik.lib.config import CONFIG from authentik.root.celery import CELERY_APP +LOGGER = get_logger() + @dataclass class BlueprintFile: @@ -54,21 +61,29 @@ def blueprints_find(): root = Path(CONFIG.y("blueprints_dir")) for file in root.glob("**/*.yaml"): path = Path(file) + LOGGER.debug("found blueprint", path=str(path)) with open(path, "r", encoding="utf-8") as blueprint_file: try: raw_blueprint = load(blueprint_file.read(), BlueprintLoader) - except YAMLError: + except YAMLError as exc: raw_blueprint = None + LOGGER.warning("failed to parse blueprint", exc=exc, path=str(path)) if not raw_blueprint: continue metadata = raw_blueprint.get("metadata", None) version = raw_blueprint.get("version", 1) if version != 1: + LOGGER.warning("invalid blueprint version", version=version, path=str(path)) continue file_hash = sha512(path.read_bytes()).hexdigest() blueprint = BlueprintFile(path.relative_to(root), version, file_hash, path.stat().st_mtime) blueprint.meta = from_dict(BlueprintMetadata, metadata) if metadata else None blueprints.append(blueprint) + LOGGER.info( + "parsed & loaded blueprint", + hash=file_hash, + path=str(path), + ) return blueprints @@ -127,10 +142,9 @@ def apply_blueprint(self: MonitoredTask, instance_pk: str): instance: BlueprintInstance = BlueprintInstance.objects.filter(pk=instance_pk).first() if not instance or not instance.enabled: return - full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(instance.path)) - file_hash = sha512(full_path.read_bytes()).hexdigest() - with open(full_path, "r", encoding="utf-8") as blueprint_file: - importer = Importer(blueprint_file.read(), instance.context) + blueprint_content = instance.retrieve() + file_hash = sha512(blueprint_content.encode()).hexdigest() + importer = Importer(blueprint_content, instance.context) valid, logs = importer.validate() if not valid: instance.status = BlueprintInstanceStatus.ERROR @@ -148,7 +162,13 @@ def apply_blueprint(self: MonitoredTask, instance_pk: str): instance.last_applied = now() instance.save() self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL)) - except (DatabaseError, ProgrammingError, InternalError, IOError) as exc: + except ( + DatabaseError, + ProgrammingError, + InternalError, + IOError, + BlueprintRetrievalFailed, + ) as exc: instance.status = BlueprintInstanceStatus.ERROR instance.save() self.set_status(TaskResult(TaskResultStatus.ERROR).with_error(exc)) diff --git a/authentik/outposts/tasks.py b/authentik/outposts/tasks.py index ce1c713a2..d3b031b13 100644 --- a/authentik/outposts/tasks.py +++ b/authentik/outposts/tasks.py @@ -1,6 +1,5 @@ """outpost tasks""" from os import R_OK, access -from os.path import expanduser from pathlib import Path from socket import gethostname from typing import Any, Optional @@ -252,13 +251,13 @@ def outpost_local_connection(): name="Local Kubernetes Cluster", local=True, kubeconfig={} ) # For development, check for the existence of a kubeconfig file - kubeconfig_path = expanduser(KUBE_CONFIG_DEFAULT_LOCATION) - if Path(kubeconfig_path).exists(): + kubeconfig_path = Path(KUBE_CONFIG_DEFAULT_LOCATION).expanduser() + if kubeconfig_path.exists(): LOGGER.debug("Detected kubeconfig") kubeconfig_local_name = f"k8s-{gethostname()}" if not KubernetesServiceConnection.objects.filter(name=kubeconfig_local_name).exists(): LOGGER.debug("Creating kubeconfig Service Connection") - with open(kubeconfig_path, "r", encoding="utf8") as _kubeconfig: + with kubeconfig_path.open("r", encoding="utf8") as _kubeconfig: KubernetesServiceConnection.objects.create( name=kubeconfig_local_name, kubeconfig=yaml.safe_load(_kubeconfig), diff --git a/tests/e2e/test_source_oauth.py b/tests/e2e/test_source_oauth.py index 1320207ee..a7b77a14b 100644 --- a/tests/e2e/test_source_oauth.py +++ b/tests/e2e/test_source_oauth.py @@ -1,5 +1,5 @@ """test OAuth Source""" -from os.path import abspath +from pathlib import Path from sys import platform from time import sleep from typing import Any, Optional @@ -116,7 +116,7 @@ class TestSourceOAuth2(SeleniumTestCase): interval=5 * 100 * 1000000, start_period=1 * 100 * 1000000, ), - "volumes": {abspath(CONFIG_PATH): {"bind": "/config.yml", "mode": "ro"}}, + "volumes": {str(Path(CONFIG_PATH).absolute()): {"bind": "/config.yml", "mode": "ro"}}, } def create_objects(self):