events: catch errors during send and re-raise as custom type

authentik/events/models.py

@@ -1,6 +1,6 @@
 """authentik events models"""
-
 from inspect import getmodule, stack
+from smtplib import SMTPException
 from typing import Optional, Union
 from uuid import uuid4
 
@@ -9,7 +9,7 @@ from django.core.exceptions import ValidationError
 from django.db import models
 from django.http import HttpRequest
 from django.utils.translation import gettext as _
-from requests import post
+from requests import RequestException, post
 from structlog.stdlib import get_logger
 
 from authentik import __version__
@@ -19,6 +19,7 @@ from authentik.core.middleware import (
 )
 from authentik.core.models import Group, User
 from authentik.events.utils import cleanse_dict, get_user, sanitize_dict
+from authentik.lib.sentry import SentryIgnoredException
 from authentik.lib.utils.http import get_client_ip
 from authentik.policies.models import PolicyBindingModel
 from authentik.stages.email.tasks import send_mail
@@ -27,6 +28,10 @@ from authentik.stages.email.utils import TemplateEmailMessage
 LOGGER = get_logger("authentik.events")
 
 
+class NotificationTransportError(SentryIgnoredException):
+    """Error raised when a notification fails to be delivered"""
+
+
 class EventAction(models.TextChoices):
     """All possible actions to save into the events log"""
 
@@ -192,6 +197,7 @@ class NotificationTransport(models.Model):
 
     def send_webhook(self, notification: "Notification") -> list[str]:
         """Send notification to generic webhook"""
+        try:
             response = post(
                 self.webhook_url,
                 json={
@@ -199,6 +205,9 @@ class NotificationTransport(models.Model):
                     "severity": notification.severity,
                 },
             )
+            response.raise_for_status()
+        except RequestException as exc:
+            raise NotificationTransportError from exc
         return [
             response.status_code,
             response.text,
@@ -235,7 +244,11 @@ class NotificationTransport(models.Model):
         if notification.event:
             body["attachments"][0]["title"] = notification.event.action
             body["attachments"][0]["text"] = notification.event.action
+        try:
             response = post(self.webhook_url, json=body)
+            response.raise_for_status()
+        except RequestException as exc:
+            raise NotificationTransportError from exc
         return [
             response.status_code,
             response.text,
@@ -257,8 +270,11 @@ class NotificationTransport(models.Model):
             },
         )
         # Email is sent directly here, as the call to send() should have been from a task.
+        try:
             # pyright: reportGeneralTypeIssues=false
             return send_mail(mail.__dict__)  # pylint: disable=no-value-for-parameter
+        except (SMTPException, ConnectionError) as exc:
+            raise NotificationTransportError from exc
 
     class Meta:
 

authentik/events/tasks.py

@@ -6,6 +6,7 @@ from authentik.events.models import (
     Event,
     Notification,
     NotificationTransport,
+    NotificationTransportError,
     NotificationTrigger,
 )
 from authentik.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
@@ -64,7 +65,12 @@ def event_trigger_handler(event_uuid: str, trigger_name: str):
             )
 
 
-@CELERY_APP.task(bind=True, base=MonitoredTask)
+@CELERY_APP.task(
+    bind=True,
+    autoretry_for=(NotificationTransportError),
+    retry_backoff=True,
+    base=MonitoredTask,
+)
 def notification_transport(
     self: MonitoredTask, notification_pk: int, transport_pk: int
 ):
@@ -77,6 +83,6 @@ def notification_transport(
         )
         transport.send(notification)
         self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL))
-    except Exception as exc:
+    except NotificationTransportError as exc:
         self.set_status(TaskResult(TaskResultStatus.ERROR).with_error(exc))
         raise exc