sources/ldap: fix syncing large LDAP directories (#6384)

* sources/ldap: fix syncing large LDAP directories

* add test

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

---------

Signed-off-by: Jens Langhammer <jens@goauthentik.io>
Co-authored-by: Jens Langhammer <jens@goauthentik.io>

authentik/sources/ldap/tasks.py

@@ -59,7 +59,7 @@ def ldap_sync_paginator(source: LDAPSource, sync: type[BaseLDAPSynchronizer]) ->
     signatures = []
     for page in sync_inst.get_objects():
         page_cache_key = CACHE_KEY_PREFIX + str(uuid4())
-        cache.set(page_cache_key, page)
+        cache.set(page_cache_key, page, 60 * 60 * int(CONFIG.get("ldap.task_timeout_hours")))
         page_sync = ldap_sync.si(source.pk, class_to_path(sync), page_cache_key)
         signatures.append(page_sync)
     return signatures
@@ -86,6 +86,12 @@ def ldap_sync(self: MonitoredTask, source_pk: str, sync_class: str, page_cache_k
         sync_inst: BaseLDAPSynchronizer = sync(source)
         page = cache.get(page_cache_key)
         if not page:
+            error_message = (
+                f"Could not find page in cache: {page_cache_key}. "
+                + "Try increasing ldap.task_timeout_hours"
+            )
+            LOGGER.warning(error_message)
+            self.set_status(TaskResult(TaskResultStatus.ERROR, [error_message]))
             return
         cache.touch(page_cache_key)
         count = sync_inst.sync(page)

authentik/sources/ldap/tests/test_sync.py

@@ -8,12 +8,14 @@ from authentik.blueprints.tests import apply_blueprint
 from authentik.core.models import Group, User
 from authentik.core.tests.utils import create_test_admin_user
 from authentik.events.models import Event, EventAction
+from authentik.events.monitored_tasks import TaskInfo, TaskResultStatus
 from authentik.lib.generators import generate_key
+from authentik.lib.utils.reflection import class_to_path
 from authentik.sources.ldap.models import LDAPPropertyMapping, LDAPSource
 from authentik.sources.ldap.sync.groups import GroupLDAPSynchronizer
 from authentik.sources.ldap.sync.membership import MembershipLDAPSynchronizer
 from authentik.sources.ldap.sync.users import UserLDAPSynchronizer
-from authentik.sources.ldap.tasks import ldap_sync_all
+from authentik.sources.ldap.tasks import ldap_sync, ldap_sync_all
 from authentik.sources.ldap.tests.mock_ad import mock_ad_connection
 from authentik.sources.ldap.tests.mock_slapd import mock_slapd_connection
 
@@ -33,6 +35,14 @@ class LDAPSyncTests(TestCase):
             additional_group_dn="ou=groups",
         )
 
+    def test_sync_missing_page(self):
+        """Test sync with missing page"""
+        connection = MagicMock(return_value=mock_ad_connection(LDAP_PASSWORD))
+        with patch("authentik.sources.ldap.models.LDAPSource.connection", connection):
+            ldap_sync.delay(self.source.pk, class_to_path(UserLDAPSynchronizer), "foo").get()
+        status = TaskInfo.by_name("ldap_sync:ldap:users:foo")
+        self.assertEqual(status.result.status, TaskResultStatus.ERROR)
+
     def test_sync_error(self):
         """Test user sync"""
         self.source.property_mappings.set(