sources/ldap(major): add sync_users and sync_groups, rewrite auth_user method

passbook/sources/ldap/auth.py

@@ -1,5 +1,6 @@
 """passbook LDAP Authentication Backend"""
 from django.contrib.auth.backends import ModelBackend
+from django.http import HttpRequest
 from structlog import get_logger
 
 from passbook.sources.ldap.connector import Connector
@@ -11,11 +12,12 @@ LOGGER = get_logger()
 class LDAPBackend(ModelBackend):
     """Authenticate users against LDAP Server"""
 
-    def authenticate(self, **kwargs):
+    def authenticate(self, request: HttpRequest, **kwargs):
         """Try to authenticate a user via ldap"""
         if 'password' not in kwargs:
             return None
         for source in LDAPSource.objects.filter(enabled=True):
+            LOGGER.debug("LDAP Auth attempt", source=source)
             _ldap = Connector(source)
             user = _ldap.auth_user(**kwargs)
             if user:

passbook/sources/ldap/connector.py

@@ -1,19 +1,15 @@
 """Wrapper for ldap3 to easily manage user"""
-from time import time
+from typing import Any, Dict, Optional
 
 import ldap3
 import ldap3.core.exceptions
 from structlog import get_logger
 
-from passbook.core.models import User
+from passbook.core.models import Group, User
-from passbook.lib.config import CONFIG
 from passbook.sources.ldap.models import LDAPSource
 
 LOGGER = get_logger()
 
-# USERNAME_FIELD = CONFIG.y('ldap.username_field', 'sAMAccountName')
-# LOGIN_FIELD = CONFIG.y('ldap.login_field', 'userPrincipalName')
-
 
 class Connector:
     """Wrapper for ldap3 to easily manage user authentication and creation"""
@@ -24,178 +20,103 @@ class Connector:
 
     def __init__(self, source: LDAPSource):
         self._source = source
-
-        if not self._source.enabled:
-            LOGGER.debug("LDAP not Enabled")
-
         self._server = ldap3.Server(source.server_uri) # Implement URI parsing
+
+    def bind(self):
+        """Bind using Source's Credentials"""
         self._connection = ldap3.Connection(self._server, raise_exceptions=True,
-                                            user=source.bind_cn,
+                                            user=self._source.bind_cn,
-                                            password=source.bind_password)
+                                            password=self._source.bind_password)
 
         self._connection.bind()
-        if source.start_tls:
+        if self._source.start_tls:
             self._connection.start_tls()
 
     @staticmethod
-    def encode_pass(password: str) -> str:
+    def encode_pass(password: str) -> bytes:
         """Encodes a plain-text password so it can be used by AD"""
         return '"{}"'.format(password).encode('utf-16-le')
 
-    def generate_filter(self, **fields):
+    @property
-        """Generate LDAP filter from **fields."""
+    def base_dn_users(self) -> str:
-        filters = []
+        """Shortcut to get full base_dn for user lookups"""
-        for item, value in fields.items():
+        return ','.join([self._source.additional_user_dn, self._source.base_dn])
-            filters.append("(%s=%s)" % (item, value))
-        ldap_filter = "(&%s)" % "".join(filters)
-        LOGGER.debug("Constructed filter: '%s'", ldap_filter)
-        return ldap_filter
 
-    def lookup(self, ldap_filter: str):
+    @property
-        """Search email in LDAP and return the DN.
+    def base_dn_groups(self) -> str:
-        Returns False if nothing was found."""
+        """Shortcut to get full base_dn for group lookups"""
-        try:
+        return ','.join([self._source.additional_group_dn, self._source.base_dn])
-            self._connection.search(self._source.search_base, ldap_filter)
-            results = self._connection.response
-            if len(results) >= 1:
-                if 'dn' in results[0]:
-                    return str(results[0]['dn'])
-        except ldap3.core.exceptions.LDAPNoSuchObjectResult as exc:
-            LOGGER.warning(exc)
-            return False
-        except ldap3.core.exceptions.LDAPInvalidDnError as exc:
-            LOGGER.warning(exc)
-            return False
-        return False
 
-    def _get_or_create_user(self, user_data):
+    def sync_groups(self):
-        """Returns a Django user for the given LDAP user data.
+        """Iterate over all LDAP Groups and create passbook_core.Group instances"""
-        If the user does not exist, then it will be created."""
+        attributes = [
-        attributes = user_data.get("attributes")
+            'objectSid', # Used as unique Identifier
-        if attributes is None:
+            'name',
-            LOGGER.warning("LDAP user attributes empty")
+            'dn',
-            return None
+        ]
-        # Create the user data.
+        groups = self._connection.extend.standard.paged_search(
-        field_map = {
+            search_base=self.base_dn_groups,
-            'username': '%(' + ')s',
+            search_filter=self._source.group_object_filter,
-            'name': '%(givenName)s %(sn)s',
+            search_scope=ldap3.SUBTREE,
-            'email': '%(mail)s',
+            attributes=ldap3.ALL_ATTRIBUTES)
-        }
+        for group in groups:
-        user_fields = {}
+            attributes = group.get('attributes', {})
-        for dj_field, ldap_field in field_map.items():
+            _, created = Group.objects.update_or_create(
-            user_fields[dj_field] = ldap_field % attributes
+                attributes__objectSid=attributes.get('objectSid', ''),
-
+                defaults=self._build_object_properties(attributes),
-        # Update or create the user.
-        user, created = User.objects.update_or_create(
-            defaults=user_fields,
-            username=user_fields.pop('username', "")
             )
+            LOGGER.debug("Synced group", group=attributes.get('name', ''), created=created)
 
-        # Update groups
+    def sync_users(self):
-        # if 'memberOf' in attributes:
+        """Iterate over all LDAP Users and create passbook_core.User instances"""
-        #     applicable_groups = LDAPGroupMapping.objects.f
+        users = self._connection.extend.standard.paged_search(
-        # ilter(ldap_dn__in=attributes['memberOf'])
+            search_base=self.base_dn_users,
-        #     for group in applicable_groups:
+            search_filter=self._source.user_object_filter,
-        #         if group.group not in user.groups.all():
+            search_scope=ldap3.SUBTREE,
-        #             user.groups.add(group.group)
+            attributes=ldap3.ALL_ATTRIBUTES)
-        #             user.save()
+        for user in users:
+            attributes = user.get('attributes', {})
+            _, created = User.objects.update_or_create(
+                attributes__objectSid=attributes.get('objectSid', ''),
+                defaults=self._build_object_properties(attributes),
+            )
+            LOGGER.debug("Synced User", user=attributes.get('name', ''), created=created)
 
-        # If the user was created, set them an unusable password.
+    def sync_membership(self):
-        if created:
+        """Iterate over all Users and assign Groups using memberOf Field"""
-            user.set_unusable_password()
+        pass
-            user.save()
-        # All done!
-        LOGGER.debug("LDAP user lookup succeeded")
-        return user
 
-    def auth_user(self, password, **filters):
+    def _build_object_properties(self, attributes: Dict[str, Any]) -> Dict[str, Dict[Any, Any]]:
+        properties = {
+            'attributes': {}
+        }
+        for mapping in self._source.property_mappings.all().select_subclasses():
+            properties[mapping.object_field] = attributes.get(mapping.ldap_property, '')
+        if 'objectSid' in attributes:
+            properties['attributes']['objectSid'] = attributes.get('objectSid')
+        properties['attributes']['distinguishedName'] = attributes.get('distinguishedName')
+        return properties
+
+    def auth_user(self, password: str, **filters: Dict[str, str]) -> Optional[User]:
         """Try to bind as either user_dn or mail with password.
         Returns True on success, otherwise False"""
-        filters.pop('request')
+        users = User.objects.filter(**filters)
-        if not self._source.enabled:
+        if not users.exists():
             return None
-        # FIXME: Adapt user_uid
+        user = users.first()
-        # email = filters.pop(CONFIG.y('passport').get('ldap').get, '')
+        if 'distinguishedName' not in user.attributes:
-        email = filters.pop('email')
+            LOGGER.debug("User doesn't have DN set, assuming not LDAP imported.", user=user)
-        user_dn = self.lookup(self.generate_filter(**{'email': email}))
-        if not user_dn:
             return None
         # Try to bind as new user
-        LOGGER.debug("Binding as '%s'", user_dn)
+        LOGGER.debug("Attempting Binding as user", user=user)
         try:
-            temp_connection = ldap3.Connection(self._server, user=user_dn,
+            temp_connection = ldap3.Connection(self._server,
+                                               user=user.attributes.get('distinguishedName'),
                                                password=password, raise_exceptions=True)
             temp_connection.bind()
-            if self._connection.search(
+            return user
-                    search_base=self._source.search_base,
-                    search_filter=self.generate_filter(**{'email': email}),
-                    search_scope=ldap3.SUBTREE,
-                    attributes=[ldap3.ALL_ATTRIBUTES, ldap3.ALL_OPERATIONAL_ATTRIBUTES],
-                    get_operational_attributes=True,
-                    size_limit=1,
-            ):
-                response = self._connection.response[0]
-                # If user has no email set in AD, use UPN
-                if 'mail' not in response.get('attributes'):
-                    response['attributes']['mail'] = response['attributes']['userPrincipalName']
-                return self._get_or_create_user(response)
-            LOGGER.warning("LDAP user lookup failed")
-            return None
         except ldap3.core.exceptions.LDAPInvalidCredentialsResult as exception:
-            LOGGER.debug("User '%s' failed to login (Wrong credentials)", user_dn)
+            LOGGER.debug("LDAPInvalidCredentialsResult", user=user)
         except ldap3.core.exceptions.LDAPException as exception:
             LOGGER.warning(exception)
         return None
-
-    def _do_modify(self, diff, **fields):
-        """Do the LDAP modification itself"""
-        user_dn = self.lookup(self.generate_filter(**fields))
-        try:
-            self._connection.modify(user_dn, diff)
-        except ldap3.core.exceptions.LDAPException as exception:
-            LOGGER.warning("Failed to modify %s ('%s'), saved to DB", user_dn, exception)
-            # Connector.handle_ldap_error(user_dn, LDAPModification.ACTION_MODIFY, diff)
-        LOGGER.debug("modified account '%s' [%s]", user_dn, ','.join(diff.keys()))
-        return 'result' in self._connection.result and self._connection.result['result'] == 0
-
-    def disable_user(self, **fields):
-        """Disables LDAP user based on mail or user_dn.
-        Returns True on success, otherwise False"""
-        diff = {
-            'userAccountControl': [(ldap3.MODIFY_REPLACE, [str(66050)])],
-        }
-        return self._do_modify(diff, **fields)
-
-    def enable_user(self, **fields):
-        """Enables LDAP user based on mail or user_dn.
-        Returns True on success, otherwise False"""
-        diff = {
-            'userAccountControl': [(ldap3.MODIFY_REPLACE, [str(66048)])],
-        }
-        return self._do_modify(diff, **fields)
-
-    def change_password(self, new_password, **fields):
-        """Changes LDAP user's password based on mail or user_dn.
-        Returns True on success, otherwise False"""
-        diff = {
-            'unicodePwd': [(ldap3.MODIFY_REPLACE, [Connector.encode_pass(new_password)])],
-        }
-        return self._do_modify(diff, **fields)
-
-    def add_to_group(self, group_dn, **fields):
-        """Adds mail or user_dn to group_dn
-        Returns True on success, otherwise False"""
-        user_dn = self.lookup(**fields)
-        diff = {
-            'member': [(ldap3.MODIFY_ADD), [user_dn]]
-        }
-        return self._do_modify(diff, user_dn=group_dn)
-
-    def remove_from_group(self, group_dn, **fields):
-        """Removes mail or user_dn from group_dn
-        Returns True on success, otherwise False"""
-        user_dn = self.lookup(**fields)
-        diff = {
-            'member': [(ldap3.MODIFY_DELETE), [user_dn]]
-        }
-        return self._do_modify(diff, user_dn=group_dn)

passbook/sources/ldap/forms.py

@@ -5,7 +5,7 @@ from django.contrib.admin.widgets import FilteredSelectMultiple
 from django.utils.translation import gettext_lazy as _
 
 from passbook.admin.forms.source import SOURCE_FORM_FIELDS
-from passbook.sources.ldap.models import LDAPSource
+from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
 
 
 class LDAPSourceForm(forms.ModelForm):
@@ -26,6 +26,7 @@ class LDAPSourceForm(forms.ModelForm):
             'group_object_filter',
             'sync_groups',
             'sync_parent_group',
+            'property_mappings',
         ]
         widgets = {
             'name': forms.TextInput(),
@@ -37,7 +38,8 @@ class LDAPSourceForm(forms.ModelForm):
             'additional_group_dn': forms.TextInput(),
             'user_object_filter': forms.TextInput(),
             'group_object_filter': forms.TextInput(),
-            'policies': FilteredSelectMultiple(_('policies'), False)
+            'policies': FilteredSelectMultiple(_('policies'), False),
+            'property_mappings': FilteredSelectMultiple(_('Property Mappings'), False)
         }
         labels = {
             'server_uri': _('Server URI'),
@@ -47,3 +49,17 @@ class LDAPSourceForm(forms.ModelForm):
             'additional_user_dn': _('Addition User DN'),
             'additional_group_dn': _('Addition Group DN'),
         }
+
+
+class LDAPPropertyMappingForm(forms.ModelForm):
+    """LDAP Property Mapping form"""
+
+    class Meta:
+
+        model = LDAPPropertyMapping
+        fields = ['name', 'ldap_property', 'object_field']
+        widgets = {
+            'name': forms.TextInput(),
+            'ldap_property': forms.TextInput(),
+            'object_field': forms.TextInput(),
+        }

passbook/sources/ldap/migrations/0002_auto_20191011_0825.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.6 on 2019-10-11 08:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('passbook_sources_ldap', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='ldappropertymapping',
+            options={'verbose_name': 'LDAP Property Mapping', 'verbose_name_plural': 'LDAP Property Mappings'},
+        ),
+    ]

passbook/sources/ldap/migrations/0003_auto_20191011_0825.py

@@ -0,0 +1,32 @@
+# Generated by Django 2.2.6 on 2019-10-11 08:25
+
+from django.apps.registry import Apps
+from django.db import migrations
+
+
+def create_default_ad_property_mappings(apps: Apps, schema_editor):
+    LDAPPropertyMapping = apps.get_model('passbook_sources_ldap', 'LDAPPropertyMapping')
+    mapping = {
+        'name': 'name',
+        'givenName': 'first_name',
+        'sn': 'last_name',
+        'sAMAccountName': 'username',
+        'mail': 'email'
+    }
+    for ldap_property, object_field in mapping.items():
+        LDAPPropertyMapping.objects.get_or_create(
+            ldap_property=ldap_property,
+            object_field=object_field,
+            defaults={
+                'name': f"Autogenerated LDAP Mapping: {ldap_property} -> {object_field}"
+            })
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('passbook_sources_ldap', '0002_auto_20191011_0825'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_default_ad_property_mappings)
+    ]

passbook/sources/ldap/migrations/0004_auto_20191011_0839.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.6 on 2019-10-11 08:39
+
+import django.core.validators
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('passbook_sources_ldap', '0003_auto_20191011_0825'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ldapsource',
+            name='server_uri',
+            field=models.TextField(validators=[django.core.validators.URLValidator(schemes=['ldap', 'ldaps'])]),
+        ),
+        migrations.AlterField(
+            model_name='ldapsource',
+            name='sync_parent_group',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='passbook_core.Group'),
+        ),
+    ]

passbook/sources/ldap/models.py

@@ -10,7 +10,7 @@ from passbook.core.models import Group, PropertyMapping, Source
 class LDAPSource(Source):
     """LDAP Authentication source"""
 
-    server_uri = models.URLField(validators=[URLValidator(schemes=['ldap', 'ldaps'])])
+    server_uri = models.TextField(validators=[URLValidator(schemes=['ldap', 'ldaps'])])
     bind_cn = models.TextField()
     bind_password = models.TextField()
     start_tls = models.BooleanField(default=False)
@@ -23,7 +23,7 @@ class LDAPSource(Source):
     group_object_filter = models.TextField()
 
     sync_groups = models.BooleanField(default=True)
-    sync_parent_group = models.ForeignKey(Group, blank=True,
+    sync_parent_group = models.ForeignKey(Group, blank=True, null=True,
                                           default=None, on_delete=models.SET_DEFAULT)
 
     form = 'passbook.sources.ldap.forms.LDAPSourceForm'
@@ -39,6 +39,17 @@ class LDAPSource(Source):
 
 
 class LDAPPropertyMapping(PropertyMapping):
+    """Map LDAP Property to User or Group object"""
 
     ldap_property = models.TextField()
     object_field = models.TextField()
+
+    form = 'passbook.sources.ldap.forms.LDAPPropertyMappingForm'
+
+    def __str__(self):
+        return f"LDAP Property Mapping {self.ldap_property} -> {self.object_field}"
+
+    class Meta:
+
+        verbose_name = _('LDAP Property Mapping')
+        verbose_name_plural = _('LDAP Property Mappings')

passbook/sources/ldap/settings.py

@@ -1,5 +1,13 @@
 """LDAP Settings"""
+from celery.schedules import crontab
 
 AUTHENTICATION_BACKENDS = [
     'passbook.sources.ldap.auth.LDAPBackend',
 ]
+
+CELERY_BEAT_SCHEDULE = {
+    'sync': {
+        'task': 'passbook.sources.ldap.tasks.sync',
+        'schedule': crontab(minute=0)  # Run every hour
+    }
+}

passbook/sources/ldap/tasks.py

@@ -0,0 +1,30 @@
+from passbook.root.celery import CELERY_APP
+from passbook.sources.ldap.connector import Connector
+from passbook.sources.ldap.models import LDAPSource
+
+
+@CELERY_APP.task()
+def sync_groups(source_pk: int):
+    """Sync LDAP Groups on background worker"""
+    source = LDAPSource.objects.get(pk=source_pk)
+    connector = Connector(source)
+    connector.bind()
+    connector.sync_groups()
+
+@CELERY_APP.task()
+def sync_users(source_pk: int):
+    """Sync LDAP Users on background worker"""
+    source = LDAPSource.objects.get(pk=source_pk)
+    connector = Connector(source)
+    connector.bind()
+    connector.sync_users()
+
+@CELERY_APP.task()
+def sync():
+    """Sync all sources"""
+    for source in LDAPSource.objects.filter(enabled=True):
+        connector = Connector(source)
+        connector.bind()
+        connector.sync_users()
+        connector.sync_groups()
+        connector.sync_membership()