From 1abd846922d7cda584d8a870d1baad8cb22d5bb2 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 9 Oct 2024 12:18:09 +0200 Subject: [PATCH 1/3] search in xapian --- dashboard/mixins.py | 15 ++++++----- dashboard/templates/base.html | 11 ++++++++ dashboard/urls.py | 1 + dashboard/views.py | 50 ++++++++++++++++++++++++++++++++++- evidence/xapian.py | 1 - 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index 8b70934..40f27bf 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -28,7 +28,7 @@ class DashboardView(LoginRequiredMixin): title = "" subtitle = "" section = "" - + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ @@ -72,14 +72,17 @@ class DetailsMixin(DashboardView, TemplateView): class InventaryMixin(DashboardView, TemplateView): def post(self, request, *args, **kwargs): - dev_ids = dict(self.request.POST).get("devices", []) - self.request.session["devices"] = dev_ids - url = self.request.POST.get("url") + post = dict(self.request.POST) + url = post.get("url") + if url: + dev_ids = post.get("devices", []) + self.request.session["devices"] = dev_ids + try: - resource = resolve(url) + resource = resolve(url[0]) if resource and dev_ids: - return redirect(url) + return redirect(url[0]) except Exception: pass return super().get(request, *args, **kwargs) diff --git a/dashboard/templates/base.html b/dashboard/templates/base.html index c8b0a38..0a38562 100644 --- a/dashboard/templates/base.html +++ b/dashboard/templates/base.html @@ -179,6 +179,17 @@ {% endblock messages %}

{{ title }}

+ +
+ {% csrf_token %} +
+ + + + +
+
+
diff --git a/dashboard/urls.py b/dashboard/urls.py index 681b73e..835e708 100644 --- a/dashboard/urls.py +++ b/dashboard/urls.py @@ -6,4 +6,5 @@ app_name = 'dashboard' urlpatterns = [ path("", views.UnassignedDevicesView.as_view(), name="unassigned_devices"), path("/", views.LotDashboardView.as_view(), name="lot"), + path("search", views.SearchView.as_view(), name="search"), ] diff --git a/dashboard/views.py b/dashboard/views.py index 7a9a397..c188cc4 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -1,7 +1,12 @@ +import json + from django.utils.translation import gettext_lazy as _ +from django.views.generic.edit import FormView from django.shortcuts import Http404 from dashboard.mixins import InventaryMixin, DetailsMixin +from evidence.models import Annotation +from evidence.xapian import search from device.models import Device from lot.models import Lot @@ -32,6 +37,49 @@ class LotDashboardView(InventaryMixin, DetailsMixin): return context def get_devices(self, user, offset, limit): - chids = self.object.devicelot_set.all().values_list("device_id", flat=True).distinct() + chids = self.object.devicelot_set.all().values_list( + "device_id", flat=True + ).distinct() + chids_page = chids[offset:offset+limit] return [Device(id=x) for x in chids_page], chids.count() + + +class SearchView(InventaryMixin): + template_name = "unassigned_devices.html" + section = "Search" + title = _("Search Devices") + breadcrumb = "Devices / Search Devices" + + def get_devices(self, user, offset, limit): + post = dict(self.request.POST) + query = post.get("search") + + if not query: + return [], 0 + + matches = search( + self.request.user.institution, + query[0], + offset, + limit + ) + + annotations = [] + for x in matches: + annotations.extend(self.get_annotations(x)) + + devices = [Device(id=x) for x in set(annotations)] + count = matches.size() + return devices, count + + def get_annotations(self, xp): + snap = xp.document.get_data() + uuid = json.loads(snap).get('uuid') + + return Annotation.objects.filter( + type=Annotation.Type.SYSTEM, + owner=self.request.user.institution, + uuid=uuid + ).values_list("value", flat=True).distinct() + diff --git a/evidence/xapian.py b/evidence/xapian.py index 27c03c5..3c0361c 100644 --- a/evidence/xapian.py +++ b/evidence/xapian.py @@ -18,7 +18,6 @@ def search(institution, qs, offset=0, limit=10): qp.set_stemmer(xapian.Stem("english")) qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME) qp.add_prefix("uuid", "uuid") - # qp.add_prefix("snapshot", "snapshot") query = qp.parse_query(qs) institution_term = "U{}".format(institution.id) final_query = xapian.Query( From e0258cbf02e8ca408825dd23bb2baa0f6f332342 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 9 Oct 2024 13:05:18 +0200 Subject: [PATCH 2/3] add search for hids --- dashboard/views.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index c188cc4..b34e2a6 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -3,6 +3,7 @@ import json from django.utils.translation import gettext_lazy as _ from django.views.generic.edit import FormView from django.shortcuts import Http404 +from django.db.models import Q from dashboard.mixins import InventaryMixin, DetailsMixin from evidence.models import Annotation @@ -40,7 +41,7 @@ class LotDashboardView(InventaryMixin, DetailsMixin): chids = self.object.devicelot_set.all().values_list( "device_id", flat=True ).distinct() - + chids_page = chids[offset:offset+limit] return [Device(id=x) for x in chids_page], chids.count() @@ -50,21 +51,24 @@ class SearchView(InventaryMixin): section = "Search" title = _("Search Devices") breadcrumb = "Devices / Search Devices" - + def get_devices(self, user, offset, limit): post = dict(self.request.POST) query = post.get("search") if not query: return [], 0 - + matches = search( self.request.user.institution, query[0], offset, limit ) - + + if not matches.size(): + return self.search_hids(query, offset, limit) + annotations = [] for x in matches: annotations.extend(self.get_annotations(x)) @@ -76,10 +80,26 @@ class SearchView(InventaryMixin): def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - + return Annotation.objects.filter( type=Annotation.Type.SYSTEM, owner=self.request.user.institution, uuid=uuid ).values_list("value", flat=True).distinct() + def search_hids(self, query, offset, limit): + qry = Q() + + for i in query[0].split(" "): + if i: + qry |= Q(value__startswith=i) + + chids = Annotation.objects.filter( + type=Annotation.Type.SYSTEM, + owner=self.request.user.institution + ).filter( + qry + ).values_list("value", flat=True).distinct() + chids_page = chids[offset:offset+limit] + + return [Device(id=x) for x in chids_page], chids.count() From 3e7b5c1df553ab2b482248057ec63391b8718c75 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 9 Oct 2024 18:00:56 +0200 Subject: [PATCH 3/3] lot over institution and not over user --- dashboard/mixins.py | 14 +++--- device/views.py | 2 +- lot/migrations/0001_initial.py | 66 +++++++++++++++++++++++++++- lot/migrations/0002_lotannotation.py | 52 ---------------------- lot/models.py | 11 +++-- lot/views.py | 37 +++++++++------- user/management/commands/add_user.py | 3 +- utils/device.py | 2 +- 8 files changed, 107 insertions(+), 80 deletions(-) delete mode 100644 lot/migrations/0002_lotannotation.py diff --git a/dashboard/mixins.py b/dashboard/mixins.py index 40f27bf..aa63d36 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -28,7 +28,7 @@ class DashboardView(LoginRequiredMixin): title = "" subtitle = "" section = "" - + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ @@ -39,13 +39,13 @@ class DashboardView(LoginRequiredMixin): 'section': self.section, 'path': resolve(self.request.path).url_name, 'user': self.request.user, - 'lot_tags': LotTag.objects.filter(owner=self.request.user) + 'lot_tags': LotTag.objects.filter(owner=self.request.user.institution) }) return context def get_session_devices(self): dev_ids = self.request.session.pop("devices", []) - + self._devices = [] for x in Annotation.objects.filter(value__in=dev_ids).filter( owner=self.request.user.institution @@ -58,7 +58,11 @@ class DetailsMixin(DashboardView, TemplateView): def get(self, request, *args, **kwargs): self.pk = kwargs['pk'] - self.object = get_object_or_404(self.model, pk=self.pk, owner=self.request.user) + self.object = get_object_or_404( + self.model, + pk=self.pk, + owner=self.request.user.institution + ) return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): @@ -78,7 +82,7 @@ class InventaryMixin(DashboardView, TemplateView): if url: dev_ids = post.get("devices", []) self.request.session["devices"] = dev_ids - + try: resource = resolve(url[0]) if resource and dev_ids: diff --git a/device/views.py b/device/views.py index 05183b0..ad13d43 100644 --- a/device/views.py +++ b/device/views.py @@ -98,7 +98,7 @@ class DetailsView(DashboardView, TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) self.object.initial() - lot_tags = LotTag.objects.filter(owner=self.request.user) + lot_tags = LotTag.objects.filter(owner=self.request.user.institution) context.update({ 'object': self.object, 'snapshot': self.object.get_last_evidence(), diff --git a/lot/migrations/0001_initial.py b/lot/migrations/0001_initial.py index 28b3e8e..e4602c5 100644 --- a/lot/migrations/0001_initial.py +++ b/lot/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-07-27 16:23 +# Generated by Django 5.0.6 on 2024-10-09 14:41 import django.db.models.deletion from django.conf import settings @@ -10,6 +10,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ("user", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] @@ -36,6 +37,15 @@ class Migration(migrations.Migration): "owner", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, + to="user.institution", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, ), ), @@ -62,6 +72,51 @@ class Migration(migrations.Migration): ), ], ), + migrations.CreateModel( + name="LotAnnotation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "type", + models.SmallIntegerField( + choices=[(0, "System"), (1, "User"), (2, "Document")] + ), + ), + ("key", models.CharField(max_length=256)), + ("value", models.CharField(max_length=256)), + ( + "lot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="lot.lot" + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="user.institution", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), migrations.CreateModel( name="LotTag", fields=[ @@ -79,6 +134,15 @@ class Migration(migrations.Migration): "owner", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, + to="user.institution", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, ), ), diff --git a/lot/migrations/0002_lotannotation.py b/lot/migrations/0002_lotannotation.py deleted file mode 100644 index aca7de1..0000000 --- a/lot/migrations/0002_lotannotation.py +++ /dev/null @@ -1,52 +0,0 @@ -# Generated by Django 5.0.6 on 2024-07-29 15:37 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("lot", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="LotAnnotation", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("created", models.DateTimeField(auto_now_add=True)), - ( - "type", - models.SmallIntegerField( - choices=[(0, "System"), (1, "User"), (2, "Document")] - ), - ), - ("key", models.CharField(max_length=256)), - ("value", models.CharField(max_length=256)), - ( - "lot", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="lot.lot" - ), - ), - ( - "owner", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL, - ), - ), - ], - ), - ] diff --git a/lot/models.py b/lot/models.py index 9078459..071d33d 100644 --- a/lot/models.py +++ b/lot/models.py @@ -6,14 +6,15 @@ from utils.constants import ( STR_EXTEND_SIZE, ) -from user.models import User +from user.models import User, Institution # from device.models import Device from evidence.models import Annotation class LotTag(models.Model): name = models.CharField(max_length=STR_SIZE, blank=False, null=False) - owner = models.ForeignKey(User, on_delete=models.CASCADE) + owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True) def __str__(self): return self.name @@ -31,7 +32,8 @@ class Lot(models.Model): code = models.CharField(max_length=STR_SIZE, blank=True, null=True) description = models.CharField(max_length=STR_SIZE, blank=True, null=True) closed = models.BooleanField(default=True) - owner = models.ForeignKey(User, on_delete=models.CASCADE) + owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True) type = models.ForeignKey(LotTag, on_delete=models.CASCADE) def add(self, v): @@ -52,7 +54,8 @@ class LotAnnotation(models.Model): created = models.DateTimeField(auto_now_add=True) lot = models.ForeignKey(Lot, on_delete=models.CASCADE) - owner = models.ForeignKey(User, on_delete=models.CASCADE) + owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True) type = models.SmallIntegerField(choices=Type) key = models.CharField(max_length=STR_EXTEND_SIZE) value = models.CharField(max_length=STR_EXTEND_SIZE) diff --git a/lot/views.py b/lot/views.py index 280450e..e9bb081 100644 --- a/lot/views.py +++ b/lot/views.py @@ -28,7 +28,8 @@ class NewLotView(DashboardView, CreateView): ) def form_valid(self, form): - form.instance.owner = self.request.user + form.instance.owner = self.request.user.institution + form.instance.user = self.request.user response = super().form_valid(form) return response @@ -68,7 +69,11 @@ class EditLotView(DashboardView, UpdateView): def get_form_kwargs(self): pk = self.kwargs.get('pk') - self.object = get_object_or_404(self.model, pk=pk) + self.object = get_object_or_404( + self.model, + owner=self.request.user.institution, + pk=pk, + ) # self.success_url = reverse_lazy('dashbiard:lot', args=[pk]) kwargs = super().get_form_kwargs() return kwargs @@ -83,8 +88,8 @@ class AddToLotView(DashboardView, FormView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - lots = Lot.objects.filter(owner=self.request.user) - lot_tags = LotTag.objects.filter(owner=self.request.user) + lots = Lot.objects.filter(owner=self.request.user.institution) + lot_tags = LotTag.objects.filter(owner=self.request.user.institution) context.update({ 'lots': lots, 'lot_tags':lot_tags, @@ -93,7 +98,7 @@ class AddToLotView(DashboardView, FormView): def get_form(self): form = super().get_form() - form.fields["lots"].queryset = Lot.objects.filter(owner=self.request.user) + form.fields["lots"].queryset = Lot.objects.filter(owner=self.request.user.institution) return form def form_valid(self, form): @@ -123,10 +128,10 @@ class LotsTagsView(DashboardView, TemplateView): def get_context_data(self, **kwargs): self.pk = kwargs.get('pk') context = super().get_context_data(**kwargs) - tag = get_object_or_404(LotTag, owner=self.request.user, id=self.pk) + tag = get_object_or_404(LotTag, owner=self.request.user.institution, id=self.pk) self.title += " {}".format(tag.name) self.breadcrumb += " {}".format(tag.name) - lots = Lot.objects.filter(owner=self.request.user).filter(type=tag) + lots = Lot.objects.filter(owner=self.request.user.institution).filter(type=tag) context.update({ 'lots': lots, 'title': self.title, @@ -144,7 +149,8 @@ class LotAddDocumentView(DashboardView, CreateView): fields = ("key", "value") def form_valid(self, form): - form.instance.owner = self.request.user + form.instance.owner = self.request.user.institution + form.instance.user = self.request.user form.instance.lot = self.lot form.instance.type = LotAnnotation.Type.DOCUMENT response = super().form_valid(form) @@ -152,7 +158,7 @@ class LotAddDocumentView(DashboardView, CreateView): def get_form_kwargs(self): pk = self.kwargs.get('pk') - self.lot = get_object_or_404(Lot, pk=pk, owner=self.request.user) + self.lot = get_object_or_404(Lot, pk=pk, owner=self.request.user.institution) self.success_url = reverse_lazy('lot:documents', args=[pk]) kwargs = super().get_form_kwargs() return kwargs @@ -166,10 +172,10 @@ class LotDocumentsView(DashboardView, TemplateView): def get_context_data(self, **kwargs): self.pk = kwargs.get('pk') context = super().get_context_data(**kwargs) - lot = get_object_or_404(Lot, owner=self.request.user, id=self.pk) + lot = get_object_or_404(Lot, owner=self.request.user.institution, id=self.pk) documents = LotAnnotation.objects.filter( lot=lot, - owner=self.request.user, + owner=self.request.user.institution, type=LotAnnotation.Type.DOCUMENT, ) context.update({ @@ -189,10 +195,10 @@ class LotAnnotationsView(DashboardView, TemplateView): def get_context_data(self, **kwargs): self.pk = kwargs.get('pk') context = super().get_context_data(**kwargs) - lot = get_object_or_404(Lot, owner=self.request.user, id=self.pk) + lot = get_object_or_404(Lot, owner=self.request.user.institution, id=self.pk) annotations = LotAnnotation.objects.filter( lot=lot, - owner=self.request.user, + owner=self.request.user.institution, type=LotAnnotation.Type.USER, ) context.update({ @@ -213,7 +219,8 @@ class LotAddAnnotationView(DashboardView, CreateView): fields = ("key", "value") def form_valid(self, form): - form.instance.owner = self.request.user + form.instance.owner = self.request.user.institution + form.instance.user = self.request.user form.instance.lot = self.lot form.instance.type = LotAnnotation.Type.USER response = super().form_valid(form) @@ -221,7 +228,7 @@ class LotAddAnnotationView(DashboardView, CreateView): def get_form_kwargs(self): pk = self.kwargs.get('pk') - self.lot = get_object_or_404(Lot, pk=pk, owner=self.request.user) + self.lot = get_object_or_404(Lot, pk=pk, owner=self.request.user.institution) self.success_url = reverse_lazy('lot:annotations', args=[pk]) kwargs = super().get_form_kwargs() return kwargs diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index f4cc780..9fd2eb6 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -43,5 +43,6 @@ class Command(BaseCommand): for tag in tags: LotTag.objects.create( name=tag, - owner=self.u + owner=self.u.institution, + user=self.u ) diff --git a/utils/device.py b/utils/device.py index f177aae..a358266 100644 --- a/utils/device.py +++ b/utils/device.py @@ -87,4 +87,4 @@ def create_index(doc, user): _uuid = doc['uuid'] ev = json.dumps(doc) - index(user, _uuid, ev) + index(user.institution, _uuid, ev)