From 6e4bb3729f5130fcbb0bf84de2f69c181caeba91 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 25 Sep 2024 21:50:26 -0300 Subject: [PATCH 01/57] lot: add a button to show/hide closed lots by default: - a lot is open - hide closed lots --- lot/models.py | 2 +- lot/templates/lots.html | 10 ++++++++++ lot/views.py | 6 ++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lot/models.py b/lot/models.py index 764e134..6c81bfd 100644 --- a/lot/models.py +++ b/lot/models.py @@ -31,7 +31,7 @@ class Lot(models.Model): name = models.CharField(max_length=STR_SIZE, blank=True, null=True) code = models.CharField(max_length=STR_SIZE, blank=True, null=True) description = models.CharField(max_length=STR_SIZE, blank=True, null=True) - closed = models.BooleanField(default=True) + closed = models.BooleanField(default=False) owner = models.ForeignKey(Institution, on_delete=models.CASCADE) user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True) type = models.ForeignKey(LotTag, on_delete=models.CASCADE) diff --git a/lot/templates/lots.html b/lot/templates/lots.html index e091dd4..2aa866d 100644 --- a/lot/templates/lots.html +++ b/lot/templates/lots.html @@ -7,6 +7,16 @@

{{ subtitle }}

+ {% if show_closed %} + + {% trans 'Hide closed lots' %} + + {% else %} + + {% trans 'Show closed lots' %} + + {% endif %} + {% trans 'Add new lot' %} diff --git a/lot/views.py b/lot/views.py index e9bb081..182aade 100644 --- a/lot/views.py +++ b/lot/views.py @@ -131,11 +131,13 @@ class LotsTagsView(DashboardView, TemplateView): tag = get_object_or_404(LotTag, owner=self.request.user.institution, id=self.pk) self.title += " {}".format(tag.name) self.breadcrumb += " {}".format(tag.name) - lots = Lot.objects.filter(owner=self.request.user.institution).filter(type=tag) + show_closed = self.request.GET.get('show_closed', 'false') == 'true' + lots = Lot.objects.filter(owner=self.request.user.institution).filter(type=tag, closed=show_closed) context.update({ 'lots': lots, 'title': self.title, - 'breadcrumb': self.breadcrumb + 'breadcrumb': self.breadcrumb, + 'show_closed': show_closed }) return context From d04216ad795f0c8d84c542b7787621c1dab54fa0 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 15 Oct 2024 13:09:34 +0200 Subject: [PATCH 02/57] api snapshot public_url bugfix with the following code the URL is more appropriate (it basically mirrors the user's request) before: http://localhost/device/{shortid} after http://localhost:8000/device/{shortid} --- api/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index 175fe3e..fe68079 100644 --- a/api/views.py +++ b/api/views.py @@ -1,6 +1,5 @@ import json -from django.conf import settings from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ @@ -82,15 +81,14 @@ def NewSnapshot(request): if not annotation: return JsonResponse({'status': 'fail'}, status=200) - url = "{}://{}{}".format( - request.scheme, - settings.DOMAIN, - reverse_lazy("device:details", args=(annotation.value,)) - ) + url_args = reverse_lazy("device:details", args=(annotation.value,)) + url = request.build_absolute_uri(url_args) + response = { "status": "success", "dhid": annotation.value[:6].upper(), "url": url, + # TODO replace with public_url when available "public_url": url } return JsonResponse(response, status=200) From 92d9837a9172976ce0ccff24b3653644a8dd56d9 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:05:43 +0200 Subject: [PATCH 03/57] save snapshots on disk from api --- api/views.py | 42 +++++++++++++++++++++++++++++++++++++++--- dhub/settings.py | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index 175fe3e..9e3c0ad 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,8 @@ +import os import json +import shutil + +from datetime import datetime from django.conf import settings from django.urls import reverse_lazy @@ -24,8 +28,37 @@ from api.models import Token from api.tables import TokensTable +def move_json(path_name, user): + tmp_snapshots = settings.SNAPSHOTS_DIR + path_dir = os.path.join(tmp_snapshots, user) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + def save_in_disk(data, user): - pass + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.SNAPSHOTS_DIR + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name + @csrf_exempt @@ -60,10 +93,11 @@ def NewSnapshot(request): ).first() if exist_annotation: - raise ValidationError("error: the snapshot {} exist".format(data['uuid'])) + txt = "error: the snapshot {} exist".format(data['uuid']) + return JsonResponse({'status': txt}, status=500) # Process snapshot - # save_in_disk(data, tk.user) + path_name = save_in_disk(data, tk.owner.institution.name) try: Build(data, tk.owner) @@ -93,6 +127,8 @@ def NewSnapshot(request): "url": url, "public_url": url } + move_json(path_name, tk.owner.institution.name) + return JsonResponse(response, status=200) diff --git a/dhub/settings.py b/dhub/settings.py index 3b48d2c..3a55985 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -62,6 +62,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) +SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") # Application definition From 5bf5bc23600dff3fe5ce0ba6786731ea501d79ac Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 16:58:51 +0200 Subject: [PATCH 04/57] save placeholders --- api/views.py | 41 +++-------------------------------------- device/forms.py | 6 +++++- dhub/settings.py | 3 ++- evidence/forms.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/api/views.py b/api/views.py index 9e3c0ad..cf80b35 100644 --- a/api/views.py +++ b/api/views.py @@ -1,11 +1,10 @@ -import os import json -import shutil -from datetime import datetime +from uuid import uuid4 from django.conf import settings from django.urls import reverse_lazy +from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt @@ -17,9 +16,8 @@ from django.views.generic.edit import ( DeleteView, UpdateView, ) -from django.http import JsonResponse -from uuid import uuid4 +from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -28,39 +26,6 @@ from api.models import Token from api.tables import TokensTable -def move_json(path_name, user): - tmp_snapshots = settings.SNAPSHOTS_DIR - path_dir = os.path.join(tmp_snapshots, user) - - if os.path.isfile(path_name): - shutil.copy(path_name, path_dir) - os.remove(path_name) - - -def save_in_disk(data, user): - uuid = data.get('uuid', '') - now = datetime.now() - year = now.year - month = now.month - day = now.day - hour = now.hour - minutes = now.minute - tmp_snapshots = settings.SNAPSHOTS_DIR - - name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" - path_dir = os.path.join(tmp_snapshots, user, "errors") - path_name = os.path.join(path_dir, name_file) - - if not os.path.isdir(path_dir): - os.system(f'mkdir -p {path_dir}') - - with open(path_name, 'w') as snapshot_file: - snapshot_file.write(json.dumps(data)) - - return path_name - - - @csrf_exempt def NewSnapshot(request): # Accept only posts diff --git a/device/forms.py b/device/forms.py index b2f0d3c..38d649e 100644 --- a/device/forms.py +++ b/device/forms.py @@ -1,5 +1,6 @@ from django import forms from utils.device import create_annotation, create_doc, create_index +from utils.save_snapshots import move_json, save_in_disk DEVICE_TYPES = [ @@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet): doc = create_doc(row) if not commit: return doc - + + path_name = save_in_disk(doc, self.user.name) create_index(doc, self.user) create_annotation(doc, user, commit=commit) + move_json(path_name, self.user.name) + return doc diff --git a/dhub/settings.py b/dhub/settings.py index 3a55985..c307073 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os import xapian from pathlib import Path @@ -62,7 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) -SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") +EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db")) # Application definition diff --git a/evidence/forms.py b/evidence/forms.py index db98f2d..a47024e 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -9,6 +9,7 @@ from utils.forms import MultipleFileField from device.models import Device from evidence.parse import Build from evidence.models import Annotation +from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): @@ -48,7 +49,9 @@ class UploadForm(forms.Form): return for ev in self.evidences: + path_name = save_in_disk(ev[1], user.institution.name) Build(ev[1], user) + move_json(path_name, user.institution.name) class UserTagForm(forms.Form): @@ -151,8 +154,10 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - cred.save() - create_index(doc, self.user) - return table + path_name = save_in_disk(doc, self.user.name) + cred.save() + create_index(doc, self.user) + move_json(path_name, self.user.name) + return table return From 69bd81289f86a18e55057d62f34c37518ef9ed41 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:56:34 +0200 Subject: [PATCH 05/57] add save snapshots for imports and uploads --- device/forms.py | 4 ++-- evidence/forms.py | 5 +++-- utils/save_snapshots.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 utils/save_snapshots.py diff --git a/device/forms.py b/device/forms.py index 38d649e..6c4d6e2 100644 --- a/device/forms.py +++ b/device/forms.py @@ -57,10 +57,10 @@ class BaseDeviceFormSet(forms.BaseFormSet): if not commit: return doc - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") create_index(doc, self.user) create_annotation(doc, user, commit=commit) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return doc diff --git a/evidence/forms.py b/evidence/forms.py index a47024e..cf90a71 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -154,10 +154,11 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") + cred.save() create_index(doc, self.user) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return table return diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py new file mode 100644 index 0000000..8c02f06 --- /dev/null +++ b/utils/save_snapshots.py @@ -0,0 +1,43 @@ +import os +import json +import shutil + +from datetime import datetime +from django.conf import settings + + +def move_json(path_name, user, place="snapshots"): + if place != "snapshots": + place = "placeholders" + + tmp_snapshots = settings.EVIDENCES_DIR + path_dir = os.path.join(tmp_snapshots, user, place) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + +def save_in_disk(data, user, place="snapshots"): + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.EVIDENCES_DIR + if place != "snapshots": + place = "placeholders" + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, place, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name From d1cd44f8f66f864b134053e96aeedf4383d4e05f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:57:45 +0200 Subject: [PATCH 06/57] remove pdbs --- login/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/login/views.py b/login/views.py index 5bf48a3..be6b009 100644 --- a/login/views.py +++ b/login/views.py @@ -65,7 +65,6 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): - import pdb; pdb.set_trace() try: response = super().form_valid(form) return response From 2d7b74c556efd5dd18ab82f685de5fab78c21131 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:17:35 +0200 Subject: [PATCH 07/57] fix bug of tag --- dashboard/views.py | 12 ++---- device/models.py | 102 +++++++++++++++++++++++++++++++++++++++++---- evidence/models.py | 22 ++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index b34e2a6..f056523 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -69,23 +69,17 @@ class SearchView(InventaryMixin): if not matches.size(): return self.search_hids(query, offset, limit) - annotations = [] + devices = [] for x in matches: - annotations.extend(self.get_annotations(x)) + devices.append(self.get_annotations(x)) - devices = [Device(id=x) for x in set(annotations)] count = matches.size() return devices, count def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - - return Annotation.objects.filter( - type=Annotation.Type.SYSTEM, - owner=self.request.user.institution, - uuid=uuid - ).values_list("value", flat=True).distinct() + return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): qry = Q() diff --git a/device/models.py b/device/models.py index 926e447..6169f2e 100644 --- a/device/models.py +++ b/device/models.py @@ -118,9 +118,32 @@ class Device: def get_unassigned(cls, institution, offset=0, limit=None): sql = """ - SELECT DISTINCT t1.value from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type} + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, @@ -144,18 +167,83 @@ class Device: def get_unassigned_count(cls, institution): sql = """ - SELECT count(DISTINCT t1.value) from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type}; + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT + COUNT(DISTINCT value) + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, ) - with connection.cursor() as cursor: cursor.execute(sql) return cursor.fetchall()[0][0] + @classmethod + def get_annotation_from_uuid(cls, uuid, institution): + sql = """ + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + AND t1.uuid = '{uuid}' + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1; + """.format( + uuid=uuid.replace("-", ""), + institution=institution.id, + type=Annotation.Type.SYSTEM, + ) + + annotations = [] + with connection.cursor() as cursor: + cursor.execute(sql) + annotations = cursor.fetchall() + + return cls(id=annotations[0][0]) + @property def is_websnapshot(self): if not self.last_evidence: diff --git a/evidence/models.py b/evidence/models.py index 667945f..ffc0d84 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,8 +126,30 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, + key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() + # annotations = Annotation.objects.filter( + # owner=user.institution, + # type=Annotation.Type.SYSTEM + # ) + + # annotations = annotations.annotate( + # priority=models.Case( + # models.When(key="CUSTOM_ID", then=1), + # models.When(key="hidalgo1", then=2), + # default=3, + # output_field=models.IntegerField(), + # ) + # ) + + # # sqlite not soport distinct('uuid') + # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # # 'uuid').values_list("uuid", flat=True) + # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # ).values_list("uuid", flat=True) + # return set(filtered_uuids) + def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From ccda270de086ee4d87769e4efcd3a9fb1101d586 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:19:09 +0200 Subject: [PATCH 08/57] clean code comment --- evidence/models.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index ffc0d84..354cb89 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -129,27 +129,6 @@ class Evidence: key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() - # annotations = Annotation.objects.filter( - # owner=user.institution, - # type=Annotation.Type.SYSTEM - # ) - - # annotations = annotations.annotate( - # priority=models.Case( - # models.When(key="CUSTOM_ID", then=1), - # models.When(key="hidalgo1", then=2), - # default=3, - # output_field=models.IntegerField(), - # ) - # ) - - # # sqlite not soport distinct('uuid') - # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # # 'uuid').values_list("uuid", flat=True) - # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # ).values_list("uuid", flat=True) - # return set(filtered_uuids) - def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From cd3cc62004bd5dc36ba75004bce07f3611896fbb Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 17:40:00 +0200 Subject: [PATCH 09/57] fix hid in details of device --- device/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/device/models.py b/device/models.py index 6169f2e..1f43476 100644 --- a/device/models.py +++ b/device/models.py @@ -90,9 +90,11 @@ class Device: def get_hids(self): annotations = self.get_annotations() + algos = list(ALGOS.keys()) + algos.append('CUSTOM_ID') self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, - key__in=ALGOS.keys(), + key__in=algos, ).values_list("value", flat=True))) def get_evidences(self): From 94fd1d685a5e31d216c9a78344dbac164d035c21 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 20:39:57 +0200 Subject: [PATCH 10/57] docker-reset: remove better the db now that in db dir there are snapshots stored --- docker-reset.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index cdbb971..bf14db2 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,13 @@ set -u set -x main() { + cd "$(dirname "${0}")" + if [ "${DETACH:-}" ]; then detach_arg='-d' fi # remove old database - sudo rm -vf db/* + sudo rm -vfr ./db/* docker compose down -v docker compose build docker compose up ${detach_arg:-} From 7b95ce054119fd668879bd099f002fa201fb4f84 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:09:16 +0200 Subject: [PATCH 11/57] docker: make default user admin --- docker/devicehub-django.entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 920cdf0..9361fdb 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,9 +24,10 @@ deploy() { INIT_ORG="${INIT_ORG:-example-org}" INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" + ADMIN='True' ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" From 11f4b647c219a768e96e79474027b83085f52b84 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:14:22 +0200 Subject: [PATCH 12/57] snapshot processing: improve warning msg add comment TODO that system annotation should happen --- evidence/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..dfc5a1f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -113,7 +113,8 @@ class Build: # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" if not mac: - print("WARNING!! No there are MAC address") + print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" ) + # TODO generate system annotation for that snapshot else: print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}") From 71f68e2d4b1f4588d22f28ce42cc46d9e0afb358 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:54:54 +0200 Subject: [PATCH 13/57] add predefined_token as an alternative to the randomly generated, which is also possible when no predefined_token is defined also update .env.example vars --- .env.example | 19 +++++++++++++++++++ docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 ++- user/management/commands/add_user.py | 12 +++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index cf39121..1495f9d 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,21 @@ DOMAIN=localhost DEMO=false + +STATIC_ROOT=/tmp/static/ +MEDIA_ROOT=/tmp/media/ +ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DOMAIN=localhost +DEBUG=True +EMAIL_HOST="mail.example.org" +EMAIL_HOST_USER="fillme_noreply" +EMAIL_HOST_PASSWORD="fillme_passwd" +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" +EMAIL_FILE_PATH="/tmp/app-messages" +ENABLE_EMAIL=false +COMMIT=$(git log --format="%H %ad" --date=iso -n 1) +PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' +# TODO review these vars +#SNAPSHOTS_DIR=/path/to/TODO +#EVIDENCES_DIR=/path/to/TODO diff --git a/docker-compose.yml b/docker-compose.yml index ee59640..7d144bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - DEMO=${DEMO:-n} + - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 9361fdb..e988d4c 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -25,9 +25,10 @@ deploy() { INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" ADMIN='True' + PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 60f3dd2..6593fee 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -17,15 +17,17 @@ class Command(BaseCommand): parser.add_argument('email', type=str, help='email') parser.add_argument('password', type=str, help='password') parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin') + parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token') def handle(self, *args, **kwargs): email = kwargs['email'] password = kwargs['password'] is_admin = kwargs['is_admin'] + predefined_token = kwargs['predefined_token'] institution = Institution.objects.get(name=kwargs['institution']) - self.create_user(institution, email, password, is_admin) + self.create_user(institution, email, password, is_admin, predefined_token) - def create_user(self, institution, email, password, is_admin): + def create_user(self, institution, email, password, is_admin, predefined_token): self.u = User.objects.create( institution=institution, email=email, @@ -34,6 +36,10 @@ class Command(BaseCommand): ) self.u.set_password(password) self.u.save() - token = uuid4() + if predefined_token: + token = predefined_token + else: + token = uuid4() + Token.objects.create(token=token, owner=self.u) print(f"TOKEN: {token}") From ce9a69d6f7f9f3cb8869ba9d8170bfc3572e0dcd Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:05:09 +0200 Subject: [PATCH 14/57] docker: bugfix wrong detection of demo --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e988d4c..2c7c77b 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" ]; then + if [ "${DEMO:-}" = 'y' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 1751dc98c2e76e8b39045a86f893dc10d1a4024a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:07:40 +0200 Subject: [PATCH 15/57] docker: use demo true/false instead of y/n --- docker-compose.yml | 2 +- docker/devicehub-django.entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d144bd..b451358 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - DEMO=${DEMO:-n} + - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 2c7c77b..452b32d 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" = 'y' ]; then + if [ "${DEMO:-}" = 'true' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 2e638e40a9756a75c1ea7742396e07635afb0a19 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:48:42 +0200 Subject: [PATCH 16/57] add ALLOWED_HOSTS in docker compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b451358..9ea039c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From d94b12d1a0cdd62cb6cfe523cfba4733b3c6c8b0 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 18:38:57 +0200 Subject: [PATCH 17/57] locale in settings --- dhub/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhub/settings.py b/dhub/settings.py index c307073..e109072 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -164,6 +164,11 @@ USE_I18N = True USE_TZ = True +USE_L10N = True +LANGUAGES = [ + ('es', 'Spanish'), + ('en', 'English'), +] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ From 1bb3b1263642c26c904082c8d2cba41922a20f49 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 19:08:00 +0200 Subject: [PATCH 18/57] more debug in the error answer from api --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index cf80b35..5aa5f87 100644 --- a/api/views.py +++ b/api/views.py @@ -66,8 +66,8 @@ def NewSnapshot(request): try: Build(data, tk.owner) - except Exception: - return JsonResponse({'status': 'fail'}, status=200) + except Exception as err: + return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( uuid=data['uuid'], From 182f85457063c7022d7b7d73a5422ad9c373265e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 09:52:09 +0200 Subject: [PATCH 19/57] add commit_id in settings --- dashboard/mixins.py | 2 ++ dhub/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index aa63d36..af01a1b 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -1,4 +1,5 @@ from django.urls import resolve +from django.conf import settings from django.shortcuts import get_object_or_404, redirect, Http404 from django.utils.translation import gettext_lazy as _ from django.core.exceptions import PermissionDenied @@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ + "commit_id": settings.COMMIT, 'title': self.title, 'subtitle': self.subtitle, 'breadcrumb': self.breadcrumb, diff --git a/dhub/settings.py b/dhub/settings.py index e109072..b96710e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -213,3 +213,4 @@ LOGGING = { SNAPSHOT_PATH="/tmp/" DATA_UPLOAD_MAX_NUMBER_FILES = 1000 +COMMIT = config('COMMIT', default='') From 8e8130fc34f566cdd68be882d7e8255d94ba4763 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:20:23 +0200 Subject: [PATCH 20/57] docker: improve debug --- docker/devicehub-django.entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 452b32d..e5773f3 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + if [ "${DEBUG:-}" = 'true' ]; then + ./manage.py print_settings + fi + # detect if existing deployment (TODO only works with sqlite) if [ -f "${program_dir}/db/db.sqlite3" ]; then echo "INFO: detected EXISTING deployment" @@ -38,7 +42,7 @@ deploy() { runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" ]; then + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO From 460fb678e9722c6668102e61e2807cba23cc5a23 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:24:40 +0200 Subject: [PATCH 21/57] settings: improve assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index b96710e..80d9f3c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From b0754d8e97f4078f14dbd70cd31f37c1cfc186cd Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:26:38 +0200 Subject: [PATCH 22/57] settings: bugfix assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index 80d9f3c..c580e6b 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From a65d78e72f1fd74ec5eb679a63c0bcc07fd42ddf Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:27:27 +0200 Subject: [PATCH 23/57] settings: bugfix format assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index c580e6b..891b714 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 03b9c5fea8fa2bf1a79ac61ba539a323010cb546 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:28:02 +0200 Subject: [PATCH 24/57] docker-compose: bugfix DOMAIN vs ALLOWED_HOSTS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ea039c..d376313 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From f87897de85ccf64d8386968091a3d33f20e334c8 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:45:08 +0200 Subject: [PATCH 25/57] add commit footer for docker --- .env.example | 1 - docker/devicehub-django.entrypoint.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1495f9d..37ed087 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ EMAIL_USE_TLS=True EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" EMAIL_FILE_PATH="/tmp/app-messages" ENABLE_EMAIL=false -COMMIT=$(git log --format="%H %ad" --date=iso -n 1) PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' # TODO review these vars #SNAPSHOTS_DIR=/path/to/TODO diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e5773f3..bdf1008 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + # TODO this is weird, find better workaround + git config --global --add safe.directory /opt/devicehub-django + export COMMIT=$(git log --format="%H %ad" --date=iso -n 1) + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py print_settings fi From a08428c90b05aa222b81b1350ca899e093dbe5e4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 12:03:26 +0200 Subject: [PATCH 26/57] commit in login page --- login/templates/login.html | 1 + login/views.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/login/templates/login.html b/login/templates/login.html index 97ba4cb..a82882e 100644 --- a/login/templates/login.html +++ b/login/templates/login.html @@ -42,4 +42,5 @@ + {% endblock %} diff --git a/login/views.py b/login/views.py index be6b009..beb0368 100644 --- a/login/views.py +++ b/login/views.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.urls import reverse_lazy from django.contrib.auth import views as auth_views from django.contrib.auth import login as auth_login @@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView): extra_context = { 'title': _('Login'), 'success_url': reverse_lazy('dashboard:unassigned_devices'), - # 'commit_id': settings.COMMIT, + 'commit_id': settings.COMMIT, } def get(self, request, *args, **kwargs): From 69a54b92fb5318aab85f9e4429d3b9ad832cdb84 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:05:43 +0200 Subject: [PATCH 27/57] save snapshots on disk from api --- api/views.py | 42 +++++++++++++++++++++++++++++++++++++++--- dhub/settings.py | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index fe68079..0ae7e04 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,8 @@ +import os import json +import shutil + +from datetime import datetime from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect @@ -23,8 +27,37 @@ from api.models import Token from api.tables import TokensTable +def move_json(path_name, user): + tmp_snapshots = settings.SNAPSHOTS_DIR + path_dir = os.path.join(tmp_snapshots, user) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + def save_in_disk(data, user): - pass + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.SNAPSHOTS_DIR + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name + @csrf_exempt @@ -59,10 +92,11 @@ def NewSnapshot(request): ).first() if exist_annotation: - raise ValidationError("error: the snapshot {} exist".format(data['uuid'])) + txt = "error: the snapshot {} exist".format(data['uuid']) + return JsonResponse({'status': txt}, status=500) # Process snapshot - # save_in_disk(data, tk.user) + path_name = save_in_disk(data, tk.owner.institution.name) try: Build(data, tk.owner) @@ -91,6 +125,8 @@ def NewSnapshot(request): # TODO replace with public_url when available "public_url": url } + move_json(path_name, tk.owner.institution.name) + return JsonResponse(response, status=200) diff --git a/dhub/settings.py b/dhub/settings.py index 3b48d2c..3a55985 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -62,6 +62,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) +SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") # Application definition From fae269eb8d415f05e3c94cba446a4d629d85fd92 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 16:58:51 +0200 Subject: [PATCH 28/57] save placeholders --- api/views.py | 41 +++-------------------------------------- device/forms.py | 6 +++++- dhub/settings.py | 3 ++- evidence/forms.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/api/views.py b/api/views.py index 0ae7e04..72b5518 100644 --- a/api/views.py +++ b/api/views.py @@ -1,10 +1,9 @@ -import os import json -import shutil -from datetime import datetime +from uuid import uuid4 from django.urls import reverse_lazy +from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt @@ -16,9 +15,8 @@ from django.views.generic.edit import ( DeleteView, UpdateView, ) -from django.http import JsonResponse -from uuid import uuid4 +from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -27,39 +25,6 @@ from api.models import Token from api.tables import TokensTable -def move_json(path_name, user): - tmp_snapshots = settings.SNAPSHOTS_DIR - path_dir = os.path.join(tmp_snapshots, user) - - if os.path.isfile(path_name): - shutil.copy(path_name, path_dir) - os.remove(path_name) - - -def save_in_disk(data, user): - uuid = data.get('uuid', '') - now = datetime.now() - year = now.year - month = now.month - day = now.day - hour = now.hour - minutes = now.minute - tmp_snapshots = settings.SNAPSHOTS_DIR - - name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" - path_dir = os.path.join(tmp_snapshots, user, "errors") - path_name = os.path.join(path_dir, name_file) - - if not os.path.isdir(path_dir): - os.system(f'mkdir -p {path_dir}') - - with open(path_name, 'w') as snapshot_file: - snapshot_file.write(json.dumps(data)) - - return path_name - - - @csrf_exempt def NewSnapshot(request): # Accept only posts diff --git a/device/forms.py b/device/forms.py index b2f0d3c..38d649e 100644 --- a/device/forms.py +++ b/device/forms.py @@ -1,5 +1,6 @@ from django import forms from utils.device import create_annotation, create_doc, create_index +from utils.save_snapshots import move_json, save_in_disk DEVICE_TYPES = [ @@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet): doc = create_doc(row) if not commit: return doc - + + path_name = save_in_disk(doc, self.user.name) create_index(doc, self.user) create_annotation(doc, user, commit=commit) + move_json(path_name, self.user.name) + return doc diff --git a/dhub/settings.py b/dhub/settings.py index 3a55985..c307073 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os import xapian from pathlib import Path @@ -62,7 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) -SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") +EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db")) # Application definition diff --git a/evidence/forms.py b/evidence/forms.py index db98f2d..a47024e 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -9,6 +9,7 @@ from utils.forms import MultipleFileField from device.models import Device from evidence.parse import Build from evidence.models import Annotation +from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): @@ -48,7 +49,9 @@ class UploadForm(forms.Form): return for ev in self.evidences: + path_name = save_in_disk(ev[1], user.institution.name) Build(ev[1], user) + move_json(path_name, user.institution.name) class UserTagForm(forms.Form): @@ -151,8 +154,10 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - cred.save() - create_index(doc, self.user) - return table + path_name = save_in_disk(doc, self.user.name) + cred.save() + create_index(doc, self.user) + move_json(path_name, self.user.name) + return table return From 003d224c3e2c7f499f1ecd401ba475f0b3cefe02 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:56:34 +0200 Subject: [PATCH 29/57] add save snapshots for imports and uploads --- device/forms.py | 4 ++-- evidence/forms.py | 5 +++-- utils/save_snapshots.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 utils/save_snapshots.py diff --git a/device/forms.py b/device/forms.py index 38d649e..6c4d6e2 100644 --- a/device/forms.py +++ b/device/forms.py @@ -57,10 +57,10 @@ class BaseDeviceFormSet(forms.BaseFormSet): if not commit: return doc - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") create_index(doc, self.user) create_annotation(doc, user, commit=commit) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return doc diff --git a/evidence/forms.py b/evidence/forms.py index a47024e..cf90a71 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -154,10 +154,11 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") + cred.save() create_index(doc, self.user) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return table return diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py new file mode 100644 index 0000000..8c02f06 --- /dev/null +++ b/utils/save_snapshots.py @@ -0,0 +1,43 @@ +import os +import json +import shutil + +from datetime import datetime +from django.conf import settings + + +def move_json(path_name, user, place="snapshots"): + if place != "snapshots": + place = "placeholders" + + tmp_snapshots = settings.EVIDENCES_DIR + path_dir = os.path.join(tmp_snapshots, user, place) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + +def save_in_disk(data, user, place="snapshots"): + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.EVIDENCES_DIR + if place != "snapshots": + place = "placeholders" + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, place, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name From 2f9b61667d38dd84a2c0e4dd9dc5568a0054ba31 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:57:45 +0200 Subject: [PATCH 30/57] remove pdbs --- login/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/login/views.py b/login/views.py index 5bf48a3..be6b009 100644 --- a/login/views.py +++ b/login/views.py @@ -65,7 +65,6 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): - import pdb; pdb.set_trace() try: response = super().form_valid(form) return response From 090ecc275fbf6b7fc876268db4492b365f8dc4c3 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:17:35 +0200 Subject: [PATCH 31/57] fix bug of tag --- dashboard/views.py | 12 ++---- device/models.py | 102 +++++++++++++++++++++++++++++++++++++++++---- evidence/models.py | 22 ++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index b34e2a6..f056523 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -69,23 +69,17 @@ class SearchView(InventaryMixin): if not matches.size(): return self.search_hids(query, offset, limit) - annotations = [] + devices = [] for x in matches: - annotations.extend(self.get_annotations(x)) + devices.append(self.get_annotations(x)) - devices = [Device(id=x) for x in set(annotations)] count = matches.size() return devices, count def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - - return Annotation.objects.filter( - type=Annotation.Type.SYSTEM, - owner=self.request.user.institution, - uuid=uuid - ).values_list("value", flat=True).distinct() + return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): qry = Q() diff --git a/device/models.py b/device/models.py index 926e447..6169f2e 100644 --- a/device/models.py +++ b/device/models.py @@ -118,9 +118,32 @@ class Device: def get_unassigned(cls, institution, offset=0, limit=None): sql = """ - SELECT DISTINCT t1.value from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type} + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, @@ -144,18 +167,83 @@ class Device: def get_unassigned_count(cls, institution): sql = """ - SELECT count(DISTINCT t1.value) from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type}; + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT + COUNT(DISTINCT value) + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, ) - with connection.cursor() as cursor: cursor.execute(sql) return cursor.fetchall()[0][0] + @classmethod + def get_annotation_from_uuid(cls, uuid, institution): + sql = """ + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + AND t1.uuid = '{uuid}' + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1; + """.format( + uuid=uuid.replace("-", ""), + institution=institution.id, + type=Annotation.Type.SYSTEM, + ) + + annotations = [] + with connection.cursor() as cursor: + cursor.execute(sql) + annotations = cursor.fetchall() + + return cls(id=annotations[0][0]) + @property def is_websnapshot(self): if not self.last_evidence: diff --git a/evidence/models.py b/evidence/models.py index 667945f..ffc0d84 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,8 +126,30 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, + key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() + # annotations = Annotation.objects.filter( + # owner=user.institution, + # type=Annotation.Type.SYSTEM + # ) + + # annotations = annotations.annotate( + # priority=models.Case( + # models.When(key="CUSTOM_ID", then=1), + # models.When(key="hidalgo1", then=2), + # default=3, + # output_field=models.IntegerField(), + # ) + # ) + + # # sqlite not soport distinct('uuid') + # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # # 'uuid').values_list("uuid", flat=True) + # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # ).values_list("uuid", flat=True) + # return set(filtered_uuids) + def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From c422eaddebb9e6d6bfab9064ff087c2614bc0c7e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:19:09 +0200 Subject: [PATCH 32/57] clean code comment --- evidence/models.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index ffc0d84..354cb89 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -129,27 +129,6 @@ class Evidence: key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() - # annotations = Annotation.objects.filter( - # owner=user.institution, - # type=Annotation.Type.SYSTEM - # ) - - # annotations = annotations.annotate( - # priority=models.Case( - # models.When(key="CUSTOM_ID", then=1), - # models.When(key="hidalgo1", then=2), - # default=3, - # output_field=models.IntegerField(), - # ) - # ) - - # # sqlite not soport distinct('uuid') - # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # # 'uuid').values_list("uuid", flat=True) - # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # ).values_list("uuid", flat=True) - # return set(filtered_uuids) - def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From d8b6d3ded61df6f5c6d769f31cedaf13dd167730 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 17:40:00 +0200 Subject: [PATCH 33/57] fix hid in details of device --- device/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/device/models.py b/device/models.py index 6169f2e..1f43476 100644 --- a/device/models.py +++ b/device/models.py @@ -90,9 +90,11 @@ class Device: def get_hids(self): annotations = self.get_annotations() + algos = list(ALGOS.keys()) + algos.append('CUSTOM_ID') self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, - key__in=ALGOS.keys(), + key__in=algos, ).values_list("value", flat=True))) def get_evidences(self): From 3d10217599d14ce44e1d1b11fb83197e018e2b10 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 20:39:57 +0200 Subject: [PATCH 34/57] docker-reset: remove better the db now that in db dir there are snapshots stored --- docker-reset.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index cdbb971..bf14db2 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,13 @@ set -u set -x main() { + cd "$(dirname "${0}")" + if [ "${DETACH:-}" ]; then detach_arg='-d' fi # remove old database - sudo rm -vf db/* + sudo rm -vfr ./db/* docker compose down -v docker compose build docker compose up ${detach_arg:-} From 0e8607d93ecf45f3573220e97144e91a619c921a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:09:16 +0200 Subject: [PATCH 35/57] docker: make default user admin --- docker/devicehub-django.entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 920cdf0..9361fdb 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,9 +24,10 @@ deploy() { INIT_ORG="${INIT_ORG:-example-org}" INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" + ADMIN='True' ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" From f757c5835656b15c8daa32dee36e26c85d8009d5 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:14:22 +0200 Subject: [PATCH 36/57] snapshot processing: improve warning msg add comment TODO that system annotation should happen --- evidence/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..dfc5a1f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -113,7 +113,8 @@ class Build: # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" if not mac: - print("WARNING!! No there are MAC address") + print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" ) + # TODO generate system annotation for that snapshot else: print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}") From 67869bc6f55e45de170f8ae2222def8fe90d053f Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:54:54 +0200 Subject: [PATCH 37/57] add predefined_token as an alternative to the randomly generated, which is also possible when no predefined_token is defined also update .env.example vars --- .env.example | 19 +++++++++++++++++++ docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 ++- user/management/commands/add_user.py | 12 +++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index cf39121..1495f9d 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,21 @@ DOMAIN=localhost DEMO=false + +STATIC_ROOT=/tmp/static/ +MEDIA_ROOT=/tmp/media/ +ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DOMAIN=localhost +DEBUG=True +EMAIL_HOST="mail.example.org" +EMAIL_HOST_USER="fillme_noreply" +EMAIL_HOST_PASSWORD="fillme_passwd" +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" +EMAIL_FILE_PATH="/tmp/app-messages" +ENABLE_EMAIL=false +COMMIT=$(git log --format="%H %ad" --date=iso -n 1) +PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' +# TODO review these vars +#SNAPSHOTS_DIR=/path/to/TODO +#EVIDENCES_DIR=/path/to/TODO diff --git a/docker-compose.yml b/docker-compose.yml index ee59640..7d144bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - DEMO=${DEMO:-n} + - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 9361fdb..e988d4c 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -25,9 +25,10 @@ deploy() { INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" ADMIN='True' + PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 60f3dd2..6593fee 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -17,15 +17,17 @@ class Command(BaseCommand): parser.add_argument('email', type=str, help='email') parser.add_argument('password', type=str, help='password') parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin') + parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token') def handle(self, *args, **kwargs): email = kwargs['email'] password = kwargs['password'] is_admin = kwargs['is_admin'] + predefined_token = kwargs['predefined_token'] institution = Institution.objects.get(name=kwargs['institution']) - self.create_user(institution, email, password, is_admin) + self.create_user(institution, email, password, is_admin, predefined_token) - def create_user(self, institution, email, password, is_admin): + def create_user(self, institution, email, password, is_admin, predefined_token): self.u = User.objects.create( institution=institution, email=email, @@ -34,6 +36,10 @@ class Command(BaseCommand): ) self.u.set_password(password) self.u.save() - token = uuid4() + if predefined_token: + token = predefined_token + else: + token = uuid4() + Token.objects.create(token=token, owner=self.u) print(f"TOKEN: {token}") From bf47d4bc5d20a2ed61cfab04540494d37d789577 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:05:09 +0200 Subject: [PATCH 38/57] docker: bugfix wrong detection of demo --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e988d4c..2c7c77b 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" ]; then + if [ "${DEMO:-}" = 'y' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 425b032273bc4f6d77ff0f4e10df3bec0740cb80 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:07:40 +0200 Subject: [PATCH 39/57] docker: use demo true/false instead of y/n --- docker-compose.yml | 2 +- docker/devicehub-django.entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d144bd..b451358 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - DEMO=${DEMO:-n} + - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 2c7c77b..452b32d 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" = 'y' ]; then + if [ "${DEMO:-}" = 'true' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From c49644ffbd80990264c43e16d3d39c70bd82e1e1 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:48:42 +0200 Subject: [PATCH 40/57] add ALLOWED_HOSTS in docker compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b451358..9ea039c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 0b55d34d74ce092de75b9285d62b946b0942d809 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 18:38:57 +0200 Subject: [PATCH 41/57] locale in settings --- dhub/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhub/settings.py b/dhub/settings.py index c307073..e109072 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -164,6 +164,11 @@ USE_I18N = True USE_TZ = True +USE_L10N = True +LANGUAGES = [ + ('es', 'Spanish'), + ('en', 'English'), +] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ From e6141bcc7fabf528e82f812b895773a3f416972f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 19:08:00 +0200 Subject: [PATCH 42/57] more debug in the error answer from api --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 72b5518..79f3592 100644 --- a/api/views.py +++ b/api/views.py @@ -65,8 +65,8 @@ def NewSnapshot(request): try: Build(data, tk.owner) - except Exception: - return JsonResponse({'status': 'fail'}, status=200) + except Exception as err: + return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( uuid=data['uuid'], From b7c4926c390f8d91375e18bf7a655d55a992739d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 09:52:09 +0200 Subject: [PATCH 43/57] add commit_id in settings --- dashboard/mixins.py | 2 ++ dhub/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index aa63d36..af01a1b 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -1,4 +1,5 @@ from django.urls import resolve +from django.conf import settings from django.shortcuts import get_object_or_404, redirect, Http404 from django.utils.translation import gettext_lazy as _ from django.core.exceptions import PermissionDenied @@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ + "commit_id": settings.COMMIT, 'title': self.title, 'subtitle': self.subtitle, 'breadcrumb': self.breadcrumb, diff --git a/dhub/settings.py b/dhub/settings.py index e109072..b96710e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -213,3 +213,4 @@ LOGGING = { SNAPSHOT_PATH="/tmp/" DATA_UPLOAD_MAX_NUMBER_FILES = 1000 +COMMIT = config('COMMIT', default='') From 3d2cb7d184c381dc0e0cbdb9e97ca18015328f6a Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:20:23 +0200 Subject: [PATCH 44/57] docker: improve debug --- docker/devicehub-django.entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 452b32d..e5773f3 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + if [ "${DEBUG:-}" = 'true' ]; then + ./manage.py print_settings + fi + # detect if existing deployment (TODO only works with sqlite) if [ -f "${program_dir}/db/db.sqlite3" ]; then echo "INFO: detected EXISTING deployment" @@ -38,7 +42,7 @@ deploy() { runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" ]; then + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO From d995db8181361a7b8faf7466c252086baff55d02 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:24:40 +0200 Subject: [PATCH 45/57] settings: improve assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index b96710e..80d9f3c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 096b83da812ef66105d88382b0f69c19d0144ed6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:26:38 +0200 Subject: [PATCH 46/57] settings: bugfix assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index 80d9f3c..c580e6b 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 1d9d100fae1d490c55c2b89f3470c0908a4d23f6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:27:27 +0200 Subject: [PATCH 47/57] settings: bugfix format assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index c580e6b..891b714 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 9f05dee6a7656179e0ec396732ca651e61434a19 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:28:02 +0200 Subject: [PATCH 48/57] docker-compose: bugfix DOMAIN vs ALLOWED_HOSTS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ea039c..d376313 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 4cf164d4bce2656c3b388435ff8dafa7eae7fa8b Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:45:08 +0200 Subject: [PATCH 49/57] add commit footer for docker --- .env.example | 1 - docker/devicehub-django.entrypoint.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1495f9d..37ed087 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ EMAIL_USE_TLS=True EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" EMAIL_FILE_PATH="/tmp/app-messages" ENABLE_EMAIL=false -COMMIT=$(git log --format="%H %ad" --date=iso -n 1) PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' # TODO review these vars #SNAPSHOTS_DIR=/path/to/TODO diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e5773f3..bdf1008 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + # TODO this is weird, find better workaround + git config --global --add safe.directory /opt/devicehub-django + export COMMIT=$(git log --format="%H %ad" --date=iso -n 1) + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py print_settings fi From c60a9dd89d368e9dcf849a4dc470c7a57545d8e4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 12:03:26 +0200 Subject: [PATCH 50/57] commit in login page --- login/templates/login.html | 1 + login/views.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/login/templates/login.html b/login/templates/login.html index 97ba4cb..a82882e 100644 --- a/login/templates/login.html +++ b/login/templates/login.html @@ -42,4 +42,5 @@ + {% endblock %} diff --git a/login/views.py b/login/views.py index be6b009..beb0368 100644 --- a/login/views.py +++ b/login/views.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.urls import reverse_lazy from django.contrib.auth import views as auth_views from django.contrib.auth import login as auth_login @@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView): extra_context = { 'title': _('Login'), 'success_url': reverse_lazy('dashboard:unassigned_devices'), - # 'commit_id': settings.COMMIT, + 'commit_id': settings.COMMIT, } def get(self, request, *args, **kwargs): From d98e8e2d48597333e06bf781ac27588fd792f0ec Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 15:24:42 +0200 Subject: [PATCH 51/57] resolve conficts --- api/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 79f3592..69da2a2 100644 --- a/api/views.py +++ b/api/views.py @@ -7,9 +7,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt -from django.core.exceptions import ValidationError from django_tables2 import SingleTableView -from django.views.generic.base import View from django.views.generic.edit import ( CreateView, DeleteView, @@ -20,7 +18,6 @@ from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build -from user.models import User from api.models import Token from api.tables import TokensTable @@ -78,7 +75,7 @@ def NewSnapshot(request): if not annotation: - return JsonResponse({'status': 'fail'}, status=200) + return JsonResponse({'status': 'fail'}, status=500) url_args = reverse_lazy("device:details", args=(annotation.value,)) url = request.build_absolute_uri(url_args) From a0548b38c7b3445261f30cd1bc6b19e894083906 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 11:24:09 +0200 Subject: [PATCH 52/57] change EreuseWorkbench for workbench-script --- evidence/models.py | 10 +++++----- evidence/parse.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index 354cb89..eadb7a7 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -67,7 +67,7 @@ class Evidence: for xa in matches: self.doc = json.loads(xa.document.get_data()) - if self.doc.get("software") == "EreuseWorkbench": + if self.doc.get("software") == "workbench-script": dmidecode_raw = self.doc["data"]["dmidecode"] self.dmi = DMIParse(dmidecode_raw) @@ -80,7 +80,7 @@ class Evidence: self.created = self.annotations.last().created def get_components(self): - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc.get('components', []) self.set_components() return self.components @@ -92,7 +92,7 @@ class Evidence: return "" return list(self.doc.get('kv').values())[0] - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['manufacturer'] return self.dmi.manufacturer().strip() @@ -104,13 +104,13 @@ class Evidence: return "" return list(self.doc.get('kv').values())[1] - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['model'] return self.dmi.model().strip() def get_chassis(self): - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['model'] chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') diff --git a/evidence/parse.py b/evidence/parse.py index dfc5a1f..4f42934 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -57,7 +57,7 @@ class Build: } def get_hid_14(self): - if self.json.get("software") == "EreuseWorkbench": + if self.json.get("software") == "workbench-script": hid = self.get_hid(self.json) else: device = self.json['device'] From 4bb71adf007eb6d30f2c233c258d8cdbd04d5ac6 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 13:27:45 +0200 Subject: [PATCH 53/57] add hours of use --- evidence/parse_details.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evidence/parse_details.py b/evidence/parse_details.py index 8465403..fd7f1c2 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -160,6 +160,7 @@ class ParseSnapshot: continue model = sm.get('model_name') manufacturer = None + hours = sm.get("power_on_time", {}).get("hours", 0) if model and len(model.split(" ")) > 1: mm = model.split(" ") model = mm[-1] @@ -175,6 +176,7 @@ class ParseSnapshot: "size": self.get_data_storage_size(sm), "variant": sm.get("firmware_version"), "interface": self.get_data_storage_interface(sm), + "hours": hours, } ) From 8d509e31c1de4d273f9f784fc6fe7fc40a6bc9a8 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 13:28:20 +0200 Subject: [PATCH 54/57] add date in list of events --- evidence/models.py | 2 +- evidence/templates/evidences.html | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index 667945f..d971ce8 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,7 +126,7 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, - ).order_by("-created").values_list("uuid", flat=True).distinct() + ).order_by("-created").values_list("uuid", "created").distinct() def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json diff --git a/evidence/templates/evidences.html b/evidence/templates/evidences.html index c67e9d9..b221d9b 100644 --- a/evidence/templates/evidences.html +++ b/evidence/templates/evidences.html @@ -14,10 +14,13 @@ {% for ev in evidences %} - {{ ev }} + {{ ev.0 }} - + {{ ev.1 }} + + + {% endfor %} From ff91bc9ad9ff0929f560e8c3bf468739ebca2259 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 13:50:01 +0200 Subject: [PATCH 55/57] change TIME_ZONE for env variable --- dhub/settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dhub/settings.py b/dhub/settings.py index 8b77fd7..23ea1a6 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -132,11 +132,14 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = "en-us" -TIME_ZONE = "UTC" +TIME_ZONE = config("TIME_ZONE", default="UTC") USE_I18N = True -USE_TZ = True +USE_TZ = False + +if TIME_ZONE == "UTC": + USE_TZ = True # Static files (CSS, JavaScript, Images) From e8d8c9670047f07bc36e92a00f7e7fb260b794eb Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 16:48:36 +0200 Subject: [PATCH 56/57] add anchor in tabs of details of devices --- device/templates/details.html | 36 ++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/device/templates/details.html b/device/templates/details.html index 37f7487..51e97f1 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -12,25 +12,25 @@
@@ -214,3 +214,25 @@
{% endblock %} + +{% block extrascript %} + +{% endblock %} + From efcb53e062b6112a100d2b40511224d4e8067a29 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 18:39:31 +0200 Subject: [PATCH 57/57] repair bad jsons --- evidence/parse.py | 9 ++++++++- evidence/parse_details.py | 8 +++++++- requirements.txt | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..8688d6f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -5,6 +5,8 @@ import hashlib from datetime import datetime from dmidecode import DMIParse +from json_repair import repair_json + from evidence.models import Annotation from evidence.xapian import index from utils.constants import ALGOS, CHASSIS_DH @@ -20,7 +22,12 @@ def get_network_cards(child, nets): def get_mac(lshw): nets = [] try: - get_network_cards(json.loads(lshw), nets) + hw = json.loads(lshw) + except json.decoder.JSONDecodeError: + hw = json.loads(repair_json(lshw)) + + try: + get_network_cards(hw, nets) except Exception as ss: print("WARNING!! {}".format(ss)) return diff --git a/evidence/parse_details.py b/evidence/parse_details.py index fd7f1c2..57ce4f9 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -3,6 +3,8 @@ import numpy as np from datetime import datetime from dmidecode import DMIParse +from json_repair import repair_json + from utils.constants import CHASSIS_DH, DATASTORAGEINTERFACE @@ -480,7 +482,11 @@ class ParseSnapshot: def loads(self, x): if isinstance(x, str): try: - return json.loads(x) + try: + hw = json.loads(lshw) + except json.decoder.JSONDecodeError: + hw = json.loads(repair_json(lshw)) + return hw except Exception as ss: print("WARNING!! {}".format(ss)) return {} diff --git a/requirements.txt b/requirements.txt index 1576d4f..32b8e86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ pandas==2.2.2 xlrd==2.0.1 odfpy==1.4.1 pytz==2024.2 - +json-repair==0.30.0