resolve conficts

as an alternative to the randomly generated, which is also possible
when no predefined_token is defined

also update .env.example vars

.env.example

@@ -1,2 +1,20 @@
 DOMAIN=localhost
 DEMO=false
+
+STATIC_ROOT=/tmp/static/
+MEDIA_ROOT=/tmp/media/
+ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1,
+DOMAIN=localhost
+DEBUG=True
+EMAIL_HOST="mail.example.org"
+EMAIL_HOST_USER="fillme_noreply"
+EMAIL_HOST_PASSWORD="fillme_passwd"
+EMAIL_PORT=587
+EMAIL_USE_TLS=True
+EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend"
+EMAIL_FILE_PATH="/tmp/app-messages"
+ENABLE_EMAIL=false
+PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150'
+# TODO review these vars
+#SNAPSHOTS_DIR=/path/to/TODO
+#EVIDENCES_DIR=/path/to/TODO

api/views.py

@@ -1,32 +1,27 @@
 import json
 
+from uuid import uuid4
+
 from django.urls import reverse_lazy
+from django.http import JsonResponse
 from django.shortcuts import get_object_or_404, redirect
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.csrf import csrf_exempt
-from django.core.exceptions import ValidationError
 from django_tables2 import SingleTableView
-from django.views.generic.base import View
 from django.views.generic.edit import (
     CreateView,
     DeleteView,
     UpdateView,
 )
-from django.http import JsonResponse
-from uuid import uuid4
 
+from utils.save_snapshots import move_json, save_in_disk
 from dashboard.mixins import DashboardView
 from evidence.models import Annotation
 from evidence.parse import Build
-from user.models import User
 from api.models import Token
 from api.tables import TokensTable
 
 
-def save_in_disk(data, user):
-    pass
-
-
 @csrf_exempt
 def NewSnapshot(request):
     # Accept only posts
@@ -59,15 +54,16 @@ def NewSnapshot(request):
     ).first()
 
     if exist_annotation:
-        raise ValidationError("error: the snapshot {} exist".format(data['uuid']))
+        txt = "error: the snapshot {} exist".format(data['uuid'])
+        return JsonResponse({'status': txt}, status=500)
 
     # Process snapshot
-    # save_in_disk(data, tk.user)
+    path_name = save_in_disk(data, tk.owner.institution.name)
 
     try:
         Build(data, tk.owner)
-    except Exception:
-        return JsonResponse({'status': 'fail'}, status=200)
+    except Exception as err:
+        return JsonResponse({'status': f"fail: {err}"}, status=500)
 
     annotation = Annotation.objects.filter(
         uuid=data['uuid'],
@@ -79,7 +75,7 @@ def NewSnapshot(request):
 
 
     if not annotation:
-        return JsonResponse({'status': 'fail'}, status=200)
+        return JsonResponse({'status': 'fail'}, status=500)
 
     url_args = reverse_lazy("device:details", args=(annotation.value,))
     url = request.build_absolute_uri(url_args)
@@ -91,6 +87,8 @@ def NewSnapshot(request):
         # TODO replace with public_url when available
         "public_url": url
     }
+    move_json(path_name, tk.owner.institution.name)
+
     return JsonResponse(response, status=200)
 
 

dashboard/mixins.py

@@ -1,4 +1,5 @@
 from django.urls import resolve
+from django.conf import settings
 from django.shortcuts import get_object_or_404, redirect, Http404
 from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import PermissionDenied
@@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context.update({
+            "commit_id": settings.COMMIT,
             'title': self.title,
             'subtitle': self.subtitle,
             'breadcrumb': self.breadcrumb,

dashboard/views.py

@@ -69,23 +69,17 @@ class SearchView(InventaryMixin):
         if not matches.size():
             return self.search_hids(query, offset, limit)
 
-        annotations = []
+        devices = []
         for x in matches:
-            annotations.extend(self.get_annotations(x))
+            devices.append(self.get_annotations(x))
 
-        devices = [Device(id=x) for x in set(annotations)]
         count = matches.size()
         return devices, count
 
     def get_annotations(self, xp):
         snap = xp.document.get_data()
         uuid = json.loads(snap).get('uuid')
-
-        return Annotation.objects.filter(
-            type=Annotation.Type.SYSTEM,
-            owner=self.request.user.institution,
-            uuid=uuid
-        ).values_list("value", flat=True).distinct()
+        return Device.get_annotation_from_uuid(uuid, self.request.user.institution)
 
     def search_hids(self, query, offset, limit):
         qry = Q()

device/forms.py

@@ -1,5 +1,6 @@
 from django import forms
 from utils.device import create_annotation, create_doc, create_index
+from utils.save_snapshots import move_json, save_in_disk
 
 
 DEVICE_TYPES = [
@@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet):
         doc = create_doc(row)
         if not commit:
             return doc
-
+        
+        path_name = save_in_disk(doc, self.user.institution.name, place="placeholder")
         create_index(doc, self.user)
         create_annotation(doc, user, commit=commit)
+        move_json(path_name, self.user.institution.name, place="placeholder")
+        
         return doc
 
 

device/models.py

@@ -90,9 +90,11 @@ class Device:
     def get_hids(self):
         annotations = self.get_annotations()
 
+        algos = list(ALGOS.keys())
+        algos.append('CUSTOM_ID')
         self.hids = list(set(annotations.filter(
             type=Annotation.Type.SYSTEM,
-            key__in=ALGOS.keys(),
+            key__in=algos,
         ).values_list("value", flat=True)))
 
     def get_evidences(self):
@@ -118,9 +120,32 @@ class Device:
     def get_unassigned(cls, institution, offset=0, limit=None):
 
         sql = """
-              SELECT DISTINCT t1.value from evidence_annotation as t1
-                left join lot_devicelot as t2 on t1.value = t2.device_id
-              where t2.device_id is null and owner_id=={institution} and type=={type}
+            WITH RankedAnnotations AS (
+                SELECT
+                    t1.value,
+                    t1.key,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY t1.uuid
+                        ORDER BY
+                            CASE
+                                WHEN t1.key = 'CUSTOM_ID' THEN 1
+                                WHEN t1.key = 'hidalgo1' THEN 2
+                                ELSE 3
+                            END,
+                            t1.created DESC
+                    ) AS row_num
+                FROM evidence_annotation AS t1
+                LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
+                WHERE t2.device_id IS NULL
+                  AND t1.owner_id = {institution}
+                  AND t1.type = {type}
+            )
+            SELECT DISTINCT
+                value
+            FROM
+                RankedAnnotations
+            WHERE
+                row_num = 1
         """.format(
             institution=institution.id,
             type=Annotation.Type.SYSTEM,
@@ -144,18 +169,83 @@ class Device:
     def get_unassigned_count(cls, institution):
 
         sql = """
-              SELECT count(DISTINCT t1.value) from evidence_annotation as t1
-                left join lot_devicelot as t2 on t1.value = t2.device_id
-              where t2.device_id is null and owner_id=={institution} and type=={type};
+            WITH RankedAnnotations AS (
+                SELECT
+                    t1.value,
+                    t1.key,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY t1.uuid
+                        ORDER BY
+                            CASE
+                                WHEN t1.key = 'CUSTOM_ID' THEN 1
+                                WHEN t1.key = 'hidalgo1' THEN 2
+                                ELSE 3
+                            END,
+                            t1.created DESC
+                    ) AS row_num
+                FROM evidence_annotation AS t1
+                LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
+                WHERE t2.device_id IS NULL
+                  AND t1.owner_id = {institution}
+                  AND t1.type = {type}
+            )
+            SELECT
+                COUNT(DISTINCT value)
+            FROM
+                RankedAnnotations
+            WHERE
+                row_num = 1
         """.format(
             institution=institution.id,
             type=Annotation.Type.SYSTEM,
         )
-
         with connection.cursor() as cursor:
             cursor.execute(sql)
             return cursor.fetchall()[0][0]
 
+    @classmethod
+    def get_annotation_from_uuid(cls, uuid, institution):
+        sql = """
+            WITH RankedAnnotations AS (
+                SELECT
+                    t1.value,
+                    t1.key,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY t1.uuid
+                        ORDER BY
+                            CASE
+                                WHEN t1.key = 'CUSTOM_ID' THEN 1
+                                WHEN t1.key = 'hidalgo1' THEN 2
+                                ELSE 3
+                            END,
+                            t1.created DESC
+                    ) AS row_num
+                FROM evidence_annotation AS t1
+                LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
+                WHERE t2.device_id IS NULL
+                  AND t1.owner_id = {institution}
+                  AND t1.type = {type}
+                  AND t1.uuid = '{uuid}'
+            )
+            SELECT DISTINCT
+                value
+            FROM
+                RankedAnnotations
+            WHERE
+                row_num = 1;
+        """.format(
+            uuid=uuid.replace("-", ""),
+            institution=institution.id,
+            type=Annotation.Type.SYSTEM,
+        )
+
+        annotations = []
+        with connection.cursor() as cursor:
+            cursor.execute(sql)
+            annotations = cursor.fetchall()
+
+        return cls(id=annotations[0][0])
+
     @property
     def is_websnapshot(self):
         if not self.last_evidence:

dhub/settings.py

@@ -10,6 +10,7 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/5.0/ref/settings/
 """
 
+import os
 import xapian
 
 from pathlib import Path
@@ -35,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY"
 print("DOMAIN: " + DOMAIN)
 
 ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv())
-assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST"
+assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}"
 
 CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv())
 
@@ -62,6 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages')
 
 ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool)
 
+EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db"))
 
 # Application definition
 
@@ -162,6 +164,11 @@ USE_I18N = True
 
 USE_TZ = True
 
+USE_L10N = True
+LANGUAGES = [
+    ('es', 'Spanish'),
+    ('en', 'English'),
+]
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/5.0/howto/static-files/
@@ -206,3 +213,4 @@ LOGGING = {
 
 SNAPSHOT_PATH="/tmp/"
 DATA_UPLOAD_MAX_NUMBER_FILES = 1000
+COMMIT = config('COMMIT', default='')

docker-compose.yml

@@ -6,7 +6,9 @@ services:
     environment:
       - DEBUG=true
       - DOMAIN=${DOMAIN:-localhost}
-      - DEMO=${DEMO:-n}
+      - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN}
+      - DEMO=${DEMO:-false}
+      - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-}
     volumes:
       - .:/opt/devicehub-django
     ports:

docker-reset.sh

@@ -9,11 +9,13 @@ set -u
 set -x
 
 main() {
+        cd "$(dirname "${0}")"
+
         if [ "${DETACH:-}" ]; then
                 detach_arg='-d'
         fi
         # remove old database
-        sudo rm -vf db/*
+        sudo rm -vfr ./db/*
         docker compose down -v
         docker compose build
         docker compose up ${detach_arg:-}

docker/devicehub-django.entrypoint.sh

@@ -12,6 +12,14 @@ check_app_is_there() {
 }
 
 deploy() {
+        # TODO this is weird, find better workaround
+        git config --global --add safe.directory /opt/devicehub-django
+        export COMMIT=$(git log --format="%H %ad" --date=iso -n 1)
+
+        if [ "${DEBUG:-}" = 'true' ]; then
+                ./manage.py print_settings
+        fi
+
         # detect if existing deployment (TODO only works with sqlite)
         if [ -f "${program_dir}/db/db.sqlite3" ]; then
                 echo "INFO: detected EXISTING deployment"
@@ -24,11 +32,13 @@ deploy() {
                 INIT_ORG="${INIT_ORG:-example-org}"
                 INIT_USER="${INIT_USER:-user@example.org}"
                 INIT_PASSWD="${INIT_PASSWD:-1234}"
+                ADMIN='True'
+                PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}"
                 ./manage.py add_institution "${INIT_ORG}"
                 # TODO: one error on add_user, and you don't add user anymore
-                ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}"
+                ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}"
 
-                if [ "${DEMO:-}" ]; then
+                if [ "${DEMO:-}" = 'true' ]; then
                         ./manage.py up_snapshots example/snapshots/ "${INIT_USER}"
                 fi
         fi
@@ -36,7 +46,7 @@ deploy() {
 
 runserver() {
         PORT="${PORT:-8000}"
-        if [ "${DEBUG:-}" ]; then
+        if [ "${DEBUG:-}" = 'true' ]; then
                 ./manage.py runserver 0.0.0.0:${PORT}
         else
                 # TODO

evidence/forms.py

@@ -9,6 +9,7 @@ from utils.forms import MultipleFileField
 from device.models import Device
 from evidence.parse import Build
 from evidence.models import Annotation
+from utils.save_snapshots import move_json, save_in_disk
 
 
 class UploadForm(forms.Form):
@@ -48,7 +49,9 @@ class UploadForm(forms.Form):
             return
 
         for ev in self.evidences:
+            path_name = save_in_disk(ev[1], user.institution.name)
             Build(ev[1], user)
+            move_json(path_name, user.institution.name)
 
 
 class UserTagForm(forms.Form):
@@ -151,8 +154,11 @@ class ImportForm(forms.Form):
 
         if commit:
             for doc, cred in table:
-              cred.save()
-              create_index(doc, self.user)
-            return table
+                path_name = save_in_disk(doc, self.user.institution.name, place="placeholder")
+
+                cred.save()
+                create_index(doc, self.user)
+                move_json(path_name, self.user.institution.name, place="placeholder")
+                return table
 
         return

evidence/models.py

@@ -126,6 +126,7 @@ class Evidence:
         return Annotation.objects.filter(
             owner=user.institution,
             type=Annotation.Type.SYSTEM,
+            key="hidalgo1",
         ).order_by("-created").values_list("uuid", flat=True).distinct()
 
     def set_components(self):

evidence/parse.py

@@ -113,7 +113,8 @@ class Build:
         # mac = get_mac2(hwinfo_raw) or ""
         mac = get_mac(lshw) or ""
         if not mac:
-            print("WARNING!! No there are MAC address")
+            print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" )
+            # TODO generate system annotation for that snapshot
         else:
             print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}")
 

login/templates/login.html

@@ -42,4 +42,5 @@
 <div id="login-footer" class="mt-3">
     <a href="{% url 'login:password_reset' %}" data-toggle="modal" data-target="#forgotPasswordModal">{% trans "Forgot your password? Click here to recover" %}</a>
 </div>
+
 {% endblock %}

login/views.py

@@ -1,5 +1,6 @@
 import logging
 
+from django.conf import settings
 from django.urls import reverse_lazy
 from django.contrib.auth import views as auth_views
 from django.contrib.auth import login as auth_login
@@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView):
     extra_context = {
         'title': _('Login'),
         'success_url': reverse_lazy('dashboard:unassigned_devices'),
-        # 'commit_id': settings.COMMIT, 
+        'commit_id': settings.COMMIT,
     }
 
     def get(self, request, *args, **kwargs):
@@ -65,7 +66,6 @@ class PasswordResetView(auth_views.PasswordResetView):
     success_url = reverse_lazy('login:password_reset_done')
 
     def form_valid(self, form):
-        import pdb; pdb.set_trace()
         try:
             response = super().form_valid(form)
             return response

user/management/commands/add_user.py

@@ -17,15 +17,17 @@ class Command(BaseCommand):
         parser.add_argument('email', type=str, help='email')
         parser.add_argument('password', type=str, help='password')
         parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin')
+        parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token')
 
     def handle(self, *args, **kwargs):
         email = kwargs['email']
         password = kwargs['password']
         is_admin = kwargs['is_admin']
+        predefined_token = kwargs['predefined_token']
         institution = Institution.objects.get(name=kwargs['institution'])
-        self.create_user(institution, email, password, is_admin)
+        self.create_user(institution, email, password, is_admin, predefined_token)
 
-    def create_user(self, institution, email, password, is_admin):
+    def create_user(self, institution, email, password, is_admin, predefined_token):
         self.u = User.objects.create(
             institution=institution,
             email=email,
@@ -34,6 +36,10 @@ class Command(BaseCommand):
         )
         self.u.set_password(password)
         self.u.save()
-        token = uuid4()
+        if predefined_token:
+            token = predefined_token
+        else:
+            token = uuid4()
+
         Token.objects.create(token=token, owner=self.u)
         print(f"TOKEN: {token}")

utils/save_snapshots.py

@@ -0,0 +1,43 @@
+import os
+import json
+import shutil
+
+from datetime import datetime
+from django.conf import settings
+
+
+def move_json(path_name, user, place="snapshots"):
+    if place != "snapshots":
+        place = "placeholders"
+
+    tmp_snapshots = settings.EVIDENCES_DIR
+    path_dir = os.path.join(tmp_snapshots, user, place)
+
+    if os.path.isfile(path_name):
+        shutil.copy(path_name, path_dir)
+        os.remove(path_name)
+
+
+def save_in_disk(data, user, place="snapshots"):
+    uuid = data.get('uuid', '')
+    now = datetime.now()
+    year = now.year
+    month = now.month
+    day = now.day
+    hour = now.hour
+    minutes = now.minute
+    tmp_snapshots = settings.EVIDENCES_DIR
+    if place != "snapshots":
+        place = "placeholders"
+
+    name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json"
+    path_dir = os.path.join(tmp_snapshots, user, place, "errors")
+    path_name = os.path.join(path_dir, name_file)
+
+    if not os.path.isdir(path_dir):
+        os.system(f'mkdir -p {path_dir}')
+
+    with open(path_name, 'w') as snapshot_file:
+        snapshot_file.write(json.dumps(data))
+
+    return path_name